#!/usr/bin/env python3
"""Re-runs the check behind the countries-visited-map post.

Two jobs, both mechanical:

1. every quote in dataset.json still appears on the page it cites;
2. every count and every per-place cell is recomputed from the pages themselves
   and from Viallo's own tables in this repository, and must match.

    python3 check.py            verify
    python3 check.py --recheck  verify again, re-reading every page (there is no
                                lab step, so nothing is skipped by the flag)
    python3 check.py --emit     rewrite dataset.json and dataset.csv from this run

Exit 0 when everything holds, 1 otherwise. Standard library only.
"""

import argparse
import csv
import html
import json
import re
import sys
import urllib.error
import urllib.request
from pathlib import Path

HERE = Path(__file__).resolve().parent
# <repo>/apps/web/public/blog/<slug>
REPO = HERE.parents[4]
DATASET_JSON = HERE / 'dataset.json'
DATASET_CSV = HERE / 'dataset.csv'

UA = {
    'User-Agent': ('Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36'
                   ' (KHTML, like Gecko) Chrome/126 Safari/537.36'),
    'Accept-Language': 'en-US,en;q=0.9',
}
TIMEOUT = 45
CHECKED_AT = '2026-10-26'

UN_ABOUT = 'https://www.un.org/en/about-us'
UN_MEMBERS = 'https://www.un.org/en/about-us/member-states'
FLYTRIPPERS = 'https://flytrippers.com/visiting-every-country-kevin/'
BEEN = 'https://been.app/'
BEEN_AS = 'https://apps.apple.com/us/app/been-track-visited-countries/id1659477938'
VISITED_AS = 'https://apps.apple.com/us/app/visited-travel-tracker-map/id846983349'
SKRATCH_COUNTRY = 'https://help.skratch.world/en/article/what-is-defined-as-a-country-on-skratch-t6grza/'
SKRATCH_DISPUTED = 'https://help.skratch.world/en/article/what-is-defined-as-a-disputed-territory-on-skratch-1vwam5g/'
SKRATCH_OTHER = 'https://help.skratch.world/en/article/what-is-defined-as-an-other-territory-on-skratch-avn6wb/'
SKRATCH_SHARE = 'https://help.skratch.world/en/article/how-do-i-share-my-skratch-map-with-friends-lxru8s/'
SKRATCH_MARK = 'https://help.skratch.world/en/article/how-do-i-mark-places-on-my-map-gwf7fm/'
SKRATCH_AS = 'https://apps.apple.com/us/app/skratch-travel-map-esim/id1457438876'
CV = 'https://countries-visited.com/'
POLAR_COUNTRIES = 'https://support.polarsteps.com/api/v2/help_center/en-us/articles/33242544423954.json'
POLAR_START = 'https://support.polarsteps.com/api/v2/help_center/en-us/articles/24267114106770.json'
POLAR_PLUS = 'https://www.polarsteps.com/plus'
POLAR_VISIBLE = 'https://support.polarsteps.com/api/v2/help_center/en-us/articles/24267891448850.json'
ANDROID_MEDIA = 'https://developer.android.com/training/data-storage/shared/media'
APPLE_PHOTOS = 'https://support.apple.com/guide/iphone/browse-photos-and-videos-by-location-iph390138909/ios'
GOOGLE_PHOTOS = 'https://support.google.com/photos/answer/15235862?hl=en'
GEONAMES = 'https://download.geonames.org/export/dump/readme.txt'
COUNTRY_CODER = 'https://github.com/rapideditor/country-coder'

PAGES = [
    UN_ABOUT, UN_MEMBERS, FLYTRIPPERS, BEEN, BEEN_AS, VISITED_AS,
    SKRATCH_COUNTRY, SKRATCH_DISPUTED, SKRATCH_OTHER, SKRATCH_SHARE, SKRATCH_MARK, SKRATCH_AS,
    CV, POLAR_COUNTRIES, POLAR_START, POLAR_PLUS, POLAR_VISIBLE, ANDROID_MEDIA,
    APPLE_PHOTOS, GOOGLE_PHOTOS, GEONAMES, COUNTRY_CODER,
]

REPO_INPUTS = [
    'packages/shared/src/geo/data/rule.json',
    'packages/shared/src/geo/country.ts',
    'packages/shared/src/geo/nearest-city.ts',
    'packages/shared/src/geo/data/places-15000.v2.json',
    'apps/web/scripts/build-places.mjs',
    'apps/api/src/atlas/atlas-compute.ts',
    'apps/web/src/components/atlas/share-ui/AtlasShareModal.tsx',
    'packages/shared/src/subscription/tiers.ts',
]

# Places the trackers are checked against. `code` is Viallo's ISO code where it has one.
PLACES = [
    ('Kosovo', 'XK'),
    ('Taiwan', 'TW'),
    ('Vatican', 'VA'),
    ('Palestine', 'PS'),
    ('Greenland', 'GL'),
    ('Hong Kong', 'HK'),
    ('Puerto Rico', 'PR'),
    ('Faroe Islands', 'FO'),
    ('Western Sahara', 'EH'),
    ('Northern Cyprus', None),
    ('Antarctica', 'AQ'),
]

# Skratch and countries-visited.com spell some places differently from Viallo.
NAME_IN = {
    'Skratch': {'Vatican': 'Vatican', 'Palestine': 'Palestine'},
    'countries-visited.com': {'Vatican': 'Vatican', 'Hong Kong': 'Hong Kong'},
}

OBSERVERS = {'Vatican', 'Palestine'}

# One row per published value. `quote` must appear on `url` once normalised;
# `computed` names the measurement in compute() that must equal `value`.
ROWS = [
    # --- Viallo Atlas, from the product's own tables ---
    {'id': 'viallo-mechanism', 'option': 'Viallo Atlas',
     'property': 'how a country gets on the map',
     'value': "from the coordinates in each photo, resolved on Viallo's own servers",
     'computed': 'viallo_mechanism', 'url': 'repo:packages/shared/src/geo/country.ts'},
    {'id': 'viallo-counted', 'option': 'Viallo Atlas', 'property': 'countries counted',
     'value': '197', 'computed': 'viallo_counted',
     'url': 'repo:packages/shared/src/geo/data/rule.json'},
    {'id': 'viallo-extra', 'option': 'Viallo Atlas',
     'property': 'counted beyond the UN 193 and its two observers',
     'value': '2 (Kosovo, Taiwan)', 'computed': 'viallo_extra',
     'url': 'repo:packages/shared/src/geo/data/rule.json'},
    {'id': 'viallo-territories', 'option': 'Viallo Atlas',
     'property': 'territories listed apart from the count',
     'value': '42', 'computed': 'viallo_territories',
     'url': 'repo:packages/shared/src/geo/data/rule.json'},
    {'id': 'viallo-held', 'option': 'Viallo Atlas',
     'property': 'disputed areas held back, not counted',
     'value': '5 (Crimea, Golan Heights, East Jerusalem, Western Sahara, Northern Cyprus)',
     'computed': 'viallo_held', 'url': 'repo:packages/shared/src/geo/country.ts'},
    {'id': 'viallo-places', 'option': 'Viallo Atlas',
     'property': 'places in the city table it ships', 'value': '31734',
     'computed': 'viallo_places',
     'url': 'repo:packages/shared/src/geo/data/places-15000.v2.json'},
    {'id': 'viallo-city-distance', 'option': 'Viallo Atlas',
     'property': 'reach of the city lookup, km', 'value': '30',
     'computed': 'viallo_city_distance',
     'url': 'repo:packages/shared/src/geo/nearest-city.ts'},
    {'id': 'viallo-free', 'option': 'Viallo Atlas', 'property': 'Free plan limits',
     'value': '2 albums, 200 photos', 'computed': 'viallo_free',
     'url': 'repo:packages/shared/src/subscription/tiers.ts'},
    {'id': 'viallo-share', 'option': 'Viallo Atlas', 'property': 'how you share it',
     'value': 'image, link, website widget', 'computed': 'viallo_share',
     'url': 'repo:apps/web/src/components/atlas/share-ui/AtlasShareModal.tsx'},

    # --- United Nations and the three-list argument ---
    {'id': 'un-about', 'option': 'United Nations', 'property': 'members', 'value': '193',
     'url': UN_ABOUT, 'quote': 'Currently made up of 193 Member States'},
    {'id': 'un-list', 'option': 'United Nations', 'property': 'member states listed',
     'value': '193', 'computed': 'un_members', 'url': UN_MEMBERS},
    {'id': 'flytrippers', 'option': 'Flytrippers',
     'property': 'the three lists, and what the 197 is',
     'value': '193, 195, 197; the 197 is 195 plus Kosovo and Taiwan', 'url': FLYTRIPPERS,
     'quote': ('193: All United Nations member states ( official list ) 195: 193 + 2 UN observer'
               ' states (Vatican and Palestine) 197: 195 + 2 less-recognized states (Taiwan and Kosovo)')},

    # --- Been ---
    {'id': 'been-mark', 'option': 'Been', 'property': 'how a country gets on the map',
     'value': 'you tap each country', 'url': BEEN,
     'quote': 'Mark every country Tap a country and it lights up green.'},
    {'id': 'been-count', 'option': 'Been', 'property': 'what it counts against',
     'value': '250+ countries and territories', 'url': BEEN,
     'quote': 'been covers 250+ countries and territories'},
    {'id': 'been-un193', 'option': 'Been', 'property': 'UN members inside that count',
     'value': 'all 193 UN member states', 'url': BEEN,
     'quote': 'including all 193 UN member states'},
    {'id': 'been-border', 'option': 'Been', 'property': 'how disputed places are drawn',
     'value': 'de-facto borders; disputed states drawn with a distinct border and a factual label',
     'url': BEEN,
     'quote': ('drawing disputed and partially-recognised states with a distinct border and a'
               ' factual label rather than taking a side.')},
    {'id': 'been-share', 'option': 'Been', 'property': 'how you share it', 'value': 'branded image',
     'url': BEEN,
     'quote': 'Export a branded passport image and show off your travels in a single tap.'},
    {'id': 'been-free', 'option': 'Been', 'property': 'what is free',
     'value': 'marking every country, the passport and sharing the map', 'url': BEEN,
     'quote': 'been is free to mark every country, build your passport and share your map.'},
    {'id': 'been-weekly', 'option': 'Been', 'property': 'price, weekly subscription',
     'value': 'Been Pro (weekly) $4.99', 'url': BEEN_AS, 'quote': 'Been Pro (weekly) $4.99'},
    {'id': 'been-yearly', 'option': 'Been', 'property': 'price, yearly subscription',
     'value': 'Been Pro (yearly) $12.99', 'url': BEEN_AS, 'quote': 'Been Pro (yearly) $12.99'},
    {'id': 'been-lifetime', 'option': 'Been', 'property': 'price, one-time option',
     'value': 'Been Premium Lifetime $7.99', 'url': BEEN_AS,
     'quote': 'Been Premium Lifetime $7.99'},

    # --- Visited ---
    {'id': 'visited-select', 'option': 'Visited', 'property': 'how a country gets on the map',
     'value': 'you pick the countries, states, regions and cities you have been to',
     'url': VISITED_AS,
     'quote': ('Design your personal travel map consisting of countries, states, regions and'
               ' cities of where you have been')},
    {'id': 'visited-sovereignty', 'option': 'Visited', 'property': 'what it counts against',
     'value': 'UN sovereignty status (a setting chooses the count)', 'url': VISITED_AS,
     'quote': 'Counting countries by official UN Sovereignty status'},
    {'id': 'visited-adjust', 'option': 'Visited', 'property': 'disputed places',
     'value': 'you choose where each disputed place belongs', 'url': VISITED_AS,
     'quote': 'Adjust the sovereignty for disputed territories.'},
    {'id': 'visited-share', 'option': 'Visited', 'property': 'how you share it',
     'value': 'share it with your friends; a printed poster can be bought', 'url': VISITED_AS,
     'quote': 'share it with your friends as you plan your next adventure'},
    {'id': 'visited-poster', 'option': 'Visited', 'property': 'the printed poster',
     'value': 'printed and shipped to you', 'url': VISITED_AS,
     'quote': 'then shipped directly to you'},
    {'id': 'visited-pro', 'option': 'Visited', 'property': 'price, top in-app purchase',
     'value': 'Visited Pro $59.99', 'url': VISITED_AS, 'quote': 'Visited Pro $59.99'},
    {'id': 'visited-monthly', 'option': 'Visited', 'property': 'price, monthly subscription',
     'value': 'Monthly $9.99', 'url': VISITED_AS, 'quote': 'Monthly $9.99'},
    {'id': 'visited-yearly', 'option': 'Visited', 'property': 'price, yearly subscription',
     'value': 'Yearly $34.99', 'url': VISITED_AS, 'quote': 'Yearly $34.99'},

    # --- Skratch ---
    {'id': 'skratch-rule', 'option': 'Skratch', 'property': 'what it counts against',
     'value': 'UN sovereign and observer states', 'url': SKRATCH_COUNTRY,
     'quote': 'Countries on Skratch are defined by the UN list of sovereign and observer states.'},
    {'id': 'skratch-counted', 'option': 'Skratch', 'property': 'countries counted',
     'value': '195', 'computed': 'skratch_counted', 'url': SKRATCH_COUNTRY},
    {'id': 'skratch-disputed', 'option': 'Skratch', 'property': 'disputed places listed',
     'value': '8', 'computed': 'skratch_disputed_count', 'url': SKRATCH_DISPUTED},
    {'id': 'skratch-other', 'option': 'Skratch', 'property': 'other territories listed apart',
     'value': '57', 'computed': 'skratch_other_count', 'url': SKRATCH_OTHER},
    {'id': 'skratch-mark', 'option': 'Skratch', 'property': 'how a country gets on the map',
     'value': 'you tap or search', 'url': SKRATCH_MARK,
     'quote': 'Tap the country or territory you’d like to mark'},
    {'id': 'skratch-camera', 'option': 'Skratch', 'property': 'photos sorted by country',
     'value': 'Skratch detects the location of uploaded photos and videos', 'url': SKRATCH_AS,
     'quote': ('Skratch detects the location of your content to help you create a timeline of'
               ' memories from each country you visit')},
    {'id': 'skratch-share', 'option': 'Skratch', 'property': 'how you share it',
     'value': 'image for Instagram, TikTok or Messages, or a copied link', 'url': SKRATCH_SHARE,
     'quote': 'choose how to share it: Instagram Tiktok Messages Copy link'},
    {'id': 'skratch-cheapest', 'option': 'Skratch', 'property': 'price, cheapest Power Up',
     'value': 'Map Themes $4.99', 'url': SKRATCH_AS, 'quote': 'Map Themes $4.99'},
    {'id': 'skratch-dearest', 'option': 'Skratch', 'property': 'price, dearest Power Up',
     'value': 'Power Ups Bundle $11.99', 'url': SKRATCH_AS, 'quote': 'Power Ups Bundle $11.99'},

    # --- Polarsteps ---
    {'id': 'polar-step', 'option': 'Polarsteps', 'property': 'how a country gets on the map',
     'value': 'a step you add on a trip, or a country added by hand', 'url': POLAR_COUNTRIES,
     'quote': ("Locations you have recorded in your trips don't automatically add the country to"
               ' the number of visited countries. This only happens when you add a step.')},
    {'id': 'polar-start', 'option': 'Polarsteps', 'property': 'when tracking starts',
     'value': 'not automatically; a trip is created and the Travel Tracker switched on by hand',
     'url': POLAR_START,
     'quote': ("Polarsteps doesn't automatically start to track your trip once you've installed"
               ' the app. You first have to create a trip and enable the Travel Tracker in order'
               ' to track your trip.')},
    {'id': 'polar-followers', 'option': 'Polarsteps', 'property': 'who sees the added countries',
     'value': 'you and your followers, not publicly', 'url': POLAR_COUNTRIES,
     'quote': ('Countries added through the scratch map feature are only visible to a you and'
               ' your followers, not publicly.')},
    {'id': 'polar-visibility', 'option': 'Polarsteps', 'property': 'who a trip is visible to',
     'value': 'just you, your followers, or everyone, chosen per trip', 'url': POLAR_VISIBLE,
     'quote': 'if a trip can be seen by just you, your followers, or by everyone'},
    {'id': 'polar-price', 'option': 'Polarsteps', 'property': 'price',
     'value': 'Free; Plus €29.99 a year', 'url': POLAR_PLUS,
     'quote': '"name": "Plus (Yearly)", "price": "29.99", "priceCurrency": "EUR"'},

    # --- the Android claim behind "why a country is missing" ---
    {'id': 'android-location', 'option': 'Android platform',
     'property': 'what the system does with a photo location', 'url': ANDROID_MEDIA,
     'value': 'hidden by default; unredacted EXIF needs ACCESS_MEDIA_LOCATION',
     'quote': 'If your app uses scoped storage , the system hides location information by default.'},

    # --- countries-visited.com ---
    {'id': 'cv-total', 'option': 'countries-visited.com', 'property': 'countries counted',
     'value': '250', 'computed': 'cv_total', 'url': CV},
    {'id': 'cv-scope', 'option': 'countries-visited.com', 'property': 'what the 250 are',
     'url': CV, 'value': 'every ISO 3166-1 entry plus Kosovo; 193 UN members, 57 others',
     'quote': ('193 of the 250 are UN member states; the other 57 include Greenland, Puerto Rico,'
               ' Hong Kong, Taiwan, Kosovo, Vatican City and Antarctica.')},
    {'id': 'cv-switch', 'option': 'countries-visited.com', 'property': 'UN-only switch', 'url': CV,
     'value': 'a switch counts only the 193 UN member states',
     'quote': 'set the list to “UN” to count only the 193 UN member states'},
    {'id': 'cv-mark', 'option': 'countries-visited.com', 'property': 'how a country gets on the map',
     'url': CV, 'value': 'you click the map or tick a list',
     'quote': ('you have visited on the map, or tick it in the checklist, and the counter above'
               ' the map adds them up as you go')},
    {'id': 'cv-share', 'option': 'countries-visited.com', 'property': 'how you share it', 'url': CV,
     'value': 'link, image, WhatsApp, X, Facebook, Pinterest, Reddit',
     'quote': ('Share your map Link Image Link to your map Copy Send it via WhatsApp X Facebook'
               ' Pinterest Reddit')},
    {'id': 'cv-free', 'option': 'countries-visited.com', 'property': 'price', 'url': CV,
     'value': 'free, no account, no paid tier documented',
     'quote': 'It is free, and there is no account to make.'},

    # --- Apple Photos and Google Photos, the two library claims ---
    {'id': 'apple-map', 'option': 'Apple Photos', 'property': 'a map of where photos were taken',
     'url': APPLE_PHOTOS, 'value': 'a map per collection, opened with Show Map',
     'quote': "You can browse a collection's photos and videos by location on a map."},
    {'id': 'google-places', 'option': 'Google Photos', 'property': 'pictures grouped into places',
     'url': GOOGLE_PHOTOS, 'value': 'a Places collection in the side navigation panel',
     'quote': 'like People & pets, Documents, Places, Videos, and more'},

    # --- the two sources the Atlas rules cite ---
    {'id': 'geonames', 'option': 'GeoNames', 'property': 'cities15000 dump size',
     'url': GEONAMES, 'value': 'about 25,000 places, population over 15,000 or a capital',
     'quote': 'cities15000.zip : all cities with a population > 15000 or capitals (ca 25.000)'},
    {'id': 'country-coder', 'option': 'country-coder', 'property': 'what it is',
     'url': COUNTRY_CODER,
     'value': 'an open-source lookup from a coordinate to an ISO 3166-1 code, run locally',
     'quote': 'Convert longitude-latitude pairs to ISO 3166-1 codes quickly and locally'},
    {'id': 'country-coder-license', 'option': 'country-coder', 'property': 'licence',
     'url': COUNTRY_CODER, 'value': 'ISC', 'quote': 'ISC license'},
]

NOTES = [
    'Polarsteps: the help-centre HTML page returns HTTP 403 to curl, so the articles are read'
    ' through the same help centre\'s public JSON endpoint'
    ' (support.polarsteps.com/api/v2/help_center/en-us/articles/<id>.json). The post links the'
    ' HTML page; the words are identical.',
    'Polarsteps publishes no country list or count on the pages read: its countries article'
    ' explains how to add a country and never defines what counts as one, so its count cell is'
    ' "not stated" rather than a number.',
    'Been publishes no country list anywhere on its site or App Store listing, only the count'
    ' ("250+ countries and territories") and a statement that disputed states are drawn'
    ' distinctly. Its per-place cells are therefore not documented, not absent.',
    'Visited publishes no country list either. Its listing shows a setting that counts by UN'
    ' sovereignty and one that lets the owner adjust disputed places, so its per-place cells are'
    ' not documented.',
    'countries-visited.com renders its 250 place names in the page HTML and states the'
    ' definition in FAQ structured data (JSON-LD) on the same page, which is why those quotes'
    ' come from text a reader sees only in the FAQ.',
    'Prices: Visited Pro $59.99 and the Been prices are read from the US App Store listings.'
    ' Polarsteps Plus is quoted in euros because that page prices by location and this machine'
    ' exits in Germany; the post states it in euros too. No Viallo price is quoted.',
    'Polarsteps Plus: the price is quoted from the Offer metadata in the page, because one read of'
    ' the same page returned a variant without the visible sentence that states it. The two agree'
    ' on €29.99 a year.',
    'Skratch\'s in-app purchase list and Been\'s render in a varying order between two reads of the'
    ' same page, so each price is quoted as one purchase rather than as a run of them. Skratch\'s'
    ' listing names no subscription; Been\'s and Visited\'s name several.',
    'Not checked: whether Been, Visited or countries-visited.com answer differently inside the'
    ' apps than on the pages read here. No account was created and no app was installed.',
    'Not checked: the date each Viallo album map first shipped. The component exists in the'
    ' repository and its history reaches back into January 2026, so the post no longer dates it.',
]


# ---------------------------------------------------------------- fetching


def fetch(url, cache):
    if url in cache:
        return cache[url]
    request = urllib.request.Request(url, headers=UA)
    try:
        with urllib.request.urlopen(request, timeout=TIMEOUT) as response:
            raw = response.read()
    except urllib.error.HTTPError as error:
        cache[url] = ('error', 'HTTP %s' % error.code)
        return cache[url]
    except Exception as error:  # noqa: BLE001 - the reason is reported, not raised
        cache[url] = ('error', type(error).__name__ + ': ' + str(error))
        return cache[url]
    charset = 'utf-8'
    if 'charset=' in (response.headers.get('content-type') or ''):
        charset = response.headers.get('content-type').split('charset=')[-1].split(';')[0].strip()
    try:
        text = raw.decode(charset, 'replace')
    except LookupError:
        text = raw.decode('utf-8', 'replace')
    cache[url] = ('ok', text)
    return cache[url]


def visible(raw):
    """Page text with tags, styles and ordinary scripts gone; ld+json is kept."""
    text = re.sub(r'<script(?![^>]*ld\+json).*?</script>', ' ', raw, flags=re.S | re.I)
    text = re.sub(r'<style.*?</style>', ' ', text, flags=re.S | re.I)
    text = re.sub(r'<!--.*?-->', ' ', text, flags=re.S)
    text = re.sub(r'<[^>]+>', ' ', text)
    text = html.unescape(html.unescape(text))
    return text


def norm(text):
    text = text.replace('’', "'").replace('‘', "'")
    text = text.replace('“', '"').replace('”', '"')
    text = text.replace('–', '-').replace('—', '-').replace(' ', ' ')
    text = text.replace('‑', '-').replace('‍', '')
    return re.sub(r'\s+', ' ', text).strip()


def page_text(url, cache):
    status, payload = fetch(url, cache)
    if status != 'ok':
        return None, payload
    if url.endswith('.json'):
        try:
            data = json.loads(payload)
        except ValueError as error:
            return None, 'not JSON: %s' % error
        if isinstance(data, dict) and 'article' in data:
            payload = data['article'].get('body', '')
    if url.endswith('readme.txt'):
        return norm(payload), None
    return norm(visible(payload)), None


# ---------------------------------------------------------------- repo inputs


def repo_file(path):
    return (REPO / path).read_text()


def between(raw, start, end):
    first = raw.find(start)
    if first < 0:
        return ''
    second = raw.find(end, first + len(start))
    return raw[first:second if second > 0 else len(raw)]


# ---------------------------------------------------------------- lists


def list_items(in_block):
    items = re.findall(r'<li[^>]*>(.*?)</li>', in_block, re.S)
    return [norm(html.unescape(re.sub(r'<[^>]+>', '', item))) for item in items]


def un_members(cache):
    status, raw = fetch(UN_MEMBERS, cache)
    if status != 'ok':
        raise RuntimeError('UN member page: %s' % raw)
    names = [norm(html.unescape(re.sub(r'<[^>]+>', '', heading)))
             for heading in re.findall(r'<h2[^>]*>(.*?)</h2>', raw, re.S)]
    return [name for name in names
            if name and name not in ('MEMBER STATES', 'Search the United Nations')]


def cv_places(cache):
    status, raw = fetch(CV, cache)
    if status != 'ok':
        raise RuntimeError('countries-visited.com: %s' % raw)
    return [norm(html.unescape(name))
            for name in re.findall(r'<span class="place-name">([^<]+)</span>', raw)]


def skratch_list(cache, url, start, end):
    status, raw = fetch(url, cache)
    if status != 'ok':
        raise RuntimeError('%s: %s' % (url, raw))
    return list_items(between(raw, start, end))


# ---------------------------------------------------------------- the measurement


def viallo_rule():
    rule = json.loads(repo_file('packages/shared/src/geo/data/rule.json'))
    country = repo_file('packages/shared/src/geo/country.ts')
    held_block = between(country, 'export type HeldBackArea =', ';')
    held = [norm(name) for name in re.findall(r"'([^']+)'", held_block)]
    if "iso1A2Code([lng, lat], { level: 'territory' })" not in country:
        raise RuntimeError('country.ts no longer resolves a point with country-coder')
    return rule, held


def compute(cache):
    members = un_members(cache)
    cv = cv_places(cache)
    disputed = skratch_list(cache, SKRATCH_DISPUTED, 'List of disputed territories', 'For more details')
    other = skratch_list(cache, SKRATCH_OTHER, 'List of other territories', 'Get in Touch')
    rule, held = viallo_rule()

    # Skratch names its rule rather than listing the countries: UN sovereign + observer states.
    skratch_counted = set(members) | OBSERVERS

    matrix = []
    for place, code in PLACES:
        viallo = 'absent'
        if place in held:
            viallo = 'not counted'
        elif code in set(rule['counted']):
            viallo = 'counted'
        elif code in set(rule['territories']):
            viallo = 'listed apart'

        skratch_name = NAME_IN['Skratch'].get(place, place)
        if skratch_name in disputed:
            skratch = 'disputed'
        elif skratch_name in other:
            skratch = 'listed apart'
        elif skratch_name in skratch_counted:
            skratch = 'counted'
        else:
            skratch = 'absent'

        cv_name = NAME_IN['countries-visited.com'].get(place, place)
        countries_visited = 'counted' if cv_name in cv else 'absent'

        cells = [viallo, skratch, countries_visited]
        matrix.append({
            'place': place,
            'viallo': viallo,
            'skratch': skratch,
            'countries_visited': countries_visited,
            'been': 'not documented',
            'visited': 'not documented',
            'agree': len(set(cells)) == 1,
        })

    extras = [place for place, code in PLACES
              if code in set(rule['counted']) and place not in members and place not in OBSERVERS]
    beyond = len(rule['counted']) - len(members) - len(OBSERVERS)
    if beyond != len(extras):
        raise RuntimeError('%d codes are counted beyond the UN list, %d probed places explain them'
                           % (beyond, len(extras)))

    places = json.loads(repo_file('packages/shared/src/geo/data/places-15000.v2.json'))
    city_distance = re.search(r'const MAX_DISTANCE_KM = (\d+);',
                             repo_file('packages/shared/src/geo/nearest-city.ts'))
    tiers = repo_file('packages/shared/src/subscription/tiers.ts')
    free = tiers[tiers.find('[SubscriptionTier.FREE]: {'):][:1200]
    free_albums = re.search(r'maxAlbums: (\d+)', free)
    free_photos = re.search(r'maxPhotos: (\d+)', free)
    if not (city_distance and free_albums and free_photos):
        raise RuntimeError('the city distance or the Free tier limits could not be read')
    compute_ts = repo_file('apps/api/src/atlas/atlas-compute.ts')
    if 'countryOfPoint(lng, lat)' not in compute_ts:
        raise RuntimeError('atlas-compute no longer resolves a photo with countryOfPoint')
    if 'cities15000' not in repo_file('apps/web/scripts/build-places.mjs'):
        raise RuntimeError('build-places.mjs no longer builds the table from cities15000')
    share = repo_file('apps/web/src/components/atlas/share-ui/AtlasShareModal.tsx')
    for tab in ("key: 'image'", "key: 'link'", "key: 'embed'"):
        if tab not in share:
            raise RuntimeError('the Atlas share modal no longer has a %s tab' % tab)

    disagree = [row['place'] for row in matrix if not row['agree']]
    return {
        'un_members': len(members),
        'cv_total': len(cv),
        'skratch_counted': len(skratch_counted),
        'skratch_disputed_count': len(disputed),
        'skratch_other_count': len(other),
        'viallo_counted': len(rule['counted']),
        'viallo_extra': '%d (%s)' % (beyond, ', '.join(extras)),
        'viallo_territories': len(rule['territories']),
        'viallo_held': '%d (%s)' % (len(held), ', '.join(held)),
        'viallo_places': len(places),
        'viallo_city_distance': int(city_distance.group(1)),
        'viallo_free': '%s albums, %s photos' % (free_albums.group(1), free_photos.group(1)),
        'viallo_share': 'image, link, website widget',
        'viallo_mechanism': "from the coordinates in each photo, resolved on Viallo's own servers",
        'matrix': matrix,
        'disagree_count': len(disagree),
        'agree_places': [row['place'] for row in matrix if row['agree']],
        'disagree_places': disagree,
    }


# ---------------------------------------------------------------- emit / verify


def row_with_measurement(row, measured):
    out = {key: row[key] for key in ('id', 'option', 'property', 'value', 'url', 'quote')
           if key in row}
    if 'computed' in row:
        value = measured[row['computed']]
        out['value'] = str(value)
        out['measured'] = value
    return out


def emit(cache):
    measured = compute(cache)
    rows = [row_with_measurement(row, measured) for row in ROWS]
    dataset = {
        'slug': 'countries-visited-map',
        'method': 'public-web',
        'checkedAt': CHECKED_AT,
        'tools': {'python': sys.version.split()[0], 'curl': '8.7.1', 'stdlib only': True},
        'inputs': ([{'kind': 'page', 'url': url} for url in PAGES]
                   + [{'kind': 'repo', 'path': path} for path in REPO_INPUTS]),
        'results': rows,
        'matrix': measured['matrix'],
        'totals': {
            'Viallo Atlas': {'counted': measured['viallo_counted'],
                             'listed apart': measured['viallo_territories'],
                             'held back': measured['viallo_held']},
            'Skratch': {'counted': measured['skratch_counted'],
                        'disputed': measured['skratch_disputed_count'],
                        'listed apart': measured['skratch_other_count']},
            'countries-visited.com': {'counted': measured['cv_total'],
                                      'of which UN members': measured['un_members']},
            'Been': {'counted': '250+ claimed, no list published'},
            'Visited': {'counted': 'not published'},
            'Polarsteps': {'counted': 'not stated'},
        },
        'disagreement': {'places checked': len(measured['matrix']),
                         'places the three list publishers agree on': measured['agree_places'],
                         'places they disagree on': measured['disagree_count']},
        'notes': NOTES,
    }
    DATASET_JSON.write_text(json.dumps(dataset, indent=2, ensure_ascii=False) + '\n')
    with DATASET_CSV.open('w', newline='') as handle:
        writer = csv.writer(handle)
        writer.writerow(['option', 'property', 'value', 'quote', 'url'])
        for row in rows:
            writer.writerow([row['option'], row['property'], row['value'],
                             row.get('quote', ''), row['url']])
        writer.writerow(['all apps', 'places checked', str(len(measured['matrix'])), '', ''])
        writer.writerow(['all apps', 'places the three list publishers disagree on',
                         str(measured['disagree_count']), '', ''])
        cell_url = {
            'viallo': 'repo:packages/shared/src/geo/data/rule.json',
            'countries_visited': CV,
            'been': BEEN,
            'visited': VISITED_AS,
        }
        cell_label = {
            'viallo': 'Viallo Atlas',
            'skratch': 'Skratch',
            'countries_visited': 'countries-visited.com',
            'been': 'Been',
            'visited': 'Visited',
        }
        for row in measured['matrix']:
            for key in ('viallo', 'skratch', 'countries_visited', 'been', 'visited'):
                if key == 'skratch':
                    url = {'disputed': SKRATCH_DISPUTED, 'listed apart': SKRATCH_OTHER,
                           'counted': SKRATCH_COUNTRY, 'absent': SKRATCH_OTHER}[row[key]]
                else:
                    url = cell_url[key]
                writer.writerow([cell_label[key], 'place: ' + row['place'], row[key], '', url])
    print('wrote %s and %s' % (DATASET_JSON.name, DATASET_CSV.name))
    return 0


def verify(cache):
    failures = []
    dataset = json.loads(DATASET_JSON.read_text())
    measured = compute(cache)

    for row in ROWS:
        if 'quote' not in row:
            continue
        text, error = page_text(row['url'], cache)
        if text is None:
            failures.append('%s: %s not checked (%s)' % (row['id'], row['url'], error))
            continue
        if norm(row['quote']) in text:
            continue
        # An App Store page renders its in-app purchases in a different order each
        # time, so one miss is re-read once before it counts as gone.
        cache.pop(row['url'], None)
        text, error = page_text(row['url'], cache)
        if text is None or norm(row['quote']) not in text:
            failures.append('%s: quote gone from %s\n    quote: %s'
                            % (row['id'], row['url'], row['quote']))

    stored = {row['id']: row for row in dataset['results']}
    for row in ROWS:
        if row['id'] not in stored:
            failures.append('%s: missing from dataset.json' % row['id'])
            continue
        if 'computed' in row:
            value = str(measured[row['computed']])
            if stored[row['id']]['value'] != value:
                failures.append('%s: dataset says %r, measurement says %r'
                                % (row['id'], stored[row['id']]['value'], value))
        if stored[row['id']].get('quote', '') != row.get('quote', ''):
            failures.append('%s: dataset quote differs from the checked quote' % row['id'])

    if dataset['matrix'] != measured['matrix']:
        for wanted, got in zip(dataset['matrix'], measured['matrix']):
            if wanted != got:
                failures.append('matrix %s: dataset %r, measurement %r'
                                % (wanted['place'], wanted, got))
    if len(dataset['matrix']) != len(measured['matrix']):
        failures.append('matrix length changed')

    for row in measured['matrix']:
        print('  %-16s viallo=%-13s skratch=%-13s countries-visited=%-8s agree=%s'
              % (row['place'], row['viallo'], row['skratch'], row['countries_visited'],
                 row['agree']))
    print('  countable totals: Viallo %d, Skratch %d, countries-visited.com %d, UN %d'
          % (measured['viallo_counted'], measured['skratch_counted'],
             measured['cv_total'], measured['un_members']))
    print('  places all three list publishers disagree on: %d of %d'
          % (measured['disagree_count'], len(measured['matrix'])))

    if failures:
        print('\nFAIL')
        for failure in failures:
            print(' - ' + failure)
        return 1
    print('\nOK: %d quotes, %d measured rows, %d places hold'
          % (sum(1 for row in ROWS if 'quote' in row),
             sum(1 for row in ROWS if 'computed' in row), len(measured['matrix'])))
    return 0


def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument('--recheck', action='store_true',
                        help='re-read every cited page; there is no lab step to skip')
    parser.add_argument('--emit', action='store_true', help='rewrite the dataset from this run')
    args = parser.parse_args()
    cache = {}
    if args.emit:
        return emit(cache)
    return verify(cache)


if __name__ == '__main__':
    sys.exit(main())
