#!/usr/bin/env python3
"""Re-run the travel-map check.

For every row in dataset.json: fetch the cited page, or read the cited file out of
this repository, and report whether the sentence that backs the cell is still there.
The Apple rows are also re-checked the other way round: the phrases that must NOT
appear on its Shared Albums pages are searched for, so a map Apple adds later turns
the row from "Not documented" into a failure to look at.

    python3 check.py            # everything
    python3 check.py --recheck  # same; the check is page reads only, there is no lab step

Exit 0 when every quote still holds and every absence still holds, 1 otherwise.
Standard library only.
"""

import csv
import html
import json
import os
import re
import sys
import urllib.error
import urllib.request
from pathlib import Path

ROOT = Path(__file__).resolve().parent
# apps/web/public/blog/travel-map -> repository root
REPO = ROOT.parents[4]
UA = ('Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 '
      '(KHTML, like Gecko) Chrome/126.0 Safari/537.36')


def normalise(text):
    """Flatten entities, tags, curly quotes, dashes and whitespace so two quotes compare equal."""
    text = html.unescape(text)
    text = re.sub(r'<(script|style|noscript)\b.*?</\1>', ' ', text, flags=re.S | re.I)
    text = re.sub(r'<[^>]+>', ' ', text)
    for a, b in (('’', "'"), ('‘', "'"), ('“', '"'), ('”', '"'),
                 ('–', '-'), ('—', '-'), (' ', ' ')):
        text = text.replace(a, b)
    text = re.sub(r'\s+', ' ', text)
    text = re.sub(r'\s+([,.;:!?)])', r'\1', text)
    text = re.sub(r'\(\s+', '(', text)
    return text.strip().lower()


def fetch(url):
    req = urllib.request.Request(url, headers={'User-Agent': UA,
                                               'Accept-Language': 'en-US,en;q=0.9'})
    with urllib.request.urlopen(req, timeout=45) as resp:
        charset = resp.headers.get_content_charset() or 'utf-8'
        return resp.status, resp.read().decode(charset, 'replace')


def page_text(url, pages):
    if url not in pages:
        try:
            pages[url] = fetch(url)
        except urllib.error.HTTPError as exc:
            pages[url] = (exc.code, '')
        except Exception as exc:  # noqa: BLE001 - any transport failure is a failed read
            pages[url] = (0, '')
            print('    fetch error for %s: %s' % (url, exc))
    status, body = pages[url]
    if status != 200 or not body:
        raise RuntimeError('HTTP %s from %s' % (status, url))
    return normalise(body)


def file_text(path, cache):
    if path not in cache:
        target = REPO / path
        if not target.exists():
            raise RuntimeError('no such file: %s' % target)
        cache[path] = normalise(target.read_text(encoding='utf-8'))
    return cache[path]


def main():
    recheck = '--recheck' in sys.argv
    data = json.loads((ROOT / 'dataset.json').read_text(encoding='utf-8'))

    pages, files = {}, {}
    found = missing = skipped = failed = 0

    print('travel-map, %d rows, checked %s%s' % (
        len(data['results']), data['checkedAt'], ' (recheck)' if recheck else ''))

    def check_quote(label, source):
        nonlocal found, missing
        quote = source['quote']
        try:
            if 'url' in source:
                haystack = page_text(source['url'], pages)
                where = source['url']
            else:
                haystack = file_text(source['file'], files)
                where = source['file']
        except RuntimeError as exc:
            missing += 1
            print('  MISSING %-58s %s' % (label, exc))
            return
        if normalise(quote) in haystack:
            found += 1
            print('  FOUND   %-58s %s' % (label, where))
        else:
            missing += 1
            print('  MISSING %-58s %r at %s' % (label, quote[:70], where))

    for row in data['results']:
        for prop in data['properties']:
            cell = row[prop]
            for source in cell['sources']:
                check_quote('%s / %s' % (row['option'], prop), source)

    print('\nphrases that must still be absent')
    for entry in data.get('absence') or []:
        try:
            haystack = page_text(entry['url'], pages)
        except RuntimeError as exc:
            failed += 1
            print('  FAIL    %-58s %s' % (entry['option'], exc))
            continue
        present = [p for p in entry['phrases'] if normalise(p) in haystack]
        if present:
            failed += 1
            print('  FAIL    %-58s %r now appears on %s' % (
                entry['option'], present[0], entry['url']))
        else:
            found += 1
            print('  OK      %-58s none of %d phrases on %s' % (
                entry['option'], len(entry['phrases']), entry['url']))

    print('\nsupporting facts')
    for fact in data.get('supporting') or []:
        for quote in fact['quotes']:
            check_quote('supporting / %s' % fact['claim'][:40],
                        {'url': fact['url'], 'quote': quote})

    print('\ndataset.csv mirrors dataset.json')
    csv_path = ROOT / 'dataset.csv'
    if not csv_path.exists():
        failed += 1
        print('  FAIL    dataset.csv is missing')
    else:
        with csv_path.open(encoding='utf-8', newline='') as handle:
            table = {r['option']: r for r in csv.DictReader(handle)}
        drift = 0
        for row in data['results']:
            cells = table.get(row['option'])
            if cells is None:
                drift += 1
                print('  FAIL    csv has no row for %s' % row['option'])
                continue
            pairs = [
                ('map_for_shared_viewer', row['map_for_shared_viewer']['value']),
                ('how_the_map_gets_there', row['how_the_map_gets_there']['value']),
                ('supporting_quote', row['map_for_shared_viewer']['sources'][0]['quote']),
                ('checked_at', data['checkedAt']),
            ]
            for column, expected in pairs:
                if normalise(cells[column]) != normalise(expected):
                    drift += 1
                    print('  FAIL    csv %s / %s carries %r, dataset says %r' % (
                        row['option'], column, cells[column][:50], expected[:50]))
        if drift:
            failed += drift
        else:
            found += 1
            print('  OK      %d rows agree' % len(table))

    if skipped:
        print('\n%d skipped' % skipped)
    print('\n%d checks passed, %d failed' % (found, missing + failed))
    return 1 if (missing or failed) else 0


if __name__ == '__main__':
    sys.exit(main())