#!/usr/bin/env python3
"""Re-run the best-photo-sharing-app-travel primary-docs check.

Fetches every URL in dataset.json and reports, per option and property and for
every supporting fact, whether the quote that backs it still appears on the page.
Usage: python3 check.py [path/to/dataset.json]
"""
import html
import json
import re
import subprocess
import sys
from pathlib import Path

UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 14_0) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Safari/605.1.15'


def fetch(url):
    proc = subprocess.run(
        ['curl', '-sL', '--compressed', '-m', '30', '-A', UA, '-H', 'Accept-Language: en-US,en',
         '-w', '\n%{http_code}', url],
        capture_output=True,
    )
    body, _, code = proc.stdout.decode('utf-8', 'ignore').rpartition('\n')
    return code.strip(), body


def to_text(page):
    page = re.sub(r'(?is)<(script|style)[^>]*>.*?</\1>', ' ', page)
    page = re.sub(r'<[^>]+>', ' ', page)
    return re.sub(r'\s+', ' ', html.unescape(page))


def norm(s):
    s = s.replace('’', "'").replace('“', '"').replace('”', '"').replace(' ', ' ')
    return re.sub(r'\s+', ' ', s).strip().lower()


def main():
    # --recheck only re-reads the pages, which is all this check does.
    args = [a for a in sys.argv[1:] if a != '--recheck']
    path = Path(args[0]) if args else Path(__file__).with_name('dataset.json')
    data = json.loads(path.read_text())
    pages = {}
    for url in data['inputs']:
        code, body = fetch(url)
        pages[url] = (code, norm(to_text(body)))
        print(f"HTTP {code}  {url}")
    print(f"\ncheckedAt in dataset: {data['checkedAt']}\n")
    found = missing = skipped = 0

    def check(label, quotes, value):
        nonlocal found, missing, skipped
        if not quotes:
            skipped += 1
            print(f"SKIP     {label}: {value}")
            return
        for q in quotes:
            code, text = pages[q['url']]
            if norm(q['quote']) in text:
                found += 1
                print(f"FOUND    {label} (HTTP {code})")
            else:
                missing += 1
                print(f"MISSING  {label} (HTTP {code}): {q['quote']!r} at {q['url']}")

    for row in data['results']:
        for prop in data['properties']:
            cell = row[prop]
            check(f"{row['option']} / {prop}", cell['quotes'], cell['value'])
    for fact in data.get('supporting', []):
        check(f"supporting / {fact['claim']}", [{'url': fact['url'], 'quote': q} for q in fact['quotes']], '')
    print(f"\n{found} quotes found, {missing} missing, {skipped} cells without a quote")
    return 1 if missing else 0


if __name__ == '__main__':
    sys.exit(main())