#!/usr/bin/env python3
"""Re-run the best-photo-sharing-app-iphone primary-docs check.

Fetches every URL in dataset.json and reports, per option and property,
whether each supporting quote still appears on the vendor's page. A url of
the form repo:<path> is read from this repository instead of fetched. A page
that refuses the request (HTTP 4xx/5xx) is reported BLOCKED: a refusal says
nothing about the quote, so it does not fail the check.
Usage: python3 check.py [path/to/dataset.json]
"""
import html
import json
import re
import subprocess
import sys
from pathlib import Path

UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 14_0) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Safari/605.1.15'


def fetch(url):
    proc = subprocess.run(
        ['curl', '-sL', '--compressed', '-m', '30', '-A', UA, '-H', 'Accept-Language: en-US,en',
         '-w', '\n%{http_code}', url],
        capture_output=True,
    )
    body, _, code = proc.stdout.decode('utf-8', 'ignore').rpartition('\n')
    return code.strip(), body


REPO = Path(__file__).resolve().parents[5]


def to_text(page):
    # JSON-LD is kept: Google One renders its prices in the browser and states
    # them, in USD, only in the page's structured data.
    ld = ' '.join(re.findall(r'(?is)<script[^>]*application/ld\+json[^>]*>(.*?)</script>', page))
    page = re.sub(r'(?is)<(script|style)[^>]*>.*?</\1>', ' ', page) + ' ' + ld
    page = re.sub(r'<[^>]+>', ' ', page)
    return re.sub(r'\s+', ' ', html.unescape(page))


def norm(s):
    return re.sub(r'\s+', ' ', s.replace('’', "'").replace(' ', ' ')).strip().lower()


def as_list(v):
    return v if isinstance(v, list) else [v]


def main():
    # --recheck only re-reads the pages, which is all this check does.
    args = [a for a in sys.argv[1:] if a != '--recheck']
    path = Path(args[0]) if args else Path(__file__).with_name('dataset.json')
    data = json.loads(path.read_text())
    pages = {}
    for url in data["inputs"] + [q["url"] for q in data.get("extraQuotes", [])]:
        if url.startswith('repo:'):
            pages[url] = ('repo', norm((REPO / url[5:]).read_text()))
            continue
        code, body = fetch(url)
        pages[url] = (code, norm(to_text(body)))
    found = missing = skipped = blocked = 0
    print(f"checkedAt in dataset: {data['checkedAt']}")
    for row in data['results']:
        for prop in data['properties']:
            cell = row[prop]
            label = f"{row['option']} / {prop}"
            if not cell.get('quote') or not cell.get('url'):
                skipped += 1
                print(f"SKIP     {label}: {cell['value']}")
                continue
            for quote, url in zip(as_list(cell['quote']), as_list(cell['url'])):
                code, text = pages[url]
                if code != 'repo' and not code.startswith('2'):
                    blocked += 1
                    print(f"BLOCKED  {label}: {url} answered HTTP {code}, quote not checked")
                elif norm(quote) in text:
                    found += 1
                    print(f"FOUND    {label} (HTTP {code})")
                else:
                    missing += 1
                    print(f"MISSING  {label} (HTTP {code}): {quote!r} at {url}")
    for extra in data.get('extraQuotes', []):
        code, text = pages[extra['url']]
        if code != 'repo' and not code.startswith('2'):
            blocked += 1
            print(f"BLOCKED  {extra['claim']}: {extra['url']} answered HTTP {code}, quote not checked")
        elif norm(extra['quote']) in text:
            found += 1
            print(f"FOUND    {extra['claim']} (HTTP {code})")
        else:
            missing += 1
            print(f"MISSING  {extra['claim']} (HTTP {code}): {extra['quote']!r} at {extra['url']}")
    print(f"\n{found} quotes found, {missing} missing, {blocked} blocked, {skipped} cells without a quote")
    return 1 if missing else 0


if __name__ == '__main__':
    sys.exit(main())