#!/usr/bin/env python3
"""Re-runs the primary-docs check for the post on Google Photos' Ask Photos search.

Fetches every page in dataset.json and reports, per table cell and per fact the
post states, whether the recorded quote still appears on that page. It also checks
that dataset.csv carries the same values as dataset.json.

This post has no lab or device step. The one account leg (searching in Ask Photos
on a Google account, 2026-10-25) needed a signed-in browser; it is printed from
the dataset's "recorded" list and never re-run.

    python3 check.py             full run
    python3 check.py --recheck   the same: every step re-reads the pages, none takes a minute

Exits 0 when every quote is found and the CSV matches, 1 otherwise.
Standard library only.
"""
import csv
import html
import json
import os
import re
import sys
import unicodedata
import urllib.error
import urllib.parse
import urllib.request

HERE = os.path.dirname(os.path.abspath(__file__))
DATASET = os.path.join(HERE, "dataset.json")
CSV_PATH = os.path.join(HERE, "dataset.csv")

UA = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
      "(KHTML, like Gecko) Chrome/126.0 Safari/537.36")
TIMEOUT = 45

QUOTE_MAP = {
    "\u2018": "'", "\u2019": "'", "\u201a": "'", "\u201b": "'",
    "\u201c": '"', "\u201d": '"', "\u201e": '"', "\u201f": '"',
    "\u00ab": '"', "\u00bb": '"',
    "\u2010": "-", "\u2011": "-", "\u2012": "-", "\u2013": "-", "\u2014": "-",
    "\u2015": "-", "\u2212": "-",
    "\u00a0": " ", "\u2002": " ", "\u2003": " ", "\u2009": " ", "\u202f": " ",
    "\u200b": "", "\ufeff": "",
}
QUOTE_RE = re.compile("|".join(re.escape(k) for k in QUOTE_MAP))


def normalise(text):
    text = QUOTE_RE.sub(lambda m: QUOTE_MAP[m.group(0)], text)
    text = unicodedata.normalize("NFKC", text)
    text = re.sub(r"\s+", " ", text)
    return text.strip()


def strip_html(raw, sep):
    raw = re.sub(r"(?is)<(script|style|noscript|svg|template)\b.*?</\1>", " ", raw)
    raw = re.sub(r"(?is)<!--.*?-->", " ", raw)
    return html.unescape(re.sub(r"(?s)<[^>]+>", sep, raw))


def variants(raw):
    # Tags as spaces, and tags removed: Google's help splits sentences across inline links.
    return [normalise(strip_html(raw, " ")), normalise(strip_html(raw, ""))]


def fetch(url, hops=4):
    # urllib follows 301/302/303/307 but not 308, which the docs hosts use.
    req = urllib.request.Request(url, headers={"User-Agent": UA, "Accept-Language": "en"})
    try:
        with urllib.request.urlopen(req, timeout=TIMEOUT) as resp:
            body = resp.read()
            charset = resp.headers.get_content_charset() or "utf-8"
            return body.decode(charset, errors="replace")
    except urllib.error.HTTPError as exc:
        target = exc.headers.get("Location") if exc.headers and exc.code in (308, 301, 302, 303, 307) else None
        if not target or hops == 0:
            raise
        return fetch(urllib.parse.urljoin(url, target), hops - 1)


def check_csv(dataset):
    if not os.path.exists(CSV_PATH):
        return ["dataset.csv is missing"]
    with open(CSV_PATH, newline="", encoding="utf-8") as fh:
        rows = list(csv.DictReader(fh))
    problems = []
    if [r["option"] for r in rows] != dataset["options"]:
        problems.append("dataset.csv options differ from dataset.json")
    for row in rows:
        for col in dataset["columns"]:
            want = [r["value"] for r in dataset["results"] if r["option"] == row["option"] and r["property"] == col]
            if want != [row.get(col)]:
                problems.append("dataset.csv %s / %s differs from dataset.json" % (row["option"], col))
    return problems


def main(argv):
    recheck = "--recheck" in argv
    with open(DATASET, encoding="utf-8") as fh:
        dataset = json.load(fh)

    print("check: %s, method %s, checkedAt %s" % (dataset["slug"], dataset["method"], dataset["checkedAt"]))
    print("mode: %s" % ("recheck" if recheck else "full"))

    rows = [("%s / %s" % (r["option"], r["property"]), r["quote"], r["url"]) for r in dataset["results"]]
    rows += [("fact / %s" % f["id"], f["quote"], f["url"]) for f in dataset["facts"]]

    pages, failures = {}, []
    for url in dict.fromkeys(u for _, _, u in rows):
        try:
            pages[url] = variants(fetch(url))
        except Exception as exc:  # HTTP, network, DNS, timeout, decode
            pages[url] = None
            failures.append("%s -> %s: %s" % (url, type(exc).__name__, exc))

    gone = 0
    for label, quote, url in rows:
        hay = pages.get(url)
        if hay is None:
            print("FETCH-FAIL %-58s %s" % (label, url))
            continue
        found = any(normalise(quote) in h for h in hay)
        print("%-10s %-58s %s" % ("PASS" if found else "FAIL", label, url))
        if not found:
            gone += 1
            failures.append("quote gone: %s: %r" % (label, quote[:90]))

    for leg in dataset.get("recorded", []):
        print("RECORDED   %s %s: %s (not re-run)" % (leg["leg"], leg["date"], leg["result"]))

    for problem in check_csv(dataset):
        print("FAIL csv: %s" % problem)
        failures.append(problem)

    print("\n%d of %d quotes found, %d problem(s)" % (len(rows) - gone, len(rows), len(failures)))
    for failure in failures:
        print("  - %s" % failure)
    return 0 if not failures else 1


if __name__ == "__main__":
    sys.exit(main(sys.argv[1:]))