#!/usr/bin/env python3
"""Re-run the check behind the Viallo post sports-team-photo-sharing.

Method: primary-docs. Every published cell of the comparison the post prints is
one sentence on the service's own help or developer page, read on 2026-10-26.
Viallo's cells and the product claims come from its own source at origin/main
through `git show`. One statement (S2) lives in a PDF and is converted with
pdftotext before the sentence is searched for.

    python3 check.py            # fetch every page, read every source
    python3 check.py --recheck  # same; no lab step exists, so identical

A cell recorded with status "not-checked" is printed with its reason and is a
recorded gap, not a failure: the page refused the fetch or does not state the
property. A page that has become unreachable, or a sentence that has gone from
it, is a failure.

Viallo rows are read at origin/main of the repository this file sits in; outside
a clone they are reported as skipped and do not count as failures.

Exit 0 when every recorded quote still appears where it was recorded, 1
otherwise. Standard library only; pdftotext is an optional external binary and
its absence is reported, never assumed.
"""

import gzip
import html
import json
import os
import re
import shutil
import subprocess
import sys
import tempfile
import urllib.error
import urllib.request

HERE = os.path.dirname(os.path.abspath(__file__))
DATASET = os.path.join(HERE, "dataset.json")
UA = (
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
    "(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
)
# Facebook answers 400 to the full Chrome string on its help host and 200 to a bare
# token; an entry that names its own user agent overrides the default.
UA_FACEBOOK = "Mozilla/5.0"
TIMEOUT = 60
REMAINDER = " (--recheck: pages re-read; this check has no lab step)"

QUOTES = {
    "‘": "'", "’": "'", "‚": "'", "‛": "'",
    "“": '"', "”": '"', "„": '"', "‟": '"',
    "«": '"', "»": '"', "′": "'", "″": '"',
}
SPACES = "     ​"


def normalise(text):
    text = html.unescape(text)
    for bad, good in QUOTES.items():
        text = text.replace(bad, good)
    for bad in SPACES:
        text = text.replace(bad, " ")
    return re.sub(r"\s+", " ", text).strip()


def page_text(raw):
    raw = re.sub(r"(?is)<(script|style|noscript|template|svg)\b[^>]*>.*?</\1>", " ", raw)
    raw = re.sub(r"(?s)<!--.*?-->", " ", raw)
    raw = re.sub(r"(?s)<[^>]+>", "\n", raw)
    return normalise(raw)


def fetch(url, ua=UA):
    req = urllib.request.Request(
        url,
        headers={
            "User-Agent": ua,
            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
            "Accept-Language": "en-US,en;q=0.9",
            # The eCFR endpoint refuses any request that does not permit compression.
            "Accept-Encoding": "gzip",
        },
    )
    with urllib.request.urlopen(req, timeout=TIMEOUT) as resp:
        body = resp.read()
        if resp.headers.get("Content-Encoding", "").lower() == "gzip":
            body = gzip.decompress(body)
        charset = resp.headers.get_content_charset() or "utf-8"
        return body.decode(charset, "replace")


def pdf_text(url):
    if shutil.which("pdftotext") is None:
        raise RuntimeError("pdftotext is not installed, so the PDF was not searched")
    req = urllib.request.Request(url, headers={"User-Agent": UA})
    with urllib.request.urlopen(req, timeout=TIMEOUT) as resp:
        body = resp.read()
    with tempfile.NamedTemporaryFile(suffix=".pdf", delete=False) as fh:
        fh.write(body)
        path = fh.name
    try:
        out = subprocess.run(["pdftotext", "-layout", path, "-"],
                             capture_output=True, check=True)
        return normalise(out.stdout.decode("utf-8", "replace"))
    finally:
        os.unlink(path)


def repo_root():
    try:
        out = subprocess.run(["git", "-C", HERE, "rev-parse", "--show-toplevel"],
                             capture_output=True, check=True)
    except (OSError, subprocess.CalledProcessError):
        return None
    return out.stdout.decode().strip() or None


VIALLO_REPO = repo_root()


def git_file(path):
    out = subprocess.run(["git", "-C", VIALLO_REPO, "show", "origin/main:" + path],
                         capture_output=True, check=True)
    return normalise(out.stdout.decode("utf-8", "replace"))


def entry_text(entry, cache):
    """Return the haystack for one entry, or raise RuntimeError."""
    source = entry.get("source") or ("git" if entry.get("path") else "url")
    if source == "git":
        if VIALLO_REPO is None:
            raise RuntimeError("needs a clone of the Viallo repository")
        key = "git:" + entry["path"]
        if key not in cache:
            cache[key] = git_file(entry["path"])
        return cache[key]
    url = entry["url"]
    key = "url:{}|{}".format(entry.get("ua", UA), url)
    if key not in cache:
        cache[key] = pdf_text(url) if url.lower().endswith(".pdf") else page_text(fetch(url, entry.get("ua", UA)))
    return cache[key]


def main():
    recheck = "--recheck" in sys.argv[1:]
    with open(DATASET, encoding="utf-8") as fh:
        data = json.load(fh)

    rows = []
    for cell in data["results"]:
        rows.append((cell, "{} / {}".format(cell["option"], cell["property"])))
    for claim in data.get("statements", []):
        rows.append((claim, "statement {}".format(claim["id"])))
    for claim in data.get("product", []):
        rows.append((dict(claim, source="git"), "product {}".format(claim["id"])))

    cache = {}
    failures = 0
    checked = 0
    skipped = 0
    for entry, label in rows:
        if entry.get("status") == "not-checked":
            skipped += 1
            print("skip  {}: {}".format(label, entry.get("reason", "not checked")))
            continue
        try:
            haystack = entry_text(entry, cache)
        except Exception as exc:  # noqa: BLE001 - every failure is reported as a line
            failures += 1
            print("FAIL  {}: {}".format(label, exc))
            continue
        checked += 1
        quote = normalise(entry.get("quote", ""))
        if quote and quote in haystack:
            print("ok    {}".format(label))
        else:
            failures += 1
            print("FAIL  {}: quote gone from {}".format(
                label, entry.get("path") or entry.get("url")))
            print("      quote: {}".format(quote[:160]))

    print()
    print("{} checked, {} failed, {} recorded as not checked, on {}".format(
        checked, failures, skipped, data["checkedAt"]) + (REMAINDER if recheck else ""))
    return 0 if failures == 0 else 1


if __name__ == "__main__":
    sys.exit(main())