#!/usr/bin/env python3
"""Re-run the evidence check behind the viallo-android-app post.

The production counts come from query.js, run read-only against the Viallo
database on 2026-10-26; this script cannot reach production. It checks that
the dataset's own numbers agree with each other and with dataset.csv, that
every quoted page still carries its sentence and, when run inside a checkout
of the repository, that every quoted source line is still there. Exits 0 when
everything holds and 1 otherwise.

    python3 check.py            # full run
    python3 check.py --recheck  # re-read the pages; there is no lab step to skip
"""
import argparse
import csv
import html
import http.cookiejar
import json
import os
import re
import sys
import time
import urllib.error
import urllib.request

HERE = os.path.dirname(os.path.abspath(__file__))
UA = (
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
    "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0 Safari/537.36"
)
OPENER = urllib.request.build_opener(
    urllib.request.HTTPCookieProcessor(http.cookiejar.CookieJar())
)
TRANSLATE = {
    "\u00a0": " ", "\u2018": "'", "\u2019": "'", "\u201c": '"', "\u201d": '"',
    "\u2013": "-", "\u2014": "-", "\u2011": "-", "\u2212": "-", "\u2026": "...",
}
MIN_ACCOUNTS = 10


def normalise(text):
    text = html.unescape(html.unescape(text))
    for src, dst in TRANSLATE.items():
        text = text.replace(src, dst)
    return re.sub(r"\s+", " ", text).strip()


def strip_tags(text):
    text = re.sub(r"(?is)<script.*?</script>", " ", text)
    text = re.sub(r"(?is)<style.*?</style>", " ", text)
    text = re.sub(r"(?is)<!--.*?-->", " ", text)
    return re.sub(r"(?s)<[^>]+>", " ", text)


def fetch(url, attempts=3):
    last = "no attempt made"
    for attempt in range(attempts):
        try:
            request = urllib.request.Request(
                url, headers={"User-Agent": UA, "Accept-Language": "en-US,en;q=0.9"}
            )
            with OPENER.open(request, timeout=45) as response:
                return response.status, response.read().decode("utf-8", "replace")
        except urllib.error.HTTPError as exc:
            return exc.code, ""
        except Exception as exc:
            last = "%s: %s" % (type(exc).__name__, exc)
            time.sleep(2 * (attempt + 1))
    return 0, last


def check_sums(dataset):
    out = []
    pd = dataset["productData"]
    rows = {r["id"]: r for r in pd["rows"]}
    for r in pd["rows"]:
        problems = []
        if r["accounts"] < MIN_ACCOUNTS:
            problems.append("fewer than %d accounts" % MIN_ACCOUNTS)
        if r["longEdgeUnder2048"] > r["uploads"]:
            problems.append("more small uploads than uploads")
        if round(100 * r["longEdgeUnder2048"] / r["uploads"], 1) != r["longEdgeUnder2048Pct"]:
            problems.append("share under 2048 px does not match its counts")
        for key in ("medianMegapixels", "medianOfAccountMedianMegapixels"):
            if round(r[key + "Raw"], 1) != r[key]:
                problems.append("%s is not the rounded raw value" % key)
        if round(r["medianMBRaw"], 2) != r["medianMB"]:
            problems.append("medianMB is not the rounded raw value")
        if r["medianShortEdgePx"] > r["medianLongEdgePx"]:
            problems.append("short edge above long edge")
        out.append(("sums:" + r["id"], not problems, "; ".join(problems) or "consistent"))
    by_cohort = {}
    for s in pd["under2048ByFileKind"]["rows"]:
        if s["accounts"] is not None and s["accounts"] < MIN_ACCOUNTS:
            out.append(("kinds:%s:%s" % (s["cohort"], s["kind"]), False, "fewer than %d accounts" % MIN_ACCOUNTS))
        by_cohort[s["cohort"]] = by_cohort.get(s["cohort"], 0) + s["uploads"]
    for cohort, total in sorted(by_cohort.items()):
        expected = rows[cohort]["longEdgeUnder2048"] if cohort in rows else None
        out.append(("kinds:" + cohort, total == expected,
                    "file kinds add up to %d of %s small uploads" % (total, expected)))
    return out


def check_csv(dataset):
    path = os.path.join(HERE, "dataset.csv")
    if not os.path.exists(path):
        return [("csv", False, "dataset.csv not found")]
    with open(path, newline="", encoding="utf-8") as handle:
        stored = {r["id"]: r for r in csv.DictReader(handle)}
    out = []
    rows = dataset["productData"]["rows"]
    for r in rows:
        c = stored.get(r["id"])
        if c is None:
            out.append(("csv:" + r["id"], False, "row missing from dataset.csv"))
            continue
        diff = [k for k in c if k in r and str(r[k]) != c[k]]
        out.append(("csv:" + r["id"], not diff, "differs in " + ", ".join(diff) if diff else "matches dataset.json"))
    for extra in sorted(set(stored) - {r["id"] for r in rows}):
        out.append(("csv:" + extra, False, "row in dataset.csv but not in dataset.json"))
    return out


def check_pages(dataset):
    cache, out = {}, []
    for row in dataset["results"]:
        url = row["url"]
        if url not in cache:
            cache[url] = fetch(url)
        status, body = cache[url]
        if status != 200:
            out.append((row["id"], False, "fetch failed (%s)" % (status or body)))
            continue
        haystack = body if row.get("kind") == "marker" else strip_tags(body)
        found = normalise(row["quote"]) in normalise(haystack)
        out.append((row["id"], found, "quote found" if found else "quote gone from %s" % url))
    return out


def find_repo():
    probe = HERE
    while probe != os.path.dirname(probe):
        if os.path.exists(os.path.join(probe, ".git")) and os.path.isdir(os.path.join(probe, "apps")):
            return probe
        probe = os.path.dirname(probe)
    return None


def check_code(dataset):
    repo = find_repo()
    out = []
    for row in dataset.get("codeResults", []):
        if repo is None:
            out.append((row["id"], None, "skipped: not inside a checkout of the repository"))
            continue
        path = os.path.join(repo, row["path"])
        if not os.path.exists(path):
            out.append((row["id"], False, "%s is gone" % row["path"]))
            continue
        with open(path, encoding="utf-8") as handle:
            found = normalise(row["quote"]) in normalise(handle.read())
        out.append((row["id"], found, "quote found in %s" % row["path"] if found else "quote gone from %s" % row["path"]))
    return out


def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--recheck", action="store_true",
                        help="re-read the cited pages; no lab step exists to skip")
    args = parser.parse_args()

    with open(os.path.join(HERE, "dataset.json"), encoding="utf-8") as handle:
        dataset = json.load(handle)

    print("viallo-android-app evidence check, checkedAt %s" % dataset["checkedAt"])
    print("mode: %s" % ("recheck" if args.recheck else "full"))
    print("")
    failures = 0
    for name, ok, detail in check_sums(dataset) + check_csv(dataset) + check_pages(dataset) + check_code(dataset):
        label = "SKIP" if ok is None else ("PASS" if ok else "FAIL")
        failures += 1 if ok is False else 0
        print("%s %-34s %s" % (label, name, detail))
    print("")
    if failures:
        print("%d check(s) failed" % failures)
        return 1
    print("all checks hold")
    return 0


if __name__ == "__main__":
    sys.exit(main())