#!/usr/bin/env python3
"""Re-run the evidence check behind the viallo-ios-app post.

Reads dataset.json beside this file and:
- re-fetches every cited page and reports whether the stored quote still appears;
- re-reads every cited source file at origin/main (skipped without a checkout);
- recomputes every production count and share from the stored query output, re-checks
  that the classes add up to their totals and that no published cell comes from fewer
  than 10 people. Production itself is not reachable from here; query.js is the
  read-only aggregate that produced the stored output.

Exits 0 when every row holds and 1 otherwise.

    python3 check.py            # full run
    python3 check.py --recheck  # re-read the pages only; there is no lab step to skip

Set VIALLO_REPO to point at a checkout of the product repository.
"""
import argparse
import csv
import html
import json
import os
import re
import subprocess
import sys
import time
import urllib.error
import urllib.request

HERE = os.path.dirname(os.path.abspath(__file__))
UA = (
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
    "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0 Safari/537.36"
)
TRANSLATE = {
    "\u00a0": " ",
    "\u2018": "'",
    "\u2019": "'",
    "\u201c": '"',
    "\u201d": '"',
    "\u2013": "-",
    "\u2014": "-",
    "\u2011": "-",
    "\u2212": "-",
    "\u2026": "...",
}


def normalise(text):
    text = html.unescape(html.unescape(text))
    for src, dst in TRANSLATE.items():
        text = text.replace(src, dst)
    return re.sub(r"\s+", " ", text).strip()


def strip_tags(text):
    text = re.sub(r"(?is)<script.*?</script>", " ", text)
    text = re.sub(r"(?is)<style.*?</style>", " ", text)
    text = re.sub(r"(?is)<!--.*?-->", " ", text)
    return re.sub(r"(?s)<[^>]+>", " ", text)


def fetch(url, attempts=3):
    last = "no attempt made"
    for attempt in range(attempts):
        try:
            request = urllib.request.Request(
                url,
                headers={"User-Agent": UA, "Accept-Language": "en-US,en;q=0.9"},
            )
            with urllib.request.urlopen(request, timeout=45) as response:
                return response.status, response.read().decode("utf-8", "replace")
        except urllib.error.HTTPError as exc:
            return exc.code, ""
        except Exception as exc:  # network, DNS, TLS
            last = "%s: %s" % (type(exc).__name__, exc)
            time.sleep(2 * (attempt + 1))
    return 0, last


def find_repo():
    repo = os.environ.get("VIALLO_REPO")
    if repo:
        return repo if os.path.exists(os.path.join(repo, ".git")) else None
    cur = HERE
    while cur != os.path.dirname(cur):
        if os.path.exists(os.path.join(cur, ".git")):
            return cur
        cur = os.path.dirname(cur)
    return None


def lookup(output, path):
    cur = output
    for part in path.split("/"):
        cur = cur[part]
    return cur


def check_product(row, output):
    if row["kind"] == "product":
        expected = str(lookup(output, row["quote"]))
    elif row["kind"] == "product-share":
        num, den = [lookup(output, p.strip()) for p in row["quote"].split(" / ")]
        expected = "%.1f%%" % (100.0 * num / den)
    elif row["kind"] == "product-mb":
        expected = "%.2f MB" % (lookup(output, row["quote"]) / 1e6)
    else:
        return False, "unknown kind %s" % row["kind"]
    if expected == row["value"]:
        return True, "matches query output"
    return False, "value %s, query output gives %s" % (row["value"], expected)


def check_sums(output):
    results = []
    for name, window in sorted(output["windows"].items()):
        opens = window["viewerOpens"]
        total = opens["all"]["opens"]
        parts = sum(v["opens"] for k, v in opens.items() if k.startswith("ua:"))
        results.append(("sum_%s_ua" % name, parts == total, "user-agent classes %d, all %d" % (parts, total)))
        split = opens["noAccount"]["opens"] + opens["signedIn"]["opens"]
        results.append(("sum_%s_account" % name, split == total, "no account + signed in %d, all %d" % (split, total)))
        photos = sum(r["photos"] for r in window["photos"])
        results.append(("sum_%s_photos" % name, photos == window["photosTotal"], "formats %d, total %d" % (photos, window["photosTotal"])))
    return results


def check_suppression(dataset):
    output = dataset["queryOutput"]
    small = []
    for row in dataset["results"]:
        if not row["kind"].startswith("product"):
            continue
        for path in [p.strip() for p in row["quote"].split(" / ")]:
            parts = path.split("/")
            if "viewerOpens" in parts:
                cell = lookup(output, "/".join(parts[:-1]))
                if min(cell["distinctIps"], cell["distinctOwners"]) < 10:
                    small.append(row["id"])
            elif parts[0] == "windows":
                window = output["windows"][parts[1]]
                people = window["albumCreators"] if "album" in parts[2].lower() else window["photoUploadersTotal"]
                if people < 10:
                    small.append(row["id"])
    if small:
        return [("suppression", False, "cells from fewer than 10 people: %s" % ", ".join(sorted(set(small))))]
    return [("suppression", True, "every published cell comes from 10 or more people")]


def check_page(row, cache):
    url = row["url"]
    if url not in cache:
        cache[url] = fetch(url)
    status, body = cache[url]
    if status != 200:
        return False, "fetch failed (%s)" % (status or "network")
    kind = row.get("kind", "html")
    haystack = body if kind in ("raw", "marker") else strip_tags(body)
    if normalise(row["quote"]) in normalise(haystack):
        return True, "quote found"
    return False, "quote gone from %s" % url


def check_code(row, repo):
    if repo is None:
        return None, "skipped: no product checkout; set VIALLO_REPO"
    try:
        out = subprocess.run(
            ["git", "-C", repo, "show", "origin/main:%s" % row["path"]],
            capture_output=True,
            text=True,
            check=False,
        )
    except Exception as exc:
        return False, "git failed: %s" % exc
    if out.returncode != 0:
        return False, "git show failed: %s" % (out.stderr.strip() or "unknown")
    if normalise(row["quote"]) in normalise(out.stdout):
        return True, "quote found in origin/main:%s" % row["path"]
    return False, "quote gone from origin/main:%s" % row["path"]


def check_csv(dataset):
    """The CSV must carry the same rows as dataset.json, or the post reads a stale table."""
    path = os.path.join(HERE, "dataset.csv")
    if not os.path.exists(path):
        return [("csv_missing", False, "dataset.csv not found")]
    with open(path, newline="", encoding="utf-8") as handle:
        rows = {r["id"]: r for r in csv.DictReader(handle)}
    problems = []
    for row in dataset["results"]:
        stored = rows.get(row["id"])
        if stored is None:
            problems.append((row["id"], False, "row missing from dataset.csv"))
        elif stored["value"] != row["value"] or stored["quote"] != row["quote"]:
            problems.append((row["id"], False, "dataset.csv differs from dataset.json"))
    extra = set(rows) - {row["id"] for row in dataset["results"]}
    for row_id in sorted(extra):
        problems.append((row_id, False, "row in dataset.csv but not in dataset.json"))
    if not problems:
        problems.append(("csv_in_sync", True, "%d rows match" % len(rows)))
    return problems


def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument(
        "--recheck",
        action="store_true",
        help="re-read the cited pages only; skip the source, count and table checks",
    )
    args = parser.parse_args()

    with open(os.path.join(HERE, "dataset.json"), encoding="utf-8") as handle:
        dataset = json.load(handle)

    repo = find_repo()
    cache = {}
    failures = 0

    print("viallo-ios-app evidence check, checkedAt %s" % dataset["checkedAt"])
    print("mode: %s" % ("recheck" if args.recheck else "full"))
    print("")

    def report(row_id, ok, detail):
        print("%s %-32s %s" % ({True: "PASS", False: "FAIL", None: "SKIP"}[ok], row_id, detail))
        return 1 if ok is False else 0

    for row in dataset["results"]:
        if row["kind"].startswith("product"):
            continue
        failures += report(row["id"], *check_page(row, cache))

    if not args.recheck:
        for row in dataset.get("codeResults", []):
            failures += report(row["id"], *check_code(row, repo))
        for row in dataset["results"]:
            if row["kind"].startswith("product"):
                failures += report(row["id"], *check_product(row, dataset["queryOutput"]))
        for row_id, ok, detail in check_sums(dataset["queryOutput"]) + check_suppression(dataset):
            failures += report(row_id, ok, detail)
        for row_id, ok, detail in check_csv(dataset):
            failures += report(row_id, ok, detail)

    print("")
    if failures:
        print("%d row(s) failed" % failures)
        return 1
    print("all rows hold")
    return 0


if __name__ == "__main__":
    sys.exit(main())
