#!/usr/bin/env python3
"""Re-check every quote the onedrive-photos-privacy dataset rests on.

Reads dataset.json next to this file. For each result row with a URL it fetches
the page today and reports whether the row's quote still appears in the page's
text (plus its JSON-LD, which carries Google One's USD price). Rows whose source
starts with ``repo:`` read Viallo's own code through ``git show origin/main:<path>``;
outside a Viallo clone they are reported as skipped, not as passed.

Exit 0 when every checked quote holds, 1 otherwise. ``--recheck`` re-reads the
same pages and repo files: this check has no lab step to skip.
"""

import gzip
import html
import json
import re
import subprocess
import sys
import time
import urllib.request
import zlib
from pathlib import Path

HERE = Path(__file__).resolve().parent
DATASET = HERE / "dataset.json"
UA = (
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
    "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36"
)
TIMEOUT = 45
DASHES = {0x2010: "-", 0x2011: "-", 0x2012: "-", 0x2013: "-", 0x2014: "-"}
QUOTES = {0x2018: "'", 0x2019: "'", 0x201C: '"', 0x201D: '"', 0x00A0: " "}


def normalize(text):
    text = html.unescape(text)
    text = re.sub(r"<[^>]+>", " ", text)
    text = text.translate(DASHES).translate(QUOTES)
    return re.sub(r"\s+", " ", text).strip()


def page_text(raw):
    ld = re.findall(
        r'<script[^>]+type="application/ld\+json"[^>]*>(.*?)</script>', raw, re.S | re.I
    )
    body = re.sub(r"<script.*?</script>", " ", raw, flags=re.S | re.I)
    body = re.sub(r"<style.*?</style>", " ", body, flags=re.S | re.I)
    return normalize(body) + " " + " ".join(normalize(block) for block in ld)


def _decode(raw, encoding):
    if raw[:2] == b"\x1f\x8b" or encoding == "gzip":
        return gzip.decompress(raw)
    if encoding == "deflate":
        return zlib.decompress(raw)
    return raw


def fetch(url):
    # One CDN edge here sometimes answers with zstd or brotli, which the
    # standard library cannot decode; the next request usually does not.
    last = None
    for attempt in range(3):
        req = urllib.request.Request(
            url,
            headers={
                "User-Agent": UA,
                "Accept-Language": "en-US,en;q=0.9",
                "Accept": "text/html,application/xhtml+xml",
                "Accept-Encoding": "identity",
            },
        )
        try:
            with urllib.request.urlopen(req, timeout=TIMEOUT) as resp:
                raw = _decode(
                    resp.read(), (resp.headers.get("Content-Encoding") or "").strip().lower()
                )
            body = raw.decode("utf-8", "replace")
            if body.count("<") > 20:
                return page_text(body)
            last = RuntimeError("body was not HTML (undecodable content encoding)")
        except Exception as exc:
            last = exc
        if attempt < 2:
            time.sleep(2)
    raise last


def repo_root():
    out = subprocess.run(
        ["git", "-C", str(HERE), "rev-parse", "--show-toplevel"], capture_output=True
    )
    if out.returncode != 0:
        return None
    root = out.stdout.decode().strip()
    probe = subprocess.run(
        ["git", "-C", root, "cat-file", "-e", "origin/main:packages/shared/src/subscription/tiers.ts"],
        capture_output=True,
    )
    return root if probe.returncode == 0 else None


def repo_text(root, path):
    out = subprocess.run(["git", "-C", root, "show", "origin/main:" + path], capture_output=True)
    if out.returncode != 0:
        raise RuntimeError(out.stderr.decode("utf-8", "replace").strip())
    return normalize(out.stdout.decode("utf-8", "replace"))


def main(argv):
    unknown = [a for a in argv[1:] if a != "--recheck"]
    if unknown:
        print("unknown argument: %s" % " ".join(unknown))
        return 1
    data = json.loads(DATASET.read_text(encoding="utf-8"))
    rows = data["results"]
    mode = "recheck" if "--recheck" in argv else "check"
    print("%s: %s, %s, checked %s, %d rows" % (
        mode, data["slug"], data["method"], data["checkedAt"], len(rows)))

    root = repo_root()
    cache = {}
    failures, skipped = [], []
    for row in rows:
        source = row["source"]
        try:
            if source.startswith("repo:"):
                if root is None:
                    skipped.append(row["id"])
                    print("skip %-20s %s (needs a Viallo clone with origin/main)" % (row["id"], row["option"]))
                    continue
                key = source
                if key not in cache:
                    cache[key] = repo_text(root, source[len("repo:"):])
            else:
                key = row["url"]
                if key not in cache:
                    cache[key] = fetch(key)
        except Exception as exc:  # a page that refuses us is a failed row, never a guess
            failures.append((row["id"], "fetch failed: %s" % exc))
            print("FAIL %-20s fetch failed (%s)" % (row["id"], exc))
            continue
        if normalize(row["quote"]) in cache[key]:
            print("ok   %-20s %s" % (row["id"], row["option"]))
        else:
            failures.append((row["id"], "quote not found on %s" % key))
            print("FAIL %-20s %-28s %s" % (row["id"], row["option"], key))

    print("-" * 70)
    if failures:
        print("%d of %d rows failed" % (len(failures), len(rows)))
        for row_id, why in failures:
            print("  %s: %s" % (row_id, why))
        return 1
    print("all %d checked rows hold%s" % (
        len(rows) - len(skipped), ", %d skipped" % len(skipped) if skipped else ""))
    return 0


if __name__ == "__main__":
    sys.exit(main(sys.argv))