#!/usr/bin/env python3
"""Re-run the check behind the without-account dataset.

Fetches every cited page and reads every cited repository file, extracts its
readable text and reports per row whether the recorded quote is still there.
Exit 0 when every row holds, 1 when any row fails.

    python3 check.py            # full run
    python3 check.py --recheck  # same; skips no step, there is no lab step

Standard library only. No account, no login, no other person's album or link.
"""

import html
import json
import os
import re
import sys
import urllib.request

HERE = os.path.dirname(os.path.abspath(__file__))
DATASET = os.path.join(HERE, "dataset.json")
CSV = os.path.join(HERE, "dataset.csv")
# apps/web/public/blog/without-account -> repository root
REPO = os.path.abspath(os.path.join(HERE, *[".."] * 5))
UA = (
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
    "(KHTML, like Gecko) Chrome/131.0 Safari/537.36"
)

QUOTES = {
    "\u2018": "'", "\u2019": "'", "\u201a": "'", "\u201b": "'",
    "\u201c": '"', "\u201d": '"', "\u201e": '"',
    "\u2013": "-", "\u2014": "-", "\u2212": "-",
    "\u00a0": " ", "\u2009": " ", "\u202f": " ",
}


def normalise(text):
    text = html.unescape(text)
    text = html.unescape(text)
    for src, dst in QUOTES.items():
        text = text.replace(src, dst)
    text = re.sub(r"\s+", " ", text)
    # a tag between a word and its punctuation leaves " ."; that is markup, not text
    text = re.sub(r"\s+([.,;:!?])", r"\1", text)
    return text.strip()


def page_text(raw):
    """Readable text of an HTML document."""
    text = re.sub(r"(?is)<(script|style|noscript|template)[^>]*>.*?</\1>", " ", raw)
    text = re.sub(r"(?s)<[^>]+>", " ", text)
    return normalise(text)


def plain_text(raw):
    return normalise(raw)


def fetch(url):
    req = urllib.request.Request(url, headers={"User-Agent": UA, "Accept-Language": "en-US,en;q=0.9"})
    with urllib.request.urlopen(req, timeout=45) as resp:
        charset = resp.headers.get_content_charset() or "utf-8"
        return resp.read().decode(charset, errors="replace")


def repo_text(path):
    with open(os.path.join(REPO, path), encoding="utf-8") as fh:
        return fh.read()


def csv_rows():
    if not os.path.exists(CSV):
        return None
    with open(CSV, encoding="utf-8") as fh:
        lines = [ln.rstrip("\n") for ln in fh if ln.strip()]
    import csv as _csv
    return list(_csv.DictReader(lines))


def main(argv):
    recheck = "--recheck" in argv
    with open(DATASET, encoding="utf-8") as fh:
        data = json.load(fh)
    print("method %s, checked %s%s" % (data["method"], data["checkedAt"],
                                       ", recheck" if recheck else ""))

    cache = {}
    failures = 0
    for row in data["results"]:
        url = row["url"]
        try:
            if url not in cache:
                if url.startswith("repo:"):
                    cache[url] = plain_text(repo_text(url[len("repo:"):]))
                else:
                    cache[url] = page_text(fetch(url))
            body = cache[url]
        except Exception as exc:  # a page that refuses to load is a failed row
            print("FAIL %-30s cannot read %s (%s)" % (row["id"], url, exc))
            failures += 1
            continue
        if normalise(row["quote"]) in body:
            print("ok   %-30s %s" % (row["id"], url))
        else:
            print("FAIL %-30s quote not found at %s" % (row["id"], url))
            failures += 1

    rows = csv_rows()
    if rows is None:
        print("FAIL dataset.csv is missing")
        failures += 1
    else:
        by_id = {r["id"]: r for r in data["results"]}
        if len(rows) != len(by_id):
            print("FAIL dataset.csv has %d rows, dataset.json has %d" % (len(rows), len(by_id)))
            failures += 1
        for row in rows:
            src = by_id.get(row["id"])
            if src is None:
                print("FAIL dataset.csv row %s is not in dataset.json" % row["id"])
                failures += 1
                continue
            for key in ("route", "property", "value", "quote", "url"):
                if row[key] != src[key]:
                    print("FAIL dataset.csv row %s differs on %s" % (row["id"], key))
                    failures += 1

    print("lab steps: none in this check" if not recheck else "lab steps skipped on --recheck: none")
    print("%d row(s) failed" % failures if failures else "every row holds")
    return 0 if failures == 0 else 1


if __name__ == "__main__":
    sys.exit(main(sys.argv[1:]))