#!/usr/bin/env python3
"""Re-run the primary-docs check behind /blog/shared-album-invite-not-working.

Every row in dataset.json carries a verbatim quote and the page it came from. This
script fetches each page (or, for the Viallo rows, reads the cited file from the
Viallo repository through git), folds whitespace, typographic quotes and HTML
entities, and reports whether the quote is still there. It also checks that every
cell of dataset.csv is the value of a row in dataset.json.

    python3 check.py            # full check
    python3 check.py --recheck  # same: this check has no lab step to skip

Exit 0 when every quote holds, 1 otherwise. Standard library only.
"""

import csv
import html
import json
import os
import re
import subprocess
import sys
import urllib.error
import urllib.request

HERE = os.path.dirname(os.path.abspath(__file__))
DATASET = os.path.join(HERE, "dataset.json")
TABLE = os.path.join(HERE, "dataset.csv")
FETCH_TIMEOUT = 45
UA = (
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
    "(KHTML, like Gecko) Chrome/126.0 Safari/537.36"
)

_TAG_BLOCK = re.compile(r"(?is)<(script|style|noscript|svg)[^>]*>.*?</\1>")
_COMMENT = re.compile(r"(?is)<!--.*?-->")
_TAG = re.compile(r"(?s)<[^>]+>")
_SPACE = re.compile(r"\s+")
_FOLD = {
    "‘": "'", "’": "'", "‚": "'", "‛": "'",
    "“": '"', "”": '"', "„": '"',
    "–": "-", "—": "-", "−": "-",
    " ": " ", " ": " ", " ": " ",
}


class Skip(Exception):
    pass


def normalise(text):
    text = html.unescape(text)
    for src, dst in _FOLD.items():
        text = text.replace(src, dst)
    return _SPACE.sub(" ", text).strip()


def html_to_text(raw):
    raw = _TAG_BLOCK.sub(" ", raw)
    raw = _COMMENT.sub(" ", raw)
    raw = re.sub(r"(?is)<br\s*/?>|</(p|div|li|tr|h[1-6]|td|th|dd|dt)>", " ", raw)
    raw = _TAG.sub(" ", raw)
    return normalise(raw)


def fetch(url):
    request = urllib.request.Request(url, headers={"User-Agent": UA, "Accept-Language": "en-US,en"})
    with urllib.request.urlopen(request, timeout=FETCH_TIMEOUT) as response:
        charset = response.headers.get_content_charset() or "utf-8"
        return response.read().decode(charset, errors="replace")


def repo_root():
    result = subprocess.run(
        ["git", "-C", HERE, "rev-parse", "--show-toplevel"], capture_output=True, text=True, check=False
    )
    root = result.stdout.strip()
    if result.returncode != 0 or not os.path.exists(os.path.join(root, "packages", "shared")):
        return None
    return root


def read_code(path_with_line, root):
    if root is None:
        raise Skip("needs the Viallo repository")
    path = path_with_line.rsplit(":", 1)[0]
    for ref in ("origin/main", "HEAD"):
        result = subprocess.run(
            ["git", "-C", root, "show", "%s:%s" % (ref, path)], capture_output=True, text=True, check=False
        )
        if result.returncode == 0:
            return result.stdout
    raise RuntimeError("git show failed for " + path)


def load_source(url, cache, root):
    if url not in cache:
        if url.startswith("code://"):
            cache[url] = normalise(read_code(url[len("code://"):], root))
        elif url.startswith("http"):
            cache[url] = html_to_text(fetch(url))
        else:
            raise RuntimeError("unsupported source: " + url)
    return cache[url]


def check_table(dataset):
    values = {(r["option"], r["property"]): r for r in dataset["results"]}
    problems = []
    with open(TABLE, encoding="utf-8", newline="") as handle:
        for row in csv.DictReader(handle):
            for column, value in row.items():
                if column == "option" or column.endswith("_source"):
                    continue
                result = values.get((row["option"], column))
                if result is None or result["value"] != value or result["url"] != row[column + "_source"]:
                    problems.append("%s / %s" % (row["option"], column))
    return problems


def main(argv):
    recheck = "--recheck" in argv[1:]
    with open(DATASET, encoding="utf-8") as handle:
        dataset = json.load(handle)

    print("method   %s" % dataset["method"])
    print("checked  %s" % dataset["checkedAt"])
    print("mode     %s" % ("recheck" if recheck else "full"))
    print("")

    root = repo_root()
    cache = {}
    failures = []
    skipped = 0
    for row in dataset["results"]:
        label = "%s / %s" % (row["option"], row["property"])
        needle = normalise(row["quote"]).lower()
        try:
            haystack = load_source(row["url"], cache, root).lower()
        except Skip as reason:
            skipped += 1
            print("skip %s -- %s" % (label, reason))
            continue
        except (urllib.error.URLError, OSError, RuntimeError) as error:
            failures.append(label)
            print("FAIL %s -- source unreachable: %s" % (label, error))
            continue
        if needle in haystack:
            print("ok   %s" % label)
        else:
            failures.append(label)
            print("FAIL %s -- quote no longer on the page: %s" % (label, row["url"]))

    table_problems = check_table(dataset)
    for label in table_problems:
        print("FAIL dataset.csv %s -- cell differs from dataset.json" % label)

    total = len(dataset["results"])
    print("")
    print("%d of %d quotes still hold, %d skipped" % (total - skipped - len(failures), total, skipped))
    print("dataset.csv %s" % ("matches dataset.json" if not table_problems else "differs from dataset.json"))
    return 1 if failures or table_problems else 0


if __name__ == "__main__":
    sys.exit(main(sys.argv))