#!/usr/bin/env python3
"""Re-run the check behind dataset.json for the iCloud-alternative comparison.

Every row is one claim: a value, the exact sentence that supports it, and the page
or repository file that sentence lives in. This script fetches each source again
and reports whether the sentence is still there. Nothing is measured locally: the
method is primary-docs, so a row passes when its quote is present in the source.

    python3 check.py             # fetch everything, verify every row
    python3 check.py --recheck   # same; there is no lab step to skip

Viallo's own rows are read with git from the repository checkout this file sits in. Run
outside a checkout (a copy downloaded from the blog), those rows are reported as skipped and
every vendor page is still checked.

Exit code 0 when every row holds and dataset.csv matches dataset.json, 1 otherwise.
"""

import argparse
import csv
import gzip
import html
import json
import os
import re
import subprocess
import sys
import time
import urllib.error
import urllib.request

HERE = os.path.dirname(os.path.abspath(__file__))
DATASET = os.path.join(HERE, "dataset.json")
CSV = os.path.join(HERE, "dataset.csv")
UA = (
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
    "(KHTML, like Gecko) Chrome/130.0 Safari/537.36"
)
PROPERTIES = [
    "viewer_account",
    "guest_can_add",
    "full_resolution",
    "photo_cap",
    "storage_owner",
]


def strip_html(raw):
    raw = re.sub(r"(?is)<(script|style|noscript)[^>]*>.*?</\1>", " ", raw)
    for entity in ("&nbsp;", "&#8203;", "​"):
        raw = raw.replace(entity, " ")
    return html.unescape(re.sub(r"(?s)<[^>]+>", " ", raw))


def normalise(text):
    text = text.replace("—", "-").replace("–", "-")
    text = text.replace("’", "'").replace("‘", "'")
    text = text.replace("“", '"').replace("”", '"')
    text = text.replace("\xa0", " ")
    return re.sub(r"\s+", " ", text).strip()


def compact(text):
    return re.sub(r"\s+", "", normalise(text))


def quote_present(quote, text, is_html=False):
    clean = normalise(strip_html(text)) if is_html else normalise(text)
    if normalise(quote) in clean:
        return True
    return compact(quote) in compact(clean)


def fetch_url(url):
    request = urllib.request.Request(url, headers={"User-Agent": UA, "Accept-Language": "en-US,en"})
    # amazon.com answers a burst of help-page requests with 403, 429 or 503.
    for attempt in range(2):
        try:
            with urllib.request.urlopen(request, timeout=60) as response:
                charset = response.headers.get_content_charset() or "utf-8"
                body = response.read()
                # amazon.com sends gzip even to a request that did not ask for it.
                if body[:2] == b"\x1f\x8b":
                    body = gzip.decompress(body)
                return body.decode(charset, "replace")
        except urllib.error.HTTPError as exc:
            if exc.code not in (403, 429, 503) or attempt == 1:
                raise
            time.sleep(15)


def repo_root():
    out = subprocess.run(
        ["git", "-C", HERE, "rev-parse", "--show-toplevel"],
        capture_output=True,
        text=True,
    )
    return out.stdout.strip() if out.returncode == 0 else None


def read_repo(repo, file_path, ref):
    out = subprocess.run(
        ["git", "-C", repo, "show", "%s:%s" % (ref, file_path)],
        capture_output=True,
        text=True,
    )
    if out.returncode != 0:
        raise RuntimeError(out.stderr.strip() or "git show failed")
    return out.stdout


def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("--recheck", action="store_true", help="re-read every page; no lab step exists here")
    parser.parse_args()

    with open(DATASET, encoding="utf-8") as handle:
        data = json.load(handle)

    sources = {}
    errors = []
    skipped = set()
    repo = repo_root()
    for entry in data["inputs"]:
        if "url" in entry:
            try:
                sources[entry["id"]] = (fetch_url(entry["url"]), True)
            except urllib.error.HTTPError as exc:
                if exc.code in (403, 429, 503):
                    errors.append("blocked, not checked: %s (%s) refused the request with %s; a refusal says nothing about the quote" % (entry["id"], entry["url"], exc.code))
                else:
                    errors.append("could not fetch %s (%s): %s" % (entry["id"], entry["url"], exc))
            except (urllib.error.URLError, TimeoutError) as exc:
                errors.append("could not fetch %s (%s): %s" % (entry["id"], entry["url"], exc))
        elif repo is None:
            skipped.add(entry["id"])
        else:
            try:
                sources[entry["id"]] = (read_repo(repo, entry["path"], entry["ref"]), False)
            except (RuntimeError, OSError) as exc:
                errors.append("could not read %s (%s): %s" % (entry["id"], entry["path"], exc))

    failures = []
    checked = 0
    for row in data["results"]:
        source_id = row["source"]
        source = sources.get(source_id)
        if source_id in skipped:
            print("skip  %s / %s (needs a Viallo checkout)" % (row["option"], row["property"]))
            continue
        if source is None:
            failures.append("%s / %s: source %s unavailable" % (row["option"], row["property"], source_id))
            continue
        checked += 1
        text, is_html = source
        if not quote_present(row["quote"], text, is_html):
            failures.append("%s / %s: quote not found in %s: %r" % (row["option"], row["property"], source_id, row["quote"]))
            continue
        print("ok    %s / %s" % (row["option"], row["property"]))

    for item in data.get("supporting", []):
        source = sources.get(item["source"])
        if item["source"] in skipped:
            print("skip  %s (needs a Viallo checkout)" % item["claim"])
            continue
        if source is None:
            failures.append("supporting %r: source %s unavailable" % (item["claim"], item["source"]))
            continue
        checked += 1
        text, is_html = source
        found = quote_present(item["quote"], text, is_html)
        if item.get("expect") == "absent":
            if found:
                failures.append("supporting %r: quote is present but expected to be absent: %r" % (item["claim"], item["quote"]))
            else:
                print("ok    absent as expected: %s" % item["claim"])
            continue
        if not found:
            failures.append("supporting %r: quote not found in %s: %r" % (item["claim"], item["source"], item["quote"]))
            continue
        print("ok    %s" % item["claim"])

    if os.path.exists(CSV):
        with open(CSV, encoding="utf-8", newline="") as handle:
            table = list(csv.reader(handle))
        header = table[0] if table else []
        if header != ["option"] + PROPERTIES:
            failures.append("dataset.csv header is %r" % (header,))
        rows = {}
        for line in table[1:]:
            if not line:
                continue
            for index, prop in enumerate(PROPERTIES, start=1):
                rows["%s|%s" % (line[0], prop)] = line[index] if index < len(line) else ""
        for row in data["results"]:
            label = "%s|%s" % (row["option"], row["property"])
            if label not in rows:
                failures.append("dataset.csv has no value for %s" % label)
                continue
            if normalise(rows[label]) != normalise(row["value"]):
                failures.append("dataset.csv value for %s differs from dataset.json" % label)
    else:
        failures.append("dataset.csv is missing")

    print("")
    for message in errors + failures:
        print("FAIL  %s" % message)
    print("checked %d quotes from %d sources, %d failures" % (checked, len(sources), len(failures) + len(errors)))
    return 0 if not errors and not failures else 1


if __name__ == "__main__":
    sys.exit(main())