#!/usr/bin/env python3
"""Re-check every quoted sentence in dataset.json against the vendor's own page.

Fetches each page live and reports, per row, whether the supporting quote still
appears. A 'Not stated' row has no quote; for it the page only has to load.
Also checks that dataset.csv carries the same values as dataset.json.

--recheck re-reads the same pages; this check has no lab step, so nothing is skipped.

Exit 0 when every row holds, 1 otherwise.
"""

import csv
import html
import json
import os
import re
import sys
import urllib.request

HERE = os.path.dirname(os.path.abspath(__file__))
DATASET = os.path.join(HERE, "dataset.json")
CSV = os.path.join(HERE, "dataset.csv")
UA = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
      "(KHTML, like Gecko) Chrome/130.0 Safari/537.36")

SWAPS = {
    "\u2018": "'", "\u2019": "'", "\u201c": '"', "\u201d": '"',
    "\u2011": "-", "\u2013": "-", "\u2014": "-", "\u00a0": " ",
    "\u200b": "", "\ufeff": "",
}


def normalise(text):
    text = html.unescape(text)
    for bad, good in SWAPS.items():
        text = text.replace(bad, good)
    text = re.sub(r"\[([^\]]+)\]\([^)]+\)", r"\1", text)  # markdown links from the text proxy
    text = text.replace("**", "").replace("`", "")
    return re.sub(r"\s+([.,;:])", r"\1", re.sub(r"\s+", " ", text)).strip()


def fetch(url, timeout=60):
    req = urllib.request.Request(url, headers={"User-Agent": UA, "Accept": "text/html,*/*"})
    with urllib.request.urlopen(req, timeout=timeout) as resp:
        raw = resp.read()
        charset = resp.headers.get_content_charset() or "utf-8"
    return raw.decode(charset, errors="ignore")


def page_text(target):
    body = fetch(target)
    if "/api/v2/help_center/" in target:
        article = json.loads(body).get("article", {})
        return re.sub(r"<[^>]+>", " ", article.get("body", ""))
    if target.startswith("https://r.jina.ai/"):
        return body
    body = re.sub(r"<script.*?</script>|<style.*?</style>", " ", body, flags=re.S)
    return re.sub(r"<[^>]+>", " ", body)


def check_csv(data):
    expected = {}
    for row in data["results"]:
        expected.setdefault(row["option"], {})[row["property"]] = row["value"]
    problems = []
    with open(CSV, encoding="utf-8", newline="") as fh:
        rows = list(csv.DictReader(fh))
    if [r["option"] for r in rows] != list(expected):
        problems.append("dataset.csv options differ from dataset.json")
    for r in rows:
        for prop, value in expected.get(r["option"], {}).items():
            if r.get(prop) != value:
                problems.append("dataset.csv {} / {} differs from dataset.json".format(r["option"], prop))
    return problems


def main():
    recheck = "--recheck" in sys.argv
    with open(DATASET, encoding="utf-8") as fh:
        data = json.load(fh)

    cache = {}
    failures = []
    for row in data["results"]:
        label = "{} / {}".format(row["option"], data["properties"][row["property"]])
        source = row.get("fetchUrl") or row["url"]
        try:
            if source not in cache:
                cache[source] = normalise(page_text(source))
            text = cache[source]
        except Exception as exc:  # a fetch that fails is a failing row, not a crash
            print("ERROR {}\n      {}".format(label, exc))
            failures.append(label)
            continue
        if row["quote"] is None:
            ok = len(text) > 200
            print("{}  {} (not stated, page loads)".format("PASS" if ok else "FAIL", label))
        else:
            ok = normalise(row["quote"]) in text
            print("{}  {}".format("PASS" if ok else "FAIL", label))
            if not ok:
                print("      quote:  {}\n      source: {}".format(row["quote"][:140], source))
        if not ok:
            failures.append(label)

    for problem in check_csv(data):
        print("FAIL  " + problem)
        failures.append(problem)

    total = len(data["results"])
    rows_failed = sum(1 for f in failures if " / " in f and not f.startswith("dataset.csv"))
    print("\n{}/{} rows hold{}".format(total - rows_failed, total, " (recheck)" if recheck else ""))
    if failures:
        print("failing: {}".format("; ".join(failures)))
    return 0 if not failures else 1


if __name__ == "__main__":
    sys.exit(main())