#!/usr/bin/env python3
"""Re-run the check behind https://www.viallo.app/blog/best-cloud-storage-for-photos.

Every result row in dataset.json is verified against the page it names: the
supporting quote must still appear in the page text after whitespace, quote-character,
HTML-entity and digit-spacing normalisation. Rows that carry `storage_gb` and `annual`
are recomputed and compared with the stored price per TB per year, so an arithmetic
slip fails the same way a vanished quote does.

dataset.csv is compared with dataset.json: every option and every price per TB in the
CSV must match the JSON row it came from.

Usage:
    python3 check.py             # full run
    python3 check.py --recheck   # same run; this check has no lab step to skip

Exit status is 0 when every checked row holds and 1 when any row fails. A row with
match "skip" is reported as SKIP with the reason recorded in dataset.json.
"""
import csv
import gzip
import html
import json
import os
import re
import subprocess
import sys
import urllib.error
import urllib.request

HERE = os.path.dirname(os.path.abspath(__file__))
DATASET = os.path.join(HERE, "dataset.json")
CSV = os.path.join(HERE, "dataset.csv")

UA = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
      "(KHTML, like Gecko) Chrome/155.0.0.0 Safari/537.36")
HEADERS = [
    "-H", "accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
    "-H", "accept-language: en-US,en;q=0.9",
    "-H", 'sec-ch-ua: "Chromium";v="155", "Not?A_Brand";v="8"',
    "-H", "sec-ch-ua-mobile: ?0",
    "-H", 'sec-ch-ua-platform: "macOS"',
    "-H", "sec-fetch-dest: document",
    "-H", "sec-fetch-mode: navigate",
    "-H", "sec-fetch-site: none",
]


def normalise(text):
    """Collapse the differences a rendered page introduces without changing words."""
    text = html.unescape(text)
    text = text.replace(" ", " ").replace(" ", " ").replace(" ", " ")
    for src, dst in (("’", "'"), ("‘", "'"), ("“", '"'), ("”", '"'),
                     ("\u2013", "-"), ("\u2014", "-"), ("…", "...")):
        text = text.replace(src, dst)
    text = re.sub(r"\s+", " ", text).strip()
    # Price cells are split across elements: "$ 2 . 49 /mo" renders as "$2.49/mo".
    text = re.sub(r"\s+([./])", r"\1", text)
    text = re.sub(r"\$\s+", "$", text)
    text = re.sub(r"(\d)\.\s+(\d)", r"\1.\2", text)
    return text


def decompress(body):
    """Some hosts gzip even when the request did not ask; urllib will not unpack that."""
    if body[:2] == b"\x1f\x8b":
        return gzip.decompress(body)
    return body


def fetch_raw(url):
    """Return the page bytes: urllib first, curl with browser headers when it refuses."""
    req = urllib.request.Request(url, headers={
        "User-Agent": UA,
        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
        "Accept-Language": "en-US,en;q=0.9",
        "Accept-Encoding": "identity",
        "sec-ch-ua": '"Chromium";v="155", "Not?A_Brand";v="8"',
        "sec-ch-ua-mobile": "?0",
        "sec-ch-ua-platform": '"macOS"',
        "sec-fetch-dest": "document",
        "sec-fetch-mode": "navigate",
        "sec-fetch-site": "none",
    })
    try:
        with urllib.request.urlopen(req, timeout=45) as resp:
            return decompress(resp.read())
    except (urllib.error.HTTPError, urllib.error.URLError):
        out = subprocess.run(["curl", "-sL", "--http2", "--compressed", "-A", UA] + HEADERS + [url],
                             capture_output=True, timeout=90)
        if out.returncode != 0 or not out.stdout:
            raise RuntimeError("curl failed with status %s" % out.returncode)
        return out.stdout


def page_text(raw):
    text = raw.decode("utf-8", "replace")
    text = re.sub(r"(?is)<(script|style|noscript|svg)[^>]*>.*?</\1>", " ", text)
    text = re.sub(r"(?i)<br\s*/?>", " ", text)
    text = re.sub(r"(?i)</(p|div|li|tr|h[1-6]|table|section|td|th)>", " ", text)
    text = re.sub(r"<[^>]+>", " ", text)
    return normalise(text)


def meta_description(raw):
    text = raw.decode("utf-8", "replace")
    found = []
    for pattern in (r'(?is)<meta[^>]+name="description"[^>]+content="([^"]*)"',
                    r'(?is)<meta[^>]+content="([^"]*)"[^>]+name="description"'):
        found += [html.unescape(m.group(1)) for m in re.finditer(pattern, text)]
    return normalise(" ".join(found))


def raw_page(raw):
    return normalise(raw.decode("utf-8", "replace"))


def quotes_of(row):
    return row.get("quotes") or [row["quote"]]


def check_row(row):
    """Return (state, detail) where state is PASS, FAIL or SKIP."""
    mode = row.get("match", "text")

    if mode == "skip":
        return "SKIP", row.get("reason", "not checked")

    if mode == "text":
        text = page_text(fetch_raw(row["url"]))
    elif mode == "meta":
        text = meta_description(fetch_raw(row["url"]))
    elif mode == "raw":
        text = raw_page(fetch_raw(row["url"]))
    else:
        return "FAIL", "unknown match type %r" % mode

    missing = [q for q in quotes_of(row) if normalise(q) not in text]
    if missing:
        return "FAIL", "quote missing"

    if "storage_gb" in row and "annual" in row:
        expected = round(row["annual"] / (row["storage_gb"] / 1000.0), 2)
        if abs(expected - float(row["per_tb"])) > 0.005:
            return "FAIL", "per TB is %s, recomputed %s" % (row["per_tb"], expected)
        return "PASS", "quote found, %s per TB per year recomputed" % expected

    return "PASS", "quote found"


def check_csv(dataset):
    """The CSV the post's table is copied from must agree with the JSON."""
    if not os.path.exists(CSV):
        return ["dataset.csv is missing"]
    with open(CSV, encoding="utf-8", newline="") as fh:
        rows = list(csv.DictReader(fh))
    problems = []
    options = {row["option"] for row in dataset["results"]}
    csv_options = {row["option"] for row in rows}
    for option in sorted(csv_options - options):
        problems.append("dataset.csv holds %r, dataset.json does not" % option)
    for option in sorted(options - csv_options):
        problems.append("dataset.json holds %r, dataset.csv does not" % option)
    per_tb = {}
    for row in dataset["results"]:
        if "per_tb" in row and row["property"].startswith("first plan at 1 TB"):
            per_tb[row["option"]] = float(row["per_tb"])
    for row in rows:
        want = per_tb.get(row["option"])
        if want is None:
            continue
        try:
            got = float(row["price_per_tb_year"])
        except ValueError:
            got = None
        if got is None or abs(got - want) > 0.005:
            problems.append("%s per TB is %s in the CSV, %s in the JSON"
                            % (row["option"], row["price_per_tb_year"], want))
    return problems


def main():
    recheck = "--recheck" in sys.argv
    with open(DATASET, encoding="utf-8") as fh:
        dataset = json.load(fh)

    rows = list(dataset["results"]) + list(dataset.get("third_party", []))
    failures = 0
    skipped = 0
    for row in rows:
        label = "%s | %s" % (row.get("option") or row.get("party"), row["property"])
        try:
            state, detail = check_row(row)
        except Exception as exc:  # a network, parse or git error is a failed row
            state, detail = "FAIL", "%s: %s" % (type(exc).__name__, exc)
        if state == "FAIL":
            failures += 1
        if state == "SKIP":
            skipped += 1
        print("%-4s  %-58s %s" % (state, label[:58], detail))
        if state == "FAIL":
            for quote in quotes_of(row):
                print("      quote: %s" % quote[:150])
            print("      url:   %s" % row["url"])

    for problem in check_csv(dataset):
        failures += 1
        print("FAIL  dataset.csv %s" % problem)

    print("\n%d row(s), %d failed, %d skipped%s"
          % (len(rows), failures, skipped,
             "  (recheck mode: no lab step in this check)" if recheck else ""))
    return 0 if failures == 0 else 1


if __name__ == "__main__":
    sys.exit(main())