#!/usr/bin/env python3
"""Re-runs the primary-docs check behind /blog/google-drive-photo-safety.

Every cell in dataset.json names a page and the sentence on it that supports the
cell. This script fetches each page (and, for the Viallo product rows, runs
`git show HEAD:<path>` in the Viallo repository), normalises whitespace, curly
quotes, dashes and HTML entities, and reports per row whether the sentence is
still there. It then rewrites dataset.csv from dataset.json.

Exit 0 when every quote holds, 1 otherwise. A page that refuses the fetch is a
failure, not a skipped cell. --recheck behaves the same: there is no lab step.
"""

import csv
import gzip
import html
import json
import os
import re
import subprocess
import sys
import urllib.error
import urllib.request
from html.parser import HTMLParser

HERE = os.path.dirname(os.path.abspath(__file__))
DATASET = os.path.join(HERE, "dataset.json")
UA = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
      "(KHTML, like Gecko) Chrome/120 Safari/537.36")


class Text(HTMLParser):
    def __init__(self):
        super().__init__()
        self.skip = 0
        self.out = []

    def handle_starttag(self, tag, attrs):
        if tag in ("script", "style", "noscript", "svg"):
            self.skip += 1
        if tag in ("p", "div", "li", "tr", "br", "h1", "h2", "h3", "h4", "td", "th", "section"):
            self.out.append(" ")

    def handle_endtag(self, tag):
        if tag in ("script", "style", "noscript", "svg") and self.skip:
            self.skip -= 1

    def handle_data(self, data):
        if not self.skip:
            self.out.append(data)


def normalise(s):
    s = html.unescape(s)
    for a, b in (("\u2019", "'"), ("\u2018", "'"), ("\u201c", '"'), ("\u201d", '"'),
                 ("\u2013", "-"), ("\u2014", "-"), ("\u00a0", " "), ("\u2026", "...")):
        s = s.replace(a, b)
    return re.sub(r"\s+", " ", s).strip()


def fetch(url, depth=0):
    headers = {"User-Agent": UA, "Accept-Encoding": "gzip"}
    if "web.archive.org" in url:
        # The Wayback Machine answers 429 to a request without a Referer.
        headers["Referer"] = "https://web.archive.org/"
    req = urllib.request.Request(url, headers=headers)
    try:
        resp = urllib.request.urlopen(req, timeout=90)
    except urllib.error.HTTPError as err:
        if err.code in (301, 302, 303, 307, 308) and depth < 5:
            return fetch(err.headers["Location"], depth + 1)
        raise
    raw = resp.read()
    if resp.headers.get("Content-Encoding") == "gzip":
        raw = gzip.decompress(raw)
    body = raw.decode("utf-8", "ignore")
    parser = Text()
    parser.feed(body)
    return normalise("".join(parser.out)), normalise(body)


def viallo_repo():
    out = subprocess.run(["git", "-C", HERE, "rev-parse", "--show-toplevel"],
                         capture_output=True, text=True)
    root = out.stdout.strip()
    if out.returncode or not os.path.exists(os.path.join(root, "apps/api/src")):
        return None
    return root


REPO = viallo_repo()
_cache = {}


def haystack(url):
    if url not in _cache:
        if url.startswith("git show HEAD:"):
            path = url.split("HEAD:", 1)[1]
            out = subprocess.run(["git", "-C", REPO, "show", "HEAD:%s" % path],
                                 capture_output=True, text=True, check=True)
            _cache[url] = [normalise(out.stdout)]
        else:
            _cache[url] = list(fetch(url))
    return _cache[url]


def check_quote(label, quote, url, failures):
    quote = (quote or "").strip()
    if not quote or not url:
        print("  %-30s SKIP  not checked" % label)
        return 0
    if url.startswith("git show") and REPO is None:
        print("  %-30s SKIP  needs a checkout of the Viallo repository" % label)
        return 0
    try:
        hay = " ".join(haystack(url))
    except Exception as exc:
        failures.append((label, url, quote, "fetch failed: %s" % exc))
        print("  %-30s FETCH-FAIL %s (%s)" % (label, url, exc))
        return 1
    if normalise(quote) in hay:
        print("  %-30s ok    %s" % (label, url))
    else:
        failures.append((label, url, quote, "quote gone"))
        print("  %-30s GONE  %s -> %r" % (label, url, quote[:70]))
    return 1


def write_csv(data):
    props = list(data["properties"])
    with open(os.path.join(HERE, "dataset.csv"), "w", newline="") as fh:
        writer = csv.writer(fh)
        writer.writerow(["option", "checked_at"] + props)
        for row in data["results"]:
            writer.writerow([row["option"], data["checkedAt"]] +
                            [row["cells"][p]["value"] for p in props])


def main():
    recheck = "--recheck" in sys.argv
    with open(DATASET) as fh:
        data = json.load(fh)
    failures = []
    checked = 0

    for row in data["results"]:
        print("\n%s" % row["option"])
        for prop, cell in row["cells"].items():
            pairs = [(cell.get("quote"), cell.get("url"))]
            pairs += [(e["quote"], e["url"]) for e in cell.get("extra", [])]
            for q, u in pairs:
                checked += check_quote(prop, q, u, failures)

    for section in ("post_claims", "product_claims"):
        print("\n%s" % section)
        for claim in data.get(section, []):
            checked += check_quote(claim["claim"][:30], claim["quote"], claim["url"], failures)

    write_csv(data)
    print("\nchecked %d quotes, %d failures%s"
          % (checked, len(failures), " (--recheck)" if recheck else ""))
    for f in failures:
        print("FAIL %s / %s / %r : %s" % f)
    return 0 if not failures else 1


if __name__ == "__main__":
    sys.exit(main())
