#!/usr/bin/env python3
"""Re-run the public-web check behind the Google Photos API lockdown post.

Fetches every URL the dataset cites, normalises the page to visible text and
reports, row by row, whether the quoted sentence is still on it. Rows in
`results` are the comparison table; rows in `claims` are the third-party
statements the post makes. A row whose `quote` is null was marked cut for want
of a source and has nothing to re-check.

    python3 check.py            # fetch and verify
    python3 check.py --recheck  # same, for the weekly recheck

Exit 0 when every quote still holds, 1 otherwise.
"""

import html
import json
import os
import re
import sys
import urllib.error
import urllib.request

USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)"
TIMEOUT = 45
HERE = os.path.dirname(os.path.abspath(__file__))

QUOTE_MAP = str.maketrans({
    "‘": "'", "’": "'", "‚": "'", "‛": "'",
    "“": '"', "”": '"', "„": '"',
    "–": "-", "—": "-", "−": "-",
    " ": " ", " ": " ", " ": " ", "": " ",
})


def normalise(text):
    text = text.translate(QUOTE_MAP)
    text = re.sub(r"\s+", " ", text)
    text = re.sub(r"\s+([,.;:!?)\]])", r"\1", text)
    return text.strip()


def visible_text(raw):
    s = raw
    s = re.sub(r"(?is)<(script|style|svg|noscript|template)\b[^>]*>.*?</\1\s*>", " ", s)
    s = re.sub(r"(?s)<!--.*?-->", " ", s)
    s = re.sub(r"(?s)<[^>]+>", " ", s)
    return html.unescape(s)


class Follow308(urllib.request.HTTPRedirectHandler):
    """Python 3.9 does not follow 308; this machine runs 3.9."""

    http_error_308 = urllib.request.HTTPRedirectHandler.http_error_302

    def redirect_request(self, req, fp, code, msg, headers, newurl):
        if code == 308:
            code = 302
        return urllib.request.HTTPRedirectHandler.redirect_request(
            self, req, fp, code, msg, headers, newurl)


OPENER = urllib.request.build_opener(Follow308)


def fetch(url):
    req = urllib.request.Request(url, headers={
        "User-Agent": USER_AGENT,
        "Accept": "text/html,application/xhtml+xml",
        "Accept-Language": "en-US,en;q=0.9",
    })
    with OPENER.open(req, timeout=TIMEOUT) as resp:
        charset = resp.headers.get_content_charset() or "utf-8"
        return resp.read().decode(charset, "replace")


def row_shape(row, kind):
    return {
        "id": row["id"],
        "kind": kind,
        "label": row.get("tool") or row.get("postClaim"),
        "verdict": row.get("verdict") or row.get("routeOut"),
        "quote": row.get("quote"),
        "url": row.get("url"),
        "absent": row.get("absent"),
        "extra": row.get("quotes") or [],
    }


def main(argv):
    recheck = "--recheck" in argv[1:]
    with open(os.path.join(HERE, "dataset.json"), encoding="utf-8") as fh:
        data = json.load(fh)

    rows = [row_shape(r, "table") for r in data["results"]]
    rows += [row_shape(c, "claim") for c in data["claims"]]

    urls = [r["url"] for r in rows if r["url"]]
    urls += [q["url"] for r in rows for q in r["extra"]]
    pages = {}
    fetch_errors = {}
    for url in sorted(set(urls)):
        try:
            pages[url] = normalise(visible_text(fetch(url)))
        except (urllib.error.URLError, urllib.error.HTTPError, OSError) as exc:
            fetch_errors[url] = str(exc)

    print("check: %s  (%s)%s" % (
        data["slug"], data["checkedAt"], "  [--recheck]" if recheck else ""))
    print("%-5s %-6s %-38s %s" % ("id", "kind", "item", "result"))

    failures = 0
    skipped = 0
    for row in rows:
        label = re.sub(r"\s+", " ", (row["label"] or ""))[:36]
        if not row["url"]:
            skipped += 1
            print("%-5s %-6s %-38s SKIP  marked %s, no source to fetch"
                  % (row["id"], row["kind"], label, row["verdict"]))
            continue
        page = pages.get(row["url"])
        if page is None:
            failures += 1
            print("%-5s %-6s %-38s FAIL  fetch: %s"
                  % (row["id"], row["kind"], label, fetch_errors.get(row["url"], "?")))
            continue

        problems = []
        quote = normalise(row["quote"] or "")
        if quote and quote not in page:
            problems.append("quote gone")
        for extra in row["extra"]:
            other = pages.get(extra["url"])
            if other is None:
                problems.append("fetch: %s" % fetch_errors.get(extra["url"], "?"))
            elif normalise(extra["quote"]) not in other:
                problems.append("quote gone  %s" % extra["url"])
        if row["absent"]:
            needle = normalise(row["absent"])
            if needle.lower() in page.lower():
                problems.append("'%s' now appears on the page" % row["absent"])

        if problems:
            failures += 1
            print("%-5s %-6s %-38s FAIL  %s  %s"
                  % (row["id"], row["kind"], label, "; ".join(problems), row["url"]))
        else:
            print("%-5s %-6s %-38s OK" % (row["id"], row["kind"], label))

    checked = len(rows) - skipped
    print("\n%d rows, %d checked, %d skipped (no source), %d failed"
          % (len(rows), checked, skipped, failures))
    if failures:
        print("RE-CHECK THE POST: a cited page no longer says what the dataset says it says.")
        return 1
    print("All quotes still appear on the pages the dataset cites.")
    return 0


if __name__ == "__main__":
    sys.exit(main(sys.argv))