#!/usr/bin/env python3
"""Re-runs the check behind dataset.json for the Viallo post "photo-privacy-checklist".

Method primary-docs: every cell and every statement in dataset.json carries the sentences
copied from the vendor's own page on 2026-10-26. This script fetches each cited page again
and reports, per row, whether every recorded sentence still appears on it. A cell whose
value is "Not checked" has no page and is skipped; a cell with a page but no sentence only
has to load.

Nothing is measured locally, so there is no lab step, and --recheck runs the same fetches;
it is accepted so the weekly recheck can call every post's script the same way.

Text is compared after HTML is reduced to plain text: scripts and tags dropped, entities
decoded, whitespace collapsed, and the space a tag boundary leaves before punctuation removed.

Some pages depend on where the request comes from: telegram.org/privacy and picsart.com
served German to the German exit this check first ran from (2026-10-26) and English to the
Netherlands exit of 2026-11-08. Those quotes are stored as a list of the German and the
English sentence, and either one on the page counts.

Exit 0 when every sentence is still on its page, 1 when any is gone or a page cannot be read.

    python3 check.py
    python3 check.py --recheck
"""
import html
import json
import os
import re
import sys
import time
import urllib.error
import urllib.parse
import urllib.request
import zlib

HERE = os.path.dirname(os.path.abspath(__file__))
DATASET = os.path.join(HERE, "dataset.json")
TIMEOUT = 60

UA_DESKTOP = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
              "(KHTML, like Gecko) Chrome/131.0 Safari/537.36")
UA_MOBILE = ("Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/605.1.15 "
             "(KHTML, like Gecko) Version/17.0 Mobile/15E148 Safari/604.1")


def headers_for(url):
    host = urllib.parse.urlsplit(url).netloc
    h = {
        "User-Agent": UA_DESKTOP,
        "Accept": "text/html,application/xhtml+xml,application/pdf",
        "Accept-Language": "en-US,en;q=0.9",
        "Referer": "https://www.google.com/",
    }
    # whatsapp.com answers 400 to a desktop user agent on its legal pages.
    if host == "www.whatsapp.com":
        h["User-Agent"] = UA_MOBILE
        h["Referer"] = "https://www.whatsapp.com/"
    # web.archive.org answers 429 to a request without a Referer of its own.
    if host == "web.archive.org":
        h["Referer"] = "https://web.archive.org/"
    return h


def fetch_bytes(url):
    req = urllib.request.Request(url, headers=headers_for(url))
    last = None
    for attempt in range(3):
        try:
            with urllib.request.urlopen(req, timeout=TIMEOUT) as r:
                body = r.read()
                # policy.pinterest.com sends gzip even when no encoding was asked for.
                if (r.headers.get("Content-Encoding") or "").lower() == "gzip":
                    body = zlib.decompress(body, 16 + zlib.MAX_WBITS)
                return body, None
        except urllib.error.HTTPError as e:
            last = "HTTP %s" % e.code
        except Exception as e:
            last = "%s: %s" % (type(e).__name__, e)
        time.sleep(3 * (attempt + 1))
    return None, last


def normalise(raw):
    t = re.sub(r"(?is)<(script|style|noscript|svg)[^>]*>.*?</\1>", " ", raw)
    t = re.sub(r"(?s)<[^>]+>", " ", t)
    t = html.unescape(t)
    t = re.sub(r"\s+", " ", t)
    return re.sub(r"\s+([,.;:!?)\]])", r"\1", t)


def pdf_has(blob, value, near):
    target = value.encode()
    for m in re.finditer(rb"stream\r?\n(.*?)endstream", blob, re.S):
        try:
            text = zlib.decompress(m.group(1))
        except Exception:
            continue
        i = text.find(target)
        while i >= 0:
            if near.encode() in text[max(0, i - 400):i]:
                return True
            i = text.find(target, i + 1)
    return False


def found(quote, page):
    """A quote is a string, or a list of the same sentence in the languages a vendor serves
    by location; any one of them on the page counts."""
    if page is None:
        return False
    alts = quote if isinstance(quote, list) else [quote]
    return any(a in page for a in alts)


def first(quote):
    return quote[0] if isinstance(quote, list) else quote


def main(argv):
    recheck = "--recheck" in argv
    with open(DATASET, encoding="utf-8") as f:
        doc = json.load(f)

    urls = []
    for r in doc["results"]:
        if r["url"] and r["url"] not in urls:
            urls.append(r["url"])
    for s in doc["statements"]:
        if s["url"] not in urls:
            urls.append(s["url"])

    pages, unreadable = {}, []
    for u in urls:
        blob, err = fetch_bytes(u)
        if err:
            pages[u] = None
            unreadable.append(u)
            print("PAGE UNREADABLE  %s  (%s)" % (u, err))
            continue
        pages[u] = normalise(blob.decode("utf-8", "ignore"))
        print("fetched %-92s %7d chars" % (u, len(pages[u])))

    checked, bad = 0, 0

    print("\n-- cells (dataset.csv)")
    for r in doc["results"]:
        label = "%-16s %-23s" % (r["option"], r["property"])
        if not r["url"]:
            print("SKIP %s %s" % (label, r["value"]))
            continue
        checked += 1
        page = pages.get(r["url"])
        missing = [q for q in r["quotes"] if not found(q, page)]
        ok = page is not None and not missing
        print("%-4s %s %s" % ("OK" if ok else "GONE", label, r["value"][:70]))
        for q in missing:
            print("       missing: %s" % first(q)[:110])
        if not ok:
            bad += 1

    print("\n-- statements")
    for s in doc["statements"]:
        checked += 1
        page = pages.get(s["url"])
        ok = found(s["quote"], page)
        print("%-4s %-28s %s" % ("OK" if ok else "GONE", s["id"], first(s["quote"])[:70]))
        if not ok:
            bad += 1

    print("\n-- pdf")
    for p in doc.get("pdf_checks", []):
        checked += 1
        blob, err = fetch_bytes(p["url"])
        ok = blob is not None and pdf_has(blob, p["value"], p["near"])
        if err:
            unreadable.append(p["url"])
            print("PAGE UNREADABLE  %s  (%s)" % (p["url"], err))
        print("%-4s %-28s %s next to %r" % ("OK" if ok else "GONE", p["id"], p["value"], p["near"]))
        if not ok:
            bad += 1

    print("\n%d of %d rows hold; %d page(s) unreadable%s"
          % (checked - bad, checked, len(unreadable), " (--recheck)" if recheck else ""))
    if bad:
        print("FAIL: a vendor changed a page or refused this client; re-read the rows above by hand.")
        return 1
    print("PASS: every recorded sentence is still on the page it cites.")
    return 0


if __name__ == "__main__":
    sys.exit(main(sys.argv[1:]))
