#!/usr/bin/env python3
"""Re-check the primary-docs table for the photo-sharing-no-compression post.

Fetches every URL named in dataset.json and reports, per row, whether its
supporting quote still appears on the page. Rows that record an absence carry no
quote; they are printed as notes and never fail the run.

    python3 check.py            # check every row
    python3 check.py --recheck  # same; this table has no lab step to skip
"""

import gzip
import html
import json
import os
import re
import sys
import time
import urllib.error
import urllib.request

HERE = os.path.dirname(os.path.abspath(__file__))
DATASET = os.path.join(HERE, "dataset.json")
BROWSER_UA = (
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
    "(KHTML, like Gecko) Chrome/126 Safari/537.36"
)
# help.instagram.com and faq.whatsapp.com return a shell with no article body to
# a browser user agent; the crawler agent gets the same page with the text.
CRAWLER_UA = "Googlebot/2.1 (+http://www.google.com/bot.html)"
CRAWLER_HOSTS = ("help.instagram.com", "faq.whatsapp.com")
TIMEOUT = 60
ATTEMPTS = 3

DROP_CHARS = "\"'`‘’“”«»"


def user_agent_for(url):
    for host in CRAWLER_HOSTS:
        if host in url:
            return CRAWLER_UA
    return BROWSER_UA


def normalise(text):
    """Whitespace, entities, tags, quote marks and dash variants removed."""
    text = html.unescape(text)
    # JSON-embedded help centres escape their markup and slashes.
    text = re.sub(r"\\u([0-9a-fA-F]{4})", lambda m: chr(int(m.group(1), 16)), text)
    text = text.replace("\\/", "/").replace('\\"', '"')
    text = re.sub(r"<[^>]*>", " ", text)
    text = html.unescape(text)
    for ch in DROP_CHARS:
        text = text.replace(ch, "")
    text = re.sub(r"[‐-―−]", "-", text)
    text = re.sub(r"\s+", " ", text)
    return text.strip().casefold()


def fetch(url):
    last_error = None
    for attempt in range(ATTEMPTS):
        request = urllib.request.Request(
            url,
            headers={
                "User-Agent": user_agent_for(url),
                "Accept": "text/html,application/xhtml+xml",
                "Accept-Language": "en-US,en;q=0.9",
                "Accept-Encoding": "gzip",
            },
        )
        try:
            with urllib.request.urlopen(request, timeout=TIMEOUT) as response:
                body = response.read()
                if response.headers.get("Content-Encoding", "").lower() == "gzip":
                    body = gzip.decompress(body)
                charset = response.headers.get_content_charset() or "utf-8"
                return body.decode(charset, "replace")
        except (urllib.error.URLError, urllib.error.HTTPError, TimeoutError, OSError) as exc:
            last_error = exc
            if attempt + 1 < ATTEMPTS:
                time.sleep(2)
    raise RuntimeError("%s: %s" % (url, last_error))


def main(argv):
    recheck = "--recheck" in argv
    with open(DATASET, encoding="utf-8") as handle:
        dataset = json.load(handle)

    pages = {}
    failures = 0
    notes = 0
    rows = dataset["results"]

    for row in rows:
        if row.get("type") == "absence" or not row.get("quote"):
            notes += 1
            print("note  %-4s %-15s %-42s no quote: %s" % (
                row["id"], row["option"], row["property"], row.get("value", "")))
            continue
        url = row["url"]
        if url not in pages:
            try:
                pages[url] = normalise(fetch(url))
            except RuntimeError as exc:
                pages[url] = None
                print("FETCH FAILED  %s" % exc)
        page = pages[url]
        if page is None:
            print("FAIL  %-4s %-15s %-42s page not fetched" % (row["id"], row["option"], row["property"]))
            failures += 1
            continue
        if normalise(row["quote"]) in page:
            print("ok    %-4s %-15s %s" % (row["id"], row["option"], row["property"]))
        else:
            print("FAIL  %-4s %-15s %-42s quote gone: %s" % (
                row["id"], row["option"], row["property"], row["quote"][:90]))
            failures += 1

    mode = "recheck" if recheck else "check"
    print("\n%s: %d rows, %d failed, %d notes, %d pages fetched" % (
        mode, len(rows), failures, notes, len(pages)))
    return 0 if failures == 0 else 1


if __name__ == "__main__":
    sys.exit(main(sys.argv[1:]))