#!/usr/bin/env python3
"""Re-run the primary-docs check behind dataset.json.

For every row it fetches the row's URL, strips the page to text, and asks whether
the row's quote still appears. That is the whole contract: exit 0 when every quote
is still on the page it cites, 1 when any is gone.

    python3 check.py             # fetch every cited page
    python3 check.py --recheck   # same; no lab step exists here to skip

Standard library only.
"""

from __future__ import annotations

import argparse
import gzip
import html
import io
import json
import re
import sys
import time
import urllib.error
import urllib.request
from html.parser import HTMLParser
from pathlib import Path

HERE = Path(__file__).resolve().parent
DATASET = HERE / "dataset.json"

UA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)"
TIMEOUT = 45
RETRIES = 3
SKIP_TAGS = {"script", "style", "noscript", "template"}

# Curly punctuation maps to ASCII so a quote written with " matches a page that
# renders a typographic quote, and vice versa.
PUNCT = str.maketrans(
    {
        "“": '"',
        "”": '"',
        "‘": "'",
        "’": "'",
        "‚": "'",
        "„": '"',
        "–": "-",
        "—": "-",
        "−": "-",
        " ": " ",
        "…": "...",
    }
)


class TextExtractor(HTMLParser):
    def __init__(self) -> None:
        super().__init__(convert_charrefs=True)
        self.chunks: list[str] = []
        self.depth = 0

    def handle_starttag(self, tag, attrs):
        if tag in SKIP_TAGS:
            self.depth += 1

    def handle_endtag(self, tag):
        if tag in SKIP_TAGS and self.depth:
            self.depth -= 1

    def handle_data(self, data):
        if not self.depth and data.strip():
            self.chunks.append(data)


def normalise(text: str) -> str:
    """Collapse whitespace, unify quotes and entities, drop space before punctuation."""
    for _ in range(2):
        text = html.unescape(text)
    text = text.translate(PUNCT)
    text = re.sub(r"\s+", " ", text)
    text = re.sub(r"\s+([.,;:!?)\]])", r"\1", text)
    return text.strip()


def page_text(raw: bytes) -> str:
    if raw[:2] == b"\x1f\x8b":
        raw = gzip.decompress(raw)
    body = raw.decode("utf-8", errors="replace")
    parser = TextExtractor()
    parser.feed(body)
    return normalise(" ".join(parser.chunks))


def fetch(url: str) -> tuple[str | None, str]:
    """Return (normalised page text, error). Error is empty on success."""
    last = ""
    for attempt in range(RETRIES):
        request = urllib.request.Request(
            url,
            headers={
                "User-Agent": UA,
                "Accept": "text/html,application/xhtml+xml",
                "Accept-Language": "en-US,en;q=0.9",
            },
        )
        try:
            with urllib.request.urlopen(request, timeout=TIMEOUT) as response:
                return page_text(response.read()), ""
        except urllib.error.HTTPError as exc:
            last = f"HTTP {exc.code}"
            if exc.code in (403, 404, 410):
                break
        except Exception as exc:  # noqa: BLE001 - any transport failure is the same report
            last = f"{type(exc).__name__}: {exc}"
        if attempt + 1 < RETRIES:
            time.sleep(2 * (attempt + 1))
    return None, last or "unreachable"


def main() -> int:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument(
        "--recheck",
        action="store_true",
        help="re-read the cited pages; there is no lab step here to skip",
    )
    args = parser.parse_args()

    data = json.loads(DATASET.read_text(encoding="utf-8"))
    rows = data["results"]

    cache: dict[str, tuple[str | None, str]] = {}
    failures: list[str] = []
    checked = 0

    for row in rows:
        url = row["url"]
        if url not in cache:
            cache[url] = fetch(url)
            if args.recheck:
                time.sleep(0.5)
        text, error = cache[url]
        for prop, spec in row["properties"].items():
            quote = normalise(spec["quote"])
            label = f"{row['option']} / {prop}"
            if text is None:
                failures.append(f"{label}: page not read ({error}) {url}")
                print(f"NOT CHECKED  {label}  <- {error}")
                continue
            checked += 1
            if quote in text:
                print(f"ok           {label}")
            else:
                failures.append(f"{label}: quote gone from {url}")
                print(f"MISSING      {label}  <- {url}")

    print()
    print(f"{checked} quotes checked across {len(cache)} pages on {data['checkedAt']}")
    if failures:
        print(f"{len(failures)} failed:")
        for line in failures:
            print(f"  - {line}")
        return 1
    print("every quote is still on the page it cites")
    return 0


if __name__ == "__main__":
    sys.exit(main())