#!/usr/bin/env python3
"""Re-verify the evidence behind the Viallo post "Does WhatsApp Compress Your Photos?".

Every row in dataset.json is re-read from its source and the quoted sentence is
looked for in it:
  page      fetched over HTTP, the quote searched in the page text;
  rendered  rendered with chrome-headless-shell (the WhatsApp Help Center draws its
            article text by script, so a plain fetch holds only the title);
  code      read from this repository's working tree.
Exits 0 when every row still holds, 1 otherwise. --recheck re-reads every page;
this dataset has no lab step to skip.

Standard library only. Set CHROME_HEADLESS to the renderer's path if it is not in
the Playwright cache.
"""

import csv
import glob
import html
import json
import os
import re
import shutil
import subprocess
import sys
import tempfile
import time
import urllib.error
import urllib.request

HERE = os.path.dirname(os.path.abspath(__file__))
REPO = os.path.abspath(os.path.join(HERE, "..", "..", "..", "..", ".."))
DATASET = os.path.join(HERE, "dataset.json")
CSV_PATH = os.path.join(HERE, "dataset.csv")
# A browser-like user agent makes faq.whatsapp.com answer 400; a bare one does not.
USER_AGENT = "Mozilla/5.0"


def norm(text):
    text = html.unescape(text)
    for src, dst in (("’", "'"), ("‘", "'"), ("“", '"'), ("”", '"'),
                     ("–", "-"), ("—", "-"), (" ", " ")):
        text = text.replace(src, dst)
    return re.sub(r"\s+", " ", text).strip()


def page_text(markup):
    markup = re.sub(r"(?is)<script.*?</script>", " ", markup)
    markup = re.sub(r"(?is)<style.*?</style>", " ", markup)
    markup = re.sub(r"(?s)<[^>]+>", " ", markup)
    return norm(markup)


def fetch(url, attempts=2):
    last = None
    for attempt in range(attempts):
        request = urllib.request.Request(url, headers={"User-Agent": USER_AGENT})
        try:
            with urllib.request.urlopen(request, timeout=30) as response:
                charset = response.headers.get_content_charset() or "utf-8"
                return response.read().decode(charset, "replace")
        except (urllib.error.URLError, urllib.error.HTTPError, TimeoutError) as error:
            last = error
            if attempt + 1 < attempts:
                time.sleep(2)
    raise RuntimeError(str(last))


def renderer():
    if os.environ.get("CHROME_HEADLESS"):
        return os.environ["CHROME_HEADLESS"]
    patterns = [
        "~/Library/Caches/ms-playwright/chromium_headless_shell-*/chrome-headless-shell-*/chrome-headless-shell",
        "~/.cache/ms-playwright/chromium_headless_shell-*/chrome-headless-shell-*/chrome-headless-shell",
    ]
    found = []
    for pattern in patterns:
        found += glob.glob(os.path.expanduser(pattern))
    return sorted(found)[-1] if found else None


def render(url):
    binary = renderer()
    if not binary:
        raise RuntimeError("chrome-headless-shell not found; set CHROME_HEADLESS")
    profile = tempfile.mkdtemp(prefix="wa-check-")
    try:
        result = subprocess.run(
            [binary, "--disable-gpu", "--lang=en-US", "--virtual-time-budget=20000",
             "--user-data-dir=" + profile, "--dump-dom", url],
            capture_output=True, text=True, timeout=120,
        )
    finally:
        shutil.rmtree(profile, ignore_errors=True)
    if result.returncode != 0 or not result.stdout:
        raise RuntimeError("render failed with exit %s" % result.returncode)
    return result.stdout


CACHE = {}


def source_text(row):
    url = row["url"]
    if row["kind"] == "code":
        with open(os.path.join(REPO, url.split(":", 1)[1]), encoding="utf-8") as handle:
            return handle.read()
    if url not in CACHE:
        CACHE[url] = page_text(render(url) if row["kind"] == "rendered" else fetch(url))
    return CACHE[url]


def check_row(row):
    """Return None when the row holds, otherwise the reason it does not."""
    if row["kind"] not in ("page", "rendered", "code"):
        return "unknown row kind %r" % row.get("kind")
    quote = row["quote"] if row["kind"] == "code" else norm(row["quote"])
    for attempt in range(2):
        try:
            text = source_text(row)
        except (RuntimeError, OSError, subprocess.TimeoutExpired) as error:
            return "source could not be read: %s" % error
        if quote in text:
            return None
        # A render that stopped before the article loaded looks like a missing quote.
        if row["kind"] != "rendered" or attempt:
            break
        CACHE.pop(row["url"], None)
    return "quote gone from %s" % row["url"]


def csv_matches(dataset):
    if not os.path.exists(CSV_PATH):
        return "dataset.csv is missing"
    with open(CSV_PATH, newline="", encoding="utf-8") as handle:
        rows = list(csv.DictReader(handle))
    expected = dataset["results"]
    if len(rows) != len(expected):
        return "dataset.csv has %d rows, dataset.json has %d" % (len(rows), len(expected))
    for index, (row, want) in enumerate(zip(rows, expected)):
        for field in ("id", "option", "property", "value", "quote", "url"):
            if row.get(field, "") != want[field]:
                return "dataset.csv row %d differs from dataset.json in %s" % (index + 2, field)
    return None


def main():
    with open(DATASET, encoding="utf-8") as handle:
        dataset = json.load(handle)

    failures = 0
    mismatch = csv_matches(dataset)
    if mismatch:
        failures += 1
        print("FAIL dataset.csv: %s" % mismatch)
    else:
        print("ok   dataset.csv matches dataset.json (%d rows)" % len(dataset["results"]))

    for row in dataset["results"]:
        reason = check_row(row)
        label = "%s [%s]" % (row["id"], row["kind"])
        if reason:
            failures += 1
            print("FAIL %-44s %s" % (label, reason))
        else:
            print("ok   %s" % label)

    print()
    if failures:
        print("%d check(s) no longer hold" % failures)
        return 1
    print("all %d rows hold, checked %s" % (len(dataset["results"]), dataset["checkedAt"]))
    return 0


if __name__ == "__main__":
    if "--recheck" in sys.argv[1:]:
        print("recheck: re-reading every cited page; no lab step to skip\n")
    sys.exit(main())