#!/usr/bin/env python3
"""Re-runs the evidence check for how-to-share-photos-privately.

Reads dataset.json, fetches every page it cites and reports, per row, whether
each supporting quote is still on that page. The two Viallo code inputs are
read from the Viallo repository this file sits in; run anywhere else, they are
skipped and reported as such.

    python3 check.py            full run
    python3 check.py --recheck  the same run (there is no lab step to skip)

Exit 0 when every quoted row and citation still holds and dataset.csv matches
dataset.json, 1 otherwise.
"""

import argparse
import csv
import html
import json
import os
import re
import subprocess
import sys
import urllib.error
import urllib.request

HERE = os.path.dirname(os.path.abspath(__file__))
DATASET = os.path.join(HERE, "dataset.json")
CSV_PATH = os.path.join(HERE, "dataset.csv")
PROPERTIES = ["viewer_account", "resolution", "end_date"]
TIMEOUT = 45


class Skipped(Exception):
    pass


def normalise(text):
    text = html.unescape(text)
    text = text.replace("\u00a0", " ").replace("\u00ad", "")
    for a, b in (
        ("\u2019", "'"),
        ("\u2018", "'"),
        ("\u201c", '"'),
        ("\u201d", '"'),
        ("\u2014", "-"),
        ("\u2013", "-"),
    ):
        text = text.replace(a, b)
    return re.sub(r"\s+", " ", text).strip()


def html_to_text(raw):
    raw = re.sub(r"<script.*?</script>", " ", raw, flags=re.S | re.I)
    raw = re.sub(r"<style.*?</style>", " ", raw, flags=re.S | re.I)
    return re.sub(r"<[^>]+>", " ", raw)


def fetch_http(url, user_agent):
    request = urllib.request.Request(url, headers={"User-Agent": user_agent})
    with urllib.request.urlopen(request, timeout=TIMEOUT) as response:
        charset = response.headers.get_content_charset() or "utf-8"
        return response.read().decode(charset, errors="replace")


def read_repo(source):
    result = subprocess.run(
        ["git", "-C", HERE, "rev-parse", "--show-toplevel"], capture_output=True, text=True
    )
    if result.returncode != 0:
        raise Skipped("not inside the Viallo repository")
    path = os.path.join(result.stdout.strip(), source["file"])
    if not os.path.exists(path):
        raise Skipped("{} not found in this checkout".format(source["file"]))
    with open(path, encoding="utf-8") as handle:
        return handle.read()


def source_text(source, user_agent):
    if source["kind"] == "http":
        return normalise(html_to_text(fetch_http(source["url"], user_agent)))
    if source["kind"] == "repo":
        return normalise(read_repo(source))
    raise RuntimeError("source {} is marked {}: {}".format(source["id"], source["kind"], source.get("note", "")))


def check_quotes(label, quotes, sources, cache, user_agent):
    ok = True
    for quote in quotes:
        source_id = quote["source"]
        source = sources.get(source_id)
        if source is None:
            print("FAIL  {:<62} unknown source {}".format(label, source_id))
            ok = False
            continue
        if source_id not in cache:
            try:
                cache[source_id] = source_text(source, user_agent)
            except (urllib.error.URLError, RuntimeError, OSError, Skipped) as error:
                cache[source_id] = error
        text = cache[source_id]
        if isinstance(text, Skipped):
            print("SKIP  {:<62} {}: {}".format(label, source_id, text))
            continue
        if isinstance(text, Exception):
            print("FAIL  {:<62} {} could not be read: {}".format(label, source_id, text))
            ok = False
            continue
        if normalise(quote["text"]) in text:
            print("OK    {:<62} quote found in {}".format(label, source_id))
        else:
            print("FAIL  {:<62} quote gone from {}".format(label, source_id))
            print("        quote: {}".format(quote["text"][:160]))
            ok = False
    return ok


def check_rows(dataset, user_agent):
    sources = {item["id"]: item for item in dataset["inputs"]}
    cache = {}
    problems = []

    for row in dataset["results"]:
        label = "{} / {}".format(row["option"], row["property"])
        if row["status"] != "ok":
            print("SKIP  {:<62} {}: {}".format(label, row["status"], row["value"]))
            continue
        if not check_quotes(label, row["quotes"], sources, cache, user_agent):
            problems.append(label)

    for citation in dataset.get("citations", []):
        label = "citation: {}".format(citation["claim"])[:62]
        if not check_quotes(label, citation["quotes"], sources, cache, user_agent):
            problems.append(label)

    return problems


def check_csv(dataset):
    if not os.path.exists(CSV_PATH):
        print("FAIL  dataset.csv is missing")
        return ["dataset.csv missing"]

    expected = {}
    order = []
    for row in dataset["results"]:
        if row["option"] not in expected:
            expected[row["option"]] = ["", "", "", row["url"]]
            order.append(row["option"])
        expected[row["option"]][PROPERTIES.index(row["property"])] = row["value"]
    expected_rows = [[option] + expected[option] for option in order]

    with open(CSV_PATH, newline="", encoding="utf-8") as handle:
        reader = csv.reader(handle)
        header = next(reader)
        actual = [line for line in reader if line]

    problems = []
    wanted_header = ["option"] + PROPERTIES + ["source_url"]
    if header != wanted_header:
        print("FAIL  dataset.csv header is {} not {}".format(header, wanted_header))
        problems.append("csv header")
    if actual != expected_rows:
        for line in actual:
            if line not in expected_rows:
                print("FAIL  dataset.csv row not in dataset.json: {}".format(line))
        for line in expected_rows:
            if line not in actual:
                print("FAIL  dataset.json row not in dataset.csv: {}".format(line))
        problems.append("csv rows")
    else:
        print("OK    dataset.csv matches dataset.json ({} rows)".format(len(actual)))
    return problems


def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("--recheck", action="store_true", help="re-read the cited pages; skip lab steps over a minute")
    args = parser.parse_args()

    with open(DATASET, encoding="utf-8") as handle:
        dataset = json.load(handle)
    user_agent = dataset["tools"]["userAgent"]

    print("check.py{}  method={}  checkedAt={}".format(
        " --recheck" if args.recheck else "", dataset["method"], dataset["checkedAt"]))
    print("")

    problems = check_rows(dataset, user_agent)
    print("")
    problems += check_csv(dataset)

    quoted = sum(1 for row in dataset["results"] if row["status"] == "ok")
    print("")
    if problems:
        print("FAILED: {} problem(s)".format(len(problems)))
        return 1
    print("All {} quoted rows and {} citations hold.".format(quoted, len(dataset.get("citations", []))))
    return 0


if __name__ == "__main__":
    sys.exit(main())