#!/usr/bin/env python3
"""Re-runs the primary-docs check behind share-photos-without-social-media:
for each of the five platforms, does its own help page still say what
dataset.csv records about the quality setting, the download-link lifetime and
the stated preparation time, and do the other pages the post cites still carry
the sentence it rests on?

Each row carries the live URL and, where the live host refuses a non-browser
fetcher, an archived copy of the same page. The row passes when every recorded
quote is found in the normalised text of the live page or of the archived copy.
Exit 0 when all rows hold, 1 otherwise.
"""

import argparse
import csv
import html
import os
import re
import sys
import time
import urllib.error
import urllib.request

UA = (
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
    "(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
)

HERE = os.path.dirname(os.path.abspath(__file__))

# One row per platform. A property with quoted=True must be found on the page;
# quoted=False records that the page states nothing for it, so it has no quote.
ROWS = [
    {
        "platform": "Instagram",
        "url": "https://help.instagram.com/181231772500920/",
        "archive": "https://web.archive.org/web/20261009062151/https://help.instagram.com/181231772500920/",
        "cells": [
            ("quality_setting", True, "select a date range, format, the notification email and media quality"),
            ("download_link_lifetime", True, "you'll have 4 days to export your information"),
            ("stated_preparation_time", True, "It may take up to 30 days for us to email you an export link."),
        ],
    },
    {
        "platform": "Facebook",
        "url": "https://www.facebook.com/help/212802592074644/",
        "archive": "https://web.archive.org/web/20261010051811/https://www.facebook.com/help/212802592074644",
        "cells": [
            ("quality_setting", True, "select a date range, format, the notification email and media quality"),
            ("download_link_lifetime", True, "you'll have 4 days to export your information"),
            ("stated_preparation_time", False, ""),
        ],
    },
    {
        "platform": "TikTok",
        "url": "https://www.tiktok.com/support/faq_detail?id=7543597460594285112",
        "archive": "https://web.archive.org/web/20260306050201/https://support.tiktok.com/en/account-and-privacy/personalized-ads-and-data/requesting-your-data",
        "cells": [
            ("quality_setting", True, "Choose which information you want to include in your file to download and select a file format."),
            ("download_link_lifetime", True, "Your file will be available for up to 4 days."),
            ("stated_preparation_time", True, "it may take a few days to prepare the file"),
        ],
    },
    {
        "platform": "Snapchat",
        "url": "https://help.snapchat.com/hc/en-us/articles/7012305371156-How-do-I-download-my-data-from-Snapchat",
        "archive": "",
        "cells": [
            ("quality_setting", True, "Select the data you would like to include in your data download."),
            ("download_link_lifetime", False, ""),
            ("stated_preparation_time", True, "We strive to deliver data within 7 days, but large downloads can take longer."),
        ],
    },
    {
        "platform": "X",
        "url": "https://help.x.com/en/managing-your-account/how-to-download-your-x-archive",
        "archive": "https://web.archive.org/web/20261015124209/https://help.x.com/en/managing-your-account/how-to-download-your-x-archive",
        "cells": [
            ("quality_setting", True, "You can also download a machine-readable archive of information associated with your account in HTML and JSON files."),
            ("download_link_lifetime", False, ""),
            ("stated_preparation_time", True, "It may take a few days for us to prepare the download of your X archive."),
        ],
    },
]

# Pages the post cites outside the export table, with the sentence each claim rests on.
SOURCES = [
    ("Gmail attachment limit", "https://support.google.com/mail/answer/6584?hl=en",
     "For personal Gmail accounts, the limit is 25 MB."),
    ("Google Photos shared album", "https://support.google.com/photos/answer/6131416?hl=en",
     "When you share an album, anyone with access to the album can view the photos and add to the album."),
    ("Google Account storage", "https://support.google.com/photos/answer/6220791?hl=en",
     "Each Google Account comes with up to 15 GB of storage"),
    ("Google Drive link sharing", "https://support.google.com/drive/answer/2494822?hl=en",
     "Anyone on the internet who has the link can open the file without signing in to a Google Account."),
    ("Dropbox view-only link", "https://help.dropbox.com/share/create-and-share-link",
     "Anyone with the link can preview the content without editing it."),
    ("iCloud Shared Albums and Advanced Data Protection", "https://support.apple.com/en-us/108756",
     "the Shared Albums feature in Photos, and sharing content with \"anyone with the link,\" don't support Advanced Data Protection."),
    ("GDPR Article 25(2)", "https://gdpr-info.eu/art-25-gdpr/",
     "by default, only personal data which are necessary for each specific purpose of the processing are processed."),
]

QUOTES = {
    "\u2018": "'", "\u2019": "'", "\u201a": "'", "\u201b": "'",
    "\u201c": '"', "\u201d": '"', "\u201e": '"', "\u201f": '"',
    "\u2013": "-", "\u2014": "-", "\u2212": "-", "\u00a0": " ",
    "\u200b": "", "\u200e": "", "\u200f": "", "\ufeff": "",
}


def normalise(raw):
    """Page text or recorded quote -> one comparable string."""
    text = html.unescape(raw)
    # Meta and TikTok ship the article inside a JSON blob in a <script>, so the
    # text arrives with escaped quotes and \uXXXX escapes still in it.
    text = re.sub(r"\\+u([0-9a-fA-F]{4})", lambda m: chr(int(m.group(1), 16)), text)
    text = re.sub(r'\\+"', '"', text)
    # The X help centre writes the company name as the escaped token >x<.
    text = text.replace(">x<", "X")
    text = re.sub(r"<style\b[^>]*>.*?</style>", " ", text, flags=re.S | re.I)
    text = re.sub(r"<!--.*?-->", " ", text, flags=re.S)
    text = re.sub(r"<[^>]+>", " ", text)
    for bad, good in QUOTES.items():
        text = text.replace(bad, good)
    return re.sub(r"\s+", " ", text).strip()


def fetch(url, timeout):
    headers = {
        "User-Agent": UA,
        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
        "Accept-Language": "en-US,en;q=0.9",
        "Accept-Encoding": "identity",
        # Meta's help centres answer 400 without these navigation headers.
        "Sec-Fetch-Mode": "navigate",
        "Sec-Fetch-Dest": "document",
        "Sec-Fetch-Site": "none",
        "Upgrade-Insecure-Requests": "1",
        "sec-ch-ua": '"Chromium";v="131", "Not_A Brand";v="24"',
        "sec-ch-ua-mobile": "?0",
        "sec-ch-ua-platform": '"macOS"',
        "Referer": "https://web.archive.org/" if "web.archive.org" in url else "",
    }
    request = urllib.request.Request(url, headers=dict((k, v) for k, v in headers.items() if v))
    # The archive rate-limits a burst of five requests; retries with a growing
    # pause are what a weekly recheck needs to get through.
    last = None
    for attempt, pause in enumerate((0, 8, 20, 40, 70)):
        if pause:
            time.sleep(pause)
        try:
            with urllib.request.urlopen(request, timeout=timeout) as response:
                return response.status, response.read().decode("utf-8", "replace")
        except urllib.error.HTTPError as error:
            last = error
            if error.code not in (429, 500, 502, 503, 504):
                raise
        except OSError as error:
            last = error
    raise last


def raw_archive(url):
    # The archive's player page is a script shell; id_ serves the captured page itself.
    return re.sub(r"(/web/\d+)/", r"\1id_/", url, count=1)


def load_dataset_rows():
    """The CSV is the results table; the quotes above must match it."""
    path = os.path.join(HERE, "dataset.csv")
    with open(path, newline="", encoding="utf-8") as handle:
        return list(csv.DictReader(handle))


def check_row(row, timeout):
    """Return (passed, source, misses, tried)."""
    misses = []
    tried = []
    candidates = [("live", row["url"])]
    if row.get("archive"):
        candidates.append(("archive", row["archive"]))
        candidates.append(("archive", raw_archive(row["archive"])))
    for kind, url in candidates:
        tried.append(url)
        try:
            status, body = fetch(url, timeout)
        except (urllib.error.URLError, urllib.error.HTTPError, OSError) as error:
            misses.append("%s: %s" % (kind, getattr(error, "code", error)))
            continue
        if status != 200:
            misses.append("%s: HTTP %s" % (kind, status))
            continue
        text = normalise(body)
        absent = [q for q, need in row["checks"] if need and q not in text]
        if not absent:
            return True, kind, [], tried
        misses.append("%s: %d quote(s) gone" % (kind, len(absent)))
        misses.extend("  missing: %r" % q for q in absent)
    return False, "none", misses, tried


def is_row_consistent_with_csv(row, csv_row):
    """The script's expectations and the published table must not drift apart."""
    problems = []
    for prop in ("quality_setting", "download_link_lifetime", "stated_preparation_time"):
        quote = csv_row[prop + "_quote"]
        value = csv_row[prop + "_value"]
        scripted = dict((name, (need, q)) for name, need, q in row["cells"])[prop]
        need, expected_quote = scripted
        if (value == "not stated") == need:
            problems.append("%s/%s: 'not stated' and a required quote disagree" % (row["platform"], prop))
        if normalise(quote) != normalise(expected_quote):
            problems.append("%s/%s: quote differs between check.py and dataset.csv" % (row["platform"], prop))
    if csv_row["url"] != row["url"]:
        problems.append("%s: live URL differs between check.py and dataset.csv" % row["platform"])
    return problems


def main():
    parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
    parser.add_argument(
        "--recheck",
        action="store_true",
        help="re-read the pages with a shorter timeout; there is no lab step to skip",
    )
    args = parser.parse_args()
    timeout = 20 if args.recheck else 45

    csv_rows = dict((r["platform"], r) for r in load_dataset_rows())
    failures = 0
    inconsistencies = 0

    print("dataset checkedAt: 2026-10-26   mode: %s   timeout: %ds"
          % ("recheck" if args.recheck else "full", timeout))
    for row in ROWS:
        csv_row = csv_rows.get(row["platform"])
        if csv_row is None:
            print("FAIL %-10s no row in dataset.csv" % row["platform"])
            failures += 1
            continue
        problems = is_row_consistent_with_csv(row, csv_row)
        if problems:
            inconsistencies += len(problems)
            for problem in problems:
                print("DRIFT %s" % problem)
        row["checks"] = [(q, need) for _, need, q in row["cells"]]
        passed, source, misses, tried = check_row(row, timeout)
        if passed:
            print("ok   %-10s quotes found on the %s page" % (row["platform"], source))
        else:
            failures += 1
            print("FAIL %-10s" % row["platform"])
            for line in misses:
                print("       %s" % line)

    source_failures = 0
    for name, url, quote in SOURCES:
        try:
            status, body = fetch(url, timeout)
            found = status == 200 and normalise(quote) in normalise(body)
            reason = "quote gone" if status == 200 else "HTTP %s" % status
        except (urllib.error.URLError, urllib.error.HTTPError, OSError) as error:
            found, reason = False, str(getattr(error, "code", error))
        where = "live"
        if not found and reason != "quote gone":
            # A bot wall on the live page (Dropbox answers 202 with a challenge) is not drift.
            try:
                status, body = fetch(raw_archive("https://web.archive.org/web/2026/" + url), timeout)
                found, where = status == 200 and normalise(quote) in normalise(body), "archive"
            except (urllib.error.URLError, urllib.error.HTTPError, OSError):
                pass
        if found:
            print("ok   source  %s (%s)" % (name, where))
        else:
            source_failures += 1
            print("FAIL source  %s: %s (%s)" % (name, reason, url))

    print("\n%d of %d platforms hold, %d of %d cited sources hold, %d drift problem(s)"
          % (len(ROWS) - failures, len(ROWS), len(SOURCES) - source_failures, len(SOURCES), inconsistencies))
    return 0 if failures == 0 and source_failures == 0 and inconsistencies == 0 else 1


if __name__ == "__main__":
    sys.exit(main())