#!/usr/bin/env python3
"""Re-run the google-tos-update-photos primary-docs check.

Reads Google's Terms of Service effective 30 July 2026, the version effective
22 May 2024 and Google's own comparison of the two from policies.google.com,
the Gemini features in Photos privacy notice, three Apple pages and one Google
blog post, and reports per row whether the supporting sentence is still there.
The terms pages are country versions: the baseline rows pin gl=TH, where they were
first read, and the country rows pin gl=US and gl=NL.

    python3 check.py             # check, then rewrite dataset.json and dataset.csv
    python3 check.py --recheck   # check only, leave the dataset files alone

Exit 0 when every row holds, 1 otherwise. There are no lab steps, so --recheck
skips nothing.
"""

import csv
import json
import re
import sys
import time
import urllib.request
from html.parser import HTMLParser
from pathlib import Path

UA = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
      "(KHTML, like Gecko) Chrome/120.0 Safari/537.36")

# policies.google.com serves a country version chosen by the reader's location. The
# baseline rows were first read from Thailand, so they pin gl=TH and hold wherever this
# runs; the country rows below pin the version they describe.
TERMS = "https://policies.google.com/terms?hl=en&gl=%s"
TERMS_2024 = "https://policies.google.com/terms/archive/20240522?hl=en&gl=%s"
TERMS_CMP = "https://policies.google.com/terms/archive/20240522-20260730?hl=en&gl=%s"
TOS, TOS_2024, TOS_CMP = TERMS % "TH", TERMS_2024 % "TH", TERMS_CMP % "TH"
TOS_US, TOS_CMP_US = TERMS % "US", TERMS_CMP % "US"
TOS_NL, TOS_CMP_NL = TERMS % "NL", TERMS_CMP % "NL"
PHOTOS = "https://support.google.com/photos/answer/15344015?hl=en"
APPLE_PRIVACY = "https://www.apple.com/privacy/"
APPLE_LEGAL = "https://www.apple.com/legal/privacy/en-ww/"
APPLE_ALBUMS = "https://support.apple.com/en-us/108916"
GPHOTOS_SCALE = ("https://blog.google/products-and-platforms/products/photos/"
                 "google-photos-10-years-tips-tricks/")

T26 = "Terms of Service, effective 30 July 2026"
T24 = "Terms of Service, effective 22 May 2024"
CMP = "Google's comparison of the two versions"
GPN = "Gemini features in Photos privacy notice"
T26_US = "Terms of Service, effective 30 July 2026, United States version"
CMP_US = "Google's comparison of the two versions, United States version"
T26_NL = "Terms of Service, effective 30 July 2026, Netherlands version"
CMP_NL = "Google's comparison of the two versions, Netherlands version"

# kind: quote   the sentence appears in the page text
#       absent  the phrase does not appear in the page text
#       count   the page text holds the word exactly `expect` times
#       ins     the sentence sits inside an element Google marks as new text
#       tags    the raw page holds exactly `expect` elements matching `quote`
#       clean   no <ins> or <del> between the two markers in `quote`
ROWS = [
    # --- the terms in force today -------------------------------------------
    dict(id="t26-effective", doc=T26, clause="effective date", value="30 July 2026",
         quote="Effective July 30, 2026", url=TOS, kind="quote"),
    dict(id="t26-licence-scope", doc=T26, clause="content license scope",
         value="same wording: worldwide, non-exclusive, royalty-free",
         quote="non-exclusive, which means you can license your content to others",
         url=TOS, kind="quote"),
    dict(id="t26-licence-develop", doc=T26, clause="license purpose: new technologies",
         value="same wording",
         quote="developing new technologies and services for Google consistent with these terms",
         url=TOS, kind="quote"),
    dict(id="t26-licence-rights", doc=T26, clause="rights granted",
         value="host, reproduce, distribute, publish, modify, create derivative works, sublicense",
         quote="modify and create derivative works based on your content",
         url=TOS, kind="quote"),
    dict(id="t26-removal", doc=T26, clause="what removing content does",
         value="Google stops showing it publicly in a reasonable amount of time",
         quote="our systems will stop making that content publicly available in a reasonable "
               "amount of time",
         url=TOS, kind="quote"),
    dict(id="t26-licence-duration", doc=T26, clause="license duration", value="same wording",
         quote="This license lasts for as long as your content is protected by intellectual "
               "property rights.",
         url=TOS, kind="quote"),
    dict(id="t26-analysis", doc=T26, clause="automated analysis of your content",
         value="same wording",
         quote="This includes using automated systems and algorithms to analyze your content",
         url=TOS, kind="quote"),
    dict(id="t26-ai-output", doc=T26, clause="AI output used to develop machine learning models",
         value="same wording",
         quote="using AI-generated content from our services to develop machine learning models "
               "or related AI technology",
         url=TOS, kind="quote"),
    dict(id="t26-jailbreak", doc=T26, clause="jailbreaking and prompt injection",
         value="same wording",
         quote="jailbreaking, adversarial prompting, or prompt injection, except as part of our "
               "safety and bug testing programs",
         url=TOS, kind="quote"),
    dict(id="t26-scraping", doc=T26, clause="automated access against robots.txt",
         value="same wording",
         quote="using automated means to access content from any of our services in violation of "
               "the machine-readable instructions on our web pages",
         url=TOS, kind="quote"),
    dict(id="t26-network-heading", doc=T26, clause="network use while you are not engaged",
         value="new section: Cellular, Wi-Fi, and other Network Connection Costs",
         quote="Cellular, Wi-Fi, and other Network Connection Costs", url=TOS, kind="quote"),
    dict(id="t26-network-use", doc=T26, clause="network use: purposes",
         value="software updates, services, security, user experience and operations, "
               "including advertising",
         quote="Google services may frequently use the Internet when you're not actively engaged "
               "with the services or your device.",
         url=TOS, kind="quote"),
    dict(id="t26-network-ads", doc=T26, clause="network use: advertising named",
         value="yes",
         quote="to improve Google's services, security, user experience, and/or Google's "
               "operations, including advertising.",
         url=TOS, kind="quote"),
    dict(id="t26-network-costs", doc=T26, clause="network use: who pays", value="the user",
         quote="You're responsible for all of these network usages and associated costs.",
         url=TOS, kind="quote"),
    dict(id="t26-train-word", doc=T26, clause="sentences with the word train",
         value="1, the robots.txt example; none about your content",
         quote="train", expect=1, url=TOS, kind="count"),
    dict(id="t26-train-robots", doc=T26, clause="where the word train appears",
         value="robots.txt example in the scraping ban",
         quote="robots.txt files that disallow crawling, training, or other activities",
         url=TOS, kind="quote"),

    # --- the version it replaced, from Google's own archive ------------------
    dict(id="t24-effective", doc=T24, clause="effective date", value="22 May 2024",
         quote="Effective May 22, 2024", url=TOS_2024, kind="quote"),
    dict(id="t24-licence-scope", doc=T24, clause="content license scope",
         value="worldwide, non-exclusive, royalty-free",
         quote="non-exclusive, which means you can license your content to others",
         url=TOS_2024, kind="quote"),
    dict(id="t24-licence-develop", doc=T24, clause="license purpose: new technologies",
         value="developing new technologies and services for Google",
         quote="developing new technologies and services for Google consistent with these terms",
         url=TOS_2024, kind="quote"),
    dict(id="t24-licence-duration", doc=T24, clause="license duration",
         value="as long as the content is protected by intellectual property rights",
         quote="This license lasts for as long as your content is protected by intellectual "
               "property rights.",
         url=TOS_2024, kind="quote"),
    dict(id="t24-analysis", doc=T24, clause="automated analysis of your content",
         value="for spam, malware and illegal content, pattern recognition and customization",
         quote="This includes using automated systems and algorithms to analyze your content",
         url=TOS_2024, kind="quote"),
    dict(id="t24-ai-output", doc=T24, clause="AI output used to develop machine learning models",
         value="prohibited",
         quote="using AI-generated content from our services to develop machine learning models "
               "or related AI technology",
         url=TOS_2024, kind="quote"),
    dict(id="t24-jailbreak", doc=T24, clause="jailbreaking and prompt injection",
         value="prohibited, except in Google's safety and bug testing programs",
         quote="jailbreaking, adversarial prompting, or prompt injection, except as part of our "
               "safety and bug testing programs",
         url=TOS_2024, kind="quote"),
    dict(id="t24-scraping", doc=T24, clause="automated access against robots.txt",
         value="prohibited",
         quote="using automated means to access content from any of our services in violation of "
               "the machine-readable instructions on our web pages",
         url=TOS_2024, kind="quote"),
    dict(id="t24-no-network", doc=T24, clause="network use while you are not engaged",
         value="no such section",
         quote="Network Connection Costs", url=TOS_2024, kind="absent"),
    dict(id="t24-train-word", doc=T24, clause="sentences with the word train",
         value="1, the robots.txt example; none about your content",
         quote="train", expect=1, url=TOS_2024, kind="count"),

    # --- Google's comparison page -------------------------------------------
    dict(id="cmp-ins-count", doc=CMP, clause="passages marked as new text", value="11",
         quote='<ins aria-label="new text">', expect=11, url=TOS_CMP, kind="tags"),
    dict(id="cmp-del-count", doc=CMP, clause="passages marked as old text", value="5",
         quote='<del aria-label="old text">', expect=5, url=TOS_CMP, kind="tags"),
    dict(id="cmp-licence-clean", doc=CMP, clause="edits inside the content license section",
         value="0",
         quote=["This license allows Google to", "Using Google services"],
         url=TOS_CMP, kind="clean"),
    dict(id="cmp-ins-network", doc=CMP, clause="new: network connection costs section",
         value="added",
         quote="Google services may frequently use the Internet when you're not actively engaged",
         url=TOS_CMP, kind="ins"),
    dict(id="cmp-ins-suitable", doc=CMP, clause="new: sentence on generated content",
         value="added",
         quote="Some content may not be suitable for everyone.", url=TOS_CMP, kind="ins"),
    dict(id="cmp-ins-advice", doc=CMP, clause="new: professional advice clause qualifier",
         value="added",
         quote="Unless explicitly permitted by service-specific additional terms",
         url=TOS_CMP, kind="ins"),
    dict(id="cmp-ins-medical", doc=CMP, clause="new: medical wording in the advice clause",
         value="added",
         quote="does not constitute medical diagnosis or treatment", url=TOS_CMP, kind="ins"),
    dict(id="cmp-ins-liability", doc=CMP, clause="new: cross-reference in business-user liability",
         value="added",
         quote="Except as stated in the For all users section above", url=TOS_CMP, kind="ins"),
    dict(id="cmp-ins-law", doc=CMP, clause="new: governing law covers the terms themselves",
         value="added",
         quote="these terms and", url=TOS_CMP, kind="ins"),

    # --- Google Photos privacy notice ----------------------------------------
    dict(id="gpn-no-direct-training", doc=GPN, clause="training on your photos themselves",
         value="not directly, for generative models outside Google Photos",
         quote="Google does not train generative AI models outside of Google Photos directly on "
               "your imagery and audio from your Google Photos library",
         url=PHOTOS, kind="quote"),
    dict(id="gpn-derived-training", doc=GPN, clause="training on derived material",
         value="may train on summaries, inferences and generated media once Photos is connected "
               "to other Google services",
         quote="In connecting to other Google services, Google may train on things like summaries, "
               "inferences, and generated media that are based on the contents of your Photos "
               "library.",
         url=PHOTOS, kind="quote"),
    dict(id="gpn-other-policies", doc=GPN, clause="whose rules apply after connecting",
         value="the connected service's policies and controls",
         quote="that other service's policies and controls apply to processing of things like "
               "your media from Google Photos and related insights",
         url=PHOTOS, kind="quote"),
    dict(id="gpn-inferences", doc=GPN, clause="what an inference is",
         value="guesses about people, such as age and locations of face groups",
         quote="Inferences include things about people, like guessing the age and locations of "
               "your top face groups and inferring insights related to your life.",
         url=PHOTOS, kind="quote"),
    dict(id="gpn-ads", doc=GPN, clause="ads", value="never used for ads, unless you connect Photos "
         "to a service that supports ads",
         quote="Your personal data in Photos is never used for ads, unless you choose to make data "
               "from Google Photos available to third party or other Google services that "
               "support ads.",
         url=PHOTOS, kind="quote"),
    dict(id="gpn-query-review", doc=GPN, clause="human review of Ask Photos queries",
         value="on unless you turn it off",
         quote="Ask Photos queries may be reviewed by humans", url=PHOTOS, kind="quote"),
    dict(id="gpn-query-review-off", doc=GPN, clause="turning query review off",
         value="possible in settings",
         quote="You may turn off query review in settings.", url=PHOTOS, kind="quote"),

    # --- Apple -----------------------------------------------------------------
    dict(id="apple-ondevice", doc="Apple Privacy page", clause="where face and scene analysis runs",
         value="on device",
         quote="happen on device instead of in the cloud", url=APPLE_PRIVACY, kind="quote"),
    dict(id="apple-no-training", doc="Apple Privacy Policy", clause="training on private personal data",
         value="not used to train Apple's foundational models",
         quote="Your private personal data is not used to train our foundational AI models.",
         url=APPLE_LEGAL, kind="quote"),
    dict(id="apple-shared-albums", doc="Apple Support, Shared Albums",
         clause="what a shared album participant needs",
         value="from iOS 27, nothing: join and contribute on the web without an Apple Account "
               "or device",
         quote="Friends and family will be able to join and contribute their own photos on the "
               "web, even without an Apple Account or Apple device.",
         url=APPLE_ALBUMS, kind="quote"),
    dict(id="apple-shared-albums-date", doc="Apple Support, Shared Albums", clause="page date",
         value="14 September 2026",
         quote="Published Date: September 14, 2026", url=APPLE_ALBUMS, kind="quote"),

    # --- country versions, added 2026-11-07 ------------------------------------
    dict(id="us-effective", doc=T26_US, clause="effective date", value="30 July 2026",
         quote="Effective July 30, 2026", url=TOS_US, kind="quote"),
    dict(id="us-licence-rights", doc=T26_US, clause="rights granted",
         value="same wording as the Thailand version",
         quote="modify and create derivative works based on your content", url=TOS_US,
         kind="quote"),
    dict(id="us-cmp-ins-count", doc=CMP_US, clause="passages marked as new text", value="10",
         quote='<ins aria-label="new text">', expect=10, url=TOS_CMP_US, kind="tags"),
    dict(id="us-cmp-del-count", doc=CMP_US, clause="passages marked as old text", value="5",
         quote='<del aria-label="old text">', expect=5, url=TOS_CMP_US, kind="tags"),
    dict(id="us-cmp-licence-clean", doc=CMP_US, clause="edits inside the content license section",
         value="0", quote=["This license allows Google to", "Using Google services"],
         url=TOS_CMP_US, kind="clean"),
    dict(id="us-cmp-ins-network", doc=CMP_US, clause="new: network connection costs section",
         value="added",
         quote="Google services may frequently use the Internet when you're not actively engaged",
         url=TOS_CMP_US, kind="ins"),
    dict(id="us-cmp-ins-medical", doc=CMP_US, clause="new: medical wording in the advice clause",
         value="added, in capitals", quote="DOES NOT CONSTITUTE MEDICAL DIAGNOSIS OR TREATMENT",
         url=TOS_CMP_US, kind="ins"),
    dict(id="us-no-liability-xref", doc=T26_US,
         clause="cross-reference in business-user liability", value="not in this version",
         quote="Except as stated in the For all users section above", url=TOS_US, kind="absent"),
    dict(id="nl-effective", doc=T26_NL, clause="effective date", value="30 July 2026",
         quote="Effective July 30, 2026", url=TOS_NL, kind="quote"),
    dict(id="nl-cmp-ins-count", doc=CMP_NL, clause="passages marked as new text", value="13",
         quote='<ins aria-label="new text">', expect=13, url=TOS_CMP_NL, kind="tags"),
    dict(id="nl-cmp-del-count", doc=CMP_NL, clause="passages marked as old text", value="13",
         quote='<del aria-label="old text">', expect=13, url=TOS_CMP_NL, kind="tags"),
    dict(id="nl-cmp-licence-clean", doc=CMP_NL, clause="edits inside the content license section",
         value="0", quote=["This license allows Google to", "Using Google services"],
         url=TOS_CMP_NL, kind="clean"),
    dict(id="nl-cmp-ins-network", doc=CMP_NL, clause="new: network connection costs section",
         value="added",
         quote="Google services may frequently use the Internet when you're not actively engaged",
         url=TOS_CMP_NL, kind="ins"),
    dict(id="nl-cmp-ins-withdrawal", doc=CMP_NL, clause="new: online withdrawal form",
         value="added", quote="exercise your right of withdrawal online", url=TOS_CMP_NL,
         kind="ins"),
    dict(id="nl-licence-modify", doc=T26_NL, clause="rights granted: modify",
         value="modify only, for example reformatting or translating",
         quote="modify your content, such as reformatting or translating it", url=TOS_NL,
         kind="quote"),
    dict(id="nl-no-derivative", doc=T26_NL, clause="rights granted: derivative works",
         value="not in this version", quote="derivative works", url=TOS_NL, kind="absent"),
    dict(id="nl-licence-develop", doc=T26_NL, clause="license purpose: new technologies",
         value="same wording",
         quote="developing new technologies and services for Google consistent with these terms",
         url=TOS_NL, kind="quote"),
    dict(id="nl-no-generated-sentence", doc=T26_NL, clause="sentence on generated content",
         value="not in this version", quote="Some content may not be suitable for everyone",
         url=TOS_NL, kind="absent"),

    # --- Google Photos scale ---------------------------------------------------
    dict(id="gphotos-users", doc="Google blog, 10 years of Google Photos", clause="monthly users",
         value="more than 1.5 billion",
         quote="Today, more than 1.5 billion people use Google Photos each month",
         url=GPHOTOS_SCALE, kind="quote"),
    dict(id="gphotos-date", doc="Google blog, 10 years of Google Photos", clause="post date",
         value="28 May 2025",
         quote="May 28, 2025", url=GPHOTOS_SCALE, kind="quote"),
]

TYPO = {"‘": "'", "’": "'", "“": '"', "”": '"',
        "–": "-", "—": "-", " ": " ", "​": ""}


class TextExtractor(HTMLParser):
    SKIP = ("script", "style", "noscript")

    def __init__(self):
        super().__init__(convert_charrefs=True)
        self.parts = []
        self.depth = 0

    def handle_starttag(self, tag, attrs):
        if tag in self.SKIP:
            self.depth += 1

    def handle_endtag(self, tag):
        if tag in self.SKIP and self.depth:
            self.depth -= 1

    def handle_data(self, data):
        if not self.depth:
            self.parts.append(data)


def normalise(text):
    for bad, good in TYPO.items():
        text = text.replace(bad, good)
    return re.sub(r"\s+", " ", text).strip()


def to_text(raw):
    parser = TextExtractor()
    parser.feed(raw)
    return normalise(" ".join(parser.parts))


_cache = {}


def fetch(url, tries=3):
    if url in _cache:
        return _cache[url]
    last = None
    for attempt in range(tries):
        try:
            req = urllib.request.Request(url, headers={"User-Agent": UA,
                                                       "Accept-Language": "en-US,en;q=0.9"})
            with urllib.request.urlopen(req, timeout=60) as resp:
                raw = resp.read().decode("utf-8", "replace")
            text = to_text(raw)
            if len(text) < 500:
                raise ValueError("page text is only %d characters" % len(text))
            _cache[url] = (raw, text)
            return _cache[url]
        except Exception as exc:  # noqa: BLE001 - the reason is reported, not swallowed
            last = exc
            time.sleep(1 + attempt)
    raise RuntimeError("%s: %s" % (url, last))


def inserted_passages(raw):
    return [to_text(m.group(1))
            for m in re.finditer(r'<ins aria-label="new text">(.*?)</ins>', raw, re.S)]


def check_row(row):
    try:
        raw, text = fetch(row["url"])
    except Exception as exc:  # noqa: BLE001
        return False, "not fetched: %s" % exc
    kind = row["kind"]
    if kind == "quote":
        return (True, "quote found") if normalise(row["quote"]) in text else (False, "quote not found")
    if kind == "absent":
        if row["quote"].lower() in text.lower():
            return False, "phrase is now on the page"
        return True, "phrase absent"
    if kind == "count":
        n = len(re.findall(r"\b%s" % re.escape(row["quote"]), text, re.I))
        return n == row["expect"], "%d occurrence(s), expected %d" % (n, row["expect"])
    if kind == "tags":
        n = raw.count(row["quote"])
        return n == row["expect"], "%d element(s), expected %d" % (n, row["expect"])
    if kind == "clean":
        start, end = row["quote"]
        a = raw.find(start)
        b = raw.find(end, a + 1)
        if a < 0 or b < 0:
            return False, "section markers not found"
        edits = raw[a:b].count("<ins") + raw[a:b].count("<del")
        return edits == 0, "%d edit(s) in the section" % edits
    if kind == "ins":
        needle = normalise(row["quote"])
        if any(needle in passage for passage in inserted_passages(raw)):
            return True, "inside new text"
        return False, "not inside any new-text passage"
    return False, "unknown kind %s" % kind


def quote_text(row):
    return row["quote"] if isinstance(row["quote"], str) else " ... ".join(row["quote"])


def write_dataset(results):
    here = Path(__file__).resolve().parent
    inputs = []
    for row in ROWS:
        if row["url"] not in inputs:
            inputs.append(row["url"])
    dataset = {
        "slug": "google-tos-update-photos",
        "method": "primary-docs",
        "checkedAt": "2026-11-07",
        "question": "What does Google's new Terms of Service change for my photos?",
        "tools": ["Python %s (urllib, html.parser, re)" % sys.version.split()[0],
                  "curl 8.7.1 for the first read of each page",
                  "check.py in this folder"],
        "inputs": inputs,
        "results": [
            {
                "id": row["id"],
                "document": row["doc"],
                "clause": row["clause"],
                "value": row["value"],
                "quote": quote_text(row),
                "url": row["url"],
                "kind": row["kind"],
                "holds": ok,
                "note": note,
            }
            for row, (ok, note) in zip(ROWS, results)
        ],
        "notes": [
            "The baseline is Google's own archived copy of the terms effective 22 May 2024 and "
            "Google's own comparison page between that version and the one effective 30 July "
            "2026, both on policies.google.com. An earlier draft of this check read the 2024 "
            "terms from a Wayback Machine capture of 1 July 2026; it answered HTTP 429 to a "
            "plain client, so it was replaced by Google's archive.",
            "Google's comparison marks 11 passages as new text and 5 as old text. One new and "
            "one old passage are the effective date; two new passages are the network costs "
            "heading and paragraph; the rest are the sentence on generated content, the "
            "professional advice qualifier and medical wording, a liability cross-reference, two "
            "commas after 'service-specific additional terms', and two governing-law insertions "
            "('these terms and', 'any').",
            "The word 'train' appears once in each version, in 'robots.txt files that disallow "
            "crawling, training, or other activities'. Neither version has a sentence about "
            "training AI on your content.",
            "The training sentence about summaries, inferences and generated media is in the "
            "Gemini features in Photos privacy notice, which carries no effective date, so this "
            "check cannot say when it was added.",
            "The Terms of Service say automated analysis customizes services including ads; the "
            "Photos notice says personal data in Photos is never used for ads unless Photos is "
            "connected to a service that supports ads. Both sentences are recorded as written.",
            "Not checked: how the Google Photos app behaves in the background (backup timing, "
            "what leaves the device and when). That needs a device test and none was run.",
            "Not checked: encryption at rest for Google Photos. No Google page stating it was "
            "fetched, so the post makes no claim about it.",
            "policies.google.com serves a country version of the terms chosen by the reader's "
            "location. The baseline rows were first read on 2026-10-26 from a client in Thailand "
            "with Accept-Language en-US; on 2026-11-07 they were read again from a runner in the "
            "Netherlands with gl=TH in the URL and every one held unchanged. Without gl=TH the "
            "same runner received the Netherlands version and seven baseline rows failed, which "
            "is how the country difference was found.",
            "Country rows added 2026-11-07, each URL pinned with gl=US or gl=NL. The United States "
            "comparison marks 10 new and 5 old passages: the Thailand edits less the liability "
            "cross-reference for business users, which is not in the United States terms, with "
            "the professional advice and medical wording in capitals. The Netherlands comparison "
            "marks 13 new and 13 old passages, among them an online withdrawal form; its content "
            "license grants the right to modify content, such as reformatting or translating "
            "it, and does not mention derivative works. All three comparisons show no edit "
            "inside the content license section.",
        ],
    }
    with open(here / "dataset.json", "w", encoding="utf-8") as fh:
        json.dump(dataset, fh, indent=1, ensure_ascii=False)
        fh.write("\n")
    with open(here / "dataset.csv", "w", encoding="utf-8", newline="") as fh:
        writer = csv.writer(fh)
        writer.writerow(["id", "document", "clause", "value", "quote", "url", "holds"])
        for row in dataset["results"]:
            writer.writerow([row["id"], row["document"], row["clause"], row["value"],
                             row["quote"], row["url"], "yes" if row["holds"] else "no"])


def main():
    recheck = "--recheck" in sys.argv[1:]
    results = []
    failures = 0
    for row in ROWS:
        ok, note = check_row(row)
        results.append((ok, note))
        failures += 0 if ok else 1
        print("%-4s %-26s %s" % ("ok" if ok else "FAIL", row["id"], note))
    print("\n%d rows, %d failed" % (len(ROWS), failures))
    if not recheck:
        write_dataset(results)
        print("dataset.json and dataset.csv rewritten")
    return 1 if failures else 0


if __name__ == "__main__":
    sys.exit(main())
