#!/usr/bin/env python3 """Re-run the family-sharing check behind dataset.json. Every results row and every cited source is fetched again, reduced to text, and must still contain each of its quotes. Rows recorded as `not checked` are listed and skipped. The lab step rebuilds the two test photos from the NASA original with sips and exiftool and recounts how many fit in Gmail's 25 MB limit. python3 check.py # pages and lab python3 check.py --recheck # pages; the lab step is skipped if it runs past 60 s Exit 0 when every quote is still on its page and the counts match, 1 otherwise. Needs Python 3.8+; the lab step also needs macOS sips and exiftool. """ import csv import html import json import math import re import shutil import subprocess import sys import tempfile import time import urllib.error import urllib.request from pathlib import Path HERE = Path(__file__).resolve().parent UA = ('Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 ' '(KHTML, like Gecko) Chrome/126.0 Safari/537.36') LIMIT_BYTES = 25 * 1000 * 1000 LAB_LIMIT_SECONDS = 60 PUNCT = str.maketrans({'\u2019': "'", '\u2018': "'", '\u201c': '"', '\u201d': '"', '\u00a0': ' ', '\u2013': '-', '\u2014': '-'}) def normalise(text): text = html.unescape(text) text = re.sub(r'<(script|style)\b.*?', ' ', text, flags=re.S | re.I) text = re.sub(r'<[^>]+>', ' ', text) text = text.translate(PUNCT) text = re.sub(r'\s+', ' ', text) text = re.sub(r'\s+([,.;:!?)])', r'\1', text) return text.strip().lower() def page(url, cache): if url not in cache: req = urllib.request.Request(url, headers={'User-Agent': UA, 'Accept-Language': 'en-US,en;q=0.9'}) try: with urllib.request.urlopen(req, timeout=45) as resp: body = resp.read().decode(resp.headers.get_content_charset() or 'utf-8', 'replace') cache[url] = (resp.status, normalise(body)) except urllib.error.HTTPError as exc: cache[url] = (exc.code, '') except Exception as exc: cache[url] = (0, str(exc)) return cache[url] def check_quotes(url, quotes, cache): status, text = page(url, cache) if status != 200: return 'FAIL', 'HTTP %s from %s' % (status, url) missing = [q for q in quotes if normalise(q) not in text] if missing: return 'FAIL', '%d of %d quotes gone from %s (first: %r)' % ( len(missing), len(quotes), url, missing[0][:70]) return 'OK', 'found on %s' % url def run_lab(step, recheck): if not (shutil.which('sips') and shutil.which('exiftool')): return [('SKIP', 'lab', 'sips or exiftool not installed')] deadline = time.time() + LAB_LIMIT_SECONDS if recheck else None out = [] with tempfile.TemporaryDirectory() as tmp: tmp = Path(tmp) try: urllib.request.urlretrieve(step['source'], tmp / 'src.jpg') cmds = [ ['sips', '-c', '5120', '3840', 'src.jpg', '--out', 'crop.jpg'], ['sips', '-z', '4032', '3024', '-s', 'formatOptions', '90', 'crop.jpg', '--out', 'photo-12mp.jpg'], ['sips', '-z', '8064', '6048', '-s', 'format', 'heic', 'crop.jpg', '--out', 'photo-48mp.heic'], ] for name in ('photo-12mp.jpg', 'photo-48mp.heic'): cmds.append(['exiftool', '-q', '-overwrite_original', '-all=', name]) cmds.append(['exiftool', '-q', '-overwrite_original', '-Make=Apple', '-Model=iPhone 15 Pro', '-Software=18.0', '-LensModel=iPhone 15 Pro back triple camera 6.765mm f/1.78', '-DateTimeOriginal=2026:10:25 09:30:00', '-CreateDate=2026:10:25 09:30:00', '-OffsetTimeOriginal=-04:00', '-GPSLatitude=28.586', '-GPSLatitudeRef=N', '-GPSLongitude=80.651', '-GPSLongitudeRef=W', '-GPSAltitude=3', '-GPSAltitudeRef=0', '-ExposureTime=1/120', '-FNumber=1.78', '-ISO=80', '-Orientation#=1', name]) for cmd in cmds: timeout = None if deadline is None else max(1, deadline - time.time()) subprocess.run(cmd, cwd=tmp, check=True, capture_output=True, timeout=timeout) except subprocess.TimeoutExpired: return [('SKIP', 'lab', 'over %d s, skipped under --recheck' % LAB_LIMIT_SECONDS)] except Exception as exc: return [('FAIL', 'lab', 'rebuild failed: %s' % exc)] for f in step['files']: size = (tmp / f['name']).stat().st_size count = math.floor(LIMIT_BYTES / size) status = 'OK' if count == f['per_25mb'] else 'FAIL' out.append((status, f['name'], '%d bytes -> %d per 25 MB (dataset: %d bytes, %d)' % ( size, count, f['bytes'], f['per_25mb']))) return out def main(): recheck = '--recheck' in sys.argv data = json.loads((HERE / 'dataset.json').read_text(encoding='utf-8')) cache, counts = {}, {'OK': 0, 'SKIP': 0, 'FAIL': 0} def report(status, label, detail): counts[status] += 1 print(' %-4s %-48s %s' % (status, label, detail)) print('family-sharing, checked %s%s' % (data['checkedAt'], ' (recheck)' if recheck else '')) for row in data['results']: label = '%s / %s' % (row['option'], row['property']) if not row['quotes']: report('SKIP', label, 'not checked: %s' % row.get('note', '')) else: status, detail = check_quotes(row['url'], row['quotes'], cache) report(status, label, detail) print('\nsources the post cites') for src in data['sources']: status, detail = check_quotes(src['url'], src['quotes'], cache) report(status, src['id'], detail) print('\nlab') for step in data['lab']: for status, label, detail in run_lab(step, recheck): report(status, label, detail) print('\ncsv') lines = (HERE / 'dataset.csv').read_text(encoding='utf-8').splitlines() rows = list(csv.DictReader(lines)) drift = 0 for row in data['results']: match = [r for r in rows if r['option'] == row['option']] if not match or match[0][row['property']] != row['value']: drift += 1 report('FAIL', '%s / %s' % (row['option'], row['property']), 'csv differs from dataset.json') if not drift: report('OK', 'dataset.csv', 'mirrors dataset.json (%d options)' % len(rows)) print('\n%d ok, %d not checked or skipped, %d failed' % (counts['OK'], counts['SKIP'], counts['FAIL'])) return 1 if counts['FAIL'] else 0 if __name__ == '__main__': sys.exit(main())