#!/usr/bin/env python3 """Re-run the check behind dataset.json for iphone-photo-privacy-settings. Two legs. Every row whose source is "page" carries one exact sentence from one URL: the script fetches each URL, extracts its text and reports whether the sentence is still there. The rows whose source is "simulator" come from a newly created iPhone 17 Pro simulator on the iOS 27.0 runtime (Xcode's simctl, plutil and a read-only sqlite3 on TCC.db): the script creates one, reads the same values from disk, compares them with deviceTest.values and deletes the device. No physical phone and no Apple Account are involved. Exit 0 when every quote holds and every device value matches, 1 otherwise. A machine without xcrun or the iOS 27.0 runtime skips the simulator leg. --recheck re-reads the pages only and skips the simulator, the one step that takes longer than a minute. """ import json import re import sys import time import unicodedata import urllib.request from html.parser import HTMLParser from pathlib import Path HERE = Path(__file__).resolve().parent UA = 'Mozilla/5.0' TIMEOUT = 45 QUOTE_MAP = { '“': '"', '”': '"', '„': '"', '‘': "'", '’': "'", '‚': "'", '–': '-', '—': '-', '−': '-', ' ': ' ', ' ': ' ', ' ': ' ', } class TextExtractor(HTMLParser): SKIP = {'script', 'style', 'noscript', 'svg'} BREAK = {'p', 'div', 'li', 'h1', 'h2', 'h3', 'h4', 'tr', 'td', 'th', 'br', 'section', 'article'} def __init__(self): super().__init__(convert_charrefs=True) self.chunks = [] self.depth = 0 def handle_starttag(self, tag, attrs): if tag in self.SKIP: self.depth += 1 if tag in self.BREAK: self.chunks.append('\n') def handle_startendtag(self, tag, attrs): if tag in self.BREAK: self.chunks.append('\n') def handle_endtag(self, tag): if tag in self.SKIP and self.depth > 0: self.depth -= 1 if tag in self.BREAK: self.chunks.append('\n') def handle_data(self, data): if not self.depth: self.chunks.append(data) def normalise(text): for bad, good in QUOTE_MAP.items(): text = text.replace(bad, good) text = unicodedata.normalize('NFKC', text) text = re.sub(r'[ \t\r\f\v]+', ' ', text) text = re.sub(r'\s*\n\s*', ' ', text) return text.strip() def page_text(html): parser = TextExtractor() parser.feed(html) return normalise(''.join(parser.chunks)) def fetch(url, attempts=3): last = None for attempt in range(attempts): try: req = urllib.request.Request(url, headers={'User-Agent': UA}) with urllib.request.urlopen(req, timeout=TIMEOUT) as resp: raw = resp.read() return raw.decode('utf-8', 'replace') except Exception as exc: # a refused or flaky page is a result, not a crash last = exc time.sleep(1 + attempt) raise RuntimeError(f'{url}: {last}') def read_simulator(): """Create a fresh iOS 27 simulator, read the values deviceTest records, delete it. Returns (values, error). A machine without Xcode or the runtime is a skip, not a failure. """ import plistlib import shutil import sqlite3 import subprocess if not shutil.which('xcrun'): return None, 'no xcrun' runtimes = subprocess.run(['xcrun', 'simctl', 'list', 'runtimes'], capture_output=True, text=True).stdout runtime = next((line.split(' - ')[-1].strip() for line in runtimes.splitlines() if line.startswith('iOS 27.0')), None) if runtime is None: return None, 'no iOS 27.0 runtime installed' create = subprocess.run(['xcrun', 'simctl', 'create', 'evidence-privacy', 'iPhone 17 Pro', runtime], capture_output=True, text=True) udid = create.stdout.strip() if create.returncode != 0 or not udid: return None, f'create failed: {create.stderr.strip()[:80]}' def plist(path): if not path.exists(): return {} with open(path, 'rb') as fh: return plistlib.load(fh) try: subprocess.run(['xcrun', 'simctl', 'boot', udid], capture_output=True, text=True) subprocess.run(['xcrun', 'simctl', 'bootstatus', udid, '-b'], capture_output=True, text=True, timeout=240) data = Path.home() / 'Library/Developer/CoreSimulator/Devices' / udid / 'data' prefs = data / 'Library/Preferences' locationd = plist(prefs / 'com.apple.locationd.plist') if not locationd: return None, 'no locationd.plist after boot' db = sqlite3.connect(f'file:{data}/Library/TCC/TCC.db?mode=ro', uri=True) try: photos = db.execute( "select count(*) from access where service like '%Photos%'").fetchone()[0] total = db.execute('select count(*) from access').fetchone()[0] finally: db.close() bundles = data / 'Containers/Bundle/Application' apps = len(list(bundles.iterdir())) if bundles.exists() else 0 listapps = subprocess.run(['xcrun', 'simctl', 'listapps', udid], capture_output=True, text=True).stdout clients = plist(data / 'Library/Caches/locationd/clients.plist') def has_key(name, *needles): return any(n.lower() in k.lower() for k in plist(prefs / name) for n in needles) return { 'osVersionOnDevice': locationd.get('LastSystemVersion'), 'photoLibraryGrants': photos, 'allTccGrants': total, 'thirdPartyAppsInstalled': apps, 'cameraAppInstalled': '"com.apple.camera"' in listapps, 'cameraLocationClient': any('camera' in k.lower() for k in clients), 'airdropModeStored': has_key('com.apple.sharingd.plist', 'discoverable'), 'hiddenLockStored': has_key('com.apple.mobileslideshow.plist', 'lock', 'faceid', 'passcode', 'auth'), 'previewsStored': has_key('com.apple.bulletinboard.plist', 'preview'), }, None finally: subprocess.run(['xcrun', 'simctl', 'shutdown', udid], capture_output=True, text=True) subprocess.run(['xcrun', 'simctl', 'delete', udid], capture_output=True, text=True) def main(): recheck = '--recheck' in sys.argv[1:] dataset = json.loads((HERE / 'dataset.json').read_text(encoding='utf-8')) rows = [r for r in dataset['results'] if r.get('source', 'page') == 'page'] sim_rows = len(dataset['results']) - len(rows) print(f"check.py {'--recheck' if recheck else ''} · {len(rows)} page rows, " f"{sim_rows} simulator or not-checked rows · checkedAt {dataset['checkedAt']}") if recheck: print('lab steps longer than a minute: none; the simulator is skipped by --recheck') pages = {} failures = 0 for row in rows: url = row['url'] if url not in pages: try: pages[url] = page_text(fetch(url)) except Exception as exc: pages[url] = None print(f'PAGE {url} ({exc})') text = pages[url] if text is None: failures += 1 print(f'FAIL {row["id"]:<22} page unreadable') continue ok = normalise(row['quote']) in text if not ok: failures += 1 print(f'{"ok " if ok else "MISS"} {row["id"]:<22} {row["verdict"]:<14} ' f'{url.split("//")[1][:52]}') device = dataset.get('deviceTest') device_failures = 0 if device: if recheck: print('SIM simulator re-read skipped by --recheck') else: values, err = read_simulator() if err: print(f'SIM simulator skipped ({err}) - a skip, not a failure') else: for key, expected in device['values'].items(): got = values.get(key) if got != expected: device_failures += 1 print(f'MISS simulator.{key:<24} {got!r} != {expected!r}') else: print(f'ok simulator.{key:<24} {got!r}') print(f'\n{len(rows) - failures}/{len(rows)} quotes still on their page') return 0 if failures == 0 and device_failures == 0 else 1 if __name__ == '__main__': sys.exit(main())