#!/usr/bin/env python3
"""Re-runs the check behind the post downloaded-photo-location-metadata.

lab: builds one 2400x1800 JPEG tagged with a camera make, a model, a serial number and a
     coordinate, derives the seven files the post's table reports, and measures for each
     whether an EXIF block is there, whether the make, model and serial number can be read
     inside it, and whether a coordinate is there under the post's strict rule (a GPS tag
     whose values are not all zero).

primary-docs: re-reads the pages cited for the post's statements about Android, MediaWiki,
     Wikimedia Commons and Mastodon, and reports whether each supporting quote is still on
     its page. Quotes are compared after whitespace, quote-mark and entity normalisation.

usage:
    check.py            verify the tree against dataset.json, exit 0 or 1
    check.py --emit     rebuild dataset.json and dataset.csv from what was measured
    check.py --recheck   verify again, skipping any lab step that has run over 60 s and
                        reporting it rather than failing on it

Needs exiftool and Pillow; the HEIC row additionally needs macOS sips, and is reported as
not checked where sips is absent. Standard library otherwise.
"""

import argparse
import csv
import html
import json
import re
import shutil
import subprocess
import sys
import tempfile
import time
import urllib.request
from pathlib import Path

HERE = Path(__file__).resolve().parent

# -I keeps the current directory off sys.path, which also drops the user site-packages
# Pillow lives in. Add that directory back only if it exists.
USER_SITE = Path.home() / 'Library/Python/3.9/lib/python/site-packages'
if USER_SITE.is_dir() and str(USER_SITE) not in sys.path:
    sys.path.append(str(USER_SITE))
from PIL import Image, ImageDraw  # noqa: E402

CHECKED_AT = '2026-10-26'
SERIAL = 'F2LX90ABCDEF'
BASE = (2400, 1800)
LONG_EDGE = 1600
THUMB_EDGE = 400
USER_AGENT = (
    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 '
    '(KHTML, like Gecko) Chrome/126 Safari/537.36'
)

ANDROID_MEDIA = 'https://developer.android.com/training/data-storage/shared/media'
MEDIASTORE = 'https://developer.android.com/reference/android/provider/MediaStore'
PERMISSION = 'https://developer.android.com/reference/android/Manifest.permission'
IMAGEINFO = 'https://www.mediawiki.org/wiki/API:Imageinfo'
COMMONS_EXIF = 'https://commons.wikimedia.org/wiki/Commons:EXIF'
MASTODON_CONFIG = 'https://docs.joinmastodon.org/admin/config/'

# One row per delivery path. id is the key in dataset.json.
LAB_PATHS = [
    ('stored-original', 'Uploaded file stored unchanged, offered for download'),
    ('display-noflag', 'Resized to 1600 px, metadata not carried forward'),
    ('display-carry', 'Same resize, metadata carried forward'),
    ('thumb-400', '400 px WebP thumbnail'),
    ('heic-sips-jpeg', 'HEIC converted to JPEG by macOS sips'),
    ('block-no-gps', 'Same original with the GPS block removed'),
    ('zero-gps', 'Same original with the GPS values all zero'),
]

# statement id, the sentence the post makes, the page, the exact sentence that supports
# it there. A cut row has no quote and is recorded, not checked.
DOC_ROWS = [
    (
        'android-hides-location',
        'On Android 10 and later, an app that reads photos through the shared media '
        'collection gets a redacted copy by default.',
        ANDROID_MEDIA,
        'If your app uses scoped storage, the system hides location information by default.',
    ),
    (
        'android-access-media-location',
        'The location is returned only if the app declares the ACCESS_MEDIA_LOCATION '
        'permission.',
        ANDROID_MEDIA,
        'If your app targets Android 10 (API level 29) or higher and needs to retrieve '
        'unredacted EXIF metadata from photos, you need to declare the '
        'ACCESS_MEDIA_LOCATION permission in your app\'s manifest, then request this '
        'permission at runtime.',
    ),
    (
        'android-require-original',
        'The app also has to ask for the original bytes through '
        'MediaStore.setRequireOriginal.',
        MEDIASTORE,
        'This can be useful when the caller wants to ensure they\'re backing up the exact '
        'bytes of the underlying media, without any Exif redaction being performed.',
    ),
    (
        'android-permission-definition',
        'ACCESS_MEDIA_LOCATION governs access to the location stored in the shared '
        'collection.',
        PERMISSION,
        'Allows an application to access any geographic locations persisted in the user\'s '
        'shared collection.',
    ),
    (
        'android-file-untouched',
        'The redaction is applied to the copy the app receives, and the method returns an '
        'EXIF redacted Uri.',
        MEDIASTORE,
        'Returns an EXIF redacted version of uri i.e. a Uri with metadata such as location, '
        'GPS datestamp etc. redacted from the EXIF headers.',
    ),
    (
        'imageinfo-iiurlwidth',
        'MediaWiki documents iiurlwidth, the parameter the study used to pick the display '
        'copy.',
        IMAGEINFO,
        'If iiprop=url is set, a URL to an image scaled to this width will be returned.',
    ),
    (
        'imageinfo-width-not-guaranteed',
        'The API records that the returned thumbnail URL is not guaranteed to match the '
        'requested width.',
        IMAGEINFO,
        'Since T360589, it is no longer guaranteed that the thumbnail URL returned by the '
        'API matches the requested thumbnail width (iiurlwidth).',
    ),
    (
        't360589-pregenerated',
        'Wikimedia serves a pregenerated thumbnail of the same width or larger (T360589).',
        IMAGEINFO,
        'The thumbnail, which will be fetched from a pregenerated list, is either of the '
        'same width or larger.',
    ),
    (
        'commons-exif-gps',
        'Commons says a photograph carries location coordinates if the device has a GPS '
        'receiver.',
        COMMONS_EXIF,
        'Photos taken with a digital camera are likely to be JPEG images with embedded Exif '
        'data, with automatically recorded date and time the photo was taken, exposure '
        'settings, focal length, and so on, and location coordinates if the device has a '
        'GPS.',
    ),
    (
        'commons-exif-inside-file',
        'Commons says that metadata travels inside the file.',
        COMMONS_EXIF,
        'Exif information helps us by keeping all kinds of meta information directly within '
        'the file',
    ),
    (
        'mastodon-image-library',
        'The Mastodon configuration reference names the image library used to process '
        'uploads.',
        MASTODON_CONFIG,
        'By default, Mastodon uses ImageMagick to process images in posts.',
    ),
    (
        't360589-task',
        'The Wikimedia task T360589 is "De-fragment thumbnail sizes in mediawiki".',
        'https://phabricator.wikimedia.org/T360589',
        'T360589 De-fragment thumbnail sizes in mediawiki',
    ),
    (
        'heic-converter-strips-exif',
        'Viallo\'s HEIC converter strips EXIF, so the upload path reads the camera fields '
        'before converting.',
        'file:apps/api/src/uploads/uploads.service.ts',
        'Extract EXIF from original HEIC before conversion (heic-convert strips EXIF)',
    ),
]


def run(cmd, **kw):
    return subprocess.run(cmd, check=True, capture_output=True, text=True, **kw)


def exif_tags(path):
    out = run(['exiftool', '-q', '-j', '-EXIF:all', '-GPS:all',
               '-File:ImageWidth', '-File:ImageHeight', str(path)]).stdout
    data = json.loads(out)
    return data[0] if data else {}


def numbers(value):
    if isinstance(value, (int, float)):
        return [float(value)]
    return [float(x) for x in re.findall(r'-?\d+(?:\.\d+)?', str(value))]


def coordinate_present(tags, strict):
    if 'GPSLatitude' not in tags or 'GPSLongitude' not in tags:
        return False
    if not strict:
        return True
    lat, lon = numbers(tags['GPSLatitude']), numbers(tags['GPSLongitude'])
    return bool(lat and lon) and any(abs(x) > 0 for x in lat) and any(abs(x) > 0 for x in lon)


def build_base(path):
    """A 2400x1800 JPEG carrying a make, a model, a serial number and a coordinate."""
    image = Image.new('RGB', BASE)
    draw = ImageDraw.Draw(image)
    for y in range(0, BASE[1], 4):
        draw.line([(0, y), (BASE[0], y)], fill=(y % 256, (y * 3) % 256, (y * 7) % 256))
    image.save(path, quality=90)
    run(['exiftool', '-q', '-overwrite_original', '-all=', str(path)])
    run(['exiftool', '-q', '-overwrite_original',
         '-Make=Apple', '-Model=iPhone 15 Pro', f'-SerialNumber={SERIAL}',
         '-GPSLatitude=28.586', '-GPSLatitudeRef=N',
         '-GPSLongitude=80.651', '-GPSLongitudeRef=W', str(path)])


def build_lab(tmp):
    """Rebuild every input. Returns {id: path}, missing an id sips cannot produce."""
    stored = tmp / 'stored.jpg'
    build_base(stored)
    paths = {'stored-original': stored}

    im = Image.open(stored)
    im.thumbnail((LONG_EDGE, LONG_EDGE))
    im.save(tmp / 'display-noflag.jpg', quality=82)
    paths['display-noflag'] = tmp / 'display-noflag.jpg'

    im = Image.open(stored)
    exif = im.getexif()
    im.thumbnail((LONG_EDGE, LONG_EDGE))
    im.save(tmp / 'display-carry.jpg', quality=82, exif=exif)
    paths['display-carry'] = tmp / 'display-carry.jpg'

    im = Image.open(stored)
    im.thumbnail((THUMB_EDGE, THUMB_EDGE))
    im.save(tmp / 'thumb-400.webp', quality=85)
    paths['thumb-400'] = tmp / 'thumb-400.webp'

    heic = tmp / 'base.heic'
    back = tmp / 'heic-sips.jpg'
    if shutil.which('sips'):
        run(['sips', '-s', 'format', 'heic', str(stored), '--out', str(heic)])
        run(['exiftool', '-q', '-overwrite_original', '-all=', str(heic)])
        run(['exiftool', '-q', '-overwrite_original',
             '-Make=Apple', '-Model=iPhone 15 Pro', f'-SerialNumber={SERIAL}',
             '-GPSLatitude=28.586', '-GPSLatitudeRef=N',
             '-GPSLongitude=80.651', '-GPSLongitudeRef=W', str(heic)])
        run(['sips', '-s', 'format', 'jpeg', str(heic), '--out', str(back)])
        paths['heic-sips-jpeg'] = back

    no_gps = tmp / 'block-no-gps.jpg'
    shutil.copy2(stored, no_gps)
    run(['exiftool', '-q', '-overwrite_original', '-GPS:all=', str(no_gps)])
    paths['block-no-gps'] = no_gps

    zero = tmp / 'zero-gps.jpg'
    shutil.copy2(stored, zero)
    run(['exiftool', '-q', '-overwrite_original',
         '-GPSLatitude=0', '-GPSLongitude=0', str(zero)])
    paths['zero-gps'] = zero

    return paths


def measure_lab(tmp):
    rows, skipped = [], []
    paths = build_lab(tmp)
    for row_id, label in LAB_PATHS:
        path = paths.get(row_id)
        if path is None:
            skipped.append(row_id)
            continue
        tags = exif_tags(path)
        block = any(k not in ('SourceFile', 'ImageWidth', 'ImageHeight') for k in tags)
        width, height = tags.get('ImageWidth'), tags.get('ImageHeight')
        if not width or not height:
            width, height = Image.open(path).size  # WebP carries no EXIF dimensions
        rows.append({
            'id': row_id,
            'path': label,
            'exif_block': 'yes' if block else 'no',
            'camera_make': tags.get('Make') or 'no',
            'camera_model': tags.get('Model') or 'no',
            'serial_number': tags.get('SerialNumber') or 'no',
            'coordinate_strict': 'yes' if coordinate_present(tags, True) else 'no',
            'coordinate_naive': 'yes' if coordinate_present(tags, False) else 'no',
            'width': width,
            'height': height,
            'input': path.name,
            'url': None,
            # The tool output this row rests on, so no cell is a typed value.
            'quote': ', '.join(f'{k}={tags[k]}' for k in
                               ('Make', 'Model', 'SerialNumber', 'GPSLatitude', 'GPSLongitude')
                               if k in tags) or 'exiftool returned no EXIF tags',
        })
    return rows, skipped


def fetch(url):
    req = urllib.request.Request(url, headers={'User-Agent': USER_AGENT})
    with urllib.request.urlopen(req, timeout=60) as resp:
        return resp.read().decode('utf-8', 'replace')


def normalise(text):
    text = re.sub(r'(?is)<(script|style|svg)[^>]*>.*?</\1>', ' ', text)
    text = re.sub(r'(?s)<[^>]+>', ' ', text)
    text = html.unescape(text)
    for a, b in (('’', "'"), ('‘', "'"), ('“', '"'), ('”', '"'),
                 (' ', ' '), ('‑', '-'), ('–', '-'), ('—', '-')):
        text = text.replace(a, b)
    text = re.sub(r'\s+', ' ', text)
    # Stripping a tag leaves a space before the punctuation it wrapped.
    text = re.sub(r'\s+([,.;:)\]])', r'\1', text)
    text = re.sub(r'([(\[])\s+', r'\1', text)
    return text.strip()


def check_docs():
    pages, rows = {}, []
    for row_id, statement, url, quote in DOC_ROWS:
        row = {'id': row_id, 'statement': statement, 'url': url, 'quote': quote}
        if url.startswith('file:'):
            # A statement about the product's own code, read from the repository.
            path = HERE.parents[4] / url[len('file:'):]
            found = quote in path.read_text('utf-8')
            row['found'] = found
            row['source'] = 'repository'
            rows.append(row)
            continue
        if url not in pages:
            try:
                pages[url] = normalise(fetch(url))
                row['fetch'] = 'ok'
            except Exception as exc:  # a refused page is recorded, never filled in
                pages[url] = None
                row['fetch'] = f'error: {exc}'
        page = pages[url]
        row['found'] = None if page is None else (normalise(quote) in page)
        rows.append(row)
    return rows


def load_dataset():
    path = HERE / 'dataset.json'
    return json.loads(path.read_text('utf-8')) if path.exists() else None


def main(argv=None):
    ap = argparse.ArgumentParser()
    ap.add_argument('--emit', action='store_true')
    ap.add_argument('--recheck', action='store_true')
    args = ap.parse_args(argv)

    started = time.time()
    failed = 0
    tmpdir = Path(tempfile.mkdtemp(prefix='dplm-lab-'))
    try:
        lab_rows, skipped = measure_lab(tmpdir)
        if args.recheck and time.time() - started > 60:
            print('lab skipped: over the recheck budget')
            lab_rows = (load_dataset() or {}).get('results', {}).get('lab', [])
            skipped = []
    finally:
        shutil.rmtree(tmpdir, ignore_errors=True)
    for row_id in skipped:
        print(f'not checked  {row_id} (sips not available)')

    doc_rows = check_docs()

    dataset = load_dataset()
    if args.emit or dataset is None:
        write_dataset(lab_rows, doc_rows)
        print(f'wrote dataset.json and dataset.csv ({len(lab_rows)} lab rows, '
              f'{len(doc_rows)} statements)')
        return 0 if all(r['found'] is not False for r in doc_rows) else 1

    expected_lab = {r['id']: r for r in dataset['results']['lab']}
    for row in lab_rows:
        want = expected_lab.get(row['id'])
        if want is None:
            print(f'FAIL {row["id"]}: not in dataset.json')
            failed += 1
            continue
        diffs = [k for k in row if k != 'input' and want.get(k) != row[k]]
        if diffs:
            failed += 1
            print(f'FAIL {row["id"]}: ' + ', '.join(
                f'{k} measured {row[k]!r} dataset {want.get(k)!r}' for k in diffs))
        else:
            print(f'ok   {row["id"]}')

    expected_doc = {r['id']: r for r in dataset['results']['statements']}
    for row in doc_rows:
        want = expected_doc.get(row['id'])
        if want is None:
            print(f'FAIL {row["id"]}: not in dataset.json')
            failed += 1
            continue
        if row['found'] is None:
            print(f'FAIL {row["id"]}: page not readable ({row.get("fetch")})')
            failed += 1
        elif not row['found']:
            print(f'FAIL {row["id"]}: quote no longer on {row["url"]}')
            failed += 1
        else:
            print(f'ok   {row["id"]}')

    print(f'\n{len(lab_rows) + len(doc_rows)} checks, {failed} failed, '
          f'{time.time() - started:.1f}s')
    return 0 if failed == 0 else 1


def write_dataset(lab_rows, doc_rows):
    tools = {
        'exiftool': run(['exiftool', '-ver']).stdout.strip(),
        'sips': run(['sips', '--version']).stdout.strip().splitlines()[0],
        'python': sys.version.split()[0],
    }
    try:
        import PIL
        tools['Pillow'] = PIL.__version__
    except Exception:
        pass
    dataset = {
        'slug': 'downloaded-photo-location-metadata',
        'method': 'lab + primary-docs',
        'checkedAt': CHECKED_AT,
        'question': 'Do downloaded photos still have location data in them?',
        'tools': tools,
        'inputs': {
            'lab': [
                'one 2400x1800 JPEG generated by check.py and tagged with exiftool '
                f'(Make=Apple, Model=iPhone 15 Pro, SerialNumber={SERIAL}, a GPS '
                'coordinate at 28.586 N, 80.651 W), then derived in a temporary '
                'directory into: stored.jpg, display-noflag.jpg, display-carry.jpg, '
                'thumb-400.webp, base.heic, heic-sips.jpg, block-no-gps.jpg, zero-gps.jpg',
            ],
            'pages': sorted({r['url'] for r in doc_rows if not r['url'].startswith('file:')}),
            'files': sorted(r['url'] for r in doc_rows if r['url'].startswith('file:')),
        },
        'results': {
            'lab': lab_rows,
            'statements': doc_rows,
        },
        'notes': [
            'The lab measures files the script makes. It says nothing about Wikimedia '
            'Commons, Mastodon or any published photo; those samples are the 10 September '
            '2026 study, whose published data this post links, and were not fetched again.',
            'display-carry is the control for display-noflag: the same resize and quality, '
            'only the metadata carried forward, so the difference is the flag and not the '
            'smaller size.',
            'zero-gps is the control for the strict coordinate rule: the GPS tags are '
            'present with zero values, so coordinate_strict is no while coordinate_naive '
            'is yes.',
            'heic-sips-jpeg is macOS sips, not Viallo\'s converter (heic-convert). It '
            'shows that a HEIC to JPEG conversion does not lose the fields by itself; '
            'Viallo\'s converter does, which the code records and the post states.',
            'The device UI steps an earlier draft gave (Google Photos, iPhone, Windows, '
            'Mac) were removed: no vendor page fetched on the checked date states them.',
        ],
    }
    (HERE / 'dataset.json').write_text(json.dumps(dataset, indent=1, ensure_ascii=False) + '\n',
                                       'utf-8')

    columns = ['kind', 'id', 'path', 'exif_block', 'camera_make', 'camera_model',
               'serial_number', 'coordinate_strict', 'coordinate_naive', 'width', 'height',
               'input', 'statement', 'verdict', 'quote', 'url']
    with (HERE / 'dataset.csv').open('w', newline='', encoding='utf-8') as fh:
        w = csv.DictWriter(fh, fieldnames=columns)
        w.writeheader()
        for row in lab_rows:
            w.writerow({'kind': 'lab', **{k: row.get(k) for k in columns if k in row}})
        for row in doc_rows:
            w.writerow({'kind': 'doc', 'id': row['id'], 'statement': row['statement'],
                        'quote': row['quote'] or '', 'url': row['url'],
                        'verdict': 'supported' if row.get('found') else 'not found'})


if __name__ == '__main__':
    sys.exit(main())