#!/usr/bin/env python3
"""Re-run the check behind /blog/ai-training, 26 October 2026.

Two parts, both reproduced from nothing:

  primary-docs  every URL in dataset.json is fetched again and every recorded quote must
                still appear on it. A row may also record strings that must be absent
                (a phrase the page did not carry on an earlier date) and a segment of the
                page inside which a list of names must not appear.
  lab           the test photos are rebuilt from two NASA public-domain images with sips
                and exiftool, put through Viallo's upload pipeline as it stands in this
                repository, and every measured value compared with dataset.json.

    python3 check.py             # fetch, rebuild and measure
    python3 check.py --recheck   # the same; no step here takes over a minute

Exit 0 when every quote is still on its page and every measurement still matches, 1 otherwise.
Needs macOS (sips), exiftool, node and network access for the first run's npm install.
"""

import gzip
import hashlib
import html
import json
import os
import re
import shutil
import subprocess
import sys
import tempfile
import urllib.request
import zlib
from html.parser import HTMLParser

HERE = os.path.dirname(os.path.abspath(__file__))
DATASET = os.path.join(HERE, 'dataset.json')
REPO = os.path.abspath(os.path.join(HERE, '..', '..', '..', '..', '..'))
CACHE = os.path.join(tempfile.gettempdir(), 'viallo-ai-training-check')
USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126 Safari/537.36'
WAYBACK_REFERER = 'https://archive.org/'
TIMEOUT = 90

QUOTE_MAP = {
    '‘': "'", '’': "'", '‚': "'", '‛': "'", '′': "'",
    '“': '"', '”': '"', '„': '"', '‟': '"', '″': '"',
    '‐': '-', '‑': '-', '‒': '-', '–': '-', '—': '-', '―': '-', '−': '-',
}


def normalise(text):
    text = html.unescape(text)
    for src, dst in QUOTE_MAP.items():
        text = text.replace(src, dst)
    text = re.sub(r'<[^>]+>', ' ', text)
    text = html.unescape(text)
    text = re.sub(r'[​‌‍]', '', text)
    text = re.sub(r'\s+', ' ', text)
    return text.strip().lower()


class TextExtractor(HTMLParser):
    def __init__(self):
        HTMLParser.__init__(self, convert_charrefs=True)
        self.parts = []
        self.skip = 0

    def handle_starttag(self, tag, attrs):
        if tag in ('script', 'style'):
            self.skip += 1

    def handle_endtag(self, tag):
        if tag in ('script', 'style') and self.skip:
            self.skip -= 1

    def handle_data(self, data):
        if not self.skip:
            self.parts.append(data)


def page_text(url):
    headers = {'User-Agent': USER_AGENT, 'Accept-Encoding': 'identity'}
    if 'web.archive.org' in url:
        headers['Referer'] = WAYBACK_REFERER
    req = urllib.request.Request(url, headers=headers)
    with urllib.request.urlopen(req, timeout=TIMEOUT) as resp:
        raw = resp.read()
        encoding = (resp.headers.get('Content-Encoding') or '').lower()
        charset = resp.headers.get_content_charset() or 'utf-8'
    if encoding == 'gzip':
        raw = gzip.decompress(raw)
    elif encoding == 'deflate':
        raw = zlib.decompress(raw, -zlib.MAX_WBITS)
    body = raw.decode(charset, 'replace')
    parser = TextExtractor()
    parser.feed(body)
    return re.sub(r'\s+', ' ', ' '.join(parser.parts))


def source_text(url):
    if url.startswith('code:'):
        with open(os.path.join(REPO, url[len('code:'):]), encoding='utf-8') as fh:
            return fh.read()
    return page_text(url)


def check_quotes(row, text, failures):
    label = '%s / %s' % (row['option'], row['property'])
    norm = normalise(text)
    for q in row.get('quotes', []):
        if normalise(q) not in norm:
            failures.append('%s: quote gone from %s\n    %r' % (label, row['url'], q))
    for q in row.get('absent', []):
        if normalise(q) in norm:
            failures.append('%s: %r is back on %s' % (label, q, row['url']))
    segment = row.get('segment')
    if segment:
        start = norm.find(normalise(segment['after']))
        end = norm.find(normalise(segment['before']), start + 1) if start >= 0 else -1
        if start < 0 or end < 0:
            failures.append('%s: the list segment %r .. %r is not on %s'
                            % (label, segment['after'], segment['before'], row['url']))
        else:
            inside = norm[start:end]
            for name in segment['absent']:
                # Word boundaries: Austria must not match Australia, Nigeria must not match Niger.
                if re.search(r'\b%s\b' % re.escape(normalise(name)), inside):
                    failures.append('%s: %r now appears in the list on %s'
                                    % (label, name, row['url']))


NODE_SCRIPT = r'''
import { createHash } from 'node:crypto';
import { readFile, writeFile } from 'node:fs/promises';
import sharp from 'sharp';
import { encode } from 'blurhash';
import convert from 'heic-convert';

const DISPLAY_JPEG_QUALITY = 82;
const PHOTO_DISPLAY_LONG_EDGE = 2048;
const USER_IMAGE_SHARP_OPTIONS = { failOn: 'none' };
const HEIC_JPEG_QUALITY = 0.92;

async function metadata(input) {
  const m = await sharp(input, USER_IMAGE_SHARP_OPTIONS).metadata();
  return { width: m.width, height: m.height, format: m.format };
}

async function generateThumbnail(input) {
  const { data, info } = await sharp(input, USER_IMAGE_SHARP_OPTIONS)
    .rotate()
    .resize(400, 400, { fit: 'inside', withoutEnlargement: true })
    .webp({ quality: 85 })
    .toBuffer({ resolveWithObject: true });
  return { buffer: data, width: info.width, height: info.height, format: info.format, size: data.length };
}

async function blurhashFor(thumbnail) {
  const { data, info } = await sharp(thumbnail.buffer)
    .resize(32, 32, { fit: 'inside' })
    .ensureAlpha()
    .raw()
    .toBuffer({ resolveWithObject: true });
  return encode(new Uint8ClampedArray(data), info.width, info.height, 4, 3);
}

async function generateDisplayRendition(input) {
  const { data, info } = await sharp(input, USER_IMAGE_SHARP_OPTIONS)
    .rotate()
    .resize(PHOTO_DISPLAY_LONG_EDGE, PHOTO_DISPLAY_LONG_EDGE, { fit: 'inside', withoutEnlargement: true })
    .jpeg({ quality: DISPLAY_JPEG_QUALITY })
    .toBuffer({ resolveWithObject: true });
  return { buffer: data, width: info.width, height: info.height, format: info.format, size: data.length };
}

const SIDE = 32;
const LOW = 8;
const COS = (() => {
  const table = new Float64Array(LOW * SIDE);
  for (let k = 0; k < LOW; k++) {
    for (let x = 0; x < SIDE; x++) {
      table[k * SIDE + x] = Math.cos(((2 * x + 1) * k * Math.PI) / (2 * SIDE));
    }
  }
  return table;
})();

async function perceptualHash(display) {
  const px = await sharp(display).greyscale().resize(SIDE, SIDE, { fit: 'fill' }).raw().toBuffer();
  const rows = new Float64Array(SIDE * LOW);
  for (let y = 0; y < SIDE; y++) {
    for (let u = 0; u < LOW; u++) {
      let sum = 0;
      for (let x = 0; x < SIDE; x++) sum += px[y * SIDE + x] * COS[u * SIDE + x];
      rows[y * LOW + u] = sum;
    }
  }
  const coefficients = new Float64Array(LOW * LOW);
  for (let v = 0; v < LOW; v++) {
    for (let u = 0; u < LOW; u++) {
      let sum = 0;
      for (let y = 0; y < SIDE; y++) sum += rows[y * LOW + u] * COS[v * SIDE + y];
      coefficients[v * LOW + u] = sum;
    }
  }
  const ac = Array.from(coefficients.subarray(1)).sort((a, b) => a - b);
  const median = (ac[31] + ac[32]) / 2;
  const bytes = Buffer.alloc(LOW);
  coefficients.forEach((c, i) => {
    if (c > median) bytes[i >> 3] |= 1 << (7 - (i & 7));
  });
  return bytes.toString('hex');
}

function hammingDistance(a, b) {
  let distance = 0;
  for (let i = 0; i < a.length; i += 8) {
    let x = parseInt(a.slice(i, i + 8), 16) ^ parseInt(b.slice(i, i + 8), 16);
    while (x) {
      x &= x - 1;
      distance++;
    }
  }
  return distance;
}

async function pipeline(path) {
  const input = await readFile(path);
  const originalMetadata = await metadata(input);
  const thumbnail = await generateThumbnail(input);
  const blurhash = await blurhashFor(thumbnail);
  const display = await generateDisplayRendition(input);
  const hash = await perceptualHash(display.buffer);
  return {
    bytes: input.length,
    sha256: createHash('sha256').update(input).digest('hex'),
    width: originalMetadata.width,
    height: originalMetadata.height,
    format: originalMetadata.format,
    thumbnail: { width: thumbnail.width, height: thumbnail.height, format: thumbnail.format, bytes: thumbnail.size },
    display: { width: display.width, height: display.height, format: display.format, bytes: display.size },
    blurhash: { value: blurhash, length: blurhash.length },
    hash,
  };
}

const config = JSON.parse(await readFile(process.argv[2], 'utf8'));
const out = {};
for (const [name, p] of Object.entries(config)) {
  if (name === 'heic') continue;
  out[name] = await pipeline(p);
}

if (config.heic) {
  const heic = await readFile(config.heic);
  const jpeg = Buffer.from(await convert({ buffer: heic, format: 'JPEG', quality: HEIC_JPEG_QUALITY }));
  const converted = config.heic + '.converted.jpg';
  await writeFile(converted, jpeg);
  out.heic_conversion = { heic_bytes: heic.length, jpeg_bytes: jpeg.length };
  out.heic_converted = await pipeline(converted);
}

out.distance_copy_half_size_q60 = hammingDistance(out.original.hash, out.copy.hash);
out.distance_copy_756x1008_q30 = hammingDistance(out.original.hash, out.copy_hard.hash);
out.distance_other_photo = hammingDistance(out.original.hash, out.other.hash);
console.log(JSON.stringify(out, null, 2));
'''


def run(cmd, cwd=None, timeout=600):
    proc = subprocess.run(cmd, cwd=cwd, capture_output=True, text=True, timeout=timeout)
    if proc.returncode != 0:
        raise RuntimeError('%s failed (%d): %s' % (' '.join(cmd[:3]), proc.returncode,
                                                   (proc.stderr or proc.stdout).strip()[-500:]))
    return proc.stdout


def sha256(path):
    h = hashlib.sha256()
    with open(path, 'rb') as fh:
        for block in iter(lambda: fh.read(1 << 20), b''):
            h.update(block)
    return h.hexdigest()


def fetch_to(url, path):
    req = urllib.request.Request(url, headers={'User-Agent': USER_AGENT, 'Accept-Encoding': 'identity'})
    with urllib.request.urlopen(req, timeout=TIMEOUT) as resp, open(path, 'wb') as out:
        while True:
            block = resp.read(1 << 20)
            if not block:
                break
            out.write(block)


def ensure_download(url, path, expected_sha, work):
    if os.path.exists(path) and sha256(path) == expected_sha:
        return
    fetch_to(url, path)
    got = sha256(path)
    if got != expected_sha:
        raise RuntimeError('%s now hashes to %s, dataset records %s' % (url, got, expected_sha))


def node_env():
    """A directory holding sharp, blurhash and heic-convert, installed once."""
    os.makedirs(CACHE, exist_ok=True)
    marker = os.path.join(CACHE, 'node_modules', 'sharp', 'package.json')
    if not os.path.exists(marker):
        run(['npm', 'install', '--prefix', CACHE, '--no-audit', '--no-fund',
             'sharp@0.33.5', 'blurhash@2.0.5', 'heic-convert@2.1.0'])
    return CACHE


def build_inputs(dataset, work):
    lab = dataset['lab']
    inputs = lab['inputs']
    src = os.path.join(work, 'src.jpg')
    ensure_download(inputs['src.jpg']['url'], src, inputs['src.jpg']['sha256'], work)

    crop = os.path.join(work, 'crop.jpg')
    photo = os.path.join(work, 'photo-12mp.jpg')
    copy = os.path.join(work, 'copy.jpg')
    copy_hard = os.path.join(work, 'copy-hard.jpg')
    heic = os.path.join(work, 'photo-48mp.heic')
    other = os.path.join(work, 'other.jpg')

    run(['sips', '-c', '5120', '3840', src, '--out', crop])
    run(['sips', '-z', '4032', '3024', '-s', 'formatOptions', '90', crop, '--out', photo])
    run(['exiftool', '-q', '-overwrite_original', '-all=', photo])
    run(['exiftool', '-q', '-overwrite_original',
         '-Make=Apple', '-Model=iPhone 15 Pro', '-Software=18.0',
         '-LensModel=iPhone 15 Pro back triple camera 6.765mm f/1.78',
         '-DateTimeOriginal=2026:10:25 09:30:00', '-CreateDate=2026:10:25 09:30:00',
         '-OffsetTimeOriginal=-04:00', '-GPSLatitude=28.586', '-GPSLatitudeRef=N',
         '-GPSLongitude=80.651', '-GPSLongitudeRef=W', '-GPSAltitude=3', '-GPSAltitudeRef=0',
         '-ExposureTime=1/120', '-FNumber=1.78', '-ISO=80', '-Orientation#=1', photo])
    run(['sips', '-z', '1512', '2016', '-s', 'formatOptions', '60', photo, '--out', copy])
    run(['sips', '-z', '1008', '756', '-s', 'formatOptions', '30', photo, '--out', copy_hard])
    run(['sips', '-z', '6048', '8064', '-s', 'format', 'heic', crop, '--out', heic])

    other_src = os.path.join(work, 'other-src.jpg')
    ensure_download(inputs['other.jpg']['url'], other_src, inputs['other.jpg']['sha256'], work)
    shutil.copyfile(other_src, other)

    built = {
        'photo-12mp.jpg': photo, 'copy.jpg': copy, 'copy-hard.jpg': copy_hard,
        'photo-48mp.heic': heic, 'other.jpg': other,
    }
    failures = []
    for name, path in built.items():
        got = sha256(path)
        if got != inputs[name]['sha256']:
            failures.append('lab: %s rebuilt to %s, dataset records %s'
                            % (name, got, inputs[name]['sha256']))
        if os.path.getsize(path) != inputs[name]['bytes']:
            failures.append('lab: %s rebuilt to %d bytes, dataset records %d'
                            % (name, os.path.getsize(path), inputs[name]['bytes']))
    return failures, {'original': photo, 'copy': copy, 'copy_hard': copy_hard,
                      'other': other, 'heic': heic}


def check_lab(dataset, failures):
    lab = dataset['lab']
    work = os.path.join(CACHE, 'work')
    os.makedirs(work, exist_ok=True)
    env_dir = node_env()

    build_failures, paths = build_inputs(dataset, work)
    if build_failures:
        failures.extend(build_failures)
        return

    config_path = os.path.join(work, 'config.json')
    with open(config_path, 'w', encoding='utf-8') as fh:
        json.dump(paths, fh)
    script_path = os.path.join(env_dir, 'measure.mjs')
    with open(script_path, 'w', encoding='utf-8') as fh:
        fh.write(NODE_SCRIPT)
    measured = json.loads(run(['node', script_path, config_path]))

    want = lab['measured']
    got = {
        'original': {k: measured['original'][k] for k in ('width', 'height', 'format', 'bytes')},
        'thumbnail': measured['original']['thumbnail'],
        'display': measured['original']['display'],
        'blurhash': measured['original']['blurhash'],
        'perceptual_hash': {'value': measured['original']['hash'],
                            'bits': len(measured['original']['hash']) * 4},
        'distance_copy_half_size_q60': measured['distance_copy_half_size_q60'],
        'distance_copy_756x1008_q30': measured['distance_copy_756x1008_q30'],
        'distance_other_photo': measured['distance_other_photo'],
        'heic': {
            'heic_bytes': measured['heic_conversion']['heic_bytes'],
            'jpeg_bytes': measured['heic_conversion']['jpeg_bytes'],
            'jpeg_width': measured['heic_converted']['width'],
            'jpeg_height': measured['heic_converted']['height'],
            'display_bytes': measured['heic_converted']['display']['bytes'],
            'thumbnail_bytes': measured['heic_converted']['thumbnail']['bytes'],
        },
    }
    for group, values in want.items():
        actual = got.get(group)
        if actual is None:
            failures.append('lab: %s was not measured' % group)
            continue
        if isinstance(values, dict):
            for key, value in values.items():
                if actual.get(key) != value:
                    failures.append('lab: %s.%s is %r, dataset records %r'
                                    % (group, key, actual.get(key), value))
        elif actual != values:
            failures.append('lab: %s is %r, dataset records %r' % (group, actual, values))
    print('lab: %d values measured from %s' % (
        sum(len(v) if isinstance(v, dict) else 1 for v in want.values()), work))


def main():
    recheck = '--recheck' in sys.argv[1:]
    with open(DATASET, encoding='utf-8') as fh:
        dataset = json.load(fh)

    failures = []
    cache = {}
    checked = 0
    for row in dataset['results']:
        url = row['url']
        if url not in cache:
            try:
                cache[url] = source_text(url)
            except Exception as exc:  # a page that refuses us is a failure, not a skip
                failures.append('%s / %s: cannot read %s (%s)'
                                % (row['option'], row['property'], url, exc))
                cache[url] = None
                continue
        if cache[url] is None:
            failures.append('%s / %s: %s was unreadable' % (row['option'], row['property'], url))
            continue
        checked += len(row.get('quotes', []))
        before = len(failures)
        check_quotes(row, cache[url], failures)
        if len(failures) == before:
            print('ok   %s / %s' % (row['option'], row['property']))

    print('%s: %d quotes across %d rows' % ('recheck' if recheck else 'full',
                                            checked, len(dataset['results'])))

    try:
        check_lab(dataset, failures)
    except Exception as exc:
        failures.append('lab: %s' % exc)

    if failures:
        print('\nFAIL')
        for line in failures:
            print('  ' + line)
        return 1
    print('PASS: every quote is still at its source and every measurement matches')
    return 0


if __name__ == '__main__':
    sys.exit(main())