#!/usr/bin/env python3
"""Re-run the share-hundreds-photos check.

Builds one zip archive of a synthetic photo album at three sizes, twice each:

  one-piece  JSZip 3.10.1, generateAsync({type: 'blob', streamFiles: true,
             compression: 'STORE'}), the call Viallo's web app used to make
  stream     client-zip 2.5.0, makeZip() written to disk entry by entry, the
             path Viallo's web app uses now

and records, per run, the peak resident memory up to the moment the archive
exists (build) and over the whole process (process, which for one-piece also
includes copying the finished archive to disk), the wall time, and whether every
file inside the archive is byte-identical to its input. The runs happen in
Node.js, not in a browser tab, so they show how memory grows with the album, not
the size at which a particular tab gives up. It needs about 20 GB of free disk.

It then fetches every page in sources.csv, next to this file, and reports per
row whether the quoted sentence behind each cell of the post's comparison table
still appears on that page.

    python3 check.py [--sizes 1.6,5,10] [--repeat 3] [--node-modules DIR] [--out results.json]
                     [--skip-lab] [--skip-docs]

DIR must contain jszip and client-zip; the default is the web app's own
node_modules (apps/web/node_modules), three levels above this file.
"""
import argparse
import csv
import html
import re
import urllib.request
import hashlib
import json
import os
import platform
import random
import resource
import shutil
import subprocess
import sys
import tempfile
import time
import zipfile
from pathlib import Path

PHOTO_BYTES = 4_000_000
SEED = 20261025

NODE_SCRIPT = r"""
import fs from 'node:fs';
import path from 'node:path';
import { Readable, Writable } from 'node:stream';
import { pathToFileURL } from 'node:url';

const [mode, nodeModules, listFile, outFile] = process.argv.slice(2);
const files = fs.readFileSync(listFile, 'utf8').split('\n').filter(Boolean);
const mtime = new Date('2026-10-25T00:00:00Z');
// Peak memory up to the moment the archive exists, before it is copied to disk.
const report = () => console.log(JSON.stringify({ build_peak_rss_bytes: process.resourceUsage().maxRSS * 1024 }));

if (mode === 'one-piece') {
  const { default: JSZip } = await import(pathToFileURL(path.join(nodeModules, 'jszip/lib/index.js')).href);
  const zip = new JSZip();
  for (const file of files) {
    zip.file(path.basename(file), fs.readFileSync(file), { date: mtime });
  }
  const blob = await zip.generateAsync({ type: 'blob', streamFiles: true, compression: 'STORE' });
  report();
  // Node's Blob.stream() hands back the whole blob as one chunk, and fs refuses a write over 2 GiB.
  const fd = fs.openSync(outFile, 'w');
  for await (const chunk of blob.stream()) {
    for (let at = 0; at < chunk.length; at += 2 ** 30) fs.writeSync(fd, chunk.subarray(at, at + 2 ** 30));
  }
  fs.closeSync(fd);
} else {
  const { makeZip } = await import(pathToFileURL(path.join(nodeModules, 'client-zip/index.js')).href);
  async function* entries() {
    for (const file of files) {
      yield {
        name: path.basename(file),
        lastModified: mtime,
        size: fs.statSync(file).size,
        input: Readable.toWeb(fs.createReadStream(file)),
      };
    }
  }
  await makeZip(entries()).pipeTo(Writable.toWeb(fs.createWriteStream(outFile)));
  report();
}
"""


def peak_rss_bytes():
    rss = resource.getrusage(resource.RUSAGE_CHILDREN).ru_maxrss
    return rss if sys.platform == 'darwin' else rss * 1024


def measure(args):
    """Runs one build in its own process so ru_maxrss covers only that build."""
    start = time.monotonic()
    proc = subprocess.run(['node', args.script, args.mode, args.node_modules, args.list, args.archive],
                          capture_output=True, text=True, timeout=1800)
    print(json.dumps({
        'exit': proc.returncode,
        'stdout': proc.stdout,
        'stderr': proc.stderr,
        'wall_s': round(time.monotonic() - start, 1),
        'peak_rss_bytes': peak_rss_bytes(),
    }))


def build_peak_mib(stdout):
    for line in stdout.splitlines():
        if line.startswith('{'):
            return round(json.loads(line)['build_peak_rss_bytes'] / 2**20)
    return None


def first_error(stderr, code):
    for line in stderr.splitlines():
        if 'Error' in line or 'FATAL' in line or 'Assertion' in line:
            return line.strip()[:300]
    return f'exit {code}'


def sha256_file(path):
    h = hashlib.sha256()
    with open(path, 'rb') as f:
        for chunk in iter(lambda: f.read(1 << 20), b''):
            h.update(chunk)
    return h.hexdigest()


def make_inputs(directory, count):
    rng = random.Random(SEED)
    hashes = {}
    for i in range(1, count + 1):
        name = f'IMG_{i:04d}.jpg'
        # Random bytes do not compress, which is how camera JPEGs behave under STORE.
        data = rng.randbytes(PHOTO_BYTES)
        (directory / name).write_bytes(data)
        hashes[name] = hashlib.sha256(data).hexdigest()
    return hashes


def archive_matches(archive, hashes, names):
    """Returns (every file identical to its input, why not)."""
    try:
        with zipfile.ZipFile(archive) as z:
            if sorted(z.namelist()) != sorted(names):
                return False, f'{len(z.namelist())} entries listed, {len(names)} expected'
            for name in names:
                h = hashlib.sha256()
                with z.open(name) as f:
                    for chunk in iter(lambda: f.read(1 << 20), b''):
                        h.update(chunk)
                if h.hexdigest() != hashes[name]:
                    return False, f'{name} differs from its input'
    except zipfile.BadZipFile as e:
        why = f'Python zipfile: {e}'
        if shutil.which('unzip'):
            test = subprocess.run(['unzip', '-tqq', str(archive)], capture_output=True, text=True)
            why += f'; unzip -t exit {test.returncode}: {(test.stdout + test.stderr).strip()[:200]}'
        return False, why
    return True, None


UA = ('Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 '
      '(KHTML, like Gecko) Version/18.0 Safari/605.1.15')


def normalise(text):
    text = text.replace('\u2019', "'").replace('\u2018', "'").replace('\u201c', '"').replace('\u201d', '"')
    return re.sub(r'\s+', ' ', text.replace('\xa0', ' ')).strip()


def page_texts(url):
    req = urllib.request.Request(url, headers={'User-Agent': UA, 'Accept-Language': 'en-US,en;q=0.9'})
    with urllib.request.urlopen(req, timeout=30) as res:
        raw = res.read().decode('utf-8', errors='replace')
    raw = re.sub(r'(?is)<(script|style|noscript)[^>]*>.*?</\1>', ' ', raw)
    # A quote can cross an inline tag, so compare against both joinings.
    return [normalise(html.unescape(re.sub(r'(?s)<[^>]+>', sep, raw))) for sep in ('', ' ')]


def check_docs(sources):
    pages, found = {}, 0
    with open(sources, newline='') as f:
        rows = list(csv.DictReader(f))
    for row in rows:
        url = row['url']
        if url not in pages:
            try:
                pages[url] = page_texts(url)
            except Exception as e:  # noqa: BLE001
                pages[url] = e
        page = pages[url]
        if isinstance(page, Exception):
            status = f'not fetched ({page})'
        elif any(normalise(row['quote']) in t for t in page):
            status, found = 'found', found + 1
        else:
            status = 'MISSING'
        print(f"{status:>8}  {row['option']} / {row['property']}: {row['quote'][:70]}  <{url}>")
    print(f'{found} of {len(rows)} quotes found')
    return found, len(rows)


def node_version():
    return subprocess.run(['node', '--version'], capture_output=True, text=True).stdout.strip()


def main():
    here = Path(__file__).resolve().parent
    parser = argparse.ArgumentParser()
    parser.add_argument('--sizes', default='1.6,5,10', help='album sizes in GB (10^9 bytes)')
    parser.add_argument('--node-modules', default=str(here.parents[2] / 'node_modules'))
    parser.add_argument('--repeat', type=int, default=3, help='runs per album size and method')
    parser.add_argument('--out', default=None, help='write the results as JSON here')
    parser.add_argument('--skip-lab', action='store_true')
    parser.add_argument('--skip-docs', action='store_true')
    parser.add_argument('--recheck', action='store_true', help='re-read the cited pages only')
    parser.add_argument('--_measure', action='store_true', help=argparse.SUPPRESS)
    parser.add_argument('--mode'), parser.add_argument('--script'), parser.add_argument('--list')
    parser.add_argument('--archive')
    args = parser.parse_args()
    if args._measure:
        return measure(args)
    if not args.skip_docs:
        check_docs(here / 'sources.csv')
    if args.skip_lab or args.recheck:
        return

    node_modules = str(Path(args.node_modules).resolve())
    sizes = [float(s) for s in args.sizes.split(',')]
    counts = [round(s * 1e9 / PHOTO_BYTES) for s in sizes]
    work = Path(tempfile.mkdtemp(prefix='share-hundreds-photos-'))
    rows = []
    try:
        script = work / 'build-zip.mjs'
        script.write_text(NODE_SCRIPT)
        photos = work / 'photos'
        photos.mkdir()
        print(f'writing {max(counts)} synthetic photos of {PHOTO_BYTES} bytes to {photos}', flush=True)
        hashes = make_inputs(photos, max(counts))
        names_all = sorted(hashes)
        for size, count in zip(sizes, counts):
            names = names_all[:count]
            listing = work / f'list-{count}.txt'
            listing.write_text('\n'.join(str(photos / n) for n in names) + '\n')
            for mode, run in ((m, n) for n in range(1, args.repeat + 1) for m in ('one-piece', 'stream')):
                archive = work / f'{mode}-{count}.zip'
                out = subprocess.run(
                    [sys.executable, __file__, '--_measure', '--mode', mode, '--script', str(script),
                     '--node-modules', node_modules, '--list', str(listing), '--archive', str(archive)],
                    capture_output=True, text=True, check=True)
                m = json.loads(out.stdout)
                finished = m['exit'] == 0 and archive.exists()
                identical, unreadable = archive_matches(archive, hashes, names) if finished else (False, None)
                row = {
                    'album_gb': size,
                    'photos': count,
                    'input_bytes': count * PHOTO_BYTES,
                    'method': mode,
                    'run': run,
                    'finished': finished,
                    'build_peak_rss_mib': build_peak_mib(m['stdout']),
                    'process_peak_rss_mib': round(m['peak_rss_bytes'] / 2**20),
                    'wall_s': m['wall_s'],
                    'archive_bytes': archive.stat().st_size if finished else None,
                    'archive_sha256': sha256_file(archive) if finished else None,
                    'files_identical_to_inputs': identical,
                    'error': unreadable if finished else first_error(m['stderr'], m['exit']),
                }
                rows.append(row)
                print(json.dumps(row), flush=True)
                if archive.exists():
                    archive.unlink()
    finally:
        shutil.rmtree(work, ignore_errors=True)

    print()
    print(f"{'album':>7} {'photos':>6} {'method':>9} {'run':>3} {'finished':>8} {'build MiB':>9} {'process MiB':>11} "
          f"{'wall s':>7} identical")
    for r in rows:
        print(f"{r['album_gb']:>5} GB {r['photos']:>6} {r['method']:>9} {r['run']:>3} {str(r['finished']):>8} "
              f"{str(r['build_peak_rss_mib']):>9} {r['process_peak_rss_mib']:>11} {r['wall_s']:>7} "
              f"{r['files_identical_to_inputs']}")
    result = {
        'node': node_version(),
        'python': platform.python_version(),
        'machine': f'{platform.system()} {platform.release()} {platform.machine()}',
        'rows': rows,
    }
    if args.out:
        Path(args.out).write_text(json.dumps(result, indent=2) + '\n')


if __name__ == '__main__':
    main()