#!/usr/bin/env python3
"""Re-runs every check behind dataset.json for wedding-photo-sharing-guide.

  python3 check.py            fetches every cited page and reads the Viallo tree
  python3 check.py --recheck  the same; there is no lab step to skip here

Exit 0 when every quote is still on its page, every Viallo line is still in the
repository this file sits in, and dataset.csv carries the same values as
dataset.json. Exit 1 otherwise. Standard library only.

Four kinds of input:
  page           HTML, reduced to text before the quote is looked for
  raw            HTML compared as served, for claims that live in metadata
  repo           a file of the repository this script sits in
  kululu-bundle  the pricing widget's JS, whose URL is hashed and read off the
                 embed page at run time

Nothing is written.
"""

import html
import json
import os
import re
import subprocess
import sys
import urllib.error
import urllib.parse
import urllib.request

HERE = os.path.dirname(os.path.abspath(__file__))
DATASET = os.path.join(HERE, 'dataset.json')
REPO = os.path.abspath(os.path.join(HERE, '..', '..', '..', '..', '..'))
UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)'
TIMEOUT = 90

# A page that answers with fewer characters than this is a script shell, a
# challenge or a login wall: no text, so no quote can be confirmed against it.
MIN_TEXT_CHARS = 200


def normalise(text):
    text = html.unescape(text)
    for a, b in (
        ('‘', "'"), ('’', "'"), ('“', '"'), ('”', '"'),
        (' ', ' '), ('​', ''), ('…', '...'),
    ):
        text = text.replace(a, b)
    text = re.sub(r'[‐-―−]', '-', text)
    text = re.sub(r'\s+', ' ', text).strip().lower()
    return text


def to_text(page):
    page = re.sub(r'(?is)<(script|style|noscript|template)[^>]*>.*?</\1>', ' ', page)
    page = re.sub(r'(?s)<[^>]+>', ' ', page)
    return normalise(page)


def fetch(url, hops=0):
    """Follows 308 too: Python 3.9's redirect handler stops at 301-303 and 307.

    This Mac's Python 3.9 links LibreSSL, which wedbox.com refuses with a
    protocol-version alert. When urllib cannot negotiate, the same GET goes
    through the system curl, which is listed in dataset tools for that reason.
    """
    if hops > 5:
        raise RuntimeError('too many redirects from %s' % url)
    req = urllib.request.Request(
        url, headers={'User-Agent': UA, 'Accept-Language': 'en-US,en;q=0.9'}
    )
    try:
        with urllib.request.urlopen(req, timeout=TIMEOUT) as resp:
            charset = resp.headers.get_content_charset() or 'utf-8'
            return resp.read().decode(charset, 'replace')
    except urllib.error.HTTPError as exc:
        target = exc.headers.get('Location')
        if exc.code in (301, 302, 303, 307, 308) and target:
            return fetch(urllib.parse.urljoin(url, target), hops + 1)
        raise
    except (urllib.error.URLError, OSError):
        out = subprocess.run(
            ['curl', '-sL', '-A', UA, '--max-time', str(TIMEOUT), url],
            capture_output=True, timeout=TIMEOUT + 30,
        )
        if out.returncode != 0 or not out.stdout:
            raise RuntimeError('urllib and curl both failed: %s'
                               % out.stderr.decode('utf-8', 'replace').strip())
        return out.stdout.decode('utf-8', 'replace')


def repo_text(path):
    with open(os.path.join(REPO, path), encoding='utf-8') as fh:
        return normalise(fh.read())


class Cache:
    def __init__(self):
        self.raw = {}

    def get(self, url):
        if url not in self.raw:
            self.raw[url] = fetch(url)
        return self.raw[url]

    def page(self, url):
        text = to_text(self.get(url))
        if len(text) < MIN_TEXT_CHARS:
            raise RuntimeError(
                'page served %d characters of text, no quote can be checked' % len(text)
            )
        return text

    def raw_text(self, url):
        return normalise(self.get(url))


def kululu_bundle(cache, embed_url):
    """The plan table is rendered by a hashed bundle; find it, then read it."""
    shell = cache.get(embed_url)
    found = re.findall(r'/static/js/main\.[0-9a-f]+\.js', shell)
    if not found:
        raise RuntimeError('no bundle URL in the pricing embed page')
    url = urllib.parse.urljoin(embed_url, found[0])
    return Cache().raw_text(url), url


def csv_mismatches(data):
    import csv
    inputs = {i['id']: i for i in data['inputs']}
    with open(os.path.join(HERE, 'dataset.csv'), newline='', encoding='utf-8') as fh:
        rows = {r['id']: r for r in csv.DictReader(fh)}
    bad = []
    for row in data['results']:
        want = {
            'option': row['option'], 'property': row['property'], 'value': row['value'],
            'quote': row['quote'] or '', 'verdict': row['verdict'],
            'url': inputs[row['url']]['url'] if row['url'] else '',
        }
        got = rows.pop(row['id'], None)
        if got is None:
            bad.append('%s: missing from dataset.csv' % row['id'])
            continue
        for key, value in want.items():
            if got[key] != value:
                bad.append('%s: dataset.csv %s differs from dataset.json' % (row['id'], key))
    bad.extend('%s: in dataset.csv only' % extra for extra in rows)
    return bad


def main(argv):
    recheck = '--recheck' in argv[1:]
    data = json.load(open(DATASET, encoding='utf-8'))
    inputs = {i['id']: i for i in data['inputs']}
    cache = Cache()

    ok, failed, skipped = 0, 0, 0
    failures = []

    for row in data['results']:
        label = '%s | %s | %s' % (row['id'], row['option'], row['property'])
        if row.get('url') is None:
            skipped += 1
            print('SKIP   %s -> %s' % (label, row.get('note', 'no page behind it')))
            continue
        src = inputs[row['url']]
        try:
            if src['kind'] == 'repo':
                text = repo_text(src['path'])
                where = src['path']
            elif src['kind'] == 'kululu-bundle':
                text, found = kululu_bundle(cache, src['url'])
                where = found
            elif src['kind'] == 'raw':
                text = cache.raw_text(src['url'])
                where = src['url']
            else:
                text = cache.page(src['url'])
                where = src['url']
        except (urllib.error.URLError, urllib.error.HTTPError, RuntimeError, OSError) as exc:
            failed += 1
            failures.append('%s: %s (%s)' % (label, exc, src.get('url', '')))
            print('FAIL   %s -> cannot read %s: %s' % (label, src.get('url', ''), exc))
            continue

        quote = normalise(row['quote'])
        if quote in text:
            absent = row.get('assertAbsent')
            if absent and normalise(absent) in text:
                failed += 1
                failures.append('%s: "%s" is present on the page' % (label, absent))
                print('FAIL   %s -> "%s" appears in %s' % (label, absent, where))
                continue
            ok += 1
            print('OK     %s -> %s' % (label, where))
        else:
            failed += 1
            failures.append('%s: quote not found on %s' % (label, where))
            print('FAIL   %s -> quote absent from %s' % (label, where))
            print('       sought: %s' % row['quote'][:200])

    for problem in csv_mismatches(data):
        failed += 1
        failures.append(problem)
        print('FAIL   ' + problem)

    print()
    print('%d ok, %d failed, %d not checked%s'
          % (ok, failed, skipped, ' (--recheck)' if recheck else ''))
    if failures:
        print('failures:')
        for f in failures:
            print('  - ' + f)
    return 0 if failed == 0 else 1


if __name__ == '__main__':
    sys.exit(main(sys.argv))