#!/usr/bin/env python3
"""Re-runs the primary-docs check behind dataset.json (best private photo sharing apps).

  python3 check.py            fetches every cited page and rewrites dataset.csv
  python3 check.py --recheck  the same; there is no lab step to skip

Exit 0 when every supporting quote is still on its page, 1 otherwise. Rows of
kind "repo" are read from the Viallo checkout this file sits in; outside one
they are reported as not checked. Standard library only.
"""

import csv
import html
import json
import os
import re
import sys
import urllib.error
import urllib.parse
import urllib.request

HERE = os.path.dirname(os.path.abspath(__file__))
DATASET = os.path.join(HERE, 'dataset.json')
CSV_OUT = os.path.join(HERE, 'dataset.csv')
REPO = os.path.abspath(os.path.join(HERE, '..', '..', '..', '..', '..'))
UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)'
TIMEOUT = 90

# A page that answers with one of these is a script shell, a challenge, or a
# login wall: no text, so no quote can be confirmed against it.
MIN_TEXT_CHARS = 200


def normalise(text):
    text = html.unescape(text)
    for a, b in (
        ('‘', "'"), ('’', "'"), ('“', '"'), ('”', '"'),
        (' ', ' '), ('​', ''), ('…', '...'),
    ):
        text = text.replace(a, b)
    text = re.sub(r'[‐-―−]', '-', text)
    text = re.sub(r'\s+', ' ', text).strip().lower()
    return text


def to_text(page):
    page = re.sub(r'(?is)<(script|style|noscript|template)[^>]*>.*?</\1>', ' ', page)
    page = re.sub(r'(?s)<[^>]+>', ' ', page)
    return normalise(page)


def fetch(url, hops=0):
    """Follows 308 too: Python 3.9's redirect handler stops at 301-303 and 307."""
    if hops > 5:
        raise RuntimeError('too many redirects from %s' % url)
    req = urllib.request.Request(
        url, headers={'User-Agent': UA, 'Accept-Language': 'en-US,en;q=0.9'}
    )
    try:
        with urllib.request.urlopen(req, timeout=TIMEOUT) as resp:
            charset = resp.headers.get_content_charset() or 'utf-8'
            return resp.read().decode(charset, 'replace')
    except urllib.error.HTTPError as exc:
        target = exc.headers.get('Location')
        if exc.code in (301, 302, 303, 307, 308) and target:
            return fetch(urllib.parse.urljoin(url, target), hops + 1)
        raise


def repo_text(path):
    with open(os.path.join(REPO, path), encoding='utf-8') as fh:
        return normalise(fh.read())


class Cache:
    def __init__(self):
        self.pages = {}

    def get(self, url):
        if url not in self.pages:
            self.pages[url] = fetch(url)
        return self.pages[url]


def main(argv):
    recheck = '--recheck' in argv[1:]
    data = json.load(open(DATASET, encoding='utf-8'))
    inputs = {i['id']: i for i in data['inputs']}
    cache = Cache()

    ok, failed, skipped = 0, 0, 0
    failures = []

    for row in data['results']:
        label = '%s | %s' % (row['option'], row['property'])
        if row.get('url') is None:
            skipped += 1
            reason = row.get('note', 'no page behind it')
            print('SKIP   %s -> %s' % (label, reason))
            continue
        src = inputs[row['url']]
        try:
            if src['kind'] == 'repo':
                if not os.path.isfile(os.path.join(REPO, src['path'])):
                    skipped += 1
                    print('SKIP   %s -> %s is only in the Viallo source tree' % (label, src['path']))
                    continue
                text = repo_text(src['path'])
                where = src['path']
            else:
                text = to_text(cache.get(src['url']))
                where = src['url']
                if len(text) < MIN_TEXT_CHARS:
                    raise RuntimeError(
                        'page served %d characters of text, no quote can be checked' % len(text)
                    )
        except (urllib.error.URLError, urllib.error.HTTPError, RuntimeError, OSError) as exc:
            failed += 1
            failures.append('%s: %s (%s)' % (label, exc, row['url']))
            print('FAIL   %s -> cannot read %s: %s' % (label, row['url'], exc))
            continue

        quote = normalise(row['quote'])
        if quote in text:
            absent = row.get('assertAbsent')
            if absent and normalise(absent) in text:
                failed += 1
                failures.append('%s: "%s" is present on the page' % (label, absent))
                print('FAIL   %s -> "%s" appears in %s' % (label, absent, where))
                continue
            ok += 1
            print('OK     %s -> %s' % (label, where))
        else:
            failed += 1
            failures.append('%s: quote not found on %s' % (label, where))
            print('FAIL   %s -> quote absent from %s' % (label, where))
            print('       sought: %s' % row['quote'][:160])

    write_csv(data)

    print()
    print('%d ok, %d failed, %d not checked%s'
          % (ok, failed, skipped, ' (--recheck)' if recheck else ''))
    if failures:
        print('failures:')
        for f in failures:
            print('  - ' + f)
    return 0 if failed == 0 else 1


def write_csv(data):
    fields = ['option', 'property', 'value', 'quote', 'url', 'verdict', 'note']
    inputs = {i['id']: i for i in data['inputs']}
    with open(CSV_OUT, 'w', newline='', encoding='utf-8') as fh:
        w = csv.DictWriter(fh, fieldnames=fields)
        w.writeheader()
        for row in data['results']:
            url = row.get('url')
            w.writerow({
                'option': row['option'],
                'property': row['property'],
                'value': row['value'],
                'quote': row['quote'] or '',
                'url': (inputs[url].get('url') or inputs[url]['path']) if url else '',
                'verdict': row['verdict'],
                'note': row.get('note', ''),
            })


if __name__ == '__main__':
    sys.exit(main(sys.argv))