#!/usr/bin/env python3
"""Re-runs the check behind the post "How to Share Photos Without an App Download".

primary-docs: every cited page is fetched again and the recorded sentence must still be on it.
Viallo rows and product claims: the cited line must still be in the repository file.
CSV: dataset.csv must still carry the same rows as dataset.json.

    python3 check.py            full check
    python3 check.py --recheck  same, skipping any lab step longer than a minute (none here)

Exit 0 when every check holds, 1 otherwise.
"""
import csv
import html
import json
import os
import re
import sys
import urllib.request

HERE = os.path.dirname(os.path.abspath(__file__))
REPO = os.path.abspath(os.path.join(HERE, '..', '..', '..', '..', '..'))
UA = 'Mozilla/5.0'
TIMEOUT = 45
RECHECK = '--recheck' in sys.argv


def norm(text):
    """Whitespace, entities, quote characters and tag-left spaces removed."""
    text = html.unescape(text or '')
    text = (text.replace('’', "'").replace('‘', "'")
                .replace('“', '"').replace('”', '"')
                .replace(' ', ' '))
    text = re.sub(r'\s+', ' ', text)
    text = re.sub(r'\s+([,.;:!?])', r'\1', text)
    return text.strip()


def to_text(raw):
    raw = re.sub(r'(?is)<(script|style|noscript|svg)[^>]*>.*?</\1>', ' ', raw)
    raw = re.sub(r'(?s)<!--.*?-->', ' ', raw)
    raw = re.sub(r'(?i)<(br|/p|/div|/li|/h[1-6]|/tr|/td|/th)\s*/?>', ' \n ', raw)
    raw = re.sub(r'(?s)<[^>]+>', ' ', raw)
    return norm(raw)


_page_cache = {}


def page(url):
    if url not in _page_cache:
        req = urllib.request.Request(url, headers={'User-Agent': UA})
        with urllib.request.urlopen(req, timeout=TIMEOUT) as resp:
            body = resp.read().decode('utf-8', 'replace')
        _page_cache[url] = to_text(body)
    return _page_cache[url]


class NotInClone(Exception):
    pass


def repo_file(path):
    full = os.path.join(REPO, path)
    if not os.path.isfile(full):
        raise NotInClone(path)
    with open(full, encoding='utf-8') as fh:
        return fh.read()


def source_text(url, source):
    """A repository file for product claims, otherwise the cited page."""
    if source:
        return repo_file(source)
    return page(url)


def main():
    with open(os.path.join(HERE, 'dataset.json'), encoding='utf-8') as fh:
        data = json.load(fh)

    print('mode: %s' % ('--recheck' if RECHECK else 'full'))
    print('method: %s   checkedAt: %s' % (data['method'], data['checkedAt']))

    failures = []
    checked = skipped = 0

    print('\n== pages: %d rows ==' % len(data['results']))
    for row in data['results']:
        label = '%-14s %-16s' % (row['option'], row['property'])
        if not row.get('checked'):
            skipped += 1
            print('  NOTCHECKED %s %s' % (label, row.get('reason', '')))
            continue
        try:
            text = norm(source_text(row.get('url', ''), row.get('source')))
        except NotInClone as exc:
            skipped += 1
            print('  NOTCHECKED %s not in a repository clone: %s' % (label, exc))
            continue
        except Exception as exc:                       # network, 404, timeout
            failures.append('%s %s: fetch failed: %s' % (label, row['property'], exc))
            print('  FAIL       %s fetch failed: %s' % (label, exc))
            continue
        if norm(row['quote']) in text:
            checked += 1
            print('  ok         %s' % label)
        else:
            failures.append('%s quote gone from %s' % (label, row['url']))
            print('  FAIL       %s quote not on page' % label)

    print('\n== third-party statements: %d ==' % len(data['third_party_statements']))
    for st in data['third_party_statements']:
        label = 'line %-4s %-10s' % (st['post_line'], st['verdict'])
        if not st.get('quote') or not (st.get('url') or st.get('source')):
            skipped += 1
            print('  NOTCHECKED %s no quote to check (%s)' % (label, st['verdict']))
            continue
        try:
            text = norm(source_text(st.get('url', ''), st.get('source')))
        except NotInClone as exc:
            skipped += 1
            print('  NOTCHECKED %s not in a repository clone: %s' % (label, exc))
            continue
        except Exception as exc:
            failures.append('line %s: fetch failed: %s' % (st['post_line'], exc))
            print('  FAIL       %s %s' % (label, exc))
            continue
        if norm(st['quote']) in text:
            checked += 1
            print('  ok         %s' % label)
        else:
            failures.append('line %s quote gone: %s' % (st['post_line'], st['url']))
            print('  FAIL       %s quote not found' % label)

    print('\n== product claims: %d ==' % len(data['product_claims']))
    for cl in data['product_claims']:
        label = '%-10s' % cl['verdict']
        try:
            text = source_text('', cl['source'])
        except NotInClone as exc:
            skipped += 1
            print('  NOTCHECKED %s not in a repository clone: %s' % (label, exc))
            continue
        except Exception as exc:
            failures.append('%s: source failed: %s' % (cl['claim'][:40], exc))
            print('  FAIL       %s %s' % (label, exc))
            continue
        if norm(cl['quote']) in norm(text):
            checked += 1
            print('  ok         %s %s' % (label, cl['claim'][:60]))
        else:
            failures.append('claim quote gone: %s' % cl['claim'][:60])
            print('  FAIL       %s %s quote not in %s' % (label, cl['claim'][:40], cl['source']))

    print('\n== dataset.csv ==')
    csv_path = os.path.join(HERE, 'dataset.csv')
    with open(csv_path, encoding='utf-8', newline='') as fh:
        rows = list(csv.DictReader(fh))
    want = {(r['option'], r['property'], r['value'], 'true' if r['checked'] else 'false')
            for r in data['results']}
    got = {(r['option'], r['property'], r['value'], r['checked'].lower()) for r in rows}
    if want == got:
        print('  ok         %d rows match dataset.json' % len(rows))
    else:
        for miss in sorted(want - got):
            print('  FAIL       missing from csv: %s' % (miss,))
        for extra in sorted(got - want):
            print('  FAIL       only in csv: %s' % (extra,))
        failures.append('dataset.csv differs from dataset.json')

    print('\nverified: %d   not checked: %d   failures: %d' % (checked, skipped, len(failures)))
    for f in failures:
        print('  - ' + f)
    return 0 if not failures else 1


if __name__ == '__main__':
    sys.exit(main())