#!/usr/bin/env python3
"""Re-run the photo-sharing-app-groups check.

Fetches every page named in dataset.json and reports, per option and property,
whether each supporting quote still appears on that page.

    python3 check.py [path/to/dataset.json]

Without an argument it reads dataset.json next to this file, or downloads it
from https://www.viallo.app/blog/photo-sharing-app-groups/dataset.json.
"""
import html
import json
import os
import re
import sys
import urllib.request

UA = ('Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 '
      '(KHTML, like Gecko) Version/18.0 Safari/605.1.15')
REMOTE = 'https://www.viallo.app/blog/photo-sharing-app-groups/dataset.json'


def fetch(url):
    req = urllib.request.Request(url, headers={'User-Agent': UA, 'Accept-Language': 'en-US,en;q=0.9'})
    with urllib.request.urlopen(req, timeout=30) as res:
        return res.read().decode('utf-8', errors='replace')


def normalise(text):
    text = text.replace('’', "'").replace('‘', "'").replace('“', '"').replace('”', '"')
    return re.sub(r'\s+', ' ', text).strip()


def page_texts(raw):
    raw = re.sub(r'(?is)<(script|style|noscript)[^>]*>.*?</\1>', ' ', raw)
    # A quote can cross an inline link, so compare against both joinings of the tags.
    return [normalise(html.unescape(re.sub(r'(?s)<[^>]+>', sep, raw))) for sep in ('', ' ')]


def load_dataset():
    args = [a for a in sys.argv[1:] if a != '--recheck']
    if args:
        return json.load(open(args[0]))
    local = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'dataset.json')
    if os.path.exists(local):
        return json.load(open(local))
    return json.loads(fetch(REMOTE))


def main():
    data = load_dataset()
    print(f"dataset checkedAt {data['checkedAt']}, method {data['method']}")
    pages = {}
    for url in data['inputs']:
        try:
            pages[url] = page_texts(fetch(url))
        except Exception as exc:
            pages[url] = exc
    found = missing = unreachable = 0
    for option, props in data['results'].items():
        for prop, cell in props.items():
            if not cell['evidence']:
                print(f"  -    {option} / {prop}: {cell['value']} (no quote)")
                continue
            for ev in cell['evidence']:
                page = pages.get(ev['url'])
                if isinstance(page, Exception) or page is None:
                    unreachable += 1
                    status = f'UNREACHABLE ({page})'
                elif any(normalise(ev['quote']) in t for t in page):
                    found += 1
                    status = 'FOUND'
                else:
                    missing += 1
                    status = 'MISSING'
                print(f"  {status:<7} {option} / {prop}: {cell['value']}\n          \"{ev['quote']}\" {ev['url']}")
    print(f'{found} quotes found, {missing} missing, {unreachable} on unreachable pages')
    return 1 if missing or unreachable else 0


if __name__ == '__main__':
    sys.exit(main())
