#!/usr/bin/env python3
"""Public reproduction, Python 3.10+ standard library.

python verify-breakout-study.py --release https://coinnudge.site/api/research/live/bitcoin-breakout-confirmation/releases/RELEASE.json
or: --release local-release.json --inputs local-input.csv

Verifies the downloaded input checksum, every qualifying candidate, spacing,
retained observations, confirmation flags, future paths and comparison counts.
Add --bootstrap (requires numpy) to independently recompute every published
primary interval and the 3/14-day failure-difference sensitivity intervals.
"""
import argparse
import csv
import hashlib
import io
import json
import math
import statistics
import urllib.request
from collections import defaultdict
from datetime import datetime, timezone
from pathlib import Path


def read(location):
    if location.startswith(("https://", "http://")):
        with urllib.request.urlopen(location, timeout=90) as response:
            return response.read()
    return Path(location).read_bytes()


def check(actual, expected, label):
    if isinstance(actual, (int, float)) and not isinstance(actual, bool) and isinstance(expected, (int, float)):
        assert math.isclose(actual, expected, rel_tol=1e-10, abs_tol=1e-9), (label, actual, expected)
    else:
        assert actual == expected, (label, actual, expected)


def pct(a, b):
    return (a / b - 1) * 100


def failed(bars, threshold):
    return next((j for j in range(1, len(bars)) if bars[j-1]["c"] < threshold and bars[j]["c"] < threshold), None)


def verify(document, raw):
    check(hashlib.sha256(raw).hexdigest(), document["input_sha256"], "input SHA256")
    start, end = document["observation_start"], document["observation_end"]
    by_symbol = defaultdict(list)
    for row in csv.DictReader(io.StringIO(raw.decode())):
        parsed = {k: float(v) if v != "" else None for k, v in row.items() if k not in ("symbol", "source_sha256")}
        by_symbol[row["symbol"]].append(parsed)
    observations = {r["observation_id"]: r for r in document["observations"]}
    expected_candidates, checked = [], 0
    for symbol, bars in by_symbol.items():
        bars.sort(key=lambda r: r["opened"])
        atr, runs, daily = [], [], defaultdict(list)
        ranges = []
        for i, row in enumerate(bars):
            tr = max(row["h"]-row["l"], abs(row["h"]-bars[i-1]["c"]), abs(row["l"]-bars[i-1]["c"])) if i else row["h"]-row["l"]
            ranges.append(tr)
            atr.append(None if i < 13 else sum(ranges[:14])/14 if i == 13 else (13*atr[-1]+tr)/14)
            runs.append(runs[-1]+1 if i and row["opened"]-bars[i-1]["opened"] == 3600 else 1)
            daily[int(row["opened"]//86400)*86400].append(row)
        day_high = {d: max(r["h"] for r in values) for d, values in daily.items() if len(values)==24 and values[0]["opened"]==d and values[-1]["opened"]==d+23*3600}
        for definition in ("24h", "20d"):
            last, blocker = -math.inf, None
            for i in range(200, len(bars)):
                row = bars[i]
                t0 = row["opened"]+3600
                if row["opened"] < start or t0 > end or runs[i] < 201:
                    continue
                if definition == "24h":
                    resistance = max(r["h"] for r in bars[i-24:i])
                else:
                    day = int(row["opened"]//86400)*86400
                    highs = [day_high.get(day-j*86400) for j in range(1, 21)]
                    if None in highs:
                        continue
                    resistance = max(highs)
                if row["c"] <= resistance:
                    continue
                identifier = f"{symbol}:{definition}:{int(t0)}"
                retained = t0-last >= 76*3600
                expected_candidates.append((identifier, retained, None if retained else blocker, resistance))
                if not retained:
                    continue
                last, blocker = t0, identifier
                event = observations[identifier]
                future = bars[i+1:i+77]
                initial = future[:4]
                contiguous = lambda seq: all(b["opened"]-a["opened"]==3600 for a,b in zip([row]+seq, seq))
                ready = len(initial)==4 and contiguous(initial) and t0+4*3600<=end
                buffer = atr[i-1]*.1
                hold = all(r["c"]>resistance+buffer for r in initial[:2]) if ready else None
                retests = [j for j in range(3) if resistance-buffer<=initial[j]["l"]<=resistance+buffer and initial[j]["c"]>resistance+buffer and initial[j+1]["c"]>resistance+buffer] if ready else []
                early = failed(initial, resistance-buffer) is not None if ready else None
                flow_valid = row["taker_buy_valid"]==1 and row["qv"]>0 and row["taker_buy_qv"] is not None and 0<=row["taker_buy_qv"]<=row["qv"]
                baseline = statistics.mean(r["qv"] for r in bars[i-24:i])
                ratio = row["qv"]/baseline if baseline>0 else None
                fields = {"reference_level":resistance, "atr_prior":atr[i-1], "buffer":buffer,
                    "decision_time":t0+4*3600, "hold_two":hold, "retest":bool(retests) if ready else None,
                    "retest_confirmation_time":t0+(retests[0]+2)*3600 if retests else None,
                    "early_failure":early, "landmark_eligible":ready and not early and initial[-1]["c"]>resistance,
                    "volume_ratio":ratio, "volume":ratio>=2 if ratio is not None else None,
                    "buy_share":row["taker_buy_qv"]/row["qv"] if flow_valid else None,
                    "buying":row["taker_buy_qv"]/row["qv"]>=.55 if flow_valid else None,
                    "hold_volume":hold and ratio>=2 if ready and ratio is not None else None,
                    "decision_close":initial[-1]["c"] if ready else None,
                    "wait_price_change_pct":pct(initial[-1]["c"],row["c"]) if ready else None}
                fields.update(hold_one=initial[0]["c"]>resistance+buffer if ready else None,
                    hold_four=all(r["c"]>resistance+buffer for r in initial) if ready else None,
                    hold_zero_buffer=all(r["c"]>resistance for r in initial[:2]) if ready else None,
                    hold_double_buffer=all(r["c"]>resistance+2*buffer for r in initial[:2]) if ready else None,
                    volume_1_5=ratio>=1.5 if ratio is not None else None)
                for horizon in (4,24,72):
                    path = future[4:4+horizon]
                    mature = t0+(4+horizon)*3600<=end
                    complete = mature and ready and len(path)==horizon and contiguous(future[:4+horizon])
                    status = "complete" if complete else "gap" if mature else "pending"
                    fields[f"status_{horizon}h"] = status
                    failure = failed(path,resistance-buffer) if complete else None
                    fields[f"failure_{horizon}h"] = failure is not None if complete else None
                    fields[f"failure_time_{horizon}h"] = path[failure]["opened"]+3600 if failure is not None else None
                    fields[f"return_{horizon}h_pct"] = pct(path[-1]["c"],initial[-1]["c"]) if complete else None
                    fields[f"mfe_{horizon}h_pct"] = max(0,pct(max(r["h"] for r in path),initial[-1]["c"])) if complete else None
                    fields[f"mae_{horizon}h_pct"] = min(0,pct(min(r["l"] for r in path),initial[-1]["c"])) if complete else None
                    fields[f"extension_{horizon}h"] = any(r["c"]>=resistance+2*atr[i-1] for r in path) if complete else None
                    fields[f"back_inside_{horizon}h"] = any(r["c"]<=resistance for r in path) if complete else None
                    fields[f"reclaimed_{horizon}h"] = any(path[j-1]["c"]>resistance+buffer and path[j]["c"]>resistance+buffer for j in range(failure+2 if failure is not None else len(path),len(path))) if complete else None
                for key, value in fields.items():
                    check(event[key], value, identifier+":"+key)
                checked += 1
    check(checked,len(observations),"retained count")
    ledger = document["raw_event_ledger"]
    check(len(ledger),len(expected_candidates),"candidate count")
    for actual, expected in zip(ledger, expected_candidates):
        for key, value in zip(("observation_id","retained","blocking_id","reference_level"),expected):
            check(actual[key],value,"candidate:"+key)
    for item in document["items"] + document.get("confirmation_sensitivity", []):
        pool = [r for r in observations.values() if r["symbol"]==item["symbol"] and r["definition"]==item["definition"] and item["calendar_start"]<=r["event_time"]<item["calendar_end"] and r["event_time"]+76*3600<=item["calendar_end"] and r["status_72h"]=="complete" and r["landmark_eligible"]]
        for prefix, condition in (("yes",True),("no",False)):
            group = [r for r in pool if r[item["method"]] is condition]
            count = sum(r["failure_24h"] for r in group)
            check(item[prefix+"_n"],len(group),"group N")
            check(item[prefix+"_failures"],count,"group failures")
            check(item[prefix+"_failure_pct"],count/len(group)*100 if group else None,"group rate")
            check(item[prefix+"_return_median_pct"],statistics.median(r["return_24h_pct"] for r in group) if group else None,"return median")
    print(f"PASS: input SHA256; {len(ledger)} candidates; {checked} retained event paths; {len(document['items'])} main + {len(document.get('confirmation_sensitivity', []))} sensitivity summaries.")
    print("Not checked by the core pass: bootstrap intervals, graphical rendering, statistical suitability or commercial data rights.")


def verify_bootstrap(document):
    import numpy as np
    cache = {}
    total = 0
    def weights_for(item, block):
        key=(item['symbol'],item['definition'],item['period'],block)
        if key in cache:
            return cache[key]
        start, end = item['calendar_start'],item['calendar_end']
        pool=[r for r in document['observations'] if r['symbol']==item['symbol'] and r['definition']==item['definition'] and start<=r['event_time']<end and r['event_time']+76*3600<=end and r['status_72h']=='complete' and r['landmark_eligible']]
        seed_label=f"{document['calculation_version']}|{item['symbol']}|{item['definition']}|{item['period']}|{block}"
        state=int.from_bytes(hashlib.sha256(seed_label.encode()).digest()[:4],'big') or 1
        if block==7:
            check(state,item['bootstrap_seed'],'bootstrap seed')
        days=math.ceil((end-start)/86400)
        days_of_events=np.array([int((r['event_time']-start)//86400) for r in pool],dtype=int)
        weights=np.zeros((2000,len(pool)),dtype=np.int16)
        for iteration in range(2000):
            selected=[]
            while len(selected)<days:
                state ^= (state<<13)&0xffffffff
                state ^= state>>17
                state ^= (state<<5)&0xffffffff
                state &= 0xffffffff
                first=state%days
                selected.extend((first+j)%days for j in range(block))
            weights[iteration]=np.bincount(selected[:days],minlength=days)[days_of_events]
        cache[key]=(pool,weights)
        return pool,weights
    def quantile(values):
        values=np.asarray(values,dtype=float)
        values=values[np.isfinite(values)]
        return np.quantile(values,[.025,.975],method='linear').tolist() if len(values)>=1900 else [None,None]
    def values_for(item,block):
        pool,weights=weights_for(item,block)
        yes=np.array([r[item['method']] is True for r in pool])
        no=np.array([r[item['method']] is False for r in pool])
        failures=np.array([r['failure_24h'] for r in pool],dtype=float)
        returns=np.array([r['return_24h_pct'] for r in pool])
        with np.errstate(divide='ignore',invalid='ignore'):
            y=(weights[:,yes]*failures[yes]).sum(axis=1)/weights[:,yes].sum(axis=1)*100
            n=(weights[:,no]*failures[no]).sum(axis=1)/weights[:,no].sum(axis=1)*100
        medians=[np.median(np.repeat(returns[yes],w[yes])) if w[yes].sum() else np.nan for w in weights]
        return {'yes':quantile(y),'no':quantile(n),'difference':quantile(y-n),'return':quantile(medians)}
    for item in document['items'] + document.get('confirmation_sensitivity', []):
        if min(item['yes_n'],item['no_n'])<20:
            for prefix in ('yes','no','difference','return'):
                check(item[prefix+'_ci_low'],None,'withheld interval')
                check(item[prefix+'_ci_high'],None,'withheld interval')
            continue
        result=values_for(item,7)
        for prefix, bounds in result.items():
            check(item[prefix+'_ci_low'],bounds[0],prefix+' interval low')
            check(item[prefix+'_ci_high'],bounds[1],prefix+' interval high')
            total+=1
    for item in document['sensitivity']:
        base=next(r for r in document['items'] if r['symbol']==item['symbol'] and r['definition']=='24h' and r['period']=='full' and r['method']==item['method'])
        if min(base['yes_n'],base['no_n'])<20:
            bounds=[None,None]
        else:
            bounds=values_for(base,item['block_days'])['difference']
        check(item['difference_ci_low'],bounds[0],'sensitivity low')
        check(item['difference_ci_high'],bounds[1],'sensitivity high')
        total+=1
    print(f'PASS: {total} uncertainty intervals, including withheld sensitivity intervals.')


if __name__ == "__main__":
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--release",required=True)
    parser.add_argument("--inputs")
    parser.add_argument("--bootstrap",action="store_true")
    args = parser.parse_args()
    document = json.loads(read(args.release))
    location = args.inputs or "https://coinnudge.site"+document["inputs_url"]
    verify(document,read(location))
    if args.bootstrap:
        verify_bootstrap(document)