"""Independent saved-data AUTHOR audit. Standard library only. Does not import
any experimental scorer, builder, generator, optimizer, task or runtime module.
Reconstructs external targets, all gates, measured J and prespecified weighting.
"""
import collections,csv,hashlib,heapq,json,math,pathlib,random,statistics,time
ROOT=pathlib.Path(__file__).resolve().parent
T_CAP=1292;GAP=1294

def load(p):return json.loads(p.read_text())
def dump(p,x):p.write_text(json.dumps(x,sort_keys=True,indent=2)+'\n')
def sha(p):return hashlib.sha256(p.read_bytes()).hexdigest()
def canonical(x):return json.dumps(x,sort_keys=True,separators=(',',':'))
def digest(x):return hashlib.sha256(canonical(x).encode()).hexdigest()
def avg(x):return sum(x)/len(x)

def fixed_sequence(commands,rng):
    t=0;out=[]
    for i,(kind,a,v) in enumerate(commands):out.append(dict(index=i,id=i%64,t=t,kind=kind,a=a,v=v));t+=GAP+rng.randrange(101)
    return out

def observations(result,stream,K):
    assert len(result['rows'])==len(stream);bank=[0]*K;gate=True;ticks=0;messages=0;measured={'write':{},'query':{}};queries=0
    for i,(e,row) in enumerate(zip(stream,result['rows'])):
        assert e['index']==i and 0<=e['id']<64 and e['id']==i%64 and 0<=e['a']<K
        if i:assert e['t']-stream[i-1]['t']>=GAP
        target=e['v'] if e['kind']=='W' else bank[e['a']];assert row[0]==target
        assert row[4]>=0;messages+=row[4];obs=row[3];other=row[5]
        good=len(obs)==1 and not other and obs[0][0]==e['id'] and obs[0][1]==e['a'] and obs[0][3]==target and 1<=obs[0][2]-e['t']<=T_CAP
        latency=obs[0][2]-e['t'] if len(obs)==1 else None
        assert row[1]==latency and row[2]==int(good);gate=gate and good
        if latency is not None:
            ticks+=latency;key='write' if e['kind']=='W' else 'query'
            if e['a'] in measured[key]:assert measured[key][e['a']]==latency
            measured[key][e['a']]=latency
        if e['kind']=='W':bank[e['a']]=e['v']
        else:queries+=1
    assert result['gate']==gate and result['actual_sum_ticks']==ticks and result['messages']==messages and result['payload_bytes']==4*messages
    assert result['trajectory_steps']>0 and len(result['trajectory_hash'])==64 and len(result['boundary_hash'])==64
    return dict(gate=gate,ticks=ticks,messages=messages,measured=measured,queries=queries)

def diagnostic_audit(record,K):
    d=record['diagnostics'];commands=[('W',a,1 if a%2==0 else -1) for a in range(K)]+[('Q',a,0) for a in range(K)]+[('W',a,-1 if a%2==0 else 1) for a in range(K)]+[('Q',a,0) for a in range(K)]
    all_result=observations(d['all_address'],fixed_sequence(commands,random.Random(431)),K);reset=observations(d['reset'],fixed_sequence([('Q',a,0) for a in range(K)],random.Random(432)),K)
    ticks=all_result['ticks']+reset['ticks'];messages=all_result['messages']+reset['messages'];gate=all_result['gate'] and reset['gate']
    for pair in range(2):
        commands=[dict(index=i,id=i,t=i*GAP,kind='W',a=i,v=(1 if (i+pair)%2==0 else -1)) for i in range(K)]
        setup=observations(d['pair_setup'][pair],commands,K);messages+=setup['messages'];ticks+=setup['ticks'];p=d['pairs'][pair];assert p['pair']==pair;pg=True
        for a,q in enumerate(p['queries']):
            target=1 if (a+pair)%2==0 else -1;assert q['target']==target and q['event']['a']==a
            e=q['event'];rs=q['responses'];good=len(rs)==1 and rs[0]['v']==target and rs[0]['a']==a and rs[0]['id']==0 and 1<=rs[0]['t']-e['t']<=T_CAP
            assert q['gate']==good;assert q['latency']==rs[0]['t']-e['t'] if len(rs)==1 else q['latency'] is None
            pg=pg and good;messages+=q['messages'];ticks+=q['latency'] or 0
        assert p['gate']==pg;gate=gate and pg
    assert d['gate']==gate and d['messages']==messages and d['actual_sum_ticks']==ticks
    return all_result

def expected_from_measurement(workload,measured,K):
    assert set(measured['write'])==set(range(K)) and set(measured['query'])==set(range(K))
    den=workload['denominator'];assert sum(workload['wi']+workload['qi'])==den and min(workload['wi']+workload['qi'])>0
    assert all(abs(x/den-y)<1e-15 for x,y in zip(workload['wi'],workload['w']))
    assert all(abs(x/den-y)<1e-15 for x,y in zip(workload['qi'],workload['q']))
    num=sum(workload['wi'][a]*measured['write'][a]+workload['qi'][a]*measured['query'][a] for a in range(K))
    return num/den,num

def saved_host_tree(h,global_tree=False):
    adj={u:[] for u in h['positions']}
    for e in h['bonds']:
        adj[e['u']].append((e['v'],e['delay']));adj[e['v']].append((e['u'],e['delay']))
    for row in adj.values():row.sort()
    root=next(t['site'] for t in h['terminals'] if t['kind']=='query');parent={}
    if global_tree:
        distance={root:0};parent={root:None};queue=[(0,root)]
        while queue:
            d,u=heapq.heappop(queue)
            if d!=distance[u]:continue
            for v,w in adj[u]:
                if d+w<distance.get(v,10**9):distance[v]=d+w;parent[v]=u;heapq.heappush(queue,(d+w,v))
    else:
        # Independent first-arrival flood reconstruction from saved geometry.
        # Registration traffic cannot change birth order: positive delays and
        # source/destination tie order precede sequence; duplicate same-source
        # birth offers do not exist. No experimental growth/scorer imported.
        queue=[(0,root,'',0)];seq=1
        while queue:
            t,u,source,_=heapq.heappop(queue)
            if u in parent:continue
            parent[u]=source or None
            for v,w in adj[u]:
                if v!=source:heapq.heappush(queue,(t+w,v,u,seq));seq+=1
    delays={tuple(sorted((e['u'],e['v']))):e['delay'] for e in h['bonds']}
    def path(u):
        p=[]
        while u is not None:p.append(u);u=parent[u]
        return list(reversed(p))
    def depth(p):return sum(delays[tuple(sorted((u,v)))] for u,v in zip(p,p[1:]))
    response=next(t['site'] for t in h['terminals'] if t['kind']=='response');writes={t['a']:t['site'] for t in h['terminals'] if t['kind']=='write'};R=depth(path(response));table={}
    for a,u in writes.items():
        p=path(u);D=depth(p);table[a]={}
        for i,v in enumerate(p):
            if v in (root,response) or v in writes.values() and v!=u:continue
            d=depth(p[:i+1]);table[a][v]=(1+D-d,2+2*d+R)
    return parent,table

def independent_assignment_bound(table,w,K):
    # Independent subset dynamic programming, rather than experimental
    # Hungarian solver. A site can serve at most one address at each update.
    sites=sorted(set(u for row in table.values() for u in row));inf=10**30;dp=[inf]*(1<<K);dp[0]=0
    for u in sites:
        nxt=dp.copy()
        for mask,value in enumerate(dp):
            if value==inf:continue
            for a in range(K):
                if not (mask>>a)&1 and u in table[a]:
                    tw,tq=table[a][u];n=mask|(1<<a);nxt[n]=min(nxt[n],value+w['wi'][a]*tw+w['qi'][a]*tq)
        dp=nxt
    return dp[-1]

def main():
    cpu=time.process_time();wall=time.perf_counter();cfg=load(ROOT/'config.json');first=load(ROOT/'optimization-freeze.json');seal=load(ROOT/'selection-seal.json')
    for name,value in first['files'].items():assert sha(ROOT/name)==value
    assert seal['unchanged_source_files']==first['files'] and seal['optimization_freeze_sha256']==sha(ROOT/'optimization-freeze.json')
    for name,value in seal['records'].items():assert sha(ROOT/name)==value
    assert len(seal['selected'])==4
    training_manifests={r['context']:r for r in load(ROOT/'training'/'manifests.json')};candidates=[];candidate_contexts=0;validation_contexts=0;candidate_failures=0
    for text in (ROOT/'training'/'candidates.txt').open():
        r=json.loads(text);assert len(r['theta'])==25 and all(-4<=x<=4 for x in r['theta']) and digest(r['theta'])==r['theta_hash'];scores=[]
        for c in r['contexts']:
            m=training_manifests[c['context']];assert m['split']=='training' and m['run']==r['run'];candidate_contexts+=1
            if c.get('diagnostic'):
                obs=observations(c['diagnostic'],m['short'],4);J,num=expected_from_measurement(m['workload'],obs['measured'],4);assert J==c['J'] and num==c['objective']['integer_cost']
                assert c['J_star']==(J if obs['gate'] else T_CAP) and c['gate']==obs['gate']
            else:assert c['failure'] and not c['gate'] and c['J_star']==T_CAP;candidate_failures+=1
            scores.append(c['J_star'])
        assert len(scores)==16 and avg(scores)==r['training_score'];candidates.append({k:r[k] for k in ('run','index','parent','theta','theta_hash','training_score','accepted')})
    assert len(candidates)==512
    for run in range(4):
        cc=[c for c in candidates if c['run']==run];assert [c['index'] for c in cc]==list(range(128));best=None
        for c in cc:
            if best is None:assert c['parent'] is None
            else:assert c['parent']==best['index']
            accepted=best is None or c['training_score']<best['training_score'];assert accepted==c['accepted']
            if accepted:best=c
        vv=load(ROOT/'training'/f'validation-run{run}.json');assert len(vv)==4;seen=set();unique=[]
        for c in sorted(cc,key=lambda c:(c['training_score'],c['index'])):
            key=tuple(c['theta'])
            if key not in seen:unique.append(c);seen.add(key)
            if len(unique)==3:break
        pool=[x['index'] for x in unique]
        if 0 not in pool:pool.append(0)
        while len(pool)<4:pool.append(0)
        assert [v['index'] for v in vv]==pool
        for v in vv:
            assert v['theta']==cc[v['index']]['theta'];scores=[]
            for c in v['contexts']:
                m=training_manifests[c['context']];assert m['split']=='validation' and m['run']==run
                if not v['cached_padding']:validation_contexts+=1
                if c.get('diagnostic'):
                    obs=observations(c['diagnostic'],m['short'],4);J,num=expected_from_measurement(m['workload'],obs['measured'],4);assert J==c['J'];assert c['J_star']==(J if obs['gate'] else T_CAP)
                else:assert c['failure'] and c['J_star']==T_CAP
                scores.append(c['J_star'])
            assert len(scores)==8 and avg(scores)==v['validation_score']
        chosen=min(vv,key=lambda v:(v['validation_score'],v['index']));assert chosen['index']==seal['selected'][run]['index'] and chosen['theta']==seal['selected'][run]['theta']
    manifests={m['context']:m for m in load(ROOT/'results'/'manifests.json')};bounds={x['context']:x for x in load(ROOT/'results'/'d-tree-optima.json')};assert len(manifests)==192 and len(bounds)==192
    tables={};b_tables={}
    for context,m in manifests.items():
        parent,table=saved_host_tree(m['host']);assert digest(parent)==m['tree_hash'];tables[context]=table
        value=independent_assignment_bound(table,m['workload'],m['setting'][1]);assert value==bounds[context]['integer_cost']
        _,bt=saved_host_tree(m['host'],True);b_tables[context]=bt
    records=[];queries=writes=ordinary_queries=ordinary_writes=replays=steps=0;gate_failures=[];seen=set();frequencies=collections.defaultdict(lambda:collections.Counter());diagnostic_queries=0
    for text in (ROOT/'results'/'conditions.txt').open():
        r=json.loads(text);m=manifests[r['context']];K=m['setting'][1];key=(r['context'],r['method']);assert key not in seen;seen.add(key);assert r['group']==m['group'] and r['setting']==m['setting'] and r['rho']==m['workload']['rho']
        assert r['host_hash']==digest(m['host'])==m['host_hash']
        if r.get('objective'):
            ds=diagnostic_audit(r,K);J,num=expected_from_measurement(m['workload'],ds['measured'],K);assert J==r['J'] and num==r['objective']['integer_cost'];gate=r['diagnostics']['gate'];realized=[]
            table=tables[r['context']] if r['method'].startswith('D') else b_tables[r['context']]
            assert len(set(r['placement'].values()))==K
            for a in range(K):
                u=r['placement'][str(a)];assert u in table[a];assert table[a][u]==(ds['measured']['write'][a],ds['measured']['query'][a])
            if r['method']=='B_cost':assert independent_assignment_bound(table,m['workload'],K)==num
            for e,stream in zip(r['episodes'],m['episodes']):
                obs=observations(e,stream,K);gate=gate and obs['gate'] and e['replay']['pass_replay'];assert obs['measured']['write']==ds['measured']['write']
                for a,lat in obs['measured']['query'].items():assert lat==ds['measured']['query'][a]
                assert e['replay']['trajectory_hash']==e['trajectory_hash'] and e['replay']['steps']==e['trajectory_steps'] and e['replay']['messages']==e['messages'];replays+=1;steps+=e['trajectory_steps']
                actual=avg([row[1] for row in e['rows'][K:]]);assert actual==e['realized_stationary_mean'];realized.append(actual)
                assert e['initialization_sum_ticks']==sum(row[1] for row in e['rows'][:K]);assert e['initialization_messages']==sum(row[4] for row in e['rows'][:K])
                q=sum(event['kind']=='Q' for event in stream[K:]);assert e['stationary_queries']==q
                queries+=obs['queries'];writes+=len(stream)-obs['queries'];ordinary_queries+=q;ordinary_writes+=64-q
                if r['method']=='D_initial':frequencies[r['rho']]['queries']+=q;frequencies[r['rho']]['writes']+=64-q
            assert len(r['episodes'])==2 and r['gate']==gate and r['J_star']==(J if gate else T_CAP)
            if r['method'].startswith('D'):assert J+1e-10>=bounds[r['context']]['J'];assert r['tree_hash']==m['tree_hash']
            r['realized_stationary_mean']=avg(realized);diagnostic_queries+=3*K+4
        else:assert r['failure'] and not r['gate'] and r['J_star']==T_CAP;r['realized_stationary_mean']=None
        if not r['gate']:gate_failures.append(dict(context=r['context'],method=r['method'],failure=r['failure']))
        # Lightweight records retained for weighting; raw rows remain on disk.
        r['ordinary_messages']=sum(e['messages']-e['initialization_messages'] for e in r['episodes'])
        r['initialization_messages']=sum(e['initialization_messages'] for e in r['episodes'])
        r['ordinary_payload_bytes']=4*r['ordinary_messages'];r['replay_messages']=sum(e['replay']['messages'] for e in r['episodes'])
        r['diagnostic_messages']=r.get('diagnostics',{}).get('messages',0)
        r['diagnostic_sum_ticks']=r.get('diagnostics',{}).get('actual_sum_ticks',0)
        r['initialization_sum_ticks']=sum(e['initialization_sum_ticks'] for e in r['episodes'])
        r['execution_cpu_seconds']=sum(e['cpu_seconds'] for e in r['episodes']);r['replay_cpu_seconds']=sum(e['replay']['cpu_seconds'] for e in r['episodes'])
        r['peak_queue']=max((e['peak_queue'] for e in r['episodes']),default=0);r['peak_serialized_state_bytes']=max((e['peak_serialized_state_bytes'] for e in r['episodes']),default=0)
        r.pop('episodes',None);r.pop('diagnostics',None);records.append(r)
    assert len(records)==2304 and all(sum(r['context']==ctx for r in records)==12 for ctx in manifests)
    lookup={(r['context'],r['method']):r for r in records};contexts=list(manifests)
    def contrasts(run,cc,comparator):
        fractions=[];ticks=[]
        for context in cc:
            a=lookup[(context,comparator)]['J_star'];b=lookup[(context,f'D_learned_{run}')]['J_star'];assert a>0
            fractions.append((a-b)/a);ticks.append(a-b)
        return dict(fraction=avg(fractions),ticks=avg(ticks),contexts=len(cc),fraction_range=[min(fractions),max(fractions)])
    per_run=[];group_table=[];setting_table=[];primary_contexts=[c for c,m in manifests.items() if m['setting'][0]==6 or m['setting'][1]==8]
    for run in range(4):
        one=dict(run=run,selection=seal['selected'][run],primary={},all_settings={},in_distribution={},group_effects=[],settings=[],placement_change_contexts=sum(lookup[(c,f'D_learned_{run}')]['placement']!=lookup[(c,'D_initial')]['placement'] for c in contexts))
        for comparator in ('D_initial','D_analytic','B_original','B_cost'):
            one['primary'][comparator]=contrasts(run,primary_contexts,comparator);one['all_settings'][comparator]=contrasts(run,contexts,comparator);one['in_distribution'][comparator]=contrasts(run,[c for c in contexts if c not in primary_contexts],comparator)
        gaps=[lookup[(c,f'D_learned_{run}')]['J_star']-bounds[c]['J'] for c in primary_contexts]
        deltas=[lookup[(c,f'D_mismatch_{run}')]['J_star']-lookup[(c,f'D_learned_{run}')]['J_star'] for c in primary_contexts]
        one['primary']['D_tree_gap']=dict(ticks=avg(gaps),fraction=avg([gap/bounds[c]['J'] for gap,c in zip(gaps,primary_contexts)]),range=[min(gaps),max(gaps)])
        one['primary']['mismatch_increase']=dict(ticks=avg(deltas),fraction=avg([delta/lookup[(c,f'D_learned_{run}')]['J_star'] for delta,c in zip(deltas,primary_contexts)]),increased=sum(d>1e-10 for d in deltas),equal=sum(abs(d)<=1e-10 for d in deltas),decreased=sum(d<-1e-10 for d in deltas))
        one['cue_changed_host_pairs']=sum(lookup[(c,f'D_learned_{run}')]['placement']!=lookup[(c.replace('rho0.25','rho0.75'),f'D_learned_{run}')]['placement'] for c in contexts if c.endswith('rho0.25'))
        one['intact_vs_mismatch_changed_contexts']=sum(lookup[(c,f'D_learned_{run}')]['placement']!=lookup[(c,f'D_mismatch_{run}')]['placement'] for c in contexts)
        for group in range(12):
            cc=[c for c in primary_contexts if manifests[c]['group']==group];row=dict(run=run,group=group,**contrasts(run,cc,'D_initial'));one['group_effects'].append(row);group_table.append(row)
        for setting in cfg['test']['settings']:
            for rho in (.25,.75):
                cc=[c for c in contexts if manifests[c]['setting']==setting and manifests[c]['workload']['rho']==rho]
                row=dict(run=run,setting=setting,rho=rho,contrasts={comp:contrasts(run,cc,comp) for comp in ('D_initial','D_analytic','B_original','B_cost')});one['settings'].append(row);setting_table.append(row)
        per_run.append(one)
    methods={}
    metrics=('active_execution_cells','wire_length','routing_bytes','persistent_logical_bytes','structure_bytes','peak_occupied','development_messages','development_bytes','development_ticks','peak_local_state_bytes','peak_total_development_state_bytes','global_builder_work','construction_cpu_seconds','construction_wall_seconds')
    for method in cfg['comparators']:
        rr=[r for r in records if r['method']==method];methods[method]=dict(conditions=len(rr),valid=sum(r['gate'] for r in rr),mean_J_star=avg([r['J_star'] for r in rr]),mean_realized_stationary=avg([r['realized_stationary_mean'] for r in rr if r['realized_stationary_mean'] is not None]),metrics={key:dict(mean=avg([r['cost'][key] for r in rr if r.get('cost')]),sum=sum(r['cost'][key] for r in rr if r.get('cost'))) for key in metrics},
            totals={key:sum(r[key] for r in rr) for key in ('ordinary_messages','ordinary_payload_bytes','initialization_messages','replay_messages','diagnostic_messages','initialization_sum_ticks','diagnostic_sum_ticks','execution_cpu_seconds','replay_cpu_seconds')},peak_queue=max(r['peak_queue'] for r in rr),peak_serialized_state_bytes=max(r['peak_serialized_state_bytes'] for r in rr))
    strongest=None
    for run in range(4):
        for c in contexts:
            learned=lookup[(c,f'D_learned_{run}')];control=lookup[(c,'B_cost')];gap=learned['J_star']-control['J_star']
            if strongest is None or gap>strongest['tick_gap']:strongest=dict(run=run,context=c,setting=manifests[c]['setting'],rho=manifests[c]['workload']['rho'],tick_gap=gap,learned_J=learned['J_star'],B_cost_J=control['J_star'],D_analytic_J=lookup[(c,'D_analytic')]['J_star'],initial_J=lookup[(c,'D_initial')]['J_star'],D_tree_optimum=bounds[c]['J'],learned_placement=learned.get('placement'),control_placement=control.get('placement'))
    mean_effects=[r['primary']['D_initial']['fraction'] for r in per_run]
    saved=dict(complete=True,observed_conditions=2304,scheduled_conditions=2304,independent_searches=4,host_groups=12,primary_contexts_per_policy=len(primary_contexts),per_run=per_run,methods=methods,primary_search_spread=[min(mean_effects),max(mean_effects)],primary_search_mean=avg(mean_effects),groups=group_table,settings=setting_table,strongest_counterexample=strongest,
               gate_failures=gate_failures,ordinary_queries=ordinary_queries,ordinary_writes=ordinary_writes,initialization_writes=writes-ordinary_writes,diagnostic_queries=diagnostic_queries,ordinary_replay_episodes=replays,trajectory_steps=steps,workload_frequencies={str(rho):dict(count,observed_query_fraction=count['queries']/(count['queries']+count['writes'])) for rho,count in frequencies.items()},candidate_evaluations=512,training_context_placements=candidate_contexts,validation_context_placements=validation_contexts,candidate_construction_failures=candidate_failures)
    dump(ROOT/'analysis.json',saved)
    with (ROOT/'group-effects.csv').open('w',newline='') as f:
        w=csv.DictWriter(f,fieldnames=['run','group','fraction','ticks','contexts','fraction_range']);w.writeheader();w.writerows(group_table)
    verified=dict(passed=True,label='independent saved-data AUTHOR reaggregation; no experimental imports; not independent reproduction',targets_and_gates_recomputed=True,expected_J_from_observed_completion_times=True,independent_tree_reconstruction_and_subset_DP_optima_verified=True,selection_and_lineage_verified=True,frozen_source_and_selection_record_hashes_match=True,ordinary_events=ordinary_queries+ordinary_writes,initialization_writes=writes-ordinary_writes,diagnostic_queries=diagnostic_queries,ordinary_replay_records=replays,recorded_trajectory_steps=steps,training_contexts=candidate_contexts,validation_contexts=validation_contexts,
                  audit_cpu_seconds=time.process_time()-cpu,audit_wall_seconds=time.perf_counter()-wall)
    dump(ROOT/'verification.json',verified);print(canonical(verified));print('primary reductions',[r['primary']['D_initial']['fraction'] for r in per_run])
if __name__=='__main__':main()