"""Metadata census and hand-verifiable numeric-scorer probes. No model/API calls."""
import json,math,statistics,itertools
from collections import Counter,defaultdict
from pathlib import Path
from numeric_scorer import _score_numeric
from plot_utils import svg,text,line,rect,circle
BASE=Path(__file__).resolve().parent
def analyze():
d=json.loads((BASE/'metadata.json').read_text());k=d['geoktools'];e=d['geoexpertqueries']
assert len(k)==85 and len(e)==25
for rows in [k,e]:assert len({r['query_id'] for r in rows})==len(rows)
assert all(r['raw_answer_matches'] for r in k)
assert all(r['answer_slots']==r['numeric_slots']+r['string_slots'] for r in k)
kc=Counter(str(r['k_level']) for r in k);tc=Counter(r['template_id'] for r in k)
assert kc=={'1':10,'2':50,'3':15,'4':10} and len(tc)==19
canonical=Counter('k2_places_dc_superlative_substate' if r['is_extra_k2_state_variant'] else r['template_id'] for r in k)
assert len(canonical)==18 and sum(r['is_extra_k2_state_variant'] for r in k)==1
# Exact worst-case composition bound for rates in [0,1], checked over all vertices.
dev=[kc[str(i)]/85 for i in range(1,5)];test=[d['heldout_k_counts'][str(i)]/210 for i in range(1,5)]
tv=sum(abs(a-b) for a,b in zip(dev,test))/2
extrema=max(abs(sum((a-b)*v for a,b,v in zip(dev,test,corner))) for corner in itertools.product([0,1],repeat=4))
assert math.isclose(tv,extrema)
example_rates=[.9,.8,.6,.4]
example_dev=sum(w*a for w,a in zip(dev,example_rates))
example_test=sum(w*a for w,a in zip(test,example_rates))
assert math.isclose(example_dev,62/85) and math.isclose(example_test,149/210)
cases=[('inside boundary',100,110,1.0),('outside boundary',100,110.01,0.8999),('zero exact',0,0,1.0),('zero near miss',0,0.01,0.0),('numeric JSON value',1000,1000,1.0),('exponent in string',1000,'1e3',0.001),('comma in string',1000,'1,000',0.001),('negative boundary',-100,-110,1.0)]
probes=[]
for name,gold,pred,expected in cases:
score=_score_numeric(gold,pred,0.1);assert math.isclose(score,expected,abs_tol=1e-12)
probes.append({'case':name,'gold':gold,'prediction':pred,'score':score})
domains={dom:dict(Counter(r['solvability'] for r in e if r['domain_cluster']==dom)) for dom in d['heldout_domain_counts']}
out={'k_counts':dict(sorted(kc.items())),'template_counts':dict(tc),'canonical_template_count':len(canonical),'canonical_template_counts':dict(canonical),'snapshot_dates':dict(Counter(r['snapshot_date'] for r in k)),'answer_slot_counts':{c:sum(r[c] for r in k) for c in ['answer_slots','numeric_slots','string_slots','zero_numeric_slots','negative_numeric_slots']},'domain_solvability':domains,'solvability':dict(Counter(r['solvability'] for r in e)),'workflow_list_items':{'min':min(r['workflow_list_items'] for r in e),'median':statistics.median(r['workflow_list_items'] for r in e),'max':max(r['workflow_list_items'] for r in e)},'domain_weights_dev':{dom:sum(v.values())/25 for dom,v in domains.items()},'domain_weights_test':{dom:n/76 for dom,n in d['heldout_domain_counts'].items()},'k_weights_dev':dev,'k_weights_test':test,'k_composition_max_pp':100*tv,'illustrative_profile':{'rates':example_rates,'dev_percent':100*example_dev,'heldout_mixture_percent':100*example_test,'gap_pp':100*(example_dev-example_test)},'numeric_probes':probes,'registry_status':'All 110 dev IDs unique; 85 parsed/raw answers agree; no agent scores computed.'}
(BASE/'results.json').write_text(json.dumps(out,indent=2)+'\n');return out
def draw(r):
body=''
for i,k in enumerate(['1','2','3','4']):
y=130+i*76;body+=text(40,y+19,'K = '+k)
for offset,weight,color,label in [(0,r['k_weights_dev'][i],'#3d5744','dev'),(27,r['k_weights_test'][i],'#b38a69','held-out')]:
body+=rect(140,y+offset,weight*1050,21,color)+text(150+weight*1050,y+offset+16,f'{weight*100:.1f}%')
body+=rect(140,457,14,14,'#3d5744')+text(161,469,'Dev: 85 released queries')+rect(460,457,14,14,'#b38a69')+text(481,469,'Held-out: 210 (counts only)')
(BASE/'composition.svg').write_text(svg('The public split has a different mixture','K counts distinct tool families, not executed calls. No per-agent performance appears here.',body,515))
body='';labels={'Natural Hazards & Crisis Resilience':'Hazards & resilience','Public Health':'Public health','Site Selection & Urban Planning':'Site selection & planning','Operational & Commercial Utilities':'Operational & commercial'}
for i,(dom,counts) in enumerate(r['domain_solvability'].items()):
y=125+i*77;body+=text(35,y+22,labels[dom]);x=290
for key,color in [('solvable','#3d5744'),('beyond_current_toolset','#c7b79f')]:
n=counts.get(key,0)
if n:body+=rect(x,y,n*42,34,color)+text(x+n*21,y+23,n,anchor='middle',fill='#ffffff' if key=='solvable' else '#292d29');x+=n*42
body+=rect(290,460,14,14,'#3d5744')+text(312,472,'Solvable (10)')+rect(500,460,14,14,'#c7b79f')+text(522,472,'Beyond current toolset (15)')
(BASE/'solvability.svg').write_text(svg('Most released expert tasks exceed the evaluated toolset','25 expert queries. Labels concern the source agent’s toolset, not universal task impossibility.',body,520))
body=''
for v in [0,.25,.5,.75,1]:
y=410-v*270;body+=line(100,y,820,y)+text(85,y+4,f'{v:g}',anchor='end')
pts=[]
for err in [0, .1, .1000000001] + [i/100 for i in range(11,101)]:
score=_score_numeric(100,100+100*err,.1);pts.append(f'{100+err*700},{410-score*270}')
body+=''
body+=line(170,120,170,415,'#b37951',1)+circle(170,140,4,'#3d5744')+''+text(184,110,'10% boundary: 1.0 → just below 0.9')
for v in [0,.1,.25,.5,.75,1]:body+=text(100+v*700,440,f'{v*100:g}%',anchor='middle')
body+=text(455,482,'Absolute relative error for a nonzero numeric gold value',anchor='middle',cls='sub')
(BASE/'numeric-tolerance.svg').write_text(svg('The offline numeric rule has a tolerance cliff','Hand-authored probes of the released _score_numeric function; not the report’s LLM judge.',body,515))
if __name__=='__main__':
r=analyze();draw(r);print(json.dumps({k:v for k,v in r.items() if k not in ['template_counts','canonical_template_counts','numeric_probes']},indent=2));print('Assertions passed. Wrote results.json and three SVG figures.')