"""Recompute the historical evaluation from the public, sanitized case table.
Run from the repository root: python3 public/research/parse-brand-classification-validation/study.py
These are historical, model-labelled cases, not a current production accuracy estimate.
"""
import csv,hashlib,json,collections
from pathlib import Path
base=Path('public/research/parse-brand-classification-validation')
rows=list(csv.DictReader((base/'cases.csv').open()))
assert len({r['case'] for r in rows})==len(rows)
assert not ({r['answer_group'] for r in rows if r['split']=='cal'} & {r['answer_group'] for r in rows if r['split']=='test'})
metrics=[]
def add(group,key,n,d=None):metrics.append(dict(group=group,metric=key,numerator=n,denominator='' if d is None else d,value=n if d is None else n/d))
def score(group,rs,label='reference_label'):
 tp=sum(r['recommendation_counted']=='1' and r[label] in ('recommended','alternative') for r in rs)
 fp=sum(r['recommendation_counted']=='1' and r[label] not in ('recommended','alternative') for r in rs)
 fn=sum(r['recommendation_counted']=='0' and r[label] in ('recommended','alternative') for r in rs)
 add(group,'cases',len(rs));add(group,'correct',len(rs)-fp-fn,len(rs));add(group,'false_positive',fp);add(group,'false_negative',fn)
 if tp+fp:add(group,'precision',tp,tp+fp)
 if tp+fn:add(group,'recall',tp,tp+fn)
 add(group,'net_overcount',fp-fn,len(rs))
for split in ['cal','test']:
 rs=[r for r in rows if r['split']==split];score(split,rs)
 if split=='test':
  score('test_original_labels',rs,'original_label')
  score('test_unambiguous',[r for r in rs if r['ambiguous']=='0'])
  for g in sorted({r['sample_group'] for r in rs}):score('test_'+g,[r for r in rs if r['sample_group']==g])
  for c in sorted({r['reference_label'] for r in rs}):
   cr=[r for r in rs if r['reference_label']==c]
   add('class_'+c,'treatment_correct',sum(r['predicted_treatment']==c for r in cr),len(cr))
   add('class_'+c,'counted_as_recommendation',sum(r['recommendation_counted']=='1' for r in cr),len(cr))
add('corpus','cases',len(rows));add('corpus','answers',len({r['answer_group'] for r in rows}));add('corpus','label_corrections',sum(r['original_label']!=r['reference_label'] for r in rows))
with (base/'dataset.csv').open('w') as f:w=csv.DictWriter(f,fieldnames=list(metrics[0]),lineterminator="\n");w.writeheader();w.writerows(metrics)
print(json.dumps({'input_sha256':hashlib.sha256((base/'cases.csv').read_bytes()).hexdigest(),'metrics':metrics},indent=2))