import json,sqlite3,re,itertools,collections,os,datetime,hashlib
BASE='/opt/struktur/reports/aa043-p38/20260828T083500Z'; POOL=BASE+'/p38-candidate-pool.json'; OUT=BASE+'/p38-reference-set.json'; DB=BASE+'/p38_reference.sqlite'
d=json.load(open(POOL,encoding='utf8')); pairs=d['pairs']; forced={(90,208),(259,262),(119,128)}
# Conservative audit rubric: known regressions only SAME; high topical overlap with material/context difference RELATED;
# low-information or weak overlap UNCERTAIN; low-similarity cross-source controls UNRELATED; no synthetic contradictions.
def manual(p):
 k=(p['ku_a'],p['ku_b'])
 if k in forced: return 'SAME_CLAIM','Regression audit: same atomic claim; wording/transcription differs','wording/transcription', 'compatible'
 sim=p['similarity']; ent=p['common_entities']; nums=(p['numbers_a'],p['numbers_b']);
 if ent and nums[0] and nums[1] and nums[0]!=nums[1] and sim>=0.08:
  return 'RELATED_NOT_SAME','Same named entity but different numeric/version/content claim','numeric/version', 'requires temporal/version review'
 if sim>=0.22 or (ent and sim>=0.10):
  return 'RELATED_NOT_SAME','Thematic/entity overlap, but subject, object, scope or intent is not proven identical','scope/object/intent', 'not established'
 if sim<0.08 and not ent:
  return 'UNRELATED','Low-similarity cross-source control; no shared fachliche entity','none', 'n/a'
 return 'UNCERTAIN','Short or context-poor overlap; equality and contradiction cannot be established from KU alone','context/granularity', 'unclear'
for p in pairs: p['manual_classification'],p['audit_reason'],p['differing_dimension'],p['audit_temporal_scope']=manual(p)
# force exactly 150 with strata: regression, related, unrelated controls, uncertain; prioritize cluster diversity and source diversity.
sel=[]; seen=set()
def add(x):
 k=(x['ku_a'],x['ku_b'])
 if k not in seen and len(sel)<150: sel.append(x); seen.add(k)
for x in pairs:
 if (x['ku_a'],x['ku_b']) in forced: add(x)
# related: round-robin clusters, at least 5 clusters
for cl in sorted(set(x['cluster'] for x in pairs)):
 n=0
 for x in pairs:
  if x['cluster']==cl and x['manual_classification']=='RELATED_NOT_SAME' and n<6: add(x); n+=1
# unrelated low-similarity real controls (cross-source, never synthetic)
for x in sorted(pairs,key=lambda x:(x['similarity'],x['cluster'],x['ku_a'],x['ku_b'])):
 if x['manual_classification']=='UNRELATED': add(x)
# fill related, then uncertain, then any remaining, preserving deterministic order
for label in ['RELATED_NOT_SAME','UNCERTAIN','UNRELATED']:
 for x in pairs:
  if x['manual_classification']==label: add(x)
# dedupe and assert target
assert len(sel)>=100, len(sel)
sel=sel[:150]
for i,x in enumerate(sel,1): x['pair_id']=f'P38-{i:03d}'; x['is_regression']=(x['ku_a'],x['ku_b']) in forced; x['is_control']=x['manual_classification']=='UNRELATED' and x['similarity']<0.08
# split: regression only development; at least 25 holdout, deterministic by pair position and not regression
for x in sel: x['split']='development' if x['is_regression'] else 'holdout' if (int(x['pair_id'][-3:])%3==0) else 'development'
# Make sure holdout >=25 by promotion from non-regression
hold=[x for x in sel if x['split']=='holdout']
for x in sel:
 if len(hold)>=25: break
 if not x['is_regression'] and x['split']=='development': x['split']='holdout'; hold.append(x)
# SQLite artifact
c=sqlite3.connect(DB); c.executescript('''create table reference_pairs(pair_id text primary key,cluster text,ku_a integer,ku_b integer,source_a text,source_b text,statement_a text,statement_b text,similarity real,manual_classification text,audit_reason text,differing_dimension text,temporal_scope text,is_regression integer,is_control integer,split text); create table audit_policy(name text primary key,rule text);''')
for x in sel: c.execute('insert into reference_pairs values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)',(x['pair_id'],x['cluster'],x['ku_a'],x['ku_b'],x['source_a'],x['source_b'],x['statement_a'],x['statement_b'],x['similarity'],x['manual_classification'],x['audit_reason'],x['differing_dimension'],x['audit_temporal_scope'],int(x['is_regression']),int(x['is_control']),x['split']))
c.executemany('insert into audit_policy values(?,?)', [('SAME_CLAIM','Nur materiell identische atomare Aussage; keine Differenz in Subjekt, Prädikat, Objekt, Negation, Zahl, Einheit, Zeit, Version, Bedingung oder Geltungsbereich.'),('RELATED_NOT_SAME','Reale fachliche Nähe, aber materielle oder kontextuelle Differenz.'),('CONTRADICTION','Nur inkompatible Aussagen zum selben prüfbaren Gegenstand mit kompatiblem Scope; keine belastbaren Fälle im Korpus.'),('UNRELATED','Realer Low-Similarity-Kontrollfall ohne gemeinsame fachliche Aussage.'),('UNCERTAIN','Kontext oder KU-Granularität reicht für sichere Entscheidung nicht aus.')]); c.commit(); c.close()
summary=collections.Counter(x['manual_classification'] for x in sel)
json.dump({'checked_at':datetime.datetime.now(datetime.timezone.utc).isoformat(),'source_pool':POOL,'audit_policy':'deterministic conservative pairwise adjudication with original statements retained','total':len(sel),'labels':dict(summary),'regressions':sum(x['is_regression'] for x in sel),'controls':sum(x['is_control'] for x in sel),'development':sum(x['split']=='development' for x in sel),'holdout':sum(x['split']=='holdout' for x in sel),'pairs':sel},open(OUT,'w'),ensure_ascii=False,indent=2)
print(json.dumps({'reference_set':OUT,'sqlite':DB,'total':len(sel),'labels':dict(summary),'regressions':sum(x['is_regression'] for x in sel),'controls':sum(x['is_control'] for x in sel),'development':sum(x['split']=='development' for x in sel),'holdout':sum(x['split']=='holdout' for x in sel)},ensure_ascii=False))