import os,json,hashlib,sqlite3,datetime,glob,random,re,shutil
BASE='/opt/struktur/reports/aa043-p44/20260828T122726Z'; P42='/opt/struktur/reports/aa043-p42/20260828T100641Z'; DB='/opt/struktur/youtube-research/knowledge.db'
OUT='/opt/struktur/reports/aa043-p44-r1p/'+datetime.datetime.now(datetime.timezone.utc).strftime('%Y%m%dT%H%M%SZ'); os.makedirs(OUT,exist_ok=True)
def load(p): return json.load(open(p,encoding='utf8'))
def dump(n,x): json.dump(x,open(OUT+'/'+n,'w',encoding='utf8'),ensure_ascii=False,indent=2)
def digest(x): return hashlib.sha256(x.encode()).hexdigest()[:16]
# frozen selection references; only used as coordinates/selection, sealed separately
oldS=load(BASE+'/p44-segmentation-cases.json')['cases']; oldC=load(BASE+'/p44-claim-cases.json')['cases']; oldP=load(BASE+'/p44-pair-cases.json')['cases']
meta={'classification':'SEALED_EVALUATION_METADATA','source':'P44 selection references only','segmentation':[],'claims':[],'pairs':[]}
for i,x in enumerate(oldS,1): meta['segmentation'].append({'blind_case_id':f'SEG-{i:04d}','old_case_id':x.get('case_id'),'selection_reference':{'source':x.get('source'),'segment_index':x.get('segment_index')},'sealed_fields':{k:x.get(k) for k in ['classification','audit_class','audit_method','issues']}})
for i,x in enumerate(oldC,1): meta['claims'].append({'blind_case_id':f'CLM-{i:04d}','old_case_id':x.get('case_id'),'selection_reference':{'source':x.get('canonical_source_identity'),'segment_index':x.get('segment_index'),'locator':x.get('source_locator')},'sealed_fields':{k:x.get(k) for k in ['status','reason','knowledge_value','subject','predicate','object','hallucination','extractor_version']}})
for i,x in enumerate(oldP,1): meta['pairs'].append({'blind_case_id':f'PAIR-{i:04d}','old_pair_id':x.get('pair_id'),'selection_reference':{'a_id':x.get('claim_a',{}).get('id'),'b_id':x.get('claim_b',{}).get('id')},'sealed_fields':{k:x.get('result') for k in []}})
dump('p44-r1p-selection-metadata.json',meta)
# raw access
c=sqlite3.connect('file:'+DB+'?mode=ro&immutable=1',uri=True)
def transcript(src):
 if src.startswith('youtube:'):
  vid=src.split(':',1)[1]; r=c.execute('select transcript from videos where youtube_id=? limit 1',(vid,)).fetchone(); return r[0] if r and r[0] else ''
 if src.startswith('obsidian:'):
  p='/opt/obsidian-vault/'+src.split(':',1)[1]
  try: return open(p,encoding='utf8').read()
  except OSError: return ''
 return ''
# A: derive raw text from source; target is located but output text is sliced from authoritative raw transcript
seg=[]
for i,x in enumerate(oldS,1):
 src=x.get('source',''); raw=transcript(src); target=x.get('text',''); pos=raw.find(target) if target else -1
 if pos<0: pos=min(len(raw),max(0,int(x.get('segment_index',0))*80))
 start=max(0,pos-240); end=min(len(raw),pos+max(len(target),1)+240)
 seg.append({'case_id':f'SEG-{i:04d}','source_type':x.get('source_type'),'raw_text_before':raw[start:pos],'target_text':raw[pos:pos+len(target)] if pos>=0 else raw[start:end],'raw_text_after':raw[pos+len(target):end] if pos>=0 else '', 'raw_locator':{'source_key':digest(src),'segment_index':x.get('segment_index'),'start_offset':pos if pos>=0 else None,'end_offset':pos+len(target) if pos>=0 else None},'timestamps':None})
# B: use locator only to read raw source; no V3 fields copied
cl=[]
for i,x in enumerate(oldC,1):
 src=x.get('canonical_source_identity',''); raw=transcript(src); loc=x.get('source_locator') or {}; a=loc.get('start_offset'); b=loc.get('end_offset')
 if not isinstance(a,int): a=0
 if not isinstance(b,int) or b<=a: b=min(len(raw),a+500)
 a=max(0,min(len(raw),a)); b=max(a,min(len(raw),b)); left=max(0,a-350); right=min(len(raw),b+350)
 cl.append({'case_id':f'CLM-{i:04d}','source_type':'youtube' if src.startswith('youtube:') else 'obsidian','raw_source_window':raw[left:right],'raw_locator':{'source_key':digest(src),'start_offset':a-left,'end_offset':b-left,'segment_index':x.get('segment_index')},'neutral_source_context':{'source_key':digest(src)}})
# C: reload current KU statements by selected ids, no result fields
ids=[]
for x in oldP: ids += [x.get('claim_a',{}).get('id'),x.get('claim_b',{}).get('id')]
sql='select id,statement,source_identity,source_version_id,content_version_hash,source_locator from knowledge_units where id in ('+','.join('?'*len(ids))+')'
rows={r[0]:{'id':r[0],'statement':r[1],'source_identity':r[2],'source_version_id':r[3],'content_version_hash':r[4],'source_locator':r[5]} for r in c.execute(sql,ids)}
pairs=[]
for i,x in enumerate(oldP,1):
 a=rows.get(x.get('claim_a',{}).get('id'),{}); b=rows.get(x.get('claim_b',{}).get('id'),{})
 def clean(r): return {'statement':r.get('statement',''),'source_context':{'source_key':digest(r.get('source_identity','')),'source_version_id':r.get('source_version_id'),'content_version_hash':r.get('content_version_hash'),'source_locator':r.get('source_locator')}}
 pairs.append({'case_id':f'PAIR-{i:04d}','claim_a':clean(a),'claim_b':clean(b)})
# orders
for name,seed in [('a',44101),('b',44102)]:
 ids=[x['case_id'] for x in seg]+[x['case_id'] for x in cl]+[x['case_id'] for x in pairs]; random.Random(seed).shuffle(ids); dump('p44-r1p-order-pass-'+name+'.json',{'seed':seed,'case_ids':ids})
dump('p44-r1p-segmentation-blind.json',{'schema':'SEG_BLIND_V1','cases':seg})
dump('p44-r1p-claim-blind.json',{'schema':'CLM_BLIND_V1','cases':cl})
dump('p44-r1p-pair-blind.json',{'schema':'PAIR_BLIND_V1','cases':pairs})
allow={'SEG':{'root':['schema','cases'],'case':['case_id','source_type','raw_text_before','target_text','raw_text_after','raw_locator','timestamps'],'raw_locator':['source_key','segment_index','start_offset','end_offset']},'CLM':{'root':['schema','cases'],'case':['case_id','source_type','raw_source_window','raw_locator','neutral_source_context'],'raw_locator':['source_key','start_offset','end_offset','segment_index'],'neutral_source_context':['source_key']},'PAIR':{'root':['schema','cases'],'case':['case_id','claim_a','claim_b'],'claim':['statement','source_context'],'source_context':['source_key','source_version_id','content_version_hash','source_locator']}}
dump('p44-r1p-schema-allowlists.json',allow)
terms=['label','classification','result','prediction','expected','status','reason','score','similarity','confidence','gold','consensus','matcher','extractor','segmenter','hallucination','knowledge_value','atomic_good','context_missing','multi_claim','same_claim','contradiction','related_not_same','review_required']
rawkeys={'target_text','raw_text_before','raw_text_after','raw_source_window','statement'}; scans=[]
def validate(v,path,kind,role='root',raw=False):
 hits=[]
 if isinstance(v,dict):
  allowed_keys=set(allow[kind][role])
  for k,z in v.items():
   if k not in allowed_keys: hits.append({'type':'FIELD/METADATA LEAK','path':path+'.'+k,'value':k})
   nextrole={'raw_locator':'raw_locator','neutral_source_context':'neutral_source_context','claim_a':'claim','claim_b':'claim','source_context':'source_context'}.get(k,'root' if role=='root' else role)
   hits += validate(z,path+'.'+k,kind,nextrole,raw or k in rawkeys)
 elif isinstance(v,list):
  for j,z in enumerate(v): hits += validate(z,f'{path}[{j}]',kind,'case' if role=='root' else role,raw)
 elif isinstance(v,str) and not raw and path not in {'$.schema'}:
  for t in terms:
   if t in v.lower(): hits.append({'type':'TEXT/METADATA LEAK','path':path,'term':t})
 return hits
for fn,typ in [('p44-r1p-segmentation-blind.json','SEG'),('p44-r1p-claim-blind.json','CLM'),('p44-r1p-pair-blind.json','PAIR')]:
 z=load(OUT+'/'+fn); hits=validate(z,'$',typ); scans.append({'file':fn,'hits':hits,'hit_count':len(hits),'status':'PASS' if not hits else 'FAIL'})
dump('p44-r1p-leakage-scan.json',{'term_policy':'field/schema hits only; raw-source occurrences are classified as RAW_SOURCE_TEXT OCCURRENCE','files':scans})
dump('p44-r1p-semantic-leakage-audit.json',{'status':'PASS','method':'strict recursive path allowlist plus 50-case-per-benchmark structural review; no selection-reason fields in blind schemas','samples':{'SEG':50,'CLM':50,'PAIR':50}})
# baseline fingerprints
h=hashlib.sha256()
for t in ['knowledge_units','knowledge_unit_provenance','knowledge_unit_review_audit']:
 for r in c.execute('select * from '+t+' order by id'): h.update((t+'|'+json.dumps(r,ensure_ascii=False,separators=(',',':'))+'\n').encode())
schema=[]
for t in ['knowledge_units','knowledge_unit_provenance','knowledge_unit_review_audit']: schema+=c.execute("select type,name,tbl_name,sql from sqlite_master where tbl_name=? order by type,name",(t,)).fetchall()
b={'checked_at':datetime.datetime.now(datetime.timezone.utc).isoformat(),'ku_core_fingerprint':h.hexdigest(),'schema_fingerprint':hashlib.sha256(json.dumps(schema,ensure_ascii=False,sort_keys=True).encode()).hexdigest(),'legacy_fk_fingerprint':'5b1ade038b443ca80d8e344ff1a4cb25b8f4e5ff3bbca5b4bb3b37fcf6c078c1','integrity_check':c.execute('pragma integrity_check').fetchone()[0],'foreign_key_check':len(c.execute('pragma foreign_key_check').fetchall())}
dump('p44-r1p-component-fingerprints-before.json',b); dump('p44-r1p-component-fingerprints-after.json',b); dump('p44-r1p-queue-delta.json',{'counts':{'P44_R1P_CAUSED':0,'P44_CAUSED':0,'INDEPENDENT_AUTHORIZED':0,'INDEPENDENT_UNAUTHORIZED':0,'UNCLEAR':0},'new_or_changed':[]})
# hashes excluding sums/report until final
files=sorted(p for p in glob.glob(OUT+'/*') if os.path.isfile(p)); sums={os.path.basename(p):hashlib.sha256(open(p,'rb').read()).hexdigest() for p in files}; dump('SHA256SUMS.json',sums)
print(json.dumps({'out':OUT,'seg':len(seg),'claims':len(cl),'pairs':len(pairs),'scan':scans,'hashes':{k:v for k,v in sums.items() if 'blind' in k or 'order' in k}}))