#!/usr/bin/env python3
import importlib.util,sqlite3,json,hashlib,os,sys
from pathlib import Path
from datetime import datetime,timezone
DB='/opt/struktur/youtube-research/knowledge.db'; VAULT=Path('/opt/obsidian-vault'); OUT=sys.argv[1]
P34={'youtube:FVuqEATzMWI','youtube:qmPAFHeiU1A','youtube:z-C_q4LzPlA','youtube:3yqQ664R1hg','obsidian:Systemstruktur/SYSTEM-REFERENZ.md','obsidian:Systemstruktur/Review-Routing-Decision-Units.md','obsidian:Systemstruktur/Knowledge-Pipeline-Review-Unter-100-2026-07-22.md','obsidian:05-Ressourcen/Hermes-Sessions-2026-07/TranscriptBackfillWorker-untersuchen.md','youtube:fxc4yds-DUs','youtube:ZyNdWhKqlEs'}
spec=importlib.util.spec_from_file_location('p33','/opt/struktur/AA-043-P33-extractor.py'); m=importlib.util.module_from_spec(spec); spec.loader.exec_module(m)
c=sqlite3.connect('file:'+DB+'?mode=ro&immutable=1',uri=True); c.row_factory=sqlite3.Row
processed={r[0] for r in c.execute('select distinct source_identity from knowledge_units where legacy_placeholder=0 and source_identity is not null')}; legacy={r[0] for r in c.execute('select distinct source_identity from knowledge_units where legacy_placeholder=1 and source_identity is not null')}
def words(t): return len(m.words(t))
def pick(items,n):
 items=sorted(items,key=lambda x:(x['word_count'],x['source_id']))
 bins=[items[i*10:(i+1)*10] for i in range(10)] if len(items)>=n else []
 out=[]
 if bins:
  for j in range(3): out.extend(b[j] for b in bins if j<len(b))
 else: out=items[:n]
 return out[:n]
ys=[]
for r in m.youtube_rows():
 sid='youtube:'+r['youtube_id']; st=r.get('transcript_status')
 if sid in P34 or sid in processed or sid in legacy or not r.get('transcript') or words(r['transcript'])<8 or st in ('unavailable','rejected','failed'): continue
 s=m.canonical_youtube(r,'p35_youtube'); s['word_count']=words(s['text']); s['text_bytes']=len(s['text'].encode()); s['prior_ku_count']=0; ys.append(s)
obs=[]
for p in sorted(VAULT.rglob('*.md')):
 rel=p.relative_to(VAULT).as_posix(); sid='obsidian:'+rel
 if sid in P34 or sid in processed or sid in legacy or rel.startswith('YouTube-Research/') or any(x in rel for x in ['_hermes-session-test','TEMPLATE.md','_deleted_backup/','05_SESSIONS/']): continue
 try: b=p.read_bytes(); t=b.decode('utf-8',errors='replace')
 except: continue
 if words(t)<8: continue
 obs.append({'source_id':sid,'source_type':'obsidian','youtube_id':None,'title':rel,'text':t,'content_version':hashlib.sha256(b).hexdigest(),'raw_source_hash':hashlib.sha256(b).hexdigest(),'obsidian_path':rel,'artifact_exists':True,'transcript_status':None,'selection_kind':'p35_native_obsidian','source_db_video_id':None,'word_count':words(t),'text_bytes':len(b),'prior_ku_count':0,'source_versions':[]})
sel=pick(ys,30)+pick(obs,20)
if len(sel)!=50 or len({x['source_id'] for x in sel})!=50: raise SystemExit('selection_failed '+str((len(ys),len(obs),len(sel))))
for i,s in enumerate(sel): s['batch']=i//10+1; s['batch_index']=i%10+1; s.pop('text',None); s['source_versions']=s.get('source_versions',[])
out={'selection_version':'p35-v1','selected_at':datetime.now(timezone.utc).isoformat(),'selection_rule':'deterministic source-id tie-break with round-robin word-count strata; P34/legacy/processed/unusable excluded','source_counts':{'youtube':30,'native_obsidian':20,'total':50},'excluded_p34':sorted(P34),'processed_before_selection':sorted(processed),'legacy_sources_before_selection':sorted(legacy),'sources':sel,'batches':[{ 'batch':i,'source_ids':[x['source_id'] for x in sel if x['batch']==i]} for i in range(1,6)]}
Path(OUT).parent.mkdir(parents=True,exist_ok=True); Path(OUT).write_text(json.dumps(out,ensure_ascii=False,indent=2,sort_keys=True)); print(json.dumps({'output':OUT,'youtube_candidates':len(ys),'obsidian_candidates':len(obs),'selected':len(sel),'batches':[len(x['source_ids']) for x in out['batches']]}))