
    )j}4                         d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlmZm	Z	 d dl
mZ dZ ed      Zh dZdZdZd	 Zd
 Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Z e!dk(  r e         yy)    N)datetimetimezone)Pathz+/opt/struktur/youtube-research/knowledge.dbz/opt/obsidian-vault>   
   BESTÄTIGTUNSICHER	GESICHERT	PLAUSIBEL   WIDERSPRÜCHLICHzp33-ku-extraction-prompt-v1u}  Du extrahierst Knowledge Units aus genau einem übergebenen kanonischen Content Record.

Regeln:
- Verwende ausschließlich Informationen aus dem übergebenen Quelltext.
- Erzeuge nur kleinste, atomare, eigenständig prüfbare fachliche Aussagen.
- Eine unabhängige Aussage pro Knowledge Unit.
- Keine Vermutungen, externen Ergänzungen oder Weltwissen.
- Keine Zusammenfassung des gesamten Dokuments.
- Führe für jede Aussage eine genaue Fundstelle (Abschnitt, Chunk oder Zeilenbereich) mit.
- Erhalte Unsicherheit, Einschränkungen und Zeitbezüge aus der Quelle.
- Glätte oder verschweige Widersprüche nicht; klassifiziere sie als WIDERSPRÜCHLICH.
- Überschriften, Stichworte und reine Entitäten ohne Aussage sind keine Knowledge Units.
- Gib strukturierte Kandidaten mit statement, source_id, source_type, content_version, source_locator, evidence_class und confidence_reason aus.
c                 H    t        j                  |       j                         S N)hashlibsha256	hexdigest)bs    %/opt/struktur/AA-043-P33-extractor.pyshar      s    7>>!$..00    c                 L    t        j                  dd|       j                         S )Nz\\s+ )resubstripss    r   normr      s    BFF7C+1133r   c                 .    t        j                  d|       S )Nu6   [A-Za-zÄÖÜäöüß0-9][A-Za-zÄÖÜäöüß0-9'_-]*)r   findallr   s    r   wordsr      s    RZZ Y[\]]r   c                 d    t        d| z   dz   |z   dz   t        |      z   j                               S )Nzv1\nz\n)r   r   encode)	source_idversion	statements      r   identityr$      s3    )#e+g5=YOWWYZZr   c                 @    t        | dz   |z   j                               S )N:)r   r    )r!   r"   s     r   import_identityr'      s    	C')11344r   c                  D    t        j                  dt        z   dz   d      S )Nzfile:z?mode=roT)uri)sqlite3connectKDB r   r   dbr.      s    ??7S=:54@@r   c                      t               } t        j                  | _        g }d}| j	                  |      D ]  }|j                  t        |              | j                          |S )Nau  SELECT v.id,v.youtube_id,v.title,v.transcript,v.transcript_hash,v.transcript_status,
                 v.transcript_source,v.transcript_updated_at,e.obsidian_path,e.content_hash,e.processed_at
          FROM videos v LEFT JOIN e2e_extractions e ON e.video_id=v.id
          WHERE v.youtube_id IS NOT NULL AND length(trim(COALESCE(v.transcript,'')))>0
          ORDER BY v.id)r.   r*   Rowrow_factoryexecuteappenddictclose)crowsqrs       r   youtube_rowsr:      sP    dA'++AM	DA
 YYq\/4;;tAw//GGId{r   c                     t               }t        j                  |_        g }d}|j	                  || | | j                  d      f      D ]  }|j                  t        |              |j                          |S )Na  SELECT sv.version_id,sv.content_hash,sv.content_version_hash,sv.import_identity,
                 sv.artifact_path,sv.created_at,r.source_identity,r.canonical_obsidian_path
          FROM source_versions sv JOIN source_processing_registry r
            ON (r.latest_observed_version_id=sv.version_id OR r.current_graph_version_id=sv.version_id)
          WHERE r.source_identity=? OR r.source_id=? OR r.canonical_obsidian_path=?
          ORDER BY sv.version_id DESC	obsidian:)	r.   r*   r0   r1   r2   removeprefixr3   r4   r5   )r!   r6   outr8   r9   s        r   source_versionsr?   "   sj    dA'++AM
C)A YYq9i1G1G1TUV 

47GGIczr   c                    | d   xs d}t        |j                  d            }| j                  d      }t        |xr t        |z  j                               }d| d   z   d| d   | j                  d      ||| j                  d	      |r|nd || j                  d
      | j                  d      || j                  d      t        d| d   z         dS )N
transcript utf-8obsidian_pathzyoutube:
youtube_idyoutube_transcripttitletranscript_hashtranscript_statustranscript_sourceidr!   source_typerE   rG   textcontent_versionraw_source_hashrD   artifact_existsrI   rJ   selection_kindsource_db_video_idr?   )r   r    getboolVAULTis_filer?   )r9   kindrN   content_hashartifactrQ   s         r   canonical_youtuber[   0   s    	
<	BDT[[)*LUU?#HBux'7&@&@&BCO"1\?2AUdefrdsEE'N$`a`e`efw`x(7HTTc !&9 :quuUhOi!quuT{Udeopqr~pe  VA	B Br   c                     | j                         }| j                  t              j                         }|j	                  dd      }d|z   dd ||t        |      t        |      |dd d dd t        d|z         dS )	NrC   replace)errorsr<   obsidianTnative_obsidianrL   )
read_bytesrelative_torV   as_posixdecoder   r?   )pathr   relrN   s       r   canonical_obsrg   ;   s~    
ooAT--e4==?ahhw_hhFi#Cjd[^#a&3q6Z]"tPTfw!%TW8XZ Zr   c                     t               } | D cg c]0  }|j                  d      rt        |d   z  j                         r/|2 }}| D cg c]1  }|j                  d      st        |d   z  j                         s0|3 }}t	        |      dk  st	        |      dk  r#t        dt	        |       dt	        |             g }||d d D cg c]  }t        |d       c}z  }|dD cg c]+  }t        |z  j                         rt        t        |z        - c}z  }t	        |      dk7  rt        d	t	        |      dz
         ||d d D cg c]  }t        |d
       c}z  }t	        |      dk7  rt        d      |D cg c]  }|d   	 }}t	        t        |            dk7  rt        d      |S c c}w c c}w c c}w c c}w c c}w c c}w )NrD         zselection impossible pure=z dual=pure_youtube_transcript)z!Systemstruktur/SYSTEM-REFERENZ.mdz/Systemstruktur/Review-Routing-Decision-Units.mdz@Systemstruktur/Knowledge-Pipeline-Review-Unter-100-2026-07-22.mdzM05-Ressourcen/Hermes-Sessions-2026-07/TranscriptBackfillWorker-untersuchen.md   z)native obsidian selection impossible got=youtube_plus_obsidian_artifact
   zexactly 10 sources not selectedr!   zsource identity collision)	r:   rT   rV   rW   lenRuntimeErrorr[   rg   set)ysr9   puredualchosenxidss          r   select_sourcesrx   B   s   ~B	ao 6E!OBT<T;];];_!	aD	a	[1551uQ=O7O6X6X6Z!	[D	[
4y{c$ik8RSVW[S\R]]cdghldmcn6o)p#pF
tBQxP! #<=PPF
 0Y  !
 !G U1W%    F 6{A~\,UVYZ`VabcVcUd*eff
dSUTUhW #CDWWF
6{Bl+LMM!'(A;(C(
3s8}b-H IIM# 
b	[ Q  X(s5   0F4 F4
F9!F9<F9F>(0GGGc                    | d   }g }d}t        |j                         d      D ]  \  }}|j                         }|rD|j                  d      s3|j                  d      s"|j                  d      s|j                  d      r^t	        j
                  d|      D ]  }t        |      j                  d	      }t        t        |            d
k  r5t	        j                  d|t        j                        r[|j                  d      s%t	        j                  d|t        j                        r|j                  |d| f          |d d S )NrN   r      #-*>z(?<=[.!?])\s+z` rl   zC^(quelle|quellen|inhalt|zusammenfassung|status|todo|hinweis)\s*[:#];u3   \b(sowohl|einerseits|andererseits|währenddessen)\bzline:   )	enumerate
splitlinesr   
startswithr   splitr   ro   r   matchIcountsearchr3   )	srcrN   r>   offsetliliner   fragsts	            r   sentence_candidatesr   W   s   	VD
C1FT__.q1 +4
**,ALL%c):all3>OSTS_S_`cSdHH-q1 	+DDz%B59~axx^`bdfdhdhiksxx}		*`bdfhfjfj kJJeB4L)*	+	+ s8Or   c                     t        j                  d| t         j                        r-t        j                  d| t         j                        rddfS ddfS y)Nu`   \b(widerspruch|widersprüchlich|unklar|nicht nachgewiesen|unsicher|möglicherweise|vermutlich)\bu   \bwidersprüchr
   r   uI   Die Quelle enthält eine explizite Unsicherheits-/Widerspruchsmarkierung.)r	   ul   Einzelquelle direkt belegt; keine unabhängige Zweitquelle im P33-Pilot nachgewiesen; daher nicht GESICHERT.)r   r   r   )stmts    r   classifyr   j   sm    	yytvz|~  }A  }A  B%'YY/@bdd%K!  ^i  i  	iQ[  ^i  i  	i Gr   c                 *    g d}t         fd|D              ry d   j                         j                  d      st        t	         d               dk  ryt        t	         d               dkD  ry	 d   j                  d
      ry d   t        vryy)Nr#   r!   rO   source_locatorevidence_classconfidence_reasonknowledge_unit_identityc              3   B   K   | ]  }j                  |         y wr   )rT   ).0kr6   s     r   	<genexpr>zvalid.<locals>.<genexpr>q   s     
%AquuQx<
%s   )Frequired_field_missingr#   zknowledge unit fromrl   )Fplaceholder_or_too_short-   )F	too_broadr&   )Fheading_liker   )Finvalid_evidence)TN)anylowerr   ro   r   endswithVALID_EVIDENCE)r6   reqs   ` r   validr   o   s     	DC

%
%%.L~(()>?3uQ{^G\C]^_C_  iI
5; !"$->~s#,@	.09Qr   c                     t        d t        |       D              }t        d t        |      D              }t        ||z        t        dt        ||z              z  S )Nc              3   <   K   | ]  }|j                           y wr   r   r   rv   s     r   r   zjaccard.<locals>.<genexpr>y   s     	&!'')	&   c              3   <   K   | ]  }|j                           y wr   r   r   s     r   r   zjaccard.<locals>.<genexpr>y   s     -JAaggi-Jr   rz   )rq   r   ro   max)ar   ABs       r   jaccardr   x   sJ    		&U1X	&&A#-Jq-J*Jqs8C#ac(O##r   c                  	   t        j                         } | j                  dd       | j                         }t	        |j
                        }|j                  dd       |dz  }t               }t        j                  |      }|j                  d       t        j                  t        j                        j                         }g }|D ]  }|j!                  d|d	   |d
   |d   |d   |d   |d   t#        |d   j%                               t#        t'        |d               |d   |d   t)        |d         |d   |d   t+        j,                  |d   d      |f       t/        |      D ]W  \  }	}
t1        |	      \  }}|	|d	   |d   |
||t3        |d	   |d   |	      d}t5        |      \  }}|j7                  ||||f       Y  g }|D ]  \  }}}}|rdnd}d}|rI|D ],  }|d   d   |d   k(  st9        |d   d   |d         dk\  s*d} n |dk(  r|j7                  ||f       |j!                  d |d	   |d   |d!   |d"   |d#   |d   |||f	      }|j:                  }|j!                  d$||d	   |d   |d!   |d   |d   |d   f        |j=                          |j?                          t        j                  |      }t        j@                  |_!        |j!                  d%      D cg c]  }tE        |       }}|j!                  d&      D cg c]  }tE        |       }}|j?                          |D cg c]  }|d   d'k(  s| }}t#        |D ch c]  }|d	   	 c}      d(k(  xr" tG        d) |D              xr t#        |      d*k(  }d+|t#        |      ||tH        tJ        d,d-d.d/t#        |      tM        d0 |D              tM        d1 |D              tM        d2 |D              tO        tP              D ci c]  tM        fd3|D               c}d4d5|rd6nd7|D cg c]  }|d	   |d   |d   |d   d8 c}d9d:d;d;d;d;d;d;d;d<d=}|d>z  jS                  t+        j,                  |dd*d?      d@A       tU        t+        j,                  tW        |      t#        |      t#        |      |dB   dC   |dB   dD   |dB   dE   |dF   dG   dHdI             y c c}w c c}w c c}w c c}w c c}w c c}w )JNz--outputT)requiredF)parentsexist_okzp33_staging.sqlitea  
    PRAGMA foreign_keys=ON;
    CREATE TABLE sources(source_id TEXT PRIMARY KEY, source_type TEXT, youtube_id TEXT, title TEXT, content_version TEXT NOT NULL, raw_source_hash TEXT, text_bytes INTEGER, word_count INTEGER, transcript_status TEXT, obsidian_path TEXT, artifact_exists INTEGER, selection_kind TEXT, source_db_video_id INTEGER, source_versions_json TEXT, frozen_at TEXT);
    CREATE TABLE candidates(candidate_id INTEGER PRIMARY KEY, source_id TEXT NOT NULL, statement TEXT, source_locator TEXT, evidence_class TEXT, confidence_reason TEXT, knowledge_unit_identity TEXT, valid_status TEXT, reject_reason TEXT, dedupe_status TEXT DEFAULT 'UNIQUE', FOREIGN KEY(source_id) REFERENCES sources(source_id));
    CREATE TABLE provenance(candidate_id INTEGER, source_id TEXT, content_version TEXT, source_locator TEXT, source_path TEXT, youtube_id TEXT, representation TEXT, FOREIGN KEY(candidate_id) REFERENCES candidates(candidate_id));
    z9INSERT INTO sources VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)r!   rM   rE   rG   rO   rP   rN   rI   rD   rQ   rR   rS   r?   )	sort_keysr   VALIDREJECTUNIQUErz   r   r#   g)\(?	DUPLICATEzINSERT INTO candidates(source_id,statement,source_locator,evidence_class,confidence_reason,knowledge_unit_identity,valid_status,reject_reason,dedupe_status) VALUES(?,?,?,?,?,?,?,?,?)r   r   r   z,INSERT INTO provenance VALUES(?,?,?,?,?,?,?)z$SELECT * FROM sources ORDER BY rowidz.SELECT * FROM candidates ORDER BY candidate_idrm   rn   c              3   &   K   | ]	  }|d      yw)rQ   Nr-   )r   r   s     r   r   zrun.<locals>.<genexpr>   s     Bf\]1EVCWBfs   rj   z1.0.0$deterministic_no_external_model_callznot queried or changedz=P33 staging extractor is provider-free; no model request made)modeconfigured_productive_modelreasonc              3   0   K   | ]  \  }}}}|sd   ywrz   Nr-   r   _oks      r   r   zrun.<locals>.<genexpr>   sE       `~  go  gh  ij  km  no  {}  ab  `~   c              3   0   K   | ]  \  }}}}|rd   ywr   r-   r   s      r   r   zrun.<locals>.<genexpr>   sE       Ln  S[  ST  UV  WY  Z[  km  MN  Lnr   c              3   2   K   | ]  }|d    dk(  sd  yw)dedupe_statusr   rz   Nr-   )r   r9   s     r   r   zrun.<locals>.<genexpr>   sE       w  FG  WX  Yh  Wi  kv  Wv  @A  ws   c              3   F   K   | ]  }|d    dk(  s|d   k(  sd  yw)valid_statusr   r   rz   Nr-   )r   r9   es     r   r   zrun.<locals>.<genexpr>   si       V	d
  ]	^	  n	o	  p	~	  n		  A
H
  n	H
  M
N
  O
_
  M
`
  b
c
  M
c
  W	X	  V	d
s   !!!z`not detected by source-only deterministic extractor; independent fact verification not performed)
candidatesr   rejects
duplicatesevidence_distributionhallucinationsCANONICAL_SOURCE_DEDUP_PASSCANONICAL_SOURCE_DEDUP_FAIL)r!   rE   rD   canonical_content_versionzEach dual representation is one canonical source row and was processed once; artifact is provenance metadata, not a second input row.)resultdual_sourcesproofr   )knowledge_units_writeshistorical_kus_changedbackfill_startedgraphiti_releasegraphiti_postsneo4j_writesgold_promotions)p33_version
checked_atsource_countsourcesr   prompt_versionpromptmodelmetricscanonical_source_dedupproduction_non_changezp33-result.json)ensure_asciiindentr   rC   )encodingr   r   r   r   r   r   )outputr   r   r   r   r   dedup)r   ),argparseArgumentParseradd_argument
parse_argsr   r   mkdirrx   r*   r+   executescriptr   nowr   utc	isoformatr2   ro   r    r   intjsondumpsr   r   r$   r   r3   r   	lastrowidcommitr5   r0   r1   r4   allPROMPT_VERSIONEXTRACTION_PROMPTsumsortedr   
write_textprintstr)apargsr>   stagingr   connfrozenallcr   r   locevr   r6   r   rejacceptedstatusdpriorcurcidr9   sources_outcand_outrt   
dedup_passr   summarys                              ` r   runr  |   s    B"//*d/"KRTR_R_RaTT[[C399T59A3OcKc7G7??7#;dT=O=O Q >	 <<%//1F4 7PSTU`Sabcdqbrst  vB  tC  DE  FM  DN  OP  Qb  Oc  de  fw  dx  y|  }~  E  }F  }M  }M  }O  yP  QT  UZ  [\  ]c  [d  Ue  Qf  gh  i|  g}  ~  @O  ~P  QT  UV  Wh  Ui  Qj  kl  m}  k~  @  AU  V  W[  Wa  Wa  bc  du  bv  AE  WF  GM  SN  	O)!, 	7FBsrlIBv2!K.cdevcw  JM  _a  v|  W_  `a  bm  `n  op  qB  oC  DF  WG  'HA8FBsT[[!Ab6	77 H 
z
!Bsw(
! )856:S8TTX_`efg`hit`uvw  yD  wE  YF  HL  YL!A5) ({HOOQqE2LL  R  TU  Va  Tb  cd  ep  cq  rs  tD  rE  FG  HX  FY  Z[  \o  Zp  qr  sL  qM  NT  UX  YZ  S[  \MMCS;XYZkXlmno  nA  BC  DS  BT  UV  Wc  Ud  ef  gw  ex  Ey  	z
z 	KKM4::<		!DGKK4#3"&,,/U"VWQaWKW#||,\]^!d1g^H^JJL	V"2 35U U!	VD	VW5Q{^56:wsBfaeBf?fwknosktvwkwJ"c'l]hv~  Q_  iz  Kq  Ph  rq  Cr  KN  OS  KT  ]`  `~  sw  `~  ]~  IL  Ln  _c  Ln  In  |  w  KS  w  |w  n
t
  u
C  n
D  P	E  i
j
  Q	R	  S	V	  V	d
  b	j	  V	d
  S	d
  Q	d
  P	E  Wy  }z  I  ^{  Ol  TX  |Y  OP  JK  LW  JX  fg  ht  fu  FG  HW  FX  uv  wH  uI  }J  |Y  bi  Tj  ]^  xy  MN  bc  uv  FG  Z[  C\  ]G	&&tzz'uTU`d'eov&w	$**s3x#g,CPTI^efo^pqx^y  EL  MV  EW  Xa  Eb  pw  xA  pB  CO  pP  Y`  ay  Yz  {C  YD  E  SX  Y  Z X^	V5 P	E  |Ys*   )SS8S#S#S(;S-%S2__main__)"r   r   r   osr   shutilr*   sysr   r   pathlibr   r,   rV   r   r   r   r   r   r   r$   r'   r.   r:   r?   r[   rg   rx   r   r   r   r   r  __name__r-   r   r   <module>r     s    < < < < ' 3"#Y. e  0 3 ][5A		BZ*&G
$&ZP Zr   