
    k<j                     ~    d dl Z d dlZd dlZd dlZd dlZd dlmZmZ d dlm	Z	 dZ
dZdZdZd Zd	 Zed
k(  r e        yy)    N)datetimetimezone)Pathz+/opt/struktur/youtube-research/knowledge.dbzC/opt/struktur/reports/aa043-p35/20260828T074105Z/p35-selection.jsonz%/opt/struktur/AA-043-P33-extractor.pyz0/opt/struktur/reports/aa043-p35/20260828T074105Zc                     t         j                  j                  ||       }t         j                  j                  |      }|j                  j                  |       |S )N)	importlibutilspec_from_file_locationmodule_from_specloaderexec_module)pnsms       9/opt/struktur/knowledge-unit-pipeline/p35_batch_runner.pyloadr      sN    77!<q	@_@_`a@badedldldxdxyzd{  EF  ~F    c                    !. t        j                         } | j                  dt        d       | j	                         .t        t        d      }t        j
                  t        t        d            }t        .fd|d   D              }|d	   }t        |      d
k(  sJ |d   D ci c]  }|d   |
 }}t        j                  dt        z   dz   d      }t        j                  |_        |j#                  d      D ch c]  }|d   	 }	}|j%                          g }
|D ]8  }|j'                  d      r|j)                  dd      d   }|j+                         D cg c]  }|d   |k(  s| }}t        |      dk7  r&t-        d|z   dz   t/        t        |            z         |j1                  |d   dt/        .j2                        z         }n,|j5                  t7        d      |j9                  d      z        }||   }|d   |d   k7  rt-        d|z         t        |j;                  |d               dk  rt-        d|z         |
j=                  |       ; g }g }|
D ]  }|j?                  |      D ]l  \  }}|jA                  |      \  }}||d   |d   ||||jC                  |d   |d   |      d}|jE                  |      \  }}|r|n|j=                  |||f       n  g }g }g }|D ]  \  }}}|D cg c]Y  \  }}|d    jG                         jI                         |d    jG                         jI                         k(  sJ|d   |d   k7  sV||f[ }}}|r$|j=                  |d   |d    |d   d   d   d!       d } |D ]/  \  }}|d"   |d"   k(  s|jK                  |d    |d          d#k\  s-|}  n | r|j=                  ||d$f       |j=                  ||f        tM        |      |	z  r6t-        d%t        jN                  tQ        tM        |      |	z              z         i d&.j2                  d'tS        jT                  tV        jX                        j[                         d(|d)d
d*t        |      t        |      z   d+t        |      d,t        |      d-t        |      d.t]        d/ |D              d0t        |      d1|d2t        |      d3dd4dd5dd6dddd7d8tQ        |j^                        D !!ci c]  !!t]        !fd9|D               c}!dddd|
D cg c]h  }|d   |d:   |d   |ja                  d      |ja                  d;      t        |j;                  |d               t        |d   jc                               d<j c}d=}"t        j                  t        d>?      }t        j                  |_        |j#                  d@       |j#                  dA       	 g }#|D ]  \  }}|j#                  dB|d"   f      je                         }$|$r|$dC   }%n|ja                  dD      xs g }&|&r|&d   ja                  dE      nd }'|jf                  d.j2                  |dF   |d   dG}(|j#                  dH|d"   |d    dIdI|d   |'|d   t        jN                  |(dJdK      dLt/        .j2                        z   dMz   tS        jT                  tV        jX                        j[                         |dF   |dN   |dO   dPdQf      })|)jh                  }%|"d3xx   dz  cc<   |#j=                  |%||f       |j#                  dR|%|d   |d   |dF   |dS   tS        jT                  tV        jX                        j[                         f      })|"d4xx   tk        d|)jl                        z  cc<    |jo                          |j#                  dA       	 |#D ]D  \  }%}}|d   js                         }*t        |dF   j)                  dd      d         }+d|+cxk  xr t        |*      k  nc xr+ |d    |ju                  |*|+dz
           jG                  dT      v },|,r|jE                  |      d   rdUndV}-|-dUk(  rdWndX}|j#                  dY|%|-|dZtS        jT                  tV        jX                        j[                         f      })|"d5xx   tk        d|)jl                        z  cc<   |)jl                  s|"d6   |-xx   dz  cc<   |j#                  d[|-dUk(  rd\nd]|%f       G |jo                          |j%                          tS        jT                  tV        jX                        j[                         |"d^<   t7        tv        d_t/        .j2                        z   d`z         jy                  t        jN                  |"dJdadb             t{        t        jN                  |"dJc             y c c}w c c}w c c}w c c}}w c c}!w c c}w #  |jq                          |j%                           xY w#  |jq                          |j%                           xY w)dNz--batchT)typerequiredp33utf8)encodingc              3   H   K   | ]  }|d    j                   k(  s|  yw)batchN)r   ).0xas     r   	<genexpr>zmain.<locals>.<genexpr>   sT       jZ  qr  FG  HO  FP  RS  RY  RY  FY  kl  jZs   ""batches
source_ids
   sources	source_idzfile:z?mode=ro&immutable=1)urizoselect distinct source_identity from knowledge_units where legacy_placeholder=0 and source_identity is not nullr   zyoutube::   
youtube_idzsource_resolution:
p35_batch_z/opt/obsidian-vaultz	obsidian:content_versionzcontent_version_drift:text   zunusable_content:)	statementr$   r*   source_locatorevidence_classconfidence_reasonknowledge_unit_identityr-   )r$   r-   other_source_idr1   g)\(?exact_or_semantic_duplicatezallowlist_contains_processed:r   
started_at	allowlistsource_countcandidate_countvalid_countreject_countduplicate_countsemantic_duplicate_countc              3   4   K   | ]  \  }}}|d k(  sd  yw)r3   r'   N )r   _rs      r   r   zmain.<locals>.<genexpr>$   sL       F~  MR  MN  OP  QR  ]^  `}  ]}  GH  F~s   cross_source_potential_countcross_source_potentialsunique_countnew_kusnew_provenancenew_review_auditreview_verdicts)PASS	TEILWEISEFAILevidencec              3   :   K   | ]  \  }}|d    k(  sd  yw)r/   r'   Nr=   )r   r>   kes      r   r   zmain.<locals>.<genexpr>$   sT       sd	  z}  z{  |}  M	N	  O	_	  M	`	  b	c	  M	c	  tu  sd	s   source_typeobsidian_path)r$   rN   r*   r(   rO   
word_count
text_bytes)provider_callsgraphiti_postsneo4j_writesqueue_writesr#      )timeoutzPRAGMA foreign_keys=ONzBEGIN IMMEDIATEz>select id from knowledge_units where knowledge_unit_identity=?idsource_versions
version_idr.   )prompt_versionsource_onlyr   r.   r*   a2  insert into knowledge_units(knowledge_unit_identity,statement,entities,relations,source_identity,source_version_id,content_version_hash,provenance,promotion_reason,created_at,source_locator,evidence_class,confidence_reason,review_state,gold_state,legacy_placeholder) values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,0)z[]F)ensure_ascii	sort_keysz P35 controlled historical batch z; no automatic Gold promotionr/   r0   PENDINGNONEzinsert or ignore into knowledge_unit_provenance(knowledge_unit_id,source_identity,source_version,source_locator,representation_type,created_at) values(?,?,?,?,?,?)selection_kindz` rG   rI   zJP35 source-only review: valid structure and exact frozen source line matchz<P35 source-only review failed exact frozen source line matchzzinsert or ignore into knowledge_unit_review_audit(knowledge_unit_id,verdict,reason,reviewer,reviewed_at) values(?,?,?,?,?)z AA-043-P35-source-only-review-v1z4update knowledge_units set review_state=? where id=?APPROVEDREJECTEDfinished_atzbatch-z-result.json   )r]   indentr^   )r]   )>argparseArgumentParseradd_argumentint
parse_argsr   EXjsonopenSELnextlensqlite3connectDBRowrow_factoryexecuteclose
startswithsplityoutube_rowsRuntimeErrorstrcanonical_youtuber   canonical_obsr   removeprefixwordsappendsentence_candidatesclassifyidentityvalidstripcasefoldjaccardsetdumpssortedr   nowr   utc	isoformatsumVALID_EVIDENCEgetencodefetchonePROMPT_VERSION	lastrowidmaxrowcountcommitrollback
splitlinesnormOUTDIR
write_textprint)/apr   Dr   idsr   metacr?   existingr#   sidyidrowsr   frozenallcrejectsstlocevreasonrL   okrraccepteddupcross_potentialr>   pspksamefoundrM   resultinsertedrowku_idsvsv_idprovcurlinesr   line_okverdictr   s/                                    `            @r   mainr      s   ryc4PTVTaTaTcRSgklnotguefy}  zC  zC  DH  IL  V\  D]  z^wx  fj  jZ  vw  xA  vB  jZ  fZ  `e  `e  fr  `s  \_  |  @C  |D  FH  |H  uH  |H!"9.Aq~a..'//'"*McBchl2m  ~E  ~I  ~Iopo|  cd  cl  cl  m^  c_  T`  ^_  UV  WX  UY  T`  KS  T`  bc  bi  bi  bk	 	S^^J
yyQ3Q^^-=!V<RUAU!!VD!V	$il,';C'?'CCD	N'RSSac!''l!:;1
//$45c6F6F{6SS
T	c&6"344LIabeIe<f6f6	Ql3Fs3J&K K	..	 	"'
 @Q%%a( @fbZZ^92fB1[>\]^o\p  CF  XZ  ou  PQ  PZ  PZ  [\  ]h  [i  jk  l}  j~  A  PB   CQ	
52bbdg55q2h?@@ "cbo UQq%  TEBrK)>)>)@)I)I)KQ{^MaMaMcMlMlMn)nsu  wB  tC  EF  GR  ES  tSB  T$  T	?!!q~!K.kopqkrstku  wB  lC  #D  E% Eeb	"#Q'@%AAQYYrR]_`al_mEnpsEs{}uz  @EE
CJJ!9:;1 HX\*I$**U[\_`c\dem\mUnJo*opp 	b  	bhll8<<&@&J&J&L  	b[Y\  	b]kln  	b  pA  BE  FJ  BK  LO  PW  LX  BX  	b  Yf  gj  ko  gp  	b  q  @C  DK  @L  	b  M^  _b  cf  _g  	b  hB  CF  F~  VY  F~  C~  	b  ]  ^a  bq  ^r  	b  sL  M\  	b  ]k  lo  px  ly  	b  zC  DE  	b  FV  WX  	b  Yk  lm  	b  n  HI  VW  _`  @a  	b  bl  n	t	  u	v	  u	E
  u	E
  n	F
  mG
  i	j	  no  ps  sd	  A	I	  sd	  pd	  nd	  mG
  	b  Y
Z
  l
m
  }
~
  NO  Y`  Za  TU  hi  ju  hv  EF  GT  EU  hi  j{  h|  JK  JO  JO  P\  J]  no  ns  ns  tC  nD  RU  VW  V]  V]  ^_  `f  ^g  Vh  Ri  wz  {|  }C  {D  {K  {K  {M  wN  [O  Za  	b
??2b!1=aiiH`>acdclclm~c(( fcayyQSTUnSoRqr{{}3	T5	 &BB"r!uyy/FRV++$qww`abr`s  GH  IZ  G[  
\D			  K  MN  Oh  Mi  jk  lw  jx  y}  ~B  CD  EP  CQ  RW  XY  Zk  Xl  mq  mw  mw  x|  JO  Z^  m_  `B  CF  GH  GN  GN  CO  `O  Po  `o  px  p|  p|  }E	  }I	  }I	  pJ	  pT	  pT	  pV	  W	X	  Y	i	  W	j	  k	l	  m	}	  k	~	  	@
  A
T
  	U
  V
_
  `
f
  Lg
  	h
C  p
s
  p
}
  p
}
  j
o
  
E  FO  
P  RS  
S  
P  U]  Ud  Ud  fk  lm  no  ep  Uqyy  {  }B  CD  EP  CQ  RS  Te  Rf  gh  iy  gz  {|  }M  {N  OW  O[  O[  \d  \h  \h  Oi  Os  Os  Ou  |v  w3  y  @P  yQ  SV  WX  Y\  Ye  Ye  Sf  yf  yQf ((* ( sieAa	
6			53q)9':'@'@Q'G'J#KUVXYUe[^_d[eUe  V[jklwjx|}  }C  }C  DI  JK  LM  JM  DN  }O  }U  }U  VZ  }[  k[W  ov  {|  {B  {B  CD  {E  FG  {H  ek  NT  ]d  mt  v|  m|  ]i  B@  V\yy  N  PU  V]  ^d  eG  HP  HT  HT  U]  Ua  Ua  Hb  Hl  Hl  Hn  Oo  p3  rx  yK  rL  NQ  RS  TW  T`  T`  Na  ra  rL	llF,-g696199E{  LS  U[  L[  ~H  ak  lq  }r  <ss ((*(,,x||"<"F"F"HF=!$vV^_bcdcjcj_kVklzVzJ{  KG  KG  HL  HR  HR  SY  gl  tu  @D  HE  KF  HM  NR  NX  NX  Y_  mr  Ns  Hte /  T` "W T mG
  Za (qwwy% (qwwy%sX   &g>g"g'*g';Ag,g,g,g2(A-g7.Gg< Dh" +>h" <#h"#i__main__)rg   importlib.utilr   rm   rr   sysr   r   pathlibr   rt   ro   rl   r   r   r   __name__r=   r   r   <module>r      sT    / / / & 06{#  Ah}  qc  jp F4tj Zr   