
    !Kj@                        d dl Zd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	Z	d dl
Z
d dlZd dlZd dlmZ d dlmZmZ dZ ed      Zej)                  dd       dZd	Zd
 Zd Z eed      Zd Z e       Zej8                  e_        ej=                  d      D  cg c]
  }  e|        c} Z e D  ch c]  } | d   r	| d   s| d    c} Z!e D  ch c]  } | d   s	| d   s| d    c} Z" ej0                   e#dd            Z$ e%e$d   xr e$d   D  cg c]  } | d   	 c}       Z& e%e$jO                  dg             Z(e!e"z  e&z  e(z  Z)d Z*g Z+ejY                         D ]  Z-de-d   z   Z.e-jO                  d      Z/e.e)v s#e-jO                  d      r e*e-d         dk  se/dv rDeja                  e-d      Z1 e*e1d          e1d!<    e2e1d    jg                               e1d"<   d e1d#<   e+ji                  e1        g Z5 ed$      Z6 e7e6jq                  d%            D ]  Z9e9ju                  e6      jw                         Z<d&e<z   Z.e.e)v s e<j{                  d'      s e>d( d)D              rMe9j                         Z@e@j                  d*d+,      ZB e*eB      dk  r|e5ji                  i de.d-d.ddd/e<d eBd0 ej                  e@      j                         d1 ej                  e@      j                         d2e<d3dddd4d5d6dd! e*eB      d" e2e@      d#d d7g         d8 ZF eFe+d9       eFe5d:      z   ZG e2eG      d;k7  s e2eGD  ch c]  } | d   	 c}       d;k7  r eHd<       eIeG      D ]-  \  ZJZ1eJd=z  d>z   e1d?<   eJd=z  d>z   e1d@<   e1j                  d d       / dA e       dBd9d:d;dC e2e+       e2e5      dDeG eLd>dE      D  cg c]!  }|eGD  cg c]  } | d?   |k(  s| d    c} dF# c} }dGZMedHz  j                   ej                  eMdIdJdK             edLz  ZP ej                  eeP       dM ZRdN ZS eRe      ZT e       ZU eSeU      ZVeUj=                  dO      j                         d    ZX eYeUj=                  dP            ZZeUj                           e       eTej                  j                  e      eX e2eV       ej                   ej                  eVdQ      jg                               j                          eRe      dR e e       j=                  dS             e       j=                  dT      D ci c]  } e^|d          |d>    c} e2e        e2e D  cg c]
  } | d   r	|  c}        e2e"      dUZ_edVz  j                   ej                  e_dIdJdK             e_dW   e_dX   k7  r eHdY      eGD  ci c]  } | d   | 
 c} Z`dZ Zag Zbg Zcdd[Zd eLd>dE      D ]]  Z@ ede@d      Z- e       Ze eSee      Zfeej=                  dO      j                         d    Zg eeej=                  dS            Zheej=                  dT      D  ci c]  }  e^| d          | d>    c} Z< eYeej=                  dP            Zieej                          e@eg e2ef      efeVk(  eieZk(  ehe< e2e-d\         d=k(  xr  e2 e%e-d\               d=k(  e-d]   egd^k(  refeVk(  reieZk(  r
e-d]   d_k(  rd`ndadb
Zjejdc   e-dc<   edde@dedfz  j                   ej                  e-dIdJdK             ebji                  e-       ejdc   d`k7  sO eHdg e^ej      z           e       Zeeej=                  dh      D  cg c]
  }  e|        c} Zkeej                           eeY      Zldi ZmekD ]  Znel emendj            ji                  en       ! g Zoelj                         D ]q  \  ZqZrerD  ch c]  } | d   	 c} Zseqs e2es      d>kD  s( ej                  erdJ      D ]4  \  ZuZ@eud   e@d   k7  seoji                  eudk   e@dk   eud   e@d   eqdl       6 s  e       dm e2ek       e2eo       e2eo      d d d dneoddo dp
Zvedqz  j                   ej                  evdIdJdK              e       Zeeej=                  d      D  cg c]
  }  e|        c} Zw eeej=                  dS            Zheej=                  dT      D  ci c]  }  e^| d          | d>    c} Z< eSee      Zxeej=                  dO      j                         d    Zyeej                          ebD  cg c]1  }  ezdr | d   D              r| ds   dtz   ezdu | d   D              z  nd 3 c} Z{ e       d;d;eb ezdv ebD               ezdw ebD               ezdx ebD               ezdy ebD               ezdz ebD               ezd{ ebD               ezd| ebD               ezd} ebD               ezd~ ebD               ej                  e{       ej                  e{       e~e{       ee{       ezd ebD              d eRe      ej                  j                  e      ey e2ex      exeVk(   e2ew       ezd ewD               ezd ewD               e ed ewD                     e ed ewD                     e ed ewD                    ehe<devdZedz  j                   ej                  edIdJdK              e ej                   e^e      d;ed   ds   ed   d   evd   ehed   d   ddI             yc c} w c c} w c c} w c c} w # eC$ r Y 	/w xY wc c} w c c} w c c} }w c c}w c c} w c c} w c c} w c c} w c c} w c c} w c c} w c c} w )    N)Path)Counterdefaultdictz+/opt/struktur/youtube-research/knowledge.dbz0/opt/struktur/reports/aa043-p39/20260828T084500ZT)parentsexist_okz%/opt/struktur/AA-043-P33-extractor.pyz2/opt/struktur/knowledge-unit-pipeline/ku_writer.pyc                      t         j                   j                  t         j                  j                        j	                         S N)datetimenowtimezoneutc	isoformat     "/opt/struktur/AA-043-P39-runner.pyr   r      s-    (##''(9(9(=(=>HHJJr   c                     t         j                  j                  ||       }t         j                  j                  |      }|j                  j                  |       |S r	   )	importlibutilspec_from_file_locationmodule_from_specloaderexec_module)pnsms       r   loadr      sN    77!<q	@_@_`a@badedldldxdxyzd{  EF  ~Fr   p33_p39c                  D    t        j                  dt        z   dz   d      S )Nzfile:z?mode=ro&immutable=1T)uri)sqlite3connectDBr   r   r   ror$      s    ,B!BtL
Lr   zselect * from knowledge_unitslegacy_placeholdersource_identityzC/opt/struktur/reports/aa043-p35/20260828T074105Z/p35-selection.jsonutf8)encodingsource_countssources	source_idexcluded_p34c                 >    t        t        j                  |             S r	   )lenr   words)ts    r   r/   r/      s    S_$r   youtube:
youtube_idtranscript_status
transcript   )unavailablerejectedfailedp39_youtubetext
word_count
text_bytesprior_ku_countz/opt/obsidian-vaultz*.md	obsidian:zYouTube-Research/c              #   ,   K   | ]  }|t         v   y wr	   )rel.0xs     r   	<genexpr>rD      s       B^q!s(  B^   )z_hermes-session-testzTEMPLATE.mdz_deleted_backup/z05_SESSIONS/zutf-8replace)errorssource_typenative_obsidiantitlecontent_versionraw_source_hashobsidian_pathartifact_existsselection_kindp39_native_obsidiansource_db_video_idsource_versionsc                     t        | d       } t        |       |k  r"t        dt        t        |       |f      z         t	        |      D cg c]  }| |t        |       z  |z      c}S c c}w )Nc                     | d   | d   fS )Nr;   r+   r   )rC   s    r   <lambda>zpick.<locals>.<lambda>    s    aoan= r   )keyzinsufficient_candidates )sortedr.   RuntimeErrorstrrange)arrr   is      r   pickr]      s`    C=>HQJl#=c3s8A,>O#OPP',Qx0!aCj1_	000s   A*   d      selection_count2      batchbatch_indexzp39-v1zideterministic word-count strata and source-id tie-break; prior P34/P35/processed/legacy/unusable excluded)youtuberI   total)rf   rI      )rd   
source_ids)selection_versionselected_atselection_ruler)   candidates_before_selectionr*   batcheszp39-selection.jsonF   ensure_asciiindent	sort_keyszknowledge.db.pre-p39.bakc                 x    t        j                  t        | d      j                               j	                         S )Nrb)hashlibsha256openread	hexdigest)r   s    r   dbhashr{   +   s(    gnnT!D\%6%6%89CCEEr   c                 `    t        d | j                  d      j                         D              S )Nc              3   2   K   | ]  }t        |        y wr	   )tuple)rB   rs     r   rD   zfkset.<locals>.<genexpr>,   s     eAuQxes   zpragma foreign_key_check)rW   executefetchall)conns    r   fksetr   ,   s%    e>X1Y1b1b1deeer   zpragma integrity_checkzselect id,source_type,video_id,obsidian_path,content_hash,graphiti_status,graphiti_release from graphiti_import_queue order by id)rs   @20124f639350968136c841c5f7221c54855a8cccfc94fe3aac423167f469a605zSselect graphiti_status,count(*) from graphiti_import_queue group by graphiti_statuszUselect graphiti_release,count(*) from graphiti_import_queue group by graphiti_release)
checked_at	db_sha256db_sizeintegrity_checkfk_countfk_fingerprint_sha256writer_sha256writer_expectedqueue_beforerelease_beforeku_total_beforereal_beforelegacy_beforezp39-live-baseline.jsonr   r   writer_hash_mismatchc                    | j                  d      r| j                  dd      d   }t        j                         D cg c]  }|d   |k(  s| }}t	        |      dk7  rt        d| z         t        j                  |d   dt        |      z         }n+t        j                  t        | j                  d      z        }t        |    }|d	   |d	   k7  rt        d
| z         t        |d         dk  rt        d| z         |S c c}w )Nr1   :rc   r2   zsource_resolution:r   
p39_batch_r>   rK   zcontent_version_drift:r:   r5   zunusable_content:)
startswithsplitr   youtube_rowsr.   rX   canonical_youtuberY   canonical_obsvaultremoveprefixmetar/   )sidrd   yr   rrr   fs          r   resolver   3   s    NN:IIc!Q!(8O1AlOQ<NQOOWaZ|$8$<==1l3u:56!	
s//<<	=q9!-..lC[\_C_6`0`	!F)Ql+>s+BCC	 Ps   DDc                   )* t         d   | dz
     d   }t        |      dk(  rt        t        |            dk(  sJ |D cg c]  }t        ||        }}g }g }|D ]  t        j                        D ]u  \  }}t        j                  |      \  }	}
|d   d   ||	|
t        j                  d   d   |      d}t        j                  |      d   r|n|j                  |f       w  g }g }g }|D ]h  \  }d }|D ]3  \  }}|d	   |d	   k(  s t        j                  |d
   |d
         dk\  s1|} n |r|j                  |f       V|j                  |f       j t        d ||D cg c]	  \  }}||f c}}z   D              j                  t        |            st        d      g }d}d}dddd}|rt        j                  t        d      }|j!                  d       |j!                  d       	 |D ]C  \  }|j!                  d|d	   f      j#                         }|r|d   }n̉j%                  d      xs g }|r|d   j%                  d      nd }t        j&                  d| |d   |d   d}|j!                  d|d	   |d
   ddd   ||d   t)        j*                  |dd      dt-        |       z   dz   t/               |d   |d   |d    d!d"f      }|j0                  }|j                  ||f       |j!                  d#|d   |d   |d   d$   t/               f      }|t3        d|j4                        z  }F |j7                          |j!                  d       	 |D ]
  \  }}t=        |d   j?                  d%d      d         }d&   jA                         }d|cxk  xr t        |      k  nc xr/ |d
   t        jC                  ||dz
           jE                  d'      v }|rt        j                  |      d   rd(nd)} || xx   dz  cc<   |j!                  d*|| | d(k(  rd+nd,d-t/               f      }|t3        d|j4                        z  }|j!                  d.| d(k(  rd/nd0|f        |j7                          |j;                          tG               }!|!j!                  d1      D "ch c]  }"|"d   	 c}")|!j!                  d2      D "ch c]  }"|"d   |"d   |"d3   |"d4   f c}"*|!j;                          tI        )fd5|D              }#tI        *fd6|D              }$|D %&cg c]M  }tK        |d&         st        |D %&cg c]  \  }%}&|%d   |d   k(  sd c}&}%      d7z  tK        |d&         z  O }'}%}}&tL        jO                  |'       i d8| d9t/               d:|d;dd<t        |      t        |      z   d=t        |      d>t        |      d?t        |      d@t        |      dAt        |      dB|dC|dD|dEtQ        tS        dF |D                    dGddHddIddddd|#|$|#dk(  r|$dk(  rdJnd)|D cg c]l  d   dK   d   j%                  dL      j%                  dM      tK        d&         t        d&   jU                               tI        fdN|D              dOn c}dP}(tV        dQ| dRdSz  jY                  t)        j*                  |(dd3dT             |(S c c}w c c}}w #  |j9                          |j;                           xY w#  |j9                          |j;                           xY wc c}"w c c}"w c c}&}%w c c}&}%}w c c}w )UNrn   rc   ri   rb   r+   rK   )	statementr+   rK   source_locatorevidence_classconfidence_reasonknowledge_unit_identityr   r   r   g)\(?c              3   ,   K   | ]  }|d    d     yw)r   r+   Nr   rA   s     r   rD   zrun_batch.<locals>.<genexpr>N   s     K!AaDKrE   allowlist_violation)PASS	TEILWEISEFAIL   )timeoutzpragma foreign_keys=onzbegin immediatez>select id from knowledge_units where knowledge_unit_identity=?rR   
version_idTr   )prompt_versionsource_onlyrd   r   rK   a2  insert into knowledge_units(knowledge_unit_identity,statement,entities,relations,source_identity,source_version_id,content_version_hash,provenance,promotion_reason,created_at,source_locator,evidence_class,confidence_reason,review_state,gold_state,legacy_placeholder) values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,0)z[]F)rq   rs   z P39 controlled historical batch z); no Cross-Source/Evidence/Gold promotionr   r   PENDINGNONEzinsert or ignore into knowledge_unit_provenance(knowledge_unit_id,source_identity,source_version,source_locator,representation_type,created_at) values(?,?,?,?,?,?)rO   r   r:   z` r   r   zzinsert or ignore into knowledge_unit_review_audit(knowledge_unit_id,verdict,reason,reviewer,reviewed_at) values(?,?,?,?,?)zMP39 source-only review: exact frozen source locator and structural validationz9P39 source-only review failed exact frozen source locatorz AA-043-P39-source-only-review-v1z4update knowledge_units set review_state=? where id=?APPROVEDREJECTEDz3select knowledge_unit_identity from knowledge_unitszeselect knowledge_unit_id,source_identity,source_version,source_locator from knowledge_unit_provenancero      c              3   8   K   | ]  \  }}|d    vsd  yw)r   rc   Nr   )rB   r   kexistings      r   rD   zrun_batch.<locals>.<genexpr>g   s"     TCAaq)B'C8'SaTs   c              3   L   K   | ]  \  }}d |d   |d   |d   fv sd  y w)Nr+   rK   r   rc   r   )rB   r   r   provseens      r   rD   zrun_batch.<locals>.<genexpr>g   s|       dIjmjklm  C  DE  FQ  DR  ST  Uf  Sg  hi  jz  h{  ~|  @H  ~Hde  dIs   $$  rd   finished_at	allowlistsource_countcandidate_countvalid_countreject_countduplicate_countsemantic_duplicate_countnew_unique_kusnew_provenancenew_review_auditreview_verdictsevidencec              3   ,   K   | ]  \  }}|d      ywr   Nr   )rB   r   r   s      r   rD   zrun_batch.<locals>.<genexpr>k   s>       nW  GJ  GH  IJ  op  qA  oB  nWrE   placeholder_counthallucination_countprovenance_error_countBATCH_IDEMPOTENCY_PASSrH   r2   rM   c              3   @   K   | ]  \  }}|d    d    k(  sd  yw)r+   rc   Nr   )rB   sskkr   s      r   rD   zrun_batch.<locals>.<genexpr>k   sb       ws  ~C  ~@  AC  SU  Va  Sb  de  fq  dr  Sr  xy  wss   )r+   rH   rK   r2   rM   r;   r<   new_kus)canonical_dedup_error_countqueue_writesgraphiti_postsneo4j_writes$idempotency_second_run_would_new_kus+idempotency_second_run_would_new_provenanceidempotencyr*   
p39-batch-02d.jsonrp   )-	selectionr.   setr   r   sentence_candidatesclassifyidentityvalidappendjaccardissubsetrX   r!   r"   r#   r   fetchonegetPROMPT_VERSIONjsondumpsrY   r   	lastrowidmaxrowcountcommitrollbackcloseintr   
splitlinesnormstripr$   sumr/   densityextenddictr   encodeOUT
write_text)+rd   writeidsr   r*   allcrejectsstlocevreasonr   accepteddupcrossfoundpspkinsertednewprovnewauditreviewcrowku_idsvsv_idprovcurlinelinesokverdictrcr   wouldnew	wouldprovr   r   densresultr   r   s+      `                                     @@r   	run_batchr   >   s   y%'"<0S2#cRUh-Y[J[2[9[$'	(q'!E
	(	(r$27
 6Q%%a( 6fbZZ^92fB1[>\]^o\p  CF  XZ  ou  PQ  PZ  PZ  [\  ]h  [i  jk  l}  j~  A  PB   CQGGAJqMDw..!u566 "cBe SQq% Eeb	"#Q'@%AAQYYrR]_`al_mEnpsEs{}uz  @EE
CJJ!u1 	Kx'0J31Q!A0J'JKKTTUXY\U]^fr  tI  gJ  aJ
ag!A!1,MV	OOBr"!AII.F$GSdIe
) qsq			RTUVoTpSrs||~C
#a&E	
 	!	'RR21		,0GSW%,,4_`aq_r  FG  HY  FZ  [T	
  H  JK  Le  Jf  gh  it  gu  vz  {  @A  BM  @N  OT  UV  Wh  Ui  jn  jt  jt  uy  GL  W[  j\  ]  @C  DI  @J  ]J  Kv  ]v  wz  w|  }~  O	  }P	  Q	R	  S	c	  Q	d	  e	f	  g	z	  e	{	  |	E
  F
L
  IM
  
N
S  V
Y
  V
c
  V
c
  P
U
  e
m
  e
t
  e
t
  v
{
  |
}
  ~

  u
@  e
A			  x  z  @A  BM  @N  OP  Qb  Oc  de  fv  dw  xy  zJ  xK  LO  LQ  yR  	SC  U\  ^a  bc  dg  dp  dp  ^q  Uq  U\q 88:))) [yuQq	Q &&s1-a0	1D69M9M9O5TUW[Tg]`af]gTg  U`lmnylz~  E  E  FK  LP  QR  LR  FS  T  Z  Z  [_  `  m`QS  tv  {|  {B  {B  CD  {E  FG  {H  jp  NT  bi  V\  ]d  Ve  gh  Vh  Ve			  O  QV  W^  ry  {A  rA  _n  GB  Ce  fi  fk  Pl  	mCc!CLL!!H199-csz  }C  tCeo  IS  TY  eZ  $[[ 88:qwwy D"**-b"cdQAaDd(  PR  PZ  PZ  [B  PC  pD  KLqrstquvwxyvz{|}~{  AB  CD  AE  qF  pDfn  FH  FN  FN  FP
T8T
T`c  dIqy  dI  aIV_ho  E  Ecdsxyz  |B  zC  tDs8G%"Rr++'FAGHMeTUV\T]N^^  E  E 	G  	G}SU  	G;s  	G>"  	GM^_bcg_hilmtiu_u  	G  wD  EH  IM  EN  	G  O]  ^a  bi  ^j  	G  k|  }@  AD  }E  	G  F`  ad  eh  ai  	G  jz  {~  G  {H  	G  IY  Za  	G  bt  u}  	G  ~O  PV  	G  Wa  bf  gn  nW  NV  nW  gW  bX  	G  Yl  mn  	G  oD  EF  	G  G_  `a  	G  @A  QR  de  uv  ^	f	  U
^
  IQ  ST  IT  Yb  de  Ye  m
E  kq  ~E  |F  yz  JK  LW  JX  gh  iv  gw  JK  L]  J^  lm  lq  lq  r~  l  PQ  PU  PU  Ve  Pf  ty  z{  |B  zC  tD  RU  VW  X^  V_  Vf  Vf  Vh  Ri  tw  ws  GO  ws  ts  }t  |F  	G
5+U##//

6u\]hl0mnw}p}Y 
) 1K )!**,	5 )!**,	5d  pD H  E |Fs]   [6.[;;E\ &D0\' 3]]/]
]]!]%]	A1]$#\$'#]
]r   r   r  r   r   r   )
rd   r   r   fk_unchangedqueue_unchangedqueuereleaseallowlist_exactr   gater&  r   r   r   zbatch_gate_fail:zzselect id,statement,source_identity,evidence_class,review_state,gold_state from knowledge_units where legacy_placeholder=0c                     t        j                  d| xs dj                         j                  dd            }dj	                  t        t        d |D                          S )Nu/   [a-zäöüß][a-zäöüß0-9_]*|\d+(?:[.,]\d+)? - c              3   ^   K   | ]%  }t        |      d kD  s|j                         s"| ' yw)ro   N)r.   isdigitrA   s     r   rD   zkey.<locals>.<genexpr>t   sa       Aj  HI  RU  VW  RX  YZ  RZ  ^_  ^g  ^g  ^i  BC  Ajs   #--)refindalllowerrF   joinrW   r   )r   r0   s     r   rV   rV   s   sq    ::@!'rARAZAZ[^_bAcdmpmumuv|  ~A  Aj  MN  Aj  ~j  wk  nl  glr   r   id)ku_aku_bsource_asource_bnormalized_claim"read_only_cross_source_observationz6not separately audited; all exact groups require auditi  )
r   modeexpanded_real_kus#exact_normalized_cross_source_pairssame_claim_candidatesrelated_not_same_candidatescontradiction_candidatesuncertain_candidatesnew_same_claim_after_p39pairsz!p39-cross-source-observation.jsonc              3   &   K   | ]	  }|d      ywr;   Nr   rB   r   s     r   rD   rD      s     OvdePQR^P_Ov   r   r   c              3   &   K   | ]	  }|d      ywrB  r   rC  s     r   rD   rD      s     !Ha!L/!HrD  c              #   &   K   | ]	  }|d      yw)r   Nr   rA   s     r   rD   rD      s,       y`  STyz  |M  zN  y`rD  c              #   &   K   | ]	  }|d      yw)r   Nr   rA   s     r   rD   rD      s0       lO  BC  mn  o|  m}  lOrD  c              #   &   K   | ]	  }|d      yw)r   Nr   rA   s     r   rD   rD      s0       ]A  tu  ^_  `n  ^o  ]ArD  c              #   &   K   | ]	  }|d      yw)r   Nr   rA   s     r   rD   rD      s0       X  rs  YZ  [l  Ym  XrD  c              #   &   K   | ]	  }|d      yw)r   Nr   rA   s     r   rD   rD      s0       YI  |}  Z[  \v  Zw  YIrD  c              #   &   K   | ]	  }|d      yw)r   Nr   rA   s     r   rD   rD      s0       ^D  wx  _`  aq  _r  ^DrD  c              #   &   K   | ]	  }|d      yw)r   Nr   rA   s     r   rD   rD      s0       \J  }~  ]^  _w  ]x  \JrD  c              #   &   K   | ]	  }|d      yw)r   Nr   rA   s     r   rD   rD      s0       ]F	  yz  ^_  `s  ^t  ]F	rD  c              #   &   K   | ]	  }|d      yw)r   Nr   rA   s     r   rD   rD      s0       [	F
  y	z	  \	]	  ^	s	  \	t	  [	F
rD  c              #   F   K   | ]  }|d    D ]  }|d   dk(  sd   yw)r*   r   r   rc   Nr   )rB   rC   r   s      r   rD   rD      sv       nk  uv  KL  MV  KW  nk  FG  [\  ]f  [g  ij  [j  op  nk  op  nks   !	!)
candidatesr   r  exact_duplicatessemantic_duplicatesr   provenance_errorsplaceholdershallucinationsdensity_batch_mediandensity_batch_meandensity_batch_mindensity_batch_maxzero_ku_sourcesc              #   ,   K   | ]  }|d    r	d  ywr%   rc   Nr   rA   s     r   rD   rD      s7       tj  {|  RS  Th  Ri  uv  tj   
c              #   ,   K   | ]  }|d    s	d  ywr\  r   rA   s     r   rD   rD      s7       wi  ~  QR  Sg  Qh  xy  wir]  c              #   &   K   | ]	  }|d      yw)review_stateNr   rA   s     r   rD   rD      s0       f  VW  @A  BP  @Q  frD  c              #   &   K   | ]	  }|d      ywr   r   rA   s     r   rD   rD      s0       h  XY  @A  BR  @S  hrD  c              #   &   K   | ]	  }|d      yw)
gold_stateNr   rA   s     r   rD   rD      s0       }b  RS  ~  @L  ~M  }brD  )r   r   r   r   r!  ku_totalreal_kuslegacyr  r   goldr#  r$  )r   sources_selectedsources_processedrn   	aggregate
live_finalcross_source_observationzp39-final-metrics.jsonrj  rk  rd  r:  r   )outselectedr   final_ku_totalcross_source_pairsr#  r   )rq   )T)importlib.utilr   r!   r   rv   ossysshutil
statistics
subprocessr-  r
   	itertoolspathlibr   collectionsr   r   r#   r   mkdirEXWRr   r   r   r$   r  Rowrow_factoryr   r   allku	processedrf  rx   p35r   priorr   p34excludedr/   ysr   r   r   r  r   r   r.   r   r   obsr   rW   rglobr   relative_toas_posixr@   r   any
read_bytesbdecoder0   	Exceptionrw   rz   r]   selrX   	enumerater\   poprZ   r   r   r   backupcopy2r{   r   prehashpreconpre_fkr   pre_integritylist	pre_queuer   pathgetsizerY   baser   r   resultsr   r   conafterfk	integrityqqrowsr&  krowsgroupsrV   rC   	obs_pairsitemsr   r[   srccombinationsaobservation
final_rowsfinal_fkfinal_integrityr   permedianmeanminr   finalprint)rC   r\   r   s   000r   <module>r     sy   d d d d d d  +0d;m6n#pspypy  CG  QUpy  qV*/cB J Fr) L$gkk		"ABC1tAwCns  Pmij{|  ~R  |S  XY  Zk  XlPQRcPd  PmY  UZ  vO  PQ  ^_  `t  ^u  z{  |M  zN  wx  yJ  wK  vO  ouDIIdXbhijruvy  {J  wK  wx  il  mv  iw  Px  de  QR  S^  Q_  Px  sylq  B  CF  CJ  CJ  KY  Z\  C]  ^  {~  ir  sy  iy  z  i  @C  iC  `h $	
	 XA,AEE*=$>8O155.%,2H2JbTwNw  zBq'qy9I<[^_`ag_h_o_o_q[r1\?  IJtu  wG  uH  LN  LU  LU  VW  LXX d()	F#	$ ]A]]5""$+c/c8Os~~&9:c  B^  U]  B^  ?^  `h	a!((7)(<	!HQJ  \[  \]+<  \\$  \wWZ  \[abc  \du  wEv}  wE  wE  FG  wH  wR  wR  wT  \  Uf  gu  gn  gu  gu  vw  gx  gB  gB  gD  \  ET  UX  \  Yj  ko  \  pC  DH  \  IY  Zo  \  pD  EI  \  JV  W\  ]^  W_  \  `l  mp  qr  ms  \  tD  EF  \  GX  Y[  \  ]]1
 	CLc#s8S=C5A;56;<PaCb=bS> ZCAaaeAg1W:"Qq/?fUYHZ Z'ce  Ny  UX  kn  wz  J{  eh  ik  el  B  CF  G  ZH  SV  sx  yz  {|  s}  a~  no  kl  SV  zh  NO  Z[  \c  Zd  fg  Zg  {|  }H  {I  zh  bi  a~  	 % %jdjjUVae&f g
%%|v||Bv'> E er
24FfV^^TlEmEvEvExyzE{}  HL  MS  M[  M[  \_  M`  Ha  ~G  ci  co  co  cq5Wrwwr7J]jvy  {A  wB  [i  [b  [i  [i  jt  jn  jt  jt  u{  FJ  jK  jR  jR  jT  [U  [_  [_  [a  rx  y{  r|  OQ  ae  fh  fj  fr  fr  sH  fI  aJ  uw  uy  uA  uA  BY	  uZ	  \[	  pq  ]`  ab  cd  ae  ]f  gh  ij  gk  ]k  \[	  n	q	  r	w	  n	x	  G
J
  W
\
  K
|
  R
S
  d
e
  f
z
  d
{
  L
M
  K
|
  G
}
  NQ  RX  NY  Z ) )*$**TuTU`d*e f$011F\9]3] #$1anQ$	
 	BG.~` 
q KAQt"$cc
ckkJb>c>l>l>nop>qIuyz}  {F  {F  G\  {]  v^st  }@  }H  }H  I`  }a  db  xy  eh  ij  kl  im  en  op  qr  os  es  db  `c  jn  or  oz  oz  {~  o  j@  di  BE  BK  BK  BM  ]^  qz  F	I	  J	Q	  F	R	  b	i	  k	q	  b	q	  D
I
  K
T
  D
T
  ]
^
  i
l
  
B  CD  EP  CQ  
R  TV  
V  
w  [^  _b  cd  ep  cq  _r  [s  uw  [w  FG  HU  FV  hq  sw  hw  |C  EK  |K  PU  W`  P`  ef  gt  eu  wO  eO  ^d  U[  T\  OS  hl  ms  ht  ^_  `f  ^g  wz  }G  HI  JM  GN  NS  {T  wT  v`  v`  ak  ae  ak  ak  lm  {@  HI  TX  aY  vZ  \c  \j  \j  kl  \mL&-?D	-I JJK D#++  /k  #l  mQa  m%  or  ox  ox  oz  CN  OS  CT  |Bl	 5As1[>*+2215 5
	lln zFBs$'(qa(3s8A:#Y##C* zcaA/00)2B2B1T7Z[\`Zamn  pA  nB  NO  Pa  Nb  vx  Dy  3zzz
 %'K`cdi`j  RU  V_  R`  y|  }F  yG  fg  CD  \]  yq  zC  DH  EH  zI  J(( 4 4ZTZZY^fgrv5w xDs{{3R'ST!d1gT*X\]`]h]h  j  ^@  YAVW  `c  `k  `k  lC  `D  GE  [\  HK  LM  NO  LP  HQ  RS  TU  RV  HV  GE  CF  PU  VY  PZ  GO  lo  lw  lw  xP  lQ  lZ  lZ  l\  ]^  l_  \k  ad  aj  aj  al HO  P  CDCOvijktiuOvLvQc!H1Y<!HHH|}}  PESSSZux  y`  X_  y`  v`  il  lO  GN  lO  iO  Z]  ]A  y@  ]A  ZA  UX  X  w~  X  U  VY  YI  AH  YI  VI  [^  ^D  |C  ^D  [D  Y\  \J  BI  \J  YJ  Z]  ]F	  ~E	  ]F	  ZF	  X	[	  [	F
  ~	E
  [	F
  X	F
  ^
o
  ^
h
  ^
o
  ^
o
  p
s
  ^
t
  JY  JT  JY  JY  Z]  J^  sv  wz  s{  PS  TW  PX  kn  nk  zA  nk  kk  hl  GM  NP  GQ  \^  \c  \c  \k  \k  ln  \o  BQ  ]`  ai  ]j  zB  DJ  zJ  VY  Zd  Ve  qt  tj  @J  tj  qj  tw  wi  CM  wi  ti  sw  x  f  [e  f  xf  sg  sw  x  h  ]g  h  xh  si  qu  v}  }b  Wa  }b  vb  qc  lm  x{  z|  Xc  d ) )*$**UUVae*f ginoyosoyoy  BE  FI  BJ  VY  di  ju  dv  wG  dH  Z_  `l  Zm  nx  Zy  OZ  [@  OA  JK  X]  ^j  Xk  lw  Xx  {y  GL  pM  jNk D  Pm  vO Px 	8 6 zh  a~ \[	  K
|
 %z db m ) U  GE Ps   l.3
l3>l3l3
l8l8#l8l=(#m
m

mm"m)m?m$
m#/m#m(m-!m2m7!m<n46nm
mm