
    ;j                     	   d dl Zd dlZd dlZd dlZd dlZd dlZd dlmZ d dl	m	Z	m
Z
 dZ ed      Zej                  d   Zh dZej                   j#                  dd	      Zej                   j'                  e      Zej*                  j-                  e        ej.                  d
ez   dz   d      Zej2                  e_        ej7                  d      D  ch c]  } | d    	 c} Zej7                  d      D  ch c]  } | d    	 c} Zd Zd Zg Z ejC                         D ]  Z"de"d   z   Z#e"jI                  d      Z%e#ev s+e#ev s'e#ev s#e"jI                  d      r ee"d         dk  se%dv rLejM                  e"d      Z' ee'd         e'd<    e(e'd   jS                               e'd<   d e'd<   e jU                  e'        g Z+ e,ej[                  d            D ]  Z.e.j_                  e      ja                         Z1de1z   Z#e#ev s(e#ev s$e#ev s e1je                  d      s e3d  d!D              rUe.ji                         Z5e5jm                  d"d#$      Z7 ee7      dk  re+jU                  i d%e#d&d'ddd(e1de7d) ejp                  e5      js                         d* ejp                  e5      js                         d+e1d,dddd-d.d/dd ee7      d e(e5      dd d0g          ee d1       ee+d2      z   Z: e(e:      d3k7  s e(e:D ch c]  }|d%   	 c}      d3k7  r& e;d4 e< e(e        e(e+       e(e:      f      z          e=e:      D ]B  \  Z>Z'e>d5z  dz   e'd6<   e>d5z  dz   e'd7<   e'j                  dd       e'jI                  d0g       e'd0<   D d8 e	j                  e
j                        j                         d9d1d2d3d: e,e       e,e       e,e      e: eCdd;      D cg c]!  }|e:D cg c]  }|d6   |k(  s|d%    c}d<# c}}d=	ZD ee      j                  j                  dd>        ee      j                   ej                  eDd?d@dA              eI ej                  e e(e        e(e+       e(e:      eDdB   D cg c]  } e(|dC          c}dD             yc c} w c c} w #  Y xY wc c}w c c}w c c}}w c c}w )E    N)Path)datetimetimezonez+/opt/struktur/youtube-research/knowledge.dbz/opt/obsidian-vault   >
   youtube:3yqQ664R1hgyoutube:FVuqEATzMWIyoutube:ZyNdWhKqlEsyoutube:fxc4yds-DUsyoutube:qmPAFHeiU1Ayoutube:z-C_q4LzPlA*obsidian:Systemstruktur/SYSTEM-REFERENZ.md8obsidian:Systemstruktur/Review-Routing-Decision-Units.mdIobsidian:Systemstruktur/Knowledge-Pipeline-Review-Unter-100-2026-07-22.mdVobsidian:05-Ressourcen/Hermes-Sessions-2026-07/TranscriptBackfillWorker-untersuchen.mdp33z%/opt/struktur/AA-043-P33-extractor.pyzfile:z?mode=ro&immutable=1T)urizoselect distinct source_identity from knowledge_units where legacy_placeholder=0 and source_identity is not nullzoselect distinct source_identity from knowledge_units where legacy_placeholder=1 and source_identity is not nullc                 >    t        t        j                  |             S N)lenmwords)ts    "/opt/struktur/AA-043-P35-select.pyr   r   
   s    S_$    c                    t        | d       } t        |       |k\  r%t        d      D cg c]  }| |dz  |dz   dz    c}ng }g }|r+t        d      D ]  |j                  fd|D                n| d | }|d | S c c}w )Nc                     | d   | d   fS )N
word_count	source_id )xs    r   <lambda>zpick.<locals>.<lambda>   s    !L/!K.!A r   )key
   r      c              3   F   K   | ]  }t        |      k  s|     y wr   )r   ).0bjs     r   	<genexpr>zpick.<locals>.<genexpr>   s     ?aAh!?s   !
!)sortedr   rangeextend)itemsnibinsoutr(   s        @r   pickr2      s    eAB36u:q=U2Y/uQrT1Q3(/b8?aSZZ?d???!9sBQ 0s   A>zyoutube:
youtube_idtranscript_status
transcript   )unavailablerejectedfailedp35_youtubetextr   
text_bytesprior_ku_countz*.mdz	obsidian:zYouTube-Research/c              #   ,   K   | ]  }|t         v   y wr   )rel)r&   r    s     r   r)   r)      s        b~opbcgjbj  b~s   )z_hermes-session-testzTEMPLATE.mdz_deleted_backup/z05_SESSIONS/zutf-8replace)errorsr   source_typeobsidiantitlecontent_versionraw_source_hashobsidian_pathartifact_existsselection_kindp35_native_obsidiansource_db_video_idsource_versions      2   zselection_failed r#   batchbatch_indexzp35-v1zldeterministic source-id tie-break with round-robin word-count strata; P34/legacy/processed/unusable excluded)youtubenative_obsidiantotal   )rP   
source_ids)	selection_versionselected_atselection_rulesource_countsexcluded_p34processed_before_selectionlegacy_sources_before_selectionsourcesbatches)parentsexist_okF   )ensure_asciiindent	sort_keysr_   rV   )outputyoutube_candidatesobsidian_candidatesselectedr_   )Jimportlib.util	importlibsqlite3jsonhashlibossyspathlibr   r   r   DBVAULTargvOUTP34utilspec_from_file_locationspecmodule_from_specr   loaderexec_moduleconnectcRowrow_factoryexecute	processedlegacyr   r2   ysyoutube_rowsrsidgetstcanonical_youtubesr   encodeappendobsr*   rglobprelative_toas_posixr?   
startswithany
read_bytesr'   decoder   sha256	hexdigestsel
SystemExitstr	enumerater/   popnowutc	isoformatr+   r1   parentmkdir
write_textdumpsprint)r   r    r/   s   000r   <module>r      s   1 1 1  &0=R8S%Y\YaYabcYdUX W^^++E2YZ^g^l^l^}^}  C  _D\]  FJ  FQ  FQ  F]  F]  ^_  F`'//'"*33=W[[q}  $U  V  WA1Q4  W	  op  ox  ox  yj  ok  `l  jk  ab  cd  ae  `l  Y_ $ 	
	 XA,AEE*=$>3J#"cVm155;NRWXYZfXgRhijRjnp  uX  oX  Zbq'qy9I<[^_`ag_h_o_o_q[r1\?  IJtu  wG  uH  LN  LU  LU  VW  LXX 	F#	$ VA]]5""$+c/c3J#"cVms~~FY7Z^a  b~  u}  b~  _~  @H	a!((7)(<	!HQJ  U[  U]:  Ul4  UPS  UTZ[\  U]no}ovo}o}~  pA  pK  pK  pM  U  N_  `n  `g  `n  `n  op  `q  `{  `{  `}  U  ~M  NQ  U  Rc  dh  U  i|  }A  U  BR  Sh  U  i}  ~B  U  CO  PU  VW  PX  U  Ye  fi  jk  fl  U  m}  ~  U  @Q  RT  U  VV 	BKSs8R<341+45r9L_`cehikelmpqtmuvyz}v~d  aA  MA  BB  <BS>  MCAaaeAg1W:"Qq/?fUYHZqrqvqv  xI  JL  rM\]^o\p  M!X\\0J0T0T0V  iW  su  HJ  SU  hV  fl  mp  fq  OU  V_  O`  CI  JP  CQ  \_  }B  CD  EF  }G  jH  xy  uv  ]`  Dr  XY  de  fm  dn  pq  dq  EF  GR  ES  Dr  ks  jH  I S	    tT  2DI4H4HTWejrs  C  JD  5E  GL  MW  MQ  MW  MW  be  {~  A  {B  Y\  ]`  Ya  mp  qt  mu  _b  cl  _m  @n  Z[  AD  EF  GS  ET  AU  @n  Xo  Mp  Gq7 W  `l(  5 Dr  jH @nsB   R*6R/+#R4
R<%
S/S=SS
S4R9S