
    辐j                        d dl Z d dlZd dlZd dlZd dlZd dlmZmZ dZdZdZ	dZ
d ZdAdZd	 Zd
 Zd ZdBdZd Zd Zd Zd Zd Z ej,                  ej.                        j1                         Z eeg d      Z ee      Zi ZeD ]  Zed   ed   ed   ed   xs i j;                  d      ed   xs i j;                  d      fZej?                  eed    ed   ed   ed   ed   ed   xs i j;                  d      ed   xs i j;                  d      d g i d
      Z e d xx   dz  cc<   e d!   jC                  ed"          ed#   red#   j;                  ed         ndZ"e d$   j;                   e#e"      d       dz   e d$    e#e"      <    ejI                         D ]2  Z e d!   dd% e d!<    e% e&e d$   jO                         d& '            e d$<   4  ee      Z(e jR                  e(_*         e&eD  ch c]W  } | d#   j;                  | d   xs i j;                  d            } e+|e,e#f      s9 e#|      j[                         sP e,|      Y c}}       Z.g Z/e.D ]  Z0g Z1d(D ]h  Z2e2 ee(      v s ee(e2      D  cg c]  } | d)   	 c} Z3d*e3v s-e1jC                  e2e(ji                  d+e2z   d,z   e0f      jk                         d    d-       j e/jC                  e0e1d.        e(jm                          ee e7e      ed/ e8ejI                               e.e/ ee       e        e        e       d0
Z9d1efd2e	ffD ]{  \  Z:Z;	  ee;      Z< e7e<       ejz                   ej|                  e<D  cg c]  } | d   | d"   | d   | d   f c} d34      j                               j                         d5e9d6e:z   <   }  eCe
d8d9:      j                   ej|                  e9d;dd3<              eE ej|                  e
e e7e       e8ejI                               e.e9j;                  d=      e9j;                  d>      d?d;@             yc c}} w c c} w c c} w # eA$ rZBd7 e#eB      ie9d6e:z   <   Y dZB[B2dZB[Bww xY w)C    N)datetimetimezonez+/opt/struktur/youtube-research/knowledge.dbz;/opt/struktur/content-extraction/data/content_extraction.dbzO/opt/struktur/reports/aa043-p34/20260827T223847Z/knowledge.db.pre-migration.bakz/tmp/aa043_p34r1_audit.jsonc                 <    t        j                  d| z   dz   d      S )Nzfile:z?mode=roT)uri)sqlite3connect)paths    $/opt/struktur/AA-043-P34-R1-audit.pyror   
   s    ??74<
2t<<    c                     t         j                  | _        | j                  ||      D cg c]  }t	        |       c}S c c}w Nr   Rowrow_factoryexecutedict)csqlargsxs       r
   rowsr      s0    ++AMYYs401DG111s   ?c                     t         j                  | _        | j                  d|z   dz         D cg c]  }t	        |       c}S c c}w )Nzpragma table_info()r   )r   tr   s      r
   colsr      s9    ++AMYY';A'=c'ABCDGCCCs   Ac                 P    | j                  d      D cg c]  }|d   	 c}S c c}w )Nz\select name from sqlite_master where type='table' and name not like 'sqlite_%' order by namer   )r   )r   r   s     r
   tablesr      s5    QYY  0N  &O  Pqt  P  P  Ps   #c                 0    t        | t              r| d d S | S )N   )
isinstancestr)vs    r
   safe_sampler$      s    !C4C.Hr   c                 n   t        |       }t        |      }|xs |}| |i g d}|D ]|  }||vrt        ||      t        |d|z   dz         |j	                  d|f      j                         xs d gd   d|d   |<   |d   |   d   D ]  }|d	   j                  d
|i|        ~ |j                          |S )N)r	   r   tables_detailforeign_keyspragma foreign_key_list(r   z*select sql from sqlite_master where name=?r   )columnsfksr   r&   r*   r'   table)r   r   r   r   r   fetchoneappendclose)r	   target_tablesr   tsdr   fks          r
   	db_schemar3      s   hA6!9M,?RmBrDA 7B;)-aa@Z[\@\]`@`9aijirir  t`  bc  ae  jf  jo  jo  jq  j{  vz  u{  }~  i  @/1O$Q'. 	7Bn$$ga%5"%56	77
 GGIaxr   c           
         t        |       }t        j                  |_        |j	                  d      D cg c]  }t        |       }}g }|D ]  }|d   }|d   }|d   t        |d|z   dz         }t        fd|D        d       }	i }
|d	 |j	                  d|j                  d	d
      z   dz   |f      j                         }|r*|j                         D ci c]  }|t        ||          }
}|j                  |||d   |	|
d        |j                          |S c c}w c c}w # t        $ r}dt        |      i}
Y d }~Td }~ww xY w)Nzpragma foreign_key_checkr+   rowidfkidr(   r   c              3   4   K   | ]  }|d    k(  s|  yw)idN ).0r   r6   s     r
   	<genexpr>zfk_rows.<locals>.<genexpr>'   s     1q1T7D=1s   zselect * from ""z""z" where rowid=?_read_errorparent)r+   r5   r>   r6   r2   child)r   r   r   r   r   r   r   nextreplacer,   keysr$   	Exceptionr"   r-   r.   )r	   r   r   
violationsresultr#   r   ridr*   r2   r?   rrker6   s                 @r
   fk_rowsrJ   !   sP   hAgkk!"+E!FGAQGJGF 	f
G*!G*c1V9d$qA[\]A]^aAa:bc1C1$7?@99.qyyT/BBCTTVYU[\eegBGGIFqaBqE 22FuFqakSU^cde	f GGIf} H G?-A)?5?s0   DAD'D",D'"D''	E0EEc           
         t        |       }g }t        |      D ]  }t        ||      D ]  }|d   j                         }d|v s	d|v s|dv s#	 |j	                  d|z   dz   |d   z   dz         j                         d   }|j	                  d	|d   z   d
z   |z   dz   |d   z   dz         j                         d   }|j                  ||d   |d   ||d         |j                          |S # t        $ r}d x}}Y d }~Ed }~ww xY w)Nnamevideo_id
youtube_id)	source_idsource_identitycanonical_urlselect count(*) from "	" where "" is not nullr   select count(distinct "	") from "type)r+   columnrW   nonnulldistinct)	r   r   r   lowerr   r,   rC   r-   r.   )	r	   r   outr   colnrY   rZ   rI   s	            r
   logical_refsr_   1   sK   hABAY v!9 	vC&k!AQ,!"3q<k7k=II&>q&@&LSQW[&XYh&hirrtuvwGYY'@V'L['XYZ'Z[f'fgjkqgr'r  tC  (C  D  M  M  O  PQ  RH

As6{#f+X_kstu	vv GGIcz !<'<wx<s   A+C**	D3C<<Dc                     d} g }t        j                  |       D ]  \  }}}|D cg c]  t        fddD              r c}|d d  |D ]  }|j                  d      sd|j	                         v sd|j	                         v s:t         j
                  j                  ||      }	 |j                  |t         j
                  j                  |      t        j                  t         j
                  j                  |      t        j                        j                         d         t!        |d 	      S c c}w # t        $ r Y w xY w)
Nz/opt/struktur/reportsc              3   B   K   | ]  }|j                         v   y wr   r[   )r:   r   r1   s     r
   r;   z#backup_inventory.<locals>.<genexpr>B   s     +\qAN+\   )logsnode_modules)z.db.bakz.sqlite	knowledgeyoutube)r	   sizemtimec                     | d   S )Nrj   r9   )r   s    r
   <lambda>z"backup_inventory.<locals>.<lambda>I   s
    QwZ r   key)oswalkanyendswithr[   r	   joinr-   getsizer   fromtimestampgetmtimer   utc	isoformatOSErrorsorted)rootfoundbasedirsfilesr1   fps        `  r
   backup_inventoryr   >   s7   	 D
E774= %T% ]q+\D[+\(\]Q 	%Azz23	9QU^bcbibibkUk'',,tA&%LL"''//!2DXMcMcdfdkdkdtdtuvdw  yA  yE  yE  NF  NP  NP  NR  "S  T		%% %/00 ^ $$s   D;D;)A?E  	EEc                  F   g d} g d}g }| D ]  }t        j                  |      D ]  \  }}}|D cg c]  }|dvs|j                  d      r| c}|d d  |D ]  }|j                  d      st         j                  j	                  ||      }	t        |	dd      j                         j                         }
t        |
d	      D ]@  \  }t        fd
|D              s|j                  |	|j                         d d d       B    |S c c}w # t        $ r Y w xY w)N)z/opt/strukturz/etc/systemdz/home/hermes)zINSERT OR REPLACE INTO videoszREPLACE INTO videoszDELETE FROM videoszPRAGMA foreign_keyszforeign_keys=zDROP TABLE videoszALTER TABLE videos)z.git__pycache__rd   reportsbackupsz.venvvenvrf   )z.pyz.sqlz.servicez.timerz.shz.tomlz.yamlz.ymlutf-8ignore)encodingerrors   c              3   ^   K   | ]$  }|j                         j                         v  & y wr   rb   )r:   termlines     r
   r;   zcode_search.<locals>.<genexpr>X   s"     JD4::<4::<7Js   *-i  )r	   r   text)ro   rp   rr   r	   rs   openread
splitlinesrC   	enumeraterq   r-   strip)rootstermshitsr{   r}   r~   r   r1   r   r   linesir   s               @r
   code_searchr   K   sA   
9E fE	D 
S!wwt} 		SODe $  N11q(qz{  {E  {E  FL  {MQ  NDG Szz"abdl'',,tA&7(CHHJUUWU'a0 SFAdJEJJAQdjjl4C>P$QRSS		S
S K N
 !*(*s"   	DDD*D	D 	D 	c                     t         j                  j                  t              sddiS t	        t              } t        |       }d|i d}|D ]  }dt        | |      i|d   |<    |D ]  }|d   |   d   D cg c]  }|d   	 }}|D cg c]  t        fdd	D              s }}|sF||d   |   d
<   |D ]n  | j                  dz   dz   z   dz         j                         d   | j                  dz   dz   z   dz   z   dz         j                         d   d|d   |   <   p  | j                          |S c c}w c c}w # t        $ r Y w xY w)NexistsFT)r   r   detailsr)   r   rL   c              3   B   K   | ]  }|j                         v   y wr   rb   )r:   qr^   s     r
   r;   zce_audit.<locals>.<genexpr>d   s     't1QWWY'trc   )rM   rN   rO   urltitleidentifier_columnsrR   rS   rT   r   rU   rV   )rY   rZ   )ro   r	   r   CEr   r   r   rq   r   r,   rC   r.   )r   r0   r1   r   r   namesr^   idcolss         ` r
   ce_auditr   \   s   77>>"x&66fAbb2FA ."4!9-)Q.  '"#I,q/)"<=Qqy== uaC't@s't$tuu28AiLO01 '3499=UVW=WXc=cde=efu=u3v33  4B  CD  4E  QR  QZ  QZ  [t  uv  [v  wB  [B  CD  [D  EP  [P  QR  [R  Sb  [b  Qc  Ql  Ql  Qn  op  Qq  )rQy\!_Q''' GGIax >u
 !&$&s%   6D>	E#E<A,E	EE)videoschaptersvideo_linksknowledge_unitssource_versionssource_processing_registryreconciliation_decisionsgraphiti_import_queuer+   r6   r>   r2   fromtor            	on_update	on_delete)
r+   r6   r>   child_columnparent_columnr   r   r   rowidsmissing_parent_valuesr   r   r5   r?   r      c                     | d    S )Nr   r9   )kvs    r
   rl   rl   x   s    xz{|x}w} r   rm   )r   r   rL   rM   zselect count(*) from z where video_id=?)r+   count)rM   
child_refs)r   r   )

checked_atschemaforeign_key_checkdistributionmissing_video_idsvideo_child_infological_referencesr   r   ce
productionbackupT)	sort_keys)fk_countfingerprintcomparison_errorwr   )r   F)ensure_asciiindentr   comparison_productioncomparison_backup)r\   r   r   r   r   prod_vs_backupr   )r   )r9   r   )Fr   jsonro   rehashlibr   r   Kr   BACKOUTr   r   r   r   r$   r3   rJ   r_   r   r   r   nowrw   rx   prod_schemaviolbyr   getrn   
setdefaultzr-   valr"   valuesr   rz   itemsr   r   r   r!   intisdigitmissing_ids
video_infovidrefsr   csr   r,   r.   lenlistprodrL   r	   vvsha256dumpsencode	hexdigestrC   rI   r   writeprint)r   r#   s   00r
   <module>r      sD   % % % '/@V!=2D P 1"  HLL((*a  p  qQZ	 SAj61X;$2':':6'BAdGMrCVCVW[C\]==c!fCFCFRUVWRXilmnio}~  @D  ~E  ~K  IK  }P  }P  Q\  }]  kl  mq  kr  kx  vx  j}  j}  ~I  jJ  RS  ]_  xz  {  |6Aq{!!!G*- zQwZ^^CFt&'(?&@&D&DSXa&PQR&RCH%S 
 @Ax["8$vaH_F`FfFfFhm}?~:q!89@ Q%w{{D  Qq1W:>>1T7=bBUBUV\B]3^Qcmnoqtuxpycz  @C  DE  @F  @N  @N  @PCF  Q  R
 7C
$ GQ&)^q)$Qqy$2BQqyyAXYZAZ[nAnpsou7v77  8B  CD  8E  %F  GG s567 	TZ^@_ostvt}t}t  pA  Va  u  Ua  bc  Ud  xH  xJ  Yd  Yf  lt  lv  w"HT?3 BIDBT]"R~w~~^h^b^h^h  bd  je  ]^klmtkuvwx  wA  BC  DL  BM  NO  PV  NW  kX  je  pt  _u  _|  _|  _~  P  PI  PI  PK  .LDt+,B Sg  $ $ZTZZ%q[_%` a jdjjD	QUVXV_V_VaQb  xC  UY  U]  U]  ^u  Uv  AE  AI  AI  J]  A^  _  mr  s  t% Q % jeA'#a&1A]4/0AsB   5:Q
0Q

Q
=Q	+Q.Q1QQQ3Q..Q3