
    Oj"                        d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	Z	d dl
Z
d dlmZ  ed      Zedz  Zedz  Z ed      Z ed      Z ed      Z ej
                         Z ej(                  ej+                               Zg Zdad	 Zd
 Zd Zd Zd Zd Zd Zd Z d Z!d Z"d Z# ejH                   e#              y# e%$ r)Z&d e'e&      da edd e       d        e"         dZ&[&ww xY w)    N)Pathz&/opt/struktur/obsidian-graphiti-importzobsidian_graphiti_import.dbzimport_run_block.jsonz//opt/struktur/graphiti/request-data/requests.dbz;/opt/struktur/graphiti/request-data/processing-events.jsonlzN/opt/struktur/knowledge-pipeline-aggregator-staging/AA-RETRY-RECOVERY-FINAL.mdc                      t         j                   j                  t         j                  j                        j	                         S N)datetimenowtimezoneutc	isoformat     #/tmp/retry_recovery_orchestrator.pyr   r      s-    (##''(9(9(=(=>HHJJr   c                  @    t        j                  dt         dd      S )Nfile:?mode=roTuri)sqlite3connectDBr   r   r   dbr      s    5H!5$?
?r   c                     t               }t        j                  |_        |j	                  d| f      j                         }|j                          |rt        |      S d S )Nzselect * from files where id=?)r   r   Rowrow_factoryexecutefetchoneclosedict)qidcrs      r   qrowr!      sa    4w{{aii0PRUQW&X&a&a&cAefelelen  CDw{|}w~  qN  JN  qNr   c                     t        j                  dt         dd      }t         j                  |_        |j                  dt        |       f      D cg c]  }t        |       }}|j                          |S c c}w )Nr   r   Tr   zDselect * from graphiti_requests where queue_id=? order by started_at)	r   r   REQr   r   r   strr   r   )r   r   xrss       r   reqrowsr'   
   s    
??U3%x(T2'++AMdedmdm  ou  wz  {~  w  vA  eB  SC_`SWXYSZ  SCr  SC  EF  EL  EL  EN  WY  PY  SCs   A9c                      t        j                  t        j                  j	                  dd      j                               S )Nzhttp://127.0.0.1:8644/health   timeout)jsonloadsurllibrequesturlopenreadr   r   r   healthr2      s/    TZZ 6 67U^` 6 a f f hiir   c                 \   t        j                  d| d   z   | d   | d   | d   d      j                         }t        j                  j                  d|ddi	      }t        j                  t        j                  j                  |d
      j                               j                  dg       S )N	obsidian_
import_keyrelative_pathcontent_hash)namer5   obsidian_pathr7   z$http://127.0.0.1:8644/episodes/checkzContent-Typezapplication/json)dataheadersr)   r*   episodes)
r,   dumpsencoder.   r/   Requestr-   r0   r1   get)rowr:   r    s      r   checkrB      sO   
jjS%66CDUfijyfz  KN  O]  K^  _  `  g  g  i  ms  m{  m{  mC  mC  Dj  pt  ~L  M_  }`  mC  ma  kl  jn  jt  jt  u{  uC  uC  uK  uK  LM  VX  uK  uY  u^  u^  u`  ja  je  je  fp  qs  jt  ctr   c                     t        t        t              dz         }|j                  t	        j
                  | dd      dz          t        j                  |t               y )Nz.tmpF   ensure_asciiindent
)r   r$   BLOCK
write_textr,   r=   osreplace)payloadtmps     r   setblockrO      sF    	#e*V
cnnTZZUZ[-\]a-abdfdndnorsxdyr   c                 F    t        ddt        |       t               d       y )NF authorized_single_retry_recovery)blockedreasonqueue_idauthorized_at)rO   r$   r   )r   s    r   
clearblockrV      s,    X8Zfijmfn  @C  @E  F  Gr   c                    g }t         j                         s|S t         j                  d      j                         D ]=  }	 t	        j
                  |      }|j                  d      | k(  r|j                  |       ? |S #  Y FxY w)NrL   )errors
request_id)EVENTSexists	read_text
splitlinesr,   r-   r@   append)ridoutliner%   s       r   active_eventsrb      sy    Y/::< T	ZZ1eeL3

1
 	 $s   :A??Bc                  N   dddt         j                   j                  t         j                  j                        j	                          ddddg} t
        r| g dz  } n| g dz  } | g dz  } t        D ]o  }| j                   d	j                  d&i d
D ci c].  }|t        |j                  |d            j                  dd      0 c}d|j                  dd      i       q t
        r| dddt
        d    dt
        d    dgz  } | ddddddddt        j                  t        j                  t        j!                               dd      dddgz  } 	 t#               }t%        |j'                  d      j)                               }|j'                  d       j+                         d!   }|j-                          | j                  d"| d#| d"       t0        j3                  d%j5                  |       d%z          y c c}w # t.        $ r}| j                  d$|        Y d }~Od }~ww xY w)'Nz# AA-RETRY-RECOVERY-FINAL u   Prüf-/Startzeit: `z` UTCz## Abschlussstatus)u$   **RETRY-RECOVERY ABGEBROCHEN –**  u   **FAIL-STOP AKTIV –**  z#**WEITERE QUELLEN NICHT GESTARTET**rd   )u*   **RETRY-RECOVERY 44/44 ERFOLGREICH –**  u1   **RETRY-BESTAND VOLLSTÄNDIG ABGEARBEITET –**  z **FAILED-BESTAND NOCH GESPERRT**rd   )z## Ausgangsbestandrd   z^retry=44, done=16, duplicate=1, failed=123, processing=0, Requests=745, active_request_count=0rd   z## Reihenfolge und Ergebnisrd   z| Queue | Pfad | Request | Run | Start | Ende | Dauer s | HTTP | Status | LLM | Edge total/completed | Rate wait s | Episode | Queue-Endstatus |z>|---:|---|---|---|---|---|---:|---:|---|---:|---|---:|---|---|z| {queue_id} | {relative_path} | {request_id} | {run_id} | {started_at} | {completed_at} | {duration} | {http_status} | {status} | {llm_request_count} | {edge_total}/{edge_completed} | {rate_gate_wait_seconds} | {episode_uuid} | {queue_status} |)rT   r6   rY   run_id
started_atcompleted_athttp_statusstatusllm_request_count
edge_totaledge_completedrate_gate_wait_secondsepisode_uuidqueue_status|/durationz## Exakter AbbruchpunktzQueue `rT   z`: rS   z%Keine weitere Quelle wurde gestartet.z## Schutz- und Endstatusup   Alle Selektionsbefehle verwendeten ausschließlich `--once --ids X --force-id X --limit 1 --hold-after-success`.uq   Keine failed-Quelle wurde als Ziel ausgewählt; die Allowlist stammte aus `files.status="retry" ORDER BY id ASC`.uf   Provider-/Rate-Gate-Status wurde vor jeder Quelle geprüft; bei Fehler wäre sofort Fail-Stop erfolgt.zRun-Block-Endzustand:z```jsonFrD   rE   z```z## Endbestandz1select status,count(*) from files group by statusz&select count(*) from graphiti_requestsr   `z`; Requests=`zEndbestand nicht lesbar: rH   r   )r   r   r   r	   r
   stoppedresultsr^   formatr$   r@   rL   r,   r=   r-   rI   r\   r   r   r   fetchallr   r   	ExceptionREPORTrJ   join)linesr%   kr   countsrequestses          r   mdr      s   #B)<X=N=N=R=RS[SdSdShSh=i=s=s=u<vv{'|}  BV  WY  ZU  D  DU  Y  Yu  @  @
  @	Q5<<  !_  !X  !_  !_  !  Q_  b`  LM  cd  eh  ij  in  in  op  qs  iu  ev  e~  e~  B  CF  eG  cG  b`  !  jk  jo  jo  pz  {}  j~  !  @	  @	Ur4
@S?TTWX_`hXiWj6k  nU  V  VU2)  ,^  `S  U}  V  XZ  \e  gk  gq  gq  rv  r|  r|  }B  }L  }L  }N  rO  ]b  jk  gl  ns  uw  yH  I  IFD!aii ST]]_`&klktkt  v^  l_  lh  lh  lj  kl  lmbj  op  ov  ov  ox  z  zF  zF  IJ  KQ  JR  R_  `h  _i  ij  Gk  zl499U#D() b`
 	E'@%DEEEs   3G8A<G= =	H$HH$c                     t               } | j                  d      D cg c]  }|d   	 }}d }| j                          t	        |      dk7  r3ddt	        |       dat        dd	t               d
       t                y|D ]O  }t        |      }t               }t        |      }|d   dk7  sB|d   r=|d   r8|s6|j                  d      r%|j                  d      s|j                  d      dk7  r|t        j                  |d   t        |d         t        |d         t	        |      |j                  d      |j                  d      |j                  d      dd      dat        dd	t!        |      t               d       t                 yt#        |      }t%        |       dt!        t&        dz        ddt!        |      dt!        |      dddg
}	t)        j(                         }
t+        j,                  |	ddd      }t)        j(                         }t#        |      }|D cg c]5  }|j                  d       |D ch c]  }|j                  d        c}vs4|7 }}}t	        |      dk(  r|d!   n|r|d!   ni }g }t        |      }|rt1        |j                  d d"            ng }|D ch c]  }|j                  d#       c}t3        d$ |D              }t5        |D cg c]F  }t!        |j                  d%d"            j7                         s-t9        |j                  d%            H c}xs dg      }t3        d& |D              }t        |      }g }|j:                  dk7  r|j=                  d'|j:                          t	        |      dk7  r|j=                  d(t	        |              |j                  d)      t!        |      k7  r#|j=                  d*|j                  d)              |j                  d      d+k7  s|j                  d,      d-k7  r5|j=                  d.|j                  d       d/|j                  d,              t	        |      dk7  r|j=                  d0t	        |              |j                  d      d1k7  r#|j=                  d2|j                  d              t?        fd3d4D              r|j=                  d5       i d)|d6|d6   d |j                  d d"      d7|j                  d7d"      d8|j                  d8d"      d9|j                  d9d"      d:tA        ||
z
  d;      d,|j                  d,d"      d|j                  dd"      d<|j                  d<d"      d%|d=|d>tA        |d;      d?|r|d   j                  d@      n|j                  d?d"      dA|j                  dd"      dBdCjC                  |      }|rtD        j=                  |       |dCjC                  |      dat        ddDt!        |      |j                  d7d"      t               dE       t                tG        t        j                  t
        |dFd              ytD        j=                  |       t        ddGt!        |      |j                  d7d"      t               dE       tG        t        j                  dH|id      dI       R t        ddJt               t	        tD              tD        D cg c]  }t!        |d)          c}dK       t                tG        t        j                  dLt	        tD              id             yc c}w c c}w c c}}w # t.        $ r g }Y w xY wc c}w c c}w c c}w )MNz5select id from files where status='retry' order by idr   F,   PRECHECKzexpected 44 retry IDs, found rT   rS   Tpreflight_failedrR   rS   
blocked_at   ri   retryr7   r5   provider_readyprovider_rate_limitedactive_request_count)ri   hashr5   r<   r   r   r   )rF   )rR   rS   rT   r   python3zimporter.pyz--oncez--idsz
--force-idz--limit1z--hold-after-successiX  )textcapture_outputr+   rY   rd   phasec              3   t   K   | ]0  }|j                  d       dk(  s|j                  d      dk(  s-d 2 yw)r   rl   outcome	completedr   N)r@   .0r%   s     r   	<genexpr>zmain.<locals>.<genexpr>6   sP       >WASTSXSXY`SacsSsxyx}x}  H  yI  KV  yVa  >Ws   888rk   c              3      K   | ]L  }|j                  d       dk(  s|j                  d      dk(  s-t        |j                  dd            dz   N yw)r   rate_gate_waitr   r   duration_msr   i  N)r@   floatr   s     r   r   zmain.<locals>.<genexpr>6   s       NH  vw  EF  EJ  EJ  KR  ES  Ue  Ee  jk  jo  jo  py  jz  |G  jG  OT  UV  UZ  UZ  [h  ij  Uk  Ol  mq  Oq  NHs   AA#Azimporter_rc=znew_requests_for_queue=rT   zqueue_id_mismatch=r   rh      zrequest=rq   z	episodes=donezqueue_endstatus=c              3   &   K   | ]  }|v  
 y wr   r   )r   r%   phasess     r   r   zmain.<locals>.<genexpr>>   s     QfQs   )edge_resolution_failededge_operation_failedterminal_edge_failure_eventr6   re   rf   rg   rr      rj   rl   rm   rn   uuidro   rS   z; source_failure)rR   rS   rT   re   r   )STOPresultsource_verified_wait_nextSUCCESS)flushretry_recovery_completed)rR   rS   r   completed_countcompleted_queue_idsCOMPLETE)$r   r   r   r   lenrt   rO   r   r   r!   r2   rB   r@   r,   r=   boolr$   r'   rV   BASEtime
subprocessrunrx   rb   summaxisdigitint
returncoder^   anyroundrz   ru   print)r   r    idsbaseline_reqr   rA   hepsbeforecmdt0proct1afterr%   ynewrrverifiedeventsrl   rk   waitsendrowrS   r   r   s                             @r   mainr   (   s   4AII&]^_qad__  |@am  BC  BI  BI  BKHbLj<YZ]^aZbYc:de'go{  J\  jm  jo  qp  hq  su  sw  @A cS
3i#68sS]G3~#6c,>OSV^_^c^cdt^uyzy~y~  @W  zX  \]  \a  \a  bx  \y  {|  \|DJJXdSVWeSfNguyz}  K  {L  vM  Y\  ]`  Ya  st  sx  sx  yI  sJ  cd  ch  ch  i@  cA  YZ  Y^  Y^  _u  Yv  0w  EJ  %K  L7  NV  bf  pB  NQ  RU  NV  dg  di  Wj  Nk  mo  mq  z{	&z#Ys4;M7NxX_`cdg`hiuvyz}v~  @I  JM  Nd  -e  jn  js  js  ju  gi  |F  |J  |J  KN  TX  hl  ux  |y  w{  ~B  ~G  ~G  ~I  {}  QX  Y\  Q]  KP  ot  cy  jk  xy  x}  x}  ~J  xK  qw  Sx  lm  TU  TY  TY  Zf  Tg  Sx  xx  de  cy  _b  cyC!SWS#b'b"(c
x35rvvl2.	/2&"(	)Q!%%.	)&#  >W  >W  ;W>  dg  KQ  h{  FG  UX  YZ  Y^  Y^  _k  ln  Yo  Up  Ux  Ux  Uz  il  mn  mr  mr  s  m@  iA  h{  hB  @A  B  dC  Yc  KN  NH  {A  NH  KH  EJ	c&2F	__adoo5F'GHXq[&--"9#c( DEVVJS!6==3EbffZFXEY1Z#[VVH{"bff]&;S&@&--RZ[][a[abj[kZllmnpntnt  vC  oD  nE  QF  CG]Av}}yX%@AZZ6!6==3CFJJxDXCY1Z#[QPQQSYS`S`a~S 
V	*S  
V	_)=  
V	l266R^_aKb  
V	cklnlrlrs{|~l  
V	  AM  NP  NT  NT  Ua  bd  Ne  
V	  ft  uw  u{  u{  |J  KM  uN  
V	  OY  Z_  `b  ce  `e  fg  Zh  
V	  iv  wy  w}  w}  ~K  LN  wO  
V	  PX  Y[  Y_  Y_  `h  ik  Yl  
V	  m@  AC  AG  AG  H[  \^  A_  
V	  `l  mw  
V	  xH  IW  
V	  Xp  qv  w|  }~  q  
V	  @N  ks  PX  YZ  P[  P_  P_  `f  Pg  y{  y  y  @N  OQ  yR  
V	  Tb  ci  cm  cm  nv  wy  cz  
V	  {C	  D	H	  D	M	  D	M	  N	T	  D	U	  
V	&
>>&sDIIf<M#N7PXdh  sC  OR  SV  OW  ac  ag  ag  hp  qs  at  BE  BG  ZH  QI  KM  KO  QV  W[  Wa  Wa  jq  {A  bB  PU  WV  QW  `a	..(d<Wcfgjckuwu{u{  }E  FH  vI  WZ  W\  $]  ^  `e  fj  fp  fp  r{  |B  qC  QV  fW  ^b  `c-c. 
T#=35cfgnco  dk  Gl  _`  HK  LM  NX  LY  HZ  Gl  m  n  pr  pt  v{  |@  |F  |F  HR  SV  W^  S_  G`  ns  |t  vu  ~3 ` Sx  cy 
RH	)  h{ GlsM   ]%]>]]] ]#.]5$.]:
]:
;]?]#]21]2	UNHANDLEDr   Tunhandled_fail_stopr   )(r   r,   r   urllib.requestr.   r   rK   signalsysr   hashlibpathlibr   r   r   rI   r#   rZ   ry   startr-   r\   originalru   rt   r   r   r!   r'   r2   rB   rO   rV   rb   r   r   exitBaseExceptionr   reprr   r   r   <module>r      sK   Z Z Z Z Z 	
23=Z8ZRbfg~b~\a  EI  J{  E|  AD  EI  JG  EH  ~D  QU  Vf  Qg  JPdiikJDJJu'89827tw J ?NY itz G*8 SXXdf N $q'	2HVky|y~=  5A  CE  CG  INNs   /C C5$C00C5