
    UjL                     6   d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlm	Z	 ej                  j                  d d       d dlZd dlmZmZmZmZmZ dZdZ e	d      Zej/                  dd	       d
 Zd Zd Zd ZddZd Zd Zd Ze dk(  r e        yy)    N)Pathz/opt/struktur/youtube-research)NativeTranscriptUnavailable	QuotaWaitTransientSupadataErroraccount_statusfetch_nativez+/opt/struktur/youtube-research/knowledge.dbz,/opt/struktur/youtube-research/.env.supadataz*/opt/struktur/youtube-research/reports/p29T)parentsexist_okc                      t         j                   j                  t         j                  j                        j	                         S N)datetimenowtimezoneutc	isoformat     /tmp/supadata_native_batch.pyr   r      s-    (##''(9(9(=(=>HHJJr   c                      t        t              j                         j                         D ]8  } | j	                  d      s| j                  dd      d   j                         c S  t        d      )NzSUPADATA_API_KEY==   SUPADATA_API_KEY_missing)r   ENV	read_text
splitlines
startswithsplitstripRuntimeError)lines    r   keyr!   	   s`    #Y  "--/ OT	__()$**S2CA2F2L2L2N+NO.//r   c                     t        |       S r   )r   )ks    r   accountr$      s    ~a((r   c                      t        j                  t        d      } t         j                  | _        | j                  d       | S )N   )timeoutzPRAGMA busy_timeout=30000)sqlite3connectDBRowrow_factoryexecute)cs    r   connr/      s.    
??2b!1=aiiHc>dmnfnr   c                     d}g }|r2|ddj                  d |D              z   dz   z  }|j                  |       |dz  }|j                  |       | j                  ||      D cg c]  }t	        |       c}S c c}w )Na,  SELECT e.job_id,e.video_id,e.youtube_id,e.status e2e_status,e.attempts,v.transcript_status,v.transcript_source,v.created_at,e.updated_at,length(trim(coalesce(v.transcript,''))) transcript_len FROM e2e_jobs e JOIN videos v ON v.id=e.video_id WHERE v.youtube_id IS NOT NULL AND length(trim(coalesce(v.transcript,'')))=0 AND coalesce(v.transcript_status,'')<>'done' AND coalesce(v.transcript_source,'') NOT IN ('supadata_native','windows_local') AND e.status IN ('blocked_transcript_fetch','local_transcript_retry','local_transcript_pending','supadata_retry')z AND e.job_id NOT IN (,c              3       K   | ]  }d   yw)?Nr   ).0_s     r   	<genexpr>zcandidates.<locals>.<genexpr>   s     *H13*Hs   )z ORDER BY CASE e.status WHEN 'blocked_transcript_fetch' THEN 0 WHEN 'local_transcript_retry' THEN 1 WHEN 'local_transcript_pending' THEN 2 ELSE 3 END,e.job_id LIMIT ?)joinextendappendr-   dict)r.   limitexclude_job_idsqparamsrs         r   
candidatesrA      s     v
#((*H*H"HH3NN!--   o  ou))Af-.Qa...s   #A8c                    	 t        ||       }d|j                  |j                  d|j                  fS # t        $ r'}|j
                  xs di |j                  fcY d }~S d }~wt        $ r'}|j
                  xs di |j                  fcY d }~S d }~wt        $ r'}|j
                  xs di |j                  fcY d }~S d }~ww xY w)N)r!      )contentlang     iW  )r   rD   languagebillabler   statusr   r   )r#   yidxexs       r   
transcriptrN      s    M!C199AJJ(OPQPZPZ$ZZ#Q"))2Bs2bkk2Q+Q?		 0SBKK ??LRYY-=#b-L&LLs>   25 	CA C C,BCCB<6C<Cc                    | j                  d|d   f      j                         }|st        d      |d   dk(  r|d   xs dj                         rdd	iS |j	                  d
      }|j	                  d      }t        |t              r|r|st        d      g }t        |      D ]  \  }}t        |t              r)t        |j	                  dd            j                         s@t        |j	                  dd            dz  }	t        |j	                  dd            dz  }
|j                  ||	|	t        |
d      z   t        |d         j                         f        dj                  d |D              j                         }|r|st        d      t        j                  |      }t        j                   t        |      |      }t#        j$                  |dd      }t        j                  |      }| j                  d       | j                  dt'               |d   f       | j                  d|d   f       | j)                  d|D cg c]  }|d   t+        |d         |d   f c}       | j                  d ||t'               |d   f       | j-                          t        |      }||d<   t        j.                  ||||      }t        j0                  ||||       t        j2                  ||d!          | j                  d"|d   |d!   t        j4                  ||||t'               f       | j                  d#t'               t'               |d   f       | j-                          d$|t7        |      t7        |      ||d%S c c}w )&NzSELECT * FROM videos WHERE id=?video_idvideo_missingtranscript_statusdonerN    resultskip_already_donerD   rE   native_transcript_unavailabletextoffsetr   i  duration c              3   &   K   | ]	  }|d      yw)   Nr   )r4   ss     r   r6   zimport_one.<locals>.<genexpr>+   s     "qt"s   empty_native_transcriptFT)ensure_ascii	sort_keysBEGINzUPDATE e2e_jobs SET status='supadata_processing',step='supadata_import',updated_at=? WHERE job_id=? AND status IN ('blocked_transcript_fetch','local_transcript_retry','local_transcript_pending','supadata_retry')job_idz0DELETE FROM transcript_segments WHERE video_id=?idzJINSERT INTO transcript_segments(video_id,start_seconds,text) VALUES(?,?,?)r   r]   zUPDATE videos SET transcript=?,transcript_hash=?,transcript_status='done',transcript_source='supadata_native',transcript_updated_at=? WHERE id=?
youtube_idzINSERT OR REPLACE INTO e2e_extractions(video_id,youtube_id,extraction_version,transcript_hash,content_hash,payload_json,obsidian_path,processed_at) VALUES(?,?,?,?,?,?,?,?)zUPDATE e2e_jobs SET status='complete',step='complete',completed_at=?,updated_at=?,lease_until=NULL,error_code=NULL,error_message=NULL WHERE job_id=?imported)rU   rH   segmentstranscript_lengthtranscript_hashobsidian_path)r-   fetchoner   r   get
isinstancelist	enumerater;   strfloatr:   maxr8   wshaextractjsondumpsr   executemanyroundcommitwrite_obsidiansync_ceregister_graphitiVERSIONlen)r.   jpayloadvrD   rE   segsir^   startdurrX   th	extractedrawchrowpaths                     r   
import_oner       s   99.*/?@IIK	o..F",(=2'D'D'FPXYlOmHm	Y	gkk&&974 t<PoCp=pg ASQq	Ad	3quuVB'7#8#>#>#@(aeeHQ %%5z!1D+Ed+Js++quSQZ'AfI(<(<(>?@A 
hh"T""((*D-F GGEE$K199T!WT2

9Z_jn8oCtutytyz}t~qs7QYY   u  wz  w|  }~  G  }H  vI  J  LM  LU  LU  VH  JK  LP  JQ  IS  LT  VW  Vc  Vc  dp  VZ  q[  QR  st  uy  sz  {@  AB  CD  AE  {F  GH  IJ  GK  rL  q[  V\  ^_  ^g  ^g  hz	  |	@
  A
C
  D
G
  D
I
  J
K
  L
P
  J
Q
  {	R
  ^S
  U
V
  U
]
  U
]
  U
_
	!Wc,1+;+;C	"R+PdRSR[R[\_`deghjRkmn  nA  nA  BF  GH  IU  GV  nW  y  {|  }A  {B  CD  EQ  CR  ST  S\  S\  ]_  `b  cf  gk  lo  lq  zr  s  uv  u~  u~  U  WZ  W\  ]`  ]b  cd  em  cn  Vo  up  rs  rz  rz  r|D	VYZ^V_rt  FJ  	K  K q[s   	N
c            
      	   t        j                         } | j                  dt        d       | j                  dt        d       | j                  dd       | j                  d	d
       | j	                         }d|j
                  cxk  rdk  rn J |j                  dk\  sJ t        d |j                  j                  d      D              }t               }t        |      }|d   dk(  r$|d   |d   |j
                  |j                  z   k\  sJ t               }t        ||j
                  |      }t        j                  t!               ||dt#        t$        dz  d      dt&               |j(                  r.t+        t        j,                  |t/        |      |dd             y g }|D ]$  }|d   |d   t!               d}		 |j1                  d|d   f      j3                         }
|
r|
d   d k(  r.|
d!   xs dj5                         rd"|	d#<   |j7                  |	       ut9        ||d         \  }}}|	j;                  ||t=        |t>              r|jA                  d$      nd t=        |t>              r|jA                  d%      nd d&       |dk(  r|	j;                  tC        |||             n|d'v r5d(|	d#<   |j1                  d)t!               |d   f       |jE                          n|d*v rOd+|	d#<   |j1                  d,t!               d-| d.| |d   f       |jE                          |j7                  |	        nd/|	d#<   |j1                  d,t!               d-| d.| |d   f       |jE                          t!               |	d3<   |j7                  |	       t/        |D cg c]  }|jA                  d#      d4k(  s| c}      |j
                  k\  s% n t        |      }t        j                  t!               ||d5t#        t$        d6z  d      dt&               t+        t        j,                  ||t/        |      |d7d             y # tF        $ ru}|	j;                  d0tI        |      jJ                  d1       |j1                  d2t!               tI        |      jJ                  |d   f       |jE                          Y d }~Kd }~ww xY wc c}w )8Nz--limit
   )typedefaultz--reserve-credits   z--exclude-job-idsrT   )r   z	--dry-run
store_true)actionr   c              3   T   K   | ]   }|j                         st        |       " y wr   )r   int)r4   rL   s     r   r6   zmain.<locals>.<genexpr>3   sL       |w  HI  mn  mt  mt  mv  }@  AB  }C  |ws   ((r1   httprC   	remaining)
created_ataccount_beforerA   zbatch-before.jsonrs      )indentr   )beforecandidate_countrA   F)r`   rc   re   )rc   re   
started_atzJSELECT transcript_status,transcript,transcript_hash FROM videos WHERE id=?rP   rR   rS   rN   rV   rU   rE   availableLangs)r   billable_requestsresponse_langavailable_langs)rF   rW   zUPDATE e2e_jobs SET status='supadata_native_unavailable',step='supadata_terminal',updated_at=?,error_code='native_unavailable',error_message='native transcript unavailable' WHERE job_id=?)i  i  rG   api_or_quota_errorzzUPDATE e2e_jobs SET status='supadata_retry',step='supadata_retry',updated_at=?,error_code=?,error_message=? WHERE job_id=?http_supadata_http_	api_errorerror)rU   error_classzUPDATE e2e_jobs SET status='supadata_retry',step='supadata_retry',updated_at=?,error_code='supadata_processing_error',error_message=? WHERE job_id=?finished_atrf   )r   account_afterresultszbatch-result.json)r   afterr   r   )&argparseArgumentParseradd_argumentr   
parse_argsr<   reserve_creditstupler=   r   r!   r$   r/   rA   rv   dumpr   openREPORTrp   dry_runprintrw   r   r-   rk   r   r:   rN   updaterm   r;   rl   r   rz   	Exceptionr   __name__)apaexcluder#   r   r.   rowsoutr   recr@   rJ   databillrM   rL   r   s                    r   mainr   2   s   ryc"Mr_rx{  EF  PG  IK  IX  IX  Yl  uw  IX  Ix  z|  zI  zI  JU  ]i  zI  zj  np  n{  n{  n}  lm  FG  IJ  IP  IP  FT  RT  FT  m  Fm  YZ  Yj  Yj  lm  Ym  m  Fm  w|  |w  MN  M^  M^  Md  Md  eh  Mi  |w  ww  ov5&F6NC$7F;<O<[`fgr`suvu|u|}~  ~O  ~O  vO  aO  O  %O6
1QWWW-4tyysu^drv9wx|  ~D  EX  ~X  Y\  y]  ef  or  0sII

Vc$iTR`efgio
 JQ(<ceL#
WYY[]^_i]j\lmvvx1
%&.AlO4Ir3P3P3RcvUXYaUbx{  yC  yC  DG  yH  JRq<16$t3::fae  KU  VZ  [_  K`vzv~v~  @F  wG  fj  [e  fj  ko  [p  }A  }E  }E  FV  }W  vz  ?{  4|ck3::j1T23&(G#h-  TQ  SV  SX  YZ  [c  Yd  Re  Jf  hi  hp  hp  hr-/CXQYY  PL  NQ  NS  V[  \b  [c  Td  gu  v|  u}  e~  @  AI  J  MK  FL  NO  NV  NV  NX  Z]  Zd  Zd  eh  Zi  kp"XAII  /k  mp  mr  uz  {A  zB  sC  FT  U[  T\  D]  ^_  `h  ^i  lj  %k  mn  mu  mu  mw U#mCJJsOS	8AEE(OZ7!	89177BEJ qz499CE%RUVW[\bcv\vwzW{  DE  NQ  RtzzF53t9WZ[inop 
 W::d2h.?.?@A199  Nd  fi  fk  lp  qs  lt  l}  l}  ~  @H  ~I  eJ  DK  MN  MU  MU  MW  MWW 
9s3   AQ!DQ:<QS	
7S	
	SA*SS__main__)r   )!r   r   hashlibrv   osr(   systimepathlibr   r   insert
e2e_workerrs   supadata_nativer   r   r   r   r   r*   r   r   mkdirr   r!   r$   r/   rA   rN   r   r   r   r   r   r   <module>r      s    9 9 9 9  2 3  x x06d#mq  s_  n`fl  bh  bn  bn  w{  EI  bn  bJ J0 )o/MK$q0 Zr   