
    $ AjE4                        d Z ddlZddlZddlZddlZddlZddlZddlZddl	m	Z	m
Z
 dZdZdZdZdZg d	Zg d
Zg dZd Z ee      Z ee      Z ee      ZdedefdZdedefdZg dZddgZdedefdZdededefdZ dedefdZ!d-dedededefdZ"ded ed!ed"ed#edefd$Z#defd%Z$d&efd'Z%d(efd)Z&e'd*k(  r e(d+        e(d,       yy).u  
Wave-2 Graphiti-Import fuer YouTube-Research — BEREINIGTE VERSION
Qualitaetsfix gegenueber Wave-1:
  - Quellenanker wird NICHT gesendet (ersetzt durch neutrale 1-Zeile)
  - Content-Cleaner filtert Video-Meta-Saetze, Channel-Provenienz, Tautologien
  - source_description enthaelt KEIN context-JSON (nur youtube_id)
  - Keine privaten Labels (LueftungsProfi etc.) erreichbar aus diesem Script
    N)datetimetimezonez+/opt/struktur/youtube-research/knowledge.dbz,/opt/obsidian-vault/YouTube-Research/Welle-2z6/opt/struktur/graphiti/youtube_research_wave2_log.jsonzhttp://127.0.0.1:8644/episodesg      @)
zv(?i)the video[^.]*(?:discusses|is about|covers|explores|explains|shows|presents|titled|hosted|posted|published)[^.]*\.zM(?i)das video[^.]*(?:handelt|erklaert|zeigt|bespricht|behandelt|wurde)[^.]*\.z6(?i)(?:this|the) (?:youtube )?video[^.]*youtube[^.]*\.zH(?i)(?:is|are) (?:hosted|published|available|featured) on youtube[^.]*\.z^(?i)(?:published|posted|created|produced|released) (?:a |the |this )?(?:youtube )?video[^.]*\.z'(?i)youtube channel[^.]*provides[^.]*\.zn(?i)(?:channel|kanal)[^.]*(?:provides|creates|produces|publishes) (?:content|tutorials|insights|videos)[^.]*\.z?(?i)the episode[^.]*(?:is about|discusses|covers|titled)[^.]*\.z(?i)(?:Webronaq|Jack Roberts|Der KI-Doktor|Mike Mildenberger|Julian Ivanov|Marc De Fanti|Philip Thomas|AI mit Arnie|Alex Sprogis|Prompt Engineer)[^.]*(?:published|posted|created|produced|made|released)[^.]*\.z(?i)(?:Webronaq|Jack Roberts|Der KI-Doktor|Mike Mildenberger|Julian Ivanov|Marc De Fanti|Philip Thomas|AI mit Arnie|Alex Sprogis|Prompt Engineer) (?:produced|created|made|released) a youtube[^.]*\.)z@(?i)\w+(?:['s])?\s+(?:creator|entwickler|author)\s+created\s+\w+z"(?i)\w+ is used in \w+ environmentz1(?i)\w+ is a (?:tool|product|service) by \w+\s*\.)	   LüftungsProfiLueftungsprofizLP-spezifischz	LP-SkillszAgent Solutionsinkasso2025zK:\\\\zprojekte-LPzprojekte-AGc                 R    | D cg c]  }t        j                  |       c}S c c}w N)recompile)lstps     //opt/struktur/graphiti/wave2_import_template.py<lambda>r   1   s    c2rzz!}2 2s   $sentencereturnc                     | j                         }t        |      dk  ryt        D ]  }|j                  |      s y t        D ]  }|j                  |      s y t
        D ]  }|j                  |      s y y)zKGibt True zurueck wenn Satz behalten werden soll, False wenn herausfiltern.   FT)striplen_VIDEO_META_REsearch_TAUTOLOGY_RE_PRIVATE_RE)r   spats      r   clean_sentencer   8   s|    A
1v{ ::a=  ::a=  ::a=     raw_contentc                    | j                  d      }g }|D ]  }|j                         }|r|j                  d      r|j                  |       8|j                  d      s|j                  d      r#|dd }t	        |      sk|j                  |       }t	        |      s|j                  |        dj                  |      S )z
    Bereinigt den Content vor dem Graphiti-POST:
    - Splittet nach Satzgrenzen
    - Filtert Video-Meta, Tautologien, private Labels
    - Gibt bereinigten Text zurueck
    
#z- z*    N)splitr   
startswithappendr   join)r   linescleaned_lineslinestrippedtexts         r   graphiti_content_cleanerr,   I   s     d#EM +::<8..s3  &t$(;(;D(AAB<Dd#$$T* h'$$T*+ 99]##r   )zGraphiti-Ready SummaryKernaussagenzBegriffe / EntitaetenzVolltext-KontextQuellenankermetac                    | j                  dd      j                  dd      j                         }| j                  dd      j                         }| j                  dd      }| j                  dd      }d| d| d	| d
| S )z2Erzeugt neutrale Quellenzeile ohne private Labels.title r   channel
youtube_idlanguagezQuelle: z
 | Kanal: z | ID: z | Sprache: )getreplacer   )r/   r1   r3   yt_idlangs        r   build_neutral_source_liner:   p   s    hhw#++,<bAGGIEhhy"%++-Ghh|R(Ehhz2&DeWJwiwug\$PPr   filepathc                    t        | dd      5 }|j                         }ddd       t        j                  ddt        j                        }t        j
                  d|t        j                        }|r|j                  d	      j                         n|j                  d
d      }g }t        j                  dt        j                        }|j                  |      D cg c]@  }|j                  d	      j                         |j                         |j                         fB }	}t        |	      D ]  \  }
\  }}t        fdt         D              r#t        fdt"        D              s<|
d	z   t%        |	      k  r|	|
d	z      d	   n
t%        |      }||| j                         }t        j                  dd|t        j                        j                         }|s|j'                  d d|         dj)                  |      }t+        |      }t-        |      }|j/                         dz   |z   }||fS # 1 sw Y   xY wc c}w )zELiest Artefakt, extrahiert nur erlaubte Sektionen, bereinigt Content.rutf-8encodingNz^---\n.*?\n---\n?r2   )flagsz^# (.+)$   r1   ?z	^## (.+)$c              3   &   K   | ]  }|v  
 y wr	    ).0bsec_names     r   	<genexpr>z+extract_graphiti_content.<locals>.<genexpr>   s     7qH}7   c              3   &   K   | ]  }|v  
 y wr	   rE   )rF   arH   s     r   rI   z+extract_graphiti_content.<locals>.<genexpr>   s     ;Q1=;rJ   z	```.*?```z## z

z

---

)openreadr
   subDOTALLr   	MULTILINEgroupr   r6   r   finditerstartend	enumerateanyBLOCKED_SECTIONSCONTENT_SECTIONSr   r%   r&   r,   r:   rstrip)r;   r/   frawtitle_matchr1   partssection_patternmsection_positionsirT   rU   
next_startcontentr   cleaned_contentneutral_sourcefinal_contentrH   s                      @r   extract_graphiti_contentrh   y   s   	hg	. !ffh &&%r3bii
@C ))Kbll;K,7Ka &&(TXXgs=SE Ejjr||<O"1":":3"?A ''!***,aggiA A A &//@%A 8!!HeS7&677;*:;;Q%6!77 (A.q1=@X 	c*%++-&&r7"))DJJLLL3xjWI678   $$U+K /{;O /t4N#**,v5FM-S As   H;!AI;Ividc                     t        j                  t              }|j                         }|j	                  dt        |       f       |j                         }|j                          |S )NzSELECT graphiti_status, quality_status, artifact_status
        FROM source_processing_registry
        WHERE source_system='youtube-research' AND source_id=?)sqlite3connectDB_PATHcursorexecutestrfetchoneclose)ri   conncurrows       r   registry_checkrv      sR    ??7#D
++-CKK BDGH;P ,,.CJJLJr   r8   status	error_msgc           	         t        j                  t              }|j                         }t	        j
                         j                         }d| }|dk(  r |j                  d|||t        |       f       n/dt        |      z   d d }|j                  d||t        |       f       |j                          |j                          y )Nyt-importeda_  UPDATE source_processing_registry SET
            quality_status='passed', graphiti_status='imported',
            graphiti_import_key=?, graphiti_actor=?,
            graphiti_group_id='youtube-research-wave-2',
            updated_at=?, notes='Wave-2 Import OK (bereinigt)'
            WHERE source_system='youtube-research' AND source_id=?
        zImport FEHLER:    zUPDATE source_processing_registry SET
            graphiti_status='failed', updated_at=?, notes=?
            WHERE source_system='youtube-research' AND source_id=?
        )rk   rl   rm   rn   r   now	isoformatro   rp   commitrr   )	ri   r8   rw   rx   rs   rt   r}   iknotes	            r   registry_updater      s    ??7#D
++-C
,,.
"
"
$CugB  "c3s8$	& "C	N2DS9  4S"	$ 	KKMJJLr   rd   sourceactorcontext_dicttimestamp_strc                    | |||j                  dd      |d}t        j                  |d      j                  d      }t        j
                  j                  t        |ddid	
      }	 t        j
                  j                  |d      5 }|j                  |j                         j                  d      fcd d d        S # 1 sw Y   y xY w# t        j                  j                  $ rN}	|	j                  r|	j                         j                  d      n
t        |	      }
|	j                   |
fcY d }	~	S d }	~	wt"        $ r}	dt        |	      fcY d }	~	S d }	~	ww xY w)Nr4   r2   )rd   r   r   context	timestampF)ensure_asciir>   zContent-Typezapplication/jsonPOST)dataheadersmethod<   )timeoutr   )r6   jsondumpsencodeurllibrequestRequestGRAPHITI_URLurlopenrw   rN   decodeerror	HTTPErrorfprp   code	Exception)rd   r   r   r   r   payloadr   reqrespebodys              r   post_episoder      s    ##L"5"G ::gE299'BD
..
 
 4!34V ! C^^##C#4 	<;;		 2 27 ;;	< 	< 	<<<!! +,44qvvxw'SVvvt| #a&ysO   )!C 
+B?5	C ?CC C E(AD1+E1E=E	EEc                      t         j                  j                  t              r/	 t	        t              5 } t        j                  |       cd d d        S i S # 1 sw Y   i S xY w# t        $ r i cY S w xY wr	   )ospathexistsLOG_PATHrM   r   loadr   )r[   s    r   load_logr      sc    	ww~~h	h $1yy|$ $ I	$ I  	I	s.   A" A		A" AA" A" "A0/A0log_datac                     t        t        dd      5 }t        j                  | |dd       d d d        y # 1 sw Y   y xY w)Nwr>   r?   Fr"   )r   indent)rM   r   r   dump)r   r[   s     r   save_logr      s7    	hg	. =!		(AE!<= = =s   6?	wave_metac                 2   t               }dx}x}}g }t        dt        j                         j	                                 t        dt        |        dt         d       t        | j                         d       D ]  }| |   \  }}}	}
t        d| d	|
        ||v r=||   j                  d
      dk(  r&t        d       |dz  }|j                  ||dd       `t        |      }|r.|d   dk(  r&t        d       |dz  }|j                  ||dd       t        j                  j                  t        |
      }t        j                  j                  |      s*t        d|        |dz  }|j                  ||dd       ||||	dd| d}t!        ||      \  }}||d<   t        |j#                               dk  r3t        dt        |              |dz  }|j                  ||dd       wt        dt        |       d       t        j                  t$        j&                        j	                         }t)        |dd| ||      \  }}t        d |        |dk(  r{|dz  }t        j                         j	                         }||||	||
d|t        |      dd!d"||<   t+        |       t-        ||d       t        d#       |j                  ||d$|d%       n||dz  }|r|d&d nd'}|||
||t        j                         j	                         d(||<   t+        |       t-        ||d)|       t        d*| d+|d&d,         |j                  ||d)d       |dk(  st/        j0                  t2                t        d-| d.| d/|        ||||fS )0zw
    wave_meta: dict mit vid_str -> (yt_id, lang, channel, filename)
    Wird vom jeweiligen Wave-Script befuellt.
    r   z*
=== Wave Graphiti-Import (bereinigt) === zZiel: z Artefakte | Vault: r    c                     t        |       S r	   )int)xs    r   r   zmain.<locals>.<lambda>   s
    #a& r   )key[z] http_statusr|   z  [SKIP] Log-DuplikatrB   skip_log)idr8   resultr{   z  [SKIP] Registry-Duplikatskip_registryz  [FEHLER] Datei fehlt: file_not_foundzyoutube-research-wave-2zwave2-)internal_idr4   r5   r3   graphiti_group_id
import_keyr1   2   z-  [FEHLER] Content zu kurz nach Bereinigung: 	too_shortz  Content nach Cleaner: z Zeichenrz   )rd   r   r   r   r   z  HTTP: T)r   r4   r1   r3   r5   filenamer   imported_atcontent_lenr   cleaner_appliedz  [OK]ok)r   r8   r   r1   NrC   )r   r4   r   r   r   r   failedz  [FEHLER] z: r   z
=== ERGEBNIS === OK:z Skip:z Fehler:)r   printr   r}   r~   r   
VAULT_PATHsortedkeysr6   r%   rv   r   r   r&   r   rh   r   r   utcr   r   r   timesleep
PAUSE_SECS)r   logok_count
fail_count
skip_countresultsvid_strr8   r9   r3   fnregr   r/   r1   graphiti_contenttsr   r   r   err_msgs                        r   mainr      s   
 *C)**H*zJG	78P8P8R7S
TU	F3y>""6zl"
EF)..*0@A O##,W#5 tWb'"RD!" C<CJNN=9S@)+!OJNN'EZPQW%3q6Z'.0!OJNN'E_UVWW\\*b)ww~~b!,RD12!OJNN'EEUVW&e W%> &ug.0
 #;2t"DW%%'(2-A#FVBWAXYZ!OJNN'E[QR(-=)>(?xHI\\(,,'113($,w-
T 	&'#MH",,.224K&ee""";"#34%>#'CJ SMGUJ7FNN'ETTYZ[!OJ$(d4CjcG)0SU)4w)1)A)A)CECJ SMGUHg>K}Bws|n=>NN'EXNO#JJz"_O#b 
"8*F:,hzl
STHj*44r   __main__uM   Dieses Script ist ein Template — Wave-2-Meta-Daten in WAVE2_META eintragen.z>Dann: from wave2_import_template import main; main(WAVE2_META)r	   ))__doc__rk   r   r
   r   r   urllib.requestr   urllib.errorr   r   rm   r   r   r   r   VIDEO_META_PATTERNSTAUTOLOGY_PATTERNSPRIVATE_LABEL_PATTERNSCOMPILEr   r   r   rp   boolr   r,   rY   rX   dictr:   tuplerh   rv   r   r   r   r   r   __name__r   rE   r   r   <module>r      s   A @ @ @ '
7;
C/

    3,-+,/0S T "$# $# $> 
 '7 QD QS Q+ s + $ + 5 + \   S # # .# s 3 #47<A2$ =t =
^5D ^5B z	
YZ	
JK r   