
    Vajp%                       d Z ddlmZ ddlZddlZddlZddlZddlZddlZddl	m
Z
mZ ddlmZmZ ddlmZ  ed      Z ed      Zh d	Z ej&                  d
ej(                        Z ej&                  dej(                        Z ej&                  dej(                        Z ej&                  dej(                        Zd Zd Zd Zd Zedk(  r e        yy)z?Read-only classifier for the automated pipeline review backlog.    )annotationsN)Counterdefaultdict)datetimetimezone)Pathz?/opt/struktur/obsidian-knowledge-pipeline/knowledge_pipeline.dbz/opt/struktur/knowledge-curator>   areatagstypetitlesourcestatusaliasesprojectversioncategorypipeline_testtest_metadatareview_requiredgraphiti_candidatezpasswort|password|secret|token|api[ _-]?key|zugang|berecht|kosten|zahlung|gesund|medizin|krank|personenbezogen|telefon|adresse|emailz`(^|/)(backup|backups|cache|tmp|temp|logs?|reports?|exports?)(/|$)|\.bak(?:\.|$)|\.backup(?:\.|$)z>Knowledge-Pipeline-Review-Backlog-Optimization-2026-07-22\.md$u   ^(Quellenanker|Mögliche Graphiti-Entities|Tools / Modelle / Dienste|Offene Fragen/Prüfbedarf|Konkrete Workflows/Schritte|Relevanz fuer unser System)$c                   t        | t        t        f      rt        j                  | ddd      S t        j                  ddt        j                  dt        | xs d            j                               j                         S )	NFT),:)ensure_ascii	sort_keys
separatorsz\s+ NFKC )
isinstancedictlistjsondumpsresubunicodedata	normalizestrstripcasefold)vs    :/opt/struktur/knowledge-curator/review_backlog_analysis.pynormr.      s`    !T$K A5SWcl)m"m66&[226#ag2,GMMOPYY[[    c                8     dj                   fddD              S )N|c              3  R   K   | ]  }t        j                  |               y wN)r.   get).0kxs     r-   	<genexpr>zkey.<locals>.<genexpr>   s#       XqDqN  Xs   $')candidate_typesubject	predicateobjectmodalitypolaritytemporal_scopeauthority_scope)join)r7   s   `r-   keyrB      s(    88  X  -W  X  X  Xr/   c                   | j                  d      xs i }t        |j                  d      xs | j                  d      xs d      }t        | j                  d            }t        | j                  d            }g }|r|dv r|j                  d       t	        |j                               dk  st	        |      d	k  r|j                  d
       t        j                  d|j                               r|j                  d       t        j                  |      s-t        j                  t        | j                  d                  r|j                  d       t        j                  t        |j                  dd                  r|j                  d       t        j                  t        |j                  dd                  r|j                  d       t        |j                  d      xs d      j                         }t        |j                  d      xs d      }t        j                  |      r&|j                  d      s|dv r|j                  d       |j                         dk(  r5d|j                         v sd|j                         v r|j                  d       |j                  d      r6t        j                  d|t        j                        r|j                  d       |S )N
provenanceevidence_textr<   r   r:   >   nonenullunknown	unbekanntmissing_or_generic_subject(      short_or_fragmentary_evidencez#{1,6}\s+[^\n]+heading_onlysensitive_or_high_impactrelative_pathtechnical_or_report_pathgenerated_pipeline_reportheadingzYouTube-Research/>      Mögliche Graphiti-EntitiesQuellenankerboilerplate_or_technical_statusquellezmarketplace:zskills.zgAbgeleitet aus Transkriptanalyse|Keine spezifischen|manuell verifizieren|nicht automatisch extrahierbar)r4   r)   r.   appendlenr*   r%   	fullmatch	SENSITIVEsearch	TECH_PATHGENERATED_REPORTBOILERPLATE_HEADING
startswithr+   I)r7   ptextr:   objreasonsrS   paths           r-   source_qualityrg      s{   eeLRAc!%%*@*YAEE(O*YWY&Zy!"GQUU8_(=CGg!FFWsHt
4::<2SB?^0_	||&tzz|4gnn^6T!1!1#aeeHo2F!GXrIsAEE/"567Hb9cs155#<=>Oj@ki &B'--/Gc!%%:P:VTV6W$$W-4??CV3W[b  gU  \U  X_  Xf  Xf  gH  XI8#4==?)Ji[_[h[h[jNjmtm{m{  }^  n_*+		  ;e  fj  km  ko  ko  1p  ry  r@  r@  Ab  rcNr/   c                 *   t        j                         } | j                  dt        t        dz               | j                  dt        t        dz               | j                  dt        t        dz               | j                         }dt         d	}t        j                  |d
      }t        j                  |_
        |j                  d      j                         }g t        t              }t        t              }|j                  d      D ]8  }	 t        j                   |d         }|t#        |         j%                  |       : |D ]Z  }t        j                   |d         }|j)                  d      xs i }	t#        |      }
||
   j%                  |d          t+        |      |j)                  dd      }|dk(  r1t-        |j)                  d            t.        vrj%                  d       t1        fddD              rd}n,sd}n't1        fddD              rd}ndv rd}n	dv rd}nd}j%                  i d|d   d |d    d!|d!   d"|	j)                  d"      d|d#|j)                  d#      d|j)                  d      d$|j)                  d$      d%|j)                  d%      d&|j)                  d&      d'|j)                  d'      d(|j)                  d(      d)|j)                  d)      d*|j)                  di       j)                  d*      d+|j)                  di       j)                  d+      d,t3        t        |	j)                  d-d                  d.|	j)                  d.      |	j)                  d/      |
|t5        |j)                  |
            d0       ] t        t              }D ]  }||d1      j%                  |d           D ]  }t3        ||d1            d2kD  rQt3        ||d1            |d3<   t7        ||d1            |d4<   |d   |d4   k7  r|d5   d6vrd7|d5<   n|d   |d4   k7  rd7|d5<   nd2|d3<   |d8   ss|d5   dk(  s|d9|d5<    dgfd;	}i d<t9        j:                  t<        j>                        jA                         d=t        t              d>t3              d |d      d) |d)      d? |d d:      d* |d*      d+ |d+      d@tC        dA D              dBtE        dC |jG                         D              dDtE        dE |jG                         D              dFtC        dG D              jI                  d:      dHtC        dI D              jI                  d:      dJtC        dK D              dLtE        dM D              dN}tK        |d@         |d@<   |d   |d<   |d)   |d)<   tM        |j                        jO                  t        jP                  |dOdPt        Q      dRz   dST       t        j                  |jR                        }|jU                  dU       |jW                  dVD cg c]N  }|d   |d    |d   |d1   |d5   t        jP                  |dW   dOX      |d,   |d.   |dY   |d3   tY        |d8         fP c}       |j[                          |j]                          dZdd[|d<    d\t3               d]dd^dg}|t_        |d@   ja                         d_ `      D 
cg c]  \  }
}da|
 db| d] c}}
z  }|g dc|d   D 
cg c]  \  }
}da|
 db| d] c}}
z   z  }|g dd|d)   D 
cg c]  \  }
}da|
 db| d] c}}
z   z  }|g dez  }tM        |jb                        jO                  dRje                  |      dRz   dST       tg        t        jP                  dfD 
ci c]  }
|
||
   
 c}
dOdPt        Q             y # t&        $ r Y w xY wc c}w c c}}
w c c}}
w c c}}
w c c}
w )hNz--jsonzreview-backlog-analysis.json)defaultz
--markdownzreview-backlog-analysis.mdz--sqlitezreview-backlog-analysis.dbzfile:z?mode=roT)urizYSELECT * FROM review_items WHERE status='pending_review' ORDER BY created_at,candidate_idzTSELECT candidate_type,candidate_json,status FROM truth_items WHERE status='approved'candidate_jsonrD   candidate_idr9   r   DocumentMetadatar;   metadata_key_requires_reviewc              3  &   K   | ]  }|v  
 y wr3    r5   zre   s     r-   r8   zmain.<locals>.<genexpr>5   s     eqG|e   )rV   rR   excluded_non_knowledgegenuine_pending_reviewc              3  &   K   | ]  }|v  
 y wr3   rp   rq   s     r-   r8   zmain.<locals>.<genexpr>7   s       Q!g  Qrs   )rJ   rM   rN   syntactically_incomplete)invalid_fragment_or_insufficient_evidencerO   rQ   +technical_path_requires_manual_confirmationrf   	chunk_keyrS   r:   r<   r=   r>   r?   r@   decision_rulecurator_versionnormalization_versionevidence_lengthrE   evidence_hashsource_content_hash)source_hashcanonical_keyre   r   approved_identical_existsr      duplicate_group_sizecanonical_candidate_idr   )rO   ru   	duplicater   "automatically_safe_reuse_candidate2   c                L     t         fdD              j                  |      S )Nc              3  H   K   | ]  }|j                        xs d   yw)z(empty)Nr4   )r5   ifields     r-   r8   z)main.<locals>.<lambda>.<locals>.<genexpr>E   s!     "L155<#<9#<"Ls   ")r   most_common)r   nitemss   ` r-   <lambda>zmain.<locals>.<lambda>E   s    7"Le"LLXXYZ[ r/   generated_atdatabasepending_totalsource_directory
categoriesc              3  &   K   | ]	  }|d      yw)r   Nrp   r5   r   s     r-   r8   zmain.<locals>.<genexpr>F   s0       Wu  jk  XY  Zd  Xe  Wu   duplicate_groupsc              3  >   K   | ]  }t        |      d kD  sd   ywr   NrY   r5   r,   s     r-   r8   zmain.<locals>.<genexpr>F   sE       Lq  ST  hk  lm  hn  op  hp  MN  Lqs   duplicate_itemsc              3  V   K   | ]!  }t        |      d kD  st        |      d z
   # ywr   r   r   s     r-   r8   zmain.<locals>.<genexpr>F   sa       Gs  UV  jm  no  jp  qr  jr  HK  LM  HN  OP  HP  Gss   ))top_documentsc              3  &   K   | ]	  }|d      yw)rf   Nrp   r   s     r-   r8   zmain.<locals>.<genexpr>F   s0       Ke  Z[  LM  NT  LU  Ker   top_headingsc              3  P   K   | ]  }|d    |j                  d      xs df   yw)rf   r:   r   Nr   r   s     r-   r8   zmain.<locals>.<genexpr>F   sh       L		  t	u	  N	O	  P	V	  N	W	  X	Y	  X	]	  X	]	  ^	g	  X	h	  X	n	  l	n	  M	o	  L		s   $&evidence_lengthsc              3  \   K   | ]$  }|d    dk(  rdn|d    dk  rdn|d    dk  rdnd & yw)	r~   r   0rK   z<40   z40-199z>=200Nrp   r   s     r-   r8   zmain.<locals>.<genexpr>F   s       j
v  kl  r
s
  t
E  r
F  HI  r
I  k
n
  XY  Zk  Xl  mo  Xo  OT  AB  CT  AU  VY  AY  u}  _f  k
f  j
vs   *,without_complete_provenancec              3  H   K   | ]  }|d     xs |d    xs |d      yw)r   r   rz   Nrp   r   s     r-   r8   zmain.<locals>.<genexpr>F   s       Xm  bc  ]^  _n  ]o  Yo  Y]  wx  yF  wG  sG  Y]  OP  Q\  O]  K]  Y]  Xms    "r   F   )r   indentri   
zutf-8)encodingae  DROP TABLE IF EXISTS classification; CREATE TABLE classification(candidate_id TEXT PRIMARY KEY,path TEXT,candidate_type TEXT,canonical_key TEXT,category TEXT,reasons_json TEXT,evidence_length INTEGER,evidence_hash TEXT,source_hash TEXT,duplicate_group_size INTEGER,approved_identical_exists INTEGER); CREATE INDEX class_category ON classification(category);z8INSERT INTO classification VALUES(?,?,?,?,?,?,?,?,?,?,?)re   )r   r   z"# Read-only Review-Backlog-Analysez	Erzeugt: zPending: **z**z## Kategorienc                    | d    | d   fS )Nr   r   rp   )rr   s    r-   r   zmain.<locals>.<lambda>L   s    Z[\]Z^Y^_`ab_cXd r/   )rB   z- `z`: **)r   z## Typenr   )r   z## Extraktionsregelnr   )r   z## Sicherheitsgrenzer   u  Diese Auswertung verändert keine Statuswerte, keine historischen Reviewentscheidungen und keine Neo4j-Daten. Kategorien sind Vorstufen für einen separaten Dry-Run; nur explizit deterministische Duplikate oder technische Fragmente dürfen später automatisch migriert werden.)	r   r   r   r   r   r   r9   r{   r   )r   )4argparseArgumentParseradd_argumentr)   
REPORT_DIR
parse_argsPIPELINE_DBsqlite3connectRowrow_factoryexecutefetchallr   r"   r#   loadsrB   rX   	Exceptionr4   rg   r.   AUTO_METADATA_KEYSanyrY   boolminr   nowr   utc	isoformatr   sumvaluesr   r!   r   
write_textr$   sqliteexecutescriptexecutemanyintcommitclosesortedr   markdownrA   print)apargsrj   crowsgroupsapprovedrr7   rb   r6   typr   dupr   topoutamdr,   r   re   s                       @@r-   mainr   (   s    B"//(3zRpGpCq/"rtv  uD  uD  EQ  Z]  ^h  iE  ^E  ZF  uD  uG  IK  IX  IX  Yc  lo  pz  {W  pW  lX  IX  IY  `b  `m  `m  `o  [_
}H%C)FqV]VaVa	
n	o	x	x	zD
E[&fT1BYYmn 	jj+,-AxA/?/F/Fq/I  
[
**Q'(
)QUU<-@-FB1#a&RXYZR[RbRbcdesctRuq!quu-=b'A3""tAEE+,>'?GY'Y[b[i[i  kI  \Je%dee  qIgo#;(  Q  (P  Q  Q  \G  S['72=WH'72=jH/h  ZnQ~%6  Zvai  ZTUVaTb  Zclmnmrmrs|m}  Z  O  PS  Z  T]  ^_  ^c  ^c  dm  ^n  Z  oz  {|  {@  {@  AL  {M  Z  NV  WX  W\  W\  ]e  Wf  Z  gq  rs  rw  rw  xB  rC  Z  DN  OP  OT  OT  U_  O`  Z  aq  rs  rw  rw  xH  rI  Z  J[  \]  \a  \a  bs  \t  Z  uD  EF  EJ  EJ  KZ  E[  Z  \m  no  ns  ns  t@  AC  nD  nH  nH  IZ  n[  Z  \s  tu  ty  ty  zF  GI  tJ  tN  tN  Of  tg  Z  hy  z}  ~A	  B	C	  B	G	  B	G	  H	W	  X	Z	  B	[	  ~\	  z]	  Z  ^	m	  n	o	  n	s	  n	s	  t	C
  n	D
  Z  S
T
  S
X
  S
X
  Y
n
  S
o
  @A  LS  _g  DH  IQ  IU  IU  VW  IX  DY  Z  	[
[ 	DCEC/*+221^3DEE Js1_%&')&)#a.@*A&BA$%`cdghijyhzd{`|AF^D_ A&>$??AjM  ZO  EO  _j  QR  S]  Q^>"a(@&AAQ\1Z=()a&'()a
m=U.U  fJWXYcWdJ 	\C 	|X\\2<<>  	|z#kJZ  	|[jknotku  	|  wG  HK  L\  H]  	|  ^m  nq  rA  nB  	|  CU  VY  Z`  ac  Vd  	|  ev  wz  {L  wM  	|  Ne  fi  jA  fB  	|  CO  PW  Wu  ot  Wu  Pu  	|  vH  IL  Lq  X[  Xb  Xb  Xd  Lq  Iq  	|  rC  DG  Gs  Z]  Zd  Zd  Zf  Gs  Ds  	|  tC  DK  Ke  _d  Ke  De  Dq  Dq  rt  Du  	|  vD	  E	L	  L		  y	~	  L		  E		  E	K
  E	K
  L
N
  E	O
  	|  P
b
  c
j
  j
v  pu  j
v  c
v  	|  wT  UX  Xm  gl  Xm  Um  	|  nu  v{  	|C3|,-CSIYEZs3C/Dqt  vE  rF\_`o\pOtzz#5RVWX\\fmnoodkk"AAOO  5\  %]MML  NS  NT  IJqQ_O`abciajklm}k~  @A  BQ  @R  ST  U_  S`  ae  ak  ak  lm  nw  lx  FK  aL  MN  O`  Ma  bc  ds  bt  uv  wD  uE  FG  H^  F_  `c  de  fA  dB  `C  OD  NT  U  WX  W_  W_  Wa  cd  cj  cj  cl,b9S=P<Q1RU`adejak`llnSopr  uD  EG  HB&\1B1H1H1JOd*e
f31QS52
ffB
cBR>STsqQCuQCr2T
TTB
)#oJ^*_31QS52+>*_
__B  B  BB""499R=#5w"G	$**  )M  N1aAh  N  \a  ij  sw  x  yC $4 NT
fT*_ Ns1   5_)A_9

_>/`
`

`)	_65_6__main__)__doc__
__future__r   r   hashlibr#   r%   r   r'   collectionsr   r   r   r   pathlibr   r   r   r   compilera   r[   r]   r^   r_   r.   rB   rg   r   __name__rp   r/   r-   <module>r      s    E " 8 8 8 , ' RS12
 t 
"**  ]  ^`  ^b  ^b  c	
"**xy{y}y}
~	]^`^b^bc BJJ   z  {}  {  {  @ \X )yV Zr/   