
    rj3                    @   d Z ddlmZ ddlZddlZddlZddlZddlZddlZddl	Z	ddl
Z
ddlZddlZddlZddlmZmZ ddlmZ ddlmZ  ed      Z ed      Z ed	      Zd
ddddZd Zd Zd Zd Zd Zd Zd Zd Z d Z!d Z"d Z#d Z$d Z%d Z&e'dk(  r e&        yy)zControlled K-DATEN pilot adapter. Original files are read-only.
Writes only to the existing knowledge.db: adapter registry, source versions,
K-DATEN FTS5 projection, KUs/provenance and the existing Graphiti queue.
    )annotationsN)datetimetimezone)Path)ElementTreez+/opt/struktur/youtube-research/knowledge.dbz/opt/raw-data/K-DATENz/opt/struktur/kdata-derivedz<http://schemas.openxmlformats.org/wordprocessingml/2006/mainz5http://schemas.openxmlformats.org/drawingml/2006/mainz9http://schemas.openxmlformats.org/spreadsheetml/2006/mainzChttp://schemas.openxmlformats.org/officeDocument/2006/relationships)waxrc                 d    t        j                  t        j                        j	                         S N)r   nowr   utc	isoformat     /home/hermes/kdata_adapter.pyr   r      s    (,,x||,6688r   c                H    t        j                  |       j                         S r   )hashlibsha256	hexdigest)bs    r   shar      s    7>>!$..00r   c                H    | j                  t              j                         S r   )relative_toRAWas_posix)ps    r   relr      s    1==%..00r   c                    d| z   S )Nzk-daten:r   )r   s    r   	source_idr!      s    A%r   c                v    t        j                  d| z   dz   |z   j                               j                         S )Nzk-daten:v1

)r   r   encoder   )shs     r   	import_idr'      s1    7>>>!+;D+@+B*J*J*LMWWYYr   c                >   | j                         }t        |       j                  j                         }d|v ryd|v sd|v sd|v ryd|v ryd	|v sd	|v sd
|v ryd|v sd|v sd|v ryt        |       j                  j	                         dk(  ryd|v sd|v ryd|v sd|v ryy)NangebotANGEBOTzab/u   auftragsbestätigungauftragsbestaetigungAUFTRAGSBESTAETIGUNG
bestellung
BESTELLUNGrechnungz/rg/RECHNUNGibninstallation	inbetriebINBETRIEBNAHME.emlEMAILprojekt_projektPROJEKTDATEN	anleitungtechnTECHNISCHE_UNTERLAGESONSTIG)casefoldr   namesuffixlower)r   r%   ns      r   classifyrC      s    ::<47<<((*N)	QJ(A-1G11LTjA\!OzQ&A+Z	QJ.A%)9AQGNNF"'!OyA~^11,Br   c                    t        j                  dd|       } t        j                  dd|       } t        j                  dd|       } t        j                  dd|       } | S )	Nz&(?i)\b[A-Z]{2}\d{2}(?:[\s\d]{13,34})\bz[REDACTED-BANKDATA]z;(?i)\b(?:iban|bic|nif|steuer(?:nummer|id))\s*[:=]?\s*[^\n]+z[REDACTED-SENSITIVE-FIELD]z&(?i)\b[\w.+-]+@[\w.-]+\.[A-Za-z]{2,}\bz[REDACTED-EMAIL]zO(?<!\d)(?:\+\d{1,3}[\s.-]?)?(?:\(?\d{2,4}\)?[\s.-]?)\d{3,4}[\s.-]?\d{3,4}(?!\d)z[REDACTED-PHONE])resub)ts    r   redactrH       sY    66
34I!L66
HIefgh66
34FqI66
\]opqr	r   c                p   t        j                  |       }t        j                  |j	                  d            }g }|j                  dt              D ]P  }dj                  d |j                  dt              D              j                         }|s@|j                  |       R |j                  dt              D ]i  }|j                  d       |j                  dt              D ]=  }|j                  d	j                  d
 |j                  dt              D                     ? k dj                  |      S )Nzword/document.xmlz.//w:p c              3  <   K   | ]  }|j                   xs d   ywrJ   Ntext.0r
   s     r   	<genexpr>zdocx_text.<locals>.<genexpr>)   s     >qaffll>   .//w:tz.//w:tblz[TABLE]z./w:tr | c              3     K   | ]>  }d j                  d |j                  dt              D              j                          @ yw)rJ   c              3  <   K   | ]  }|j                   xs d   ywrL   rM   rO   s     r   rQ   z&docx_text.<locals>.<genexpr>.<genexpr>.   s      O!2 OrR   rS   N)joinfindallNSstrip)rP   tcs     r   rQ   zdocx_text.<locals>.<genexpr>.   s4     z\^ Orzz(27N OOUUWzs   AAz./w:tcr#   )
zipfileZipFileET
fromstringreadrX   rY   rW   rZ   append)r   zrootoutparatxttabletrs           r   	docx_textri   &   s    
??1BMM!&&1D*EFtB\\(2& Tgg>DLL"$=>>DDF#C ll:b) |U**YMM(2& |b::ejjzbdblblmuvxbyzz{|| 			#r   c           	        t        j                  |       }g }	 t        j                  |j	                  d            }|D cg c](  }dj                  d |j                         D              * }}g }t        d |j                         D              D ]6  }t        j                  |j	                  |            }|j                  dt        |      j                  z   dz          |j                  d      D ]  }g }	|j                  d      D ]  }
|
j                  d	      }|dn|j                  xs d}|
j!                  d
      dk(  r5|j#                         r%t%        |      t'        |      k  r|t%        |         }|	j                  |
j!                  dd       d|         |	s|j                  dj                  |	              9 dj                  |      S c c}w # t        $ r Y ~w xY w)Nzxl/sharedStrings.xmlrJ   c              3  t   K   | ]0  }|j                   j                  d       s|j                  xs d 2 yw)z}trJ   N)tagendswithrN   rO   s     r   rQ   zxlsx_text.<locals>.<genexpr>3   s0     C|UVghglglguguvzg{AFFLbLC|s    88c              3  h   K   | ]*  }|j                  d       s|j                  d      s'| , yw)zxl/worksheets/sheetz.xmlN)
startswithrm   rO   s     r   rQ   zxlsx_text.<locals>.<genexpr>6   s+     j!q||<Q/RWXWaWabhWiAjs   222z[SHEET ]z	.//{*}rowz{*}cz{*}vrG   r%   r   ?=rT   r#   )r\   r]   r^   r_   r`   rW   iterKeyErrorsortednamelistra   r   stemrX   findrN   getisdigitintlen)r   rb   sharedsrsird   r?   rc   rowvalscellvvalues                r   	xlsx_textr   0   s   
??1bv]]166012"  HJ  <K  BDBGGC|Z\ZaZaZcC|<|  <KF  <Kjqzz|jj (T	}}QVVD\"$CJJyd/H/L$M\\+& (c
4{{6" 0t
iiAQYrAFFLbxx}cemmo#e*S[2HvVYZ_V`OaKK488C$%Qug./0 3::ejj&'(( 			# <Ks#   (G" -G.G" G" "	G/.G/c           	         t        j                  ddddt        |       dgddd      }|j                  d	k(  r|j                  S d
S )Ntimeout8s	pdftotextz-layout-T   )capture_outputrN   r   r   rJ   )
subprocessrunstr
returncodestdout)r   r   s     r   pdf_textr   @   sC    >>9T+iAsCSW]ajlmLL!O++r   c                   t        j                  | j                               }g }|d   r|j                  d|d   z          |d   r|j                  d|d   z          |d   r|j                  d|d   z          |d   r|j                  d|d   z          |j	                         }t        |t              r|j                  |       nnt        |t              r^|D ]Y  }|j                         d	k(  s|j                  |j	                  d
      j                  |j                         xs dd             [ dj                  |      S )NFromzFrom: TozTo: DatezDate: Subjectz	Subject: z
text/plainT)decodeutf8replacer#   )emailmessage_from_bytes
read_bytesra   get_payload
isinstancer   listget_content_typer   get_content_charsetrW   )r   mbodypayloadr
   s        r   eml_textr   C   s   ALLN+"TfIdkk(1V9,-dGDKKqw'fIdkk(1V9,-iL[956	
wsDKK0 EaL(Q]]$]5O5V5VWXWlWlWnWxrx  zC  6D  *EE		$r   c                    | j                   j                         }|dk(  rt        |       S |dk(  rt        |       S |dk(  rt	        |       S |dk(  rt        |       S |dv r| j                  dd      S y	)
Nz.docxz.xlsxz.pdfr5   )z.txtz.csvr   r   )encodingerrorsrJ   )r@   rA   ri   r   r   r   	read_text)r   exts     r   extractr   O   sp    XX^^L	!$L	!$Kx{"Kx{"?	
fI	66
r   c                H    | j                  d       | j                  d       y )NaG  CREATE TABLE IF NOT EXISTS kdata_documents(
 id INTEGER PRIMARY KEY AUTOINCREMENT, source_identity TEXT NOT NULL UNIQUE,
 project TEXT NOT NULL, original_path TEXT NOT NULL, relative_path TEXT NOT NULL,
 document_type TEXT NOT NULL, file_extension TEXT NOT NULL, file_size INTEGER NOT NULL,
 original_mtime TEXT NOT NULL, sha256 TEXT NOT NULL, source_version INTEGER NOT NULL,
 import_identity TEXT NOT NULL UNIQUE, imported_at TEXT NOT NULL,
 extracted_artifact_path TEXT, extracted_artifact_hash TEXT, sensitivity_status TEXT NOT NULL,
 extraction_status TEXT NOT NULL, notes TEXT)zCREATE VIRTUAL TABLE IF NOT EXISTS kdata_fts USING fts5(
 doc_id UNINDEXED, source_identity UNINDEXED, project UNINDEXED,
 document_type UNINDEXED, original_path UNINDEXED, content))execute)cs    r   setupr   X   s'     1 2  ? @r   c                 d  & t        j                         } | j                  dd       | j                  dd       | j                         }t        |j
                  z  }t        d |j                  d      D              }g }t               }|j                  r|D ]  }t        |      }|j                         }t        |      }	|j                  t        |      |t        |      t!        |      t#        j$                  |j'                         j(                  t*        j,                        j/                         t1        |      d	t!        |	      |	rt3        |	      |	k7  rd
ndd	        t5        t7        j8                  |j
                  t!        |      t!        |      d|ddd             y t;        j<                  t>              }
t:        j@                  |
_!        tE        |
       	 |D ]  }t        |      }t        |      }|j                         }t1        |      }t#        j$                  |j'                         j(                  t*        j,                        j/                         }|jF                  jI                         }t        |      }t        |      }	|	rt3        |	      nd}||	k7  s|dv rd
nd}|
jK                  d|f      jM                         }|r|d   |k7  r|d   d	z   n|r|d   nd	}tO        ||      }d }d }|rtP        dz  tS        jT                  |jW                               jY                         d d dz   |jZ                  j]                  dd      z   dz   z  }d|z   dz   t_        |      z   dz   |z   d z   t_        |      z   d!z   |z   d"z   |z   d#z   |z   d$z   }t1        |jW                               }|||t!        |      ||||t!        |	      ||rt_        |      nd d%}|j                  |       |j                  r|r0|j`                  jc                  dd&       |je                  d'(       |
jK                  d)|d*t_        |      |||t!        |      ||||||rt_        |      nd |||	rd+nd,d-f       |
jK                  d.|f      jM                         d/   }|
jK                  d0t_        |      f       |
jK                  d1t_        |      |d*|t_        |      |xs d2f       |
jK                  d3d4|f      jM                         }|rN|d/   }|
jK                  d5|jf                  ||rd6nd7|rt_        |      nd |d8d9||d:||rt_        |      nd d;|f       nm|
jK                  d<d4|||jf                  ||rd6nd7|rt_        |      nd |d8d9d=||d:||rt_        |      nd d;f       |
jK                  d>      jM                         d/   }|
jK                  d?||f      jM                         }|r|d@   }nZ|
jK                  dA||||||rt_        |      nd |rdBnd,d;dCt_        |      |dDf       |
jK                  d>      jM                         d/   }|r~|ji                         D cg c]#  }|jk                         s|jk                         % }}|D cg c]  }tm        jn                  dE|      s| } }tq        | d dF       D ]
  \  }!&dG|z   dHz   tS        jT                  &jW                               jY                         z   }"dIt_        ts        &fdJtq        |      D        |!d	z               z   }#t7        j8                  ||t_        |      |rt_        |      nd ||dK|dLddM      }$|
jK                  dN|"&dOdO||||$dP||#dQdRd8dSf       |
jK                  dT|"f      jM                         d/   }%|
jK                  dU|%|t_        |      |#dV|f        |s|
jK                  dWd4d t_        |      ||d9||dG|z   dHz   |z   ||dXdYdDf        |j                  r|
ju                          n|
jw                          |
jy                          t5        t7        j8                  |j
                  t!        |      t!        |      |j                  |ddd             y c c}w c c}w # |
jy                          w xY w)ZNz	--projectz10 Projekte/Eisen, Heinz)defaultz	--dry-run
store_true)actionc              3  B   K   | ]  }|j                         s|  y wr   )is_file)rP   r   s     r   rQ   zmain.<locals>.<genexpr>g   s     !L		!!Ls   *   
RESTRICTEDNORMAL)	source_identityrelative_pathdocument_typesizemtimer   source_version
text_charssensitivityT)projectfilesrecordsdry_runreportF   )ensure_asciiindentrJ   )r6   r0   r*   r,   r.   z5select * from kdata_documents where source_identity=?r   r   zEisen-Heinz   r    _z.mdz---
source_identity: z&
project: Eisen, Heinz
original_path: z
document_type: z
source_version: z
source_hash: z
sensitivity: z
---

r#   )r   r   r   r   r   r   r   import_identityr   r   artifact)parentsexist_okr   )r   a  INSERT INTO kdata_documents(source_identity,project,original_path,relative_path,document_type,file_extension,file_size,original_mtime,sha256,source_version,import_identity,imported_at,extracted_artifact_path,extracted_artifact_hash,sensitivity_status,extraction_status,notes) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) ON CONFLICT(source_identity) DO UPDATE SET project=excluded.project,original_path=excluded.original_path,document_type=excluded.document_type,file_extension=excluded.file_extension,file_size=excluded.file_size,original_mtime=excluded.original_mtime,sha256=excluded.sha256,source_version=excluded.source_version,import_identity=excluded.import_identity,imported_at=excluded.imported_at,extracted_artifact_path=excluded.extracted_artifact_path,extracted_artifact_hash=excluded.extracted_artifact_hash,sensitivity_status=excluded.sensitivity_status,extraction_status=excluded.extraction_status,notes=excluded.noteszEisen, Heinz	EXTRACTED
REGISTEREDzOriginal remains unchangedz6select id from kdata_documents where source_identity=?r   z$delete from kdata_fts where doc_id=?zminsert into kdata_fts(doc_id,source_identity,project,document_type,original_path,content) values(?,?,?,?,?,?)z4[binary/image source registered; text not extracted]zOselect id from source_processing_registry where source_system=? and source_id=?zk-datena  update source_processing_registry set source_title=?,source_hash=?,artifact_status=?,artifact_path=?,artifact_hash=?,quality_status=?,graphiti_status=?,processed_at=?,updated_at=?,notes=?,source_identity=?,canonical_obsidian_path=?,current_state=? where id=?created
registeredPENDINGqueuedz!K-DATEN pilot; original immutableOBSERVEDaC  insert into source_processing_registry(source_system,source_id,source_external_id,source_title,source_hash,artifact_status,artifact_path,artifact_hash,quality_status,graphiti_status,graphiti_actor,processed_at,updated_at,notes,source_identity,canonical_obsidian_path,current_state) values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)zkdata-adapterzselect last_insert_rowid()zLselect * from source_versions where registry_id=? and content_version_hash=?
version_idzinsert into source_versions(registry_id,content_hash,content_version_hash,import_identity,version_number,artifact_path,artifact_state,reconciliation_state,graphiti_state,original_import_path,created_at,promotion_policy) values(?,?,?,?,?,?,?,?,?,?,?,?)CREATEDQUEUEDREVIEW_REQUIREDu_   (?i)(smartfan|lüftungsgerät|lüftungsanlage|menge|stück|stck|lieferung|auftragsbestätigung)(   zkdata::zline:c              3  :   K   | ]  \  }}|k(  s|d z     yw)r   Nr   )rP   jrb   lines      r   rQ   zmain.<locals>.<genexpr>   s<       ]Iehefgh  AB  DH  AH]^_`]`  ]Is   
zkdata-adapter-v1)r   r   original_pathartifact_pathartifact_hashsource_hashadapterr   )r   	sort_keysa<  insert or ignore into knowledge_units(knowledge_unit_identity,statement,entities,relations,source_identity,source_version_id,content_version_hash,provenance,promotion_reason,created_at,source_locator,evidence_class,confidence_reason,review_state,gold_state,legacy_placeholder) values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,0)z[]zK-DATEN pilot source extraction	PLAUSIBELz!source-extracted; review requiredNONEz>select id from knowledge_units where knowledge_unit_identity=?zinsert or ignore into knowledge_unit_provenance(knowledge_unit_id,source_identity,source_version,source_locator,representation_type,created_at) values(?,?,?,?,?,?)zkdata-extracteda"  insert or ignore into graphiti_import_queue(source_type,video_id,obsidian_path,content_hash,file_modified_at,graphiti_status,created_at,updated_at,reconciliation_record_id,source_version_id,import_identity,knowledge_unit_identity,action,promotion_policy) values(?,?,?,?,?,?,?,?,?,?,?,?,?,?)
__source__CREATE)=argparseArgumentParseradd_argument
parse_argsr   r   ru   rglobr   r   r   r   r   ra   r!   rC   r|   r   fromtimestampstatst_mtimer   r   r   r   rH   printjsondumpssqlite3connectDBRowrow_factoryr   r@   rA   r   fetchoner'   DERIVEDr   r   r$   r   rw   r   r   parentmkdir
write_textr?   
splitlinesrZ   rE   search	enumeratenextrollbackcommitclose)'apr	   rc   r   r   stampr   r   rawrN   r   sidohmtr   typcleansensoldversioniidr   ahcontentrecdocidregridsvvidr
   lines
candidatesidxidentlocprovkur   s'                                         @r   mainr#  e   s   r{C]^`b`o`op{  DP`o  aQ  UW  Ub  Ub  Ud  ST	!))m6!LTZZ_!LLUUWf_b_dY^II oaV1Sgaj&--S\]^S_pq  CK  LM  CN  VY  Z]  V^  go  g}  g}  ~  ~D  ~D  ~F  ~O  ~O  PX  P\  P\  g]  gg  gg  gi  sv  wz  s{  MN  \_  `d  \e  DH  MS  TX  MY  [_  M_  t@  em  An  3oo

aiiE
S[[_iop  D  LM  	N  O  QW
??2gkk58& "vaV11S1<<>sc#h28CYCYZ[Z`Z`ZbZkZkltlxlxCy  DD  DD  DF  LM  LT  LT  LZ  LZ  L\  HK  bj  kl  bm  ^a  t{  |}  t~  os  VZ  FL  MQ  FR  `b  @Ets.p'pv~4yyH#PYY[3  BE  JM  NV  JW  Y[  J[fijzf{|}f}  {~  be  fv  bw  DE]d  MV  WZ  []  M^  IL8T"m$gnnQXXZ&@&J&J&LSb&QRU&UVWV\V\VdVdehilVm&mns&stH  W  X[  [  \F  F  GJ  KL  GM  M  Na  a  be  e  fz  z  {~  F  {G  G  HY  Y  Z\  \  ]n  n  os  s  t    @E  E  FJ  Jv}  OR  SZ  Sa  Sa  Sc  Od  LNa3s8\^hj  }D  WZ  hk  lp  hq  @D  ai  PS  T\  P]  os  t3	==ii%%dD%A8CVCVW^hnCVCo99  u  wz  {I  JM  NO  JP  QR  SV  WZ  [^  _b  [c  df  gi  jq  ru  v{  MU  |  @H  |I  [_  `b  cg  w{  hs  AM  Nj  vk  l	
KSF	S	\	\	^_`	a5993SZMB99|  B  CH  I  JM  N\  ]`  ad  ef  ag  hm  hg  qg  ~h  iyybdmnocpqzz|3	s1v3qyy  "f  hi  hn  hn  oq  G  r{  MY  ks  Z]  ^f  Zg  y}  ~@  AJ  KS  TY  Z_  `C  DG  Ya  HK  LT  HU  gk  lv  wz  g{  |II  T  V_  `a  bc  de  dj  dj  km  {C  nw  IU  go  VY  Zb  Vc  uy  z|  }F  GO  P_  `e  fk  lO  PS  em  TW  X`  Ta  sw  xB	  UC	  D	  J	K	  J	S	  J	S	  T	p	  J	q	  J	z	  J	z	  J	|	  }	~	  J		  F	I	ii^`cdf_ghqqs2b#II  L  NQ  RT  UW  X[  \c  u}  dg  hp  dq  CG  U]  HQ  co  pz  {C  DG  HI  DJ  KP  Qb  Mc  d  jk  js  js  tP  jQ  jZ  jZ  j\  ]^  j_  fi#..0
>AGGI1779
>E
>   UaBII  /Q  RS  %T  UJ  Uj"o. hDC<GNN4;;=9CCEEU7SVW[  ]Iluv{l|  ]I  JM  NO  JO  XP  TQ  LQs  X\  Xb  Xb  vy  KR  cf  gh  ci  KS  z}  ~F  zG  Y]  np  A  L^  or  cs  AF  QU  XV  SWYY  N  PU  VZ  [_  `d  eh  il  mo  pt  uV  W\  ]`  al  mP  QZ  [a  Ob  c			RTYS[\eeghijRYY  u  wy  z}  ~A  BI  ~J  KN  O`  af  vg  h	h
 II  w  yB  CG  HK  LT  HU  VX  Y[  \d  ej  kp  qy  z}  q}  ~A  qA  BD  qD  EH  IL  MY  Zb  ct  xu  vE"vF YYqzz|xxz	
tzzQYYs5zCKZ[ZcZcmst  CH  PQ  R  S ? U 
s?   &Q=b #b9bb b-b1D b A"b 
b b/__main__)(__doc__
__future__r   r   csvr   r   r   	mimetypesosrE   r   r   r\   r   r   pathlibr   	xml.etreer   r^   r   r   r   rY   r   r   r   r!   r'   rC   rH   ri   r   r   r   r   r   r#  __name__r   r   r   <module>r-     s    # [ [ [ [ [ [ '  '56	 !*+F  LC  HC  HM  N 8 0 0 % Y

	 ,@/S` Zr   