
    ljL                     @    d dl Z dZ e j                  d      Zd Zd Zy)    Nz%P43-R2-segmenter-v2-deterministic-1.0u   [.!?。！？:]$c                    | xs d} g }d}t        | j                  d            D ]k  \  }}|j                  d      }|}	|	t        |      z   }
|t        |      z  }|j	                         sF|j                  ||	|
|j	                         d d d       m g }d}|t        |      k  r||   }|d   }t        j                  d|      }t        |      dkD  r|d	   }t        |      D ]  \  }}|j	                         s|}|t        |      dz   z  }|j                  t        |      |j	                         |d
   g|t        |d   |t        |      z         |d   |d   d|||g d        |dz  }|dz   t        |      k  rt        |      dk  rt        j                  |      sz||dz      d   d d j                         r^||dz      }|dz   |d   z   }|j                  t        |      ||d
   |d
   g|d	   |d   |d   |d   d||||d
   |d
   gd       |dz  }|j                  t        |      ||d
   g|d	   |d   |d   |d   d|||g d       |dz  }|t        |      k  r|S )N r   Tz
)raw_segment_indexstart_offset
end_offsettextstart_timestampend_timestampr   u   (?<=[.!?。！？])\s+   r   r   r   r	   r
   )raw_segment_indicesr   r   r	   r
   )segment_indexr   source_locatorcanonical_source_identitycontent_versionsource_typereassembled_fromP       )	enumerate
splitlinesrstriplenstripappendresplitminPUNCTsearchislower)r   sourceversionr   rawposilinesstartendoutcurtxtpartsoffjpstnxtjoineds                        +/opt/struktur/AA-043-P43-R2-segmenter-v2.pysplit_sourcer6      s<   	DS	CDOOD12 ]$
++f
SueCFl#CTNC779cjjauZ]efelelen  BF  W[  "\  ]] 	CaA
CH*FFC2884Mc+RUu:a<N#C ' G!wwy(SVAX

CHAGGIor  tG  pH  oI  Y[  il  mp  q}  m~  A  BE  FG  BH  H  iI  \_  `q  \r  CF  GV  CW  XX  u{  NU  do  CE  F  GG qDA(Q3s3x<CHRKS0Ac!A#hvFVWYXYFZFbFbFdAaCCSV!4&JJChkl  iA  BE  FY  BZ  h[  kn  o}  k~  LO  P\  L]  ps  tE  pF  WZ  [j  Wk  Ql  IO  bi  xC  X[  \o  Xp  qt  uH  qI  WJ  K  L  NO  QR  NR  NO  T\

CHCadexay`z  KN  O]  K^  lo  p|  l}  PS  Te  Pf  wz  {J  wK  JL  io  BI  Xc  wy  z  	{  }~  @A  }A  }~ CH* J    c                 t   g }| D ]/  }|d   }g }t        |      dkD  r|j                  d       t        |      dk  r|j                  d       t        j                  |      st        |      dkD  r|j                  d       t	        j
                  d|t        j                        r|j                  d	       t	        j                  d
|t        j                        r|j                  d       t	        j
                  d|      r|j                  d       |s|j                  |d   |||d   d       2 |S )Nr   iX  OVERLONG_SEGMENT   SHORT_FRAGMENT(   MISSING_PUNCTUATIONzK^(https?://|subscribe|like and subscribe|thanks for watching|in this video)HEADING_OR_METADATAz\b(uh|um|you know|so so so)\bASR_FRAGMENTu
   ^[-*•]\sLIST_STRUCTUREr   r   )r   issuesr   r   )r   r   r   r    r   matchI)segmentsrowsr(   trA   s        r5   audit_segmentsrG      s	   	D }
F)BVq6#:v}}%78q6"9fmm$45||A3q6"9fmm<Q.R88bcdegeieijlrlyly  {P  mQ995a=v}}^?\88M!$fmm4D&E4;;/0BFZ[jklyjz{|} Kr7   )r   VERSIONcompiler   r6   rG    r7   r5   <module>rK      s(    	/bjj$%0r7   