Explorer
/opt/struktur/AA-043-P43-R2-segmenter-v2.py
← Zurück ↓ Download
import re
VERSION='P43-R2-segmenter-v2-deterministic-1.0'
PUNCT=re.compile(r'[.!?。!?:]$')

def split_source(text,source,version,source_type):
    text=text or ''; raw=[]
    # Preserve line/character offsets; transcript lines are kept as atomic inputs first.
    pos=0
    for i,line in enumerate(text.splitlines(True)):
        s=line.rstrip('\r\n'); start=pos; end=start+len(s); pos+=len(line)
        if s.strip(): raw.append({'raw_segment_index':i,'start_offset':start,'end_offset':end,'text':s.strip(),'start_timestamp':None,'end_timestamp':None})
    out=[]; i=0
    while i<len(raw):
        cur=raw[i]; txt=cur['text']; parts=re.split(r'(?<=[.!?。!?])\s+',txt)
        if len(parts)>1:
            off=cur['start_offset']
            for j,p in enumerate(parts):
                if not p.strip(): continue
                st=off; off+=len(p)+1
                out.append({'segment_index':len(out),'text':p.strip(),'source_locator':{'raw_segment_indices':[cur['raw_segment_index']],'start_offset':st,'end_offset':min(cur['end_offset'],st+len(p)),'start_timestamp':cur['start_timestamp'],'end_timestamp':cur['end_timestamp']},'canonical_source_identity':source,'content_version':version,'source_type':source_type,'reassembled_from':[]})
            i+=1; continue
        # conservative join only when current line is an obvious fragment and next is immediate
        if i+1<len(raw) and len(txt)<80 and not PUNCT.search(txt) and raw[i+1]['text'][:1].islower():
            nxt=raw[i+1]; joined=txt+' '+nxt['text']
            out.append({'segment_index':len(out),'text':joined,'source_locator':{'raw_segment_indices':[cur['raw_segment_index'],nxt['raw_segment_index']],'start_offset':cur['start_offset'],'end_offset':nxt['end_offset'],'start_timestamp':cur['start_timestamp'],'end_timestamp':nxt['end_timestamp']},'canonical_source_identity':source,'content_version':version,'source_type':source_type,'reassembled_from':[cur['raw_segment_index'],nxt['raw_segment_index']]}); i+=2; continue
        out.append({'segment_index':len(out),'text':txt,'source_locator':{'raw_segment_indices':[cur['raw_segment_index']],'start_offset':cur['start_offset'],'end_offset':cur['end_offset'],'start_timestamp':cur['start_timestamp'],'end_timestamp':cur['end_timestamp']},'canonical_source_identity':source,'content_version':version,'source_type':source_type,'reassembled_from':[]}); i+=1
    return out

def audit_segments(segments):
    rows=[]
    for s in segments:
        t=s['text']; issues=[]
        if len(t)>600: issues.append('OVERLONG_SEGMENT')
        if len(t)<25: issues.append('SHORT_FRAGMENT')
        if not PUNCT.search(t) and len(t)>40: issues.append('MISSING_PUNCTUATION')
        if re.match(r'^(https?://|subscribe|like and subscribe|thanks for watching|in this video)',t,re.I): issues.append('HEADING_OR_METADATA')
        if re.search(r'\b(uh|um|you know|so so so)\b',t,re.I): issues.append('ASR_FRAGMENT')
        if re.match(r'^[-*•]\s',t): issues.append('LIST_STRUCTURE')
        if issues: rows.append({'segment_index':s['segment_index'],'issues':issues,'text':t,'source_type':s['source_type']})
    return rows