import re
VERSION='P43-R2-segmenter-v2-deterministic-1.0'
PUNCT=re.compile(r'[.!?。!?:]$')
def split_source(text,source,version,source_type):
text=text or ''; raw=[]
# Preserve line/character offsets; transcript lines are kept as atomic inputs first.
pos=0
for i,line in enumerate(text.splitlines(True)):
s=line.rstrip('\r\n'); start=pos; end=start+len(s); pos+=len(line)
if s.strip(): raw.append({'raw_segment_index':i,'start_offset':start,'end_offset':end,'text':s.strip(),'start_timestamp':None,'end_timestamp':None})
out=[]; i=0
while i<len(raw):
cur=raw[i]; txt=cur['text']; parts=re.split(r'(?<=[.!?。!?])\s+',txt)
if len(parts)>1:
off=cur['start_offset']
for j,p in enumerate(parts):
if not p.strip(): continue
st=off; off+=len(p)+1
out.append({'segment_index':len(out),'text':p.strip(),'source_locator':{'raw_segment_indices':[cur['raw_segment_index']],'start_offset':st,'end_offset':min(cur['end_offset'],st+len(p)),'start_timestamp':cur['start_timestamp'],'end_timestamp':cur['end_timestamp']},'canonical_source_identity':source,'content_version':version,'source_type':source_type,'reassembled_from':[]})
i+=1; continue
# conservative join only when current line is an obvious fragment and next is immediate
if i+1<len(raw) and len(txt)<80 and not PUNCT.search(txt) and raw[i+1]['text'][:1].islower():
nxt=raw[i+1]; joined=txt+' '+nxt['text']
out.append({'segment_index':len(out),'text':joined,'source_locator':{'raw_segment_indices':[cur['raw_segment_index'],nxt['raw_segment_index']],'start_offset':cur['start_offset'],'end_offset':nxt['end_offset'],'start_timestamp':cur['start_timestamp'],'end_timestamp':nxt['end_timestamp']},'canonical_source_identity':source,'content_version':version,'source_type':source_type,'reassembled_from':[cur['raw_segment_index'],nxt['raw_segment_index']]}); i+=2; continue
out.append({'segment_index':len(out),'text':txt,'source_locator':{'raw_segment_indices':[cur['raw_segment_index']],'start_offset':cur['start_offset'],'end_offset':cur['end_offset'],'start_timestamp':cur['start_timestamp'],'end_timestamp':cur['end_timestamp']},'canonical_source_identity':source,'content_version':version,'source_type':source_type,'reassembled_from':[]}); i+=1
return out
def audit_segments(segments):
rows=[]
for s in segments:
t=s['text']; issues=[]
if len(t)>600: issues.append('OVERLONG_SEGMENT')
if len(t)<25: issues.append('SHORT_FRAGMENT')
if not PUNCT.search(t) and len(t)>40: issues.append('MISSING_PUNCTUATION')
if re.match(r'^(https?://|subscribe|like and subscribe|thanks for watching|in this video)',t,re.I): issues.append('HEADING_OR_METADATA')
if re.search(r'\b(uh|um|you know|so so so)\b',t,re.I): issues.append('ASR_FRAGMENT')
if re.match(r'^[-*•]\s',t): issues.append('LIST_STRUCTURE')
if issues: rows.append({'segment_index':s['segment_index'],'issues':issues,'text':t,'source_type':s['source_type']})
return rows