Explorer
/opt/struktur/AA-043-P43-R2-extractor-v3.py
← Zurück ↓ Download
import re
VERSION='P43-R2-extractor-v3-deterministic-1.0'
NON=re.compile(r'^(hi|hello|thanks|subscribe|like and subscribe|click|welcome|in this video|today we|sponsor|brought to you|chapter|table of contents)',re.I)
FACT=re.compile(r'\b(is|are|means|requires|supports|uses|has|can|must|should|costs|includes|allows|depends|because|therefore|wenn|ist|sind|muss|soll|kann|verwendet|benötigt|enthält)\b',re.I)
def classify(seg,window):
 t=seg['text'].strip(); low=t.lower(); reason=[]
 if not t or len(t)<25: return 'REJECT','OTHER_NON_KNOWLEDGE'
 if NON.search(t): return 'REJECT','TEMPLATE'
 if re.search(r'\b(subscribe|comment below|link in description|sponsor)\b',low): return 'REJECT','CTA'
 if re.search(r'\b(first|second|next section|chapter)\b',low) and len(t)<100: return 'REJECT','NAVIGATION'
 if not FACT.search(t): return 'REVIEW_REQUIRED','CONTEXT_MISSING'
 if re.search(r'\band\b.*\b(and|but)\b',low) or re.search(r'\b(because|therefore)\b',low): return 'REVIEW_REQUIRED','MULTI_CLAIM'
 if len(t)>500: return 'REVIEW_REQUIRED','OVERLONG'
 return 'ACCEPT','ATOMIC_GOOD'
def extract(segments,source,version):
 out=[]
 for i,s in enumerate(segments):
  win=[x['text'] for x in segments[max(0,i-1):min(len(segments),i+2)]]
  status,reason=classify(s,win)
  r={'segment_index':s['segment_index'],'status':status,'reason':reason,'original_span':s['text'],'canonical_statement':s['text'],'subject':None,'predicate':None,'object':None,'qualifiers':[],'negation':bool(re.search(r'\b(no|not|never|kein|nicht)\b',s['text'],re.I)),'numeric_values':re.findall(r'\b\d+(?:[.,]\d+)?\b',s['text']),'units':re.findall(r'\b(?:%|seconds?|minutes?|hours?|euro|dollars?|kg|mb)\b',s['text'],re.I),'temporal_scope':None,'conditions':None,'canonical_entities':[],'context_window':win,'source_locator':s['source_locator'],'canonical_source_identity':source,'content_version':version,'extractor_version':VERSION,'hallucination':False}
  out.append(r)
 return out