#!/usr/bin/env python3
"""Shared Supadata Native client for the e2e worker and maintenance batch."""
import datetime, os, time
from dataclasses import dataclass
from pathlib import Path
from typing import Any
import requests
API = 'https://api.supadata.ai/v1'
ENV = '/opt/struktur/youtube-research/.env.supadata'
class SupadataError(RuntimeError):
def __init__(self, message, *, status=None, billable=None, kind='provider_error'):
super().__init__(message); self.status=status; self.billable=billable; self.kind=kind
class NativeTranscriptUnavailable(SupadataError): pass
class QuotaWait(SupadataError): pass
class TransientSupadataError(SupadataError): pass
@dataclass
class NativeResult:
segments: list[tuple[int,float,float,str]]
language: str
billable: str|None
http_status: int
headers: dict[str,str]
content: list[dict[str,Any]]
def api_key(path=ENV):
for line in Path(path).read_text().splitlines():
if line.startswith('SUPADATA_API_KEY='):
value=line.split('=',1)[1].strip()
if value: return value
raise RuntimeError('SUPADATA_API_KEY_missing')
def account_status(key=None):
key=key or api_key(); r=requests.get(API+'/me',headers={'x-api-key':key},timeout=30)
try: data=r.json() if r.headers.get('content-type','').startswith('application/json') else {}
except ValueError: data={}
maxc=data.get('maxCredits'); used=data.get('usedCredits')
return {'http':r.status_code,'plan':data.get('plan'),'usedCredits':used,'maxCredits':maxc,
'available': maxc-used if isinstance(maxc,int) and isinstance(used,int) else None}
def _headers(r):
return {k.lower():v for k,v in r.headers.items() if k.lower() in ('content-type','x-billable-requests','x-billable-request','retry-after')}
def fetch_native(video_id, *, key=None, timeout=90):
key=key or api_key(); params={'url':f'https://www.youtube.com/watch?v={video_id}','mode':'native','text':'false','lang':'de'}
try: r=requests.get(API+'/transcript',params=params,headers={'x-api-key':key},timeout=timeout)
except requests.RequestException as ex: raise TransientSupadataError(type(ex).__name__,kind='network_error') from ex
bill=r.headers.get('x-billable-requests') or r.headers.get('x-billable-request'); headers=_headers(r)
try: data=r.json() if r.headers.get('content-type','').startswith('application/json') else {}
except ValueError: data={}
if r.status_code==202 and data.get('jobId'):
for _ in range(120):
time.sleep(1)
try: z=requests.get(API+'/transcript/'+str(data['jobId']),headers={'x-api-key':key},timeout=30)
except requests.RequestException as ex: raise TransientSupadataError(type(ex).__name__,kind='network_error') from ex
bill=bill or z.headers.get('x-billable-requests') or z.headers.get('x-billable-request'); headers.update(_headers(z))
if z.status_code==200:
r=z; data=z.json() if z.headers.get('content-type','').startswith('application/json') else {}; break
if z.status_code not in (202,206): r=z; break
if r.status_code in (402,429): raise QuotaWait(f'http_{r.status_code}',status=r.status_code,billable=bill,kind='quota')
if r.status_code==206: raise NativeTranscriptUnavailable('native transcript unavailable',status=r.status_code,billable=bill,kind='native_unavailable')
if r.status_code!=200:
if r.status_code>=500: raise TransientSupadataError(f'http_{r.status_code}',status=r.status_code,billable=bill,kind='provider_5xx')
raise TransientSupadataError(f'http_{r.status_code}',status=r.status_code,billable=bill,kind='provider_error')
content=data.get('content'); lang=data.get('lang')
if not isinstance(content,list) or not content or not lang:
raise NativeTranscriptUnavailable('native transcript unavailable',status=200,billable=bill,kind='native_unavailable')
segs=[]
for i,s in enumerate(content):
if not isinstance(s,dict) or not str(s.get('text','')).strip(): continue
try: start=float(s.get('offset',0))/1000; dur=float(s.get('duration',0))/1000
except (TypeError,ValueError): continue
segs.append((i,start,start+max(dur,0),str(s['text']).strip()))
if not segs: raise NativeTranscriptUnavailable('native transcript unavailable',status=200,billable=bill,kind='native_unavailable')
return NativeResult(segs, str(lang), bill, r.status_code, headers, content)