Explorer
/proc/66/root/tmp/supadata_native.py
← Zurück ↓ Download
#!/usr/bin/env python3
"""Shared Supadata Native client for the e2e worker and maintenance batch."""
import datetime, os, time
from dataclasses import dataclass
from pathlib import Path
from typing import Any
import requests

API = 'https://api.supadata.ai/v1'
ENV = '/opt/struktur/youtube-research/.env.supadata'

class SupadataError(RuntimeError):
    def __init__(self, message, *, status=None, billable=None, kind='provider_error'):
        super().__init__(message); self.status=status; self.billable=billable; self.kind=kind
class NativeTranscriptUnavailable(SupadataError): pass
class QuotaWait(SupadataError): pass
class TransientSupadataError(SupadataError): pass

@dataclass
class NativeResult:
    segments: list[tuple[int,float,float,str]]
    language: str
    billable: str|None
    http_status: int
    headers: dict[str,str]
    content: list[dict[str,Any]]

def api_key(path=ENV):
    for line in Path(path).read_text().splitlines():
        if line.startswith('SUPADATA_API_KEY='):
            value=line.split('=',1)[1].strip()
            if value: return value
    raise RuntimeError('SUPADATA_API_KEY_missing')

def account_status(key=None):
    key=key or api_key(); r=requests.get(API+'/me',headers={'x-api-key':key},timeout=30)
    try: data=r.json() if r.headers.get('content-type','').startswith('application/json') else {}
    except ValueError: data={}
    maxc=data.get('maxCredits'); used=data.get('usedCredits')
    return {'http':r.status_code,'plan':data.get('plan'),'usedCredits':used,'maxCredits':maxc,
            'available': maxc-used if isinstance(maxc,int) and isinstance(used,int) else None}

def _headers(r):
    return {k.lower():v for k,v in r.headers.items() if k.lower() in ('content-type','x-billable-requests','x-billable-request','retry-after')}

def fetch_native(video_id, *, key=None, timeout=90):
    key=key or api_key(); params={'url':f'https://www.youtube.com/watch?v={video_id}','mode':'native','text':'false','lang':'de'}
    try: r=requests.get(API+'/transcript',params=params,headers={'x-api-key':key},timeout=timeout)
    except requests.RequestException as ex: raise TransientSupadataError(type(ex).__name__,kind='network_error') from ex
    bill=r.headers.get('x-billable-requests') or r.headers.get('x-billable-request'); headers=_headers(r)
    try: data=r.json() if r.headers.get('content-type','').startswith('application/json') else {}
    except ValueError: data={}
    if r.status_code==202 and data.get('jobId'):
        for _ in range(120):
            time.sleep(1)
            try: z=requests.get(API+'/transcript/'+str(data['jobId']),headers={'x-api-key':key},timeout=30)
            except requests.RequestException as ex: raise TransientSupadataError(type(ex).__name__,kind='network_error') from ex
            bill=bill or z.headers.get('x-billable-requests') or z.headers.get('x-billable-request'); headers.update(_headers(z))
            if z.status_code==200:
                r=z; data=z.json() if z.headers.get('content-type','').startswith('application/json') else {}; break
            if z.status_code not in (202,206): r=z; break
    if r.status_code in (402,429): raise QuotaWait(f'http_{r.status_code}',status=r.status_code,billable=bill,kind='quota')
    if r.status_code==206: raise NativeTranscriptUnavailable('native transcript unavailable',status=r.status_code,billable=bill,kind='native_unavailable')
    if r.status_code!=200:
        if r.status_code>=500: raise TransientSupadataError(f'http_{r.status_code}',status=r.status_code,billable=bill,kind='provider_5xx')
        raise TransientSupadataError(f'http_{r.status_code}',status=r.status_code,billable=bill,kind='provider_error')
    content=data.get('content'); lang=data.get('lang')
    if not isinstance(content,list) or not content or not lang:
        raise NativeTranscriptUnavailable('native transcript unavailable',status=200,billable=bill,kind='native_unavailable')
    segs=[]
    for i,s in enumerate(content):
        if not isinstance(s,dict) or not str(s.get('text','')).strip(): continue
        try: start=float(s.get('offset',0))/1000; dur=float(s.get('duration',0))/1000
        except (TypeError,ValueError): continue
        segs.append((i,start,start+max(dur,0),str(s['text']).strip()))
    if not segs: raise NativeTranscriptUnavailable('native transcript unavailable',status=200,billable=bill,kind='native_unavailable')
    return NativeResult(segs, str(lang), bill, r.status_code, headers, content)