Explorer
/opt/struktur/lead-engine/discovery/orchestrator.py
← Zurück ↓ Download
"""
orchestrator.py -- Koordiniert Discovery fuer einen Kontakt.

Ablauf:
1. Suchanfragen bauen
2. DuckDuckGo durchsuchen
3. Top-URLs scrapen
4. Text + Standort extrahieren
5. Fallback: low_data_lead wenn < 500 Zeichen

Gibt zurueck:
    {
        "text_blob":  str,   -- gesammelter Text (alle Quellen)
        "location":   str|None,
        "sources":    list,  -- genutzte URLs
        "low_data":   bool,  -- True wenn < 500 Zeichen
    }
"""

import logging
from .query_builder import build_queries
from .search import search
from .scraper import fetch
from .parser import extract_text, extract_location

logger = logging.getLogger(__name__)

MIN_TEXT_LENGTH = 500


def run_discovery(company_name: str, website: str = "") -> dict:
    """
    Fuehrt vollstaendige Multi-Source-Discovery durch.

    Args:
        company_name: Name der Firma
        website:      Bekannte Website (wird zuerst gescrapt)

    Returns:
        Dict mit text_blob, location, sources, low_data
    """
    collected_text = []
    found_location = None
    sources_used = []

    # 0. Bekannte Website zuerst scrapen (hoechste Prioritaet)
    if website:
        html = fetch(website)
        if html:
            text = extract_text(html)
            if text:
                collected_text.append(text)
                sources_used.append(website)
                if not found_location:
                    found_location = extract_location(text)
                logger.debug(f"Website direkt: {len(text)} Zeichen")

    # 1. Suchanfragen bauen
    queries = build_queries(company_name)

    # 2. Suchen + Scrapen
    seen_urls = set(sources_used)
    for q in queries:
        links = search(q)

        for link in links:
            if link in seen_urls:
                continue
            seen_urls.add(link)

            html = fetch(link)
            if not html:
                continue

            text = extract_text(html)
            if not text:
                continue

            collected_text.append(text)
            sources_used.append(link)

            if not found_location:
                found_location = extract_location(text)

        # Fruehzeitig abbrechen wenn genuegend Text gesammelt
        total = sum(len(t) for t in collected_text)
        if total >= 5000:
            logger.debug(f"Genug Text gesammelt ({total} Zeichen) — breche fruehzeitig ab")
            break

    text_blob = " ".join(collected_text)
    low_data = len(text_blob) < MIN_TEXT_LENGTH

    logger.info(
        f"Discovery abgeschlossen: {len(text_blob)} Zeichen | "
        f"{len(sources_used)} Quellen | low_data={low_data}"
    )

    return {
        "text_blob": text_blob,
        "location":  found_location,
        "sources":   sources_used,
        "low_data":  low_data,
    }