Explorer
/opt/struktur/lead-engine/discovery/scraper.py
← Zurück ↓ Download
"""
scraper.py -- Holt HTML-Inhalt einer URL.
"""

import logging
import requests

logger = logging.getLogger(__name__)

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
                  "AppleWebKit/537.36 (KHTML, like Gecko) "
                  "Chrome/120.0.0.0 Safari/537.36"
}


def fetch(url: str, timeout: int = 10) -> str | None:
    """
    Holt HTML einer URL.

    Returns:
        HTML-String oder None bei Fehler
    """
    try:
        resp = requests.get(url, timeout=timeout, headers=HEADERS,
                            verify=False, allow_redirects=True)
        if resp.status_code == 200:
            return resp.text
        logger.debug(f"HTTP {resp.status_code} fuer {url}")
        return None
    except Exception as e:
        logger.debug(f"Fetch fehlgeschlagen ({url}): {e}")
        return None