Explorer
/tmp/published_at_normalizer.py
← Zurück ↓ Download
"""Normalisierung belastbarer Veröffentlichungszeitpunkte.

Rohwerte werden direkt am Intake bzw. in aa045_enrich normalisiert. Ein relativer
Wert darf ausschließlich gegen den tatsächlichen observed_at-Zeitpunkt gerechnet
werden; ohne belastbares Quelldatum wird NULL zurückgegeben.
"""
from __future__ import annotations

import calendar
import email.utils
import re
from datetime import datetime, timedelta, timezone


_RELATIVE = re.compile(
    r"^(?:(?P<de>vor)\s+(?P<de_n>\d+)\s*(?P<de_u>sekunde?n?|minute?n?|stunde?n?|tage?n?|woche?n?|monate?n?|jahre?n?)|"
    r"(?P<es>hace)\s+(?P<es_n>\d+)\s*(?P<es_u>segundo?s?|minuto?s?|hora?s?|día?s?|dias?|semana?s?|mes(?:es)?|año?s?|anos?)|"
    r"(?P<en_n>\d+)\s*(?P<en_u>second?s?|minute?s?|hour?s?|day?s?|week?s?|month?s?|year?s?)\s+ago)$",
    re.IGNORECASE,
)


def _parse_reference(value: str | None) -> datetime | None:
    if not value:
        return None
    raw = str(value).strip()
    try:
        dt = datetime.fromisoformat(raw.replace("Z", "+00:00"))
    except ValueError:
        return None
    if dt.tzinfo is None:
        dt = dt.replace(tzinfo=timezone.utc)
    return dt.astimezone(timezone.utc)


def _add_months(dt: datetime, months: int) -> datetime:
    index = dt.year * 12 + (dt.month - 1) - months
    year, month0 = divmod(index, 12)
    month = month0 + 1
    day = min(dt.day, calendar.monthrange(year, month)[1])
    return dt.replace(year=year, month=month, day=day)


def _relative_to_iso(raw: str, reference: datetime) -> str | None:
    text = " ".join(str(raw).strip().lower().split())
    if text in {"yesterday", "gestern", "ayer"}:
        return (reference - timedelta(days=1)).isoformat()
    if text in {"today", "heute", "hoy"}:
        return reference.isoformat()
    match = _RELATIVE.match(text)
    if not match:
        return None
    if match.group("de"):
        number, unit = int(match.group("de_n")), match.group("de_u")
        lang = "de"
    elif match.group("es"):
        number, unit = int(match.group("es_n")), match.group("es_u")
        lang = "es"
    else:
        number, unit = int(match.group("en_n")), match.group("en_u")
        lang = "en"

    if lang == "de":
        if unit.startswith("sek"): delta = timedelta(seconds=number)
        elif unit.startswith("min"): delta = timedelta(minutes=number)
        elif unit.startswith("stund"): delta = timedelta(hours=number)
        elif unit.startswith("tag"): delta = timedelta(days=number)
        elif unit.startswith("woch"): delta = timedelta(weeks=number)
        elif unit.startswith("monat"): return _add_months(reference, number).isoformat()
        elif unit.startswith("jahr"): return _add_months(reference, number * 12).isoformat()
        else: return None
    elif lang == "es":
        if unit.startswith("seg"): delta = timedelta(seconds=number)
        elif unit.startswith("min"): delta = timedelta(minutes=number)
        elif unit.startswith("hora"): delta = timedelta(hours=number)
        elif unit.startswith("día") or unit.startswith("dia"): delta = timedelta(days=number)
        elif unit.startswith("sem"): delta = timedelta(weeks=number)
        elif unit.startswith("mes"): return _add_months(reference, number).isoformat()
        elif unit.startswith("año") or unit.startswith("ano"): return _add_months(reference, number * 12).isoformat()
        else: return None
    else:
        if unit.startswith("second"): delta = timedelta(seconds=number)
        elif unit.startswith("minute"): delta = timedelta(minutes=number)
        elif unit.startswith("hour"): delta = timedelta(hours=number)
        elif unit.startswith("day"): delta = timedelta(days=number)
        elif unit.startswith("week"): delta = timedelta(weeks=number)
        elif unit.startswith("month"): return _add_months(reference, number).isoformat()
        elif unit.startswith("year"): return _add_months(reference, number * 12).isoformat()
        else: return None
    return (reference - delta).isoformat()


def normalize_published_at(raw: object, observed_at: str | None) -> str | None:
    """Return canonical UTC ISO-8601 or None.

    Accepted inputs are ISO-8601, RFC-822/RSS dates, and explicit German,
    Spanish, or English relative source values. Unknown/empty values become NULL.
    """
    if raw is None:
        return None
    text = str(raw).strip()
    if not text:
        return None

    dt = _parse_reference(text)
    if dt is not None:
        return dt.isoformat()

    try:
        dt = email.utils.parsedate_to_datetime(text)
        if dt.tzinfo is None:
            dt = dt.replace(tzinfo=timezone.utc)
        return dt.astimezone(timezone.utc).isoformat()
    except (TypeError, ValueError, IndexError, OverflowError):
        pass

    reference = _parse_reference(observed_at)
    if reference is None:
        return None
    return _relative_to_iso(text, reference)