Explorer
/opt/struktur/worker-watcher/src/worker_watcher/alerts.py
← Zurück ↓ Download
"""Read-only alert summary for external dashboards."""

from collections.abc import Iterable, Mapping
from datetime import UTC, datetime
from typing import Any

from .timeutil import iso, parse_iso

ACTIVE_HEALTH_STATUSES = frozenset({"failed", "degraded", "stale", "unknown"})
HEALTH_LABELS = {
    "healthy": "fehlerfrei",
    "degraded": "gestört",
    "failed": "fehlgeschlagen",
    "stale": "veraltet",
    "unknown": "unbekannt",
}


def _value(row: Mapping[str, Any], key: str) -> Any:
    return row.get(key)


def _evidence(row: Mapping[str, Any]) -> dict[str, Any]:
    value = _value(row, "evidence_json")
    if isinstance(value, list):
        return {str(item.get("key")): item.get("value") for item in value if isinstance(item, dict) and "key" in item}
    return {}


def _problem_reason(row: Mapping[str, Any]) -> str:
    reason = str(_value(row, "status_reason") or "")
    if reason == "run failed with exit code 15":
        return "Ausführung mit Exitcode 15 fehlgeschlagen"
    if reason.startswith("run failed with exit code "):
        return f"{reason.removeprefix('run failed with ')} fehlgeschlagen"
    translations = {
        "reported source status": "Gemeldeter Quellenstatus",
        "collector could not read the configured source": "Konfigurierte Quelle nicht verfügbar",
        "last known job status retained; current observation unavailable": "Letzter bekannter Status; aktuelle Beobachtung nicht verfügbar",
        "expected run is overdue": "Erwarteter Lauf ist überfällig",
        "source was readable but did not provide enough evidence": "Quelle lieferte nicht genügend Nachweise",
        "no expected activity for more than 60 seconds": "Keine erwartete Aktivität seit mehr als 60 Sekunden",
    }
    if reason in translations:
        return translations[reason]
    evidence = _evidence(row)
    if evidence.get("systemd_ActiveState") or evidence.get("systemd_SubState"):
        active = str(evidence.get("systemd_ActiveState") or "unbekannt")
        sub = str(evidence.get("systemd_SubState") or "unbekannt")
        return f"Systemd-Zustand: {active}/{sub}"
    health = str(_value(row, "health_status") or "unknown")
    return f"Aktueller Zustand: {HEALTH_LABELS.get(health, 'unbekannt')}"


def _problem_since(row: Mapping[str, Any], events_by_instance: Mapping[int, str]) -> datetime | None:
    event_time = events_by_instance.get(int(_value(row, "instance_id")))
    if event_time:
        parsed = parse_iso(event_time)
        if parsed is not None:
            return parsed
    if _value(row, "health_status") == "failed":
        return parse_iso(_value(row, "last_failure_at"))
    return None


def _is_active_alert(row: Mapping[str, Any]) -> bool:
    return (_value(row, "health_status") in ACTIVE_HEALTH_STATUSES
            or _value(row, "observation_status") == "unavailable"
            or _value(row, "status_reason") == "expected run is overdue")


def build_alert_summary(
    health: Mapping[str, Any],
    job_rows: Iterable[Mapping[str, Any]],
    status_events: Iterable[Mapping[str, Any]],
    now: datetime,
    cycle_interval_seconds: int,
) -> dict[str, Any]:
    """Build a compact external summary without changing watcher state or data models."""
    now = now.astimezone(UTC)
    rows = list(job_rows)
    event_rows = list(status_events)
    events_by_instance: dict[int, str] = {}
    for row in rows:
        instance_id = _value(row, "instance_id")
        if instance_id is None:
            continue
        matching = [event for event in event_rows
                    if event.get("instance_id") == instance_id
                    and event.get("new_health_status") == _value(row, "health_status")
                    and event.get("new_observation_status") == _value(row, "observation_status")]
        if matching and matching[-1].get("occurred_at"):
            events_by_instance[int(instance_id)] = str(matching[-1]["occurred_at"])

    active_rows = [row for row in rows if _is_active_alert(row)]
    last_cycle_finished = parse_iso(health.get("last_cycle_finished_at"))
    age_seconds = None if last_cycle_finished is None else max(0, int((now - last_cycle_finished).total_seconds()))
    stale_after_seconds = max(1, int(cycle_interval_seconds) * 3)
    data_is_current = last_cycle_finished is not None and age_seconds is not None and age_seconds <= stale_after_seconds

    failed = sum(1 for row in rows if _value(row, "health_status") == "failed")
    degraded = sum(1 for row in rows if _value(row, "health_status") == "degraded")
    stale = sum(1 for row in rows if _value(row, "health_status") == "stale")
    unknown = sum(1 for row in rows if _value(row, "health_status") == "unknown")
    unavailable = sum(1 for row in rows if _value(row, "observation_status") == "unavailable")
    overdue = sum(1 for row in rows if _value(row, "status_reason") == "expected run is overdue")
    critical_alert = any(_value(row, "health_status") == "failed" or _value(row, "severity") == "critical" for row in active_rows)

    reasons: list[str] = []
    if failed:
        reasons.append(f"{failed} fehlgeschlagene Aufgabe" + ("n" if failed != 1 else ""))
    if degraded:
        reasons.append(f"{degraded} gestörte Aufgabe" + ("n" if degraded != 1 else ""))
    if stale:
        reasons.append(f"{stale} veraltete Beobachtung" + ("en" if stale != 1 else ""))
    if unknown:
        reasons.append(f"{unknown} unbekannte Aufgabe" + ("n" if unknown != 1 else ""))
    if unavailable:
        reasons.append(f"{unavailable} nicht verfügbare Beobachtung" + ("en" if unavailable != 1 else ""))
    if overdue:
        reasons.append(f"{overdue} überfällige Aufgabe" + ("n" if overdue != 1 else ""))
    if int(health.get("collectors_failed", 0) or 0):
        count = int(health["collectors_failed"])
        reasons.append(f"{count} fehlgeschlagener Collector" + ("s" if count != 1 else ""))

    if health.get("database_status") == "error" or not data_is_current:
        overall_status = "unavailable"
        if last_cycle_finished is None:
            reasons.insert(0, "Noch keine abgeschlossene Worker-Prüfung")
        else:
            reasons.insert(0, "Worker-Überwachung nicht aktuell")
    elif critical_alert:
        overall_status = "critical"
    elif active_rows or health.get("watcher_status") == "degraded":
        overall_status = "warning"
    elif health.get("watcher_status") == "healthy":
        overall_status = "healthy"
    else:
        overall_status = "unknown"

    problems: list[dict[str, Any]] = []
    for row in active_rows:
        problem_since = _problem_since(row, events_by_instance)
        problems.append({
            "job_key": _value(row, "job_key"),
            "display_name": _value(row, "name"),
            "health_status": _value(row, "health_status"),
            "observation_status": _value(row, "observation_status"),
            "severity": _value(row, "severity"),
            "problem_since": iso(problem_since),
            "age_seconds": None if problem_since is None else max(0, int((now - problem_since).total_seconds())),
            "reason": _problem_reason(row),
        })
    problems.sort(key=lambda item: (item["problem_since"] is None, item["problem_since"] or "", str(item["job_key"])))

    return {
        "overall_status": overall_status,
        "overall_reasons": reasons,
        "watcher_status": health.get("watcher_status", "unknown"),
        "jobs_total": int(health.get("jobs_total", len(rows)) or 0),
        "healthy": int(health.get("jobs_healthy", 0) or 0),
        "degraded": degraded,
        "failed": failed,
        "unknown": unknown,
        "stale": stale,
        "unavailable": unavailable,
        "overdue": overdue,
        "active_alerts": len(active_rows),
        "oldest_problem": problems[0] if problems else None,
        "last_cycle_finished_at": health.get("last_cycle_finished_at"),
        "last_successful_cycle_at": health.get("last_successful_cycle_at"),
        "data_age_seconds": age_seconds,
        "data_stale_after_seconds": stale_after_seconds,
        "data_is_current": data_is_current,
        "generated_at": iso(now),
    }