Explorer
/tmp/aa045/multi_cluster.py
← Zurück ↓ Download
"""
AA-045: Multi-Source-Themencluster.
Gruppiert aktuelle Signale über Quellengrenzen hinweg (Jaccard auf topic+summary),
vergibt cluster_id und erhöht radar_score je zusätzlicher unabhängiger Quelle.
Konfiguration: sources_multi.yaml -> radar.*
"""

import sys
import uuid
import re
from pathlib import Path
sys.path.insert(0, str(Path(__file__).parent.parent))

import yaml
from datetime import datetime, timezone, timedelta

from sma_database import SIGNALS_DB_PATH

CONFIG_PATH = Path(__file__).parent / "sources_multi.yaml"

_STOP = {
    "de", "die", "das", "der", "und", "oder", "in", "auf", "ist", "von", "mit",
    "zu", "ein", "eine", "sich", "an", "nach", "bei", "hat", "im", "für", "aus",
    "the", "a", "and", "or", "is", "of", "for", "to", "el", "la", "los", "las",
    "en", "un", "una", "por", "para", "con", "que", "se", "del",
}


def _words(text: str) -> set:
    t = re.sub(r"[^a-z0-9äöüß\s]", " ", (text or "").lower())
    return {w for w in t.split() if w not in _STOP and len(w) > 3}


def _jaccard(a: set, b: set) -> float:
    if not a or not b:
        return 0.0
    return len(a & b) / len(a | b)


def run_clustering() -> dict:
    with open(CONFIG_PATH, encoding="utf-8") as fh:
        cfg = yaml.safe_load(fh)
    rcfg = cfg["radar"]
    window_h = int(rcfg.get("cluster_window_hours", 96))
    thr = float(rcfg.get("cluster_jaccard_threshold", 0.30))
    bonus_per = float(rcfg.get("multi_source_bonus_per_source", 6.0))
    bonus_max = float(rcfg.get("multi_source_bonus_max", 18.0))

    import sqlite3
    conn = sqlite3.connect(str(SIGNALS_DB_PATH))
    conn.row_factory = sqlite3.Row
    cutoff = (datetime.now(timezone.utc) - timedelta(hours=window_h)).isoformat()

    rows = conn.execute("""
        SELECT signal_id, source_platform, source_type, topic, short_summary,
               cluster_id, radar_score
        FROM signals
        WHERE created_at > ? AND status NOT IN ('ignoriert', 'veroeffentlicht')
    """, (cutoff,)).fetchall()

    # Union-Find über Jaccard-Ähnlichkeit
    parent = {r["signal_id"]: r["signal_id"] for r in rows}

    def find(x):
        while parent[x] != x:
            parent[x] = parent[parent[x]]
            x = parent[x]
        return x

    def union(a, b):
        ra, rb = find(a), find(b)
        if ra != rb:
            parent[rb] = ra

    enriched = []
    for r in rows:
        words = _words(f"{r['topic']} {r['short_summary']}")
        enriched.append((r, words))

    for i in range(len(enriched)):
        for j in range(i + 1, len(enriched)):
            r1, w1 = enriched[i]
            r2, w2 = enriched[j]
            if _jaccard(w1, w2) >= thr:
                union(r1["signal_id"], r2["signal_id"])

    clusters = {}
    for sid in parent:
        clusters.setdefault(find(sid), []).append(sid)

    now = datetime.now(timezone.utc).isoformat()
    n_clusters = n_bonus = 0
    for members in clusters.values():
        if len(members) < 2:
            continue  # Einzelsignal braucht keinen Cluster
        platforms = set()
        for m in members:
            row = next(r for r, _ in enriched if r["signal_id"] == m)
            p = row["source_platform"] or row["source_type"]
            platforms.add(p)
        n_indep = len(platforms)
        if n_indep < 2:
            continue  # gleiche Quelle -> kein Multi-Source-Bonus
        cluster_id = f"cl_{uuid.uuid4().hex[:10]}"
        bonus = min(bonus_per * (n_indep - 1), bonus_max)
        for m in members:
            conn.execute("""
                UPDATE signals SET cluster_id = ?,
                    radar_score = MIN(COALESCE(radar_score,0) + ?, 100.0)
                WHERE signal_id = ?
            """, (cluster_id, bonus, m))
        n_clusters += 1
        n_bonus += len(members)
        print(f"[CLUSTER] {cluster_id}: {len(members)} Signale, "
              f"{n_indep} Plattformen, +{bonus:.0f} Score")

    conn.commit()
    conn.close()
    return {"clusters": n_clusters, "signals_boosted": n_bonus,
            "window_hours": window_h, "threshold": thr}


if __name__ == "__main__":
    import json
    print(json.dumps(run_clustering(), indent=2))