"""
RSS Intake -- Google News RSS + Mallorca-spezifische Feeds
Kein API-Key noetig. Alle Feeds oeffentlich zugaenglich.
"""
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).parent.parent))
import re as _re
import html as _html
import urllib.request
import xml.etree.ElementTree as ET
def _clean_html(raw: str) -> str:
"""Entfernt HTML-Tags, dekodiert Entities, normalisiert Whitespace."""
if not raw:
return ""
text = _re.sub(r"<(script|style)[^>]*>.*?</(script|style)>", " ", raw, flags=_re.DOTALL | _re.IGNORECASE)
text = _re.sub(r"<[^>]+>", " ", text)
# Unclosed tags (truncated HTML): ab '<' abschneiden
if "<" in text:
text = text[:text.index("<")]
text = _html.unescape(text)
text = " ".join(text.split())
return text
from datetime import datetime, timezone, timedelta
from signal_db import insert_signal
from classification.signal_classifier import score_signal
from classification.translator import translate_to_de
from intake.image_extractor import extract_image_url
import email.utils
def _pubdate_to_iso(pubdate_str):
if not pubdate_str:
return ""
try:
dt = email.utils.parsedate_to_datetime(pubdate_str)
return dt.isoformat()
except Exception:
return ""
RSS_FEEDS = [
{
"url": "https://news.google.com/rss/search?q=schimmel+mallorca&hl=de&gl=DE&ceid=DE:de",
"name": "Google News: Schimmel Mallorca (DE)",
"language": "de",
"base_category": "schimmel_risiko",
},
{
"url": "https://news.google.com/rss/search?q=feuchtigkeit+immobilien+mallorca&hl=de&gl=DE&ceid=DE:de",
"name": "Google News: Feuchte Immobilien Mallorca (DE)",
"language": "de",
"base_category": "feuchte",
},
{
"url": "https://news.google.com/rss/search?q=kondensation+fenster+schimmel&hl=de&gl=DE&ceid=DE:de",
"name": "Google News: Kondensation Fenster (DE)",
"language": "de",
"base_category": "feuchte",
},
{
"url": "https://news.google.com/rss/search?q=moho+mallorca&hl=es&gl=ES&ceid=ES:es",
"name": "Google News: Moho Mallorca (ES)",
"language": "es",
"base_category": "schimmel_risiko",
},
{
"url": "https://news.google.com/rss/search?q=humedad+vivienda+mallorca&hl=es&gl=ES&ceid=ES:es",
"name": "Google News: Humedad Vivienda Mallorca (ES)",
"language": "es",
"base_category": "feuchte",
},
{
"url": "https://news.google.com/rss/search?q=black+mold+holiday+home&hl=en&gl=GB&ceid=GB:en",
"name": "Google News: Black Mold Holiday Home (EN)",
"language": "en",
"base_category": "schimmel_risiko",
},
{
"url": "https://news.google.com/rss/search?q=lluvia+mallorca+baleares&hl=es&gl=ES&ceid=ES:es",
"name": "Google News: Lluvia Mallorca (ES)",
"language": "es",
"base_category": "wetter_klima",
},
{
"url": "https://news.google.com/rss/search?q=alquiler+vacacional+problemas+mallorca&hl=es&gl=ES&ceid=ES:es",
"name": "Google News: Alquiler Vacacional Mallorca (ES)",
"language": "es",
"base_category": "ferienimmobilie",
},
{
"url": "https://www.mallorcazeitung.es/rss",
"name": "Mallorca Zeitung RSS",
"language": "de",
"base_category": "mallorca_news",
},
# --- Direkte Verlagsquellen (Bilder via enclosure) ---
{
"url": "https://www.diariodemallorca.es/rss",
"name": "Diario de Mallorca RSS",
"language": "es",
"base_category": "mallorca_news",
},
# --- Konkurrenz / Marktbeobachtung ---
{
"url": "https://news.google.com/rss/search?q=murprotec+mallorca+baleares&hl=es&gl=ES&ceid=ES:es",
"name": "Konkurrenz: Murprotec Baleares",
"language": "es",
"base_category": "konkurrenz",
"competition": True,
},
{
"url": "https://news.google.com/rss/search?q=sanierung+feuchte+schimmel+mallorca&hl=de&gl=DE&ceid=DE:de",
"name": "Konkurrenz: Sanierung Mallorca (DE)",
"language": "de",
"base_category": "konkurrenz",
"competition": True,
},
{
"url": "https://news.google.com/rss/search?q=empresa+humedades+mallorca+baleares&hl=es&gl=ES&ceid=ES:es",
"name": "Konkurrenz: Empresa Humedades Mallorca",
"language": "es",
"base_category": "konkurrenz",
"competition": True,
},
]
DECAY_HOURS = {
"wetter_klima": 24,
"schimmel_risiko": 72,
"feuchte": 72,
"ferienimmobilie": 168,
"mallorca_news": 48,
"konkurrenz": 168,
}
HEADERS = {"User-Agent": "Mozilla/5.0 (compatible; MallorcaRadar/1.0)"}
# Mindest-Score fuer Speicherung
MIN_SCORE = 31
def fetch_feed(url: str) -> list:
try:
req = urllib.request.Request(url, headers=HEADERS)
with urllib.request.urlopen(req, timeout=15) as resp:
raw = resp.read()
root = ET.fromstring(raw)
items = []
for item in root.findall(".//item"):
desc = _clean_html(item.findtext("description", "").strip())
items.append({
"title": item.findtext("title", "").strip(),
"link": item.findtext("link", "").strip(),
"description": desc,
"pubDate": item.findtext("pubDate", "").strip(),
"image_url": extract_image_url(item, desc),
})
return items
except Exception as e:
print(f"[RSS] Fehler beim Laden von {url}: {e}")
return []
def process_feed(feed_config: dict) -> int:
items = fetch_feed(feed_config["url"])
saved = 0
now = datetime.now(timezone.utc)
decay_h = DECAY_HOURS.get(feed_config["base_category"], 48)
expires = (now + timedelta(hours=decay_h)).isoformat()
for item in items:
title = item["title"]
link = item["link"]
desc = item.get("description", "")
pub = item.get("pubDate", "")
text = f"{title} {desc}"
# Vollstaendige Score-Daten holen (inkl. quality + breakdown)
sd = score_signal(
text,
language=feed_config["language"],
base_category=feed_config["base_category"],
source_url=link,
timestamp=pub,
)
if sd["radar_score"] < MIN_SCORE:
continue
signal = {
"source_type": "rss",
"source_name": feed_config["name"],
"source_url": link,
"language": feed_config["language"],
"signal_category": sd["signal_category"],
"topic": title[:200],
"short_summary": (desc[:500] if desc else title[:200]),
"extracted_hook": sd["extracted_hook"],
"emotional_direction": sd["emotional_direction"],
"urgency_level": sd["urgency_level"],
"mallorca_relevance": sd["mallorca_relevance"],
"seasonal_relevance": 0,
"risk_relevance": sd["risk_relevance"],
"estimated_noise_level": sd["estimated_noise_level"],
"duplicate_cluster": "",
"suggested_case_types": sd["suggested_case_types"],
"suggested_platforms": sd["suggested_platforms"],
"suggested_cta": sd["suggested_cta"],
"confidence_score": sd["confidence_score"],
"radar_score": sd["radar_score"],
"decay_rate": 1.5,
"expires_at": expires,
"timestamp": _pubdate_to_iso(pub), "published_at": _pubdate_to_iso(pub), "expires_at": expires,
# Qualitaetsfaktoren -- direkt gespeichert
"virality_level": sd["virality_level"],
"emotionality_level": sd["emotionality_level"],
"comment_potential": sd["comment_potential"],
"content_potential": sd["content_potential"],
# Vollstaendiger Breakdown
"score_breakdown": sd["score_breakdown"],
"topic_de": translate_to_de(title, feed_config["language"]),
"image_url": item.get("image_url", ""),
"image_source": ("rss_media" if item.get("image_url") else ""),
"image_found": 1 if item.get("image_url") else 0,
# Phase 4: Platform Fit + Themenklasse + Konkurrenz
"platform_linkedin": sd.get("platform_linkedin", 1),
"platform_facebook": sd.get("platform_facebook", 1),
"platform_instagram": sd.get("platform_instagram", 1),
"platform_tiktok": sd.get("platform_tiktok", 1),
"topic_class": sd.get("topic_class", "sekundaer"),
"competition_signal": 1 if feed_config.get("competition") else sd.get("competition_signal", 0),
}
result = insert_signal(signal)
if result is None:
print(f"[RSS] DUPLIKAT: '{title[:55]}'")
else:
print(f"[RSS] {'+canonical' if True else 'NEU'}: '{title[:55]}' score={sd['radar_score']:.0f} V{sd['virality_level']}|E{sd['emotionality_level']}|K{sd['comment_potential']}|C{sd['content_potential']}")
saved += 1
return saved
def run():
print(f"[RSS] Starte {len(RSS_FEEDS)} Feeds...")
total = 0
for feed in RSS_FEEDS:
print(f"[RSS] >> {feed['name']}")
n = process_feed(feed)
total += n
print(f"[RSS] Gesamt verarbeitet: {total} Signale")
return total
if __name__ == "__main__":
run()