Explorer
/tmp/patch_aa022_rss.py
← Zurück ↓ Download
from pathlib import Path
p=Path('/opt/struktur/social-media-radar/intake/rss_intake.py')
s=p.read_text()
s=s.replace('import urllib.request\n', 'import urllib.request\nimport time\n', 1)
s=s.replace('from signal_db import insert_signal\n', 'from signal_db import insert_signal, record_source_run\n', 1)
anchor='''    {
        "url": "https://www.mallorcazeitung.es/rss",
'''
insert='''    {
        "url": "https://www.solerpalau.com/blog/es-es/feed/",
        "name": "Soler & Palau Blog RSS (ES) — ventilación/CO2/humedad",
        "language": "es",
        "base_category": "feuchte",
        "professional": True,
    },
    {
        "url": "https://www.thebesa.com/besa-blogs/rss.xml",
        "name": "BESA Building Engineering RSS (EN) — IAQ/ventilation",
        "language": "en",
        "base_category": "feuchte",
        "professional": True,
    },
    {
        "url": "https://www.idae.es/rss.xml",
        "name": "IDAE RSS (ES) — climatización/edificios",
        "language": "es",
        "base_category": "wetter_klima",
        "professional": True,
    },
'''
assert anchor in s
s=s.replace(anchor,insert+anchor,1)
s=s.replace('def process_feed(feed_config: dict) -> int:\n    items   = fetch_feed(feed_config["url"])\n    saved   = 0\n', 'def process_feed(feed_config: dict) -> dict:\n    items   = fetch_feed(feed_config["url"])\n    saved = duplicates = rejected = 0\n    parsed = len(items)\n', 1)
s=s.replace('        if sd["radar_score"] < MIN_SCORE:\n            continue\n', '        if sd["radar_score"] < MIN_SCORE:\n            rejected += 1\n            continue\n', 1)
s=s.replace('        if result is None:\n            print(f"[RSS] DUPLIKAT: \'{title[:55]}\'")\n', '        if result is None:\n            duplicates += 1\n            print(f"[RSS] DUPLIKAT: \'{title[:55]}\'")\n', 1)
s=s.replace('    return saved\n\n\ndef run():\n', '    return {"seen": len(items), "parsed": parsed, "new": saved, "duplicates": duplicates, "rejected": rejected}\n\n\ndef run():\n', 1)
s=s.replace('''    total = 0
    for feed in RSS_FEEDS:
        print(f"[RSS] >> {feed['name']}")
        n = process_feed(feed)
        total += n
''', '''    total = 0
    for feed in RSS_FEEDS:
        print(f"[RSS] >> {feed['name']}")
        t0 = time.monotonic()
        stats = process_feed(feed)
        total += stats["new"]
        record_source_run(feed["name"], t0, stats["seen"], stats["new"],
                          stats["duplicates"], rejected=stats["rejected"],
                          quota_usage=f"parsed={stats['parsed']};canonicalized={stats['parsed']}")
''', 1)
p.write_text(s)