from pathlib import Path
p=Path('/opt/struktur/social-media-radar/intake/rss_intake.py')
s=p.read_text()
s=s.replace('import urllib.request\n', 'import urllib.request\nimport time\n', 1)
s=s.replace('from signal_db import insert_signal\n', 'from signal_db import insert_signal, record_source_run\n', 1)
anchor=''' {
"url": "https://www.mallorcazeitung.es/rss",
'''
insert=''' {
"url": "https://www.solerpalau.com/blog/es-es/feed/",
"name": "Soler & Palau Blog RSS (ES) — ventilación/CO2/humedad",
"language": "es",
"base_category": "feuchte",
"professional": True,
},
{
"url": "https://www.thebesa.com/besa-blogs/rss.xml",
"name": "BESA Building Engineering RSS (EN) — IAQ/ventilation",
"language": "en",
"base_category": "feuchte",
"professional": True,
},
{
"url": "https://www.idae.es/rss.xml",
"name": "IDAE RSS (ES) — climatización/edificios",
"language": "es",
"base_category": "wetter_klima",
"professional": True,
},
'''
assert anchor in s
s=s.replace(anchor,insert+anchor,1)
s=s.replace('def process_feed(feed_config: dict) -> int:\n items = fetch_feed(feed_config["url"])\n saved = 0\n', 'def process_feed(feed_config: dict) -> dict:\n items = fetch_feed(feed_config["url"])\n saved = duplicates = rejected = 0\n parsed = len(items)\n', 1)
s=s.replace(' if sd["radar_score"] < MIN_SCORE:\n continue\n', ' if sd["radar_score"] < MIN_SCORE:\n rejected += 1\n continue\n', 1)
s=s.replace(' if result is None:\n print(f"[RSS] DUPLIKAT: \'{title[:55]}\'")\n', ' if result is None:\n duplicates += 1\n print(f"[RSS] DUPLIKAT: \'{title[:55]}\'")\n', 1)
s=s.replace(' return saved\n\n\ndef run():\n', ' return {"seen": len(items), "parsed": parsed, "new": saved, "duplicates": duplicates, "rejected": rejected}\n\n\ndef run():\n', 1)
s=s.replace(''' total = 0
for feed in RSS_FEEDS:
print(f"[RSS] >> {feed['name']}")
n = process_feed(feed)
total += n
''', ''' total = 0
for feed in RSS_FEEDS:
print(f"[RSS] >> {feed['name']}")
t0 = time.monotonic()
stats = process_feed(feed)
total += stats["new"]
record_source_run(feed["name"], t0, stats["seen"], stats["new"],
stats["duplicates"], rejected=stats["rejected"],
quota_usage=f"parsed={stats['parsed']};canonicalized={stats['parsed']}")
''', 1)
p.write_text(s)