#!/usr/bin/env python3
"""
Improved Facebook intake for SMA Radar
Multiple search backends and entity-specific queries
"""
import sys
import re
import time
import json
import base64
import urllib.parse
import urllib.request
import xml.etree.ElementTree as ET
from datetime import datetime, timezone, timedelta
from pathlib import Path
sys.path.insert(0, str(Path(__file__).parent.parent))
try:
import yaml
except ImportError:
yaml = None
from signal_db import insert_signal, record_source_run
from classification.signal_classifier import score_signal
# Multiple search backends configuration
SEARCH_BACKENDS = [
{
'name': 'bing_rss',
'url_template': 'https://www.bing.com/search?q={query}&format=rss&count=25',
'enabled': True,
'timeout': 15,
'parse_func': 'parse_bing_rss'
},
{
'name': 'duckduckgo_html',
'url_template': 'https://duckduckgo.com/html/?q={query}',
'enabled': False, # Blocked from VPS per testing
'timeout': 15,
'parse_func': 'parse_duckduckgo_html'
},
{
'name': 'startpage_html',
'url_template': 'https://www.startpage.com/sp/search?q={query}',
'enabled': False, # Blocked from VPS per testing
'timeout': 15,
'parse_func': 'parse_startpage_html'
}
]
_FB_SKIP = re.compile(r'/login|register|m\.me|l\.facebook|/facebook/?$|sharer|plugins')
def parse_bing_rss(raw_bytes):
"""Parse Bing RSS results"""
try:
root = ET.fromstring(raw_bytes)
results = []
for item in root.findall('.//item'):
link = (item.findtext('link', '') or '').strip()
title = (item.findtext('title', '') or '').strip()
desc = (item.findtext('description', '') or '').strip()
pub_date = (item.findtext('pubDate', '') or '').strip()
if not link or 'facebook.com' not in link:
continue
if _FB_SKIP.search(link) or link.rstrip('/') == 'https://www.facebook.com':
continue
results.append({
'title': title,
'url': link,
'description': desc,
'published': pub_date,
'source': 'bing_rss'
})
return results
except Exception as e:
print(f"[FB-IMPROVED] Bing RSS parse error: {e}")
return []
def parse_duckduckgo_html(raw_bytes):
"""Parse DuckDuckGo HTML results (placeholder)"""
# Would implement HTML parsing here if not blocked
return []
def parse_startpage_html(raw_bytes):
"""Parse Startpage HTML results (placeholder)"""
# Would implement HTML parsing here if not blocked
return []
def get_search_backends():
"""Get list of enabled search backends"""
return [backend for backend in SEARCH_BACKENDS if backend['enabled']]
def fb_multi_backend_search(entity, backend):
"""Search Facebook using specified backend"""
name = entity['canonical_name']
handle = entity.get('facebook_handle', '')
# Build query variations
queries = []
# Entity-specific queries
if handle:
queries.extend([
f'site:facebook.com/{handle}',
f'site:facebook.com/{handle}/posts',
f'"{name}" site:facebook.com',
f'"{name}" site:facebook.com Mallorca',
f'"{name}" site:facebook.com humedad',
f'"{name}" site:facebook.com moho',
f'"{name}" site:facebook.com propiedad',
f'"{name}" site:facebook.com propiedad gestion',
f'"{name}" site:facebook.com mantenimiento',
])
else:
# Fallback to name-based queries
queries.extend([
f'"{name}" site:facebook.com',
f'"{name}" site:facebook.com Mallorca',
f'"{name}" site:facebook.com humedad',
f'"{name}" site:facebook.com moho',
f'"{name}" site:facebook.com propiedad',
])
# Add general Mallorca property queries
queries.extend([
'site:facebook.com "Mallorca" humedad',
'site:facebook.com "Mallorca" moho',
'site:facebook.com "Mallorca" propiedad gestion',
'site:facebook.com "Mallorca" mantenimiento',
'site:facebook.com "Mallorca" casa vigilada',
'site:facebook.com "Mallorca" ausencia prolongada',
])
# Limit queries per run to avoid rate limiting
max_queries = 5
queries = queries[:max_queries]
all_results = []
for query in queries:
url = backend['url_template'].format(query=urllib.parse.quote(query))
req = urllib.request.Request(url, headers={
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124 Safari/537.36',
'Accept-Language': 'de-DE,de;q=0.9',
})
try:
with urllib.request.urlopen(req, timeout=backend['timeout']) as resp:
raw = resp.read()
if backend['parse_func'] == 'parse_bing_rss':
results = parse_bing_rss(raw)
else:
# Placeholder for other parsers
results = []
# Filter and deduplicate results
seen_urls = set()
for result in results:
url_norm = result['url'].strip().lower().rstrip('/')
if url_norm not in seen_urls and 'facebook.com' in url_norm:
seen_urls.add(url_norm)
result['query'] = query
result['backend'] = backend['name']
all_results.append(result)
print(f"[FB-IMPROVED] {entity['canonical_name']} | {backend['name']} | '{query[:50]}...' -> {len(results)} results")
except Exception as e:
print(f"[FB-IMPROVED] {entity['canonical_name']} | {backend['name']} | ERROR '{query[:50]}...': {e}")
return all_results
def entity_to_signal(entity, fb_result):
"""Convert Facebook result to signal format"""
text = f"{fb_result['title']} {fb_result['description'][:300]}"
sd = score_signal(text, language="de", base_category="feuchte",
source_url=fb_result['url'])
if sd['radar_score'] < 31:
return None
expires = (datetime.now(timezone.utc) + timedelta(hours=240)).isoformat()
signal = {
'source_type': 'facebook_public',
'source_platform': 'facebook',
'external_id': '',
'observed_at': fb_result.get('published', datetime.now(timezone.utc).isoformat()),
'source_name': f"FB-IMPROVED: {entity['canonical_name']} ({fb_result['backend']})",
'source_url': fb_result['url'],
'language': (entity.get('languages') or ['de'])[0],
'signal_category': sd['signal_category'] or 'fb_oeffentlich',
'topic': fb_result['title'][:200] or '(Facebook-Diskussion)',
'short_summary': (fb_result['description'] or fb_result['title'])[:500],
'extracted_hook': sd['extracted_hook'],
'emotional_direction': sd['emotional_direction'],
'urgency_level': sd['urgency_level'],
'mallorca_relevance': sd['mallorca_relevance'],
'risk_relevance': sd['risk_relevance'],
'estimated_noise_level': sd['estimated_noise_level'],
'suggested_case_types': sd['suggested_case_types'],
'suggested_platforms': sd['suggested_platforms'],
'suggested_cta': sd['suggested_cta'],
'confidence_score': sd['confidence_score'],
'radar_score': sd['radar_score'],
'decay_rate': 1.0,
'expires_at': expires,
'virality_level': sd['virality_level'],
'emotionality_level': sd['emotionality_level'],
'comment_potential': sd['comment_potential'],
'content_potential': sd['content_potential'],
'score_breakdown': sd['score_breakdown'],
'platform_facebook': 1,
'topic_class': 'sozial',
'retrieval_mode': 'search_index',
'watch_entity_id': entity['entity_id'],
'watch_sector': entity['sector'],
'watch_priority': entity.get('priority', 'normal'),
}
return signal
def run_improved_facebook(cfg):
"""Run improved Facebook intake"""
fcfg = cfg['sources']['facebook']
if not fcfg.get('enabled'):
return 0
backends = get_search_backends()
if not backends:
print("[FB-IMPROVED] No search backends enabled")
return 0
# Load entities
try:
with open('/opt/struktur/social-media-radar/watch_entities.yaml', encoding='utf-8') as fh:
data = yaml.safe_load(fh)
entities = [e for e in data.get('watch_entities', [])
if e.get('active') and e.get('verification', {}).get('status') == 'verified']
except Exception as e:
print(f"[FB-IMPROVED] Failed to load entities: {e}")
return 0
# Rotation: high priority first
entities.sort(key=lambda e: (0 if e.get('priority') == 'high' else 1, e['entity_id']))
# Process entities in batches to avoid overload
entities_per_run = 5 # Process 5 entities per run
day_key = datetime.now(timezone.utc).strftime('%Y%m%d%H')
offset = int(day_key) % max(len(entities), 1)
rotated = entities[offset:] + entities[:offset]
batch = rotated[:entities_per_run]
print(f"[FB-IMPROVED] Processing {len(batch)} entities this run (rotation)")
total_saved = 0
total_seen = 0
total_new = 0
total_dup = 0
total_rej = 0
t0 = time.time()
for entity in batch:
print(f"[FB-IMPROVED] Processing entity: {entity['canonical_name']} ({entity['entity_id']})")
# Try each backend until we get results
entity_results = []
for backend in backends:
results = fb_multi_backend_search(entity, backend)
entity_results.extend(results)
# If we got good results from this backend, we can stop trying others
if len(results) >= 3: # Got decent results
break
if not entity_results:
print(f"[FB-IMPROVED] {entity['canonical_name']}: No results from any backend")
continue
# Process results into signals
for fb_result in entity_results:
total_seen += 1
signal = entity_to_signal(entity, fb_result)
if signal is None:
total_rej += 1
continue
# Insert signal
signal_id = insert_signal(signal)
if signal_id is None:
total_dup += 1
else:
total_new += 1
total_saved += 1
print(f"[FB-IMPROVED] NEU: {entity['canonical_name']} | {signal['topic'][:60]} | {signal['source_url'][:80]}")
# Record source run
record_source_run(
'facebook_improved',
time.time() - t0,
total_seen,
total_new,
total_dup,
clustered=0,
rejected=total_rej,
error=None
)
print(f"[FB-IMPROVED] Summary: seen={total_seen} new={total_new} dup={total_dup} rej={total_rej} saved={total_saved}")
return total_saved
if __name__ == '__main__':
# Test standalone
cfg = {'sources': {'facebook': {'enabled': True}}}
run_improved_facebook(cfg)