"""
AA-045-F6: Enhanced Watch-Entity Intake for Facebook and Property Management
Improved Facebook search with multiple backends and entity-specific queries
Enhanced Property Management with sitemap discovery, CMS endpoints, and service detection
"""
import sys
import re
import time
import json
import base64
import urllib.parse
import urllib.request
import xml.etree.ElementTree as ET
from datetime import datetime, timezone, timedelta
from pathlib import Path
sys.path.insert(0, str(Path(__file__).parent.parent))
try:
import yaml
except ImportError:
yaml = None
from signal_db import insert_signal, record_source_run
from classification.signal_classifier import score_signal
CONFIG_PATH = Path(__file__).parent.parent / "watch_entities.yaml"
HEADERS = {
"User-Agent": ("Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
"AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124 Safari/537.36"),
"Accept-Language": "de-DE,de;q=0.9,es;q=0.8,en;q=0.7",
}
FB_SKIP = re.compile(r"/login|register|m\.me|l\.facebook|sharer|plugins|/facebook/?$")
GNEWS_SKIP = re.compile(r"login|register|/facebook/?$")
# Service-related keywords for Property Management
SERVICE_KEYWORDS = [
'property check', 'key holding', 'home watch', 'absence management',
'preventive maintenance', 'humidity check', 'mould prevention', 'ventilation',
'air conditioning maintenance', 'water leak', 'storm check', 'pool', 'garden',
'renovation', 'emergency service', 'owner service', 'arrival preparation',
'departure check', 'winter check', 'summer check', 'hauskontrolle',
'wartung', 'instandhaltung', 'service', 'pflegeservice', 'hauswart',
'schlüssel hinterlegung', 'notdienst', 'feuchtigkeitsmessung',
'schimmel untersuchung', 'luftqualität', 'klimanalyse'
]
# CMS endpoints to check
CMS_ENDPOINTS = [
'/wp-json/wp/v2/posts',
'/wp-json/wp/v2/pages',
'/feed/',
'/rss/',
'/blog/feed/',
'/blog/rss/',
'/news/feed/',
'/articles/feed/',
'/press/feed/',
]
# Search backends for Facebook
FACEBOOK_BACKENDS = [
{
'name': 'bing_rss',
'url_template': 'https://www.bing.com/search?q={query}&format=rss&count=25',
'enabled': True,
'timeout': 15,
}
]
def load_entities():
with open(CONFIG_PATH, encoding="utf-8") as fh:
data = yaml.safe_load(fh)
ents = [e for e in data.get("watch_entities", [])
if e.get("active") and e.get("verification", {}).get("status") == "verified"]
# Rotation: high priorisiert, stabile Reihenfolge über Laufnummer
settings = data.get("watch_settings", {})
per_run = int(settings.get("entities_per_run", 8))
run_no = int(datetime.now(timezone.utc).strftime("%Y%m%d%H")) % max(len(ents), 1)
ents.sort(key=lambda e: (0 if e.get("priority") == "high" else 1, e["entity_id"]))
rotated = ents[run_no:] + ents[:run_no]
return rotated[:per_run], len(ents)
def _get(url: str, timeout: int = 15) -> bytes | None:
try:
req = urllib.request.Request(url, headers=HEADERS)
with urllib.request.urlopen(req, timeout=timeout) as resp:
return resp.read()
except Exception as e:
print(f"[WATCH] HTTP-Fehler {url[:70]}: {e}")
return None
def _og_image(html_bytes: bytes) -> tuple[str, str]:
if not html_bytes:
return "", ""
h = html_bytes.decode("utf-8", errors="ignore")
for pat in (
r'<meta[^>]+property=["\']og:image["\'][^>]+content=["\']([^"\'>\s]+)',
r'<meta[^>]+content=["\']([^"\'>\s]+)["\'][^>]+property=["\']og:image["\']',
r'<meta[^>]+name=["\']twitter:image["\'][^>]+content=["\']([^"\'>\s]+)',
r'<meta[^>]+content=["\']([^"\'>\s]+)["\'][^>]+name=["\']twitter:image["\']',
):
m = re.search(pat, h, re.IGNORECASE)
if m:
u = m.group(1).strip()
if u.startswith("http"):
src = "twitter:image" if "twitter" in pat.lower() else "og:image"
return u, src
return "", ""
def _is_scheinsignal(title: str, url: str) -> str | None:
"""AA-045-F5 §29: erkennt nicht-postwürdige Treffer. Liefert Grund oder None."""
t = (title or "").strip()
tl = t.lower()
if not t or t.startswith("(ohne Titel)"):
return "ohne Titel"
if "instagram.com" in url:
return "Instagram-Seite"
if ".webflow." in tl:
return "CSS-Klassentitel"
# Startseiten-/Sprachvarianten ohne konkreten Beitrag
if any(x in url for x in ("?lang=", "/en-gb", "/en/", "/de/", "/es/", "/ru/", "/sv/")) and len(t) < 30:
return "Startseiten-/Sprachvariante"
return None
def is_service_related(title: str, content: str = "") -> bool:
"""Check if content is related to property management services"""
text = f"{title} {content}".lower()
return any(keyword in text for keyword in SERVICE_KEYWORDS)
def extract_links_from_html(html_content: bytes, base_url: str) -> list:
"""Extract relevant links from HTML content"""
if not html_content:
return []
try:
html = html_content.decode('utf-8', errors='ignore')
except:
return []
# Find all href attributes
href_pattern = r'href=["\']([^"\']+)["\']'
raw_links = re.findall(href_pattern, html, re.IGNORECASE)
# Process and filter links
processed_links = []
try:
base_parsed = urllib.parse.urlparse(base_url)
base_domain = f"{base_parsed.scheme}://{base_parsed.netloc}"
except:
return processed_links
for link in raw_links:
# Skip empty, javascript, mailto, tel links
if not link or link.startswith(('javascript:', 'mailto:', 'tel:', '#')):
continue
# Convert relative URLs to absolute
if link.startswith('//'):
link = f"{base_parsed.scheme}:{link}"
elif link.startswith('/'):
link = f"{base_domain}{link}"
elif not link.startswith(('http://', 'https://')):
# Relative path
link = urllib.parse.urljoin(base_url, link)
# Only include links from the same domain
try:
link_parsed = urllib.parse.urlparse(link)
if link_parsed.netloc == base_parsed.netloc:
processed_links.append(link)
except:
continue
return list(set(processed_links)) # Remove duplicates
def check_cms_endpoints(base_url: str) -> list:
"""Check CMS endpoints for recent posts"""
results = []
for endpoint in CMS_ENDPOINTS[:6]: # Limit to avoid too many requests
url = urllib.parse.urljoin(base_url, endpoint)
content = _get(url, timeout=10)
if content:
content_type = "unknown" # We don't have headers here, but we can try to detect
# Try to parse as JSON
if endpoint.endswith('.json') or 'wp-json' in endpoint:
try:
data = json.loads(content.decode('utf-8'))
if isinstance(data, list):
for item in data[:5]: # Limit to 5 items
if isinstance(item, dict):
title = item.get('title', {}).get('rendered', '') if isinstance(item.get('title'), dict) else str(item.get('title', ''))
link = item.get('link', '')
date = item.get('date', '')
excerpt = item.get('excerpt', {}).get('rendered', '') if isinstance(item.get('excerpt'), dict) else str(item.get('excerpt', ''))
if title and link:
results.append({
'title': title,
'url': link,
'date': date,
'summary': excerpt,
'source': endpoint
})
except:
pass
# Try to parse as XML/RSS
else:
try:
root = ET.fromstring(content)
# Handle different feed formats
items = []
# RSS 2.0
items.extend(root.findall('.//item'))
# Atom
items.extend(root.findall('.//{http://www.w3.org/2005/Atom}entry'))
for item in items[:5]: # Limit to 5 items
title_elem = item.find('title') or item.find('./{http://www.w3.org/2005/Atom}title')
link_elem = item.find('link') or item.find('./{http://www.w3.org/2005/Atom}link')
date_elem = item.find('pubDate') or item.find('./{http://www.w3.org/2005/Atom}published') or item.find('./{http://purl.org/rss/1.0/modules/dc/}date')
desc_elem = item.find('description') or item.find('./{http://www.w3.org/2005/Atom}summary')
title = title_elem.text.strip() if title_elem is not None and title_elem.text else ''
link = ''
if link_elem is not None:
link = link_elem.get('href') or link_elem.text or ''
date = date_elem.text.strip() if date_elem is not None and date_elem.text else ''
summary = desc_elem.text.strip() if desc_elem is not None and desc_elem.text else ''
if title and link:
results.append({
'title': title,
'url': link,
'date': date,
'summary': summary,
'source': endpoint
})
except:
pass
return results
def process_sitemap(sitemap_url: str) -> list:
"""Process sitemap for relevant URLs"""
content = _get(sitemap_url, timeout=15)
if not content:
return []
results = []
try:
root = ET.fromstring(content)
# Handle sitemap index
if root.tag.endswith('sitemapindex'):
sitemap_elements = root.findall('.//{*}sitemap')
for sitemap_elem in sitemap_elements[:3]: # Limit to 3 sub-sitemaps
loc_elem = sitemap_elem.find('{*}loc')
if loc_elem is not None and loc_elem.text:
sub_sitemap_url = loc_elem.text.strip()
sub_results = process_sitemap(sub_sitemap_url)
results.extend(sub_results)
# Handle URL set
elif root.tag.endswith('urlset'):
url_elements = root.findall('.//{*}url')
for url_elem in url_elements[:20]: # Limit to 20 URLs per sitemap
loc_elem = url_elem.find('{*}loc')
lastmod_elem = url_elem.find('{*}lastmod')
if loc_elem is not None and loc_elem.text:
url = loc_elem.text.strip()
lastmod = lastmod_elem.text.strip() if lastmod_elem is not None and lastmod_elem.text else ''
# Check if URL looks relevant to property management services
if any(keyword in url.lower() for keyword in [
'blog', 'news', 'article', 'service', 'wartung', 'pflege',
'check', 'inspection', 'wartung', 'service', 'notfall',
'feuchte', 'schimmel', 'luft', 'klima', 'pool', 'garten',
'wartung', 'instandhaltung', 'hauswart', 'schluessel',
'feuchtigkeit', 'schimmel', 'luftqualitaet'
]):
results.append({
'title': url.split('/')[-1].replace('-', ' ').replace('_', ' ').title(),
'url': url,
'date': lastmod,
'summary': '',
'source': 'sitemap'
})
except:
pass
return results
def facebook_multi_backend_search(entity: dict) -> list:
"""Search Facebook using multiple backends and entity-specific queries"""
name = entity['canonical_name']
handle = entity.get('facebook_handle', '')
# Build query variations
queries = []
# Entity-specific queries
if handle:
queries.extend([
f'site:facebook.com/{handle}',
f'site:facebook.com/{handle}/posts',
f'"{name}" site:facebook.com',
f'"{name}" site:facebook.com Mallorca',
f'"{name}" site:facebook.com humedad',
f'"{name}" site:facebook.com moho',
f'"{name}" site:facebook.com propiedad',
f'"{name}" site:facebook.com propiedad gestion',
f'"{name}" site:facebook.com mantenimiento',
])
else:
# Fallback to name-based queries
queries.extend([
f'"{name}" site:facebook.com',
f'"{name}" site:facebook.com Mallorca',
f'"{name}" site:facebook.com humedad',
f'"{name}" site:facebook.com moho',
f'"{name}" site:facebook.com propiedad',
])
# Add general Mallorca property queries
queries.extend([
'site:facebook.com "Mallorca" humedad',
'site:facebook.com "Mallorca" moho',
'site:facebook.com "Mallorca" propiedad gestion',
'site:facebook.com "Mallorca" mantenimiento',
'site:facebook.com "Mallorca" casa vigilada',
'site:facebook.com "Mallorca" ausencia prolongada',
])
# Limit queries per run to avoid rate limiting
max_queries = 5
queries = queries[:max_queries]
all_results = []
backends = [b for b in FACEBOOK_BACKENDS if b['enabled']]
for query in queries:
for backend in backends:
url = backend['url_template'].format(query=urllib.parse.quote(query))
raw = _get(url, timeout=backend['timeout'])
if raw:
try:
root = ET.fromstring(raw)
results = []
for item in root.findall('.//item'):
link = (item.findtext('link', '') or '').strip()
title = (item.findtext('title', '') or '').strip()
desc = (item.findtext('description', '') or '').strip()
pub_date = (item.findtext('pubDate', '') or '').strip()
if not link or 'facebook.com' not in link:
continue
if FB_SKIP.search(link) or link.rstrip('/') == 'https://www.facebook.com':
continue
results.append({
'title': title,
'url': link,
'description': desc,
'published': pub_date,
'query': query,
'backend': backend['name']
})
# Filter and deduplicate results
seen_urls = set()
filtered_results = []
for result in results:
url_norm = result['url'].strip().lower().rstrip('/')
if url_norm not in seen_urls and 'facebook.com' in url_norm:
seen_urls.add(url_norm)
filtered_results.append(result)
all_results.extend(filtered_results)
print(f"[WATCH-IMPROVED FB] {entity['canonical_name']} | {backend['name']} | '{query[:50]}...' -> {len(filtered_results)} results")
# If we got good results from this backend, we can stop trying others for this query
if len(filtered_results) >= 3:
break
except Exception as e:
print(f"[WATCH-IMPROVED FB] {entity['canonical_name']} | {backend['name']} | Parse error '{query[:50]}...': {e}")
else:
print(f"[WATCH-IMPROVED FB] {entity['canonical_name']} | {backend['name']} | No response '{query[:50]}...'")
return all_results
def property_management_discovery(entity: dict) -> list:
"""Discover Property Management content via sitemaps, CMS endpoints, and service detection"""
website = entity.get('website', '')
if not website:
return []
results = []
# 1. Check sitemap.xml and sitemap_index.xml
for sitemap_path in ['/sitemap.xml', '/sitemap_index.xml']:
sitemap_url = urllib.parse.urljoin(website, sitemap_path)
sitemap_results = process_sitemap(sitemap_url)
results.extend(sitemap_results)
if sitemap_results:
print(f"[WATCH-IMPROVED PM] {entity['canonical_name']} | Sitemap {sitemap_path} -> {len(sitemap_results)} results")
# 2. Check CMS endpoints
cms_results = check_cms_endpoints(website)
results.extend(cms_results)
if cms_results:
print(f"[WATCH-IMPROVED PM] {entity['canonical_name']} | CMS endpoints -> {len(cms_results)} results")
# 3. Check blog page directly for recent posts
blog_url = entity.get('blog_url')
if blog_url:
blog_content = _get(blog_url, timeout=12)
if blog_content:
links = extract_links_from_html(blog_content, blog_url)
# Process blog links for recent posts
blog_count = 0
for link in links[:10]: # Limit to 10 blog links
if any(keyword in link.lower() for keyword in ['post', 'article', 'blog', 'news']):
link_content = _get(link, timeout=10)
if link_content:
title_elem = re.search(r'<title>([^<]{5,150})</title>', link_content.decode('utf-8', errors='ignore'), re.IGNORECASE)
title = title_elem.group(1).strip() if title_elem else link.split('/')[-1].replace('-', ' ').title()
# Extract meta description or first paragraph as summary
desc_match = re.search(r'<meta[^>]+name=["\']description["\'][^>]+content=["\']([^"\']*)["\'>]', link_content.decode('utf-8', errors='ignore'), re.IGNORECASE)
summary = desc_match.group(1) if desc_match else ''
if not summary:
# Try to get first paragraph
p_match = re.search(r'<p[^>]*>([^<]{20,200})</p>', link_content.decode('utf-8', errors='ignore'), re.IGNORECASE)
summary = p_match.group(1) if p_match else ''
results.append({
'title': title,
'url': link,
'date': '', # Would need more complex parsing
'summary': summary,
'source': 'blog_page'
})
blog_count += 1
if blog_count > 0:
print(f"[WATCH-IMPROVED PM] {entity['canonical_name']} | Blog page -> {blog_count} results")
return results
def _store_improved(entity: dict, platform: str, url: str, title: str, summary: str,
image_url: str = "", image_source: str = "", retrieval_mode: str = "direct",
watch_entity_data: dict = None) -> str | None:
"""Store an improved watch entity signal with enhanced metadata"""
skip_reason = _is_scheinsignal(title, url)
if skip_reason:
return None # kein Signal, kein Fehler (§29)
# Determine if content is service-related for Property Management boost
service_boost = 0
if entity.get('sector') in ['property_management', 'villa_management', 'makler_property_management']:
if is_service_related(title, summary):
service_boost = 15
expires = (datetime.now(timezone.utc) + timedelta(hours=336)).isoformat()
# Prepare base signal data for scoring
text_for_scoring = f"{title} {summary}"
sd = score_signal(text_for_scoring, language="de", base_category="feuchte",
source_url=url)
# Apply service boost for Property Management
final_score = min(sd['radar_score'] + service_boost, 100)
if final_score < 31:
return None # Below threshold
signal = {
"source_type": "watch_entity",
"source_platform": platform,
"external_id": "",
"observed_at": datetime.now(timezone.utc).isoformat(),
"source_name": f"WATCH-IMPROVED: {entity['canonical_name']} ({entity['sector']})",
"source_url": url,
"language": (entity.get("languages") or ["de"])[0],
"signal_category": sd["signal_category"] or f"{platform}_watch",
"topic": (title or "(ohne Titel)")[:200],
"short_summary": (summary or title or "")[:500],
"extracted_hook": sd["extracted_hook"],
"emotional_direction": sd["emotional_direction"],
"urgency_level": sd["urgency_level"],
"mallorca_relevance": sd["mallorca_relevance"],
"risk_relevance": sd["risk_relevance"],
"estimated_noise_level": sd["estimated_noise_level"],
"suggested_case_types": sd["suggested_case_types"],
"suggested_platforms": sd["suggested_platforms"],
"suggested_cta": sd["suggested_cta"],
"confidence_score": sd["confidence_score"],
"radar_score": final_score,
"decay_rate": 0.8 if platform == "web" else 1.0,
"expires_at": expires,
"virality_level": sd["virality_level"],
"emotionality_level": sd["emotionality_level"],
"comment_potential": sd["comment_potential"],
"content_potential": sd["content_potential"],
"score_breakdown": sd["score_breakdown"],
"platform_facebook": 1,
"topic_class": "sozial",
"retrieval_mode": retrieval_mode,
"watch_entity_id": entity["entity_id"],
"watch_sector": entity["sector"],
"watch_priority": entity.get("priority", "normal"),
"image_url": image_url,
"image_source": image_source,
"image_found": 1 if image_url else 0,
}
# Add watch entity specific data if provided
if watch_entity_data:
signal.update(watch_entity_data)
return insert_signal(signal)
def poll_facebook_improved(entity: dict) -> int:
"""Improved Facebook polling with multiple backends and entity-specific queries"""
results = facebook_multi_backend_search(entity)
if not results:
return 0
saved = 0
for result in results:
# Get image if available (try to extract from the Facebook page)
image_url, image_source = "", ""
# Note: We don't scrape Facebook pages for images due to restrictions
sid = _store_improved(
entity,
"facebook",
result["url"],
result["title"],
result["description"],
image_url,
image_source,
retrieval_mode="search_index",
watch_entity_data={
"external_id": "", # Facebook doesn't provide easy external ID without API
# Search-index pubDate is not proof of the Facebook post publication time.
# Leave it NULL unless a reliable public post timestamp is available.
}
)
if sid:
saved += 1
print(f"[WATCH-IMPROVED FB] NEU: {entity['canonical_name']} | {result['title'][:60]}")
return saved
def poll_property_management_improved(entity: dict) -> int:
"""Improved Property Management polling with sitemap discovery and service detection"""
results = property_management_discovery(entity)
if not results:
return 0
saved = 0
for result in results:
# Try to get image from the page
image_url, image_source = "", ""
page_content = _get(result["url"], timeout=10)
if page_content:
image_url, image_source = _og_image(page_content)
sid = _store_improved(
entity,
"web",
result["url"],
result["title"],
result["summary"],
image_url,
image_source,
retrieval_mode=result.get("source", "direct")
)
if sid:
saved += 1
service_tag = " [SERVICE]" if is_service_related(result["title"], result["summary"]) else ""
print(f"[WATCH-IMPROVED PM] NEU: {entity['canonical_name']} | {result['title'][:60]}{service_tag}")
return saved
def poll_youtube(entity: dict) -> int:
"""Existing YouTube polling (unchanged)"""
cid = entity.get("youtube_channel_id")
if not cid:
return 0
saved = 0
raw = _get(f"https://www.youtube.com/feeds/videos.xml?channel_id={cid}")
if not raw:
return 0
try:
root = ET.fromstring(raw)
except ET.ParseError:
return 0
ns = {"a": "http://www.w3.org/2005/Atom",
"yt": "http://www.youtube.com/xml/schemas/2015"}
for e in root.findall("a:entry", ns)[:5]:
vid = e.findtext("yt:videoId", "", ns)
if not vid:
continue
title = (e.findtext("a:title", "", ns) or "").strip()
thumb = f"https://i.ytimg.com/vi/{vid}/hqdefault.jpg"
sid = _store_improved(entity, "youtube", f"https://www.youtube.com/watch?v={vid}",
title, title, thumb, retrieval_mode="youtube_rss")
if sid:
saved += 1
return saved
def poll_website(entity: dict) -> int:
"""Existing website polling (enhanced with service detection)"""
saved = 0
blog = entity.get("blog_url")
if not blog:
return 0
# RSS-Versuch an üblichen Pfaden
rss_candidates = [
entity.get("rss_url"),
blog.rstrip("/") + "/feed/",
blog.rstrip("/") + "/rss",
blog.rstrip("/") + "?format=rss",
]
items = []
feed_url_used = None
for cand in filter(None, rss_candidates):
raw = _get(cand, timeout=12)
if raw:
try:
root = ET.fromstring(raw)
items = root.findall(".//item")[:5]
feed_url_used = cand
break
except ET.ParseError:
continue
if items:
for it in items:
title = (it.findtext("title", "") or "").strip()
link = (it.findtext("link", "") or "").strip()
desc = re.sub(r"<[^>]+>", " ", it.findtext("description", "") or "")
desc = re.sub(r"\s+", " ", desc).strip()
img = ""
enc = it.find("enclosure")
if enc is not None and (enc.get("type") or "").startswith("image"):
img = enc.get("url", "")
if not img:
img, _ = _og_image(_get(link, timeout=10))
if link.startswith("http"):
sid = _store_improved(entity, "web", link, title, desc[:400], img, img and "og:image" or "")
if sid:
saved += 1
else:
# Blog-Seite ohne RSS: Links + og:image der Seite
raw = _get(blog, timeout=12)
if raw:
h = raw.decode("utf-8", errors="ignore")
links = re.findall(r'href="(https?://[^"]+' + re.escape(
urllib.parse.urlparse(blog).netloc.split(".")[-2] if "." in urllib.parse.urlparse(blog).netloc else ""
) + r'[^"]*)"', h)
seen = set()
for u in links[:8]:
if u in seen or u.rstrip("/") == blog.rstrip("/"):
continue
seen.add(u)
page = _get(u, timeout=10)
img, src = _og_image(page)
tmatch = re.search(rb"<title>([^<]{5,150})", page or b"")
title = tmatch.group(1).decode("utf-8", "ignore").strip() if tmatch else u.split("/")[-1]
sid = _store_improved(entity, "web", u, title, title, img, src)
if sid:
saved += 1
time.sleep(0.2)
return saved
def run_all(limit_entities: int | None = None) -> int:
if yaml is None:
print("[WATCH] PyYAML fehlt")
return 0
ents, total_ents = load_entities()
if limit_entities:
ents = ents[:limit_entities]
print(f"[WATCH-IMPROVED] {len(ents)}/{total_ents} Entities dieser Lauf "
f"(Rotation, verified+active)")
t0 = time.monotonic()
stats = {"seen": 0, "new": 0}
for e in ents:
eid = e["entity_id"]
try:
n_web = poll_website(e)
n_fb = poll_facebook_improved(e)
n_yt = poll_youtube(e)
n_pm = poll_property_management_improved(e) # New improved PM polling
new = n_web + n_fb + n_yt + n_pm
stats["new"] += new
stats["seen"] += 1
record_source_run(f"watch:{eid}", t0, 4, new, 0, 0, 0, None)
print(f"[WATCH-IMPROVED] {eid}: web={n_web} fb={n_fb} yt={n_yt} pm={n_pm}")
except Exception as exc:
print(f"[WATCH-IMPROVED] Fehler bei {eid}: {exc}")
record_source_run(f"watch:{eid}", t0, 0, 0, 0, 0, 0, error=str(exc))
print(f"[WATCH-IMPROVED] Gesamt neu: {stats['new']}")
return stats["new"]
if __name__ == "__main__":
import argparse
ap = argparse.ArgumentParser()
ap.add_argument("--limit", type=int, default=None)
args = ap.parse_args()
run_all(args.limit)