
    j+                        d Z ddlZddlZddlZddlZddlZddlZddlZddl	m
c mZ ddlmZmZmZ ddlmZ ej$                  j'                  d e ee      j,                  j,                               	 ddlZddlmZmZ  ee      j,                  j,                  dz  Zddd	Z ej<                  d
      Z ej<                  d      Z de!e"   fdZ#d!dede$de%dz  fdZ&de%de'eef   fdZ(	 d"de"dededededededz  fdZ)de"de$fdZ*de"de$fdZ+de"de$fdZ,d#de$dz  de$fdZ-e.dk(  rLddl/Z/ e/j`                         Z1e1je                  de$d        e1jg                         Z4 e-e4jj                         yy# e$ r dZY w xY w)$u  
AA-045-F5: Watch-Entity Intake — Branchen-/Wettbewerber-Beobachtung.
Liest watch_entities.yaml, pollt je aktiver verifizierter Entity:
  - Website/Blog/RSS (kontrollierte Abfrage, og:image)
  - YouTube Channel-RSS (falls channel_id)
  - Facebook gezielt loginfrei via Bing-RSS: site:facebook.com "<Name>"
Rotation: entities_per_run pro Lauf; high zuerst. Idempotent über URL-Dedup.
Signale: source_type='watch_entity', source_platform=plattform.
    N)datetimetimezone	timedelta)Path)insert_signalrecord_source_runzwatch_entities.yamlzUMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/124 Safari/537.36zde,es;q=0.9,en;q=0.8)z
User-AgentzAccept-Languagez=/login|register|m\.me|l\.facebook|sharer|plugins|/facebook/?$zlogin|register|/facebook/?$returnc                     t        t        d      5 } t        j                  |       }d d d        j	                  dg       D cg c]9  }|j	                  d      r&|j	                  di       j	                  d      dk(  r|; }}|j	                  di       }t        |j	                  d	d
            }t        t        j                  t        j                        j                  d            t        t        |      d      z  }|j                  d        ||d  |d | z   }|d | t        |      fS # 1 sw Y   
xY wc c}w )Nutf-8)encodingwatch_entitiesactiveverificationstatusverifiedwatch_settingsentities_per_run   z%Y%m%d%H   c                 <    | j                  d      dk(  rdnd| d   fS )Npriorityhighr   r   	entity_id)get)es    /tmp/watch_intake.py<lambda>zload_entities.<locals>.<lambda>1   s!    !%%
"3v"=Q1anU     )key)openCONFIG_PATHyaml	safe_loadr   intr   nowr   utcstrftimemaxlensort)fhdatar   entssettingsper_runrun_norotateds           r   load_entitiesr2   (   s    	kG	, "~~b!"xx 0"5 Z!uuX155#<#@#@#Jj#X  ZD Z xx("-H(,,1156Ghll+44Z@ACD	STDUUFIIUIV67md7Fm+G8Gc$i''" "Zs   D0>D=0D:urltimeoutc                 0   	 t         j                  j                  | t              }t         j                  j	                  ||      5 }|j                         cd d d        S # 1 sw Y   y xY w# t        $ r}t        d| d d  d|        Y d }~y d }~ww xY w)N)headersr4   z[WATCH] HTTP-Fehler F   : )urllibrequestRequestHEADERSurlopenread	Exceptionprint)r3   r4   reqrespr   s        r   _getrD   6   s    nn$$S'$:^^##C#9 	T99;	 	 	 $S"XJb45s6   AA. A"	A. "A+'A. +A. .	B7BB
html_bytesc                 
   | sy| j                  dd      }dD ]h  }t        j                  ||t        j                        }|s+|j	                  d      j                         }|j                  d      s\d|v rd	nd
}||fc S  y)N) rG   r   ignoreerrors)zC<meta[^>]+property=["\']og:image["\'][^>]+content=["\']([^"\'>\s]+)zH<meta[^>]+content=["\']([^"\'>\s]+)["\'][^>]+property=["\']og:image["\']zD<meta[^>]+name=["\']twitter:image["\'][^>]+content=["\']([^"\'>\s]+)r   httptwitterztwitter:imageog:image)decoderesearch
IGNORECASEgroupstrip
startswith)rE   hpatmusrcs         r   	_og_imagerZ   @   s    '(3A 

 IIc1bmm,
  "A||F#)2c)9oz#v
 r   entityplatformtitlesummary	image_urlc                    t        j                  t        j                        t	        d      z   j                         }i ddd|dddt        j                  t        j                        j                         d	d
| d    d| d    dd|d| j                  d      xs dgd   ddd|xs ddd d|xs |xs ddd d|dd ddd dd!| j                  d"      d#k(  rd$ndd%d&d'd&d(di d)d*d+d,d-d.d/d0d1d2d3|d4d&d5d&d6d7d8d7d9t        j                  | d:   | d   | j                  d;d<      d=      d>d&d?d&d@dAdB|dC|dDk(  r|rdDn|rdEnddF|rd&nd}t        |      S )GuF   Speichert einen Watch-Treffer als Signal (idempotent über URL-Dedup).iP  )hourssource_typewatch_entitysource_platformexternal_idrG   observed_atsource_namezWATCH: canonical_namez (sector)
source_urllanguage	languagesder   signal_categorybranche_watchtopicz(ohne Titel)N   short_summaryi  extracted_hook   emotional_directionneutralurgency_levelmallorca_relevance	geographymallorca   risk_relevancer   estimated_noise_levelsuggested_case_typessuggested_platformscockpitsuggested_cta
beobachtenconfidence_scoregffffff?radar_scoreg      D@
decay_rateg?
expires_atvirality_levelemotionality_levelcomment_potential   content_potentialscore_breakdownr   r   normal)rc   ri   r   platform_facebookplatform_linkedintopic_classsozialr_   image_sourceyoutuberM   image_found)
r   r%   r   r&   r   	isoformatr   jsondumpsr   )r[   r\   r3   r]   r^   r_   expiressignals           r   _storer   R   s    ||HLL)IC,@@KKMG&~&8& 	r& 	x||HLL1;;=	&
 	(8!9 :"VH=M<NaP& 	c& 	VZZ,6:& 	?& 	%)>4C0& 	'0U0b$37& 	%+& 	y& 	& 	FJJ{$;z$Ibq& 	!&  	 !&" 	#&$ 	y%&& 	'&( 	C)&* 	t+&, 	c-&. 	g/&0 	!1&2 	a3&4 	Q5&6 	Q7&8 	4::vk7J171A39::j(3S'U V9&> 	Q?&@ 	QA&B 	xC&D 	YE&F 	h)&;	(1*rI&J 	Iq1K&FN   r   c           	         | j                  d      }|syd}t        d|       }|sy	 t        j                  |      }ddd}|j                  d|      d d D ]^  }|j                  d	d
|      }|s|j                  dd
|      xs d
j                         }d| d}	t        | dd| |||	      }
|
sZ|dz  }` |S # t        j                  $ r Y yw xY w)Nyoutube_channel_idr   z4https://www.youtube.com/feeds/videos.xml?channel_id=zhttp://www.w3.org/2005/Atomz'http://www.youtube.com/xml/schemas/2015)aytza:entry   z
yt:videoIdrG   za:titlezhttps://i.ytimg.com/vi/z/hqdefault.jpgr   z https://www.youtube.com/watch?v=r   )	r   rD   ET
fromstring
ParseErrorfindallfindtextrS   r   )r[   cidsavedrawrootnsr   vidr]   thumbsids              r   poll_youtuber      s   
**)
*CE
EcUK
LC}}S! -9
;B\\)R(!, 	jjr2.Ir2.4";;=)#n=VY*J3%(PE5*QJE	 L == s   B; ;CCc                    | d   }t         j                  j                  d| d      }d| d}t        |      }|sy	 t	        j
                  |      }d}|j                  d      D ]  }|j                  dd	      xs d	j                         }|j                  d
d	      xs d	j                         }	|j                  dd	      xs d	j                         }
|rd|vrvt        j                  |      r|	j                         j                  dd	      |j                         j                  dd	      k(  rt        | d||	xs | d|
      }|s|dz  } |S # t        j                  $ r Y yw xY w)Nrh   zsite:facebook.com ""zhttps://www.bing.com/search?q=z&format=rss&count=20r   .//itemlinkrG   r]   descriptionzfacebook.com facebookz auf Facebookr   )r:   parsequoterD   r   r   r   r   r   rS   FB_SKIPrP   lowerreplacer   )r[   nameqr3   r   r   r   itemr   r]   descr   s               r   poll_facebookr      sn   "#D0a89A*1#-A
BC
s)C}}S! EY' fb)/R668w+1r88:mR06B==?~T1>>$ ;;=  b)TZZ\-A-A#r-JJVZu/N4&8NPTUQJE L% == s   E E,+E,c           
      n   d}| j                  d      }|sy| j                  d      |j                  d      dz   |j                  d      dz   |j                  d      dz   g}g }d }t        d |      D ]?  }t        |d	      }|s	 t	        j
                  |      }|j                  d
      d d }|} n |r8|D ]0  }	|	j                  dd      xs dj                         }
|	j                  dd      xs dj                         }t        j                  dd|	j                  dd      xs d      }t        j                  dd|      j                         }d}|	j                  d      }|6|j                  d      xs dj                  d      r|j                  dd      }|st        t        |d	            \  }}|j                  d      st        | d||
|d d |      }|s,|dz  }3 |S t        |d	      }|r|j!                  dd      }t        j                  dt        j"                  d t$        j&                  j)                  |      j*                  v r;t$        j&                  j)                  |      j*                  j-                  d       d!   nd      z   d"z   |      }t/               }|d d# D ]  }||v s#|j                  d      |j                  d      k(  r+|j1                  |       t        |d	      }t        |      \  }}t        j2                  d$|xs d%      }|r/|j5                  d      j!                  dd      j                         n|j7                  dd      d&   }
t        | d||
|
|      }|r|dz  }t9        j:                  d'        |S # t        j                  $ r Y Aw xY w)(Nr   blog_urlrss_url/z/feed/z/rssz?format=rss   r7   r   r   r]   rG   r   z<[^>]+>r   r   z\s+	enclosuretypeimager3   
   rK   webi  r   r   rH   rI   zhref="(https?://[^"]+.z[^"]*)"r   s   <title>([^<]{5,150})r   g?)r   rstripfilterrD   r   r   r   r   r   rS   rO   subfindrT   rZ   r   rN   escaper:   r   urlparsenetlocsplitsetaddrP   rR   rsplittimesleep)r[   r   blogrss_candidatesitemsfeed_url_usedcandr   r   itr]   r   r   imgenc_r   rU   linksseenrX   page_srctmatchs                           r   poll_websiter      s}   E::j!D 	

9C8#C6!C=(	N EMt^, 	4$}}S)Y/3 $	  	B[["-3::<EKK+1r88:D66*c2;;}b+I+ORPD66&#t,224DC''+&CCGGFO$9r#E#Eg#NggeR("4b#9:Qv&VUD%dsSIQJE	H L' 4$

78
4AJJ7"))EHFLLLaLabfLgLnLnEn%%d+2288=bAtv;   E 5D2AY  9S1A AAr*%dO	T#;T[SIMSQ..wAGGIYZYaYabeghYijlYmVUAueSAQJE

3  LO == s   +NN43N4limit_entitiesc                 f   t         t        d       yt               \  }}| r|d |  }t        dt        |       d| d       t	        j                         }ddd}|D ]w  }|d   }	 t        |      }t        |      }t        |      }	||z   |	z   }
|dxx   |
z  cc<   |d	xx   d
z  cc<   t        d| |d|
dddd        t        d| d| d| d|	        y t        d|d           |d   S # t        $ r:}t        d| d|        t        d| |dddddt        |             Y d }~d }~ww xY w)Nz[WATCH] PyYAML fehltr   z[WATCH] r   z1 Entities dieser Lauf (Rotation, verified+active))r   newr   r   r   r   zwatch:   z: web=z fb=z yt=z[WATCH] Fehler bei r9   )errorz[WATCH] Gesamt neu: )r"   rA   r2   r)   r   r   r   r   r   r@   str)r   r-   
total_entst0statsr   eidn_webn_fbn_ytr   excs               r   run_allr     su   |$%$D*O^$	HSYKq -( ) *	Bq!E Qn	Q OE #D?D$,%C%LCL&MQMsenb!S!Q4HHSEwd4&TFCDQ 
 u
/0<	  	Q'uBse45senb!Q1as3xPP	Qs   &A/C--	D060D++D0__main__z--limit)r   default)   )rG   )N)6__doc__sysrO   r   r   base64urllib.parser:   urllib.requestxml.etree.ElementTreeetreeElementTreer   r   r   r   pathlibr   pathinsertr   __file__parentr"   ImportError	signal_dbr   r   r!   r=   compiler   
GNEWS_SKIPlistdictr2   r$   bytesrD   tuplerZ   r   r   r   r   r   __name__argparseArgumentParserapadd_argument
parse_argsargslimit r   r   <module>r     s    	      " " 2 2  3tH~,,334 5 78n##**-BBA-
 "**U
VRZZ67
(tDz (c C  % E#s(O & +!4 +!3 +!S +! +!s +!+!#&:+!` # :$ 3 @= =# =DC$J # : z	 	 	 	"BOOICO6==?DDJJ K  Ds   5E: :FF