
    <j                    h   d Z ddlmZ ddlZddlZddlZddlZddlZddlZ	ddl
Z	ddlZ	ddlmZ ddlmZmZ ddlmZ ddlmZ ddlZ ee      j,                  Z ed      Z ee      ej4                  vr"ej4                  j7                  d ee             dd	lmZ dd
lmZm Z  	 ddl!Z!dZ#e G d d             Z$ G d de      Z%d Z&ddZ'd Z(d Z)d Z*ddZ+e,dk(  rG ejZ                         Z.e.j_                  dd       e.ja                         Z1 e+e1jd                         yy# e"$ r dZ!Y w xY w)a  Public website collector for SMA-MAS.

Robots-aware, rate-limited, read-only HTML intake. It discovers sitemap/start URLs,
extracts article metadata and sends only MAS-relevant candidates through signal_db's
canonical dedupe port. No login, cookie, CAPTCHA or paywall bypass.
    )annotationsN)	dataclass)datetimetimezone)
HTMLParser)Pathz /opt/struktur/social-media-radar)assess_mas_relevance)insert_signalrecord_source_runzZSMA-MAS-WebsiteCollector/1.0 (+public; no-login; contact: ops@agentsolutions-mallorca.com)c                  V    e Zd ZU ded<   ded<   ded<   dZded<   dZded<   dZded<   y	)
Pagestrurltitletext published_at	image_urlauthorN)__name__
__module____qualname____annotations__r   r   r        $/home/hermes/website_intake_final.pyr   r      s*    )#X|C2"Ic4DQSfSmr   r   c                  0     e Zd Z fdZd Zd Zd Z xZS )	Extractorc                    t         |   d       g | _        g | _        i | _        d| _        d| _        d| _        d| _        d| _	        y )NT)convert_charrefsr   Fr   )
super__init__r   r   meta	canonicalin_titleskipdater   )self	__class__s    r   r"   zExtractor.__init__    sw    'D)b4:B$)RT	egVZVdw|imiv  IJ  C  H  VX  LP  LU  fh  Z^  Zer   c                   t        |      }|j                         }|dv r| xj                  dz  c_        |dk(  rd| _        |dk(  ri|j	                  d      xs |j	                  d      xs dj                         }|j	                  d	d      j                         }|r|r|| j                  |<   |d
k(  rM|j	                  d      xs dj                         dk(  r&|j	                  dd      j                         | _        y y y )N>   navsvgstylefooterheaderscriptnoscript   r   Tr#   propertynamer   contentlinkrelr$   href)dictlowerr&   r%   getstripr#   r$   )r(   tagattrsakeyvals         r   handle_starttagzExtractor.handle_starttag"   s    K!SYY[GGTU'\4=&[	j0QUU6]0b7793quuYr?R?X?X?Z3	c$))C.&[aeeEl(b//1;>quuU[\^O_OeOeOg>[r   c                    |j                         dk(  rd| _        |j                         dv r#| j                  r| xj                  dz  c_        y y y )Nr   F>   r+   r,   r-   r.   r/   r0   r1   r2   )r:   r%   r&   )r(   r=   s     r   handle_endtagzExtractor.handle_endtag*   sF    YY['$-YY[OOTXT]T]^b^g^gij^j^gT]Or   c                    | j                   ry dj                  |j                               }|sy | j                  r| j                  j                  |       | j                  j                  |       y )N )r&   joinsplitr%   r   appendr   )r(   datavs      r   handle_datazExtractor.handle_data-   sN    	YYvHHTZZ\!	
6	]]4::$$Q'))1r   )r   r   r   r"   rB   rD   rL   __classcell__)r)   s   @r   r   r      s    hhkr   r   c                    t         st        d      t        j                  t        |       j	                               xs i }|j                  di       j                  di       j                  dg       S )Nz&PyYAML required for sources_multi.yamlsourceswebsite)yamlRuntimeError	safe_loadr   	read_textr;   )pathrJ   s     r   cfg_sourcesrV   4   s[    L!IJJ
nnT$Z))+,22""9R044YrBBr   c                2   t         j                  j                  | t        dd      }t         j                  j	                  ||      5 }|j                         |j                         |j                  j                         fcd d d        S # 1 sw Y   y xY w)Nz)text/html,application/xml;q=0.9,*/*;q=0.1)z
User-AgentAccept)headers)timeout)	urllibrequestRequestUAurlopengeturlreadrY   get_content_type)r   rZ   reqrs       r   fetchre   9   sr    ^^CbBm(nonnS1 :Q	
AFFHQYY779	9: : :s   
9BBc                   t         j                  j                  |       j                  j	                         j                  d      d   }t         j                  j                  |d         }t        |j                  d      xs- |j                  j	                         j                  d      d   g      }||vry|j                  d      xs dg}t        fd|D              S )	N:r   	start_urlallowed_hostsFallowed_paths/c              3  T   K   | ]  }j                   j                  |       ! y w)N)rU   
startswith).0xps     r   	<genexpr>zallowed.<locals>.<genexpr>C   s      0QAFFa 0s   %()	r[   parseurlsplitnetlocr:   rH   setr;   any)r   sourcehostbaseri   pathsrp   s         @r   allowedr{   >   s    	<<AHHNN$4$:$:3$?$BtllF;/0FJJD_  EGdhdododududwd}d}  B  eC  DE  eF  dG  AH-Uzz/"+se0%000r   c                   g }| j                  dg       D ]_  }	 t        |      \  }}}|j                  dd      }|t        j                  d|t        j
                        d | j                  dd       z  }a |D cg c]  }t        ||       s| c}S # t        $ r}t        d| d|        Y d }~d }~ww xY wc c}w )	Nsitemapsutf-8replacez<loc>\s*(.*?)\s*</loc>max_sitemap_urlsd   z  sitemap error : )	r;   re   decoderefindallI	Exceptionprintr{   )	rw   outsm_rawctyperooteus	            r   urls_from_sitemapr   E   s    ::j$ =R=Ry;1Sszz')<$"**.tBDD
9:]6::FXY\;]
^^3=
 -q71V,-- 
<!1"Rs;<<<-s$   AB;B;B;	B8B33B8c                   t               }|j                  |j                  dd             |j                  }dj	                  |j
                        j                         xs |j                  dd      }dj	                  |j                        }|j                  d      xs( |j                  d      xs |j                  d      xs d}|j                  d	      xs |j                  d
      xs d}|r t        j                  j                  | |      }t        |j                  xs | ||d d |||j                  dd            S )Nr~   r   rF   zog:titler   zarticle:published_timer'   pubdatezog:imageztwitter:imagei  r   )r   feedr   r#   rG   r   r<   r;   r   r[   rr   urljoinr   r$   )r   r   ro   mr   r   pubimgs           r   rr   rr   N   s    ;szz')4566!''"((*BaeeJr.B5	hhqvvUU#$OfOy9IORUU:6!%%06BV\\!!#c*Q[[Cd5Dk#c!%%:LMMr   3/opt/struktur/social-media-radar/sources_multi.yamlc                   dx}x}x}}t        |       D ]g  }|j                  dd      st        j                         }dx}x}x}	}
d }	 g }||d   gz  }|t	        |      z  }g }t               }|D ]w  }t        j                  |j                         j                  d      j                               j                         }||vsV|j                  |       |j                  |       y |d t        |j                  dd             D ].  }t        ||      s|dz  }	 t!        |      \  }}}d	|vr|j#                  d
      s<t%        ||      }t'        |j(                  dz   |j*                  z   |j(                  |j*                  d d d      }|j,                  s|
dz  }
|xs |}i dt        j                  |j                               j                         d d dt        t        j                               ddd|d   d|d|j                  dd      d|j                  dd      d|j(                  d d d|j*                  d d d|j(                  d d  d!d"d#d$d%|j                  d%d&      d'd(d)d*d+d,d-t/        j0                  d.d/g      t/        j0                  d0d1g      d2d3d4d5t        t        j                               d6z   d7|j2                  |j2                  rd8nd9d:	}t5        |      }|r|dz  }n|	dz  }	t        j:                  t=        |j                  d=d>                   1 	 tA        |d   ||||	|
|||@	       t9        |d   |||	|
|dA       ||z  }||z  }||	z  }||
z  }j t9        ||||dB       y # t6        $ r }t9        d;| d<|        |
dz  }
Y d }~d }~ww xY w# t6        $ r5}t?        |      }t9        d?|j                  d       d<|        Y d }~d }~ww xY w)CNr   enabledTrh   rk   	max_pages   r2   html)z.htmlz.htmrF   i  )topicshort_summary)fields	signal_id    	timestampsource_typerP   source_namer4   
source_urllanguagedesignal_categorycategoryr      r   i  extracted_hook   emotional_directionproblemurgency_level   mallorca_relevanceg?seasonal_relevanceg333333?risk_relevanceg      ?estimated_noiseg      ?suggested_case_typesfeuchtefallimmobilienpruefungfacebooklinkedinberatungg?g        daysi:	 neuzwebsite:og-imager   )	suggested_platformssuggested_ctaconfidence_scoreradar_score
decay_rate
expires_atstatusr   image_sourcez  page error r   rate_limit_secondsg      ?z  source error )rejectederroritems_parseditems_canonicalized)seennew	duplicater   r   )r   r   r   r   )!rV   r;   time	monotonicr   ru   hashlibsha256r<   rstripencode	hexdigestaddrI   intr{   re   endswithrr   r	   r   r   relevantjsondumpsr   r
   r   r   sleepfloatr   r   )config	total_new
total_seen	total_dup	total_rejrw   t0r   r   duprejerr
candidates	seen_urls	seen_hashr   hr   finalr   r   pager7   rJ   resultr   s                             r   runr   W   s	   *+++:+i	6" @V	Id	#H	^^")t)C)Ct3V:&%&&:"6**:935 ?qnnQWWY%%c*1134>>@A	)--*9+;+;A+>? 8VZZB789 <s3vx!GDE3Z_U3ueENN3C$DX
c
T	djjnTYY6tzzbfbkbklqmqbr>s	tSLLa\cU |;w~~elln5??A#2F  |TWX\XaXaXcTd  |fst}  |  L  MS  TZ  M[  |  \h  in  |  oy  z@  zD  zD  EO  PT  zU  |  Vg  hn  hr  hr  s}  ~G  hH  |  IP  QU  Q[  Q[  \`  ]`  Qa  |  bq  rv  r{  r{  |@  }@  rA  |  BR  SW  S]  S]  ^b  _b  Sc  |  dy  zC  |  DS  TU  |  Vj  kq  ku  ku  vJ  KN  kO  |  Pd  eh  |  iy  z}  |  ~O  PT  |  Uk  lp  lv  lv  xE	  F	Z	  w[	  l\	  |  s	w	  s	}	  s	}	  	I
  J
T
  ~	U
  s	V
  g
q
  EI  X[  io  }@  AE  AJ  AJ  AL  }M  NU  }U  _d  qu  q  q  ei  es  es  Oa  y{  |T$VsAvs
q&#JJuVZZ 4S9:;<" F6N2d3sCcW[pstvt##UXYZd*9c>9)S.)C=@> z	i9UV D5=Rs!;<c1fcD	UCF5?6::f;M:NbQRPS)T#U#UUsd   
A<N<AN< $NN<AN#N<$E&N
1N<	N9N4/N<4N99N<<	O:+O55O:__main__z--config)default)   )r   )3__doc__
__future__r   argparser   r   r   r   urllib.parser[   urllib.requesturllib.robotparserdataclassesr   r   r   html.parserr   pathlibr   sys__file__parentROOTRADARr   rU   insertmas_relevancer	   	signal_dbr
   r   rQ   ImportErrorr^   r   r   rV   re   r{   r   rr   r   r   ArgumentParserapadd_argument
parse_argsargsr   r   r   r   <module>r	     sK   # Z Z Z Z ! ' "  
	(^
-.u:SXXsxxqU< . 6 `
T T T
 *C
:
1.N!WF ZHboojAvow}  ~K  ~K  ~Mx|  NQ  RV  R]  R]  N^ I  
s   &D' 'D10D1