
    ;gj                     Z   d Z ddlZddlZddlZddlZddlZddlmZmZ ddl	m
Z
 ej                  j                  dd       dddZd	Zd
Z e
d      ZdefdZdedefdZdededefdZdefdZdedefdZddededefdZedk(  r$ddlZ e ej<                   e       d             yy)aV  
AA-045-F4: Link-/Bild-Anreicherung fuer bestehende Radar-Signale.
- Google-News-Aggregator-URL -> final_article_url aufloesen (mit Cache)
- Zielartikel laden -> og:image/twitter:image
- YouTube-Thumbnails als Sicherheitsnetz
Kontrollierter Backfill: nur aktuelle Signale ohne Bild, kurze Timeouts,
kein historischer Blind-Crawl. Idempotent.
    N)datetimetimezone)Pathz /opt/struktur/social-media-radarziMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124 Safari/537.36zde,es;q=0.9,en;q=0.8)z
User-AgentzAccept-Language
   i   z*/var/lib/sma-data/gnews_redirect_cache.txtreturnc                      i } t         j                         rFt         j                         j                         D ]!  }d|v s|j	                  dd      \  }}|| |<   # | S )N	   )
CACHE_FILEexists	read_text
splitlinessplit)cachelinekvs       /tmp/aa045f4_enrich.py_load_cacher      s_    E((*557 	Dt|zz$*1a	 L    r   r   c                     t         j                  d      5 }|j                  |  d| d       d d d        y # 1 sw Y   y xY w)Nar	   
)r   openwrite)r   r   fhs      r   _save_cacher   )   s=    		  
A3b2     s	   7A urlr   c                 n   | |v r||    S d}	 t         j                  j                  | t              }t         j                  j	                  |t
              5 }|j                  t              j                  dd      }|j                  rd|j                  vr|j                  }ddd       |sYt        j                  d	      xs. t        j                  d
|      xs t        j                  d|      }|r|j                  d      }|rt        | |       ||| <   |S # 1 sw Y   yxY w# t        $ r Y ,w xY w)zDLoest eine Google-News-URL zum Zielartikel auf (caching, kein Loop). headerstimeoututf-8replaceerrorsznews.google.comNzdata-n-au="([^"]+)"z href="(https://[^"]+)"[^>]*jslogz:<a[^>]+href="(https?://(?!www\.google|news\.google)[^"]+)"r
   )urllibrequestRequestHEADERSurlopenTIMEOUTreadMAX_HTMLdecoder   researchgroup	Exceptionr   )r   r   finalreqresphtmlms          r   resolve_gnewsr;   .   s$   
e|SzEnn$$S'$:^^##C#9 	!T99X&--gi-HDxx-TXX=	! 		0$7 _		=tD_		WY]^  
 Cc
L	! 	!  s,   A
D( AD$A#D( D%!D( (	D43D4c                    	 t         j                  j                  | t              }t         j                  j	                  |t
              5 }|j                  t              j                  dd      }ddd       dD ]h  }t        j                  |t        j                        }|s+|j                  d	      j                         }|j                  d
      s\d|v rdnd}||fc S  y# 1 sw Y   wxY w# t        $ r Y yw xY w)z>og:image / twitter:image aus einer Seite (klein, mit Timeout).r!   r#   r%   r&   r'   N)r    r    )zC<meta[^>]+property=["\']og:image["\'][^>]+content=["\']([^"\'>\s]+)zH<meta[^>]+content=["\']([^"\'>\s]+)["\'][^>]+property=["\']og:image["\']zD<meta[^>]+name=["\']twitter:image["\'][^>]+content=["\']([^"\'>\s]+)r
   httptwitterztwitter:imagezog:image)r)   r*   r+   r,   r-   r.   r/   r0   r1   r5   r2   r3   
IGNORECASEr4   strip
startswith)r   r7   r8   r9   patr:   usrcs           r   fetch_og_imagerE   H   s    nn$$S'$:^^##C#9 	IT99X&--gi-HD	I 

 IIc4/
  "A||F#)2c)9oz#v
 	I 	I s*   A
C5 'C)3C5 )C2.C5 5	D Dvideo_idc                 <   d|  d}	 t         j                  j                  |d      }t         j                  j                  |d      5 }|j                  dk(  xr d|j
                  j                  d	d
      v cd d d        S # 1 sw Y   y xY w# t        $ r Y yw xY w)Nhttps://i.ytimg.com/vi//hqdefault.jpgHEAD)method   r#      imagezContent-Typer    F)r)   r*   r+   r-   statusr"   getr5   )rF   r   r7   r8   s       r   yt_thumb_okrQ   ^   s    #H:^
<Cnn$$S$8^^##C#3 	Zt;;#%Y'T\\5E5EnVX5Y*Y	Z 	Z 	Z s0   AB 
/B9	B BB B 	BBdayslimitc           	         dd l }|j                  d      }|j                  |_        ddddd}t	        j
                  t        j                        j                         }t	        j
                  t        j                        j                  j                  t        d      j                         | dz  z
  t        j                        j                         }t               }|j                  d||f      j                         }|D ]u  }	t!        j"                  d|	d	   xs d
      }
|
s#t%        |
j'                  d            s>|j                  dd|
j'                  d       d||	d   f       |dxx   dz  cc<   w |j                  d||f      j                         }|D ]  }		 t)        |	d	   |      }|st+        |      \  }}dg|g}}|r|g dz  }|||dgz  }|||	d   gz  }|j                  ddj-                  |       d|       |dxx   |rdndz  cc<   t        j.                  d        |j                  d||f      j                         }|D ]i  }		 t+        |	d	         \  }}|r'|j                  d||||	d   f       |dxx   dz  cc<   n|j                  d||	d   f       t        j.                  d       k |j3                          |j                  d|f      j5                         d   }|j                  d|f      j5                         d   }|j7                          ||d <   ||d!<   |S # t0        $ r |dxx   dz  cc<   Y w xY w# t0        $ r |dxx   dz  cc<   Y w xY w)"Nr   z/var/lib/sma-data/signals.db)youtubegnews	direct_ogr(   timeiQ )tzz
        SELECT signal_id, source_url FROM signals
        WHERE source_type='youtube' AND COALESCE(image_found,0)=0
          AND created_at > ? LIMIT ?z[?&]v=([A-Za-z0-9_-]{11})
source_urlr    r
   zkUPDATE signals SET image_url=?, image_source='youtube', image_found=1, image_checked_at=? WHERE signal_id=?rH   rI   	signal_idrU   z
        SELECT signal_id, source_url FROM signals
        WHERE source_name LIKE '%Google News%'
          AND (COALESCE(image_found,0)=0 OR COALESCE(final_article_url,'')='')
          AND created_at > ? LIMIT ?zfinal_article_url=?)zimage_url=?zimage_source=?zimage_found=1zUPDATE signals SET z, z&, image_checked_at=? WHERE signal_id=?rV   g333333?r(   z
        SELECT signal_id, source_url FROM signals
        WHERE source_type='rss' AND source_name NOT LIKE '%Google News%'
          AND COALESCE(image_found,0)=0 AND created_at > ? LIMIT ?zcUPDATE signals SET image_url=?, image_source=?, image_found=1, image_checked_at=? WHERE signal_id=?rW   z7UPDATE signals SET image_checked_at=? WHERE signal_id=?z1SELECT COUNT(*) FROM signals WHERE created_at > ?zCSELECT COUNT(*) FROM signals WHERE image_found=1 AND created_at > ?total_periodwith_img_after)sqlite3connectRowrow_factoryr   nowr   utc	isoformat	__class__fromtimestamp
__import__rX   r   executefetchallr2   r3   rQ   r4   r;   rE   joinsleepr5   commitfetchoneclose)rR   rS   r^   connstatsrb   cutoffr   rowsrr:   r6   imgrD   setsvalstotalwith_imgs                     r   run_backfillry   h   s   ??9:D{{DAACE
,,x||
$
.
.
0Cll8<<(22}Z/446E(,,}WXaXaXc ME << ( +1%: ;C(* 	  "II2AlO4IrJQWWQZ(LL8*1771:,nEsAkN[] )!" << ( +1%	: ;C(*	 	
  !	!!!L/59E%e,HC/05'$DJJc1%S!K.))DLL.tyy.??efhlm'N3aA-NJJsO!$ << F IOPUX YaX`Xb 	  !	!%ao6HC<#sAkN35 k"a'"V!1[>24JJsO!  	KKM LLLviXaacdefE||adjclmvvxyz{HJJL!E.&E
LA  	!(Oq O	!*  	!(Oq O	!s,   $L66A:L6A&M6MMM-,M-__main__   )indent)   <   )__doc__sysr2   rX   urllib.requestr)   urllib.errorr   r   pathlibr   pathinsertr,   r.   r0   r   dictr   strr   r;   rE   boolrQ   intry   __name__jsonprintdumps r   r   <module>r      s     	    '  5 6U-
  >?
T  3  3  
s 4 C 4 ,# $ Ms Ms MD M` z	*$**\^A
./ r   