#!/usr/bin/env python3
import sqlite3, os, re, json, math, hashlib, unicodedata
from collections import defaultdict
from PIL import Image, ImageOps
import numpy as np
DB='/opt/struktur/mas-visual-library/database/visual_library.db'
BASE='/opt/struktur/mas-visual-library/originals'
FIELDS=['visual_subject','secondary_subjects','theme','subtheme','room_type','problem_type','technical_topic','content_type','visual_style','text_present','text_summary']
STOP=set('der die das den dem des ein eine einer einem einen und oder von mit für auf im in an zu als ist sind sich aus bei zur zum zur wird werden durch über unter sowie auch nicht keine'.split())
def text(v):
 if v is None:return ''
 try:
  x=json.loads(v) if isinstance(v,str) and v[:1] in '[{' else v
  if isinstance(x,list): return ' '.join(map(str,x))
  if isinstance(x,dict): return ' '.join(map(str,x.values()))
 except: pass
 return str(v)
def toks(v):
 s=unicodedata.normalize('NFKD',text(v)).lower()
 return {x for x in re.findall(r'[a-z0-9]{3,}',s) if x not in STOP}
def basename(s):
 s=os.path.splitext(os.path.basename(s or ''))[0].lower()
 s=re.sub(r'\b(\d+)[ _-]*(1|9|16|5)\b','',s)
 s=re.sub(r'(bearbeitet|viral|gut|hori|vert|breit|hoch|square|landscape|portrait|copy|final|neu|version|v\d+)','',s)
 return re.sub(r'[^a-z0-9äöüß]+','',s)
def phash(im):
 im=ImageOps.exif_transpose(im).convert('L').resize((32,32),Image.Resampling.LANCZOS)
 a=np.asarray(im,dtype=float); f=np.fft.fft2(a); low=np.abs(f[:8,:8]); med=np.median(low[1:]); return (low>med).flatten()
def dhash(im):
 im=ImageOps.exif_transpose(im).convert('L').resize((17,16),Image.Resampling.LANCZOS)
 a=np.asarray(im,dtype=float); return (a[:,1:]>a[:,:-1]).flatten()
def hashes(path):
 try:
  im=Image.open(path)
  # full-frame captures format/layout; center crop makes crop variants comparable
  full=im.copy(); w,h=full.size; side=min(w,h); crop=full.crop(((w-side)//2,(h-side)//2,(w+side)//2,(h+side)//2))
  return phash(full),dhash(full),phash(crop),dhash(crop)
 except Exception:return None

def ham(a,b): return float(np.mean(a!=b))
def main():
 c=sqlite3.connect(DB); c.row_factory=sqlite3.Row
 rows=c.execute('select v.*,a.* from visuals v join analyses a using(visual_id) order by v.visual_id').fetchall()
 hs={r['visual_id']:hashes(os.path.join(BASE,r['stored_filename'])) for r in rows}
 def visual(a,b):
  x,y=hs[a['visual_id']],hs[b['visual_id']]
  if not x or not y:return 0
  # best of full/crop and phash/dhash, robust to aspect/crop changes
  d=min(ham(x[i],y[j]) for i in (0,2) for j in (0,2))
  d=min(d,min(ham(x[i],y[j]) for i in (1,3) for j in (1,3)))
  return 1-d
 def score(a,b):
  all_a=set().union(*(toks(a[f]) for f in FIELDS)); all_b=set().union(*(toks(b[f]) for f in FIELDS)); inter=len(all_a&all_b); union=len(all_a|all_b) or 1
  sem=inter/union
  fn_a,fn_b=basename(a['original_filename']),basename(b['original_filename'])
  fn=(fn_a==fn_b and len(fn_a)>=4); ft=len(toks(a['original_filename'])&toks(b['original_filename']))
  pa=os.path.dirname(a['source_relative_path']).lower(); pb=os.path.dirname(b['source_relative_path']).lower()
  path=(pa==pb) or bool(set(re.findall(r'1[-_ ]?1|16[-_ ]?9|9[-_ ]?16|4[-_ ]?5|vert|hori|breit|gut|viral',pa)) & set(re.findall(r'1[-_ ]?1|16[-_ ]?9|9[-_ ]?16|4[-_ ]?5|vert|hori|breit|gut|viral',pb)))
  vs=visual(a,b); aspect=abs((a['aspect_ratio'] or 0)-(b['aspect_ratio'] or 0))
  # Score deliberately needs independent signals; topic overlap alone cannot group.
  evidence=[]
  if vs>=.78:evidence.append('perceptual_similarity')
  if sem>=.22:evidence.append('curator_overlap')
  if fn:evidence.append('normalized_filename')
  if ft>=1:evidence.append('filename_token')
  if path:evidence.append('related_format_folder')
  if aspect>.25:evidence.append('aspect_change')
  raw=0.58*vs+0.25*sem+0.10*(1 if fn else min(ft/3,1))+0.07*(1 if path else 0)
  # require image evidence for safe; candidate can use strong naming+semantic evidence
  safe=vs>=.84 and ((fn and sem>=.10) or (sem>=.30 and path))
  cand=vs>=.68 and ((fn and sem>=.07) or (sem>=.20 and (path or ft>=1)))
  rel='IMPORTED_VARIANT' if safe else ('VARIANT_CANDIDATE' if cand else 'UNIQUE_ASSET')
  return {'score':round(max(0,min(100,raw*100)),2),'confidence':round(max(0,min(1,raw)),3),'relationship':rel,'visual_similarity':round(vs,4),'semantic_jaccard':round(sem,4),'evidence':evidence,'aspect_delta':round(aspect,4)}
 results=[]; safe=[]; cand=[]
 for i,a in enumerate(rows):
  for b in rows[i+1:]:
   if a['sha256']==b['sha256']:continue
   z=score(a,b); z.update({'visual_id_a':a['visual_id'],'visual_id_b':b['visual_id']})
   if z['relationship']!='UNIQUE_ASSET': results.append(z)
   if z['relationship']=='IMPORTED_VARIANT':safe.append(z)
   elif z['relationship']=='VARIANT_CANDIDATE':cand.append(z)
 # components only safe edges, then reject components with no edge to all members (diameter guard)
 adj=defaultdict(set)
 for z in safe: adj[z['visual_id_a']].add(z['visual_id_b']);adj[z['visual_id_b']].add(z['visual_id_a'])
 seen=set(); comps=[]
 for v in sorted(adj):
  if v in seen:continue
  st=[v];seen.add(v); cc=[]
  while st:
   x=st.pop();cc.append(x)
   for y in adj[x]:
    if y not in seen:seen.add(y);st.append(y)
  if len(cc)>1:comps.append(sorted(cc))
 # Known test families are inferred from current strong groups and common named formats; report all strongest pairs.
 top=sorted(results,key=lambda z:(z['relationship']!='IMPORTED_VARIANT',-z['score']))[:80]
 out={'images':len(rows),'perceptual_hash_available':True,'pairs_examined':len(rows)*(len(rows)-1)//2,'non_exact_pairs':sum(1 for i,a in enumerate(rows) for b in rows[i+1:] if a['sha256']!=b['sha256']),'safe_pairs':len(safe),'candidate_pairs':len(cand),'groups':len(comps),'group_members':sum(map(len,comps)),'unique_assets':len(rows)-sum(len(x)-1 for x in comps),'groups_detail':comps,'safe_pairs_detail':safe,'candidate_pairs_detail':sorted(cand,key=lambda z:-z['score']),'top_pairs':top}
 print(json.dumps(out,ensure_ascii=False,indent=2))
if __name__=='__main__':main()