
    w~ojj                     4   d dl Z d dlZd dlZd dlZd dlmZmZ d dlmZm	Z	 d dl
mZ d dlmZmZ ddlmZ ddlmZmZ dd	lmZ dd
lmZ ddlmZmZ  ej6                  e      ZdZdedee e!ef      fdZ"dede e!ef   fdZ#dede$e   fdZ%dede!de&ddfdZ'y)    N)BaseHTTPRequestHandlerThreadingHTTPServer)EventThread)Any)parse_qsurlparse   )build_alert_summary)parse_agent_registrationparse_remote_event)_row)utc_now)Watcherbuild_collectoruPn  <!doctype html>
<html lang="de">
<head>
  <meta charset="utf-8">
  <meta name="viewport" content="width=device-width, initial-scale=1">
  <title>Worker-Überwachung · Agent Solutions</title>
  <style>
    :root { color-scheme: dark; --bg:#081416; --panel:#0d2022; --line:#244044; --text:#f4e4d2; --muted:#9bb0ae; --green:#54d98a; --yellow:#f4c95d; --red:#ff7777; --blue:#8dc8ff; }
    * { box-sizing:border-box; } body { margin:0; background:var(--bg); color:var(--text); font:15px/1.45 system-ui,-apple-system,sans-serif; }
    header { border-bottom:1px solid var(--line); }
    .header-inner { width:min(1280px,90vw); margin:0 auto; padding:24px 0 20px; display:flex; justify-content:space-between; gap:20px; align-items:center; }
    .header-title { flex:1 1 auto; min-width:0; }
    .header-actions { display:flex; align-items:center; gap:16px; flex:0 1 auto; min-width:0; }
    .refresh-meta { color:var(--muted); font-size:12px; line-height:1.45; text-align:right; }
    .refresh-meta div + div { margin-top:2px; }
    .refresh-message { color:inherit; margin-left:6px; }
    .refresh-message.error { color:var(--red); }
    #refresh { min-width:144px; display:inline-flex; align-items:center; justify-content:center; white-space:nowrap; }
    .spinner { display:inline-block; flex:0 0 12px; width:12px; height:12px; margin-right:6px; border:2px solid currentColor; border-right-color:transparent; border-radius:50%; vertical-align:-2px; animation:spin .7s linear infinite; }
    .spinner[hidden] { display:inline-block; visibility:hidden; animation:none; }
    @keyframes spin { to { transform:rotate(360deg); } }
    h1 { margin:0; font-size:28px; } h2 { margin:0 0 16px; font-size:19px; } .sub { color:var(--muted); margin-top:4px; }
    main { width:min(1280px,90vw); margin:28px auto 50px; } .grid { display:grid; grid-template-columns:repeat(auto-fit,minmax(140px,1fr)); gap:12px; margin-bottom:24px; }
    .card,.section { background:var(--panel); border:1px solid var(--line); border-radius:10px; } .card { padding:16px; }
    .label { color:var(--muted); font-size:12px; text-transform:uppercase; letter-spacing:.08em; } .value { font-size:28px; margin-top:4px; }
    .section { padding:20px; margin-bottom:20px; overflow:auto; } table { border-collapse:collapse; width:100%; min-width:700px; } .job-table { min-width:0; table-layout:fixed; } th,td { text-align:left; padding:10px 8px; border-bottom:1px solid var(--line); vertical-align:top; } th { color:var(--muted); font-size:12px; text-transform:uppercase; letter-spacing:.06em; white-space:nowrap; } .job-table th { white-space:normal; line-height:1.25; } .job-table th,.job-table td { overflow-wrap:anywhere; word-break:normal; }
    .job-table:not(.project-table) th:nth-child(1),.job-table:not(.project-table) td:nth-child(1) { width:14%; } .job-table:not(.project-table) th:nth-child(2),.job-table:not(.project-table) td:nth-child(2) { width:22%; } .job-table:not(.project-table) th:nth-child(3),.job-table:not(.project-table) td:nth-child(3) { width:6%; } .job-table:not(.project-table) th:nth-child(4),.job-table:not(.project-table) td:nth-child(4) { width:10%; } .job-table:not(.project-table) th:nth-child(5),.job-table:not(.project-table) td:nth-child(5) { width:6%; } .job-table:not(.project-table) th:nth-child(6),.job-table:not(.project-table) td:nth-child(6) { width:9%; } .job-table:not(.project-table) th:nth-child(7),.job-table:not(.project-table) td:nth-child(7) { width:8%; } .job-table:not(.project-table) th:nth-child(8),.job-table:not(.project-table) td:nth-child(8) { width:9%; } .job-table:not(.project-table) th:nth-child(9),.job-table:not(.project-table) td:nth-child(9) { width:7%; } .job-table:not(.project-table) th:nth-child(10),.job-table:not(.project-table) td:nth-child(10) { width:9%; }
    .job-table.project-table th:nth-child(1),.job-table.project-table td:nth-child(1) { width:8%; } .job-table.project-table th:nth-child(2),.job-table.project-table td:nth-child(2) { width:14%; } .job-table.project-table th:nth-child(3),.job-table.project-table td:nth-child(3) { width:20%; } .job-table.project-table th:nth-child(4),.job-table.project-table td:nth-child(4) { width:6%; } .job-table.project-table th:nth-child(5),.job-table.project-table td:nth-child(5) { width:9%; } .job-table.project-table th:nth-child(6),.job-table.project-table td:nth-child(6) { width:6%; } .job-table.project-table th:nth-child(7),.job-table.project-table td:nth-child(7) { width:8%; } .job-table.project-table th:nth-child(8),.job-table.project-table td:nth-child(8) { width:7%; } .job-table.project-table th:nth-child(9),.job-table.project-table td:nth-child(9) { width:8%; } .job-table.project-table th:nth-child(10),.job-table.project-table td:nth-child(10) { width:7%; } .job-table.project-table th:nth-child(11),.job-table.project-table td:nth-child(11) { width:7%; }
    .sort-button { color:inherit; background:transparent; border:0; border-radius:4px; padding:2px 0; font:inherit; letter-spacing:inherit; text-transform:inherit; text-align:left; } .sort-button:hover,.sort-button:focus-visible { color:var(--text); outline:2px solid var(--blue); outline-offset:2px; } .sort-arrow { color:var(--blue); margin-left:4px; }
    .pill { display:inline-block; padding:3px 9px; border-radius:999px; border:1px solid var(--line); font-size:12px; } .healthy,.online,.running,.completed { color:var(--green); border-color:#246b48; } .failed,.stale,.stopped { color:var(--red); border-color:#713b3b; } .degraded,.unknown,.unavailable,.waiting,.paused,.scheduled { color:var(--yellow); border-color:#725d2c; } .fresh { color:var(--green); } .muted { color:var(--muted); }
    button { color:var(--blue); background:transparent; border:1px solid var(--line); border-radius:6px; padding:8px 12px; cursor:pointer; } button:hover { border-color:var(--blue); } button:disabled { opacity:.7; cursor:wait; }
    .empty { color:var(--muted); padding:10px 0; } @media (max-width:850px) { .grid { grid-template-columns:repeat(2,1fr); } .header-inner { align-items:flex-start; flex-direction:column; } .header-actions { width:100%; justify-content:space-between; } } @media (max-width:500px) { .grid { grid-template-columns:1fr; } .header-actions { align-items:flex-start; flex-direction:column; } .refresh-meta { text-align:left; } }
  </style>
</head>
<body>
  <header><div class="header-inner"><div class="header-title"><h1>Worker-Überwachung</h1><div class="sub">Zentrale Nur-Lese-Überwachung von KI-Workern und Automationen</div></div><div class="header-actions"><div class="refresh-meta"><div>Zuletzt aktualisiert: <time id="last-updated">Unbekannt</time></div><div>Letzte Watcher-Prüfung: <time id="last-watcher-cycle">Unbekannt</time><span id="refresh-message" class="refresh-message" aria-live="polite"></span></div></div><button id="refresh" type="button" aria-busy="false"><span id="refresh-spinner" class="spinner" aria-hidden="true" hidden></span><span id="refresh-label">Aktualisieren</span></button></div></div></header>
  <main>
    <div class="grid" id="summary"></div>
    <section class="section"><h2>Zentrale Infrastruktur</h2><div id="jobs-core" class="empty">Lade Daten …</div></section>
    <section class="section"><h2>Projekt-Worker</h2><div id="jobs-projects" class="empty">Lade Daten …</div></section>
    <section class="section"><h2>Automationen und Hintergrund-Worker</h2><div id="jobs-automation" class="empty">Lade Daten …</div></section>
    <section class="section"><h2>Weitere überwachte Aufgaben</h2><div id="jobs-other" class="empty">Lade Daten …</div></section>
    <section class="section"><h2>Ausführungsorte und Remote-Agenten</h2><div id="agents" class="empty">Lade Daten …</div></section>
    <section class="section"><h2>Letzte Ereignisse</h2><div id="events" class="empty">Lade Daten …</div></section>
  </main>
  <script>
    const esc = value => String(value ?? '—').replace(/[&<>"']/g, c => ({'&':'&amp;','<':'&lt;','>':'&gt;','"':'&quot;',"'":'&#039;'}[c]));
    const statusTranslations = {
      healthy:'Fehlerfrei', degraded:'Gestört', failed:'Fehlgeschlagen', unknown:'Unbekannt',
      fresh:'Aktuell', unavailable:'Nicht verfügbar', stale:'Veraltet', running:'Läuft', stopped:'Gestoppt',
      waiting:'Wartet', paused:'Pausiert', scheduled:'Geplant', completed:'Abgeschlossen',
      critical:'Kritisch', warning:'Warnung', info:'Information', online:'Online', offline:'Offline',
      pending:'Ausstehend', activated:'Aktiviert', inactive:'Inaktiv', active:'Aktiv', dead:'Beendet',
      exited:'Beendet', activating:'Wird gestartet', deactivating:'Wird beendet', restarting:'Wird neu gestartet',
      created:'Erstellt', starting:'Wird gestartet', unhealthy:'Fehlerhaft', none:'Nicht geprüft',
      success:'Erfolgreich', failure:'Fehlerhaft', timeout:'Zeitüberschreitung', signal:'Signal', error:'Fehler'
    };
    const eventTranslations = {
      new_instance:'Neue Instanz', status_changed:'Zustand geändert', error_raised:'Fehler aufgetreten',
      error_resolved:'Fehler behoben', collector_failed:'Collector fehlgeschlagen', collector_succeeded:'Collector erfolgreich'
    };
    const statusLabel = value => statusTranslations[String(value ?? '').toLowerCase()] || value || '—';
    const eventLabel = value => eventTranslations[String(value ?? '').toLowerCase()] || statusLabel(value);
    const pill = value => `<span class="pill ${esc(value)}">${esc(statusLabel(value))}</span>`;
    const get = path => fetch(path, {cache:'no-store'}).then(r => { if (!r.ok) throw new Error(`HTTP ${r.status}`); return r.json(); });
    const tableStates = {};
    const collator = new Intl.Collator('de-DE', {sensitivity:'base', numeric:true});
    const healthRank = {failed:0, degraded:1, unknown:2, stale:3, healthy:4};
    const observationRank = {unavailable:0, stale:1, fresh:2};
    const operationalRank = {stopped:0, paused:1, waiting:2, scheduled:3, running:4, completed:5, unknown:6};
    const severityRank = {critical:0, warning:1, info:2};
    const agentRank = {offline:0, pending:1, online:2};
    const emptyValue = value => value === null || value === undefined || value === '' || Number.isNaN(value);
    const compareValues = (left, right, column) => {
      const leftEmpty = emptyValue(left.value);
      const rightEmpty = emptyValue(right.value);
      if (leftEmpty || rightEmpty) return leftEmpty === rightEmpty ? 0 : (leftEmpty ? 1 : -1);
      let result = 0;
      if (column.type === 'number' || column.type === 'date' || column.type === 'duration') {
        result = Number(left.value) - Number(right.value);
      } else if (column.type === 'status') {
        const ranks = column.rank || {};
        result = (ranks[String(left.value).toLowerCase()] ?? 999) - (ranks[String(right.value).toLowerCase()] ?? 999);
      } else {
        result = collator.compare(String(left.value), String(right.value));
      }
      return result;
    };
    const defaultJobCompare = (left, right) => {
      for (const [key, column] of [['health', {type:'status', rank:healthRank}], ['activity', {type:'date'}], ['task', {type:'text'}]]) {
        const result = compareValues(left.cells[key], right.cells[key], column);
        if (result) return result;
      }
      return 0;
    };
    const defaultEventCompare = (left, right) => compareValues(right.cells.time, left.cells.time, {type:'date'});
    const defaultAgentCompare = (left, right) => {
      const status = compareValues(left.cells.status, right.cells.status, {type:'status', rank:agentRank});
      return status || compareValues(right.cells.heartbeat, left.cells.heartbeat, {type:'date'});
    };
    const sortRows = (rows, columns, state, defaultCompare) => {
      const ordered = [...rows];
      if (!state.key) return ordered.sort(defaultCompare);
      const column = columns.find(item => item.key === state.key);
      if (!column) return ordered.sort(defaultCompare);
      return ordered.sort((left, right) => compareValues(left.cells[state.key], right.cells[state.key], column) * state.direction);
    };
    const sortableHeader = (target, column, state) => {
      const active = state.key === column.key && state.direction !== 0;
      const arrow = active ? (state.direction === 1 ? ' ▲' : ' ▼') : '';
      const direction = active ? (state.direction === 1 ? 'absteigend' : 'aufsteigend') : 'aufsteigend';
      return `<th scope="col" aria-sort="${active ? (state.direction === 1 ? 'ascending' : 'descending') : 'none'}"><button class="sort-button" type="button" data-sort-target="${esc(target)}" data-sort-key="${esc(column.key)}" aria-label="Nach ${esc(column.label)} ${direction} sortieren" title="Nach ${esc(column.label)} sortieren">${esc(column.label)}<span class="sort-arrow" aria-hidden="true">${arrow}</span></button></th>`;
    };
    function table(target, columns, rows, defaultCompare) {
      if (!rows.length) { document.getElementById(target).innerHTML = '<div class="empty">Noch keine Daten vorhanden.</div>'; return; }
      const available = new Set(columns.map(column => column.key));
      const state = tableStates[target] || {key:null, direction:0};
      if (state.key && !available.has(state.key)) { state.key = null; state.direction = 0; }
      tableStates[target] = state;
      const ordered = sortRows(rows, columns, state, defaultCompare);
      const markup = ordered.map(row => `<tr>${columns.map(column => `<td>${row.cells[column.key].html}</td>`).join('')}</tr>`).join('');
      const container = document.getElementById(target);
      const tableClass = target.startsWith('jobs-') ? `job-table${target === 'jobs-projects' ? ' project-table' : ''}` : '';
      container.innerHTML = `<table class="${tableClass}"><thead><tr>${columns.map(column => sortableHeader(target, column, state)).join('')}</tr></thead><tbody>${markup}</tbody></table>`;
      container.querySelectorAll('.sort-button').forEach(button => button.addEventListener('click', () => {
        const key = button.dataset.sortKey;
        const current = tableStates[target];
        if (current.key !== key) { current.key = key; current.direction = 1; }
        else if (current.direction === 1) current.direction = -1;
        else { current.key = null; current.direction = 0; }
        refresh();
      }));
    }
    const projectNames = {lead:'Lead Engine',social:'Social Media',telefon:'Telefon Agent',vento:'Vento',youtube:'YouTube Research',content:'Content Extraction',solar:'Solar',mirofish:'MiroFish'};
    const projectName = job => projectNames[job.job_key.split('.')[1]] || 'Projekt';
    const cell = (value, html) => ({value, html: html ?? esc(value)});
    const statusCell = value => cell(value, pill(value));
    const formatDate = value => {
      if (!value) return '—';
      const date = new Date(value);
      if (Number.isNaN(date.getTime())) return value;
      return `${new Intl.DateTimeFormat('de-DE', {dateStyle:'medium', timeStyle:'medium', timeZone:'UTC'}).format(date)} UTC`;
    };
    const parseTime = value => {
      if (!value) return null;
      const timestamp = new Date(value).getTime();
      return Number.isNaN(timestamp) ? null : timestamp;
    };
    const relativeDuration = milliseconds => {
      const seconds = Math.max(0, Math.floor(milliseconds / 1000));
      if (seconds < 60) return `vor ${seconds} ${seconds === 1 ? 'Sekunde' : 'Sekunden'}`;
      const minutes = Math.floor(seconds / 60);
      if (minutes < 60) return `vor ${minutes} ${minutes === 1 ? 'Minute' : 'Minuten'}`;
      const hours = Math.floor(minutes / 60);
      if (hours < 24) return `vor ${hours} ${hours === 1 ? 'Stunde' : 'Stunden'}`;
      const days = Math.floor(hours / 24);
      return `vor ${days} ${days === 1 ? 'Tag' : 'Tagen'}`;
    };
    const evidenceValue = (job, key) => (job.evidence_json || []).find(item => item.key === key)?.value;
    // Ersatzreihenfolge: fachliche Aktivität, Laufende, Erfolg, Lebenszeichen, Systemd-Aktivierung.
    const activityData = job => {
      const raw = job.last_activity_at || job.last_finished_at || job.last_started_at || job.last_success_at || job.heartbeat_at || evidenceValue(job, 'systemd_ActiveEnterTimestamp');
      const timestamp = parseTime(raw);
      return {timestamp, label: timestamp === null ? 'Unbekannt' : formatDate(timestamp)};
    };
    const nextRunData = job => {
      const timestamp = parseTime(job.expected_next_run_at);
      if (timestamp !== null) {
        return timestamp <= Date.now()
          ? cell(timestamp, `Überfällig seit ${relativeDuration(Date.now() - timestamp)}`)
          : cell(timestamp, formatDate(timestamp));
      }
      const continuous = job.operational_state === 'running' && ['service', 'worker'].includes(job.job_type);
      return cell(null, continuous ? 'Dauerbetrieb' : 'Kein Zeitplan bekannt');
    };
    const sourceStateLabel = value => statusLabel(value);
    const reasonText = value => {
      const text = String(value ?? '');
      const translations = {
        'reported source status':'Gemeldeter Quellenstatus',
        'source is healthy':'Quelle ist fehlerfrei',
        'run failed with exit code 15':'Ausführung mit Exitcode 15 fehlgeschlagen',
        'collector could not read the configured source':'Collector konnte die konfigurierte Quelle nicht lesen',
        'last known job status retained; current observation unavailable':'Letzter bekannter Aufgabenstatus beibehalten; aktuelle Beobachtung nicht verfügbar',
        'source was readable but did not provide enough evidence':'Quelle war lesbar, lieferte aber nicht genügend Nachweise',
        'remote agent heartbeat overdue':'Lebenszeichen des Remote-Agenten überfällig'
      };
      const missingSource = text.match(/^error: no such object: (.+)$/i);
      return translations[text] || (missingSource ? `Fehler: Quelle nicht gefunden: ${missingSource[1]}` : text || '—');
    };
    const reason = job => {
      if (job.observation_status === 'unavailable') return `Quelle nicht verfügbar: ${job.source_reference || job.worker_key}`;
      if (job.status_reason === 'reported source status') {
        const source = evidenceValue(job, 'source');
        const active = evidenceValue(job, 'systemd_ActiveState');
        const sub = evidenceValue(job, 'systemd_SubState');
        const result = evidenceValue(job, 'systemd_Result');
        const state = evidenceValue(job, 'reported_state');
        const container = evidenceValue(job, 'container_name');
        const cstate = evidenceValue(job, 'container_state');
        const chealth = evidenceValue(job, 'container_health');
        if (active || sub) return `Systemd ${sourceStateLabel(active)}/${sourceStateLabel(sub)}${result ? `, Ergebnis ${sourceStateLabel(result)}` : ''}`;
        if (container || cstate) return `Docker ${container || '—'}: ${sourceStateLabel(cstate)}${chealth ? ` (${sourceStateLabel(chealth)})` : ''}`;
        if (source === 'procfs') return evidenceValue(job, 'process_exists') ? 'Prozess vorhanden' : 'Prozess nicht vorhanden';
        return state ? `Gemeldeter Zustand: ${sourceStateLabel(state)}` : 'Quelle meldet normalen Zustand';
      }
      return reasonText(job.status_reason);
    };
    const jobColumns = [
      {key:'task', label:'Aufgabe / Instanz', type:'text'},
      {key:'worker', label:'Worker / Dienst', type:'text'},
      {key:'location', label:'Ort', type:'text'},
      {key:'activity', label:'Letzte Aktivität', type:'date'},
      {key:'since', label:'Seit', type:'duration'},
      {key:'operational', label:'Betriebszustand', type:'status', rank:operationalRank},
      {key:'health', label:'Gesundheit', type:'status', rank:healthRank},
      {key:'observation', label:'Beobachtung', type:'status', rank:observationRank},
      {key:'next', label:'Nächster Lauf / Überfällig', type:'date'},
      {key:'reason', label:'Grund', type:'text'}
    ];
    const projectColumns = [{key:'project', label:'Projekt', type:'text'}, ...jobColumns];
    const jobCells = job => {
      const activity = activityData(job);
      const since = activity.timestamp === null ? cell(null, 'Unbekannt') : cell(Date.now() - activity.timestamp, relativeDuration(Date.now() - activity.timestamp));
      const taskValue = `${job.name} ${job.job_key}`;
      const workerValue = job.worker_key || job.source_reference || '';
      const reasonValue = reason(job);
      return {
        task: cell(taskValue, `${esc(job.name)}<br><span class="muted">${esc(job.job_key)}</span>`),
        worker: cell(workerValue, `${esc(workerValue)}${job.source_reference && job.source_reference !== workerValue ? `<br><span class="muted">${esc(job.source_reference)}</span>` : ''}`),
        location: cell(job.location_name || job.host || null),
        activity: cell(activity.timestamp, esc(activity.label)),
        since,
        operational: statusCell(job.operational_state),
        health: statusCell(job.health_status),
        observation: statusCell(job.observation_status),
        next: nextRunData(job),
        reason: cell(reasonValue, esc(reasonValue))
      };
    };
    const jobRow = job => ({cells: jobCells(job)});
    const projectJobRow = job => ({cells: {project: cell(projectName(job)), ...jobCells(job)}});
    const agentColumns = [
      {key:'agent', label:'Agent', type:'text'}, {key:'environment', label:'Umgebung', type:'text'},
      {key:'host', label:'Host', type:'text'}, {key:'runtime', label:'Laufzeit', type:'text'},
      {key:'status', label:'Zustand', type:'status', rank:agentRank}, {key:'heartbeat', label:'Letzte Rückmeldung', type:'date'}
    ];
    const agentRow = agent => {
      const heartbeat = parseTime(agent.last_heartbeat_at);
      return {cells:{
        agent:cell(agent.agent_name, `${esc(agent.agent_name)}<br><span class="muted">${esc(agent.agent_key)}</span>`),
        environment:cell(agent.environment), host:cell(agent.host), runtime:cell(agent.runtime),
        status:statusCell(agent.status), heartbeat:cell(heartbeat, esc(heartbeat === null ? 'Unbekannt' : formatDate(heartbeat)))
      }};
    };
    const eventColumns = [
      {key:'time', label:'Zeit', type:'date'}, {key:'type', label:'Typ', type:'text'},
      {key:'severity', label:'Schweregrad', type:'status', rank:severityRank}, {key:'component', label:'Komponente', type:'text'},
      {key:'reason', label:'Grund', type:'text'}
    ];
    const eventRow = event => {
      const time = parseTime(event.occurred_at);
      const reasonValue = reasonText(event.reason || event.message);
      return {cells:{
        time:cell(time, esc(time === null ? 'Unbekannt' : formatDate(time))), type:cell(eventLabel(event.event_type)),
        severity:statusCell(event.new_severity || event.severity), component:cell(event.component || event.instance_id),
        reason:cell(reasonValue, esc(reasonValue))
      }};
    };
    const refreshButton = document.getElementById('refresh');
    const refreshLabel = document.getElementById('refresh-label');
    const refreshSpinner = document.getElementById('refresh-spinner');
    const refreshMessage = document.getElementById('refresh-message');
    const lastUpdated = document.getElementById('last-updated');
    const lastWatcherCycle = document.getElementById('last-watcher-cycle');
    let refreshInFlight = null;
    let hasRenderedData = false;
    let lastSuccessfulRefreshAt = null;

    const formatLocalTimestamp = value => {
      if (!value) return 'Unbekannt';
      const timestamp = new Date(value).getTime();
      if (Number.isNaN(timestamp)) return 'Unbekannt';
      return new Intl.DateTimeFormat('de-DE', {
        day:'2-digit', month:'2-digit', year:'numeric', hour:'2-digit', minute:'2-digit', second:'2-digit',
        hourCycle:'h23', timeZone:'Europe/Madrid'
      }).format(new Date(timestamp));
    };
    const updateRefreshMeta = () => {
      lastUpdated.textContent = lastSuccessfulRefreshAt
        ? `${formatLocalTimestamp(lastSuccessfulRefreshAt)} · ${relativeRefreshAge((Date.now() - lastSuccessfulRefreshAt) / 1000)}`
        : 'Unbekannt';
    };
    const relativeRefreshAge = seconds => {
      const value = Math.max(0, Math.floor(Number(seconds)));
      if (value < 60) return 'gerade eben';
      if (value < 3600) return `vor ${Math.floor(value / 60)} Min.`;
      if (value < 86400) return `vor ${Math.floor(value / 3600)} Std.`;
      return `vor ${Math.floor(value / 86400)} T.`;
    };
    const setRefreshUi = loading => {
      refreshButton.disabled = loading;
      refreshButton.setAttribute('aria-busy', String(loading));
      refreshSpinner.hidden = !loading;
      refreshLabel.textContent = 'Aktualisieren';
    };
    let refreshMessageTimer = null;
    const setRefreshMessage = (message, error = false, duration = 0) => {
      if (refreshMessageTimer) window.clearTimeout(refreshMessageTimer);
      refreshMessage.textContent = message;
      refreshMessage.classList.toggle('error', error);
      if (duration > 0) {
        refreshMessageTimer = window.setTimeout(() => {
          refreshMessage.textContent = '';
          refreshMessage.classList.remove('error');
          refreshMessageTimer = null;
        }, duration);
      }
    };

    function refresh(manual = false) {
      if (refreshInFlight) return refreshInFlight;
      setRefreshUi(true);
      if (manual) setRefreshMessage('Wird aktualisiert …');
      refreshInFlight = Promise.all([get('/api/v1/status'), get('/api/v1/jobs'), get('/api/v1/agents'), get('/api/v1/events?limit=25')])
        .then(([status, jobs, agents, events]) => {
          const s = status;
          document.getElementById('summary').innerHTML = [['Watcher',statusLabel(s.watcher_status)],['Aufgaben',s.jobs_total],['Fehlerfrei',s.jobs_healthy],['Gestört',s.jobs_degraded],['Fehlgeschlagen',s.jobs_failed],['Unbekannt',s.jobs_unknown],['Agenten',agents.items.length]].map(x => `<div class="card"><div class="label">${esc(x[0])}</div><div class="value">${esc(x[1])}</div></div>`).join('');
          const groups = {core:[], projects:[], automation:[], other:[]};
          jobs.items.forEach(j => { const key = j.job_key; if (key.startsWith('project.')) groups.projects.push(j); else if (key.startsWith('automation.')) groups.automation.push(j); else if (key.startsWith('agent.') || key.startsWith('dashboard.') || key.startsWith('portal.') || key.startsWith('admin.') || key.startsWith('commander.') || key.startsWith('editor.') || key.startsWith('watcher.') || key.startsWith('process.')) groups.core.push(j); else groups.other.push(j); });
          table('jobs-core',jobColumns,groups.core.map(jobRow),defaultJobCompare);
          table('jobs-projects',projectColumns,groups.projects.map(projectJobRow),defaultJobCompare);
          table('jobs-automation',jobColumns,groups.automation.map(jobRow),defaultJobCompare);
          table('jobs-other',jobColumns,groups.other.map(jobRow),defaultJobCompare);
          table('agents',agentColumns,agents.items.map(agentRow),defaultAgentCompare);
          table('events',eventColumns,events.items.map(eventRow),defaultEventCompare);
          hasRenderedData = true;
          lastSuccessfulRefreshAt = Date.now();
          lastUpdated.dateTime = new Date(lastSuccessfulRefreshAt).toISOString();
          updateRefreshMeta();
          lastWatcherCycle.textContent = formatLocalTimestamp(s.last_cycle_finished_at);
          lastWatcherCycle.dateTime = s.last_cycle_finished_at || '';
          if (manual) setRefreshMessage('Aktualisiert', false, 2000);
          else setRefreshMessage('');
        })
        .catch(error => {
          if (!hasRenderedData) document.getElementById('summary').innerHTML = `<div class="card"><div class="label">Verbindung</div><div class="value">Fehler</div><div class="muted">${esc(error instanceof Error ? error.message : error)}</div></div>`;
          setRefreshMessage('Aktualisierung fehlgeschlagen', true, 2000);
        })
        .finally(() => {
          setRefreshUi(false);
          refreshInFlight = null;
        });
      return refreshInFlight;
    }
    refreshButton.addEventListener('click', () => refresh(true));
    refresh();
    setInterval(refresh, 15000);
    setInterval(updateRefreshMeta, 1000);
  </script>
</body>
</html>watcherreturnc                 `   g }| j                   j                  D ]   }| j                  j                  d|j                  f      j                         }	 t        |j                        j                  }|j                  |j                  |j                  |j                  t        |j                        |d}|8|j                  |d   |d   t        |d         |d   |d   |d	   |d
   d       n|j                  d d d d d ddd       |j!                  |        |S # t        $ r d}Y w xY w)NzLSELECT * FROM collector_runs WHERE collector_name=? ORDER BY id DESC LIMIT 1unknown)nametypeenabledtimeout_secondsdefinitions_countversion
started_atfinished_atsuccess
error_typeerror_message	jobs_seenobservations_created)last_run_atlast_finished_atlast_successlast_error_typelast_error_messager!   r"   r   )config
collectors
connectionexecuter   fetchoner   r   r   
ValueErrorr   r   lendefinitionsupdateboolappend)r   rowsr(   latestr   items         G/opt/struktur/hermes-workspace/worker-watcher/src/worker_watcher/api.pycollector_config_rowsr7     sG   !#D..++ ##++Z]c]h]h\j

(* 		 %fkk2::G KKKK~~%55!$V%7%7!8 
 KK%l3$*=$9 $VI%6 7#),#7&,_&=#K0(./E(F  KK$X\,0[\124 5 	D=> K3  	 G	 s   DD-,D-c                 b   | j                         }| j                  j                         D cg c]  }t        |       }}| j                  j                  d      D cg c]  }t        |       }}t        |||t               | j                  j                  j                        S c c}w c c}w )Nz4SELECT * FROM status_events ORDER BY occurred_at ASC)health
repositorystatus_rowsr   r*   r+   r   r   r(   r   cycle_interval_seconds)r   r9   rowjobseventss        r6   alert_summaryr@     s    ^^F!(!3!3!?!?!AB#DIBDB#*#5#5#=#=>t#uvCd3ivFvvtVWY@V@V@m@mnn Cvs   B'B,c                 *      G  fddt               }|S )Nc                       e Zd ZdededdfdZdededdfdZded	eddfd
Zde	eef   dz  f fdZ
def fdZd fdZd fdZy)create_handler.<locals>.Handlerformatargsr   Nc                      y )N )selfrD   rE   s      r6   log_messagez+create_handler.<locals>.Handler.log_message  s        statuspayloadc                 >   t        j                  |dt              j                  d      }| j	                  |       | j                  dd       | j                  dt        t        |                   | j                          | j                  j                  |       y )NF)ensure_asciidefaultutf-8Content-Typezapplication/json; charset=utf-8Content-Length)
jsondumpsstrencodesend_responsesend_headerr.   end_headerswfilewrite)rH   rK   rL   bodys       r6   	send_jsonz)create_handler.<locals>.Handler.send_json  su    ::gE3GNNwWDv&^-NO-s3t9~>JJT"rJ   r\   c                 
   |j                  d      }| j                  |       | j                  dd       | j                  dt        t	        |                   | j                          | j                  j                  |       y )NrP   rQ   ztext/html; charset=utf-8rR   )rV   rW   rX   rU   r.   rY   rZ   r[   )rH   rK   r\   encodeds       r6   	send_htmlz)create_handler.<locals>.Handler.send_html  sg    kk'*Gv&^-GH-s3w</@AJJW%rJ   c                 d   	 t        | j                  j                  dd            }|dk  s#|j                  j                  j
                  kD  ry t        j                  | j                  j                  |            }t        |t              r|S d S # t        t        j                  f$ r Y y w xY w)NrR   0r   )intheadersgetr(   r   max_remote_event_bytesrS   loadsrfileread
isinstancedictr-   JSONDecodeError)rH   lengthrL   r   s      r6   	read_jsonz)create_handler.<locals>.Handler.read_json  s    T\\--.>DEQ;&7>>+A+A+X+X"X**TZZ__V%<=",Wd";wEE 4 45 s   AB ?B B B/.B/c                     t         j                  j                  j                  j                  j
                        }|sy| j                  j                  dd      }t        j                  |d|       S )NTAuthorization zBearer )	osenvironre   r(   r   agent_auth_token_envrd   secretscompare_digest)rH   expectedprovidedr   s      r6   authorized_agent_requestz8create_handler.<locals>.Handler.authorized_agent_request  s[    zz~~gnn&<&<&Q&QRH||''<H))(ghZ4HIIrJ   c           
      
   t        | j                        }|j                  j                  d      xs d}|dk(  r| j                  dt               y |dk(  r| j                  dddi       y |dk(  rJ	 j                  j                  d      j                          | j                  ddj                  d	       y |dk(  s|dk(  r!| j                  dj                                y |dk(  r| j                  dt                     y |dk(  rt        |j                         }j"                  j%                         }|j'                  dd g      d   }|r |D cg c]  }|d   |k(  s	|d   |k(  s| }}| j                  dd|D cg c]  }t)        |       c}i       y |dk(  rD| j                  ddj"                  j+                         D cg c]  }t)        |       c}i       y |dk(  rD| j                  ddj"                  j-                         D cg c]  }t)        |       c}i       y |dk(  r| j                  ddt/              i       y |dk(  rwt1        t        |j                         j'                  ddg      d         }| j                  ddj"                  j3                  |      D cg c]  }t)        |       c}i       y |j5                  d      r]|j7                  dd      d   t9        fdt/              D        d       }	|	| j                  ddd i       y | j                  d|	       y |d!k(  rwt1        t        |j                         j'                  ddg      d         }| j                  ddj"                  j;                  |      D cg c]  }t)        |       c}i       y |j5                  d"      r	 t1        |j7                  dd      d         }
j                  j                  d$|
f      j                         }|r"| j                  d|rt)        |      ndd#i       y | j                  ddd#i       y |d%k(  rwt1        t        |j                         j'                  dd&g      d         }| j                  ddj"                  j?                  |      D cg c]  }t)        |       c}i       y | j                  ddd'i       y # t        $ r2}| j                  d
dt        |      j                  d       Y d }~y d }~ww xY wc c}w c c}w c c}w c c}w c c}w c c}w # t<        $ r | j                  ddd#i       Y y w xY wc c}w )(N/   z/health/liverK   livez/health/readyzSELECT 1ready)rK   schema_versioni  	not_ready)rK   errorz/healthz/api/v1/statusz/api/v1/alerts/summaryz/api/v1/jobsr   health_statusoperational_stateitemsz/api/v1/workersz/api/v1/agentsz/api/v1/collectorsz/api/v1/collectors/runslimitd   z/api/v1/collectors/r
   c              3   4   K   | ]  }|d    k(  s|  yw)r   NrG   ).0r=   collector_names     r6   	<genexpr>z9create_handler.<locals>.Handler.do_GET.<locals>.<genexpr>  s     lScRXk]kNkSls     r   zcollector not foundz/api/v1/remote-eventsz/api/v1/jobs/zinstance not foundz4SELECT * FROM job_status_current WHERE instance_id=?z/api/v1/events2   	not found) r	   pathrstripr`   WATCHER_HTMLr]   r*   r+   r,   r   	Exceptionr   __name__r9   r@   r   queryr:   r;   re   r   worker_rows
agent_rowsr7   rc   collector_rows
startswithrsplitnextremote_event_rowsr-   r?   )rH   parsedr   excr   r3   rK   r=   r   r5   instance_idr   r   s              @r6   do_GETz&create_handler.<locals>.Handler.do_GET  s   dii(F;;%%c*1cDs{sL1'sXv$67(^&&..z:CCENN37gNdNd(ef "d.>&>sGNN$4511sM'$:;' .))5578dV4Q7+/xC33G63QUXYlUmqwUwCxDxsWD.IStCy.I$JK**sWGDVDVDbDbDd.eStCy.e$fg))sWGDVDVDaDaDc.dStCy.d$ef--sW.CG.L$MN22HV\\266wFqIJsWGDVDVDeDefkDl.mStCy.m$no!67!%S!!4Q!7l,A',Jlnrs<NN32G(HINN3-00HV\\266wFqIJsWGDVDVDhDhinDo.pStCy.p$qr1"%dkk#q&9!&<"=K ((001gjuiwx  B  B  D^asDI7DX:YZgkguguvy  }D  FZ  |[  h\))HV\\266wEaHIsWGDVDVD]D]^cDd.eStCy.e$fgsWk$:;Y ! ^NN3;cI[I[(\]]^ y.I.e.d
 /n /q " NN32F(GH /fs[   3AS  T.TT#T(T-6T2 T7T< 0U 	T)(TT<UUc                    t        | j                        j                  j                  d      xs d}|dvr| j                  dddi       y | j	                         s| j                  dddi       y j
                  j                  j                  s| j                  dddi       y | j                         }|| j                  d	dd
i       y 	 |j                  d      r=t        |      }j                  |      }| j                  dd||j                  d       y t        |      }| j                  dj                  |             y # t        t         t"        f$ r'}| j                  ddt%        |      i       Y d }~y d }~ww xY w)Nr{   >   /api/v1/agents/events/api/v1/agents/registerr   r   r   i  zagent authentication requiredzremote ingest disabledi  z+JSON object required and limited to 512 KiBz	/register   T)acceptedagent_id	agent_keyr|   i  )r	   r   r   r]   ry   r(   r   remote_ingest_enabledrn   endswithr   register_remote_agentr   r   ingest_remote_eventKeyError	TypeErrorr-   rU   )rH   r   rL   registrationr   eventr   r   s          r6   do_POSTz'create_handler.<locals>.Handler.do_POST	  sV   DII&++2237>3DOOsWk$:;002sW.M$NO>>))??sW.F$GHnn&GsW.[$\]	9==-#;G#DL&<<\JHNN3Tx^j^t^t(uv.w7ENN3(C(CE(JKi4 9sWc#h$7889s   AE
 ,E
 
FF  Fr   N)r   
__module____qualname__rU   r   rI   rc   r]   r`   rk   rn   r1   ry   r   r   )r   s   r6   HandlerrC     s    	c 	# 	$ 		#C 	## 	#$ 	#	&C 	&s 	&t 	&	tCH~4 		Jd 	J7	<r	9rJ   r   )r   )r   r   s   ` r6   create_handlerr     s    v9( v9p NrJ   hostportc                     t               d fd}t        |dd      }|j                          t        ||ft	                     }	 |j                          j                          |j                  d       |j                          y # j                          |j                  d       |j                          w xY w)Nc                      j                         sR	 j                           j                  j                  j                  j                          j                         sQy y # t        $ r t        j	                  d       Y bw xY w)Nzwatcher cycle failed)	is_setrun_oncer   LOGGER	exceptionwaitr(   r   r<   )stopr   s   r6   	scan_loopzserve.<locals>.scan_loop)  sf    ++-9  " IIgnn,,CCD ++-  9  !789s   A% %BBzwatcher-cycleT)targetr   daemon   )timeoutr   )	r   r   startr   r   serve_foreversetjoinserver_close)r   r   r   r   scan_threadserverr   s   `     @r6   server   &  s    7DE 	MK $~g/FGF
# 	
#s   
B 4C)(rS   loggingrr   ru   http.serverr   r   	threadingr   r   typingr   urllib.parser   r	   alertsr   remoter   r   reportsr   timeutilr   r   r   r   	getLoggerr   r   r   listrk   rU   r7   r@   r   r   rc   r   rG   rJ   r6   <module>r      s      	  C #  + ' @   -			8	$iX!7 !tDcN/C !Ho7 otCH~ oyG y-C(D yx7 # S T rJ   