From 2d1e4aa98fadfdc9b13623fd5edb44c9270fee40 Mon Sep 17 00:00:00 2001 From: librelad Date: Fri, 17 Jul 2026 22:03:02 +0100 Subject: [PATCH] feat(health): self-healing control-plane watchdog + crash-loop failure cap MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An offline trivy install crash-looped (server FATALs when it can't fetch the vuln DB), and on rootless docker the restart storm churned the shared network's port-forwarder until the WebUI's own published host port was torn down — the WebUI stayed healthy INSIDE its container but was unreachable from the host, with nothing detecting or healing it. Three fixes, in the house self-healing style (mirrors the network-drift trio): 1. Control-plane health checker wired into the existing task-processor idle poll (maybeRegenPoll), no new daemon. dockerHealthScan (read-only) detects daemon down, a WebUI running-but-host-port-unreachable (the port-forward corruption), and crash-looping containers. webuiSystemHealthCheck writes frontend/data/system/health_status.json + self-dispatches a heal — the user can't click a button on a dead WebUI, so the poll drives the fix. Frontend health-notifier surfaces a topbar badge + dashboard banner + details panel. 2. Failure cap, enforced centrally by dockerHealthHeal (task-gated): stops crash-loopers (removing the churn), restarts the WebUI to re-publish a lost port forward, and — only if that fails — recycles the rootless daemon and restarts the core container. Caps every app immediately, no template churn. 3. Trivy no longer crash-loops offline: the server runs in a shell retry-loop so the container stays Up and quietly retries on a backoff instead of exiting FATAL. Verified: container stays Up across repeated DB-download failures. Core WebUI compose gains restart: unless-stopped so it self-recovers after a reboot / daemon recycle instead of staying down. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: librelad --- containers/libreportal/docker-compose.yml | 4 + .../frontend/core/boot/js/system-loader.js | 1 + .../health-notifier/css/health-notifier.css | 247 +++++++++++++ .../health-notifier/js/health-notifier.js | 336 ++++++++++++++++++ containers/libreportal/frontend/index.html | 1 + containers/trivy/docker-compose.yml | 17 +- .../commands/system/cli_system_commands.sh | 25 ++ .../cli/commands/system/cli_system_header.sh | 2 + scripts/docker/health/docker_health_heal.sh | 83 +++++ scripts/docker/health/docker_health_scan.sh | 97 +++++ scripts/source/files/arrays/files_docker.sh | 2 + scripts/source/files/arrays/files_webui.sh | 1 + .../source/files/arrays/function_manifest.sh | 21 ++ scripts/task/crontab_task_processor.sh | 6 + .../generators/system/webui_system_health.sh | 131 +++++++ 15 files changed, 973 insertions(+), 1 deletion(-) create mode 100644 containers/libreportal/frontend/core/health-notifier/css/health-notifier.css create mode 100644 containers/libreportal/frontend/core/health-notifier/js/health-notifier.js create mode 100644 scripts/docker/health/docker_health_heal.sh create mode 100644 scripts/docker/health/docker_health_scan.sh create mode 100644 scripts/webui/data/generators/system/webui_system_health.sh diff --git a/containers/libreportal/docker-compose.yml b/containers/libreportal/docker-compose.yml index 537e634..162c97a 100644 --- a/containers/libreportal/docker-compose.yml +++ b/containers/libreportal/docker-compose.yml @@ -10,6 +10,10 @@ services: build: context: . image: libreportal-service:latest + # The control plane self-recovers after a host reboot or a rootless-daemon + # recycle (the health-heal path). "no" would leave the WebUI down until + # someone started it by hand — exactly the outage this policy prevents. + restart: unless-stopped user: "USER_DATA" #LIBREPORTAL|USER_TAG|USER_DATA group_add: - SOCKET_GID_DATA #LIBREPORTAL|SOCKET_GID_TAG|SOCKET_GID_DATA diff --git a/containers/libreportal/frontend/core/boot/js/system-loader.js b/containers/libreportal/frontend/core/boot/js/system-loader.js index 5701d2a..a71c205 100755 --- a/containers/libreportal/frontend/core/boot/js/system-loader.js +++ b/containers/libreportal/frontend/core/boot/js/system-loader.js @@ -70,6 +70,7 @@ class SystemLoader { '/core/topbar/js/topbar.js', '/core/update-notifier/js/update-notifier.js', '/core/network-notifier/js/network-notifier.js', + '/core/health-notifier/js/health-notifier.js', '/core/topbar/js/mobile-menu.js' ] }); diff --git a/containers/libreportal/frontend/core/health-notifier/css/health-notifier.css b/containers/libreportal/frontend/core/health-notifier/css/health-notifier.css new file mode 100644 index 0000000..f9f8099 --- /dev/null +++ b/containers/libreportal/frontend/core/health-notifier/css/health-notifier.css @@ -0,0 +1,247 @@ +/* Health Notifier — topbar badge, dashboard banner, and details panel. + Driven by js/health-notifier.js and /data/system/health_status.json. Uses a + warm amber-orange identity hue (--page-health) so a "control plane needs + attention" signal reads distinctly from the amber update pill and the rose + network badge. Falls back gracefully if the token is absent. */ + +/* ---- Topbar badge -------------------------------------------------------- */ + +.health-badge { + display: inline-flex; + align-items: center; + gap: 6px; + padding: 7px 12px; + border: 1px solid var(--page-health, #d9822b); + border-radius: 6px; + background: rgba(var(--page-health-rgb, 217, 130, 43), 0.14); + color: var(--page-health, #d9822b); + font-weight: 600; + font-size: 0.85rem; + cursor: pointer; + transition: background 0.2s, transform 0.1s; + white-space: nowrap; +} + +.health-badge:hover { background: rgba(var(--page-health-rgb, 217, 130, 43), 0.24); } +.health-badge:active { transform: scale(0.97); } + +.health-badge-dot { + width: 8px; + height: 8px; + border-radius: 50%; + background: var(--page-health, #d9822b); + box-shadow: 0 0 0 0 rgba(var(--page-health-rgb, 217, 130, 43), 0.6); + animation: health-badge-pulse 2s infinite; +} + +@keyframes health-badge-pulse { + 0% { box-shadow: 0 0 0 0 rgba(var(--page-health-rgb, 217, 130, 43), 0.6); } + 70% { box-shadow: 0 0 0 7px rgba(var(--page-health-rgb, 217, 130, 43), 0); } + 100% { box-shadow: 0 0 0 0 rgba(var(--page-health-rgb, 217, 130, 43), 0); } +} + +@media (prefers-reduced-motion: reduce) { + .health-badge-dot { animation: none; } +} + +/* ---- Dashboard banner ---------------------------------------------------- */ + +.health-banner { + display: flex; + align-items: center; + gap: 16px; + margin-bottom: 20px; + padding: 16px 20px; + border: 1px solid var(--page-health, #d9822b); + border-left-width: 4px; + border-radius: 10px; + background: rgba(var(--page-health-rgb, 217, 130, 43), 0.1); + color: var(--text-primary, #fff); +} + +.health-banner-icon { + display: flex; + align-items: center; + justify-content: center; + flex: 0 0 auto; + color: var(--page-health, #d9822b); +} + +.health-banner-text { flex: 1 1 auto; min-width: 0; } +.health-banner-title { font-weight: 700; font-size: 1rem; } + +.health-banner-sub { + margin-top: 2px; + font-size: 0.85rem; + color: var(--text-muted, #9aa); +} + +.health-banner-actions { + display: flex; + gap: 8px; + flex: 0 0 auto; +} + +/* ---- Shared action buttons ----------------------------------------------- */ + +.health-btn-primary, +.health-btn-secondary { + padding: 8px 16px; + border-radius: 6px; + font-weight: 600; + font-size: 0.85rem; + cursor: pointer; + transition: background 0.2s, border-color 0.2s; + border: 1px solid transparent; + white-space: nowrap; +} + +.health-btn-primary { + background: var(--page-health, #d9822b); + color: var(--text-on-accent, #fff); +} + +.health-btn-primary:hover { filter: brightness(1.08); } + +.health-btn-secondary { + background: transparent; + border-color: var(--border-color, rgba(255, 255, 255, 0.2)); + color: var(--text-primary, #fff); +} + +.health-btn-secondary:hover { background: var(--surface-hover, rgba(255, 255, 255, 0.08)); } + +/* ---- Details panel (modal) ----------------------------------------------- */ + +.health-panel-overlay { + position: fixed; + inset: 0; + z-index: 10000; + display: flex; + align-items: center; + justify-content: center; + padding: 20px; + background: rgba(0, 0, 0, 0.55); + backdrop-filter: blur(2px); +} + +.health-panel { + width: 100%; + max-width: 460px; + border: 1px solid var(--card-border, var(--border-color, rgba(255, 255, 255, 0.15))); + border-radius: 12px; + background: var(--card-bg, var(--surface-bg-solid, #1b1f2a)); + box-shadow: var(--card-shadow, 0 20px 60px rgba(0, 0, 0, 0.45)); + color: var(--text-primary, #fff); + overflow: hidden; +} + +.health-panel-header { + display: flex; + align-items: center; + justify-content: space-between; + padding: 16px 20px; + border-bottom: 1px solid var(--border-color, rgba(255, 255, 255, 0.1)); +} + +.health-panel-header h3 { margin: 0; font-size: 1.05rem; } + +.health-panel-close { + border: none; + background: transparent; + color: var(--text-muted, #9aa); + font-size: 1.5rem; + line-height: 1; + cursor: pointer; + padding: 0 4px; +} + +.health-panel-close:hover { color: var(--text-primary, #fff); } + +.health-panel-status { + padding: 14px 20px 0; + font-size: 0.9rem; + color: var(--text-secondary, var(--text-muted, #9aa)); +} + +.health-panel-status.is-issue { + color: var(--page-health, #d9822b); + font-weight: 600; +} + +.health-panel-error { + margin: 12px 20px 0; + padding: 8px 12px; + border-radius: 6px; + font-size: 0.82rem; + background: rgba(var(--status-danger-rgb, 220, 53, 69), 0.12); + color: var(--status-danger, #dc3545); +} + +.health-panel-rows { + margin: 14px 0 0; + padding: 0 20px; +} + +.health-panel-row { + display: flex; + justify-content: space-between; + gap: 16px; + padding: 8px 0; + border-bottom: 1px solid var(--border-subtle, rgba(255, 255, 255, 0.06)); + font-size: 0.88rem; +} + +.health-panel-row:last-child { border-bottom: none; } +.health-panel-row dt { color: var(--text-muted, #9aa); margin: 0; } + +.health-panel-row dd { + margin: 0; + font-weight: 600; + text-align: right; + word-break: break-word; +} + +.health-panel-row dd.is-bad { color: var(--status-danger, #dc3545); } +.health-panel-row dd.is-good { color: var(--status-success, #38a169); } + +/* affected-containers list */ +.health-panel-apps { + margin: 14px 20px 0; + border: 1px solid var(--border-subtle, rgba(255, 255, 255, 0.08)); + border-radius: 8px; + overflow: hidden; +} + +.health-panel-app { + display: flex; + justify-content: space-between; + gap: 12px; + padding: 8px 12px; + font-size: 0.84rem; + border-bottom: 1px solid var(--border-subtle, rgba(255, 255, 255, 0.06)); +} + +.health-panel-app:last-child { border-bottom: none; } +.health-panel-app-name { font-weight: 600; } +.health-panel-app-meta { color: var(--text-muted, #9aa); font-family: var(--font-mono, monospace); } + +.health-panel-note { + margin: 14px 20px 0; + font-size: 0.8rem; + color: var(--text-muted, #9aa); + line-height: 1.45; +} + +.health-panel-actions { + display: flex; + justify-content: flex-end; + gap: 8px; + padding: 18px 20px 20px; +} + +@media (max-width: 600px) { + .health-banner { flex-wrap: wrap; } + .health-banner-actions { width: 100%; } + .health-banner-actions button { flex: 1 1 auto; } +} diff --git a/containers/libreportal/frontend/core/health-notifier/js/health-notifier.js b/containers/libreportal/frontend/core/health-notifier/js/health-notifier.js new file mode 100644 index 0000000..f6a7d83 --- /dev/null +++ b/containers/libreportal/frontend/core/health-notifier/js/health-notifier.js @@ -0,0 +1,336 @@ +// Health Notifier +// ----------------------------------------------------------------------------- +// Surfaces control-plane health across the WebUI — the rootless docker layer +// going bad in a way that threatens the WebUI itself: its published host port +// torn down (container healthy inside, unreachable outside), or a container +// crash-looping and churning the shared network. Two surfaces, shown ONLY when +// there's a real issue to act on: +// * a badge in the global topbar (after the update pill / network badge), and +// * a banner on the dashboard. +// Both are driven by /data/system/health_status.json, written host-side by +// webuiSystemHealthCheck (scripts/webui/data/generators/system/). +// +// The backend already SELF-HEALS most of this (the task-processor poll dispatches +// `system health heal` when the WebUI's own port is down — you can't click a +// button on a dead WebUI). These surfaces are for visibility and for a manual +// "Repair now" when the WebUI is still reachable (e.g. a crash-loop caught early). +// +// Actions go through the normal task pipeline so progress streams on Tasks: +// * "Repair now" -> task `libreportal system health heal` +// * "Re-check" -> task `libreportal system health check` +// +// This file owns no detection logic — it only reads the status file. + +class HealthNotifier { + constructor() { + this.status = null; + this.fetching = null; + this.pollMs = 60 * 1000; // re-read the status file every minute + this.pollTimer = null; + this.started = false; + } + + // ---- data ---------------------------------------------------------------- + + async fetchStatus() { + if (this.fetching) return this.fetching; + this.fetching = (async () => { + try { + const s = await fetch('/data/system/health_status.json', { cache: 'no-store' }) + .then(r => r.ok ? r.json() : null).catch(() => null); + if (s !== null) this.status = s; // keep last-good on a failed fetch + return this.status; + } catch { + return null; + } finally { + this.fetching = null; + } + })(); + return this.fetching; + } + + async refresh() { + await this.fetchStatus(); + this.renderTopbarBadge(); + this.renderDashboardBanner(); + } + + // ---- lifecycle ----------------------------------------------------------- + + start() { + if (this.started) return; + this.started = true; + + this.refresh(); + + // Topbar HTML and this script load independently; retry until + // .topbar-controls exists so the badge appears regardless of the race. + let tries = 0; + const ensure = setInterval(() => { + if (document.querySelector('.topbar-controls')) { this.renderTopbarBadge(); clearInterval(ensure); } + else if (++tries > 30) clearInterval(ensure); // ~15s ceiling + }, 500); + + if (this.pollTimer) clearInterval(this.pollTimer); + this.pollTimer = setInterval(() => this.refresh(), this.pollMs); + + // Re-read the status as soon as a health heal/check task finishes so the + // badge clears without waiting for the next poll. + window.taskRefresh?.register({ + id: 'health-badge', + match: (d) => d.action === 'system_health_heal' + || /^libreportal system health\b/.test((d.task && d.task.command) || d.command || ''), + run: () => this.refresh(), + debounceMs: 1500, + }); + } + + // Called by TopbarComponent.init() once the topbar DOM exists. + onTopbarReady() { + this.renderTopbarBadge(); + this.refresh(); + } + + _hasIssues() { return !!(this.status && this.status.issues_found === true); } + + // ---- topbar badge -------------------------------------------------------- + + renderTopbarBadge() { + const controls = document.querySelector('.topbar-controls'); + if (!controls) return; + + let badge = document.getElementById('health-badge'); + + if (!this._hasIssues()) { + if (badge) badge.remove(); + return; + } + + if (!badge) { + badge = document.createElement('button'); + badge.id = 'health-badge'; + badge.className = 'health-badge'; + badge.type = 'button'; + badge.addEventListener('click', () => this.openPanel()); + // Sit after the network badge (or the update pill) so the pills keep a + // stable order — update, network, health — rather than racing firstChild. + const anchor = document.getElementById('network-badge') || document.getElementById('update-pill'); + if (anchor && anchor.parentNode === controls) controls.insertBefore(badge, anchor.nextSibling); + else controls.insertBefore(badge, controls.firstChild); + } + + const n = Array.isArray(this.status.crash_loops) ? this.status.crash_loops.length : 0; + badge.title = 'Control plane needs attention'; + badge.setAttribute('aria-label', badge.title); + badge.innerHTML = ` + + + System`; + } + + // ---- dashboard banner ---------------------------------------------------- + + renderDashboardBanner() { + const main = document.querySelector('.dashboard-main'); + if (!main) return; // not on the dashboard + + let banner = document.getElementById('health-banner'); + + // Attention-only: render nothing unless there's a real issue. + if (!this._hasIssues()) { + if (banner) banner.remove(); + return; + } + + if (!banner) { + banner = document.createElement('div'); + banner.id = 'health-banner'; + // Attention-only, so it leads the dashboard when an issue is present. If + // the network banner is also present, sit just after it. + const netBanner = document.getElementById('network-banner'); + if (netBanner && netBanner.parentNode === main) main.insertBefore(banner, netBanner.nextSibling); + else main.insertBefore(banner, main.firstChild); + } + + const s = this.status; + const sub = s.summary ? this._escape(s.summary) : 'The docker control plane needs attention'; + + const heartIcon = ` + + + `; + + banner.className = 'health-banner'; + banner.innerHTML = ` + +
+
Control plane attention needed
+
${sub}
+
+
+ + ${s.can_auto_heal ? '' : ''} +
`; + + const details = banner.querySelector('#health-banner-details'); + if (details) details.addEventListener('click', () => this.openPanel()); + const healBtn = banner.querySelector('#health-banner-heal'); + if (healBtn) healBtn.addEventListener('click', () => this.runHeal()); + } + + // ---- details panel (self-contained modal) -------------------------------- + + openPanel() { + this.closePanel(); + const s = this.status || {}; + const webui = s.webui || {}; + const loops = Array.isArray(s.crash_loops) ? s.crash_loops : []; + + const overlay = document.createElement('div'); + overlay.id = 'health-panel-overlay'; + overlay.className = 'health-panel-overlay'; + overlay.addEventListener('click', (e) => { if (e.target === overlay) this.closePanel(); }); + + const reachTxt = webui.reachable === true ? 'reachable' + : webui.reachable === false ? 'unreachable' + : 'unknown'; + const reachCls = webui.reachable === true ? 'is-good' + : webui.reachable === false ? 'is-bad' : ''; + + const rows = [ + ['Docker daemon', s.daemon_ok ? 'up' : 'unreachable', s.daemon_ok ? 'is-good' : 'is-bad'], + ['WebUI container', webui.running ? 'running' : (webui.present ? 'not running' : 'missing'), + webui.running ? 'is-good' : 'is-bad'], + ['WebUI host port', webui.port ? `${webui.port} (${reachTxt})` : reachTxt, reachCls], + ['Crash-loops', String(loops.length), loops.length ? 'is-bad' : ''], + ['Last checked', this._formatTime(s.checked_at), ''], + ]; + + const loopList = loops.length + ? `
+ ${loops.map(l => ` +
+ ${this._escape(l.container || l.app || '')} + restarts: ${this._escape(String(l.restart_count ?? '?'))} +
`).join('')} +
` + : ''; + + const statusLine = this._hasIssues() + ? (s.summary || 'The docker control plane needs attention.') + : 'Control plane healthy.'; + + overlay.innerHTML = ` + `; + + document.body.appendChild(overlay); + overlay.querySelector('#health-panel-close').addEventListener('click', () => this.closePanel()); + overlay.querySelector('#health-panel-check').addEventListener('click', () => this.checkNow()); + const heal = overlay.querySelector('#health-panel-heal'); + if (heal) heal.addEventListener('click', () => this.runHeal()); + + this._escHandler = (e) => { if (e.key === 'Escape') this.closePanel(); }; + document.addEventListener('keydown', this._escHandler); + } + + closePanel() { + const overlay = document.getElementById('health-panel-overlay'); + if (overlay) overlay.remove(); + if (this._escHandler) { + document.removeEventListener('keydown', this._escHandler); + this._escHandler = null; + } + } + + // ---- actions ------------------------------------------------------------- + + async runHeal() { + this.closePanel(); + try { + await this._createTask('libreportal system health heal'); + this._toast('Control-plane repair started — follow progress on the Tasks page.', 'info'); + this._goToTasks(); + } catch (e) { + this._toast('Could not start the repair: ' + e.message, 'error'); + } + } + + async checkNow() { + try { + await this._createTask('libreportal system health check'); + this._toast('Re-checking control-plane health…', 'info'); + setTimeout(() => this.refresh(), 4000); + } catch (e) { + this._toast('Could not re-check health: ' + e.message, 'error'); + } + } + + // ---- helpers ------------------------------------------------------------- + + async _createTask(command) { + if (window.tasksManager?.taskManager?.createTask) { + return window.tasksManager.taskManager.createTask(command, 'system_health_heal', null, ''); + } + if (typeof TaskManager !== 'undefined') { + return new TaskManager().createTask(command, 'system_health_heal', null, ''); + } + const res = await fetch('/api/tasks', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ command, type: 'system_health_heal', app: null, config: '' }) + }); + if (!res.ok) throw new Error(`HTTP ${res.status}`); + return res.json(); + } + + _goToTasks() { + if (window.librePortalSPA?.navigate) window.librePortalSPA.navigate('/tasks'); + else if (typeof navigateToRoute === 'function') navigateToRoute('/tasks'); + else window.location.href = '/tasks'; + } + + _toast(message, type = 'info') { + const ns = window.notificationSystem || window.ensureNotificationSystem?.(); + if (ns?.show) ns.show(message, type); + else console.log(`[health] ${message}`); + } + + _formatTime(iso) { + if (!iso) return '—'; + const d = new Date(iso); + if (isNaN(d.getTime())) return '—'; + return d.toLocaleString(); + } + + _escape(str) { + return String(str).replace(/[&<>"']/g, (c) => ( + { '&': '&', '<': '<', '>': '>', '"': '"', "'": ''' }[c] + )); + } +} + +window.healthNotifier = window.healthNotifier || new HealthNotifier(); + +if (document.readyState === 'loading') { + document.addEventListener('DOMContentLoaded', () => window.healthNotifier.start()); +} else { + window.healthNotifier.start(); +} diff --git a/containers/libreportal/frontend/index.html b/containers/libreportal/frontend/index.html index 4230f46..478e049 100755 --- a/containers/libreportal/frontend/index.html +++ b/containers/libreportal/frontend/index.html @@ -40,6 +40,7 @@ +