Merge claude/1
This commit is contained in:
librelad
commit
57f800ca1f
@ -10,6 +10,10 @@ services:
|
||||
build:
|
||||
context: .
|
||||
image: libreportal-service:latest
|
||||
# The control plane self-recovers after a host reboot or a rootless-daemon
|
||||
# recycle (the health-heal path). "no" would leave the WebUI down until
|
||||
# someone started it by hand — exactly the outage this policy prevents.
|
||||
restart: unless-stopped
|
||||
user: "USER_DATA" #LIBREPORTAL|USER_TAG|USER_DATA
|
||||
group_add:
|
||||
- SOCKET_GID_DATA #LIBREPORTAL|SOCKET_GID_TAG|SOCKET_GID_DATA
|
||||
|
||||
@ -70,6 +70,7 @@ class SystemLoader {
|
||||
'/core/topbar/js/topbar.js',
|
||||
'/core/update-notifier/js/update-notifier.js',
|
||||
'/core/network-notifier/js/network-notifier.js',
|
||||
'/core/health-notifier/js/health-notifier.js',
|
||||
'/core/topbar/js/mobile-menu.js'
|
||||
]
|
||||
});
|
||||
|
||||
@ -0,0 +1,247 @@
|
||||
/* Health Notifier — topbar badge, dashboard banner, and details panel.
|
||||
Driven by js/health-notifier.js and /data/system/health_status.json. Uses a
|
||||
warm amber-orange identity hue (--page-health) so a "control plane needs
|
||||
attention" signal reads distinctly from the amber update pill and the rose
|
||||
network badge. Falls back gracefully if the token is absent. */
|
||||
|
||||
/* ---- Topbar badge -------------------------------------------------------- */
|
||||
|
||||
.health-badge {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
gap: 6px;
|
||||
padding: 7px 12px;
|
||||
border: 1px solid var(--page-health, #d9822b);
|
||||
border-radius: 6px;
|
||||
background: rgba(var(--page-health-rgb, 217, 130, 43), 0.14);
|
||||
color: var(--page-health, #d9822b);
|
||||
font-weight: 600;
|
||||
font-size: 0.85rem;
|
||||
cursor: pointer;
|
||||
transition: background 0.2s, transform 0.1s;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.health-badge:hover { background: rgba(var(--page-health-rgb, 217, 130, 43), 0.24); }
|
||||
.health-badge:active { transform: scale(0.97); }
|
||||
|
||||
.health-badge-dot {
|
||||
width: 8px;
|
||||
height: 8px;
|
||||
border-radius: 50%;
|
||||
background: var(--page-health, #d9822b);
|
||||
box-shadow: 0 0 0 0 rgba(var(--page-health-rgb, 217, 130, 43), 0.6);
|
||||
animation: health-badge-pulse 2s infinite;
|
||||
}
|
||||
|
||||
@keyframes health-badge-pulse {
|
||||
0% { box-shadow: 0 0 0 0 rgba(var(--page-health-rgb, 217, 130, 43), 0.6); }
|
||||
70% { box-shadow: 0 0 0 7px rgba(var(--page-health-rgb, 217, 130, 43), 0); }
|
||||
100% { box-shadow: 0 0 0 0 rgba(var(--page-health-rgb, 217, 130, 43), 0); }
|
||||
}
|
||||
|
||||
@media (prefers-reduced-motion: reduce) {
|
||||
.health-badge-dot { animation: none; }
|
||||
}
|
||||
|
||||
/* ---- Dashboard banner ---------------------------------------------------- */
|
||||
|
||||
.health-banner {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 16px;
|
||||
margin-bottom: 20px;
|
||||
padding: 16px 20px;
|
||||
border: 1px solid var(--page-health, #d9822b);
|
||||
border-left-width: 4px;
|
||||
border-radius: 10px;
|
||||
background: rgba(var(--page-health-rgb, 217, 130, 43), 0.1);
|
||||
color: var(--text-primary, #fff);
|
||||
}
|
||||
|
||||
.health-banner-icon {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
flex: 0 0 auto;
|
||||
color: var(--page-health, #d9822b);
|
||||
}
|
||||
|
||||
.health-banner-text { flex: 1 1 auto; min-width: 0; }
|
||||
.health-banner-title { font-weight: 700; font-size: 1rem; }
|
||||
|
||||
.health-banner-sub {
|
||||
margin-top: 2px;
|
||||
font-size: 0.85rem;
|
||||
color: var(--text-muted, #9aa);
|
||||
}
|
||||
|
||||
.health-banner-actions {
|
||||
display: flex;
|
||||
gap: 8px;
|
||||
flex: 0 0 auto;
|
||||
}
|
||||
|
||||
/* ---- Shared action buttons ----------------------------------------------- */
|
||||
|
||||
.health-btn-primary,
|
||||
.health-btn-secondary {
|
||||
padding: 8px 16px;
|
||||
border-radius: 6px;
|
||||
font-weight: 600;
|
||||
font-size: 0.85rem;
|
||||
cursor: pointer;
|
||||
transition: background 0.2s, border-color 0.2s;
|
||||
border: 1px solid transparent;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.health-btn-primary {
|
||||
background: var(--page-health, #d9822b);
|
||||
color: var(--text-on-accent, #fff);
|
||||
}
|
||||
|
||||
.health-btn-primary:hover { filter: brightness(1.08); }
|
||||
|
||||
.health-btn-secondary {
|
||||
background: transparent;
|
||||
border-color: var(--border-color, rgba(255, 255, 255, 0.2));
|
||||
color: var(--text-primary, #fff);
|
||||
}
|
||||
|
||||
.health-btn-secondary:hover { background: var(--surface-hover, rgba(255, 255, 255, 0.08)); }
|
||||
|
||||
/* ---- Details panel (modal) ----------------------------------------------- */
|
||||
|
||||
.health-panel-overlay {
|
||||
position: fixed;
|
||||
inset: 0;
|
||||
z-index: 10000;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
padding: 20px;
|
||||
background: rgba(0, 0, 0, 0.55);
|
||||
backdrop-filter: blur(2px);
|
||||
}
|
||||
|
||||
.health-panel {
|
||||
width: 100%;
|
||||
max-width: 460px;
|
||||
border: 1px solid var(--card-border, var(--border-color, rgba(255, 255, 255, 0.15)));
|
||||
border-radius: 12px;
|
||||
background: var(--card-bg, var(--surface-bg-solid, #1b1f2a));
|
||||
box-shadow: var(--card-shadow, 0 20px 60px rgba(0, 0, 0, 0.45));
|
||||
color: var(--text-primary, #fff);
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
.health-panel-header {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
padding: 16px 20px;
|
||||
border-bottom: 1px solid var(--border-color, rgba(255, 255, 255, 0.1));
|
||||
}
|
||||
|
||||
.health-panel-header h3 { margin: 0; font-size: 1.05rem; }
|
||||
|
||||
.health-panel-close {
|
||||
border: none;
|
||||
background: transparent;
|
||||
color: var(--text-muted, #9aa);
|
||||
font-size: 1.5rem;
|
||||
line-height: 1;
|
||||
cursor: pointer;
|
||||
padding: 0 4px;
|
||||
}
|
||||
|
||||
.health-panel-close:hover { color: var(--text-primary, #fff); }
|
||||
|
||||
.health-panel-status {
|
||||
padding: 14px 20px 0;
|
||||
font-size: 0.9rem;
|
||||
color: var(--text-secondary, var(--text-muted, #9aa));
|
||||
}
|
||||
|
||||
.health-panel-status.is-issue {
|
||||
color: var(--page-health, #d9822b);
|
||||
font-weight: 600;
|
||||
}
|
||||
|
||||
.health-panel-error {
|
||||
margin: 12px 20px 0;
|
||||
padding: 8px 12px;
|
||||
border-radius: 6px;
|
||||
font-size: 0.82rem;
|
||||
background: rgba(var(--status-danger-rgb, 220, 53, 69), 0.12);
|
||||
color: var(--status-danger, #dc3545);
|
||||
}
|
||||
|
||||
.health-panel-rows {
|
||||
margin: 14px 0 0;
|
||||
padding: 0 20px;
|
||||
}
|
||||
|
||||
.health-panel-row {
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
gap: 16px;
|
||||
padding: 8px 0;
|
||||
border-bottom: 1px solid var(--border-subtle, rgba(255, 255, 255, 0.06));
|
||||
font-size: 0.88rem;
|
||||
}
|
||||
|
||||
.health-panel-row:last-child { border-bottom: none; }
|
||||
.health-panel-row dt { color: var(--text-muted, #9aa); margin: 0; }
|
||||
|
||||
.health-panel-row dd {
|
||||
margin: 0;
|
||||
font-weight: 600;
|
||||
text-align: right;
|
||||
word-break: break-word;
|
||||
}
|
||||
|
||||
.health-panel-row dd.is-bad { color: var(--status-danger, #dc3545); }
|
||||
.health-panel-row dd.is-good { color: var(--status-success, #38a169); }
|
||||
|
||||
/* affected-containers list */
|
||||
.health-panel-apps {
|
||||
margin: 14px 20px 0;
|
||||
border: 1px solid var(--border-subtle, rgba(255, 255, 255, 0.08));
|
||||
border-radius: 8px;
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
.health-panel-app {
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
gap: 12px;
|
||||
padding: 8px 12px;
|
||||
font-size: 0.84rem;
|
||||
border-bottom: 1px solid var(--border-subtle, rgba(255, 255, 255, 0.06));
|
||||
}
|
||||
|
||||
.health-panel-app:last-child { border-bottom: none; }
|
||||
.health-panel-app-name { font-weight: 600; }
|
||||
.health-panel-app-meta { color: var(--text-muted, #9aa); font-family: var(--font-mono, monospace); }
|
||||
|
||||
.health-panel-note {
|
||||
margin: 14px 20px 0;
|
||||
font-size: 0.8rem;
|
||||
color: var(--text-muted, #9aa);
|
||||
line-height: 1.45;
|
||||
}
|
||||
|
||||
.health-panel-actions {
|
||||
display: flex;
|
||||
justify-content: flex-end;
|
||||
gap: 8px;
|
||||
padding: 18px 20px 20px;
|
||||
}
|
||||
|
||||
@media (max-width: 600px) {
|
||||
.health-banner { flex-wrap: wrap; }
|
||||
.health-banner-actions { width: 100%; }
|
||||
.health-banner-actions button { flex: 1 1 auto; }
|
||||
}
|
||||
@ -0,0 +1,336 @@
|
||||
// Health Notifier
|
||||
// -----------------------------------------------------------------------------
|
||||
// Surfaces control-plane health across the WebUI — the rootless docker layer
|
||||
// going bad in a way that threatens the WebUI itself: its published host port
|
||||
// torn down (container healthy inside, unreachable outside), or a container
|
||||
// crash-looping and churning the shared network. Two surfaces, shown ONLY when
|
||||
// there's a real issue to act on:
|
||||
// * a badge in the global topbar (after the update pill / network badge), and
|
||||
// * a banner on the dashboard.
|
||||
// Both are driven by /data/system/health_status.json, written host-side by
|
||||
// webuiSystemHealthCheck (scripts/webui/data/generators/system/).
|
||||
//
|
||||
// The backend already SELF-HEALS most of this (the task-processor poll dispatches
|
||||
// `system health heal` when the WebUI's own port is down — you can't click a
|
||||
// button on a dead WebUI). These surfaces are for visibility and for a manual
|
||||
// "Repair now" when the WebUI is still reachable (e.g. a crash-loop caught early).
|
||||
//
|
||||
// Actions go through the normal task pipeline so progress streams on Tasks:
|
||||
// * "Repair now" -> task `libreportal system health heal`
|
||||
// * "Re-check" -> task `libreportal system health check`
|
||||
//
|
||||
// This file owns no detection logic — it only reads the status file.
|
||||
|
||||
class HealthNotifier {
|
||||
constructor() {
|
||||
this.status = null;
|
||||
this.fetching = null;
|
||||
this.pollMs = 60 * 1000; // re-read the status file every minute
|
||||
this.pollTimer = null;
|
||||
this.started = false;
|
||||
}
|
||||
|
||||
// ---- data ----------------------------------------------------------------
|
||||
|
||||
async fetchStatus() {
|
||||
if (this.fetching) return this.fetching;
|
||||
this.fetching = (async () => {
|
||||
try {
|
||||
const s = await fetch('/data/system/health_status.json', { cache: 'no-store' })
|
||||
.then(r => r.ok ? r.json() : null).catch(() => null);
|
||||
if (s !== null) this.status = s; // keep last-good on a failed fetch
|
||||
return this.status;
|
||||
} catch {
|
||||
return null;
|
||||
} finally {
|
||||
this.fetching = null;
|
||||
}
|
||||
})();
|
||||
return this.fetching;
|
||||
}
|
||||
|
||||
async refresh() {
|
||||
await this.fetchStatus();
|
||||
this.renderTopbarBadge();
|
||||
this.renderDashboardBanner();
|
||||
}
|
||||
|
||||
// ---- lifecycle -----------------------------------------------------------
|
||||
|
||||
start() {
|
||||
if (this.started) return;
|
||||
this.started = true;
|
||||
|
||||
this.refresh();
|
||||
|
||||
// Topbar HTML and this script load independently; retry until
|
||||
// .topbar-controls exists so the badge appears regardless of the race.
|
||||
let tries = 0;
|
||||
const ensure = setInterval(() => {
|
||||
if (document.querySelector('.topbar-controls')) { this.renderTopbarBadge(); clearInterval(ensure); }
|
||||
else if (++tries > 30) clearInterval(ensure); // ~15s ceiling
|
||||
}, 500);
|
||||
|
||||
if (this.pollTimer) clearInterval(this.pollTimer);
|
||||
this.pollTimer = setInterval(() => this.refresh(), this.pollMs);
|
||||
|
||||
// Re-read the status as soon as a health heal/check task finishes so the
|
||||
// badge clears without waiting for the next poll.
|
||||
window.taskRefresh?.register({
|
||||
id: 'health-badge',
|
||||
match: (d) => d.action === 'system_health_heal'
|
||||
|| /^libreportal system health\b/.test((d.task && d.task.command) || d.command || ''),
|
||||
run: () => this.refresh(),
|
||||
debounceMs: 1500,
|
||||
});
|
||||
}
|
||||
|
||||
// Called by TopbarComponent.init() once the topbar DOM exists.
|
||||
onTopbarReady() {
|
||||
this.renderTopbarBadge();
|
||||
this.refresh();
|
||||
}
|
||||
|
||||
_hasIssues() { return !!(this.status && this.status.issues_found === true); }
|
||||
|
||||
// ---- topbar badge --------------------------------------------------------
|
||||
|
||||
renderTopbarBadge() {
|
||||
const controls = document.querySelector('.topbar-controls');
|
||||
if (!controls) return;
|
||||
|
||||
let badge = document.getElementById('health-badge');
|
||||
|
||||
if (!this._hasIssues()) {
|
||||
if (badge) badge.remove();
|
||||
return;
|
||||
}
|
||||
|
||||
if (!badge) {
|
||||
badge = document.createElement('button');
|
||||
badge.id = 'health-badge';
|
||||
badge.className = 'health-badge';
|
||||
badge.type = 'button';
|
||||
badge.addEventListener('click', () => this.openPanel());
|
||||
// Sit after the network badge (or the update pill) so the pills keep a
|
||||
// stable order — update, network, health — rather than racing firstChild.
|
||||
const anchor = document.getElementById('network-badge') || document.getElementById('update-pill');
|
||||
if (anchor && anchor.parentNode === controls) controls.insertBefore(badge, anchor.nextSibling);
|
||||
else controls.insertBefore(badge, controls.firstChild);
|
||||
}
|
||||
|
||||
const n = Array.isArray(this.status.crash_loops) ? this.status.crash_loops.length : 0;
|
||||
badge.title = 'Control plane needs attention';
|
||||
badge.setAttribute('aria-label', badge.title);
|
||||
badge.innerHTML = `
|
||||
<span class="health-badge-dot" aria-hidden="true"></span>
|
||||
<svg width="15" height="15" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" aria-hidden="true">
|
||||
<path d="M22 12h-4l-3 9L9 3l-3 9H2"></path>
|
||||
</svg>
|
||||
<span class="health-badge-text">System</span>`;
|
||||
}
|
||||
|
||||
// ---- dashboard banner ----------------------------------------------------
|
||||
|
||||
renderDashboardBanner() {
|
||||
const main = document.querySelector('.dashboard-main');
|
||||
if (!main) return; // not on the dashboard
|
||||
|
||||
let banner = document.getElementById('health-banner');
|
||||
|
||||
// Attention-only: render nothing unless there's a real issue.
|
||||
if (!this._hasIssues()) {
|
||||
if (banner) banner.remove();
|
||||
return;
|
||||
}
|
||||
|
||||
if (!banner) {
|
||||
banner = document.createElement('div');
|
||||
banner.id = 'health-banner';
|
||||
// Attention-only, so it leads the dashboard when an issue is present. If
|
||||
// the network banner is also present, sit just after it.
|
||||
const netBanner = document.getElementById('network-banner');
|
||||
if (netBanner && netBanner.parentNode === main) main.insertBefore(banner, netBanner.nextSibling);
|
||||
else main.insertBefore(banner, main.firstChild);
|
||||
}
|
||||
|
||||
const s = this.status;
|
||||
const sub = s.summary ? this._escape(s.summary) : 'The docker control plane needs attention';
|
||||
|
||||
const heartIcon = `
|
||||
<svg width="22" height="22" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2">
|
||||
<path d="M22 12h-4l-3 9L9 3l-3 9H2"></path>
|
||||
</svg>`;
|
||||
|
||||
banner.className = 'health-banner';
|
||||
banner.innerHTML = `
|
||||
<div class="health-banner-icon" aria-hidden="true">${heartIcon}</div>
|
||||
<div class="health-banner-text">
|
||||
<div class="health-banner-title">Control plane attention needed</div>
|
||||
<div class="health-banner-sub">${sub}</div>
|
||||
</div>
|
||||
<div class="health-banner-actions">
|
||||
<button type="button" class="health-btn-secondary" id="health-banner-details">Details</button>
|
||||
${s.can_auto_heal ? '<button type="button" class="health-btn-primary" id="health-banner-heal">Repair now</button>' : ''}
|
||||
</div>`;
|
||||
|
||||
const details = banner.querySelector('#health-banner-details');
|
||||
if (details) details.addEventListener('click', () => this.openPanel());
|
||||
const healBtn = banner.querySelector('#health-banner-heal');
|
||||
if (healBtn) healBtn.addEventListener('click', () => this.runHeal());
|
||||
}
|
||||
|
||||
// ---- details panel (self-contained modal) --------------------------------
|
||||
|
||||
openPanel() {
|
||||
this.closePanel();
|
||||
const s = this.status || {};
|
||||
const webui = s.webui || {};
|
||||
const loops = Array.isArray(s.crash_loops) ? s.crash_loops : [];
|
||||
|
||||
const overlay = document.createElement('div');
|
||||
overlay.id = 'health-panel-overlay';
|
||||
overlay.className = 'health-panel-overlay';
|
||||
overlay.addEventListener('click', (e) => { if (e.target === overlay) this.closePanel(); });
|
||||
|
||||
const reachTxt = webui.reachable === true ? 'reachable'
|
||||
: webui.reachable === false ? 'unreachable'
|
||||
: 'unknown';
|
||||
const reachCls = webui.reachable === true ? 'is-good'
|
||||
: webui.reachable === false ? 'is-bad' : '';
|
||||
|
||||
const rows = [
|
||||
['Docker daemon', s.daemon_ok ? 'up' : 'unreachable', s.daemon_ok ? 'is-good' : 'is-bad'],
|
||||
['WebUI container', webui.running ? 'running' : (webui.present ? 'not running' : 'missing'),
|
||||
webui.running ? 'is-good' : 'is-bad'],
|
||||
['WebUI host port', webui.port ? `${webui.port} (${reachTxt})` : reachTxt, reachCls],
|
||||
['Crash-loops', String(loops.length), loops.length ? 'is-bad' : ''],
|
||||
['Last checked', this._formatTime(s.checked_at), ''],
|
||||
];
|
||||
|
||||
const loopList = loops.length
|
||||
? `<div class="health-panel-apps">
|
||||
${loops.map(l => `
|
||||
<div class="health-panel-app">
|
||||
<span class="health-panel-app-name">${this._escape(l.container || l.app || '')}</span>
|
||||
<span class="health-panel-app-meta">restarts: ${this._escape(String(l.restart_count ?? '?'))}</span>
|
||||
</div>`).join('')}
|
||||
</div>`
|
||||
: '';
|
||||
|
||||
const statusLine = this._hasIssues()
|
||||
? (s.summary || 'The docker control plane needs attention.')
|
||||
: 'Control plane healthy.';
|
||||
|
||||
overlay.innerHTML = `
|
||||
<div class="health-panel" role="dialog" aria-modal="true" aria-label="Control plane health">
|
||||
<div class="health-panel-header">
|
||||
<h3>Control Plane Health</h3>
|
||||
<button type="button" class="health-panel-close" id="health-panel-close" aria-label="Close">×</button>
|
||||
</div>
|
||||
<div class="health-panel-status ${this._hasIssues() ? 'is-issue' : ''}">${this._escape(statusLine)}</div>
|
||||
${s.error ? `<div class="health-panel-error">${this._escape(s.error)}</div>` : ''}
|
||||
<dl class="health-panel-rows">
|
||||
${rows.map(([k, v, cls]) => `<div class="health-panel-row"><dt>${this._escape(k)}</dt><dd class="${cls || ''}">${this._escape(v)}</dd></div>`).join('')}
|
||||
</dl>
|
||||
${loopList}
|
||||
${s.can_auto_heal && this._hasIssues() ? '<p class="health-panel-note">Repair stops any crash-looping container and, if the WebUI’s host port is down, restarts it — recycling the rootless docker daemon only if needed. Progress streams on the Tasks page.</p>' : ''}
|
||||
<div class="health-panel-actions">
|
||||
<button type="button" class="health-btn-secondary" id="health-panel-check">Re-check</button>
|
||||
${s.can_auto_heal && this._hasIssues() ? '<button type="button" class="health-btn-primary" id="health-panel-heal">Repair now</button>' : ''}
|
||||
</div>
|
||||
</div>`;
|
||||
|
||||
document.body.appendChild(overlay);
|
||||
overlay.querySelector('#health-panel-close').addEventListener('click', () => this.closePanel());
|
||||
overlay.querySelector('#health-panel-check').addEventListener('click', () => this.checkNow());
|
||||
const heal = overlay.querySelector('#health-panel-heal');
|
||||
if (heal) heal.addEventListener('click', () => this.runHeal());
|
||||
|
||||
this._escHandler = (e) => { if (e.key === 'Escape') this.closePanel(); };
|
||||
document.addEventListener('keydown', this._escHandler);
|
||||
}
|
||||
|
||||
closePanel() {
|
||||
const overlay = document.getElementById('health-panel-overlay');
|
||||
if (overlay) overlay.remove();
|
||||
if (this._escHandler) {
|
||||
document.removeEventListener('keydown', this._escHandler);
|
||||
this._escHandler = null;
|
||||
}
|
||||
}
|
||||
|
||||
// ---- actions -------------------------------------------------------------
|
||||
|
||||
async runHeal() {
|
||||
this.closePanel();
|
||||
try {
|
||||
await this._createTask('libreportal system health heal');
|
||||
this._toast('Control-plane repair started — follow progress on the Tasks page.', 'info');
|
||||
this._goToTasks();
|
||||
} catch (e) {
|
||||
this._toast('Could not start the repair: ' + e.message, 'error');
|
||||
}
|
||||
}
|
||||
|
||||
async checkNow() {
|
||||
try {
|
||||
await this._createTask('libreportal system health check');
|
||||
this._toast('Re-checking control-plane health…', 'info');
|
||||
setTimeout(() => this.refresh(), 4000);
|
||||
} catch (e) {
|
||||
this._toast('Could not re-check health: ' + e.message, 'error');
|
||||
}
|
||||
}
|
||||
|
||||
// ---- helpers -------------------------------------------------------------
|
||||
|
||||
async _createTask(command) {
|
||||
if (window.tasksManager?.taskManager?.createTask) {
|
||||
return window.tasksManager.taskManager.createTask(command, 'system_health_heal', null, '');
|
||||
}
|
||||
if (typeof TaskManager !== 'undefined') {
|
||||
return new TaskManager().createTask(command, 'system_health_heal', null, '');
|
||||
}
|
||||
const res = await fetch('/api/tasks', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ command, type: 'system_health_heal', app: null, config: '' })
|
||||
});
|
||||
if (!res.ok) throw new Error(`HTTP ${res.status}`);
|
||||
return res.json();
|
||||
}
|
||||
|
||||
_goToTasks() {
|
||||
if (window.librePortalSPA?.navigate) window.librePortalSPA.navigate('/tasks');
|
||||
else if (typeof navigateToRoute === 'function') navigateToRoute('/tasks');
|
||||
else window.location.href = '/tasks';
|
||||
}
|
||||
|
||||
_toast(message, type = 'info') {
|
||||
const ns = window.notificationSystem || window.ensureNotificationSystem?.();
|
||||
if (ns?.show) ns.show(message, type);
|
||||
else console.log(`[health] ${message}`);
|
||||
}
|
||||
|
||||
_formatTime(iso) {
|
||||
if (!iso) return '—';
|
||||
const d = new Date(iso);
|
||||
if (isNaN(d.getTime())) return '—';
|
||||
return d.toLocaleString();
|
||||
}
|
||||
|
||||
_escape(str) {
|
||||
return String(str).replace(/[&<>"']/g, (c) => (
|
||||
{ '&': '&', '<': '<', '>': '>', '"': '"', "'": ''' }[c]
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
window.healthNotifier = window.healthNotifier || new HealthNotifier();
|
||||
|
||||
if (document.readyState === 'loading') {
|
||||
document.addEventListener('DOMContentLoaded', () => window.healthNotifier.start());
|
||||
} else {
|
||||
window.healthNotifier.start();
|
||||
}
|
||||
@ -40,6 +40,7 @@
|
||||
<link rel="stylesheet" href="/components/updater/css/updater.css">
|
||||
<link rel="stylesheet" href="/core/update-notifier/css/update-notifier.css">
|
||||
<link rel="stylesheet" href="/core/network-notifier/css/network-notifier.css">
|
||||
<link rel="stylesheet" href="/core/health-notifier/css/health-notifier.css">
|
||||
<script>
|
||||
// Inline data-theme bootstrap — runs before any rendering so the right
|
||||
// palette tokens resolve on first paint. Synchronously injects a
|
||||
|
||||
@ -7,7 +7,22 @@ services:
|
||||
container_name: trivy-service
|
||||
image: aquasec/trivy:latest
|
||||
restart: unless-stopped
|
||||
command: server --listen 0.0.0.0:4954
|
||||
# Keep the container process ALIVE even when the vuln DB can't be fetched.
|
||||
# `trivy server` FATAL-exits if it can't download the DB on first boot (no
|
||||
# internet / registry unreachable); bare, restart:unless-stopped then crash-
|
||||
# loops it, and on rootless docker that restart storm churns the shared
|
||||
# network's port-forwarder until the WebUI's own published port is torn down.
|
||||
# Wrapping the server in a shell retry-loop means the container stays Up and
|
||||
# quietly retries on a backoff instead of exiting — no restart, no churn. The
|
||||
# backend already reports the scanner as db_updating/absent until a DB lands.
|
||||
entrypoint: ["/bin/sh", "-c"]
|
||||
command:
|
||||
- |
|
||||
while true; do
|
||||
trivy server --listen 0.0.0.0:4954
|
||||
echo "[libreportal] trivy server exited ($?) — vuln DB unavailable or offline; staying up, retrying in ${TRIVY_RETRY_SECS:-300}s (scanning paused until the DB is reachable)."
|
||||
sleep "${TRIVY_RETRY_SECS:-300}"
|
||||
done
|
||||
# GLUETUN_OFF_BEGIN
|
||||
ports:
|
||||
- "PORTS_DATA_1" #LIBREPORTAL|PORTS_TAG_1|PORTS_DATA_1
|
||||
|
||||
@ -109,6 +109,31 @@ cliHandleSystemCommands()
|
||||
esac
|
||||
;;
|
||||
|
||||
"health")
|
||||
# libreportal system health check [force] (read-only, rewrites
|
||||
# health_status.json — used by the task-processor poll + WebUI;
|
||||
# self-dispatches a heal when the control plane is unreachable)
|
||||
# libreportal system health heal (mutating — stops
|
||||
# crash-loopers, repairs the WebUI port forward, recycles the
|
||||
# rootless daemon; routes through the task system like network heal)
|
||||
case "$initial_command3" in
|
||||
"check")
|
||||
webuiSystemHealthCheck "${initial_command4:-force}"
|
||||
;;
|
||||
"heal")
|
||||
if [[ "$LIBREPORTAL_TASK_EXEC" == "1" ]]; then
|
||||
dockerHealthHeal
|
||||
else
|
||||
cliTaskRun "libreportal system health heal" "system_health_heal" "" ""
|
||||
fi
|
||||
;;
|
||||
*)
|
||||
isNotice "Invalid health command: $initial_command3"
|
||||
cliShowSystemHelp
|
||||
;;
|
||||
esac
|
||||
;;
|
||||
|
||||
"image")
|
||||
# libreportal system image rm [--force] <comma-separated ids>
|
||||
case "$initial_command3" in
|
||||
|
||||
@ -15,5 +15,7 @@ cliShowSystemHelp()
|
||||
echo " libreportal system image rm [--force] <ids> - Remove specific images (comma-separated ids)"
|
||||
echo " libreportal system network check - Re-scan for apps stranded off the docker subnet"
|
||||
echo " libreportal system network heal [app] - Re-IP stranded apps from the current subnet (ports kept)"
|
||||
echo " libreportal system health check - Check the rootless docker / WebUI control plane"
|
||||
echo " libreportal system health heal - Stop crash-loops + repair the WebUI port forward"
|
||||
echo ""
|
||||
}
|
||||
|
||||
83
scripts/docker/health/docker_health_heal.sh
Normal file
83
scripts/docker/health/docker_health_heal.sh
Normal file
@ -0,0 +1,83 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Rootless-docker / control-plane health heal — the mutating half of the
|
||||
# detector. Runs ONLY through the task system (see cli_system_commands.sh
|
||||
# `health heal`, which enqueues unless LIBREPORTAL_TASK_EXEC=1), never a direct
|
||||
# API. It re-scans before each heavier step so it does the least it can.
|
||||
#
|
||||
# Escalation ladder:
|
||||
# 1) Stop crash-loopers — an app FATAL-ing on every boot (e.g. trivy offline)
|
||||
# churns the rootless port-forwarder; stopping it removes the churn. This is
|
||||
# the failure cap: a broken opt-in app can't take the control plane down.
|
||||
# (unless-stopped honours a manual stop across a daemon recycle, so a
|
||||
# stopped crash-looper stays down and can't resume the churn.)
|
||||
# 2) Restart the WebUI container — re-publishes a mildly-lost port forward.
|
||||
# 3) Recycle the rootless docker daemon — rebuilds the netns/port-forward state
|
||||
# when churn has already corrupted it (a container restart alone won't fix
|
||||
# it), then explicitly starts the core container (it may carry no restart
|
||||
# policy, so a daemon recycle would otherwise leave it down).
|
||||
#
|
||||
# Re-runs the read-only check at the end to rewrite health_status.json (the badge
|
||||
# clears, or stays if anything is still unhealed).
|
||||
#
|
||||
# dockerHealthHeal
|
||||
dockerHealthHeal() {
|
||||
isHeader "Healing docker / control-plane health"
|
||||
|
||||
dockerHealthScan # populate HEALTH_* (call direct, NOT in $(...))
|
||||
|
||||
if [[ "$HEALTH_DAEMON_OK" != "true" ]]; then
|
||||
isError "Docker daemon unreachable — cannot heal from here."
|
||||
declare -f webuiSystemHealthCheck >/dev/null 2>&1 && webuiSystemHealthCheck "force" >/dev/null 2>&1
|
||||
return 1
|
||||
fi
|
||||
|
||||
local rootless="true"
|
||||
[[ "${CFG_DOCKER_INSTALL_TYPE:-rootless}" == "rootless" ]] || rootless="false"
|
||||
|
||||
local acted=0
|
||||
|
||||
# 1) Stop crash-loopers (the failure cap).
|
||||
local row cname
|
||||
for row in "${HEALTH_CRASHLOOPS[@]}"; do
|
||||
cname="${row#*|}"; cname="${cname%%|*}" # app|container|rc -> container
|
||||
[[ "$cname" =~ ^[A-Za-z0-9][A-Za-z0-9_.-]*$ ]] || continue
|
||||
isNotice "Stopping crash-looping container '$cname' (restart storm churning the network)…"
|
||||
dockerCommandRun "docker stop '$cname'" >/dev/null 2>&1
|
||||
((acted++))
|
||||
done
|
||||
|
||||
# 2/3) Restore the WebUI host port if the container is up but unreachable.
|
||||
if [[ "$HEALTH_WEBUI_RUNNING" == "true" && "$HEALTH_WEBUI_REACHABLE" == "false" ]]; then
|
||||
local webui; webui="$(_healthWebuiContainer)"
|
||||
isNotice "WebUI up but host port ${HEALTH_WEBUI_PORT:-?} unreachable — restarting '$webui' to re-publish…"
|
||||
dockerCommandRun "docker restart '$webui'" >/dev/null 2>&1
|
||||
((acted++))
|
||||
sleep 4
|
||||
dockerHealthScan
|
||||
|
||||
if [[ "$HEALTH_WEBUI_REACHABLE" != "true" && "$rootless" == "true" ]]; then
|
||||
isNotice "Still unreachable — recycling the rootless docker daemon to rebuild port-forward state…"
|
||||
dockerCommandRun "systemctl --user restart docker.service" >/dev/null 2>&1
|
||||
sleep 8
|
||||
# The core container may have no restart policy — bring it back explicitly.
|
||||
dockerCommandRun "docker start '$webui'" >/dev/null 2>&1
|
||||
sleep 4
|
||||
((acted++))
|
||||
fi
|
||||
fi
|
||||
|
||||
# Rewrite the status file + report what (if anything) remains.
|
||||
declare -f webuiSystemHealthCheck >/dev/null 2>&1 && webuiSystemHealthCheck "force" >/dev/null 2>&1
|
||||
dockerHealthScan
|
||||
|
||||
if [[ "$HEALTH_WEBUI_RUNNING" == "true" && "$HEALTH_WEBUI_REACHABLE" == "false" ]]; then
|
||||
isError "Health heal ran ${acted} action(s); WebUI still unreachable on port ${HEALTH_WEBUI_PORT:-?} — manual inspection needed."
|
||||
return 1
|
||||
fi
|
||||
if (( acted == 0 )); then
|
||||
isSuccessful "Nothing to heal — control plane healthy."
|
||||
else
|
||||
isSuccessful "Health heal complete — ${acted} action(s); control plane reachable."
|
||||
fi
|
||||
}
|
||||
97
scripts/docker/health/docker_health_scan.sh
Normal file
97
scripts/docker/health/docker_health_scan.sh
Normal file
@ -0,0 +1,97 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Read-only rootless-docker / control-plane health scan — the shared detection
|
||||
# used by both the WebUI status generator (webuiSystemHealthCheck) and the heal
|
||||
# verb (system health heal), so the two never diverge.
|
||||
#
|
||||
# dockerHealthScan sets these globals (call it DIRECTLY, never in $(...) — a
|
||||
# subshell would drop them):
|
||||
# HEALTH_DAEMON_OK "true"/"false" — rootless docker daemon reachable
|
||||
# HEALTH_WEBUI_PRESENT "true"/"false" — the core WebUI container exists
|
||||
# HEALTH_WEBUI_RUNNING "true"/"false" — ...and is running (not exited/restarting)
|
||||
# HEALTH_WEBUI_PORT — its published host port (docker port), "" if none
|
||||
# HEALTH_WEBUI_REACHABLE "true"/"false"/"unknown" — that host port accepts a TCP connect
|
||||
# HEALTH_SCAN_ERROR — human note when the daemon is off (else "")
|
||||
# HEALTH_CRASHLOOPS (array) — "app|container|restartcount" per container
|
||||
# stuck restarting (State=restarting and
|
||||
# RestartCount >= CFG_HEALTH_CRASHLOOP_LIMIT):
|
||||
# a crash-loop churning the shared network.
|
||||
#
|
||||
# The incident this guards: an app crash-loop (trivy, offline, FATAL on every
|
||||
# boot) churned the rootless port-forwarder until the WebUI's published port was
|
||||
# torn down — the container stayed healthy INSIDE, but nothing on the host could
|
||||
# reach it. So we probe the host-visible port forward, not just container state.
|
||||
#
|
||||
# Nothing here mutates state.
|
||||
|
||||
# The core WebUI container + its internal port (the control plane we must keep
|
||||
# reachable). Kept as tiny functions so a rename only touches one place.
|
||||
_healthWebuiContainer() { echo "libreportal-service"; }
|
||||
_healthWebuiInternalPort() { echo "1111"; }
|
||||
|
||||
# TCP-connect probe from the host (manager context). Success => the published
|
||||
# port forward is live. Uses bash /dev/tcp (always present) with a timeout so a
|
||||
# black-holed forward can't hang the poll.
|
||||
_healthTcpReachable() {
|
||||
local host="$1" port="$2"
|
||||
[[ -n "$port" ]] || return 2
|
||||
timeout 4 bash -c "exec 3<>/dev/tcp/${host}/${port}" 2>/dev/null
|
||||
}
|
||||
|
||||
dockerHealthScan() {
|
||||
HEALTH_DAEMON_OK="false"; HEALTH_WEBUI_PRESENT="false"; HEALTH_WEBUI_RUNNING="false"
|
||||
HEALTH_WEBUI_PORT=""; HEALTH_WEBUI_REACHABLE="unknown"; HEALTH_SCAN_ERROR=""
|
||||
HEALTH_CRASHLOOPS=()
|
||||
|
||||
local limit="${CFG_HEALTH_CRASHLOOP_LIMIT:-3}"
|
||||
[[ "$limit" =~ ^[0-9]+$ ]] || limit=3
|
||||
|
||||
# Daemon reachable? Never alarm on what we can't verify — a daemon blip (or a
|
||||
# mid-recycle window) is transient, not a conflict.
|
||||
if ! dockerCommandRun "docker info" >/dev/null 2>&1; then
|
||||
HEALTH_SCAN_ERROR="docker daemon unreachable"
|
||||
return 0
|
||||
fi
|
||||
HEALTH_DAEMON_OK="true"
|
||||
|
||||
# Crash-loopers: containers docker reports as "restarting" whose RestartCount
|
||||
# has already climbed past the limit — i.e. actively churning, not a one-off
|
||||
# restart. Container name is enough to name the offender in the badge; derive
|
||||
# a friendly app label by trimming the conventional "-service" suffix.
|
||||
local names name rc app
|
||||
names=$(dockerCommandRun "docker ps -a --filter status=restarting --format '{{.Names}}'" 2>/dev/null)
|
||||
while IFS= read -r name; do
|
||||
[[ -n "$name" ]] || continue
|
||||
rc=$(dockerCommandRun "docker inspect --format '{{.RestartCount}}' '$name'" 2>/dev/null | tr -dc '0-9')
|
||||
[[ -n "$rc" ]] || rc=0
|
||||
if (( rc >= limit )); then
|
||||
app="${name%-service}"
|
||||
HEALTH_CRASHLOOPS+=("${app}|${name}|${rc}")
|
||||
fi
|
||||
done <<< "$names"
|
||||
|
||||
# Core WebUI container: present? running? host port reachable?
|
||||
local webui iport wstate
|
||||
webui="$(_healthWebuiContainer)"
|
||||
iport="$(_healthWebuiInternalPort)"
|
||||
|
||||
wstate=$(dockerCommandRun "docker inspect --format '{{.State.Status}}' '$webui'" 2>/dev/null | tr -d '[:space:]')
|
||||
if [[ -z "$wstate" ]]; then
|
||||
return 0 # container not found — nothing more to probe
|
||||
fi
|
||||
HEALTH_WEBUI_PRESENT="true"
|
||||
[[ "$wstate" == "running" ]] && HEALTH_WEBUI_RUNNING="true"
|
||||
|
||||
# docker's view of the published host port (e.g. "1111/tcp -> 0.0.0.0:9781").
|
||||
# Note: this reports the INTENDED mapping even when the rootless forward has
|
||||
# been torn down — which is exactly why we then TCP-probe it for real.
|
||||
HEALTH_WEBUI_PORT=$(dockerCommandRun "docker port '$webui' '$iport'" 2>/dev/null | head -1 | sed -n 's/.*:\([0-9][0-9]*\)$/\1/p')
|
||||
|
||||
if [[ "$HEALTH_WEBUI_RUNNING" == "true" && -n "$HEALTH_WEBUI_PORT" ]]; then
|
||||
if _healthTcpReachable "127.0.0.1" "$HEALTH_WEBUI_PORT"; then
|
||||
HEALTH_WEBUI_REACHABLE="true"
|
||||
else
|
||||
HEALTH_WEBUI_REACHABLE="false"
|
||||
fi
|
||||
fi
|
||||
}
|
||||
@ -36,6 +36,8 @@ docker_scripts=(
|
||||
"docker/compose/setup_compose_yml.sh"
|
||||
"docker/compose/update_and_start.sh"
|
||||
"docker/compose/update_compose_yml.sh"
|
||||
"docker/health/docker_health_heal.sh"
|
||||
"docker/health/docker_health_scan.sh"
|
||||
"docker/install/rooted/rooted_docker_check.sh"
|
||||
"docker/install/rooted/rooted_docker_compose.sh"
|
||||
"docker/install/rooted/rooted_docker.sh"
|
||||
|
||||
@ -30,6 +30,7 @@ webui_scripts=(
|
||||
"webui/data/generators/peers/webui_peers.sh"
|
||||
"webui/data/generators/system/webui_ssh_access.sh"
|
||||
"webui/data/generators/system/webui_system_disk.sh"
|
||||
"webui/data/generators/system/webui_system_health.sh"
|
||||
"webui/data/generators/system/webui_system_info.sh"
|
||||
"webui/data/generators/system/webui_system_memory.sh"
|
||||
"webui/data/generators/system/webui_system_metrics.sh"
|
||||
|
||||
@ -405,6 +405,8 @@ declare -gA LP_FN_MAP=(
|
||||
[dockerContainerOwner]="function/permission/libreportal_folders.sh"
|
||||
[dockerCopyBuildContext]="docker/compose/copy_build_context.sh"
|
||||
[dockerDeleteData]="docker/app/uninstall/delete_data.sh"
|
||||
[dockerHealthHeal]="docker/health/docker_health_heal.sh"
|
||||
[dockerHealthScan]="docker/health/docker_health_scan.sh"
|
||||
[dockerInstallApp]="docker/app/functions/function_install_app.sh"
|
||||
[dockerPruneAppNetworks]="docker/network/network_prune.sh"
|
||||
[dockerRemoveApp]="docker/app/docker/remove_app.sh"
|
||||
@ -429,6 +431,7 @@ declare -gA LP_FN_MAP=(
|
||||
[endStart]="start/start_end.sh"
|
||||
[engineBackupApp]="backup/engine/engine_dispatch.sh"
|
||||
[engineBackupSystem]="backup/engine/engine_dispatch.sh"
|
||||
[_engineCachedPull]="backup/engine/engine_dispatch.sh"
|
||||
[engineCheckAllLocations]="backup/engine/engine_dispatch.sh"
|
||||
[engineCheckLocation]="backup/engine/engine_dispatch.sh"
|
||||
[engineDispatch]="backup/engine/engine_dispatch.sh"
|
||||
@ -499,6 +502,9 @@ declare -gA LP_FN_MAP=(
|
||||
[healthLogInfo]="task/crontab_check_processor.sh"
|
||||
[healthLogSuccess]="task/crontab_check_processor.sh"
|
||||
[healthLogWarning]="task/crontab_check_processor.sh"
|
||||
[_healthTcpReachable]="docker/health/docker_health_scan.sh"
|
||||
[_healthWebuiContainer]="docker/health/docker_health_scan.sh"
|
||||
[_healthWebuiInternalPort]="docker/health/docker_health_scan.sh"
|
||||
[hostAppInstall]="install/host_app.sh"
|
||||
[hostSshAuthKeysFile]="ssh/host_access.sh"
|
||||
[hostSshEnsureDir]="ssh/host_access.sh"
|
||||
@ -972,6 +978,7 @@ declare -gA LP_FN_MAP=(
|
||||
[webuiSystemApps]="webui/data/generators/system/webui_system_metrics.sh"
|
||||
[webuiSystemAppStorage]="webui/data/generators/system/webui_system_metrics.sh"
|
||||
[webuiSystemDisk]="webui/data/generators/system/webui_system_disk.sh"
|
||||
[webuiSystemHealthCheck]="webui/data/generators/system/webui_system_health.sh"
|
||||
[webuiSystemInfo]="webui/data/generators/system/webui_system_info.sh"
|
||||
[webuiSystemMemory]="webui/data/generators/system/webui_system_memory.sh"
|
||||
[webuiSystemMetrics]="webui/data/generators/system/webui_system_metrics.sh"
|
||||
@ -1390,6 +1397,8 @@ declare -gA LP_FN_ROOT=(
|
||||
[dockerContainerOwner]="scripts"
|
||||
[dockerCopyBuildContext]="scripts"
|
||||
[dockerDeleteData]="scripts"
|
||||
[dockerHealthHeal]="scripts"
|
||||
[dockerHealthScan]="scripts"
|
||||
[dockerInstallApp]="scripts"
|
||||
[dockerPruneAppNetworks]="scripts"
|
||||
[dockerRemoveApp]="scripts"
|
||||
@ -1414,6 +1423,7 @@ declare -gA LP_FN_ROOT=(
|
||||
[endStart]="scripts"
|
||||
[engineBackupApp]="scripts"
|
||||
[engineBackupSystem]="scripts"
|
||||
[_engineCachedPull]="scripts"
|
||||
[engineCheckAllLocations]="scripts"
|
||||
[engineCheckLocation]="scripts"
|
||||
[engineDispatch]="scripts"
|
||||
@ -1484,6 +1494,9 @@ declare -gA LP_FN_ROOT=(
|
||||
[healthLogInfo]="scripts"
|
||||
[healthLogSuccess]="scripts"
|
||||
[healthLogWarning]="scripts"
|
||||
[_healthTcpReachable]="scripts"
|
||||
[_healthWebuiContainer]="scripts"
|
||||
[_healthWebuiInternalPort]="scripts"
|
||||
[hostAppInstall]="scripts"
|
||||
[hostSshAuthKeysFile]="scripts"
|
||||
[hostSshEnsureDir]="scripts"
|
||||
@ -1957,6 +1970,7 @@ declare -gA LP_FN_ROOT=(
|
||||
[webuiSystemApps]="scripts"
|
||||
[webuiSystemAppStorage]="scripts"
|
||||
[webuiSystemDisk]="scripts"
|
||||
[webuiSystemHealthCheck]="scripts"
|
||||
[webuiSystemInfo]="scripts"
|
||||
[webuiSystemMemory]="scripts"
|
||||
[webuiSystemMetrics]="scripts"
|
||||
@ -2408,6 +2422,8 @@ dockerConfigSetupToContainer() { unset -f dockerConfigSetupToContainer; __lpAuto
|
||||
dockerContainerOwner() { unset -f dockerContainerOwner; __lpAutoload "${install_scripts_dir}function/permission/libreportal_folders.sh"; dockerContainerOwner "$@"; }
|
||||
dockerCopyBuildContext() { unset -f dockerCopyBuildContext; __lpAutoload "${install_scripts_dir}docker/compose/copy_build_context.sh"; dockerCopyBuildContext "$@"; }
|
||||
dockerDeleteData() { unset -f dockerDeleteData; __lpAutoload "${install_scripts_dir}docker/app/uninstall/delete_data.sh"; dockerDeleteData "$@"; }
|
||||
dockerHealthHeal() { unset -f dockerHealthHeal; __lpAutoload "${install_scripts_dir}docker/health/docker_health_heal.sh"; dockerHealthHeal "$@"; }
|
||||
dockerHealthScan() { unset -f dockerHealthScan; __lpAutoload "${install_scripts_dir}docker/health/docker_health_scan.sh"; dockerHealthScan "$@"; }
|
||||
dockerInstallApp() { unset -f dockerInstallApp; __lpAutoload "${install_scripts_dir}docker/app/functions/function_install_app.sh"; dockerInstallApp "$@"; }
|
||||
dockerPruneAppNetworks() { unset -f dockerPruneAppNetworks; __lpAutoload "${install_scripts_dir}docker/network/network_prune.sh"; dockerPruneAppNetworks "$@"; }
|
||||
dockerRemoveApp() { unset -f dockerRemoveApp; __lpAutoload "${install_scripts_dir}docker/app/docker/remove_app.sh"; dockerRemoveApp "$@"; }
|
||||
@ -2432,6 +2448,7 @@ emailValidation() { unset -f emailValidation; __lpAutoload "${install_scripts_di
|
||||
endStart() { unset -f endStart; __lpAutoload "${install_scripts_dir}start/start_end.sh"; endStart "$@"; }
|
||||
engineBackupApp() { unset -f engineBackupApp; __lpAutoload "${install_scripts_dir}backup/engine/engine_dispatch.sh"; engineBackupApp "$@"; }
|
||||
engineBackupSystem() { unset -f engineBackupSystem; __lpAutoload "${install_scripts_dir}backup/engine/engine_dispatch.sh"; engineBackupSystem "$@"; }
|
||||
_engineCachedPull() { unset -f _engineCachedPull; __lpAutoload "${install_scripts_dir}backup/engine/engine_dispatch.sh"; _engineCachedPull "$@"; }
|
||||
engineCheckAllLocations() { unset -f engineCheckAllLocations; __lpAutoload "${install_scripts_dir}backup/engine/engine_dispatch.sh"; engineCheckAllLocations "$@"; }
|
||||
engineCheckLocation() { unset -f engineCheckLocation; __lpAutoload "${install_scripts_dir}backup/engine/engine_dispatch.sh"; engineCheckLocation "$@"; }
|
||||
engineDispatch() { unset -f engineDispatch; __lpAutoload "${install_scripts_dir}backup/engine/engine_dispatch.sh"; engineDispatch "$@"; }
|
||||
@ -2502,6 +2519,9 @@ healthLogError() { unset -f healthLogError; __lpAutoload "${install_scripts_dir}
|
||||
healthLogInfo() { unset -f healthLogInfo; __lpAutoload "${install_scripts_dir}task/crontab_check_processor.sh"; healthLogInfo "$@"; }
|
||||
healthLogSuccess() { unset -f healthLogSuccess; __lpAutoload "${install_scripts_dir}task/crontab_check_processor.sh"; healthLogSuccess "$@"; }
|
||||
healthLogWarning() { unset -f healthLogWarning; __lpAutoload "${install_scripts_dir}task/crontab_check_processor.sh"; healthLogWarning "$@"; }
|
||||
_healthTcpReachable() { unset -f _healthTcpReachable; __lpAutoload "${install_scripts_dir}docker/health/docker_health_scan.sh"; _healthTcpReachable "$@"; }
|
||||
_healthWebuiContainer() { unset -f _healthWebuiContainer; __lpAutoload "${install_scripts_dir}docker/health/docker_health_scan.sh"; _healthWebuiContainer "$@"; }
|
||||
_healthWebuiInternalPort() { unset -f _healthWebuiInternalPort; __lpAutoload "${install_scripts_dir}docker/health/docker_health_scan.sh"; _healthWebuiInternalPort "$@"; }
|
||||
hostAppInstall() { unset -f hostAppInstall; __lpAutoload "${install_scripts_dir}install/host_app.sh"; hostAppInstall "$@"; }
|
||||
hostSshAuthKeysFile() { unset -f hostSshAuthKeysFile; __lpAutoload "${install_scripts_dir}ssh/host_access.sh"; hostSshAuthKeysFile "$@"; }
|
||||
hostSshEnsureDir() { unset -f hostSshEnsureDir; __lpAutoload "${install_scripts_dir}ssh/host_access.sh"; hostSshEnsureDir "$@"; }
|
||||
@ -2975,6 +2995,7 @@ webuiSyncAppIcons() { unset -f webuiSyncAppIcons; __lpAutoload "${install_script
|
||||
webuiSystemApps() { unset -f webuiSystemApps; __lpAutoload "${install_scripts_dir}webui/data/generators/system/webui_system_metrics.sh"; webuiSystemApps "$@"; }
|
||||
webuiSystemAppStorage() { unset -f webuiSystemAppStorage; __lpAutoload "${install_scripts_dir}webui/data/generators/system/webui_system_metrics.sh"; webuiSystemAppStorage "$@"; }
|
||||
webuiSystemDisk() { unset -f webuiSystemDisk; __lpAutoload "${install_scripts_dir}webui/data/generators/system/webui_system_disk.sh"; webuiSystemDisk "$@"; }
|
||||
webuiSystemHealthCheck() { unset -f webuiSystemHealthCheck; __lpAutoload "${install_scripts_dir}webui/data/generators/system/webui_system_health.sh"; webuiSystemHealthCheck "$@"; }
|
||||
webuiSystemInfo() { unset -f webuiSystemInfo; __lpAutoload "${install_scripts_dir}webui/data/generators/system/webui_system_info.sh"; webuiSystemInfo "$@"; }
|
||||
webuiSystemMemory() { unset -f webuiSystemMemory; __lpAutoload "${install_scripts_dir}webui/data/generators/system/webui_system_memory.sh"; webuiSystemMemory "$@"; }
|
||||
webuiSystemMetrics() { unset -f webuiSystemMetrics; __lpAutoload "${install_scripts_dir}webui/data/generators/system/webui_system_metrics.sh"; webuiSystemMetrics "$@"; }
|
||||
|
||||
@ -519,6 +519,12 @@ maybeRegenPoll() {
|
||||
# so almost every tick is a single stat() and an early return. Eligible signed
|
||||
# hotfixes are enqueued as ordinary tasks by the scan, never applied inline.
|
||||
command -v libreportal >/dev/null 2>&1 && libreportal updater check auto >/dev/null 2>&1 || true
|
||||
# Control-plane health (rootless docker port-forward + container crash-loops).
|
||||
# Same shape as the drift check: runs every poll, self-throttled inside the CLI
|
||||
# to CFG_HEALTH_CHECK_INTERVAL, cheap no-op most ticks. When it finds the
|
||||
# WebUI's host port down or a container crash-looping it self-dispatches
|
||||
# `system health heal` — the user can't click a heal button on a dead WebUI.
|
||||
command -v libreportal >/dev/null 2>&1 && libreportal system health check >/dev/null 2>&1 || true
|
||||
}
|
||||
|
||||
# ============================================================================
|
||||
|
||||
131
scripts/webui/data/generators/system/webui_system_health.sh
Normal file
131
scripts/webui/data/generators/system/webui_system_health.sh
Normal file
@ -0,0 +1,131 @@
|
||||
#!/bin/bash
|
||||
|
||||
# WebUI rootless-docker / control-plane health detector.
|
||||
# Writes frontend/data/system/health_status.json so the dashboard + topbar can
|
||||
# surface the control plane going unhealthy — the WebUI's published host port
|
||||
# torn down (container healthy inside, unreachable outside) or a container
|
||||
# crash-looping and churning the shared network.
|
||||
#
|
||||
# Read-only w.r.t. the system: it inspects docker and writes a status file. The
|
||||
# actual fix is the locked-down `libreportal system health heal`.
|
||||
#
|
||||
# Scheduling: invoked from the task processor's idle poll (~60s) via
|
||||
# `libreportal system health check`. Self-throttled to CFG_HEALTH_CHECK_INTERVAL
|
||||
# so most calls no-op. Pass "force" to bypass (manual re-check / post-heal).
|
||||
#
|
||||
# Unlike the network badge, this one SELF-DISPATCHES a heal: when the WebUI's own
|
||||
# port is down the user can't click a button to fix it, so on an auto-healable
|
||||
# problem the check enqueues `system health heal` itself (throttled by a cooldown
|
||||
# stamp so it can never stack heals).
|
||||
webuiSystemHealthCheck() {
|
||||
local force_flag="$1"
|
||||
|
||||
local system_dir="$containers_dir/libreportal/frontend/data/system"
|
||||
local final_file="${system_dir}/health_status.json"
|
||||
local stamp_file="${system_dir}/.health_check_stamp"
|
||||
local heal_stamp="${system_dir}/.health_heal_stamp"
|
||||
local interval="${CFG_HEALTH_CHECK_INTERVAL:-120}"
|
||||
local heal_cooldown="${CFG_HEALTH_HEAL_COOLDOWN:-300}"
|
||||
|
||||
createFolders "quiet" "$sudo_user_name" "$system_dir"
|
||||
|
||||
local do_run="false"
|
||||
if [[ "$force_flag" == "force" || ! -f "$final_file" || ! -f "$stamp_file" ]]; then
|
||||
do_run="true"
|
||||
else
|
||||
local _now _last; _now=$(date +%s); _last=$(stat -c '%Y' "$stamp_file" 2>/dev/null || echo 0)
|
||||
(( _now - _last >= interval )) && do_run="true"
|
||||
fi
|
||||
[[ "$do_run" == "true" ]] || return 0
|
||||
runFileOp touch "$stamp_file" 2>/dev/null || true
|
||||
|
||||
# Read-only scan — call directly (NOT in $(...)): it sets HEALTH_* globals +
|
||||
# the HEALTH_CRASHLOOPS array, which a subshell would discard.
|
||||
dockerHealthScan
|
||||
|
||||
local healthy="true" issues_found="false" can_auto_heal="false"
|
||||
local error_json="null" summary=""
|
||||
local crash_json=""
|
||||
|
||||
if [[ "$HEALTH_DAEMON_OK" != "true" ]]; then
|
||||
# Daemon unreachable — neutral status; never alarm on what we can't check.
|
||||
error_json="\"${HEALTH_SCAN_ERROR//\"/\\\"}\""
|
||||
summary="Docker daemon unreachable"
|
||||
else
|
||||
local row app cname rc _a _c
|
||||
for row in "${HEALTH_CRASHLOOPS[@]}"; do
|
||||
IFS='|' read -r app cname rc <<< "$row"
|
||||
[[ -n "$cname" ]] || continue
|
||||
[[ "$rc" =~ ^[0-9]+$ ]] || rc=0
|
||||
_a=${app//\\/\\\\}; _a=${_a//\"/\\\"}
|
||||
_c=${cname//\\/\\\\}; _c=${_c//\"/\\\"}
|
||||
crash_json+="${crash_json:+,}"$'\n'" {\"app\": \"${_a}\", \"container\": \"${_c}\", \"restart_count\": ${rc}}"
|
||||
done
|
||||
|
||||
local webui_unreachable="false"
|
||||
[[ "$HEALTH_WEBUI_RUNNING" == "true" && "$HEALTH_WEBUI_REACHABLE" == "false" ]] && webui_unreachable="true"
|
||||
|
||||
if [[ -n "$crash_json" || "$webui_unreachable" == "true" ]]; then
|
||||
healthy="false"; issues_found="true"; can_auto_heal="true"
|
||||
if [[ "$webui_unreachable" == "true" ]]; then
|
||||
summary="WebUI unreachable on host port ${HEALTH_WEBUI_PORT:-?} (container is up) — port forward needs repair"
|
||||
else
|
||||
summary="Crash-looping container(s) churning the network"
|
||||
fi
|
||||
else
|
||||
summary="Control plane healthy"
|
||||
fi
|
||||
fi
|
||||
|
||||
local crash_arr="[]"
|
||||
[[ -n "$crash_json" ]] && crash_arr="[${crash_json}"$'\n'" ]"
|
||||
|
||||
local reachable_json
|
||||
case "$HEALTH_WEBUI_REACHABLE" in
|
||||
true) reachable_json="true" ;;
|
||||
false) reachable_json="false" ;;
|
||||
*) reachable_json="null" ;;
|
||||
esac
|
||||
|
||||
local port_json="null"
|
||||
[[ "$HEALTH_WEBUI_PORT" =~ ^[0-9]+$ ]] && port_json="$HEALTH_WEBUI_PORT"
|
||||
|
||||
local temp_file; temp_file="$(mktemp)"
|
||||
cat << EOF > "$temp_file"
|
||||
{
|
||||
"healthy": ${healthy},
|
||||
"issues_found": ${issues_found},
|
||||
"can_auto_heal": ${can_auto_heal},
|
||||
"daemon_ok": ${HEALTH_DAEMON_OK},
|
||||
"webui": {
|
||||
"container": "$(_healthWebuiContainer)",
|
||||
"present": ${HEALTH_WEBUI_PRESENT},
|
||||
"running": ${HEALTH_WEBUI_RUNNING},
|
||||
"port": ${port_json},
|
||||
"reachable": ${reachable_json}
|
||||
},
|
||||
"crash_loops": ${crash_arr},
|
||||
"summary": "${summary//\"/\\\"}",
|
||||
"error": ${error_json},
|
||||
"checked_at": "$(date -Iseconds)"
|
||||
}
|
||||
EOF
|
||||
runFileWrite "$final_file" < "$temp_file"; rm -f "$temp_file"
|
||||
|
||||
# Self-dispatch a heal on an auto-healable problem, throttled by a cooldown
|
||||
# stamp. The WebUI can't be clicked when its own port is down, so the poll
|
||||
# must drive the fix — this is the backend counterpart of the "Heal now"
|
||||
# button. The stamp is touched BEFORE enqueue so a heal-triggered re-check
|
||||
# (which force-runs this generator) can't stack a second heal.
|
||||
if [[ "$can_auto_heal" == "true" ]]; then
|
||||
local _hnow _hlast=0
|
||||
_hnow=$(date +%s)
|
||||
[[ -f "$heal_stamp" ]] && _hlast=$(stat -c '%Y' "$heal_stamp" 2>/dev/null || echo 0)
|
||||
if (( _hnow - _hlast >= heal_cooldown )); then
|
||||
runFileOp touch "$heal_stamp" 2>/dev/null || true
|
||||
if declare -f cliTaskRun >/dev/null 2>&1; then
|
||||
cliTaskRun "libreportal system health heal" "system_health_heal" "" "--detach" >/dev/null 2>&1 || true
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
}
|
||||
Loading…
x
Reference in New Issue
Block a user