feat(webui): collect host + per-app metrics with history ring buffer
Add webui_system_metrics.sh, run each minute from webuiSystemUpdate: - whole-server snapshot (metrics.json): CPU% + load, memory + swap, per-mount disk + inodes, network rx/tx rate, docker summary - capped ring buffer (metrics_history.json, 24h default) for trend charts - per-app docker stats grouped by compose project (metrics_apps.json) plus a short per-app history (metrics_apps_history.json) for sparklines CPU% and network rate use stateful deltas stashed beside the JSON; all host metrics read from /proc and docker via runFileOp, so it works rootless. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com> Signed-off-by: librelad <librelad@digitalangels.vip>
This commit is contained in:
librelad
parent
0b64b364f9
commit
bade6eaacb
@ -30,6 +30,7 @@ webui_scripts=(
|
||||
"webui/data/generators/system/webui_system_disk.sh"
|
||||
"webui/data/generators/system/webui_system_info.sh"
|
||||
"webui/data/generators/system/webui_system_memory.sh"
|
||||
"webui/data/generators/system/webui_system_metrics.sh"
|
||||
"webui/data/generators/system/webui_system_update.sh"
|
||||
"webui/data/lock/webui_check_update_lock.sh"
|
||||
"webui/data/lock/webui_create_update_lock.sh"
|
||||
|
||||
295
scripts/webui/data/generators/system/webui_system_metrics.sh
Normal file
295
scripts/webui/data/generators/system/webui_system_metrics.sh
Normal file
@ -0,0 +1,295 @@
|
||||
#!/bin/bash
|
||||
|
||||
# WebUI System Metrics Generator
|
||||
# ---------------------------------------------------------------------------
|
||||
# Produces the data behind the Admin → System page:
|
||||
# metrics.json — current whole-server snapshot (CPU/mem/disk/net/docker)
|
||||
# metrics_history.json — capped ring buffer of compact samples (for trend charts)
|
||||
# metrics_apps.json — current per-app Docker resource snapshot
|
||||
# metrics_apps_history.json — short per-app ring buffer (for per-app sparklines)
|
||||
#
|
||||
# Runs every minute from the same cron as webuiSystemUpdate. CPU% and network
|
||||
# rate are rate-of-change values, so we stash the previous raw counters in
|
||||
# hidden state files next to the JSON and diff against them each run (the
|
||||
# existing update checker uses the same .stamp idiom in this directory).
|
||||
# Everything is read from /proc + the docker socket, so it works rootless.
|
||||
|
||||
# Read the aggregate CPU jiffy counters from /proc/stat.
|
||||
# Echoes: "<total> <idle>"
|
||||
_metricsReadCpu() {
|
||||
awk '/^cpu /{
|
||||
idle = $5 + $6 # idle + iowait
|
||||
total = 0
|
||||
for (i = 2; i <= NF; i++) total += $i
|
||||
print total, idle
|
||||
exit
|
||||
}' /proc/stat 2>/dev/null
|
||||
}
|
||||
|
||||
webuiSystemMetrics() {
|
||||
local system_dir="${containers_dir}libreportal/frontend/data/system"
|
||||
createFolders "quiet" "$sudo_user_name" "$system_dir"
|
||||
|
||||
local now_epoch; now_epoch=$(date +%s)
|
||||
local now_iso; now_iso=$(date -Iseconds)
|
||||
|
||||
# --- CPU --------------------------------------------------------------
|
||||
local cpu_cores; cpu_cores=$(nproc 2>/dev/null || echo 1)
|
||||
local cpu_state="$system_dir/.metrics_cpu_prev"
|
||||
local cur_cpu; cur_cpu=$(_metricsReadCpu)
|
||||
local cur_total cur_idle
|
||||
cur_total=$(echo "$cur_cpu" | awk '{print $1}')
|
||||
cur_idle=$(echo "$cur_cpu" | awk '{print $2}')
|
||||
|
||||
local cpu_percent="0.0"
|
||||
if [[ -f "$cpu_state" ]]; then
|
||||
local prev_total prev_idle
|
||||
prev_total=$(awk '{print $1}' "$cpu_state" 2>/dev/null)
|
||||
prev_idle=$(awk '{print $2}' "$cpu_state" 2>/dev/null)
|
||||
if [[ -n "$prev_total" && -n "$cur_total" && "$cur_total" -gt "$prev_total" ]]; then
|
||||
cpu_percent=$(awk -v t="$cur_total" -v i="$cur_idle" -v pt="$prev_total" -v pi="$prev_idle" \
|
||||
'BEGIN { dt = t - pt; di = i - pi; if (dt <= 0) { print "0.0" } else { printf "%.1f", (1 - di/dt) * 100 } }')
|
||||
fi
|
||||
fi
|
||||
[[ -n "$cur_total" ]] && echo "$cur_total $cur_idle" > "$cpu_state" 2>/dev/null
|
||||
|
||||
# Load average (1/5/15) and 1-min load as a % of total cores.
|
||||
local load1 load5 load15
|
||||
read -r load1 load5 load15 _ < /proc/loadavg 2>/dev/null
|
||||
load1="${load1:-0}"; load5="${load5:-0}"; load15="${load15:-0}"
|
||||
local load1_pct
|
||||
load1_pct=$(awk -v l="$load1" -v c="$cpu_cores" 'BEGIN { if (c <= 0) c = 1; v = l / c * 100; if (v > 100) v = 100; printf "%.1f", v }')
|
||||
|
||||
# --- Memory + swap ----------------------------------------------------
|
||||
local mem_total mem_avail swap_total swap_free
|
||||
mem_total=$(awk '/^MemTotal:/{print $2*1024}' /proc/meminfo)
|
||||
mem_avail=$(awk '/^MemAvailable:/{print $2*1024}' /proc/meminfo)
|
||||
swap_total=$(awk '/^SwapTotal:/{print $2*1024}' /proc/meminfo)
|
||||
swap_free=$(awk '/^SwapFree:/{print $2*1024}' /proc/meminfo)
|
||||
mem_total="${mem_total:-0}"; mem_avail="${mem_avail:-0}"
|
||||
swap_total="${swap_total:-0}"; swap_free="${swap_free:-0}"
|
||||
local mem_used=$(( mem_total - mem_avail ))
|
||||
local swap_used=$(( swap_total - swap_free ))
|
||||
local mem_percent="0.0" swap_percent="0.0"
|
||||
[[ "$mem_total" -gt 0 ]] && mem_percent=$(awk -v u="$mem_used" -v t="$mem_total" 'BEGIN{printf "%.1f", u/t*100}')
|
||||
[[ "$swap_total" -gt 0 ]] && swap_percent=$(awk -v u="$swap_used" -v t="$swap_total" 'BEGIN{printf "%.1f", u/t*100}')
|
||||
|
||||
# --- Disks (real filesystems only) -----------------------------------
|
||||
# Build a JSON array of mounts with size/used/percent + inode %.
|
||||
local disks_json="[]"
|
||||
disks_json=$(df -PB1 -x tmpfs -x devtmpfs -x squashfs -x overlay -x aufs 2>/dev/null \
|
||||
| tail -n +2 \
|
||||
| while read -r fs blocks used avail pcent mount; do
|
||||
[[ -z "$mount" ]] && continue
|
||||
local ipct
|
||||
ipct=$(df -Pi "$mount" 2>/dev/null | tail -1 | awk '{gsub(/%/,"",$5); print $5+0}')
|
||||
printf '{"mount":"%s","total":%s,"used":%s,"percent":%s,"inode_percent":%s}\n' \
|
||||
"$mount" "${blocks:-0}" "${used:-0}" "$(echo "${pcent:-0}" | tr -d '%')" "${ipct:-0}"
|
||||
done | jq -s '.' 2>/dev/null)
|
||||
[[ -z "$disks_json" ]] && disks_json="[]"
|
||||
# Root mount percent powers the compact history sample.
|
||||
local root_percent
|
||||
root_percent=$(echo "$disks_json" | jq -r '(.[] | select(.mount=="/") | .percent) // (.[0].percent) // 0' 2>/dev/null)
|
||||
root_percent="${root_percent:-0}"
|
||||
|
||||
# --- Network throughput (aggregate, bytes/sec) -----------------------
|
||||
local net_state="$system_dir/.metrics_net_prev"
|
||||
local cur_rx cur_tx
|
||||
read -r cur_rx cur_tx < <(awk -F'[: ]+' '
|
||||
NR>2 && $2 !~ /^lo$/ { rx += $3; tx += $11 }
|
||||
END { print rx+0, tx+0 }' /proc/net/dev 2>/dev/null)
|
||||
cur_rx="${cur_rx:-0}"; cur_tx="${cur_tx:-0}"
|
||||
local net_rx_rate=0 net_tx_rate=0
|
||||
if [[ -f "$net_state" ]]; then
|
||||
local prev_t prev_rx prev_tx
|
||||
read -r prev_t prev_rx prev_tx < "$net_state"
|
||||
local dt=$(( now_epoch - ${prev_t:-now_epoch} ))
|
||||
if [[ "$dt" -gt 0 ]]; then
|
||||
net_rx_rate=$(awk -v c="$cur_rx" -v p="${prev_rx:-0}" -v d="$dt" 'BEGIN{v=(c-p)/d; if(v<0)v=0; printf "%.0f", v}')
|
||||
net_tx_rate=$(awk -v c="$cur_tx" -v p="${prev_tx:-0}" -v d="$dt" 'BEGIN{v=(c-p)/d; if(v<0)v=0; printf "%.0f", v}')
|
||||
fi
|
||||
fi
|
||||
echo "$now_epoch $cur_rx $cur_tx" > "$net_state" 2>/dev/null
|
||||
|
||||
# --- Docker summary ---------------------------------------------------
|
||||
local d_running=0 d_total=0 d_images=0 d_volumes=0
|
||||
if command -v docker &>/dev/null; then
|
||||
local states
|
||||
states=$(runFileOp docker ps -a --format '{{.State}}' 2>/dev/null)
|
||||
if [[ -n "$states" ]]; then
|
||||
d_total=$(echo "$states" | grep -c . )
|
||||
d_running=$(echo "$states" | grep -c '^running$')
|
||||
fi
|
||||
d_images=$(runFileOp docker images -q 2>/dev/null | grep -c . )
|
||||
d_volumes=$(runFileOp docker volume ls -q 2>/dev/null | grep -c . )
|
||||
fi
|
||||
|
||||
# --- Write current snapshot ------------------------------------------
|
||||
local tmp; tmp=$(mktemp)
|
||||
cat > "$tmp" << EOF
|
||||
{
|
||||
"cpu": {
|
||||
"percent": $cpu_percent,
|
||||
"cores": $cpu_cores,
|
||||
"load1": $load1,
|
||||
"load5": $load5,
|
||||
"load15": $load15,
|
||||
"load1_percent": $load1_pct
|
||||
},
|
||||
"memory": {
|
||||
"total": $mem_total,
|
||||
"used": $mem_used,
|
||||
"available": $mem_avail,
|
||||
"percent": $mem_percent,
|
||||
"swap_total": $swap_total,
|
||||
"swap_used": $swap_used,
|
||||
"swap_percent": $swap_percent
|
||||
},
|
||||
"disks": $disks_json,
|
||||
"network": {
|
||||
"rx_rate": $net_rx_rate,
|
||||
"tx_rate": $net_tx_rate
|
||||
},
|
||||
"docker": {
|
||||
"containers_running": $d_running,
|
||||
"containers_total": $d_total,
|
||||
"images": $d_images,
|
||||
"volumes": $d_volumes
|
||||
},
|
||||
"updated": "$now_iso"
|
||||
}
|
||||
EOF
|
||||
runFileWrite "$system_dir/metrics.json" < "$tmp"; rm -f "$tmp"
|
||||
|
||||
# --- Append to the history ring buffer -------------------------------
|
||||
local cap="${CFG_METRICS_HISTORY_CAP:-1440}" # 24h at 1 sample/min
|
||||
local hist="$system_dir/metrics_history.json"
|
||||
local existing="{}"
|
||||
[[ -s "$hist" ]] && existing=$(cat "$hist" 2>/dev/null)
|
||||
local sample
|
||||
sample=$(jq -n \
|
||||
--argjson t "$now_epoch" \
|
||||
--argjson cpu "$cpu_percent" \
|
||||
--argjson mem "$mem_percent" \
|
||||
--argjson swap "$swap_percent" \
|
||||
--argjson disk "$root_percent" \
|
||||
--argjson load1 "$load1" \
|
||||
--argjson rx "$net_rx_rate" \
|
||||
--argjson tx "$net_tx_rate" \
|
||||
'{t:$t, cpu:$cpu, mem:$mem, swap:$swap, disk:$disk, load1:$load1, net_rx:$rx, net_tx:$tx}' 2>/dev/null)
|
||||
|
||||
local tmp2; tmp2=$(mktemp)
|
||||
if echo "$existing" | jq \
|
||||
--argjson p "$sample" \
|
||||
--argjson cap "$cap" \
|
||||
--arg now "$now_iso" \
|
||||
'{cap:$cap, points: (((.points // []) + [$p]) | if length > $cap then .[-$cap:] else . end), updated:$now}' \
|
||||
> "$tmp2" 2>/dev/null; then
|
||||
runFileWrite "$hist" < "$tmp2"
|
||||
fi
|
||||
rm -f "$tmp2"
|
||||
|
||||
webuiSystemApps "$now_epoch" "$now_iso" "$system_dir"
|
||||
}
|
||||
|
||||
# Per-app Docker resource snapshot, grouped by compose project. Apps without a
|
||||
# compose project label fall back to their container name so nothing is lost.
|
||||
webuiSystemApps() {
|
||||
local now_epoch="$1" now_iso="$2" system_dir="$3"
|
||||
[[ -z "$system_dir" ]] && system_dir="${containers_dir}libreportal/frontend/data/system"
|
||||
command -v docker &>/dev/null || return 0
|
||||
|
||||
# name|project|state|status for every container (running or not)
|
||||
local meta
|
||||
meta=$(runFileOp docker ps -a --format '{{.Names}}|{{.Label "com.docker.compose.project"}}|{{.State}}|{{.Status}}' 2>/dev/null)
|
||||
[[ -z "$meta" ]] && { echo '{"apps":[],"updated":"'"$now_iso"'"}' | runFileWrite "$system_dir/metrics_apps.json"; return 0; }
|
||||
|
||||
# name|cpu%|memUsage|mem%|netIO for running containers
|
||||
local stats
|
||||
stats=$(runFileOp docker stats --no-stream --no-trunc \
|
||||
--format '{{.Name}}|{{.CPUPerc}}|{{.MemUsage}}|{{.MemPerc}}|{{.NetIO}}' 2>/dev/null)
|
||||
|
||||
# Aggregate into JSON via awk, keyed by project.
|
||||
local apps_json
|
||||
apps_json=$(awk -v meta="$meta" -v stats="$stats" '
|
||||
function tobytes(s, num, unit, m) {
|
||||
if (match(s, /[0-9.]+/) == 0) return 0
|
||||
num = substr(s, RSTART, RLENGTH) + 0
|
||||
unit = substr(s, RSTART + RLENGTH); gsub(/[ \t]/, "", unit)
|
||||
m = 1
|
||||
if (unit ~ /^[Kk]i/) m = 1024
|
||||
else if (unit ~ /^[Mm]i/) m = 1024^2
|
||||
else if (unit ~ /^[Gg]i/) m = 1024^3
|
||||
else if (unit ~ /^[Tt]i/) m = 1024^4
|
||||
else if (unit ~ /^[Kk]/) m = 1000
|
||||
else if (unit ~ /^[Mm]/) m = 1000^2
|
||||
else if (unit ~ /^[Gg]/) m = 1000^3
|
||||
else if (unit ~ /^[Tt]/) m = 1000^4
|
||||
return num * m
|
||||
}
|
||||
BEGIN {
|
||||
n = split(meta, mlines, "\n")
|
||||
for (i = 1; i <= n; i++) {
|
||||
if (mlines[i] == "") continue
|
||||
split(mlines[i], f, "|")
|
||||
cname = f[1]; proj = f[2]; state = f[3]
|
||||
if (proj == "") proj = cname
|
||||
c2p[cname] = proj
|
||||
total[proj]++
|
||||
if (state == "running") running[proj]++
|
||||
seen[proj] = 1
|
||||
}
|
||||
s = split(stats, slines, "\n")
|
||||
for (i = 1; i <= s; i++) {
|
||||
if (slines[i] == "") continue
|
||||
split(slines[i], g, "|")
|
||||
cname = g[1]
|
||||
proj = (cname in c2p) ? c2p[cname] : cname
|
||||
cpu = g[2]; gsub(/%/, "", cpu); cpu += 0
|
||||
memp = g[4]; gsub(/%/, "", memp); memp += 0
|
||||
cpus[proj] += cpu
|
||||
mems[proj] += tobytes(g[3])
|
||||
memps[proj] += memp
|
||||
split(g[5], nio, "/")
|
||||
netrx[proj] += tobytes(nio[1])
|
||||
nettx[proj] += tobytes(nio[2])
|
||||
seen[proj] = 1
|
||||
}
|
||||
first = 1
|
||||
printf "["
|
||||
for (p in seen) {
|
||||
st = (running[p] > 0) ? "running" : "stopped"
|
||||
if (!first) printf ","
|
||||
first = 0
|
||||
printf "{\"app\":\"%s\",\"containers\":%d,\"running\":%d,\"cpu_percent\":%.1f,\"mem_bytes\":%.0f,\"mem_percent\":%.1f,\"net_rx\":%.0f,\"net_tx\":%.0f,\"status\":\"%s\"}", \
|
||||
p, total[p]+0, running[p]+0, cpus[p]+0, mems[p]+0, memps[p]+0, netrx[p]+0, nettx[p]+0, st
|
||||
}
|
||||
printf "]"
|
||||
}')
|
||||
[[ -z "$apps_json" ]] && apps_json="[]"
|
||||
# Sort by CPU desc so the busiest apps surface first.
|
||||
apps_json=$(echo "$apps_json" | jq 'sort_by(-.cpu_percent)' 2>/dev/null || echo "$apps_json")
|
||||
|
||||
local tmp; tmp=$(mktemp)
|
||||
jq -n --argjson apps "$apps_json" --arg now "$now_iso" '{apps:$apps, updated:$now}' > "$tmp" 2>/dev/null \
|
||||
&& runFileWrite "$system_dir/metrics_apps.json" < "$tmp"
|
||||
rm -f "$tmp"
|
||||
|
||||
# Per-app history: append one {t,cpu,mem} per app, capped short.
|
||||
local cap="${CFG_METRICS_APP_HISTORY_CAP:-120}"
|
||||
local hist="$system_dir/metrics_apps_history.json"
|
||||
local existing="{}"
|
||||
[[ -s "$hist" ]] && existing=$(cat "$hist" 2>/dev/null)
|
||||
local tmp2; tmp2=$(mktemp)
|
||||
if echo "$existing" | jq \
|
||||
--argjson cur "$apps_json" \
|
||||
--argjson t "$now_epoch" \
|
||||
--argjson cap "$cap" \
|
||||
--arg now "$now_iso" \
|
||||
'reduce $cur[] as $a ((.apps // {}); .[$a.app] = (((.[$a.app] // []) + [{t:$t, cpu:$a.cpu_percent, mem:$a.mem_bytes}]) | if length > $cap then .[-$cap:] else . end))
|
||||
| {cap:$cap, apps:., updated:$now}' \
|
||||
> "$tmp2" 2>/dev/null; then
|
||||
runFileWrite "$hist" < "$tmp2"
|
||||
fi
|
||||
rm -f "$tmp2"
|
||||
}
|
||||
@ -7,6 +7,7 @@ webuiSystemUpdate() {
|
||||
webuiSystemInfo
|
||||
webuiSystemDisk
|
||||
webuiSystemMemory
|
||||
webuiSystemMetrics
|
||||
webuiSystemUpdateCheck
|
||||
isSuccessful "System information updated!"
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user