perf(backup): fetch snapshots once per location, filter in jq — kill the O(apps×locations) restic storm
The backup-data refresh in the WebUI update chain shelled out to `restic snapshots` once PER APP (dashboard, app-status) and once per (host, app) (migrate) — every call a `sudo`+`restic` spawn that re-opens and decrypts the repo. With N apps and L locations that's O(N·L) (app-status) plus O(hosts·apps·L) (migrate) spawns on every update — the silent stall after "Refreshed backup dashboard data...". One `restic snapshots --json` already returns the whole repo, so fetch it ONCE per location and derive the per-app / per-host / system views in jq. Validated against a live restic 0.19 repo: the jq filter reproduces `restic snapshots --tag <t> --host <h>` exactly (incl. empty-host = all hosts, and newest = last), and count/latest extraction matches the old grep. - dashboard: one primary-location fetch → all app tiles + system status (was N snapshot calls + 1). - app-status: one fetch per location → filter per app (was N×L calls); also de-recursed into a flat app loop. - migrate: filter the already-fetched per-location list (was a spawn per host×app). Fixes a latent bug too — `grep -oc` over single-line JSON always reported a count of 1; jq `length` is now accurate. restic snapshot spawns per refresh drop from ~O(N·L + hosts·apps·L) to ~O(L). `restic stats` (still run per location in locations + dashboard) is a separate, non-multiplicative cost left untouched to preserve its semantics. Signed-off-by: librelad <librelad@digitalangels.vip> Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
librelad
parent
c3315d9fbf
commit
10846b120e
@ -2,54 +2,74 @@
|
||||
|
||||
webuiGenerateBackupAppStatus()
|
||||
{
|
||||
local app_name="$1"
|
||||
local app_name="${1:-}"
|
||||
local output_dir="$containers_dir/libreportal/frontend/data/backup/generated/apps"
|
||||
runFileOp mkdir -p "$output_dir"
|
||||
|
||||
if [[ -z "$app_name" ]]; then
|
||||
if [[ -f "$docker_dir/$db_file" ]]; then
|
||||
while IFS= read -r a; do
|
||||
[[ -n "$a" ]] && webuiGenerateBackupAppStatus "$a"
|
||||
done < <(runInstallOp sqlite3 "$docker_dir/$db_file" "SELECT name FROM apps WHERE status = 1;" 2>/dev/null)
|
||||
fi
|
||||
return 0
|
||||
# Resolve the app list: a single app (direct call — e.g. right after that
|
||||
# app's backup) or every installed app (routine WebUI refresh).
|
||||
local apps=()
|
||||
if [[ -n "$app_name" ]]; then
|
||||
apps=("$app_name")
|
||||
elif [[ -f "$docker_dir/$db_file" ]]; then
|
||||
while IFS= read -r a; do
|
||||
[[ -n "$a" ]] && apps+=("$a")
|
||||
done < <(runInstallOp sqlite3 "$docker_dir/$db_file" "SELECT name FROM apps WHERE status = 1;" 2>/dev/null)
|
||||
fi
|
||||
[[ ${#apps[@]} -eq 0 ]] && return 0
|
||||
|
||||
local output_file="$output_dir/${app_name}.json"
|
||||
local temp_file="${output_file}.tmp.$$"
|
||||
|
||||
local content="{"
|
||||
content+="\"app\":\"$app_name\","
|
||||
content+="\"generated_at\":\"$(date -Iseconds)\","
|
||||
content+="\"host\":\"${CFG_INSTALL_NAME:-libreportal}\","
|
||||
content+="\"snapshots\":["
|
||||
|
||||
local first=true
|
||||
# Fetch each enabled location's snapshot list ONCE, then filter per app
|
||||
# in-process. A single `restic snapshots --json` returns the whole repo,
|
||||
# so this replaces the old O(apps x locations) restic spawns with one
|
||||
# restic call per location. jq reproduces restic's --tag/--host filtering.
|
||||
local host="${CFG_INSTALL_NAME}"
|
||||
local loc_idxs=()
|
||||
declare -A loc_name loc_snaps
|
||||
local idx
|
||||
while IFS= read -r idx; do
|
||||
[[ -z "$idx" ]] && continue
|
||||
local snaps
|
||||
snaps=$(engineSnapshotsJson "$idx" "$app_name" "$CFG_INSTALL_NAME" 2>/dev/null)
|
||||
[[ -z "$snaps" || "$snaps" == "[]" ]] && continue
|
||||
local count latest_id latest_time
|
||||
count=$(echo "$snaps" | grep -o '"short_id"' | wc -l | tr -d ' ')
|
||||
latest_id=$(echo "$snaps" | grep -o '"short_id":"[^"]*"' | tail -1 | cut -d'"' -f4)
|
||||
latest_time=$(echo "$snaps" | grep -o '"time":"[^"]*"' | tail -1 | cut -d'"' -f4)
|
||||
|
||||
local name_esc
|
||||
name_esc=$(printf '%s' "$(resticLocationName "$idx")" | sed 's/\\/\\\\/g; s/"/\\"/g')
|
||||
|
||||
$first || content+=","
|
||||
first=false
|
||||
content+="{"
|
||||
content+="\"location_idx\":$idx,"
|
||||
content+="\"location_name\":\"$name_esc\","
|
||||
content+="\"count\":$count,"
|
||||
content+="\"latest_id\":\"$latest_id\","
|
||||
content+="\"latest_time\":\"$latest_time\""
|
||||
content+="}"
|
||||
loc_idxs+=("$idx")
|
||||
loc_name["$idx"]=$(resticLocationName "$idx")
|
||||
local raw
|
||||
raw=$(engineSnapshotsJson "$idx" 2>/dev/null)
|
||||
[[ -z "$raw" ]] && raw="[]"
|
||||
loc_snaps["$idx"]="$raw"
|
||||
done < <(resticEnabledLocations)
|
||||
content+="]}"
|
||||
|
||||
echo "$content" | runFileWrite "$output_file"
|
||||
local app
|
||||
for app in "${apps[@]}"; do
|
||||
local content="{"
|
||||
content+="\"app\":\"$app\","
|
||||
content+="\"generated_at\":\"$(date -Iseconds)\","
|
||||
content+="\"host\":\"${host:-libreportal}\","
|
||||
content+="\"snapshots\":["
|
||||
|
||||
local first=true
|
||||
for idx in "${loc_idxs[@]}"; do
|
||||
local count latest_id latest_time
|
||||
IFS=$'\t' read -r count latest_id latest_time <<<"$(printf '%s' "${loc_snaps[$idx]}" | jq -r \
|
||||
--arg tag "app=$app" --arg host "$host" '
|
||||
[ .[] | select($host=="" or .hostname==$host)
|
||||
| select((.tags // []) | index($tag)) ] as $f
|
||||
| ($f[-1] // {}) as $l
|
||||
| "\($f|length)\t\($l.short_id // "")\t\($l.time // "")"' 2>/dev/null)"
|
||||
[[ -z "$count" || "$count" == "0" ]] && continue
|
||||
|
||||
local name_esc
|
||||
name_esc=$(printf '%s' "${loc_name[$idx]}" | sed 's/\\/\\\\/g; s/"/\\"/g')
|
||||
|
||||
$first || content+=","
|
||||
first=false
|
||||
content+="{"
|
||||
content+="\"location_idx\":$idx,"
|
||||
content+="\"location_name\":\"$name_esc\","
|
||||
content+="\"count\":$count,"
|
||||
content+="\"latest_id\":\"$latest_id\","
|
||||
content+="\"latest_time\":\"$latest_time\""
|
||||
content+="}"
|
||||
done
|
||||
content+="]}"
|
||||
|
||||
echo "$content" | runFileWrite "$output_dir/${app}.json"
|
||||
done
|
||||
}
|
||||
|
||||
@ -48,11 +48,20 @@ webuiGenerateBackupDashboard()
|
||||
done < <(resticEnabledLocations)
|
||||
locations_json+="]"
|
||||
|
||||
# Fetch the primary location's snapshots ONCE, then derive every app tile
|
||||
# and the system-config status below by filtering it in jq — instead of a
|
||||
# restic spawn per app plus one for system. jq reproduces restic's
|
||||
# --tag/--host filtering (empty host = no host filter, as restic does).
|
||||
local primary_idx primary_snaps="[]"
|
||||
primary_idx=$(resticEnabledLocations | head -1)
|
||||
if [[ -n "$primary_idx" ]]; then
|
||||
primary_snaps=$(engineSnapshotsJson "$primary_idx" 2>/dev/null)
|
||||
[[ -z "$primary_snaps" ]] && primary_snaps="[]"
|
||||
fi
|
||||
|
||||
local apps_json="["
|
||||
first=true
|
||||
if [[ -f "$docker_dir/$db_file" ]]; then
|
||||
local primary_idx
|
||||
primary_idx=$(resticEnabledLocations | head -1)
|
||||
while IFS= read -r app; do
|
||||
[[ -z "$app" ]] && continue
|
||||
# The WebUI app is reproducible and skipped by backupAllApps — its
|
||||
@ -60,12 +69,11 @@ webuiGenerateBackupDashboard()
|
||||
# perpetually-"No backup yet" tile for it.
|
||||
[[ "$app" == "libreportal" ]] && continue
|
||||
local latest_id="" latest_time=""
|
||||
if [[ -n "$primary_idx" ]]; then
|
||||
local snap_json
|
||||
snap_json=$(engineSnapshotsJson "$primary_idx" "$app" "$CFG_INSTALL_NAME" 2>/dev/null)
|
||||
latest_id=$(echo "$snap_json" | grep -o '"short_id":"[^"]*"' | tail -1 | cut -d'"' -f4)
|
||||
latest_time=$(echo "$snap_json" | grep -o '"time":"[^"]*"' | tail -1 | cut -d'"' -f4)
|
||||
fi
|
||||
IFS=$'\t' read -r latest_id latest_time <<<"$(printf '%s' "$primary_snaps" | jq -r \
|
||||
--arg tag "app=$app" --arg host "$CFG_INSTALL_NAME" '
|
||||
[ .[] | select($host=="" or .hostname==$host)
|
||||
| select((.tags // []) | index($tag)) ] | (.[-1] // {})
|
||||
| "\(.short_id // "")\t\(.time // "")"' 2>/dev/null)"
|
||||
$first || apps_json+=","
|
||||
first=false
|
||||
apps_json+="{\"app\":\"$app\",\"latest_snapshot\":\"$latest_id\",\"latest_time\":\"$latest_time\"}"
|
||||
@ -75,14 +83,11 @@ webuiGenerateBackupDashboard()
|
||||
|
||||
# System-config backup status (tag system=config) — its own tracked item.
|
||||
local system_id="" system_time=""
|
||||
local sys_idx
|
||||
sys_idx=$(resticEnabledLocations | head -1)
|
||||
if [[ -n "$sys_idx" ]]; then
|
||||
local sys_json
|
||||
sys_json=$(engineSystemSnapshotsJson "$sys_idx" "$CFG_INSTALL_NAME" 2>/dev/null)
|
||||
system_id=$(echo "$sys_json" | grep -o '"short_id":"[^"]*"' | tail -1 | cut -d'"' -f4)
|
||||
system_time=$(echo "$sys_json" | grep -o '"time":"[^"]*"' | tail -1 | cut -d'"' -f4)
|
||||
fi
|
||||
IFS=$'\t' read -r system_id system_time <<<"$(printf '%s' "$primary_snaps" | jq -r \
|
||||
--arg tag "system=config" --arg host "$CFG_INSTALL_NAME" '
|
||||
[ .[] | select($host=="" or .hostname==$host)
|
||||
| select((.tags // []) | index($tag)) ] | (.[-1] // {})
|
||||
| "\(.short_id // "")\t\(.time // "")"' 2>/dev/null)"
|
||||
|
||||
local content="{"
|
||||
content+="\"generated_at\":\"$generated_at\","
|
||||
|
||||
@ -89,15 +89,18 @@ webuiGenerateBackupMigrate()
|
||||
local app
|
||||
while IFS= read -r app; do
|
||||
[[ -z "$app" ]] && continue
|
||||
local app_snaps_json
|
||||
app_snaps_json=$(engineSnapshotsJson "$idx" "$app" "$host" 2>/dev/null)
|
||||
local count
|
||||
count=$(printf '%s' "$app_snaps_json" | grep -oc '"short_id":"' || echo 0)
|
||||
(( count == 0 )) && continue
|
||||
|
||||
local latest_id latest_date
|
||||
latest_id=$(printf '%s' "$app_snaps_json" | grep -o '"short_id":"[^"]*"' | tail -1 | cut -d'"' -f4)
|
||||
latest_date=$(printf '%s' "$app_snaps_json" | grep -o '"time":"[^"]*"' | tail -1 | cut -d'"' -f4)
|
||||
# Filter the location's already-fetched snapshot list in-process
|
||||
# (jq == restic --tag/--host) rather than a restic spawn per
|
||||
# (host, app). Also gives an accurate count — the old grep -oc
|
||||
# over single-line JSON always reported 1.
|
||||
local count latest_id latest_date
|
||||
IFS=$'\t' read -r count latest_id latest_date <<<"$(printf '%s' "$all_json" | jq -r \
|
||||
--arg tag "app=$app" --arg host "$host" '
|
||||
[ .[] | select($host=="" or .hostname==$host)
|
||||
| select((.tags // []) | index($tag)) ] as $f
|
||||
| ($f[-1] // {}) as $l
|
||||
| "\($f|length)\t\($l.short_id // "")\t\($l.time // "")"' 2>/dev/null)"
|
||||
[[ -z "$count" || "$count" == "0" ]] && continue
|
||||
|
||||
local opt_out
|
||||
opt_out=$(migrateUrlRewriteEnabled "$app" 2>/dev/null)
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user