From 10846b120e36b38526516e58075a173e7557d55b Mon Sep 17 00:00:00 2001 From: librelad Date: Mon, 6 Jul 2026 18:19:58 +0100 Subject: [PATCH] =?UTF-8?q?perf(backup):=20fetch=20snapshots=20once=20per?= =?UTF-8?q?=20location,=20filter=20in=20jq=20=E2=80=94=20kill=20the=20O(ap?= =?UTF-8?q?ps=C3=97locations)=20restic=20storm?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The backup-data refresh in the WebUI update chain shelled out to `restic snapshots` once PER APP (dashboard, app-status) and once per (host, app) (migrate) — every call a `sudo`+`restic` spawn that re-opens and decrypts the repo. With N apps and L locations that's O(N·L) (app-status) plus O(hosts·apps·L) (migrate) spawns on every update — the silent stall after "Refreshed backup dashboard data...". One `restic snapshots --json` already returns the whole repo, so fetch it ONCE per location and derive the per-app / per-host / system views in jq. Validated against a live restic 0.19 repo: the jq filter reproduces `restic snapshots --tag --host ` exactly (incl. empty-host = all hosts, and newest = last), and count/latest extraction matches the old grep. - dashboard: one primary-location fetch → all app tiles + system status (was N snapshot calls + 1). - app-status: one fetch per location → filter per app (was N×L calls); also de-recursed into a flat app loop. - migrate: filter the already-fetched per-location list (was a spawn per host×app). Fixes a latent bug too — `grep -oc` over single-line JSON always reported a count of 1; jq `length` is now accurate. restic snapshot spawns per refresh drop from ~O(N·L + hosts·apps·L) to ~O(L). `restic stats` (still run per location in locations + dashboard) is a separate, non-multiplicative cost left untouched to preserve its semantics. Signed-off-by: librelad Co-Authored-By: Claude Opus 4.8 (1M context) --- .../backup/webui_backup_app_status.sh | 100 +++++++++++------- .../backup/webui_backup_dashboard.sh | 37 ++++--- .../generators/backup/webui_backup_migrate.sh | 21 ++-- 3 files changed, 93 insertions(+), 65 deletions(-) diff --git a/scripts/webui/data/generators/backup/webui_backup_app_status.sh b/scripts/webui/data/generators/backup/webui_backup_app_status.sh index 8e9fcc5..b1cde78 100644 --- a/scripts/webui/data/generators/backup/webui_backup_app_status.sh +++ b/scripts/webui/data/generators/backup/webui_backup_app_status.sh @@ -2,54 +2,74 @@ webuiGenerateBackupAppStatus() { - local app_name="$1" + local app_name="${1:-}" local output_dir="$containers_dir/libreportal/frontend/data/backup/generated/apps" runFileOp mkdir -p "$output_dir" - if [[ -z "$app_name" ]]; then - if [[ -f "$docker_dir/$db_file" ]]; then - while IFS= read -r a; do - [[ -n "$a" ]] && webuiGenerateBackupAppStatus "$a" - done < <(runInstallOp sqlite3 "$docker_dir/$db_file" "SELECT name FROM apps WHERE status = 1;" 2>/dev/null) - fi - return 0 + # Resolve the app list: a single app (direct call — e.g. right after that + # app's backup) or every installed app (routine WebUI refresh). + local apps=() + if [[ -n "$app_name" ]]; then + apps=("$app_name") + elif [[ -f "$docker_dir/$db_file" ]]; then + while IFS= read -r a; do + [[ -n "$a" ]] && apps+=("$a") + done < <(runInstallOp sqlite3 "$docker_dir/$db_file" "SELECT name FROM apps WHERE status = 1;" 2>/dev/null) fi + [[ ${#apps[@]} -eq 0 ]] && return 0 - local output_file="$output_dir/${app_name}.json" - local temp_file="${output_file}.tmp.$$" - - local content="{" - content+="\"app\":\"$app_name\"," - content+="\"generated_at\":\"$(date -Iseconds)\"," - content+="\"host\":\"${CFG_INSTALL_NAME:-libreportal}\"," - content+="\"snapshots\":[" - - local first=true + # Fetch each enabled location's snapshot list ONCE, then filter per app + # in-process. A single `restic snapshots --json` returns the whole repo, + # so this replaces the old O(apps x locations) restic spawns with one + # restic call per location. jq reproduces restic's --tag/--host filtering. + local host="${CFG_INSTALL_NAME}" + local loc_idxs=() + declare -A loc_name loc_snaps local idx while IFS= read -r idx; do [[ -z "$idx" ]] && continue - local snaps - snaps=$(engineSnapshotsJson "$idx" "$app_name" "$CFG_INSTALL_NAME" 2>/dev/null) - [[ -z "$snaps" || "$snaps" == "[]" ]] && continue - local count latest_id latest_time - count=$(echo "$snaps" | grep -o '"short_id"' | wc -l | tr -d ' ') - latest_id=$(echo "$snaps" | grep -o '"short_id":"[^"]*"' | tail -1 | cut -d'"' -f4) - latest_time=$(echo "$snaps" | grep -o '"time":"[^"]*"' | tail -1 | cut -d'"' -f4) - - local name_esc - name_esc=$(printf '%s' "$(resticLocationName "$idx")" | sed 's/\\/\\\\/g; s/"/\\"/g') - - $first || content+="," - first=false - content+="{" - content+="\"location_idx\":$idx," - content+="\"location_name\":\"$name_esc\"," - content+="\"count\":$count," - content+="\"latest_id\":\"$latest_id\"," - content+="\"latest_time\":\"$latest_time\"" - content+="}" + loc_idxs+=("$idx") + loc_name["$idx"]=$(resticLocationName "$idx") + local raw + raw=$(engineSnapshotsJson "$idx" 2>/dev/null) + [[ -z "$raw" ]] && raw="[]" + loc_snaps["$idx"]="$raw" done < <(resticEnabledLocations) - content+="]}" - echo "$content" | runFileWrite "$output_file" + local app + for app in "${apps[@]}"; do + local content="{" + content+="\"app\":\"$app\"," + content+="\"generated_at\":\"$(date -Iseconds)\"," + content+="\"host\":\"${host:-libreportal}\"," + content+="\"snapshots\":[" + + local first=true + for idx in "${loc_idxs[@]}"; do + local count latest_id latest_time + IFS=$'\t' read -r count latest_id latest_time <<<"$(printf '%s' "${loc_snaps[$idx]}" | jq -r \ + --arg tag "app=$app" --arg host "$host" ' + [ .[] | select($host=="" or .hostname==$host) + | select((.tags // []) | index($tag)) ] as $f + | ($f[-1] // {}) as $l + | "\($f|length)\t\($l.short_id // "")\t\($l.time // "")"' 2>/dev/null)" + [[ -z "$count" || "$count" == "0" ]] && continue + + local name_esc + name_esc=$(printf '%s' "${loc_name[$idx]}" | sed 's/\\/\\\\/g; s/"/\\"/g') + + $first || content+="," + first=false + content+="{" + content+="\"location_idx\":$idx," + content+="\"location_name\":\"$name_esc\"," + content+="\"count\":$count," + content+="\"latest_id\":\"$latest_id\"," + content+="\"latest_time\":\"$latest_time\"" + content+="}" + done + content+="]}" + + echo "$content" | runFileWrite "$output_dir/${app}.json" + done } diff --git a/scripts/webui/data/generators/backup/webui_backup_dashboard.sh b/scripts/webui/data/generators/backup/webui_backup_dashboard.sh index fe91b9e..800ef0d 100644 --- a/scripts/webui/data/generators/backup/webui_backup_dashboard.sh +++ b/scripts/webui/data/generators/backup/webui_backup_dashboard.sh @@ -48,11 +48,20 @@ webuiGenerateBackupDashboard() done < <(resticEnabledLocations) locations_json+="]" + # Fetch the primary location's snapshots ONCE, then derive every app tile + # and the system-config status below by filtering it in jq — instead of a + # restic spawn per app plus one for system. jq reproduces restic's + # --tag/--host filtering (empty host = no host filter, as restic does). + local primary_idx primary_snaps="[]" + primary_idx=$(resticEnabledLocations | head -1) + if [[ -n "$primary_idx" ]]; then + primary_snaps=$(engineSnapshotsJson "$primary_idx" 2>/dev/null) + [[ -z "$primary_snaps" ]] && primary_snaps="[]" + fi + local apps_json="[" first=true if [[ -f "$docker_dir/$db_file" ]]; then - local primary_idx - primary_idx=$(resticEnabledLocations | head -1) while IFS= read -r app; do [[ -z "$app" ]] && continue # The WebUI app is reproducible and skipped by backupAllApps — its @@ -60,12 +69,11 @@ webuiGenerateBackupDashboard() # perpetually-"No backup yet" tile for it. [[ "$app" == "libreportal" ]] && continue local latest_id="" latest_time="" - if [[ -n "$primary_idx" ]]; then - local snap_json - snap_json=$(engineSnapshotsJson "$primary_idx" "$app" "$CFG_INSTALL_NAME" 2>/dev/null) - latest_id=$(echo "$snap_json" | grep -o '"short_id":"[^"]*"' | tail -1 | cut -d'"' -f4) - latest_time=$(echo "$snap_json" | grep -o '"time":"[^"]*"' | tail -1 | cut -d'"' -f4) - fi + IFS=$'\t' read -r latest_id latest_time <<<"$(printf '%s' "$primary_snaps" | jq -r \ + --arg tag "app=$app" --arg host "$CFG_INSTALL_NAME" ' + [ .[] | select($host=="" or .hostname==$host) + | select((.tags // []) | index($tag)) ] | (.[-1] // {}) + | "\(.short_id // "")\t\(.time // "")"' 2>/dev/null)" $first || apps_json+="," first=false apps_json+="{\"app\":\"$app\",\"latest_snapshot\":\"$latest_id\",\"latest_time\":\"$latest_time\"}" @@ -75,14 +83,11 @@ webuiGenerateBackupDashboard() # System-config backup status (tag system=config) — its own tracked item. local system_id="" system_time="" - local sys_idx - sys_idx=$(resticEnabledLocations | head -1) - if [[ -n "$sys_idx" ]]; then - local sys_json - sys_json=$(engineSystemSnapshotsJson "$sys_idx" "$CFG_INSTALL_NAME" 2>/dev/null) - system_id=$(echo "$sys_json" | grep -o '"short_id":"[^"]*"' | tail -1 | cut -d'"' -f4) - system_time=$(echo "$sys_json" | grep -o '"time":"[^"]*"' | tail -1 | cut -d'"' -f4) - fi + IFS=$'\t' read -r system_id system_time <<<"$(printf '%s' "$primary_snaps" | jq -r \ + --arg tag "system=config" --arg host "$CFG_INSTALL_NAME" ' + [ .[] | select($host=="" or .hostname==$host) + | select((.tags // []) | index($tag)) ] | (.[-1] // {}) + | "\(.short_id // "")\t\(.time // "")"' 2>/dev/null)" local content="{" content+="\"generated_at\":\"$generated_at\"," diff --git a/scripts/webui/data/generators/backup/webui_backup_migrate.sh b/scripts/webui/data/generators/backup/webui_backup_migrate.sh index dd69d35..51c85ee 100644 --- a/scripts/webui/data/generators/backup/webui_backup_migrate.sh +++ b/scripts/webui/data/generators/backup/webui_backup_migrate.sh @@ -89,15 +89,18 @@ webuiGenerateBackupMigrate() local app while IFS= read -r app; do [[ -z "$app" ]] && continue - local app_snaps_json - app_snaps_json=$(engineSnapshotsJson "$idx" "$app" "$host" 2>/dev/null) - local count - count=$(printf '%s' "$app_snaps_json" | grep -oc '"short_id":"' || echo 0) - (( count == 0 )) && continue - - local latest_id latest_date - latest_id=$(printf '%s' "$app_snaps_json" | grep -o '"short_id":"[^"]*"' | tail -1 | cut -d'"' -f4) - latest_date=$(printf '%s' "$app_snaps_json" | grep -o '"time":"[^"]*"' | tail -1 | cut -d'"' -f4) + # Filter the location's already-fetched snapshot list in-process + # (jq == restic --tag/--host) rather than a restic spawn per + # (host, app). Also gives an accurate count — the old grep -oc + # over single-line JSON always reported 1. + local count latest_id latest_date + IFS=$'\t' read -r count latest_id latest_date <<<"$(printf '%s' "$all_json" | jq -r \ + --arg tag "app=$app" --arg host "$host" ' + [ .[] | select($host=="" or .hostname==$host) + | select((.tags // []) | index($tag)) ] as $f + | ($f[-1] // {}) as $l + | "\($f|length)\t\($l.short_id // "")\t\($l.time // "")"' 2>/dev/null)" + [[ -z "$count" || "$count" == "0" ]] && continue local opt_out opt_out=$(migrateUrlRewriteEnabled "$app" 2>/dev/null)