From 71a02374d10421fd9db8599cbeb7c50991f6fbab Mon Sep 17 00:00:00 2001 From: librelad Date: Fri, 17 Jul 2026 21:56:55 +0100 Subject: [PATCH] perf(backup): raw-data stats, dedupe repo stats, SSH connection reuse MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up to the backup-refresh throttle/dedupe, cutting the cost of the remote pulls that do still happen. * restic stats now runs in --mode raw-data (restic_check.sh). The default restore-size mode walks every snapshot's tree to sum logical file sizes — the slowest restic op — just to fill a size readout. raw-data reads the index only and reports the repository's actual deduplicated on-disk size, which is exactly what the dashboard already labels "deduplicated, encrypted". raw-data omits total_file_count, so the per-location card now shows that location's snapshot count (already loaded client-side, and more useful for a backup repo) instead of a file count. * engineLocationStats now shares the same per-refresh memoiser as engineSnapshotsJson (engine_dispatch.sh). Both the locations and dashboard generators call it per location, so repo stats went from two restic calls per location per refresh to one. Factored the cache into _engineCachedPull. * SSH connection reuse for SFTP locations (backup_ssh.sh): ControlMaster=auto with a self-reaping ControlPersist master, so the several restic subprocesses a refresh/backup spawns against one location share a single authenticated connection instead of a fresh handshake each — the dominant per-call cost on a high-latency link. Toggle via CFG_BACKUP_SSH_MULTIPLEX. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: librelad --- configs/backup/backup_engine | 1 + .../backup/dashboard/js/backup-dashboard.js | 2 +- scripts/backup/engine/backup_ssh.sh | 16 ++++++- scripts/backup/engine/engine_dispatch.sh | 45 ++++++++++++------- scripts/backup/engine/restic_check.sh | 8 +++- .../backup/webui_backup_dashboard.sh | 7 +-- .../backup/webui_backup_locations.sh | 5 +-- 7 files changed, 56 insertions(+), 28 deletions(-) diff --git a/configs/backup/backup_engine b/configs/backup/backup_engine index 29c8d36..14cbb5a 100644 --- a/configs/backup/backup_engine +++ b/configs/backup/backup_engine @@ -8,3 +8,4 @@ CFG_BACKUP_STRATEGY=auto # Backup Strategy CFG_BACKUP_VERIFY_AFTER=true # Verify After Backup - Run integrity check after each backup CFG_BACKUP_VERIFY_DATA_PERCENT=5 # Verify Data Sample % - Percentage of repo data to checksum-verify weekly CFG_BACKUP_PARALLEL_REPOS=true # Parallel Repos - Push to all enabled locations in parallel +CFG_BACKUP_SSH_MULTIPLEX=true # SSH Connection Reuse - Share one SSH connection across the restic calls to each SFTP location (faster on high-latency links). Disable if your SSH server rejects multiplexing. diff --git a/containers/libreportal/frontend/components/backup/dashboard/js/backup-dashboard.js b/containers/libreportal/frontend/components/backup/dashboard/js/backup-dashboard.js index aeb18b6..ce99fbe 100644 --- a/containers/libreportal/frontend/components/backup/dashboard/js/backup-dashboard.js +++ b/containers/libreportal/frontend/components/backup/dashboard/js/backup-dashboard.js @@ -62,7 +62,7 @@ Object.assign(BackupPage.prototype, {
${this.formatBytes(parseInt(r.total_size_bytes) || 0)}
- ${r.total_files || 0} files + ${(() => { const n = this.snapshotsByLoc[r.idx]?.snapshots?.length ?? 0; return `${n} snapshot${n === 1 ? '' : 's'}`; })()}
`).join(''); diff --git a/scripts/backup/engine/backup_ssh.sh b/scripts/backup/engine/backup_ssh.sh index 2251a2f..b05f596 100644 --- a/scripts/backup/engine/backup_ssh.sh +++ b/scripts/backup/engine/backup_ssh.sh @@ -24,7 +24,21 @@ backupSshCommand() pass=$(resticLocationField "$idx" SSH_PASS) [[ -z "$auth" ]] && auth=key - local base="ssh -p $port -o StrictHostKeyChecking=accept-new" + # Connection multiplexing: a single WebUI refresh (repo stats + snapshot + # list) and each backup push run several restic subprocesses against the + # same location; ControlMaster lets them share one authenticated SSH + # connection instead of a fresh handshake each — the dominant per-call cost + # on a high-latency link. auto = reuse an existing master or open one; the + # master self-reaps ControlPersist seconds after its last channel closes. + # %C keeps the socket path short and unique per (host,port,user). Opt out + # with CFG_BACKUP_SSH_MULTIPLEX=false. + local mux="" + if [[ "${CFG_BACKUP_SSH_MULTIPLEX:-true}" != "false" ]]; then + local mux_dir="${TMPDIR:-/tmp}"; mux_dir="${mux_dir%/}" + mux=" -o ControlMaster=auto -o ControlPath=${mux_dir}/lp-bkmux-%C -o ControlPersist=30" + fi + + local base="ssh -p $port -o StrictHostKeyChecking=accept-new${mux}" [[ "$mode" == "sftp" ]] && local suffix=" -s sftp" || local suffix="" if [[ "$auth" == "password" ]]; then diff --git a/scripts/backup/engine/engine_dispatch.sh b/scripts/backup/engine/engine_dispatch.sh index 616f4ee..0b54449 100644 --- a/scripts/backup/engine/engine_dispatch.sh +++ b/scripts/backup/engine/engine_dispatch.sh @@ -34,6 +34,28 @@ engineDispatch() "$fn" "$@" } +# Transparent per-refresh memoiser for read-only remote pulls. The WebUI backup +# refresh reads the same restic data from several generators per location — the +# snapshot list (dashboard/snapshots/app-status/migrate) and repo stats +# (locations + dashboard) — which on a remote (SSH) repo is one round-trip each. +# When LP_SNAP_CACHE_DIR is set (webui_updater wraps the refresh chain with it), +# the first successful pull for a cache key is written to a file the siblings +# reuse; empty/failed pulls fall through so a transient error is never cached. +# Unset dir → every call runs live. +_engineCachedPull() { + local _key="$1"; shift + if [[ -n "${LP_SNAP_CACHE_DIR:-}" ]]; then + local _cf="${LP_SNAP_CACHE_DIR}/${_key}" + [[ -s "$_cf" ]] && { cat "$_cf"; return 0; } + local _out _rc + _out=$("$@"); _rc=$? + [[ $_rc -eq 0 && -n "$_out" ]] && printf '%s' "$_out" > "$_cf" 2>/dev/null + printf '%s' "$_out" + return $_rc + fi + "$@" +} + # ---- Idx-scoped dispatchers ---------------------------------------------------- # Local/removable-drive safety guard runs before init, readiness, and any backup @@ -43,7 +65,7 @@ engineInitLocation() { local i="$1"; backupLocationLocalGuard "$i" || re engineEnsureLocationReady() { local i="$1"; backupLocationLocalGuard "$i" || return 1; engineDispatch "$(engineForLocation "$i")EnsureLocationReady" "$i"; } enginePasswordEnsure() { local i="$1"; engineDispatch "$(engineForLocation "$i")PasswordEnsure" "$i"; } engineLocationUri() { local i="$1"; engineDispatch "$(engineForLocation "$i")LocationUri" "$i"; } -engineLocationStats() { local i="$1"; engineDispatch "$(engineForLocation "$i")LocationStats" "$i"; } +engineLocationStats() { local i="$1"; _engineCachedPull "stats_${i}.json" engineDispatch "$(engineForLocation "$i")LocationStats" "$i"; } engineEnvExport() { local i="$1"; engineDispatch "$(engineForLocation "$i")EnvExport" "$i"; } engineEnvUnset() { local i="$1"; engineDispatch "$(engineForLocation "${i:-1}")EnvUnset"; } @@ -52,23 +74,14 @@ engineBackupSystem() { local i="$1"; shift; backupLocationLocalGuard "$i engineRestoreSystemLatest() { local i="$1"; shift; engineDispatch "$(engineForLocation "$i")RestoreSystemLatest" "$i" "$@"; } engineRestoreSnapshot() { local i="$1"; shift; engineDispatch "$(engineForLocation "$i")RestoreSnapshot" "$i" "$@"; } engineSnapshotLatestId() { local i="$1"; shift; engineDispatch "$(engineForLocation "$i")SnapshotLatestId" "$i" "$@"; } -# Whole-repo snapshot list. The WebUI backup refresh pulls this the same way -# (no filters) from four generators per location — dashboard, snapshots, -# app-status and migrate — which on a remote (SSH) repo is four identical restic -# round-trips. When LP_SNAP_CACHE_DIR is set (webui_updater wraps the refresh -# chain with it), memoise the first unfiltered pull per location to a file the -# sibling generators reuse; filtered/parameterised calls (extra args) and any -# failed/empty pull always fall through to a live restic call. +# Whole-repo snapshot list. Only the unfiltered pull (no extra args) is memoised +# — the four generators that read it that way per location share one round-trip; +# filtered/parameterised calls always run live. engineSnapshotsJson() { local i="$1"; shift - if [[ $# -eq 0 && -n "${LP_SNAP_CACHE_DIR:-}" ]]; then - local _cf="${LP_SNAP_CACHE_DIR}/snapshots_${i}.json" - [[ -s "$_cf" ]] && { cat "$_cf"; return 0; } - local _out _rc - _out=$(engineDispatch "$(engineForLocation "$i")SnapshotsJson" "$i"); _rc=$? - [[ $_rc -eq 0 && -n "$_out" ]] && printf '%s' "$_out" > "$_cf" 2>/dev/null - printf '%s' "$_out" - return $_rc + if [[ $# -eq 0 ]]; then + _engineCachedPull "snapshots_${i}.json" engineDispatch "$(engineForLocation "$i")SnapshotsJson" "$i" + return $? fi engineDispatch "$(engineForLocation "$i")SnapshotsJson" "$i" "$@" } diff --git a/scripts/backup/engine/restic_check.sh b/scripts/backup/engine/restic_check.sh index f6e43b0..df429d7 100644 --- a/scripts/backup/engine/restic_check.sh +++ b/scripts/backup/engine/restic_check.sh @@ -42,7 +42,13 @@ resticLocationStats() { local idx="$1" resticEnvExport "$idx" || return 1 - runBackupOp restic stats --json --no-lock 2>/dev/null + # raw-data mode reads the index only and reports the repository's actual + # deduplicated on-disk size — fast, and the number the dashboard already + # labels "deduplicated, encrypted". The default restore-size mode instead + # walks every snapshot's tree to sum logical file sizes: the slowest restic + # op, and pointless for a size readout. (raw-data omits total_file_count; + # the dashboard shows snapshot count for that slot instead.) + runBackupOp restic stats --json --no-lock --mode raw-data 2>/dev/null local rc=$? resticEnvUnset return $rc diff --git a/scripts/webui/data/generators/backup/webui_backup_dashboard.sh b/scripts/webui/data/generators/backup/webui_backup_dashboard.sh index 800ef0d..f1400a2 100644 --- a/scripts/webui/data/generators/backup/webui_backup_dashboard.sh +++ b/scripts/webui/data/generators/backup/webui_backup_dashboard.sh @@ -21,13 +21,11 @@ webuiGenerateBackupDashboard() uri=$(resticLocationUri "$idx") append_only="false"; resticLocationAppendOnly "$idx" && append_only="true" - local stats_json total_size="0" total_files="0" + local stats_json total_size="0" stats_json=$(engineLocationStats "$idx" 2>/dev/null) if [[ -n "$stats_json" ]]; then total_size=$(echo "$stats_json" | grep -o '"total_size":[0-9]*' | head -1 | cut -d':' -f2) - total_files=$(echo "$stats_json" | grep -o '"total_file_count":[0-9]*' | head -1 | cut -d':' -f2) [[ -z "$total_size" ]] && total_size="0" - [[ -z "$total_files" ]] && total_files="0" fi local name_esc uri_esc @@ -42,8 +40,7 @@ webuiGenerateBackupDashboard() locations_json+="\"type\":\"$(resticLocationType "$idx")\"," locations_json+="\"uri\":\"$uri_esc\"," locations_json+="\"append_only\":$append_only," - locations_json+="\"total_size_bytes\":$total_size," - locations_json+="\"total_files\":$total_files" + locations_json+="\"total_size_bytes\":$total_size" locations_json+="}" done < <(resticEnabledLocations) locations_json+="]" diff --git a/scripts/webui/data/generators/backup/webui_backup_locations.sh b/scripts/webui/data/generators/backup/webui_backup_locations.sh index 52908e8..4a01739 100644 --- a/scripts/webui/data/generators/backup/webui_backup_locations.sh +++ b/scripts/webui/data/generators/backup/webui_backup_locations.sh @@ -40,15 +40,13 @@ webuiGenerateBackupLocations() pass_exists="true" fi - local total_size="0" total_files="0" + local total_size="0" if [[ "$enabled" == "true" && "$pass_exists" == "true" ]]; then local stats_json stats_json=$(engineLocationStats "$idx" 2>/dev/null) if [[ -n "$stats_json" ]]; then total_size=$(echo "$stats_json" | grep -o '"total_size":[0-9]*' | head -1 | cut -d':' -f2) - total_files=$(echo "$stats_json" | grep -o '"total_file_count":[0-9]*' | head -1 | cut -d':' -f2) [[ -z "$total_size" ]] && total_size="0" - [[ -z "$total_files" ]] && total_files="0" fi fi @@ -80,7 +78,6 @@ webuiGenerateBackupLocations() content+="\"custom_retention\":$custom_retention," content+="\"password_exists\":$pass_exists," content+="\"total_size_bytes\":$total_size," - content+="\"total_files\":$total_files," content+="\"path\":\"$(printf '%s' "$(resticLocationField "$idx" PATH)" | sed 's/\\/\\\\/g; s/"/\\"/g')\"," content+="\"path_mode\":\"$(resticLocationField "$idx" PATH_MODE)\"," content+="\"ssh_user\":\"$(printf '%s' "$(resticLocationField "$idx" SSH_USER)" | sed 's/\\/\\\\/g; s/"/\\"/g')\","