perf(backup): raw-data stats, dedupe repo stats, SSH connection reuse
Follow-up to the backup-refresh throttle/dedupe, cutting the cost of the remote pulls that do still happen. * restic stats now runs in --mode raw-data (restic_check.sh). The default restore-size mode walks every snapshot's tree to sum logical file sizes — the slowest restic op — just to fill a size readout. raw-data reads the index only and reports the repository's actual deduplicated on-disk size, which is exactly what the dashboard already labels "deduplicated, encrypted". raw-data omits total_file_count, so the per-location card now shows that location's snapshot count (already loaded client-side, and more useful for a backup repo) instead of a file count. * engineLocationStats now shares the same per-refresh memoiser as engineSnapshotsJson (engine_dispatch.sh). Both the locations and dashboard generators call it per location, so repo stats went from two restic calls per location per refresh to one. Factored the cache into _engineCachedPull. * SSH connection reuse for SFTP locations (backup_ssh.sh): ControlMaster=auto with a self-reaping ControlPersist master, so the several restic subprocesses a refresh/backup spawns against one location share a single authenticated connection instead of a fresh handshake each — the dominant per-call cost on a high-latency link. Toggle via CFG_BACKUP_SSH_MULTIPLEX. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Signed-off-by: librelad <librelad@digitalangels.vip>
This commit is contained in:
librelad
parent
f81bcbbd21
commit
71a02374d1
@ -8,3 +8,4 @@ CFG_BACKUP_STRATEGY=auto # Backup Strategy
|
||||
CFG_BACKUP_VERIFY_AFTER=true # Verify After Backup - Run integrity check after each backup
|
||||
CFG_BACKUP_VERIFY_DATA_PERCENT=5 # Verify Data Sample % - Percentage of repo data to checksum-verify weekly
|
||||
CFG_BACKUP_PARALLEL_REPOS=true # Parallel Repos - Push to all enabled locations in parallel
|
||||
CFG_BACKUP_SSH_MULTIPLEX=true # SSH Connection Reuse - Share one SSH connection across the restic calls to each SFTP location (faster on high-latency links). Disable if your SSH server rejects multiplexing.
|
||||
|
||||
@ -62,7 +62,7 @@ Object.assign(BackupPage.prototype, {
|
||||
</div>
|
||||
<div class="backup-repo-row-meta">
|
||||
${this.formatBytes(parseInt(r.total_size_bytes) || 0)}<br>
|
||||
<span class="backup-card-hint">${r.total_files || 0} files</span>
|
||||
<span class="backup-card-hint">${(() => { const n = this.snapshotsByLoc[r.idx]?.snapshots?.length ?? 0; return `${n} snapshot${n === 1 ? '' : 's'}`; })()}</span>
|
||||
</div>
|
||||
</div>
|
||||
`).join('');
|
||||
|
||||
@ -24,7 +24,21 @@ backupSshCommand()
|
||||
pass=$(resticLocationField "$idx" SSH_PASS)
|
||||
[[ -z "$auth" ]] && auth=key
|
||||
|
||||
local base="ssh -p $port -o StrictHostKeyChecking=accept-new"
|
||||
# Connection multiplexing: a single WebUI refresh (repo stats + snapshot
|
||||
# list) and each backup push run several restic subprocesses against the
|
||||
# same location; ControlMaster lets them share one authenticated SSH
|
||||
# connection instead of a fresh handshake each — the dominant per-call cost
|
||||
# on a high-latency link. auto = reuse an existing master or open one; the
|
||||
# master self-reaps ControlPersist seconds after its last channel closes.
|
||||
# %C keeps the socket path short and unique per (host,port,user). Opt out
|
||||
# with CFG_BACKUP_SSH_MULTIPLEX=false.
|
||||
local mux=""
|
||||
if [[ "${CFG_BACKUP_SSH_MULTIPLEX:-true}" != "false" ]]; then
|
||||
local mux_dir="${TMPDIR:-/tmp}"; mux_dir="${mux_dir%/}"
|
||||
mux=" -o ControlMaster=auto -o ControlPath=${mux_dir}/lp-bkmux-%C -o ControlPersist=30"
|
||||
fi
|
||||
|
||||
local base="ssh -p $port -o StrictHostKeyChecking=accept-new${mux}"
|
||||
[[ "$mode" == "sftp" ]] && local suffix=" -s sftp" || local suffix=""
|
||||
|
||||
if [[ "$auth" == "password" ]]; then
|
||||
|
||||
@ -34,6 +34,28 @@ engineDispatch()
|
||||
"$fn" "$@"
|
||||
}
|
||||
|
||||
# Transparent per-refresh memoiser for read-only remote pulls. The WebUI backup
|
||||
# refresh reads the same restic data from several generators per location — the
|
||||
# snapshot list (dashboard/snapshots/app-status/migrate) and repo stats
|
||||
# (locations + dashboard) — which on a remote (SSH) repo is one round-trip each.
|
||||
# When LP_SNAP_CACHE_DIR is set (webui_updater wraps the refresh chain with it),
|
||||
# the first successful pull for a cache key is written to a file the siblings
|
||||
# reuse; empty/failed pulls fall through so a transient error is never cached.
|
||||
# Unset dir → every call runs live.
|
||||
_engineCachedPull() {
|
||||
local _key="$1"; shift
|
||||
if [[ -n "${LP_SNAP_CACHE_DIR:-}" ]]; then
|
||||
local _cf="${LP_SNAP_CACHE_DIR}/${_key}"
|
||||
[[ -s "$_cf" ]] && { cat "$_cf"; return 0; }
|
||||
local _out _rc
|
||||
_out=$("$@"); _rc=$?
|
||||
[[ $_rc -eq 0 && -n "$_out" ]] && printf '%s' "$_out" > "$_cf" 2>/dev/null
|
||||
printf '%s' "$_out"
|
||||
return $_rc
|
||||
fi
|
||||
"$@"
|
||||
}
|
||||
|
||||
# ---- Idx-scoped dispatchers ----------------------------------------------------
|
||||
|
||||
# Local/removable-drive safety guard runs before init, readiness, and any backup
|
||||
@ -43,7 +65,7 @@ engineInitLocation() { local i="$1"; backupLocationLocalGuard "$i" || re
|
||||
engineEnsureLocationReady() { local i="$1"; backupLocationLocalGuard "$i" || return 1; engineDispatch "$(engineForLocation "$i")EnsureLocationReady" "$i"; }
|
||||
enginePasswordEnsure() { local i="$1"; engineDispatch "$(engineForLocation "$i")PasswordEnsure" "$i"; }
|
||||
engineLocationUri() { local i="$1"; engineDispatch "$(engineForLocation "$i")LocationUri" "$i"; }
|
||||
engineLocationStats() { local i="$1"; engineDispatch "$(engineForLocation "$i")LocationStats" "$i"; }
|
||||
engineLocationStats() { local i="$1"; _engineCachedPull "stats_${i}.json" engineDispatch "$(engineForLocation "$i")LocationStats" "$i"; }
|
||||
engineEnvExport() { local i="$1"; engineDispatch "$(engineForLocation "$i")EnvExport" "$i"; }
|
||||
engineEnvUnset() { local i="$1"; engineDispatch "$(engineForLocation "${i:-1}")EnvUnset"; }
|
||||
|
||||
@ -52,23 +74,14 @@ engineBackupSystem() { local i="$1"; shift; backupLocationLocalGuard "$i
|
||||
engineRestoreSystemLatest() { local i="$1"; shift; engineDispatch "$(engineForLocation "$i")RestoreSystemLatest" "$i" "$@"; }
|
||||
engineRestoreSnapshot() { local i="$1"; shift; engineDispatch "$(engineForLocation "$i")RestoreSnapshot" "$i" "$@"; }
|
||||
engineSnapshotLatestId() { local i="$1"; shift; engineDispatch "$(engineForLocation "$i")SnapshotLatestId" "$i" "$@"; }
|
||||
# Whole-repo snapshot list. The WebUI backup refresh pulls this the same way
|
||||
# (no filters) from four generators per location — dashboard, snapshots,
|
||||
# app-status and migrate — which on a remote (SSH) repo is four identical restic
|
||||
# round-trips. When LP_SNAP_CACHE_DIR is set (webui_updater wraps the refresh
|
||||
# chain with it), memoise the first unfiltered pull per location to a file the
|
||||
# sibling generators reuse; filtered/parameterised calls (extra args) and any
|
||||
# failed/empty pull always fall through to a live restic call.
|
||||
# Whole-repo snapshot list. Only the unfiltered pull (no extra args) is memoised
|
||||
# — the four generators that read it that way per location share one round-trip;
|
||||
# filtered/parameterised calls always run live.
|
||||
engineSnapshotsJson() {
|
||||
local i="$1"; shift
|
||||
if [[ $# -eq 0 && -n "${LP_SNAP_CACHE_DIR:-}" ]]; then
|
||||
local _cf="${LP_SNAP_CACHE_DIR}/snapshots_${i}.json"
|
||||
[[ -s "$_cf" ]] && { cat "$_cf"; return 0; }
|
||||
local _out _rc
|
||||
_out=$(engineDispatch "$(engineForLocation "$i")SnapshotsJson" "$i"); _rc=$?
|
||||
[[ $_rc -eq 0 && -n "$_out" ]] && printf '%s' "$_out" > "$_cf" 2>/dev/null
|
||||
printf '%s' "$_out"
|
||||
return $_rc
|
||||
if [[ $# -eq 0 ]]; then
|
||||
_engineCachedPull "snapshots_${i}.json" engineDispatch "$(engineForLocation "$i")SnapshotsJson" "$i"
|
||||
return $?
|
||||
fi
|
||||
engineDispatch "$(engineForLocation "$i")SnapshotsJson" "$i" "$@"
|
||||
}
|
||||
|
||||
@ -42,7 +42,13 @@ resticLocationStats()
|
||||
{
|
||||
local idx="$1"
|
||||
resticEnvExport "$idx" || return 1
|
||||
runBackupOp restic stats --json --no-lock 2>/dev/null
|
||||
# raw-data mode reads the index only and reports the repository's actual
|
||||
# deduplicated on-disk size — fast, and the number the dashboard already
|
||||
# labels "deduplicated, encrypted". The default restore-size mode instead
|
||||
# walks every snapshot's tree to sum logical file sizes: the slowest restic
|
||||
# op, and pointless for a size readout. (raw-data omits total_file_count;
|
||||
# the dashboard shows snapshot count for that slot instead.)
|
||||
runBackupOp restic stats --json --no-lock --mode raw-data 2>/dev/null
|
||||
local rc=$?
|
||||
resticEnvUnset
|
||||
return $rc
|
||||
|
||||
@ -21,13 +21,11 @@ webuiGenerateBackupDashboard()
|
||||
uri=$(resticLocationUri "$idx")
|
||||
append_only="false"; resticLocationAppendOnly "$idx" && append_only="true"
|
||||
|
||||
local stats_json total_size="0" total_files="0"
|
||||
local stats_json total_size="0"
|
||||
stats_json=$(engineLocationStats "$idx" 2>/dev/null)
|
||||
if [[ -n "$stats_json" ]]; then
|
||||
total_size=$(echo "$stats_json" | grep -o '"total_size":[0-9]*' | head -1 | cut -d':' -f2)
|
||||
total_files=$(echo "$stats_json" | grep -o '"total_file_count":[0-9]*' | head -1 | cut -d':' -f2)
|
||||
[[ -z "$total_size" ]] && total_size="0"
|
||||
[[ -z "$total_files" ]] && total_files="0"
|
||||
fi
|
||||
|
||||
local name_esc uri_esc
|
||||
@ -42,8 +40,7 @@ webuiGenerateBackupDashboard()
|
||||
locations_json+="\"type\":\"$(resticLocationType "$idx")\","
|
||||
locations_json+="\"uri\":\"$uri_esc\","
|
||||
locations_json+="\"append_only\":$append_only,"
|
||||
locations_json+="\"total_size_bytes\":$total_size,"
|
||||
locations_json+="\"total_files\":$total_files"
|
||||
locations_json+="\"total_size_bytes\":$total_size"
|
||||
locations_json+="}"
|
||||
done < <(resticEnabledLocations)
|
||||
locations_json+="]"
|
||||
|
||||
@ -40,15 +40,13 @@ webuiGenerateBackupLocations()
|
||||
pass_exists="true"
|
||||
fi
|
||||
|
||||
local total_size="0" total_files="0"
|
||||
local total_size="0"
|
||||
if [[ "$enabled" == "true" && "$pass_exists" == "true" ]]; then
|
||||
local stats_json
|
||||
stats_json=$(engineLocationStats "$idx" 2>/dev/null)
|
||||
if [[ -n "$stats_json" ]]; then
|
||||
total_size=$(echo "$stats_json" | grep -o '"total_size":[0-9]*' | head -1 | cut -d':' -f2)
|
||||
total_files=$(echo "$stats_json" | grep -o '"total_file_count":[0-9]*' | head -1 | cut -d':' -f2)
|
||||
[[ -z "$total_size" ]] && total_size="0"
|
||||
[[ -z "$total_files" ]] && total_files="0"
|
||||
fi
|
||||
fi
|
||||
|
||||
@ -80,7 +78,6 @@ webuiGenerateBackupLocations()
|
||||
content+="\"custom_retention\":$custom_retention,"
|
||||
content+="\"password_exists\":$pass_exists,"
|
||||
content+="\"total_size_bytes\":$total_size,"
|
||||
content+="\"total_files\":$total_files,"
|
||||
content+="\"path\":\"$(printf '%s' "$(resticLocationField "$idx" PATH)" | sed 's/\\/\\\\/g; s/"/\\"/g')\","
|
||||
content+="\"path_mode\":\"$(resticLocationField "$idx" PATH_MODE)\","
|
||||
content+="\"ssh_user\":\"$(printf '%s' "$(resticLocationField "$idx" SSH_USER)" | sed 's/\\/\\\\/g; s/"/\\"/g')\","
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user