fix(backup): honest failures, rootless docker in capture paths, speedtest key capture
- backup_files.sh / backup_db.sh: every docker exec/run in the capture, sidecar-discovery, rehydrate and DB-import paths now goes through runFileOp — bare docker can't reach the rootless daemon socket, which made live capture fail (and silently bounce containers) on every rootless install, and would have broken DB restores the same way. - capture/rehydrate stderr is kept and printed on failure instead of being discarded, with a clear message when the image has no tar. - backup_app_start.sh: when no location produced a complete snapshot the backup now returns 1 — the task is marked failed instead of logging a nonexistent/incomplete backup as a success and skipping verification. - restic engine: on restic exit 3 the orphan incomplete snapshot is called out explicitly so nobody restores it believing it is whole. - speedtest: capture /config through the container (root-owned TLS key and logrotate state are unreadable from the host), which also flips its auto strategy to live — no more container stop per backup. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
librelad
parent
b3b9f9a18b
commit
a86c142e74
@ -30,6 +30,9 @@ services:
|
||||
labels:
|
||||
libreportal.category: "CATEGORY_DATA" #LIBREPORTAL|CATEGORY_TAG|CATEGORY_DATA
|
||||
libreportal.title: "TITLE_DATA" #LIBREPORTAL|TITLE_TAG|TITLE_DATA
|
||||
# /config holds root-and-app-owned files (TLS key, logs) the backup user
|
||||
# can't read from the host; capture them through the container instead.
|
||||
libreportal.backup.files: "speedtest-service:/config:config"
|
||||
traefik.enable: TRAEFIK_ENABLE_DATA #LIBREPORTAL|TRAEFIK_ENABLE_TAG|TRAEFIK_ENABLE_DATA
|
||||
# TRAEFIK_PORT_1_BEGIN
|
||||
traefik.http.routers.speedtest-service.entrypoints: web,websecure
|
||||
|
||||
@ -76,15 +76,21 @@ backupAppStart()
|
||||
local primary_snapshot_id=""
|
||||
local primary_idx=""
|
||||
local first_loc=true
|
||||
local snap_ok=0 snap_failed=0
|
||||
local idx
|
||||
while IFS= read -r idx; do
|
||||
[[ -z "$idx" ]] && continue
|
||||
local snap_id
|
||||
snap_id=$(engineBackupApp "$idx" "$stored_app_name" "$manifest_sha")
|
||||
if [[ "$first_loc" == true && -n "$snap_id" ]]; then
|
||||
primary_snapshot_id="$snap_id"
|
||||
primary_idx="$idx"
|
||||
first_loc=false
|
||||
if [[ -n "$snap_id" ]]; then
|
||||
snap_ok=$((snap_ok + 1))
|
||||
if [[ "$first_loc" == true ]]; then
|
||||
primary_snapshot_id="$snap_id"
|
||||
primary_idx="$idx"
|
||||
first_loc=false
|
||||
fi
|
||||
else
|
||||
snap_failed=$((snap_failed + 1))
|
||||
fi
|
||||
done < <(resticEnabledLocations)
|
||||
|
||||
@ -104,6 +110,19 @@ backupAppStart()
|
||||
echo ""
|
||||
backupAppRunHook "$stored_app_name" post
|
||||
|
||||
# Containers are back up; now be honest about the snapshot result. With no
|
||||
# good snapshot on any location there is nothing to verify, retain, or
|
||||
# record — returning non-zero here is what marks the task failed instead of
|
||||
# logging a backup that doesn't exist (or is missing files) as a success.
|
||||
if [[ $snap_ok -eq 0 ]]; then
|
||||
isError "Backup of $stored_app_name FAILED — no complete snapshot was created on any location (see errors above)"
|
||||
menu_number=0
|
||||
return 1
|
||||
fi
|
||||
if [[ $snap_failed -gt 0 ]]; then
|
||||
isNotice "Backup of $stored_app_name succeeded on $snap_ok location(s) but failed on $snap_failed — check the errors above"
|
||||
fi
|
||||
|
||||
if [[ "$CFG_BACKUP_VERIFY_AFTER" == "true" && -n "$primary_snapshot_id" ]]; then
|
||||
((menu_number++))
|
||||
echo ""
|
||||
|
||||
@ -185,16 +185,16 @@ _backupDbImport()
|
||||
local kind="$1" container="$2" dump="$3"
|
||||
case "$kind" in
|
||||
postgres)
|
||||
runFileOp gzip -dc "$dump" | docker exec -i "$container" sh -c \
|
||||
runFileOp gzip -dc "$dump" | runFileOp docker exec -i "$container" sh -c \
|
||||
'export PGPASSWORD="${POSTGRES_PASSWORD:-}"; psql -v ON_ERROR_STOP=1 -U "${POSTGRES_USER:-postgres}" -d "${POSTGRES_DB:-${POSTGRES_USER:-postgres}}"' >/dev/null 2>&1 ;;
|
||||
mongo)
|
||||
# --drop replaces each collection as it is restored, which is what
|
||||
# makes a re-run idempotent (the retry loop in the caller depends on
|
||||
# that, exactly like pg_dump --clean --if-exists).
|
||||
runFileOp gzip -dc "$dump" | docker exec -i "$container" sh -c \
|
||||
runFileOp gzip -dc "$dump" | runFileOp docker exec -i "$container" sh -c \
|
||||
"$_backup_mongo_auth_sh"' mongorestore "$@" --archive --drop --quiet' >/dev/null 2>&1 ;;
|
||||
*)
|
||||
runFileOp gzip -dc "$dump" | docker exec -i "$container" sh -c \
|
||||
runFileOp gzip -dc "$dump" | runFileOp docker exec -i "$container" sh -c \
|
||||
'RP="${MARIADB_ROOT_PASSWORD:-$MYSQL_ROOT_PASSWORD}"; (mariadb -uroot -p"$RP" 2>/dev/null || mysql -uroot -p"$RP")' >/dev/null 2>&1 ;;
|
||||
esac
|
||||
}
|
||||
|
||||
@ -54,6 +54,13 @@ resticBackupAppToLocation()
|
||||
else
|
||||
isError "Backup to $loc_name failed for $app_name" >&2
|
||||
echo "$output" | tail -10 >&2
|
||||
# restic exit 3 = snapshot written but some source files were
|
||||
# unreadable. It stays in the repo but is NOT reported as this run's
|
||||
# result — an incomplete snapshot must never be recorded as a good
|
||||
# backup (it can be missing exactly the files that matter, e.g. keys).
|
||||
if [[ $rc -eq 3 && -n "$snapshot_id" ]]; then
|
||||
isError "An INCOMPLETE snapshot ${snapshot_id:0:8} was still written to $loc_name — it is missing the unreadable files listed above; do not rely on it" >&2
|
||||
fi
|
||||
fi
|
||||
|
||||
resticEnvUnset
|
||||
@ -96,6 +103,9 @@ resticBackupSystemToLocation()
|
||||
else
|
||||
isError "System config backup to $loc_name failed" >&2
|
||||
echo "$output" | tail -10 >&2
|
||||
if [[ $rc -eq 3 && -n "$snapshot_id" ]]; then
|
||||
isError "An INCOMPLETE snapshot ${snapshot_id:0:8} was still written to $loc_name — it is missing the unreadable files listed above; do not rely on it" >&2
|
||||
fi
|
||||
fi
|
||||
|
||||
resticEnvUnset
|
||||
|
||||
@ -86,7 +86,15 @@ backupFilesCapture()
|
||||
runFileOp rm -rf "$stage" 2>/dev/null
|
||||
runFileOp mkdir -p "$stage"
|
||||
# Read in the container's namespace, write the plain tree to staging.
|
||||
if docker exec "$container" tar -C "$cpath" -cf - . 2>/dev/null | runFileOp tar -xf - -C "$stage" 2>/dev/null; then
|
||||
# docker must go through runFileOp: on rootless installs the daemon
|
||||
# socket lives in the install user's runtime dir and only the wrapper
|
||||
# sets DOCKER_HOST for it — bare `docker` can't connect at all. Keep
|
||||
# stderr in a scratch file so a failure can say why.
|
||||
local err_file capture_status
|
||||
err_file=$(mktemp "${TMPDIR:-/tmp}/lp-capture-XXXXXX.err")
|
||||
runFileOp docker exec "$container" tar -C "$cpath" -cf - . 2>"$err_file" | runFileOp tar -xf - -C "$stage" 2>>"$err_file"
|
||||
capture_status=("${PIPESTATUS[@]}")
|
||||
if [[ ${capture_status[0]} -eq 0 && ${capture_status[1]} -eq 0 ]]; then
|
||||
# The capture preserves the app's ownership (e.g. www-data, 0640),
|
||||
# which the backup user still couldn't read. Hand the staging tree to
|
||||
# the backup user so restic can read it; modes are unchanged, so the
|
||||
@ -102,7 +110,7 @@ backupFilesCapture()
|
||||
if [[ -z "$uid" || -z "$gid" ]]; then
|
||||
local meta_file="$app_dir/$backup_files_stage_subdir/$subdir.lp-owner"
|
||||
local discovered
|
||||
discovered=$(docker exec "$container" stat -c '%u:%g' "$cpath" 2>/dev/null)
|
||||
discovered=$(runFileOp docker exec "$container" stat -c '%u:%g' "$cpath" 2>/dev/null)
|
||||
if [[ -n "$discovered" ]]; then
|
||||
echo "$discovered" | runFileWrite "$meta_file" 2>/dev/null
|
||||
runFileOp chown "$docker_install_user":"$docker_install_user" "$meta_file" 2>/dev/null
|
||||
@ -111,9 +119,18 @@ backupFilesCapture()
|
||||
|
||||
isSuccessful "captured $subdir ($(du -sh "$stage" 2>/dev/null | cut -f1))"
|
||||
else
|
||||
isError "capture of $subdir from $container failed"
|
||||
if grep -q "executable file not found" "$err_file" 2>/dev/null; then
|
||||
isError "capture of $subdir from $container failed — the image has no tar, so live capture cannot work for this app"
|
||||
else
|
||||
isError "capture of $subdir from $container failed:"
|
||||
fi
|
||||
local err_line
|
||||
while IFS= read -r err_line; do
|
||||
isError " $err_line"
|
||||
done < <(head -5 "$err_file" 2>/dev/null)
|
||||
rc=1
|
||||
fi
|
||||
rm -f "$err_file"
|
||||
done < <(backupFilesDescriptors "$app")
|
||||
|
||||
return $rc
|
||||
@ -169,12 +186,22 @@ restoreFilesRehydratePreStart()
|
||||
# Helper runs as in-namespace root: it can clear/create the dir under the
|
||||
# app dir, extract the streamed tree, and chown to the app's uid:gid
|
||||
# (which maps to the right owner in rooted and rootless alike).
|
||||
if runFileOp tar -C "$stage" -cf - . 2>/dev/null | docker run --rm -i \
|
||||
# docker goes through runFileOp for the rootless socket, same as capture.
|
||||
local err_file restore_status
|
||||
err_file=$(mktemp "${TMPDIR:-/tmp}/lp-rehydrate-XXXXXX.err")
|
||||
runFileOp tar -C "$stage" -cf - . 2>"$err_file" | runFileOp docker run --rm -i \
|
||||
-v "$app_dir:/parent" "$backup_files_helper_image" \
|
||||
sh -c "rm -rf '/parent/$subdir' && mkdir -p '/parent/$subdir' && tar -C '/parent/$subdir' -xf - && chown -R $uid:$gid '/parent/$subdir'" 2>/dev/null; then
|
||||
sh -c "rm -rf '/parent/$subdir' && mkdir -p '/parent/$subdir' && tar -C '/parent/$subdir' -xf - && chown -R $uid:$gid '/parent/$subdir'" 2>>"$err_file"
|
||||
restore_status=("${PIPESTATUS[@]}")
|
||||
if [[ ${restore_status[0]} -eq 0 && ${restore_status[1]} -eq 0 ]]; then
|
||||
isSuccessful "restored $subdir"
|
||||
else
|
||||
isError "restoring $subdir failed"
|
||||
isError "restoring $subdir failed:"
|
||||
local err_line
|
||||
while IFS= read -r err_line; do
|
||||
isError " $err_line"
|
||||
done < <(head -5 "$err_file" 2>/dev/null)
|
||||
fi
|
||||
rm -f "$err_file"
|
||||
done < <(backupFilesDescriptors "$app")
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user