From a86c142e74414dc77e44fe430c38b0cfe0ffb2cc Mon Sep 17 00:00:00 2001 From: librelad Date: Fri, 21 Aug 2026 00:31:46 +0100 Subject: [PATCH] fix(backup): honest failures, rootless docker in capture paths, speedtest key capture MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - backup_files.sh / backup_db.sh: every docker exec/run in the capture, sidecar-discovery, rehydrate and DB-import paths now goes through runFileOp — bare docker can't reach the rootless daemon socket, which made live capture fail (and silently bounce containers) on every rootless install, and would have broken DB restores the same way. - capture/rehydrate stderr is kept and printed on failure instead of being discarded, with a clear message when the image has no tar. - backup_app_start.sh: when no location produced a complete snapshot the backup now returns 1 — the task is marked failed instead of logging a nonexistent/incomplete backup as a success and skipping verification. - restic engine: on restic exit 3 the orphan incomplete snapshot is called out explicitly so nobody restores it believing it is whole. - speedtest: capture /config through the container (root-owned TLS key and logrotate state are unreadable from the host), which also flips its auto strategy to live — no more container stop per backup. Co-Authored-By: Claude Fable 5 --- containers/speedtest/docker-compose.yml | 3 ++ scripts/backup/app/backup_app_start.sh | 27 ++++++++++++++--- scripts/backup/db/backup_db.sh | 6 ++-- scripts/backup/engine/restic_backup.sh | 10 +++++++ scripts/backup/files/backup_files.sh | 39 +++++++++++++++++++++---- 5 files changed, 72 insertions(+), 13 deletions(-) diff --git a/containers/speedtest/docker-compose.yml b/containers/speedtest/docker-compose.yml index 1b1f71c..3a99b3f 100644 --- a/containers/speedtest/docker-compose.yml +++ b/containers/speedtest/docker-compose.yml @@ -30,6 +30,9 @@ services: labels: libreportal.category: "CATEGORY_DATA" #LIBREPORTAL|CATEGORY_TAG|CATEGORY_DATA libreportal.title: "TITLE_DATA" #LIBREPORTAL|TITLE_TAG|TITLE_DATA + # /config holds root-and-app-owned files (TLS key, logs) the backup user + # can't read from the host; capture them through the container instead. + libreportal.backup.files: "speedtest-service:/config:config" traefik.enable: TRAEFIK_ENABLE_DATA #LIBREPORTAL|TRAEFIK_ENABLE_TAG|TRAEFIK_ENABLE_DATA # TRAEFIK_PORT_1_BEGIN traefik.http.routers.speedtest-service.entrypoints: web,websecure diff --git a/scripts/backup/app/backup_app_start.sh b/scripts/backup/app/backup_app_start.sh index 9df0eed..6374e44 100755 --- a/scripts/backup/app/backup_app_start.sh +++ b/scripts/backup/app/backup_app_start.sh @@ -76,15 +76,21 @@ backupAppStart() local primary_snapshot_id="" local primary_idx="" local first_loc=true + local snap_ok=0 snap_failed=0 local idx while IFS= read -r idx; do [[ -z "$idx" ]] && continue local snap_id snap_id=$(engineBackupApp "$idx" "$stored_app_name" "$manifest_sha") - if [[ "$first_loc" == true && -n "$snap_id" ]]; then - primary_snapshot_id="$snap_id" - primary_idx="$idx" - first_loc=false + if [[ -n "$snap_id" ]]; then + snap_ok=$((snap_ok + 1)) + if [[ "$first_loc" == true ]]; then + primary_snapshot_id="$snap_id" + primary_idx="$idx" + first_loc=false + fi + else + snap_failed=$((snap_failed + 1)) fi done < <(resticEnabledLocations) @@ -104,6 +110,19 @@ backupAppStart() echo "" backupAppRunHook "$stored_app_name" post + # Containers are back up; now be honest about the snapshot result. With no + # good snapshot on any location there is nothing to verify, retain, or + # record — returning non-zero here is what marks the task failed instead of + # logging a backup that doesn't exist (or is missing files) as a success. + if [[ $snap_ok -eq 0 ]]; then + isError "Backup of $stored_app_name FAILED — no complete snapshot was created on any location (see errors above)" + menu_number=0 + return 1 + fi + if [[ $snap_failed -gt 0 ]]; then + isNotice "Backup of $stored_app_name succeeded on $snap_ok location(s) but failed on $snap_failed — check the errors above" + fi + if [[ "$CFG_BACKUP_VERIFY_AFTER" == "true" && -n "$primary_snapshot_id" ]]; then ((menu_number++)) echo "" diff --git a/scripts/backup/db/backup_db.sh b/scripts/backup/db/backup_db.sh index d24c3aa..df158f9 100644 --- a/scripts/backup/db/backup_db.sh +++ b/scripts/backup/db/backup_db.sh @@ -185,16 +185,16 @@ _backupDbImport() local kind="$1" container="$2" dump="$3" case "$kind" in postgres) - runFileOp gzip -dc "$dump" | docker exec -i "$container" sh -c \ + runFileOp gzip -dc "$dump" | runFileOp docker exec -i "$container" sh -c \ 'export PGPASSWORD="${POSTGRES_PASSWORD:-}"; psql -v ON_ERROR_STOP=1 -U "${POSTGRES_USER:-postgres}" -d "${POSTGRES_DB:-${POSTGRES_USER:-postgres}}"' >/dev/null 2>&1 ;; mongo) # --drop replaces each collection as it is restored, which is what # makes a re-run idempotent (the retry loop in the caller depends on # that, exactly like pg_dump --clean --if-exists). - runFileOp gzip -dc "$dump" | docker exec -i "$container" sh -c \ + runFileOp gzip -dc "$dump" | runFileOp docker exec -i "$container" sh -c \ "$_backup_mongo_auth_sh"' mongorestore "$@" --archive --drop --quiet' >/dev/null 2>&1 ;; *) - runFileOp gzip -dc "$dump" | docker exec -i "$container" sh -c \ + runFileOp gzip -dc "$dump" | runFileOp docker exec -i "$container" sh -c \ 'RP="${MARIADB_ROOT_PASSWORD:-$MYSQL_ROOT_PASSWORD}"; (mariadb -uroot -p"$RP" 2>/dev/null || mysql -uroot -p"$RP")' >/dev/null 2>&1 ;; esac } diff --git a/scripts/backup/engine/restic_backup.sh b/scripts/backup/engine/restic_backup.sh index 024abf4..1a4d847 100644 --- a/scripts/backup/engine/restic_backup.sh +++ b/scripts/backup/engine/restic_backup.sh @@ -54,6 +54,13 @@ resticBackupAppToLocation() else isError "Backup to $loc_name failed for $app_name" >&2 echo "$output" | tail -10 >&2 + # restic exit 3 = snapshot written but some source files were + # unreadable. It stays in the repo but is NOT reported as this run's + # result — an incomplete snapshot must never be recorded as a good + # backup (it can be missing exactly the files that matter, e.g. keys). + if [[ $rc -eq 3 && -n "$snapshot_id" ]]; then + isError "An INCOMPLETE snapshot ${snapshot_id:0:8} was still written to $loc_name — it is missing the unreadable files listed above; do not rely on it" >&2 + fi fi resticEnvUnset @@ -96,6 +103,9 @@ resticBackupSystemToLocation() else isError "System config backup to $loc_name failed" >&2 echo "$output" | tail -10 >&2 + if [[ $rc -eq 3 && -n "$snapshot_id" ]]; then + isError "An INCOMPLETE snapshot ${snapshot_id:0:8} was still written to $loc_name — it is missing the unreadable files listed above; do not rely on it" >&2 + fi fi resticEnvUnset diff --git a/scripts/backup/files/backup_files.sh b/scripts/backup/files/backup_files.sh index bc24fe7..bc6b1c8 100644 --- a/scripts/backup/files/backup_files.sh +++ b/scripts/backup/files/backup_files.sh @@ -86,7 +86,15 @@ backupFilesCapture() runFileOp rm -rf "$stage" 2>/dev/null runFileOp mkdir -p "$stage" # Read in the container's namespace, write the plain tree to staging. - if docker exec "$container" tar -C "$cpath" -cf - . 2>/dev/null | runFileOp tar -xf - -C "$stage" 2>/dev/null; then + # docker must go through runFileOp: on rootless installs the daemon + # socket lives in the install user's runtime dir and only the wrapper + # sets DOCKER_HOST for it — bare `docker` can't connect at all. Keep + # stderr in a scratch file so a failure can say why. + local err_file capture_status + err_file=$(mktemp "${TMPDIR:-/tmp}/lp-capture-XXXXXX.err") + runFileOp docker exec "$container" tar -C "$cpath" -cf - . 2>"$err_file" | runFileOp tar -xf - -C "$stage" 2>>"$err_file" + capture_status=("${PIPESTATUS[@]}") + if [[ ${capture_status[0]} -eq 0 && ${capture_status[1]} -eq 0 ]]; then # The capture preserves the app's ownership (e.g. www-data, 0640), # which the backup user still couldn't read. Hand the staging tree to # the backup user so restic can read it; modes are unchanged, so the @@ -102,7 +110,7 @@ backupFilesCapture() if [[ -z "$uid" || -z "$gid" ]]; then local meta_file="$app_dir/$backup_files_stage_subdir/$subdir.lp-owner" local discovered - discovered=$(docker exec "$container" stat -c '%u:%g' "$cpath" 2>/dev/null) + discovered=$(runFileOp docker exec "$container" stat -c '%u:%g' "$cpath" 2>/dev/null) if [[ -n "$discovered" ]]; then echo "$discovered" | runFileWrite "$meta_file" 2>/dev/null runFileOp chown "$docker_install_user":"$docker_install_user" "$meta_file" 2>/dev/null @@ -111,9 +119,18 @@ backupFilesCapture() isSuccessful "captured $subdir ($(du -sh "$stage" 2>/dev/null | cut -f1))" else - isError "capture of $subdir from $container failed" + if grep -q "executable file not found" "$err_file" 2>/dev/null; then + isError "capture of $subdir from $container failed — the image has no tar, so live capture cannot work for this app" + else + isError "capture of $subdir from $container failed:" + fi + local err_line + while IFS= read -r err_line; do + isError " $err_line" + done < <(head -5 "$err_file" 2>/dev/null) rc=1 fi + rm -f "$err_file" done < <(backupFilesDescriptors "$app") return $rc @@ -169,12 +186,22 @@ restoreFilesRehydratePreStart() # Helper runs as in-namespace root: it can clear/create the dir under the # app dir, extract the streamed tree, and chown to the app's uid:gid # (which maps to the right owner in rooted and rootless alike). - if runFileOp tar -C "$stage" -cf - . 2>/dev/null | docker run --rm -i \ + # docker goes through runFileOp for the rootless socket, same as capture. + local err_file restore_status + err_file=$(mktemp "${TMPDIR:-/tmp}/lp-rehydrate-XXXXXX.err") + runFileOp tar -C "$stage" -cf - . 2>"$err_file" | runFileOp docker run --rm -i \ -v "$app_dir:/parent" "$backup_files_helper_image" \ - sh -c "rm -rf '/parent/$subdir' && mkdir -p '/parent/$subdir' && tar -C '/parent/$subdir' -xf - && chown -R $uid:$gid '/parent/$subdir'" 2>/dev/null; then + sh -c "rm -rf '/parent/$subdir' && mkdir -p '/parent/$subdir' && tar -C '/parent/$subdir' -xf - && chown -R $uid:$gid '/parent/$subdir'" 2>>"$err_file" + restore_status=("${PIPESTATUS[@]}") + if [[ ${restore_status[0]} -eq 0 && ${restore_status[1]} -eq 0 ]]; then isSuccessful "restored $subdir" else - isError "restoring $subdir failed" + isError "restoring $subdir failed:" + local err_line + while IFS= read -r err_line; do + isError " $err_line" + done < <(head -5 "$err_file" 2>/dev/null) fi + rm -f "$err_file" done < <(backupFilesDescriptors "$app") }