From 94c9e83c42dc5039b9d4fbf35f0aaa8625c559f3 Mon Sep 17 00:00:00 2001 From: librelad Date: Sat, 23 May 2026 18:15:53 +0100 Subject: [PATCH] feat(backup): container-side capture of private app files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reads files the backup user can't see from the host (container-owned, e.g. Nextcloud's www-data data dir) by streaming them out THROUGH the container (docker exec tar) — no host root, no host read perms, works rooted + rootless. Extracts to staging as plain files so restic keeps full dedup + per-file restore (not a piped tar blob); the live path is excluded from the snapshot. Restore streams the staging copy back through a throwaway in-namespace container that recreates the tree with the app's uid:gid. Declared via a libreportal.backup.files compose label; Nextcloud (html, 33:33) is the first to use it. Live capture failure falls back to stop-snapshot-start. Co-Authored-By: Claude Opus 4.7 Signed-off-by: librelad --- containers/nextcloud/docker-compose.yml | 1 + scripts/backup/app/backup_app_start.sh | 17 ++- scripts/backup/db/backup_db.sh | 1 + scripts/backup/files/backup_files.sh | 136 ++++++++++++++++++ scripts/restore/restore_app_start.sh | 3 +- scripts/source/files/arrays/files_backup.sh | 1 + .../data/generators/apps/webui_config.sh | 2 +- 7 files changed, 152 insertions(+), 9 deletions(-) create mode 100644 scripts/backup/files/backup_files.sh diff --git a/containers/nextcloud/docker-compose.yml b/containers/nextcloud/docker-compose.yml index efb0c21..f35d8f7 100644 --- a/containers/nextcloud/docker-compose.yml +++ b/containers/nextcloud/docker-compose.yml @@ -32,6 +32,7 @@ services: libreportal.category: "CATEGORY_DATA" #LIBREPORTAL|CATEGORY_TAG|CATEGORY_DATA libreportal.title: "TITLE_DATA" #LIBREPORTAL|TITLE_TAG|TITLE_DATA libreportal.backup.db: "mariadb:nextcloud-db:db_data:" + libreportal.backup.files: "nextcloud-service:/var/www/html:html:33:33" traefik.enable: TRAEFIK_ENABLE_DATA #LIBREPORTAL|TRAEFIK_ENABLE_TAG|TRAEFIK_ENABLE_DATA # TRAEFIK_PORT_1_BEGIN traefik.http.routers.nextcloud-service.entrypoints: web,websecure diff --git a/scripts/backup/app/backup_app_start.sh b/scripts/backup/app/backup_app_start.sh index 0de894c..819b81b 100755 --- a/scripts/backup/app/backup_app_start.sh +++ b/scripts/backup/app/backup_app_start.sh @@ -40,9 +40,9 @@ backupAppStart() if [[ "$strategy" == "pause-snapshot-unpause" ]]; then dockerComposePause "$stored_app_name" 2>/dev/null || dockerComposeDown "$stored_app_name" elif [[ "$strategy" == "live" ]]; then - isNotice "Live strategy — containers stay running; databases dumped consistently" - if ! backupDbDump "$stored_app_name"; then - isError "Live database dump failed — falling back to stop-snapshot-start for safety" + isNotice "Live strategy — containers stay running; databases dumped + private files captured via their containers" + if ! backupDbDump "$stored_app_name" || ! backupFilesCapture "$stored_app_name"; then + isError "Live capture failed — falling back to stop-snapshot-start for safety" sudo rm -rf "${containers_dir:?}$stored_app_name/.lp-backup" strategy="stop-snapshot-start" dockerComposeDown "$stored_app_name" @@ -63,12 +63,15 @@ backupAppStart() echo "" echo "---- $menu_number. Snapshotting to all enabled locations" echo "" - # On the live path the raw DB data dirs are torn and superseded by the - # dumps written above — exclude them so the snapshot carries only the - # consistent copy. Other strategies quiesced the DB, so keep everything. + # On the live path the raw DB data dirs (superseded by the dumps) and the + # private file trees (superseded by the container-side captures) are excluded + # so the snapshot carries only the consistent copies. Other strategies + # quiesced the app, so keep everything. backup_exclude_paths="" if [[ "$strategy" == "live" ]]; then - backup_exclude_paths=$(backupDbExcludePaths "$stored_app_name") + backup_exclude_paths=$(printf '%s\n%s\n' \ + "$(backupDbExcludePaths "$stored_app_name")" \ + "$(backupFilesExcludePaths "$stored_app_name")") fi local primary_snapshot_id="" local primary_idx="" diff --git a/scripts/backup/db/backup_db.sh b/scripts/backup/db/backup_db.sh index cd8a67c..9b408c2 100644 --- a/scripts/backup/db/backup_db.sh +++ b/scripts/backup/db/backup_db.sh @@ -78,6 +78,7 @@ backupAppLiveCapable() { local app="$1" if backupDbHasDescriptors "$app"; then return 0; fi + if declare -f backupFilesHasDescriptors >/dev/null 2>&1 && backupFilesHasDescriptors "$app"; then return 0; fi if backupAppIsLiveSafe "$app"; then return 0; fi return 1 } diff --git a/scripts/backup/files/backup_files.sh b/scripts/backup/files/backup_files.sh new file mode 100644 index 0000000..c134507 --- /dev/null +++ b/scripts/backup/files/backup_files.sh @@ -0,0 +1,136 @@ +#!/bin/bash + +# Container-side file capture. +# +# Some apps store private data the backup user can't read from the host: in +# rooted Docker the files are owned by the container's real UIDs (e.g. Nextcloud +# data is www-data 0750), and in rootless they're owned by mapped sub-UIDs — in +# both cases restic, running as the unprivileged backup user, gets "permission +# denied" and the snapshot silently drops that data. +# +# So we read those paths the same way we dump databases: THROUGH the container. +# `docker exec tar` runs in the container's namespace and sees every file as +# the app's own user, needing no host root and no host read access (works +# identically rooted and rootless). We extract the stream to a staging dir as +# PLAIN FILES (not a piped tar blob) so restic keeps full per-file dedup and +# per-file restore; the live path is then excluded from the snapshot. On restore +# we stream the staging copy back through a throwaway container that recreates +# the tree with the app's ownership in-namespace — again no host root. +# +# Declared per app as a compose label (multiple allowed): +# +# labels: +# libreportal.backup.files: "::::" +# +# container service to exec/read through +# container_path path inside the container to capture (a bind-mount target) +# host_subdir app-dir-relative dir it maps to (excluded from the snapshot) +# uid:gid ownership to restore the files as (the app's runtime user) +# +# Example (Nextcloud): +# "nextcloud-service:/var/www/html:html:33:33" + +# Staging lives at the app root (never inside an excluded path) so it rides in +# the snapshot alongside the DB dumps under .lp-backup/. +backup_files_stage_subdir=".lp-backup/files" + +# Tiny image used as a throwaway, in-namespace extractor on restore. +backup_files_helper_image="busybox" + +backupFilesDescriptors() +{ + local app="$1" + local compose="$containers_dir$app/docker-compose.yml" + [[ -f "$compose" ]] || return 0 + + grep -E '^[[:space:]]*libreportal\.backup\.files[[:space:]]*:' "$compose" 2>/dev/null \ + | sed -E 's/^[[:space:]]*libreportal\.backup\.files[[:space:]]*:[[:space:]]*//' \ + | sed -E 's/[[:space:]]*#.*$//' \ + | sed -E 's/^["'\'']//; s/["'\'']$//' \ + | sed -E 's/[[:space:]]+$//' +} + +backupFilesHasDescriptors() +{ + local app="$1" + if [[ -n "$(backupFilesDescriptors "$app")" ]]; then return 0; fi + return 1 +} + +# Capture each declared path live, through its container, into staging. No host +# root, no host read perms. +backupFilesCapture() +{ + local app="$1" + local app_dir="$containers_dir$app" + local desc container cpath subdir uid gid stage rc=0 + + backupFilesHasDescriptors "$app" || return 0 + + while IFS= read -r desc; do + [[ -z "$desc" ]] && continue + IFS=':' read -r container cpath subdir uid gid <<< "$desc" + [[ -z "$container" || -z "$cpath" || -z "$subdir" ]] && { isError "Bad backup.files descriptor: $desc"; rc=1; continue; } + stage="$app_dir/$backup_files_stage_subdir/$subdir" + + isNotice "Capturing $subdir from $container — live, via container" + rm -rf "$stage" 2>/dev/null + mkdir -p "$stage" + # Read in the container's namespace, write the plain tree to staging. + if docker exec "$container" tar -C "$cpath" -cf - . 2>/dev/null | tar -xf - -C "$stage" 2>/dev/null; then + isSuccessful "captured $subdir ($(du -sh "$stage" 2>/dev/null | cut -f1))" + else + isError "capture of $subdir from $container failed" + rc=1 + fi + done < <(backupFilesDescriptors "$app") + + return $rc +} + +# Live paths the staging copies supersede — excluded from the snapshot. +backupFilesExcludePaths() +{ + local app="$1" + local app_dir="$containers_dir$app" + local desc container cpath subdir uid gid + + while IFS= read -r desc; do + [[ -z "$desc" ]] && continue + IFS=':' read -r container cpath subdir uid gid <<< "$desc" + [[ -n "$subdir" ]] && echo "$app_dir/$subdir" + done < <(backupFilesDescriptors "$app") +} + +# Pre-start restore: rebuild each captured tree at its host path with the app's +# ownership, by extracting through a throwaway container running in-namespace. +# Runs after the snapshot is laid down (staging present, live path absent) and +# before the app starts. No host root. +restoreFilesRehydratePreStart() +{ + local app="$1" + local app_dir="$containers_dir$app" + local desc container cpath subdir uid gid stage + + backupFilesHasDescriptors "$app" || return 0 + + while IFS= read -r desc; do + [[ -z "$desc" ]] && continue + IFS=':' read -r container cpath subdir uid gid <<< "$desc" + stage="$app_dir/$backup_files_stage_subdir/$subdir" + [[ -d "$stage" ]] || { isNotice "No captured files for $subdir — skipping"; continue; } + uid="${uid:-0}"; gid="${gid:-0}" + + isNotice "Restoring $subdir as $uid:$gid — via container" + # Helper runs as in-namespace root: it can clear/create the dir under the + # app dir, extract the streamed tree, and chown to the app's uid:gid + # (which maps to the right owner in rooted and rootless alike). + if tar -C "$stage" -cf - . 2>/dev/null | docker run --rm -i \ + -v "$app_dir:/parent" "$backup_files_helper_image" \ + sh -c "rm -rf '/parent/$subdir' && mkdir -p '/parent/$subdir' && tar -C '/parent/$subdir' -xf - && chown -R $uid:$gid '/parent/$subdir'" 2>/dev/null; then + isSuccessful "restored $subdir" + else + isError "restoring $subdir failed" + fi + done < <(backupFilesDescriptors "$app") +} diff --git a/scripts/restore/restore_app_start.sh b/scripts/restore/restore_app_start.sh index 70ab420..201889f 100644 --- a/scripts/restore/restore_app_start.sh +++ b/scripts/restore/restore_app_start.sh @@ -79,9 +79,10 @@ restoreAppStart() ((menu_number++)) echo "" - echo "---- $menu_number. Rehydrating databases (pre-start)" + echo "---- $menu_number. Rehydrating databases + files (pre-start)" echo "" restoreDbRehydratePreStart "$stored_app_name" + restoreFilesRehydratePreStart "$stored_app_name" ((menu_number++)) echo "" diff --git a/scripts/source/files/arrays/files_backup.sh b/scripts/source/files/arrays/files_backup.sh index 7d4f4d1..ac151f5 100755 --- a/scripts/source/files/arrays/files_backup.sh +++ b/scripts/source/files/arrays/files_backup.sh @@ -38,6 +38,7 @@ backup_scripts=( "backup/engine/restic_install.sh" "backup/engine/restic_restore.sh" "backup/engine/restic_snapshots.sh" + "backup/files/backup_files.sh" "backup/locations/location_add.sh" "backup/locations/location_loader.sh" "backup/locations/location_migrate.sh" diff --git a/scripts/webui/data/generators/apps/webui_config.sh b/scripts/webui/data/generators/apps/webui_config.sh index bbf7d40..63b3b5d 100644 --- a/scripts/webui/data/generators/apps/webui_config.sh +++ b/scripts/webui/data/generators/apps/webui_config.sh @@ -187,7 +187,7 @@ EOF # strategy option where it would be unsafe. local backup_live_capable="false" if [[ -f "$compose_file" ]]; then - if grep -qE '^[[:space:]]*libreportal\.backup\.db[[:space:]]*:' "$compose_file" 2>/dev/null \ + if grep -qE '^[[:space:]]*libreportal\.backup\.(db|files)[[:space:]]*:' "$compose_file" 2>/dev/null \ || grep -qE '^[[:space:]]*libreportal\.backup\.live[[:space:]]*:[[:space:]]*["'\'']?true' "$compose_file" 2>/dev/null; then backup_live_capable="true" fi