Merge claude/2

Reads files the backup user can't see from the host (container-owned, e.g.
Nextcloud's www-data data dir) by streaming them out THROUGH the container
(docker exec tar) — no host root, no host read perms, works rooted + rootless.
Extracts to staging as plain files so restic keeps full dedup + per-file
restore (not a piped tar blob); the live path is excluded from the snapshot.
Restore streams the staging copy back through a throwaway in-namespace
container that recreates the tree with the app's uid:gid.

Declared via a libreportal.backup.files compose label; Nextcloud (html, 33:33)
is the first to use it. Live capture failure falls back to stop-snapshot-start.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
Signed-off-by: librelad <librelad@digitalangels.vip>

containers/nextcloud/docker-compose.yml

@@ -32,6 +32,7 @@ services:
       libreportal.category: "CATEGORY_DATA" #LIBREPORTAL|CATEGORY_TAG|CATEGORY_DATA
       libreportal.title: "TITLE_DATA" #LIBREPORTAL|TITLE_TAG|TITLE_DATA
       libreportal.backup.db: "mariadb:nextcloud-db:db_data:"
+      libreportal.backup.files: "nextcloud-service:/var/www/html:html:33:33"
       traefik.enable: TRAEFIK_ENABLE_DATA #LIBREPORTAL|TRAEFIK_ENABLE_TAG|TRAEFIK_ENABLE_DATA
       # TRAEFIK_PORT_1_BEGIN
       traefik.http.routers.nextcloud-service.entrypoints: web,websecure

scripts/backup/app/backup_app_start.sh

@@ -40,9 +40,9 @@ backupAppStart()
     if [[ "$strategy" == "pause-snapshot-unpause" ]]; then
         dockerComposePause "$stored_app_name" 2>/dev/null || dockerComposeDown "$stored_app_name"
     elif [[ "$strategy" == "live" ]]; then
-        isNotice "Live strategy — containers stay running; databases dumped consistently"
+        isNotice "Live strategy — containers stay running; databases dumped + private files captured via their containers"
-        if ! backupDbDump "$stored_app_name"; then
+        if ! backupDbDump "$stored_app_name" || ! backupFilesCapture "$stored_app_name"; then
-            isError "Live database dump failed — falling back to stop-snapshot-start for safety"
+            isError "Live capture failed — falling back to stop-snapshot-start for safety"
             sudo rm -rf "${containers_dir:?}$stored_app_name/.lp-backup"
             strategy="stop-snapshot-start"
             dockerComposeDown "$stored_app_name"
@@ -63,12 +63,15 @@ backupAppStart()
     echo ""
     echo "---- $menu_number. Snapshotting to all enabled locations"
     echo ""
-    # On the live path the raw DB data dirs are torn and superseded by the
+    # On the live path the raw DB data dirs (superseded by the dumps) and the
-    # dumps written above — exclude them so the snapshot carries only the
+    # private file trees (superseded by the container-side captures) are excluded
-    # consistent copy. Other strategies quiesced the DB, so keep everything.
+    # so the snapshot carries only the consistent copies. Other strategies
+    # quiesced the app, so keep everything.
     backup_exclude_paths=""
     if [[ "$strategy" == "live" ]]; then
-        backup_exclude_paths=$(backupDbExcludePaths "$stored_app_name")
+        backup_exclude_paths=$(printf '%s\n%s\n' \
+            "$(backupDbExcludePaths "$stored_app_name")" \
+            "$(backupFilesExcludePaths "$stored_app_name")")
     fi
     local primary_snapshot_id=""
     local primary_idx=""

scripts/backup/db/backup_db.sh

@@ -78,6 +78,7 @@ backupAppLiveCapable()
 {
     local app="$1"
     if backupDbHasDescriptors "$app"; then return 0; fi
+    if declare -f backupFilesHasDescriptors >/dev/null 2>&1 && backupFilesHasDescriptors "$app"; then return 0; fi
     if backupAppIsLiveSafe "$app"; then return 0; fi
     return 1
 }

scripts/backup/files/backup_files.sh

@@ -0,0 +1,136 @@
+#!/bin/bash
+
+# Container-side file capture.
+#
+# Some apps store private data the backup user can't read from the host: in
+# rooted Docker the files are owned by the container's real UIDs (e.g. Nextcloud
+# data is www-data 0750), and in rootless they're owned by mapped sub-UIDs — in
+# both cases restic, running as the unprivileged backup user, gets "permission
+# denied" and the snapshot silently drops that data.
+#
+# So we read those paths the same way we dump databases: THROUGH the container.
+# `docker exec <c> tar` runs in the container's namespace and sees every file as
+# the app's own user, needing no host root and no host read access (works
+# identically rooted and rootless). We extract the stream to a staging dir as
+# PLAIN FILES (not a piped tar blob) so restic keeps full per-file dedup and
+# per-file restore; the live path is then excluded from the snapshot. On restore
+# we stream the staging copy back through a throwaway container that recreates
+# the tree with the app's ownership in-namespace — again no host root.
+#
+# Declared per app as a compose label (multiple allowed):
+#
+#   labels:
+#     libreportal.backup.files: "<container>:<container_path>:<host_subdir>:<uid>:<gid>"
+#
+#   container       service to exec/read through
+#   container_path  path inside the container to capture (a bind-mount target)
+#   host_subdir     app-dir-relative dir it maps to (excluded from the snapshot)
+#   uid:gid         ownership to restore the files as (the app's runtime user)
+#
+# Example (Nextcloud):
+#   "nextcloud-service:/var/www/html:html:33:33"
+
+# Staging lives at the app root (never inside an excluded path) so it rides in
+# the snapshot alongside the DB dumps under .lp-backup/.
+backup_files_stage_subdir=".lp-backup/files"
+
+# Tiny image used as a throwaway, in-namespace extractor on restore.
+backup_files_helper_image="busybox"
+
+backupFilesDescriptors()
+{
+    local app="$1"
+    local compose="$containers_dir$app/docker-compose.yml"
+    [[ -f "$compose" ]] || return 0
+
+    grep -E '^[[:space:]]*libreportal\.backup\.files[[:space:]]*:' "$compose" 2>/dev/null \
+        | sed -E 's/^[[:space:]]*libreportal\.backup\.files[[:space:]]*:[[:space:]]*//' \
+        | sed -E 's/[[:space:]]*#.*$//' \
+        | sed -E 's/^["'\'']//; s/["'\'']$//' \
+        | sed -E 's/[[:space:]]+$//'
+}
+
+backupFilesHasDescriptors()
+{
+    local app="$1"
+    if [[ -n "$(backupFilesDescriptors "$app")" ]]; then return 0; fi
+    return 1
+}
+
+# Capture each declared path live, through its container, into staging. No host
+# root, no host read perms.
+backupFilesCapture()
+{
+    local app="$1"
+    local app_dir="$containers_dir$app"
+    local desc container cpath subdir uid gid stage rc=0
+
+    backupFilesHasDescriptors "$app" || return 0
+
+    while IFS= read -r desc; do
+        [[ -z "$desc" ]] && continue
+        IFS=':' read -r container cpath subdir uid gid <<< "$desc"
+        [[ -z "$container" || -z "$cpath" || -z "$subdir" ]] && { isError "Bad backup.files descriptor: $desc"; rc=1; continue; }
+        stage="$app_dir/$backup_files_stage_subdir/$subdir"
+
+        isNotice "Capturing $subdir from $container — live, via container"
+        rm -rf "$stage" 2>/dev/null
+        mkdir -p "$stage"
+        # Read in the container's namespace, write the plain tree to staging.
+        if docker exec "$container" tar -C "$cpath" -cf - . 2>/dev/null | tar -xf - -C "$stage" 2>/dev/null; then
+            isSuccessful "captured $subdir ($(du -sh "$stage" 2>/dev/null | cut -f1))"
+        else
+            isError "capture of $subdir from $container failed"
+            rc=1
+        fi
+    done < <(backupFilesDescriptors "$app")
+
+    return $rc
+}
+
+# Live paths the staging copies supersede — excluded from the snapshot.
+backupFilesExcludePaths()
+{
+    local app="$1"
+    local app_dir="$containers_dir$app"
+    local desc container cpath subdir uid gid
+
+    while IFS= read -r desc; do
+        [[ -z "$desc" ]] && continue
+        IFS=':' read -r container cpath subdir uid gid <<< "$desc"
+        [[ -n "$subdir" ]] && echo "$app_dir/$subdir"
+    done < <(backupFilesDescriptors "$app")
+}
+
+# Pre-start restore: rebuild each captured tree at its host path with the app's
+# ownership, by extracting through a throwaway container running in-namespace.
+# Runs after the snapshot is laid down (staging present, live path absent) and
+# before the app starts. No host root.
+restoreFilesRehydratePreStart()
+{
+    local app="$1"
+    local app_dir="$containers_dir$app"
+    local desc container cpath subdir uid gid stage
+
+    backupFilesHasDescriptors "$app" || return 0
+
+    while IFS= read -r desc; do
+        [[ -z "$desc" ]] && continue
+        IFS=':' read -r container cpath subdir uid gid <<< "$desc"
+        stage="$app_dir/$backup_files_stage_subdir/$subdir"
+        [[ -d "$stage" ]] || { isNotice "No captured files for $subdir — skipping"; continue; }
+        uid="${uid:-0}"; gid="${gid:-0}"
+
+        isNotice "Restoring $subdir as $uid:$gid — via container"
+        # Helper runs as in-namespace root: it can clear/create the dir under the
+        # app dir, extract the streamed tree, and chown to the app's uid:gid
+        # (which maps to the right owner in rooted and rootless alike).
+        if tar -C "$stage" -cf - . 2>/dev/null | docker run --rm -i \
+                -v "$app_dir:/parent" "$backup_files_helper_image" \
+                sh -c "rm -rf '/parent/$subdir' && mkdir -p '/parent/$subdir' && tar -C '/parent/$subdir' -xf - && chown -R $uid:$gid '/parent/$subdir'" 2>/dev/null; then
+            isSuccessful "restored $subdir"
+        else
+            isError "restoring $subdir failed"
+        fi
+    done < <(backupFilesDescriptors "$app")
+}

scripts/restore/restore_app_start.sh

@@ -79,9 +79,10 @@ restoreAppStart()
 
     ((menu_number++))
     echo ""
-    echo "---- $menu_number. Rehydrating databases (pre-start)"
+    echo "---- $menu_number. Rehydrating databases + files (pre-start)"
     echo ""
     restoreDbRehydratePreStart "$stored_app_name"
+    restoreFilesRehydratePreStart "$stored_app_name"
 
     ((menu_number++))
     echo ""

scripts/source/files/arrays/files_backup.sh

@@ -38,6 +38,7 @@ backup_scripts=(
     "backup/engine/restic_install.sh"
     "backup/engine/restic_restore.sh"
     "backup/engine/restic_snapshots.sh"
+    "backup/files/backup_files.sh"
     "backup/locations/location_add.sh"
     "backup/locations/location_loader.sh"
     "backup/locations/location_migrate.sh"

scripts/webui/data/generators/apps/webui_config.sh

@@ -187,7 +187,7 @@ EOF
         # strategy option where it would be unsafe.
         local backup_live_capable="false"
         if [[ -f "$compose_file" ]]; then
-            if grep -qE '^[[:space:]]*libreportal\.backup\.db[[:space:]]*:' "$compose_file" 2>/dev/null \
+            if grep -qE '^[[:space:]]*libreportal\.backup\.(db|files)[[:space:]]*:' "$compose_file" 2>/dev/null \
                || grep -qE '^[[:space:]]*libreportal\.backup\.live[[:space:]]*:[[:space:]]*["'\'']?true' "$compose_file" 2>/dev/null; then
                 backup_live_capable="true"
             fi