27ad517626
Adds CFG_<APP>_BACKUP_STRATEGY (default auto) so an app's backup strategy can be overridden from its Advanced config tab, taking precedence over the global default. Added to the 10 live-capable apps, so the dropdown's 'live' option only appears where it actually works. - backupResolveStrategy now checks the per-app override before the global value. - backupAppLiveCapable / backupAppStrategyOptions expose capability + the valid option set; predicate helpers hardened with explicit returns so they behave identically with or without shell errexit. - BACKUP_STRATEGY field mapping (select, advanced) renders the dropdown. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com> Signed-off-by: librelad <librelad@digitalangels.vip>
332 lines
13 KiB
Bash
332 lines
13 KiB
Bash
#!/bin/bash
|
|
|
|
# Live, consistent database backups.
|
|
#
|
|
# A file-level snapshot of a running database is "torn" — pages can be
|
|
# half-written when restic reads them, so the restored copy may not even
|
|
# mount. The fix is a *logical* dump taken while the service keeps running:
|
|
# mysqldump --single-transaction / pg_dump / sqlite3 .backup all produce a
|
|
# transactionally-consistent file with zero downtime. We snapshot that dump
|
|
# (and exclude the raw data dir, which is now redundant and unreliable), then
|
|
# replay it on restore.
|
|
#
|
|
# Apps declare their databases as compose labels so the metadata travels with
|
|
# the app (the compose is always copied to the install dir and always lives in
|
|
# the snapshot). One label per database:
|
|
#
|
|
# labels:
|
|
# libreportal.backup.db: "<kind>:<container>:<datadir>:<path>"
|
|
#
|
|
# kind mysql | mariadb | postgres | sqlite
|
|
# container service container_name to `docker exec` into (server engines)
|
|
# datadir app-dir-relative folder holding raw DB files, excluded on live
|
|
# path app-dir-relative path to the sqlite file (sqlite only)
|
|
#
|
|
# Examples:
|
|
# "mysql:nextcloud-db:db_data:" MariaDB/MySQL in nextcloud-db, raw db_data/ excluded
|
|
# "postgres:mastodon-db:postgres_data:" Postgres in mastodon-db
|
|
# "sqlite:::data/gitea.db" sqlite file at data/gitea.db
|
|
#
|
|
# An app with no database can still opt into live snapshots (its files are
|
|
# static enough to capture safely) with:
|
|
# labels:
|
|
# libreportal.backup.live: "true"
|
|
|
|
# Subdir (relative to the app dir) where consistent dumps are written. It sits
|
|
# at the app root so it is never inside an excluded datadir, and rides along in
|
|
# the snapshot.
|
|
backup_db_dump_subdir=".lp-backup/db"
|
|
|
|
# Emit one "kind:container:datadir:path" line per declared database, read from
|
|
# the *installed* compose so it reflects what is actually deployed.
|
|
backupDbDescriptors()
|
|
{
|
|
local app="$1"
|
|
local compose="$containers_dir$app/docker-compose.yml"
|
|
[[ -f "$compose" ]] || return 0
|
|
|
|
grep -E '^[[:space:]]*libreportal\.backup\.db[[:space:]]*:' "$compose" 2>/dev/null \
|
|
| sed -E 's/^[[:space:]]*libreportal\.backup\.db[[:space:]]*:[[:space:]]*//' \
|
|
| sed -E 's/[[:space:]]*#.*$//' \
|
|
| sed -E 's/^["'\'']//; s/["'\'']$//' \
|
|
| sed -E 's/[[:space:]]+$//'
|
|
}
|
|
|
|
backupDbHasDescriptors()
|
|
{
|
|
local app="$1"
|
|
if [[ -n "$(backupDbDescriptors "$app")" ]]; then return 0; fi
|
|
return 1
|
|
}
|
|
|
|
# True when the app carries `libreportal.backup.live: "true"` — i.e. its data is
|
|
# safe to snapshot while running even though it has no database to dump.
|
|
backupAppIsLiveSafe()
|
|
{
|
|
local app="$1"
|
|
local compose="$containers_dir$app/docker-compose.yml"
|
|
[[ -f "$compose" ]] || return 1
|
|
if grep -qE '^[[:space:]]*libreportal\.backup\.live[[:space:]]*:[[:space:]]*["'\'']?true' "$compose" 2>/dev/null; then
|
|
return 0
|
|
fi
|
|
return 1
|
|
}
|
|
|
|
# An app can be backed up live without downtime when we can make it consistent:
|
|
# it has a dumpable database, or it is explicitly blessed live-safe.
|
|
backupAppLiveCapable()
|
|
{
|
|
local app="$1"
|
|
if backupDbHasDescriptors "$app"; then return 0; fi
|
|
if backupAppIsLiveSafe "$app"; then return 0; fi
|
|
return 1
|
|
}
|
|
|
|
# Strategy options valid for one app, in .config "[a:b|c:d]" syntax. live is
|
|
# offered only where the app can actually do it, so the UI never shows a choice
|
|
# that would just fall back to stop.
|
|
backupAppStrategyOptions()
|
|
{
|
|
local app="$1"
|
|
local opts="auto:Automatic — recommended|stop-snapshot-start:Stop → snapshot → start|pause-snapshot-unpause:Pause → snapshot → unpause"
|
|
if backupAppLiveCapable "$app"; then
|
|
opts="$opts|live:Live — no downtime"
|
|
fi
|
|
echo "$opts"
|
|
return 0
|
|
}
|
|
|
|
# Resolve the effective strategy for one app. Order of precedence:
|
|
# 1. per-app override CFG_<APP>_BACKUP_STRATEGY (advanced, defaults to auto)
|
|
# 2. global default CFG_BACKUP_STRATEGY (auto)
|
|
# An explicit stop/pause/live is honoured as-is; "auto" goes live only where the
|
|
# app is live-capable and otherwise uses the always-safe stop-snapshot-start.
|
|
backupResolveStrategy()
|
|
{
|
|
local app="$1"
|
|
local override_key="CFG_${app^^}_BACKUP_STRATEGY"
|
|
local s="${!override_key}"
|
|
if [[ -z "$s" || "$s" == "auto" ]]; then
|
|
s="${CFG_BACKUP_STRATEGY:-auto}"
|
|
fi
|
|
|
|
case "$s" in
|
|
live|pause-snapshot-unpause|stop-snapshot-start)
|
|
echo "$s"; return 0 ;;
|
|
esac
|
|
if backupAppLiveCapable "$app"; then
|
|
echo "live"
|
|
else
|
|
echo "stop-snapshot-start"
|
|
fi
|
|
return 0
|
|
}
|
|
|
|
# Deterministic dump filename for a descriptor — backup writes it, restore reads
|
|
# it, both deriving the same name from the descriptor with no side metadata.
|
|
_backupDbDumpName()
|
|
{
|
|
local kind="$1" container="$2" path="$3"
|
|
case "$kind" in
|
|
sqlite) echo "sqlite-$(echo "$path" | tr '/' '_').sqlite.gz" ;;
|
|
*) echo "db-${container}.sql.gz" ;;
|
|
esac
|
|
}
|
|
|
|
# Wait until a server database accepts connections (it has just been started
|
|
# fresh on restore, or is mid-load on a busy host).
|
|
_backupDbWaitReady()
|
|
{
|
|
local kind="$1" container="$2" tries=30
|
|
local i
|
|
for ((i = 0; i < tries; i++)); do
|
|
case "$kind" in
|
|
postgres)
|
|
docker exec "$container" sh -c 'pg_isready -U "${POSTGRES_USER:-postgres}" -q' >/dev/null 2>&1 && return 0 ;;
|
|
*)
|
|
docker exec "$container" sh -c 'mariadb-admin ping -uroot -p"${MARIADB_ROOT_PASSWORD:-$MYSQL_ROOT_PASSWORD}" 2>/dev/null || mysqladmin ping -uroot -p"${MARIADB_ROOT_PASSWORD:-$MYSQL_ROOT_PASSWORD}"' >/dev/null 2>&1 && return 0 ;;
|
|
esac
|
|
sleep 2
|
|
done
|
|
return 1
|
|
}
|
|
|
|
# Dump every declared database for an app to consistent files inside the app
|
|
# dir, while the containers keep running. Called on the live path only.
|
|
backupDbDump()
|
|
{
|
|
local app="$1"
|
|
local app_dir="$containers_dir$app"
|
|
local dump_dir="$app_dir/$backup_db_dump_subdir"
|
|
local desc kind container datadir path dump rc=0
|
|
|
|
backupDbHasDescriptors "$app" || return 0
|
|
|
|
sudo mkdir -p "$dump_dir"
|
|
|
|
while IFS= read -r desc; do
|
|
[[ -z "$desc" ]] && continue
|
|
IFS=':' read -r kind container datadir path <<< "$desc"
|
|
dump="$dump_dir/$(_backupDbDumpName "$kind" "$container" "$path")"
|
|
|
|
case "$kind" in
|
|
postgres)
|
|
isNotice "Dumping postgres ($container) — live, consistent"
|
|
if docker exec "$container" sh -c \
|
|
'export PGPASSWORD="${POSTGRES_PASSWORD:-}"; pg_dump --clean --if-exists -U "${POSTGRES_USER:-postgres}" -d "${POSTGRES_DB:-${POSTGRES_USER:-postgres}}"' \
|
|
2>/dev/null | gzip | sudo tee "$dump" >/dev/null; then
|
|
isSuccessful "postgres dump written ($container)"
|
|
else
|
|
isError "postgres dump failed ($container)"; rc=1
|
|
fi
|
|
;;
|
|
mysql|mariadb)
|
|
isNotice "Dumping $kind ($container) — live, consistent"
|
|
if docker exec "$container" sh -c \
|
|
'RP="${MARIADB_ROOT_PASSWORD:-$MYSQL_ROOT_PASSWORD}"; DB="${MARIADB_DATABASE:-$MYSQL_DATABASE}"; (mariadb-dump -uroot -p"$RP" --single-transaction --routines --triggers --databases "$DB" 2>/dev/null || mysqldump -uroot -p"$RP" --single-transaction --routines --triggers --databases "$DB")' \
|
|
2>/dev/null | gzip | sudo tee "$dump" >/dev/null; then
|
|
isSuccessful "$kind dump written ($container)"
|
|
else
|
|
isError "$kind dump failed ($container)"; rc=1
|
|
fi
|
|
;;
|
|
sqlite)
|
|
isNotice "Dumping sqlite ($path) — live, consistent"
|
|
local src="$app_dir/$path"
|
|
if [[ ! -f "$src" ]]; then
|
|
# Declared but not found — could be a fresh app, or a wrong
|
|
# path. Treat as a dump failure so the caller falls back to
|
|
# the safe stop-snapshot-start rather than snapshotting a
|
|
# live sqlite file untorn.
|
|
isError "sqlite file $path not found — cannot dump"
|
|
rc=1
|
|
continue
|
|
fi
|
|
# .backup takes a consistent copy even while the app writes.
|
|
local tmp="$dump_dir/.$(basename "$path").tmp"
|
|
if sudo sqlite3 "$src" ".backup '$tmp'" 2>/dev/null && sudo gzip -c "$tmp" | sudo tee "$dump" >/dev/null; then
|
|
sudo rm -f "$tmp"
|
|
isSuccessful "sqlite dump written ($path)"
|
|
else
|
|
sudo rm -f "$tmp"
|
|
isError "sqlite dump failed ($path)"; rc=1
|
|
fi
|
|
;;
|
|
*)
|
|
isError "Unknown db kind '$kind' for $app — skipping"; rc=1 ;;
|
|
esac
|
|
done < <(backupDbDescriptors "$app")
|
|
|
|
sudo chown -R "$docker_install_user":"$docker_install_user" "$dump_dir" 2>/dev/null
|
|
return $rc
|
|
}
|
|
|
|
# Absolute paths to exclude from a live snapshot: the raw data dirs / sqlite
|
|
# files the dumps supersede. Echoed one per line for the engine adapters.
|
|
backupDbExcludePaths()
|
|
{
|
|
local app="$1"
|
|
local app_dir="$containers_dir$app"
|
|
local desc kind container datadir path
|
|
|
|
while IFS= read -r desc; do
|
|
[[ -z "$desc" ]] && continue
|
|
IFS=':' read -r kind container datadir path <<< "$desc"
|
|
case "$kind" in
|
|
sqlite)
|
|
[[ -n "$path" ]] || continue
|
|
echo "$app_dir/$path"
|
|
echo "$app_dir/$path-wal"
|
|
echo "$app_dir/$path-shm"
|
|
;;
|
|
*)
|
|
[[ -n "$datadir" ]] || continue
|
|
echo "$app_dir/$datadir"
|
|
;;
|
|
esac
|
|
done < <(backupDbDescriptors "$app")
|
|
}
|
|
|
|
# Pre-start restore step. Runs after the snapshot is laid down but before the
|
|
# containers come up:
|
|
# server remove the (absent or stale) raw data dir so the engine first-run
|
|
# init builds a clean, empty database for us to load into.
|
|
# sqlite put the consistent dump back at the real path so the app opens it.
|
|
restoreDbRehydratePreStart()
|
|
{
|
|
local app="$1"
|
|
local app_dir="$containers_dir$app"
|
|
local dump_dir="$app_dir/$backup_db_dump_subdir"
|
|
local desc kind container datadir path dump
|
|
|
|
backupDbHasDescriptors "$app" || return 0
|
|
|
|
while IFS= read -r desc; do
|
|
[[ -z "$desc" ]] && continue
|
|
IFS=':' read -r kind container datadir path <<< "$desc"
|
|
dump="$dump_dir/$(_backupDbDumpName "$kind" "$container" "$path")"
|
|
|
|
case "$kind" in
|
|
sqlite)
|
|
[[ -f "$dump" ]] || { isNotice "No sqlite dump for $path — leaving app to initialise"; continue; }
|
|
sudo rm -f "$app_dir/$path" "$app_dir/$path-wal" "$app_dir/$path-shm"
|
|
sudo mkdir -p "$(dirname "$app_dir/$path")"
|
|
sudo gzip -dc "$dump" | sudo tee "$app_dir/$path" >/dev/null
|
|
sudo chown -R "$docker_install_user":"$docker_install_user" "$(dirname "$app_dir/$path")"
|
|
isSuccessful "sqlite $path rehydrated from dump"
|
|
;;
|
|
*)
|
|
[[ -f "$dump" ]] || { isNotice "No dump for $container — keeping restored data dir as-is"; continue; }
|
|
[[ -n "$datadir" ]] && sudo rm -rf "${app_dir:?}/$datadir"
|
|
isNotice "Cleared $datadir — $container will init fresh, then load the dump"
|
|
;;
|
|
esac
|
|
done < <(backupDbDescriptors "$app")
|
|
}
|
|
|
|
# Post-start restore step. Server engines load their dump into the freshly
|
|
# initialised database once it is accepting connections. sqlite is already in
|
|
# place from the pre-start step, so it is a no-op here.
|
|
restoreDbReplayPostStart()
|
|
{
|
|
local app="$1"
|
|
local app_dir="$containers_dir$app"
|
|
local dump_dir="$app_dir/$backup_db_dump_subdir"
|
|
local desc kind container datadir path dump
|
|
|
|
backupDbHasDescriptors "$app" || return 0
|
|
|
|
while IFS= read -r desc; do
|
|
[[ -z "$desc" ]] && continue
|
|
IFS=':' read -r kind container datadir path <<< "$desc"
|
|
dump="$dump_dir/$(_backupDbDumpName "$kind" "$container" "$path")"
|
|
[[ "$kind" == "sqlite" ]] && continue
|
|
[[ -f "$dump" ]] || continue
|
|
|
|
isNotice "Waiting for $container to accept connections"
|
|
if ! _backupDbWaitReady "$kind" "$container"; then
|
|
isError "$container never became ready — dump not loaded; data dir left for manual recovery"
|
|
continue
|
|
fi
|
|
|
|
case "$kind" in
|
|
postgres)
|
|
if sudo gzip -dc "$dump" | docker exec -i "$container" sh -c \
|
|
'export PGPASSWORD="${POSTGRES_PASSWORD:-}"; psql -U "${POSTGRES_USER:-postgres}" -d "${POSTGRES_DB:-${POSTGRES_USER:-postgres}}"' >/dev/null 2>&1; then
|
|
isSuccessful "postgres dump loaded into $container"
|
|
else
|
|
isError "Loading postgres dump into $container failed"
|
|
fi
|
|
;;
|
|
mysql|mariadb)
|
|
if sudo gzip -dc "$dump" | docker exec -i "$container" sh -c \
|
|
'RP="${MARIADB_ROOT_PASSWORD:-$MYSQL_ROOT_PASSWORD}"; (mariadb -uroot -p"$RP" 2>/dev/null || mysql -uroot -p"$RP")' >/dev/null 2>&1; then
|
|
isSuccessful "$kind dump loaded into $container"
|
|
else
|
|
isError "Loading $kind dump into $container failed"
|
|
fi
|
|
;;
|
|
esac
|
|
done < <(backupDbDescriptors "$app")
|
|
}
|