69f7289b4a
- Add libreportal.backup.db labels to the MariaDB/Postgres apps (nextcloud, owncloud, bookstack, mastodon, invidious) so they back up live + consistent. - If a declared dump cannot be taken (DB down, wrong path), the backup falls back to stop-snapshot-start for that run instead of snapshotting torn data — a misconfiguration degrades to 'safe with downtime', never to 'unsafe'. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com> Signed-off-by: librelad <librelad@digitalangels.vip>
297 lines
12 KiB
Bash
297 lines
12 KiB
Bash
#!/bin/bash
|
|
|
|
# Live, consistent database backups.
|
|
#
|
|
# A file-level snapshot of a running database is "torn" — pages can be
|
|
# half-written when restic reads them, so the restored copy may not even
|
|
# mount. The fix is a *logical* dump taken while the service keeps running:
|
|
# mysqldump --single-transaction / pg_dump / sqlite3 .backup all produce a
|
|
# transactionally-consistent file with zero downtime. We snapshot that dump
|
|
# (and exclude the raw data dir, which is now redundant and unreliable), then
|
|
# replay it on restore.
|
|
#
|
|
# Apps declare their databases as compose labels so the metadata travels with
|
|
# the app (the compose is always copied to the install dir and always lives in
|
|
# the snapshot). One label per database:
|
|
#
|
|
# labels:
|
|
# libreportal.backup.db: "<kind>:<container>:<datadir>:<path>"
|
|
#
|
|
# kind mysql | mariadb | postgres | sqlite
|
|
# container service container_name to `docker exec` into (server engines)
|
|
# datadir app-dir-relative folder holding raw DB files, excluded on live
|
|
# path app-dir-relative path to the sqlite file (sqlite only)
|
|
#
|
|
# Examples:
|
|
# "mysql:nextcloud-db:db_data:" MariaDB/MySQL in nextcloud-db, raw db_data/ excluded
|
|
# "postgres:mastodon-db:postgres_data:" Postgres in mastodon-db
|
|
# "sqlite:::data/gitea.db" sqlite file at data/gitea.db
|
|
#
|
|
# An app with no database can still opt into live snapshots (its files are
|
|
# static enough to capture safely) with:
|
|
# labels:
|
|
# libreportal.backup.live: "true"
|
|
|
|
# Subdir (relative to the app dir) where consistent dumps are written. It sits
|
|
# at the app root so it is never inside an excluded datadir, and rides along in
|
|
# the snapshot.
|
|
backup_db_dump_subdir=".lp-backup/db"
|
|
|
|
# Emit one "kind:container:datadir:path" line per declared database, read from
|
|
# the *installed* compose so it reflects what is actually deployed.
|
|
backupDbDescriptors()
|
|
{
|
|
local app="$1"
|
|
local compose="$containers_dir$app/docker-compose.yml"
|
|
[[ -f "$compose" ]] || return 0
|
|
|
|
grep -E '^[[:space:]]*libreportal\.backup\.db[[:space:]]*:' "$compose" 2>/dev/null \
|
|
| sed -E 's/^[[:space:]]*libreportal\.backup\.db[[:space:]]*:[[:space:]]*//' \
|
|
| sed -E 's/[[:space:]]*#.*$//' \
|
|
| sed -E 's/^["'\'']//; s/["'\'']$//' \
|
|
| sed -E 's/[[:space:]]+$//'
|
|
}
|
|
|
|
backupDbHasDescriptors()
|
|
{
|
|
local app="$1"
|
|
[[ -n "$(backupDbDescriptors "$app")" ]]
|
|
}
|
|
|
|
# True when the app carries `libreportal.backup.live: "true"` — i.e. its data is
|
|
# safe to snapshot while running even though it has no database to dump.
|
|
backupAppIsLiveSafe()
|
|
{
|
|
local app="$1"
|
|
local compose="$containers_dir$app/docker-compose.yml"
|
|
[[ -f "$compose" ]] || return 1
|
|
grep -qE '^[[:space:]]*libreportal\.backup\.live[[:space:]]*:[[:space:]]*["'\'']?true' "$compose" 2>/dev/null
|
|
}
|
|
|
|
# Resolve the effective strategy for one app. Explicit settings are honoured as
|
|
# power-user overrides; the default "auto" goes live only where we can guarantee
|
|
# consistency (a dumpable database, or an app blessed live-safe) and otherwise
|
|
# falls back to the always-safe stop-snapshot-start.
|
|
backupResolveStrategy()
|
|
{
|
|
local app="$1"
|
|
local s="${CFG_BACKUP_STRATEGY:-auto}"
|
|
case "$s" in
|
|
live|pause-snapshot-unpause|stop-snapshot-start)
|
|
echo "$s"; return 0 ;;
|
|
esac
|
|
if backupDbHasDescriptors "$app" || backupAppIsLiveSafe "$app"; then
|
|
echo "live"
|
|
else
|
|
echo "stop-snapshot-start"
|
|
fi
|
|
}
|
|
|
|
# Deterministic dump filename for a descriptor — backup writes it, restore reads
|
|
# it, both deriving the same name from the descriptor with no side metadata.
|
|
_backupDbDumpName()
|
|
{
|
|
local kind="$1" container="$2" path="$3"
|
|
case "$kind" in
|
|
sqlite) echo "sqlite-$(echo "$path" | tr '/' '_').sqlite.gz" ;;
|
|
*) echo "db-${container}.sql.gz" ;;
|
|
esac
|
|
}
|
|
|
|
# Wait until a server database accepts connections (it has just been started
|
|
# fresh on restore, or is mid-load on a busy host).
|
|
_backupDbWaitReady()
|
|
{
|
|
local kind="$1" container="$2" tries=30
|
|
local i
|
|
for ((i = 0; i < tries; i++)); do
|
|
case "$kind" in
|
|
postgres)
|
|
docker exec "$container" sh -c 'pg_isready -U "${POSTGRES_USER:-postgres}" -q' >/dev/null 2>&1 && return 0 ;;
|
|
*)
|
|
docker exec "$container" sh -c 'mariadb-admin ping -uroot -p"${MARIADB_ROOT_PASSWORD:-$MYSQL_ROOT_PASSWORD}" 2>/dev/null || mysqladmin ping -uroot -p"${MARIADB_ROOT_PASSWORD:-$MYSQL_ROOT_PASSWORD}"' >/dev/null 2>&1 && return 0 ;;
|
|
esac
|
|
sleep 2
|
|
done
|
|
return 1
|
|
}
|
|
|
|
# Dump every declared database for an app to consistent files inside the app
|
|
# dir, while the containers keep running. Called on the live path only.
|
|
backupDbDump()
|
|
{
|
|
local app="$1"
|
|
local app_dir="$containers_dir$app"
|
|
local dump_dir="$app_dir/$backup_db_dump_subdir"
|
|
local desc kind container datadir path dump rc=0
|
|
|
|
backupDbHasDescriptors "$app" || return 0
|
|
|
|
sudo mkdir -p "$dump_dir"
|
|
|
|
while IFS= read -r desc; do
|
|
[[ -z "$desc" ]] && continue
|
|
IFS=':' read -r kind container datadir path <<< "$desc"
|
|
dump="$dump_dir/$(_backupDbDumpName "$kind" "$container" "$path")"
|
|
|
|
case "$kind" in
|
|
postgres)
|
|
isNotice "Dumping postgres ($container) — live, consistent"
|
|
if docker exec "$container" sh -c \
|
|
'export PGPASSWORD="${POSTGRES_PASSWORD:-}"; pg_dump --clean --if-exists -U "${POSTGRES_USER:-postgres}" -d "${POSTGRES_DB:-${POSTGRES_USER:-postgres}}"' \
|
|
2>/dev/null | gzip | sudo tee "$dump" >/dev/null; then
|
|
isSuccessful "postgres dump written ($container)"
|
|
else
|
|
isError "postgres dump failed ($container)"; rc=1
|
|
fi
|
|
;;
|
|
mysql|mariadb)
|
|
isNotice "Dumping $kind ($container) — live, consistent"
|
|
if docker exec "$container" sh -c \
|
|
'RP="${MARIADB_ROOT_PASSWORD:-$MYSQL_ROOT_PASSWORD}"; DB="${MARIADB_DATABASE:-$MYSQL_DATABASE}"; (mariadb-dump -uroot -p"$RP" --single-transaction --routines --triggers --databases "$DB" 2>/dev/null || mysqldump -uroot -p"$RP" --single-transaction --routines --triggers --databases "$DB")' \
|
|
2>/dev/null | gzip | sudo tee "$dump" >/dev/null; then
|
|
isSuccessful "$kind dump written ($container)"
|
|
else
|
|
isError "$kind dump failed ($container)"; rc=1
|
|
fi
|
|
;;
|
|
sqlite)
|
|
isNotice "Dumping sqlite ($path) — live, consistent"
|
|
local src="$app_dir/$path"
|
|
if [[ ! -f "$src" ]]; then
|
|
# Declared but not found — could be a fresh app, or a wrong
|
|
# path. Treat as a dump failure so the caller falls back to
|
|
# the safe stop-snapshot-start rather than snapshotting a
|
|
# live sqlite file untorn.
|
|
isError "sqlite file $path not found — cannot dump"
|
|
rc=1
|
|
continue
|
|
fi
|
|
# .backup takes a consistent copy even while the app writes.
|
|
local tmp="$dump_dir/.$(basename "$path").tmp"
|
|
if sudo sqlite3 "$src" ".backup '$tmp'" 2>/dev/null && sudo gzip -c "$tmp" | sudo tee "$dump" >/dev/null; then
|
|
sudo rm -f "$tmp"
|
|
isSuccessful "sqlite dump written ($path)"
|
|
else
|
|
sudo rm -f "$tmp"
|
|
isError "sqlite dump failed ($path)"; rc=1
|
|
fi
|
|
;;
|
|
*)
|
|
isError "Unknown db kind '$kind' for $app — skipping"; rc=1 ;;
|
|
esac
|
|
done < <(backupDbDescriptors "$app")
|
|
|
|
sudo chown -R "$docker_install_user":"$docker_install_user" "$dump_dir" 2>/dev/null
|
|
return $rc
|
|
}
|
|
|
|
# Absolute paths to exclude from a live snapshot: the raw data dirs / sqlite
|
|
# files the dumps supersede. Echoed one per line for the engine adapters.
|
|
backupDbExcludePaths()
|
|
{
|
|
local app="$1"
|
|
local app_dir="$containers_dir$app"
|
|
local desc kind container datadir path
|
|
|
|
while IFS= read -r desc; do
|
|
[[ -z "$desc" ]] && continue
|
|
IFS=':' read -r kind container datadir path <<< "$desc"
|
|
case "$kind" in
|
|
sqlite)
|
|
[[ -n "$path" ]] || continue
|
|
echo "$app_dir/$path"
|
|
echo "$app_dir/$path-wal"
|
|
echo "$app_dir/$path-shm"
|
|
;;
|
|
*)
|
|
[[ -n "$datadir" ]] || continue
|
|
echo "$app_dir/$datadir"
|
|
;;
|
|
esac
|
|
done < <(backupDbDescriptors "$app")
|
|
}
|
|
|
|
# Pre-start restore step. Runs after the snapshot is laid down but before the
|
|
# containers come up:
|
|
# server remove the (absent or stale) raw data dir so the engine first-run
|
|
# init builds a clean, empty database for us to load into.
|
|
# sqlite put the consistent dump back at the real path so the app opens it.
|
|
restoreDbRehydratePreStart()
|
|
{
|
|
local app="$1"
|
|
local app_dir="$containers_dir$app"
|
|
local dump_dir="$app_dir/$backup_db_dump_subdir"
|
|
local desc kind container datadir path dump
|
|
|
|
backupDbHasDescriptors "$app" || return 0
|
|
|
|
while IFS= read -r desc; do
|
|
[[ -z "$desc" ]] && continue
|
|
IFS=':' read -r kind container datadir path <<< "$desc"
|
|
dump="$dump_dir/$(_backupDbDumpName "$kind" "$container" "$path")"
|
|
|
|
case "$kind" in
|
|
sqlite)
|
|
[[ -f "$dump" ]] || { isNotice "No sqlite dump for $path — leaving app to initialise"; continue; }
|
|
sudo rm -f "$app_dir/$path" "$app_dir/$path-wal" "$app_dir/$path-shm"
|
|
sudo mkdir -p "$(dirname "$app_dir/$path")"
|
|
sudo gzip -dc "$dump" | sudo tee "$app_dir/$path" >/dev/null
|
|
sudo chown -R "$docker_install_user":"$docker_install_user" "$(dirname "$app_dir/$path")"
|
|
isSuccessful "sqlite $path rehydrated from dump"
|
|
;;
|
|
*)
|
|
[[ -f "$dump" ]] || { isNotice "No dump for $container — keeping restored data dir as-is"; continue; }
|
|
[[ -n "$datadir" ]] && sudo rm -rf "${app_dir:?}/$datadir"
|
|
isNotice "Cleared $datadir — $container will init fresh, then load the dump"
|
|
;;
|
|
esac
|
|
done < <(backupDbDescriptors "$app")
|
|
}
|
|
|
|
# Post-start restore step. Server engines load their dump into the freshly
|
|
# initialised database once it is accepting connections. sqlite is already in
|
|
# place from the pre-start step, so it is a no-op here.
|
|
restoreDbReplayPostStart()
|
|
{
|
|
local app="$1"
|
|
local app_dir="$containers_dir$app"
|
|
local dump_dir="$app_dir/$backup_db_dump_subdir"
|
|
local desc kind container datadir path dump
|
|
|
|
backupDbHasDescriptors "$app" || return 0
|
|
|
|
while IFS= read -r desc; do
|
|
[[ -z "$desc" ]] && continue
|
|
IFS=':' read -r kind container datadir path <<< "$desc"
|
|
dump="$dump_dir/$(_backupDbDumpName "$kind" "$container" "$path")"
|
|
[[ "$kind" == "sqlite" ]] && continue
|
|
[[ -f "$dump" ]] || continue
|
|
|
|
isNotice "Waiting for $container to accept connections"
|
|
if ! _backupDbWaitReady "$kind" "$container"; then
|
|
isError "$container never became ready — dump not loaded; data dir left for manual recovery"
|
|
continue
|
|
fi
|
|
|
|
case "$kind" in
|
|
postgres)
|
|
if sudo gzip -dc "$dump" | docker exec -i "$container" sh -c \
|
|
'export PGPASSWORD="${POSTGRES_PASSWORD:-}"; psql -U "${POSTGRES_USER:-postgres}" -d "${POSTGRES_DB:-${POSTGRES_USER:-postgres}}"' >/dev/null 2>&1; then
|
|
isSuccessful "postgres dump loaded into $container"
|
|
else
|
|
isError "Loading postgres dump into $container failed"
|
|
fi
|
|
;;
|
|
mysql|mariadb)
|
|
if sudo gzip -dc "$dump" | docker exec -i "$container" sh -c \
|
|
'RP="${MARIADB_ROOT_PASSWORD:-$MYSQL_ROOT_PASSWORD}"; (mariadb -uroot -p"$RP" 2>/dev/null || mysql -uroot -p"$RP")' >/dev/null 2>&1; then
|
|
isSuccessful "$kind dump loaded into $container"
|
|
else
|
|
isError "Loading $kind dump into $container failed"
|
|
fi
|
|
;;
|
|
esac
|
|
done < <(backupDbDescriptors "$app")
|
|
}
|