fc169e7a4a
The live Nextcloud 31→34 climb left 4.4 GB of images behind — one per rung, each ~1.5 GB, all still present after it finished. On a small VPS that is the difference between working and full. `system reclaim` cannot help: it collects DANGLING images, and every rung is a real tag, so all of them stay tagged and stay on disk. (Rolling apps never hit this — moving a floating tag orphans the old image, which reclaim then collects. It is specific to laddering.) After a SUCCESSFUL climb only, remove the images stepped through, keeping the immediately-previous version so a roll-back needs no download. CFG_UPDATER_UPGRADE_PRUNE=false keeps everything. Never runs on failure, where the older images are exactly what recovery may need. Tested: a 3-rung climb removes 31 and 32 and keeps 33; a single-step climb removes nothing (its previous version IS the rollback target); the config switch disables it. Found by looking at the box after the first real ladder run — the feature worked, and then quietly cost 4.4 GB. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
221 lines
10 KiB
Bash
221 lines
10 KiB
Bash
#!/bin/bash
|
|
|
|
# Stepped upgrade engine — climbs a version ladder one rung at a time.
|
|
# ---------------------------------------------------------------------------
|
|
# For apps that cannot skip a release (Nextcloud refuses outright; databases
|
|
# refuse via their data directory), moving 31 -> 34 is not one update but three,
|
|
# each with a migration that must COMPLETE before the next begins.
|
|
#
|
|
# The per-rung contract, and every part of it is load-bearing:
|
|
#
|
|
# snapshot (fail-closed) -> set version -> pull -> up -> VERIFY -> next rung
|
|
#
|
|
# On any failure, at any point: restore THIS rung's snapshot, put the version
|
|
# back, stop, and leave the app on the last version it verified at. The ladder
|
|
# never continues past a doubt.
|
|
#
|
|
# Why a snapshot per rung rather than one at the start: upstream migrations are
|
|
# usually one-way. Nextcloud 32's schema changes cannot be undone by putting the
|
|
# 31 image back. So the recovery guarantee is "restore the snapshot taken sixty
|
|
# seconds ago", not "undo the upgrade" — which only works if each rung has its
|
|
# own restore point.
|
|
#
|
|
# Deliberately NOT automatic. CFG_<APP>_UPDATE_TYPE=auto applies patches within
|
|
# a line; crossing versions on stateful data stays a decision a person makes,
|
|
# after reading release notes. The updater surfaces "34 available"; this runs
|
|
# only when asked.
|
|
|
|
_updaterUpgradeGenDir() { echo "${containers_dir%/}/libreportal/frontend/data/updater/generated"; }
|
|
|
|
# Rewrite the anchor image AND its version sentinel, so the live compose stays
|
|
# self-consistent. updaterSetAnchorRef preserves the trailing comment verbatim,
|
|
# which would leave the sentinel advertising the OLD version — and the next
|
|
# config-driven regeneration would then quietly revert the app. Both or neither.
|
|
updaterSetAnchorVersion() {
|
|
local app="$1" newtag="$2"
|
|
local compose="${containers_dir%/}/$app/docker-compose.yml"
|
|
[ -f "$compose" ] || return 1
|
|
local svc="${app//_/-}-service"
|
|
local up; up="$(printf '%s' "$app" | tr '[:lower:]' '[:upper:]')"
|
|
local tmp; tmp="$(mktemp)"
|
|
awk -v s="$svc" -v tag="$newtag" -v key="${up}_VERSION_TAG" '
|
|
!done && seen && /^[[:space:]]*image:/ {
|
|
match($0,/^[[:space:]]*/); ind=substr($0,1,RLENGTH)
|
|
line=$0; sub(/^[[:space:]]*image:[[:space:]]*/,"",line)
|
|
sub(/[[:space:]]*#.*$/,"",line); gsub(/["'"'"']/,"",line)
|
|
repo=line; sub(/:[^:\/]*$/,"",repo) # strip the old tag
|
|
printf "%simage: %s:%s #LIBREPORTAL|%s|%s\n", ind, repo, tag, key, tag
|
|
done=1; next
|
|
}
|
|
$0 ~ ("^[[:space:]]*" s ":") { seen=1 }
|
|
{ print }
|
|
' "$compose" > "$tmp" || { rm -f "$tmp"; return 1; }
|
|
grep -q "image:.*:${newtag}" "$tmp" || { rm -f "$tmp"; return 1; }
|
|
runFileWrite "$compose" < "$tmp"; local rc=$?
|
|
rm -f "$tmp"
|
|
|
|
# Keep the config key in step when the app has one, so the WebUI's Version
|
|
# field shows what is actually deployed rather than what it used to be.
|
|
local cfgkey="CFG_${up}_VERSION"
|
|
if [ -n "${!cfgkey+x}" ] && declare -f updateConfigOption >/dev/null 2>&1; then
|
|
updateConfigOption "$cfgkey" "$newtag" >/dev/null 2>&1 || true
|
|
fi
|
|
return $rc
|
|
}
|
|
|
|
# Current anchor tag for an app, straight from its live compose.
|
|
updaterCurrentTag() {
|
|
local app="$1"
|
|
local compose="${containers_dir%/}/$app/docker-compose.yml"
|
|
[ -f "$compose" ] || return 1
|
|
updaterTagOf "$(updaterPrimaryImage "$app" "$compose")"
|
|
}
|
|
|
|
# updaterUpgradeApp <app> [target-tag] [--dry-run]
|
|
# Walks the ladder. Returns 0 only if every rung verified.
|
|
updaterUpgradeApp() {
|
|
local app="$1" target="${2:-}" mode="${3:-}"
|
|
[ "$target" = "--dry-run" ] && { mode="--dry-run"; target=""; }
|
|
|
|
local app_dir="${containers_dir%/}/$app"
|
|
[ -d "$app_dir" ] || { isError "App '$app' is not installed."; return 1; }
|
|
|
|
local cur; cur="$(updaterCurrentTag "$app")"
|
|
[ -n "$cur" ] || { isError "Could not read $app's current version tag."; return 1; }
|
|
|
|
local anchor repo
|
|
anchor="$(updaterPrimaryImage "$app" "$app_dir/docker-compose.yml")"
|
|
repo="$(updaterRepoTag "$anchor")"; repo="${repo%:*}"
|
|
|
|
# GATE 1 — a stepped upgrade without a real verifier is a guess. The generic
|
|
# health check cannot see a half-finished migration, so refusing here is the
|
|
# difference between this being a safety feature and a liability.
|
|
if ! updaterHasVerifier "$app"; then
|
|
isError "$app has no upgrade verifier, so a stepped upgrade cannot be confirmed safe."
|
|
isNotice "Add ${app}_upgrade_verify (see cli_updater_verify.sh) before laddering this app."
|
|
return 1
|
|
fi
|
|
|
|
# GATE 2 — the ladder must be computable end to end. updaterVersionLadder
|
|
# returns non-zero rather than guessing when it cannot reach the target.
|
|
local -a rungs=()
|
|
if ! mapfile -t rungs < <(updaterVersionLadder "$cur" "$repo" "$target") || (( ${#rungs[@]} == 0 )); then
|
|
if [ -n "$target" ]; then
|
|
isError "No safe path from $cur to $target could be determined — not attempting it."
|
|
isNotice "Upgrade these by hand, one release at a time, if you are sure."
|
|
return 1
|
|
fi
|
|
isSuccessful "$app is already on the newest release line ($cur)."
|
|
return 0
|
|
fi
|
|
|
|
isHeader "Upgrade plan for $app"
|
|
isNotice "$(updaterLadderSummary "$cur" "${rungs[@]}")"
|
|
isNotice "Each step: snapshot → pull → start → verify. A failure stops the ladder and restores that step."
|
|
|
|
if [ "$mode" = "--dry-run" ]; then
|
|
isSuccessful "Dry run — nothing was changed."
|
|
return 0
|
|
fi
|
|
|
|
local timeout="${CFG_UPDATER_UPGRADE_TIMEOUT:-900}"
|
|
local from="$cur" rung done_n=0
|
|
for rung in "${rungs[@]}"; do
|
|
isHeader "$app: $from → $rung (step $((done_n + 1)) of ${#rungs[@]})"
|
|
|
|
# 1. Snapshot THIS rung. Fail-closed: no snapshot, no step.
|
|
isNotice "Snapshotting $app before $rung…"
|
|
if ! backupAppStart "$app" >/dev/null 2>&1; then
|
|
isError "Pre-step snapshot failed — stopping with $app on $from."
|
|
updaterRecordHistory "$app" "upgrade" "$from" "$rung" "aborted-no-snapshot" "" "" "" "manual"
|
|
return 1
|
|
fi
|
|
|
|
# 2. Move the version.
|
|
if ! updaterSetAnchorVersion "$app" "$rung"; then
|
|
isError "Could not set $app to $rung — stopping, nothing changed."
|
|
updaterRecordHistory "$app" "upgrade" "$from" "$rung" "aborted-set-version" "" "" "" "manual"
|
|
return 1
|
|
fi
|
|
|
|
# 3. Pull + start.
|
|
if ! updaterComposePull "$app" || ! dockerComposeUp "$app" >/dev/null 2>&1; then
|
|
isError "$app failed to start on $rung — rolling this step back."
|
|
_updaterUpgradeRollbackStep "$app" "$from" "$rung"
|
|
return 1
|
|
fi
|
|
|
|
# 4. VERIFY. The rung is not done until the app says so itself.
|
|
if ! updaterVerifyUpgrade "$app" "$rung" "$timeout"; then
|
|
isError "$app did not verify on $rung — rolling this step back."
|
|
_updaterUpgradeRollbackStep "$app" "$from" "$rung"
|
|
return 1
|
|
fi
|
|
|
|
updaterRecordHistory "$app" "upgrade" "$from" "$rung" "ok" "" "" "" "manual"
|
|
isSuccessful "$app is verified on $rung."
|
|
from="$rung"; done_n=$((done_n + 1))
|
|
done
|
|
|
|
isSuccessful "$app upgraded through ${done_n} version(s) — now on $from, verified."
|
|
_updaterUpgradePruneImages "$app" "$cur" "${rungs[@]}"
|
|
webuiUpdaterScan >/dev/null 2>&1 || true
|
|
return 0
|
|
}
|
|
|
|
# Drop the images the climb left behind. A 3-rung Nextcloud upgrade downloads
|
|
# ~1.5 GB per rung and keeps every one — 4.4 GB of superseded images after a
|
|
# single upgrade, which on a small VPS is the difference between working and
|
|
# full. `system reclaim` cannot help: it collects DANGLING images, and each rung
|
|
# is a distinct tag, so they are all still tagged and all still there. (Rolling
|
|
# apps do not have this problem — moving a floating tag orphans the old image,
|
|
# which reclaim then collects.)
|
|
#
|
|
# Only ever after a SUCCESSFUL climb, and the immediately-previous version is
|
|
# KEPT as the roll-back target so recovery does not depend on the network.
|
|
# CFG_UPDATER_UPGRADE_PRUNE=false to keep everything.
|
|
_updaterUpgradePruneImages() {
|
|
local app="$1" start="$2"; shift 2
|
|
local -a climbed=("$@")
|
|
[[ "${CFG_UPDATER_UPGRADE_PRUNE:-true}" == "true" ]] || { isNotice "Keeping superseded images (CFG_UPDATER_UPGRADE_PRUNE=false)."; return 0; }
|
|
(( ${#climbed[@]} >= 2 )) || return 0 # one step: previous IS the rollback target
|
|
|
|
local anchor repo
|
|
anchor="$(updaterPrimaryImage "$app" "${containers_dir%/}/$app/docker-compose.yml")"
|
|
repo="$(updaterRepoTag "$anchor")"; repo="${repo%:*}"
|
|
|
|
# Everything we moved off, minus the last one (kept for rollback).
|
|
local -a superseded=("$start" "${climbed[@]:0:$(( ${#climbed[@]} - 1 ))}")
|
|
unset 'superseded[-1]'
|
|
(( ${#superseded[@]} > 0 )) || return 0
|
|
|
|
local tag removed=0
|
|
for tag in "${superseded[@]}"; do
|
|
[[ -n "$tag" ]] || continue
|
|
if runFileOp docker image rm "${repo}:${tag}" >/dev/null 2>&1; then
|
|
removed=$((removed + 1))
|
|
fi
|
|
done
|
|
(( removed > 0 )) && isSuccessful "Removed $removed superseded image(s); kept ${climbed[-2]} for roll-back."
|
|
return 0
|
|
}
|
|
|
|
# Undo one failed rung: put the version back, restore the snapshot taken moments
|
|
# ago, start it, and record what happened. Best effort by nature — if the
|
|
# restore itself fails the user is told plainly rather than reassured.
|
|
_updaterUpgradeRollbackStep() {
|
|
local app="$1" from="$2" failed="$3"
|
|
isNotice "Restoring $app to $from…"
|
|
updaterSetAnchorVersion "$app" "$from" || isError "Could not put $app's version back to $from — check its compose file."
|
|
if restoreAppStart "$app" latest "" >/dev/null 2>&1; then
|
|
dockerComposeUp "$app" >/dev/null 2>&1 || true
|
|
updaterRecordHistory "$app" "upgrade" "$from" "$failed" "rolled-back" "" "" "" "manual"
|
|
isSuccessful "$app restored to $from from its pre-step snapshot."
|
|
isNotice "The ladder stopped here. Read $failed's release notes before trying again."
|
|
return 0
|
|
fi
|
|
updaterRecordHistory "$app" "upgrade" "$from" "$failed" "rollback-failed" "" "" "" "manual"
|
|
isError "Could not restore $app automatically. Its data snapshot is intact — restore it from the Backups page."
|
|
return 1
|
|
}
|