#!/bin/bash # Stepped upgrade engine — climbs a version ladder one rung at a time. # --------------------------------------------------------------------------- # For apps that cannot skip a release (Nextcloud refuses outright; databases # refuse via their data directory), moving 31 -> 34 is not one update but three, # each with a migration that must COMPLETE before the next begins. # # The per-rung contract, and every part of it is load-bearing: # # snapshot (fail-closed) -> set version -> pull -> up -> VERIFY -> next rung # # On any failure, at any point: restore THIS rung's snapshot, put the version # back, stop, and leave the app on the last version it verified at. The ladder # never continues past a doubt. # # Why a snapshot per rung rather than one at the start: upstream migrations are # usually one-way. Nextcloud 32's schema changes cannot be undone by putting the # 31 image back. So the recovery guarantee is "restore the snapshot taken sixty # seconds ago", not "undo the upgrade" — which only works if each rung has its # own restore point. # # Deliberately NOT automatic. CFG__UPDATE_TYPE=auto applies patches within # a line; crossing versions on stateful data stays a decision a person makes, # after reading release notes. The updater surfaces "34 available"; this runs # only when asked. _updaterUpgradeGenDir() { echo "${containers_dir%/}/libreportal/frontend/data/updater/generated"; } # Rewrite the anchor image AND every image locked in step with it, plus their # version sentinels, so the live compose stays self-consistent. # # THE LOCK-STEP PROBLEM. Some apps are one product shipped as many images: # Stoat is nine stoatchat services released together, all on v0.15.1, and they # expect matching versions of each other. Moving only the anchor would put api # on v0.16 while events stayed on v0.15.1 — precisely the API/events mismatch # that once justified keeping the app off automatic updates. So the set has to # move together or not at all. # # The set is DERIVED, not configured, because the compose already states it: # a service is locked in step with the anchor when it carries a version # sentinel, currently sits on the SAME tag, and lives under the same registry # namespace. Both conditions are needed and each rejects a real case here: # ghcr.io/stoatchat/events:v0.15.1 same ns, same tag -> moves # ghcr.io/stoatchat/livekit-server:v1.9.13 same ns, other tag -> stays # vectorim/element-web:v1.12.25 other ns -> stays # mongo:8.0 no namespace -> stays # A sidecar that coincidentally shares a version number is excluded by the # namespace test; a sibling on its own release cadence by the tag test. # # The anchor itself is found by its BARE _VERSION_TAG sentinel, not by # guessing "-service". That guess was wrong for every app that names its # services anything else — matrix (matrix-synapse) and stoat (api) among them — # and since nothing matched, the rewrite silently changed no lines and the # upgrade aborted at "could not set version". Dry runs never showed it: they # return before this point. updaterSetAnchorVersion() { local app="$1" newtag="$2" local compose="${containers_dir%/}/$app/docker-compose.yml" [ -f "$compose" ] || return 1 local up; up="$(printf '%s' "$app" | tr '[:lower:]' '[:upper:]')" # Anchor line -> its current tag and registry namespace. local aline; aline="$(grep -E "#LIBREPORTAL\|${up}_VERSION_TAG\|" "$compose" 2>/dev/null | head -1)" [ -n "$aline" ] || return 1 # tr, not sed, to strip quotes: in a sed bracket expression \047 is not an # octal escape but the literal characters \ 0 4 7, so it silently deleted # every 0, 4 and 7 in the ref -- v0.15.1 became v.15.1. awk does honour the # escape, which is why the same idiom is fine below. local aref; aref="$(printf '%s' "$aline" | sed -E 's/^[[:space:]]*image:[[:space:]]*//; s/[[:space:]]*#.*$//' | tr -d "\"' ")" local atag="${aref##*:}"; local arepo="${aref%:*}" case "$aref" in */*:*|*:*) : ;; *) return 1 ;; esac local ans=""; case "$arepo" in */*) ans="${arepo%/*}" ;; esac [ -n "$atag" ] || return 1 local tmp; tmp="$(mktemp)" awk -v newtag="$newtag" -v atag="$atag" -v ans="$ans" ' /^[[:space:]]*image:/ && index($0, "#LIBREPORTAL|") && index($0, "_VERSION_TAG|") { match($0,/^[[:space:]]*/); ind=substr($0,1,RLENGTH) # key = the sentinel this line owns; each keeps its own. k=$0; sub(/^.*#LIBREPORTAL\|/,"",k); sub(/\|.*$/,"",k) ref=$0; sub(/^[[:space:]]*image:[[:space:]]*/,"",ref) sub(/[[:space:]]*#.*$/,"",ref); gsub(/["\047]/,"",ref) if (ref ~ /:/) { tag=ref; sub(/^.*:/,"",tag) repo=ref; sub(/:[^:\/]*$/,"",repo) ns=""; if (repo ~ /\//) { ns=repo; sub(/\/[^\/]*$/,"",ns) } if (tag == atag && ns == ans) { printf "%simage: %s:%s #LIBREPORTAL|%s|%s\n", ind, repo, newtag, k, newtag changed++ next } } } { print } END { if (!changed) exit 3 } ' "$compose" > "$tmp" || { rm -f "$tmp"; return 1; } grep -q "image:.*:${newtag}" "$tmp" || { rm -f "$tmp"; return 1; } runFileWrite "$compose" < "$tmp"; local rc=$? rm -f "$tmp" # Keep the config keys in step so the WebUI's Version field shows what is # actually deployed. Every sentinel now on the new tag gets its key set, not # just the anchor's — otherwise the next config-driven regeneration would # quietly pull the locked-step services back to the old version. if declare -f updateConfigOption >/dev/null 2>&1; then local k cfgkey while IFS= read -r k; do [ -n "$k" ] || continue cfgkey="CFG_${k%_VERSION_TAG}_VERSION" [ -n "${!cfgkey+x}" ] && updateConfigOption "$cfgkey" "$newtag" >/dev/null 2>&1 done < <(grep -oE "#LIBREPORTAL\|[A-Z0-9_]+_VERSION_TAG\|${newtag}" "$compose" 2>/dev/null \ | sed -E 's/#LIBREPORTAL\|//; s/\|.*$//' | sort -u) fi return $rc } # Current anchor tag for an app, straight from its live compose. updaterCurrentTag() { local app="$1" local compose="${containers_dir%/}/$app/docker-compose.yml" [ -f "$compose" ] || return 1 updaterTagOf "$(updaterPrimaryImage "$app" "$compose")" } # updaterUpgradeApp [target-tag] [--dry-run] # Walks the ladder. Returns 0 only if every rung verified. updaterUpgradeApp() { local app="$1" target="${2:-}" mode="${3:-}" [ "$target" = "--dry-run" ] && { mode="--dry-run"; target=""; } local app_dir="${containers_dir%/}/$app" [ -d "$app_dir" ] || { isError "App '$app' is not installed."; return 1; } local cur; cur="$(updaterCurrentTag "$app")" [ -n "$cur" ] || { isError "Could not read $app's current version tag."; return 1; } local anchor repo anchor="$(updaterPrimaryImage "$app" "$app_dir/docker-compose.yml")" repo="$(updaterRepoTag "$anchor")"; repo="${repo%:*}" # GATE 1 — a stepped upgrade without a real verifier is a guess. The generic # health check cannot see a half-finished migration, so refusing here is the # difference between this being a safety feature and a liability. if ! updaterHasVerifier "$app"; then isError "$app has no upgrade verifier, so a stepped upgrade cannot be confirmed safe." isNotice "Add ${app}_upgrade_verify (see cli_updater_verify.sh) before laddering this app." return 1 fi # GATE 2 — the ladder must be computable end to end. updaterVersionLadder # returns non-zero rather than guessing when it cannot reach the target. local -a rungs=() if ! mapfile -t rungs < <(updaterVersionLadder "$cur" "$repo" "$target") || (( ${#rungs[@]} == 0 )); then if [ -n "$target" ]; then isError "No safe path from $cur to $target could be determined — not attempting it." isNotice "Upgrade these by hand, one release at a time, if you are sure." return 1 fi isSuccessful "$app is already on the newest release line ($cur)." return 0 fi isHeader "Upgrade plan for $app" isNotice "$(updaterLadderSummary "$cur" "${rungs[@]}")" isNotice "Each step: snapshot → pull → start → verify. A failure stops the ladder and restores that step." if [ "$mode" = "--dry-run" ]; then isSuccessful "Dry run — nothing was changed." return 0 fi local timeout="${CFG_UPDATER_UPGRADE_TIMEOUT:-900}" local from="$cur" rung done_n=0 for rung in "${rungs[@]}"; do isHeader "$app: $from → $rung (step $((done_n + 1)) of ${#rungs[@]})" # 1. Snapshot THIS rung. Fail-closed: no snapshot, no step. isNotice "Snapshotting $app before $rung…" if ! backupAppStart "$app" >/dev/null 2>&1; then isError "Pre-step snapshot failed — stopping with $app on $from." updaterRecordHistory "$app" "upgrade" "$from" "$rung" "aborted-no-snapshot" "" "" "" "${UPDATER_UPGRADE_TRIGGER:-manual}" return 1 fi # 2. Move the version. if ! updaterSetAnchorVersion "$app" "$rung"; then isError "Could not set $app to $rung — stopping, nothing changed." updaterRecordHistory "$app" "upgrade" "$from" "$rung" "aborted-set-version" "" "" "" "${UPDATER_UPGRADE_TRIGGER:-manual}" return 1 fi # 3. Pull + start. if ! updaterComposePull "$app" || ! dockerComposeUp "$app" >/dev/null 2>&1; then isError "$app failed to start on $rung — rolling this step back." _updaterUpgradeRollbackStep "$app" "$from" "$rung" return 1 fi # 4. VERIFY. The rung is not done until the app says so itself. if ! updaterVerifyUpgrade "$app" "$rung" "$timeout"; then isError "$app did not verify on $rung — rolling this step back." _updaterUpgradeRollbackStep "$app" "$from" "$rung" return 1 fi updaterRecordHistory "$app" "upgrade" "$from" "$rung" "ok" "" "" "" "${UPDATER_UPGRADE_TRIGGER:-manual}" isSuccessful "$app is verified on $rung." from="$rung"; done_n=$((done_n + 1)) done isSuccessful "$app upgraded through ${done_n} version(s) — now on $from, verified." _updaterUpgradePruneImages "$app" "$cur" "${rungs[@]}" webuiUpdaterScan >/dev/null 2>&1 || true return 0 } # Drop the images the climb left behind. A 3-rung Nextcloud upgrade downloads # ~1.5 GB per rung and keeps every one — 4.4 GB of superseded images after a # single upgrade, which on a small VPS is the difference between working and # full. `system reclaim` cannot help: it collects DANGLING images, and each rung # is a distinct tag, so they are all still tagged and all still there. (Rolling # apps do not have this problem — moving a floating tag orphans the old image, # which reclaim then collects.) # # Only ever after a SUCCESSFUL climb, and the immediately-previous version is # KEPT as the roll-back target so recovery does not depend on the network. # CFG_UPDATER_UPGRADE_PRUNE=false to keep everything. _updaterUpgradePruneImages() { local app="$1" start="$2"; shift 2 local -a climbed=("$@") [[ "${CFG_UPDATER_UPGRADE_PRUNE:-true}" == "true" ]] || { isNotice "Keeping superseded images (CFG_UPDATER_UPGRADE_PRUNE=false)."; return 0; } (( ${#climbed[@]} >= 2 )) || return 0 # one step: previous IS the rollback target local anchor repo anchor="$(updaterPrimaryImage "$app" "${containers_dir%/}/$app/docker-compose.yml")" repo="$(updaterRepoTag "$anchor")"; repo="${repo%:*}" # Everything we moved off, minus the last one (kept for rollback). local -a superseded=("$start" "${climbed[@]:0:$(( ${#climbed[@]} - 1 ))}") unset 'superseded[-1]' (( ${#superseded[@]} > 0 )) || return 0 local tag removed=0 for tag in "${superseded[@]}"; do [[ -n "$tag" ]] || continue if runFileOp docker image rm "${repo}:${tag}" >/dev/null 2>&1; then removed=$((removed + 1)) fi done (( removed > 0 )) && isSuccessful "Removed $removed superseded image(s); kept ${climbed[-2]} for roll-back." return 0 } # Undo one failed rung: put the version back, restore the snapshot taken moments # ago, start it, and record what happened. Best effort by nature — if the # restore itself fails the user is told plainly rather than reassured. _updaterUpgradeRollbackStep() { local app="$1" from="$2" failed="$3" isNotice "Restoring $app to $from…" updaterSetAnchorVersion "$app" "$from" || isError "Could not put $app's version back to $from — check its compose file." if restoreAppStart "$app" latest "" >/dev/null 2>&1; then dockerComposeUp "$app" >/dev/null 2>&1 || true updaterRecordHistory "$app" "upgrade" "$from" "$failed" "rolled-back" "" "" "" "${UPDATER_UPGRADE_TRIGGER:-manual}" isSuccessful "$app restored to $from from its pre-step snapshot." isNotice "The ladder stopped here. Read $failed's release notes before trying again." return 0 fi updaterRecordHistory "$app" "upgrade" "$from" "$failed" "rollback-failed" "" "" "" "${UPDATER_UPGRADE_TRIGGER:-manual}" isError "Could not restore $app automatically. Its data snapshot is intact — restore it from the Backups page." return 1 }