LibrePortal/scripts/cli/commands/updater/cli_updater_upgrade.sh

#!/bin/bash

# Stepped upgrade engine — climbs a version ladder one rung at a time.
# ---------------------------------------------------------------------------
# For apps that cannot skip a release (Nextcloud refuses outright; databases
# refuse via their data directory), moving 31 -> 34 is not one update but three,
# each with a migration that must COMPLETE before the next begins.
#
# The per-rung contract, and every part of it is load-bearing:
#
#   snapshot (fail-closed) -> set version -> pull -> up -> VERIFY -> next rung
#
# On any failure, at any point: restore THIS rung's snapshot, put the version
# back, stop, and leave the app on the last version it verified at. The ladder
# never continues past a doubt.
#
# Why a snapshot per rung rather than one at the start: upstream migrations are
# usually one-way. Nextcloud 32's schema changes cannot be undone by putting the
# 31 image back. So the recovery guarantee is "restore the snapshot taken sixty
# seconds ago", not "undo the upgrade" — which only works if each rung has its
# own restore point.
#
# Deliberately NOT automatic. CFG_<APP>_UPDATE_TYPE=auto applies patches within
# a line; crossing versions on stateful data stays a decision a person makes,
# after reading release notes. The updater surfaces "34 available"; this runs
# only when asked.

_updaterUpgradeGenDir() { echo "$(webuiDir)/frontend/data/updater/generated"; }

# Rewrite the anchor image AND every image locked in step with it, plus their
# version sentinels, so the live compose stays self-consistent.
#
# THE LOCK-STEP PROBLEM. Some apps are one product shipped as many images:
# Stoat is nine stoatchat services released together, all on v0.15.1, and they
# expect matching versions of each other. Moving only the anchor would put api
# on v0.16 while events stayed on v0.15.1 — precisely the API/events mismatch
# that once justified keeping the app off automatic updates. So the set has to
# move together or not at all.
#
# The set is DERIVED, not configured, because the compose already states it:
# a service is locked in step with the anchor when it carries a version
# sentinel, currently sits on the SAME tag, and lives under the same registry
# namespace. Both conditions are needed and each rejects a real case here:
#   ghcr.io/stoatchat/events:v0.15.1      same ns, same tag  -> moves
#   ghcr.io/stoatchat/livekit-server:v1.9.13  same ns, other tag -> stays
#   vectorim/element-web:v1.12.25         other ns           -> stays
#   mongo:8.0                             no namespace       -> stays
# A sidecar that coincidentally shares a version number is excluded by the
# namespace test; a sibling on its own release cadence by the tag test.
#
# The anchor itself is found by its BARE <APP>_VERSION_TAG sentinel, not by
# guessing "<app>-service". That guess was wrong for every app that names its
# services anything else — matrix (matrix-synapse) and stoat (api) among them —
# and since nothing matched, the rewrite silently changed no lines and the
# upgrade aborted at "could not set version". Dry runs never showed it: they
# return before this point.
updaterSetAnchorVersion() {
    local app="$1" newtag="$2"
    local compose="$(appDir "$app")/docker-compose.yml"
    [ -f "$compose" ] || return 1
    local up; up="$(printf '%s' "$app" | tr '[:lower:]' '[:upper:]')"

    # Anchor line -> its current tag and registry namespace.
    local aline; aline="$(grep -E "#LIBREPORTAL\|${up}_VERSION_TAG\|" "$compose" 2>/dev/null | head -1)"
    [ -n "$aline" ] || return 1
    # tr, not sed, to strip quotes: in a sed bracket expression \047 is not an
    # octal escape but the literal characters \ 0 4 7, so it silently deleted
    # every 0, 4 and 7 in the ref -- v0.15.1 became v.15.1. awk does honour the
    # escape, which is why the same idiom is fine below.
    local aref; aref="$(printf '%s' "$aline" | sed -E 's/^[[:space:]]*image:[[:space:]]*//; s/[[:space:]]*#.*$//' | tr -d "\"' ")"
    local atag="${aref##*:}"; local arepo="${aref%:*}"
    case "$aref" in */*:*|*:*) : ;; *) return 1 ;; esac
    local ans=""; case "$arepo" in */*) ans="${arepo%/*}" ;; esac
    [ -n "$atag" ] || return 1

    local tmp; tmp="$(mktemp)"
    awk -v newtag="$newtag" -v atag="$atag" -v ans="$ans" '
        /^[[:space:]]*image:/ && index($0, "#LIBREPORTAL|") && index($0, "_VERSION_TAG|") {
            match($0,/^[[:space:]]*/); ind=substr($0,1,RLENGTH)
            # key = the sentinel this line owns; each keeps its own.
            k=$0; sub(/^.*#LIBREPORTAL\|/,"",k); sub(/\|.*$/,"",k)
            ref=$0; sub(/^[[:space:]]*image:[[:space:]]*/,"",ref)
            sub(/[[:space:]]*#.*$/,"",ref); gsub(/["\047]/,"",ref)
            if (ref ~ /:/) {
                tag=ref; sub(/^.*:/,"",tag)
                repo=ref; sub(/:[^:\/]*$/,"",repo)
                ns=""; if (repo ~ /\//) { ns=repo; sub(/\/[^\/]*$/,"",ns) }
                if (tag == atag && ns == ans) {
                    printf "%simage: %s:%s #LIBREPORTAL|%s|%s\n", ind, repo, newtag, k, newtag
                    changed++
                    next
                }
            }
        }
        { print }
        END { if (!changed) exit 3 }
    ' "$compose" > "$tmp" || { rm -f "$tmp"; return 1; }
    grep -q "image:.*:${newtag}" "$tmp" || { rm -f "$tmp"; return 1; }
    runFileWrite "$compose" < "$tmp"; local rc=$?
    rm -f "$tmp"

    # Keep the config keys in step so the WebUI's Version field shows what is
    # actually deployed. Every sentinel now on the new tag gets its key set, not
    # just the anchor's — otherwise the next config-driven regeneration would
    # quietly pull the locked-step services back to the old version.
    if declare -f updateConfigOption >/dev/null 2>&1; then
        local k cfgkey
        while IFS= read -r k; do
            [ -n "$k" ] || continue
            cfgkey="CFG_${k%_VERSION_TAG}_VERSION"
            [ -n "${!cfgkey+x}" ] && updateConfigOption "$cfgkey" "$newtag" >/dev/null 2>&1
        done < <(grep -oE "#LIBREPORTAL\|[A-Z0-9_]+_VERSION_TAG\|${newtag}" "$compose" 2>/dev/null \
                 | sed -E 's/#LIBREPORTAL\|//; s/\|.*$//' | sort -u)
    fi
    return $rc
}

# Current anchor tag for an app, straight from its live compose.
updaterCurrentTag() {
    local app="$1"
    local compose="$(appDir "$app")/docker-compose.yml"
    [ -f "$compose" ] || return 1
    updaterTagOf "$(updaterPrimaryImage "$app" "$compose")"
}

# updaterUpgradeApp <app> [target-tag] [--dry-run]
# Walks the ladder. Returns 0 only if every rung verified.
updaterUpgradeApp() {
    local app="$1" target="${2:-}" mode="${3:-}"
    [ "$target" = "--dry-run" ] && { mode="--dry-run"; target=""; }

    local app_dir="$(appDir "$app")"
    [ -d "$app_dir" ] || { isError "App '$app' is not installed."; return 1; }

    local cur; cur="$(updaterCurrentTag "$app")"
    [ -n "$cur" ] || { isError "Could not read $app's current version tag."; return 1; }

    local anchor repo
    anchor="$(updaterPrimaryImage "$app" "$app_dir/docker-compose.yml")"
    repo="$(updaterRepoTag "$anchor")"; repo="${repo%:*}"

    # GATE 1 — a stepped upgrade without a real verifier is a guess. The generic
    # health check cannot see a half-finished migration, so refusing here is the
    # difference between this being a safety feature and a liability.
    if ! updaterHasVerifier "$app"; then
        isError "$app has no upgrade verifier, so a stepped upgrade cannot be confirmed safe."
        isNotice "Add ${app}_upgrade_verify (see cli_updater_verify.sh) before laddering this app."
        return 1
    fi

    # GATE 2 — the ladder must be computable end to end. updaterVersionLadder
    # returns non-zero rather than guessing when it cannot reach the target.
    local -a rungs=()
    if ! mapfile -t rungs < <(updaterVersionLadder "$cur" "$repo" "$target") || (( ${#rungs[@]} == 0 )); then
        if [ -n "$target" ]; then
            isError "No safe path from $cur to $target could be determined — not attempting it."
            isNotice "Upgrade these by hand, one release at a time, if you are sure."
            return 1
        fi
        isSuccessful "$app is already on the newest release line ($cur)."
        return 0
    fi

    isHeader "Upgrade plan for $app"
    isNotice "$(updaterLadderSummary "$cur" "${rungs[@]}")"
    isNotice "Each step: snapshot → pull → start → verify. A failure stops the ladder and restores that step."

    if [ "$mode" = "--dry-run" ]; then
        isSuccessful "Dry run — nothing was changed."
        return 0
    fi

    local timeout="${CFG_UPDATER_UPGRADE_TIMEOUT:-900}"
    local from="$cur" rung done_n=0
    for rung in "${rungs[@]}"; do
        isHeader "$app: $from → $rung  (step $((done_n + 1)) of ${#rungs[@]})"

        # 1. Snapshot THIS rung. Fail-closed: no snapshot, no step.
        isNotice "Snapshotting $app before $rung…"
        if ! backupAppStart "$app" >/dev/null 2>&1; then
            isError "Pre-step snapshot failed — stopping with $app on $from."
            updaterRecordHistory "$app" "upgrade" "$from" "$rung" "aborted-no-snapshot" "" "" "" "${UPDATER_UPGRADE_TRIGGER:-manual}"
            return 1
        fi

        # 2. Move the version.
        if ! updaterSetAnchorVersion "$app" "$rung"; then
            isError "Could not set $app to $rung — stopping, nothing changed."
            updaterRecordHistory "$app" "upgrade" "$from" "$rung" "aborted-set-version" "" "" "" "${UPDATER_UPGRADE_TRIGGER:-manual}"
            return 1
        fi

        # 3. Pull + start.
        if ! updaterComposePull "$app" || ! dockerComposeUp "$app" >/dev/null 2>&1; then
            isError "$app failed to start on $rung — rolling this step back."
            _updaterUpgradeRollbackStep "$app" "$from" "$rung"
            return 1
        fi

        # 4. VERIFY. The rung is not done until the app says so itself.
        if ! updaterVerifyUpgrade "$app" "$rung" "$timeout"; then
            isError "$app did not verify on $rung — rolling this step back."
            _updaterUpgradeRollbackStep "$app" "$from" "$rung"
            return 1
        fi

        updaterRecordHistory "$app" "upgrade" "$from" "$rung" "ok" "" "" "" "${UPDATER_UPGRADE_TRIGGER:-manual}"
        isSuccessful "$app is verified on $rung."
        from="$rung"; done_n=$((done_n + 1))
    done

    isSuccessful "$app upgraded through ${done_n} version(s) — now on $from, verified."
    _updaterUpgradePruneImages "$app" "$cur" "${rungs[@]}"
    webuiUpdaterScan >/dev/null 2>&1 || true
    return 0
}

# Drop the images the climb left behind. A 3-rung Nextcloud upgrade downloads
# ~1.5 GB per rung and keeps every one — 4.4 GB of superseded images after a
# single upgrade, which on a small VPS is the difference between working and
# full. `system reclaim` cannot help: it collects DANGLING images, and each rung
# is a distinct tag, so they are all still tagged and all still there. (Rolling
# apps do not have this problem — moving a floating tag orphans the old image,
# which reclaim then collects.)
#
# Only ever after a SUCCESSFUL climb, and the immediately-previous version is
# KEPT as the roll-back target so recovery does not depend on the network.
# CFG_UPDATER_UPGRADE_PRUNE=false to keep everything.
_updaterUpgradePruneImages() {
    local app="$1" start="$2"; shift 2
    local -a climbed=("$@")
    [[ "${CFG_UPDATER_UPGRADE_PRUNE:-true}" == "true" ]] || { isNotice "Keeping superseded images (CFG_UPDATER_UPGRADE_PRUNE=false)."; return 0; }
    (( ${#climbed[@]} >= 2 )) || return 0          # one step: previous IS the rollback target

    local anchor repo
    anchor="$(updaterPrimaryImage "$app" "$(appDir "$app")/docker-compose.yml")"
    repo="$(updaterRepoTag "$anchor")"; repo="${repo%:*}"

    # Everything we moved off, minus the last one (kept for rollback).
    local -a superseded=("$start" "${climbed[@]:0:$(( ${#climbed[@]} - 1 ))}")
    unset 'superseded[-1]'
    (( ${#superseded[@]} > 0 )) || return 0

    local tag removed=0
    for tag in "${superseded[@]}"; do
        [[ -n "$tag" ]] || continue
        if runFileOp docker image rm "${repo}:${tag}" >/dev/null 2>&1; then
            removed=$((removed + 1))
        fi
    done
    (( removed > 0 )) && isSuccessful "Removed $removed superseded image(s); kept ${climbed[-2]} for roll-back."
    return 0
}

# Undo one failed rung: put the version back, restore the snapshot taken moments
# ago, start it, and record what happened. Best effort by nature — if the
# restore itself fails the user is told plainly rather than reassured.
_updaterUpgradeRollbackStep() {
    local app="$1" from="$2" failed="$3"
    isNotice "Restoring $app to $from…"
    updaterSetAnchorVersion "$app" "$from" || isError "Could not put $app's version back to $from — check its compose file."
    if restoreAppStart "$app" latest "" >/dev/null 2>&1; then
        dockerComposeUp "$app" >/dev/null 2>&1 || true
        updaterRecordHistory "$app" "upgrade" "$from" "$failed" "rolled-back" "" "" "" "${UPDATER_UPGRADE_TRIGGER:-manual}"
        isSuccessful "$app restored to $from from its pre-step snapshot."
        isNotice "The ladder stopped here. Read $failed's release notes before trying again."
        return 0
    fi
    updaterRecordHistory "$app" "upgrade" "$from" "$failed" "rollback-failed" "" "" "" "${UPDATER_UPGRADE_TRIGGER:-manual}"
    isError "Could not restore $app automatically. Its data snapshot is intact — restore it from the Backups page."
    return 1
}