#!/bin/bash # Upgrade verification — "did that rung actually land?" # --------------------------------------------------------------------------- # A stepped upgrade is only as safe as its verification. Stepping 31 -> 32 -> 33 # is arithmetic; knowing that 32 FINISHED before touching 33 is the whole ball # game, because the dangerous state is invisible from the outside: Nextcloud # runs its migration on boot and can sit in maintenance mode, or fail halfway, # while Docker cheerfully reports the container healthy. Advance a rung then and # you have skipped a migration on live data. # # So "the container is up" is explicitly NOT accepted as proof for an app that # declares a verifier. Contract: # # _upgrade_verify -> 0 = verified # # It must poll until the deadline and return 0 ONLY when it can positively # confirm the app is serving at the expected version with no migration # outstanding. Any other outcome — unhealthy, indeterminate, timed out — must # return non-zero. Uncertainty is a failure here, not a maybe: the engine # aborts and restores rather than guessing, which is the only honest reading # when the alternative risks someone's data. # # Apps with no verifier fall back to updaterVerifyGeneric, which is deliberately # conservative and is NOT sufficient for a stepped upgrade — the engine refuses # to ladder an app that has not declared a real one. # Container name for an app's ANCHOR service — the one whose image carries the # app's own version. Read from the compose rather than assumed, because # "-service" is a convention and not every app follows it: matrix names its # anchor service matrix-synapse and stoat names its api, so the assumption # inspected a container that does not exist, found no state at all, and the # generic verifier could only ever time out on exactly the stateful apps that # most need verifying. Falls back to the convention when there is no compose. _updaterPrimaryContainer() { local app="$1" local compose="$(appDir "$app")/docker-compose.yml" local fallback; fallback="$(printf '%s-service' "${app//_/-}")" [ -f "$compose" ] || { printf '%s' "$fallback"; return 0; } local up; up="$(printf '%s' "$app" | tr '[:lower:]' '[:upper:]')" # The service block that owns the bare _VERSION_TAG sentinel, then that # block's container_name -- compose's own answer for what the container is # called. \047 is an apostrophe: the program is single-quoted, so it cannot # contain one literally. local name name="$(awk -v key="#LIBREPORTAL|${up}_VERSION_TAG|" ' /^[[:space:]]{2,4}[a-zA-Z0-9_-]+:[[:space:]]*(#|$)/ { if (found) { if (cname != "") print cname; else print svc; exit } s=$1; sub(/:.*/,"",s); gsub(/[[:space:]]/,"",s) svc=s; cname="" } /^[[:space:]]*container_name:/ { cname=$2; gsub(/["\047]/,"",cname) } index($0,key) { found=1 } END { if (found) { if (cname != "") print cname; else print svc } } ' "$compose" 2>/dev/null | head -1 | tr -d '[:space:]')" [ -n "$name" ] && printf '%s' "$name" || printf '%s' "$fallback" } # Generic health: running, not restarting, healthcheck (if any) reporting # healthy, and STILL true after a settle period — a crash-loop looks perfect # in the instant between restarts. Good enough for a single in-place update, # never good enough to justify climbing another rung. updaterVerifyGeneric() { local app="$1" deadline="${3:-$(( $(date +%s) + 120 ))}" local c; c="$(_updaterPrimaryContainer "$app")" local stable_needed=3 stable=0 while [ "$(date +%s)" -lt "$deadline" ]; do local state health restarts state="$(dockerCommandRun "docker inspect --format '{{.State.Status}}' $c" 2>/dev/null | tr -d '\r')" health="$(dockerCommandRun "docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' $c" 2>/dev/null | tr -d '\r')" restarts="$(dockerCommandRun "docker inspect --format '{{.RestartCount}}' $c" 2>/dev/null | tr -d '\r')" if [ "$state" = "running" ] && { [ "$health" = "healthy" ] || [ "$health" = "none" ]; }; then stable=$((stable + 1)) [ -z "${_uv_restarts:-}" ] && _uv_restarts="$restarts" # A restart during the settle window means it is looping, not up. [ "$restarts" != "$_uv_restarts" ] && stable=0 && _uv_restarts="$restarts" (( stable >= stable_needed )) && { unset _uv_restarts; return 0; } else stable=0 fi sleep 5 done unset _uv_restarts return 1 } # Does this app ship a real verifier? The stepped engine requires one. # # "Not currently defined" is NOT the same as "not shipped", and conflating the # two is the most expensive mistake this file can make. The CLI runs lazy # (LP_LAZY=1), where the container scan is skipped entirely and every function # has to arrive through function_manifest.sh — a BUILD-time artifact. Any app # that came into existence after that build has no stub, so its verifier reads # as absent while sitting on disk two directories away. That is exactly every # multi-instance clone, and the consequences both point the wrong way: GATE 1 # refuses an upgrade for an app that is fully verifiable, and the auto-ladder's # `updaterHasVerifier || continue` drops it in silence, for good. # # So consult the disk before answering no, and source what is actually there — # the same file the eager loader would have picked up. Cheap (one stat for the # common case) and self-healing regardless of how stale the manifest is, which # matters because a LibrePortal self-update overwrites the manifest with the # shipped copy and drops every instance entry again. updaterHasVerifier() { local app="$1" declare -F "${app}_upgrade_verify" >/dev/null 2>&1 && return 0 local hooks="${install_containers_dir%/}/$app/scripts/${app}_upgrade_hooks.sh" [ -f "$hooks" ] || return 1 source "$hooks" 2>/dev/null || return 1 declare -F "${app}_upgrade_verify" >/dev/null 2>&1 } # updaterVerifyUpgrade [timeout-secs] # Dispatches to the app's verifier, falling back to the generic check. Returns # 0 only on positive confirmation. updaterVerifyUpgrade() { local app="$1" expected="$2" timeout="${3:-600}" local deadline=$(( $(date +%s) + timeout )) if updaterHasVerifier "$app"; then isNotice "Verifying $app is serving $expected (up to ${timeout}s)…" if "${app}_upgrade_verify" "$app" "$expected" "$deadline"; then isSuccessful "$app verified at $expected." return 0 fi isError "$app did NOT verify at $expected — treating as failed." return 1 fi isNotice "$app has no upgrade verifier; using the generic health check." updaterVerifyGeneric "$app" "$expected" "$deadline" } # --------------------------------------------------------------------------- # Shared HTTP version verification. # # The strongest thing a verifier can say is "the app itself reports the version # we asked for, and kept reporting it." Most apps expose that over HTTP, so the # per-app hook becomes three facts — port, path, where the version lives — and # the polling, extraction, comparison and stability rules live here once. # # Probed from the HOST against the container's PUBLISHED port, deliberately, not # via `docker exec … curl`: half these images ship no curl at all (mattermost is # one), so exec-based probes are a coin flip on the vendor's base image. The # host has curl, and a published port is something LibrePortal already # guarantees for every app with a web interface. # First published host port for a container's internal port. Empty if unmapped. _updaterPublishedPortFor() { local c="$1" internal="$2" dockerCommandRun "docker port $c $internal" 2>/dev/null \ | tr -d '\r' | grep -oE '[0-9]+$' | head -1 } # Pull a version out of an HTTP response. "json:" reads a top-level string # field; "header:" reads a response header (case-insensitively). _updaterExtractVersion() { local resp="$1" spec="$2" case "$spec" in json:*) local k="${spec#json:}" printf '%s' "$resp" | tr -d '\r' \ | grep -oE "\"$k\"[[:space:]]*:[[:space:]]*\"[^\"]*\"" | head -1 \ | sed -E 's/.*:[[:space:]]*"([^"]*)"/\1/' ;; header:*) local h="${spec#header:}" printf '%s' "$resp" | tr -d '\r' | grep -i "^${h}:" | head -1 \ | sed -E 's/^[^:]*:[[:space:]]*//' ;; esac } # Do a tag and a self-reported version agree on every component BOTH of them # state? Tags and running versions are rarely the same precision: # v1.158.0 vs 1.158.0 -> yes (the v is noise) # 11.9 vs 11.9.1 -> yes (the tag is a line; the build is more precise) # 8.7.0 vs 8.7 -> yes (the app reports less precision than the tag) # 11.9 vs 11.10 -> NO # Comparing only the shared prefix is what makes one rule work for all of them; # demanding string equality would fail every app above except Synapse. _updaterVersionAgrees() { local a b a="$(printf '%s' "$1" | grep -oE '[0-9]+' | tr '\n' ' ')" b="$(printf '%s' "$2" | grep -oE '[0-9]+' | tr '\n' ' ')" local -a A=($a) B=($b) local n=${#A[@]}; [ ${#B[@]} -lt "$n" ] && n=${#B[@]} [ "$n" -gt 0 ] || return 1 local i for ((i=0; i # 0 only when the app reports a version agreeing with the tag, three polls # running. Three because one lucky answer during a rolling restart proves # nothing — the old container can still be serving while the new one boots. updaterVerifyHttpVersion() { local app="$1" expected="$2" deadline="$3" iport="$4" path="$5" extract="$6" local c; c="$(_updaterPrimaryContainer "$app")" local port; port="$(_updaterPublishedPortFor "$c" "$iport")" if [ -z "$port" ]; then isError "$app: container $c publishes no host port for $iport — cannot verify." return 1 fi local stable=0 need=3 last="" resp ver while [ "$(date +%s)" -lt "$deadline" ]; do resp="$(curl -fsS -i --max-time 6 "http://127.0.0.1:${port}${path}" 2>/dev/null)" ver="$(_updaterExtractVersion "$resp" "$extract")" last="reported=${ver:-none}" if [ -n "$ver" ] && _updaterVersionAgrees "$expected" "$ver"; then stable=$((stable + 1)) if (( stable >= need )); then isSuccessful "$app reports $ver, agreeing with $expected, and held it." return 0 fi else stable=0 fi sleep 5 done isError "$app did not report $expected before the deadline.${last:+ Last probe: $last}" return 1 }