8b5e02c760
The main sweep — ~260 call sites across ~100 files move from string
concatenation on a single root to appDir/storageAppDirs/storageAppConfigs.
On a single-root install the resolved paths are identical, so this is a
no-op until a location is registered.
Enumerators were the interesting half. `for d in "$containers_dir"/*/`
appears in the menus, the registry/artifact scanners and the DNS setup —
and a shell glob cannot list a rootless 751 tree at all, which is the
same bug config_find_file.sh already documents in a comment. Routing them
through storageAppDirs (which enumerates as the owning user) fixes that
alongside the multi-root work.
Three places needed judgement rather than substitution:
db_app_scan.sh deletes database rows and port allocations for apps whose
folder is missing, and reaps "empty" app dirs. With a storage location
unmounted, every app on it looks exactly like that. Each of those
branches now gates on appStorageAvailable first — an app on an unplugged
drive is skipped with a notice, never deleted.
instance_create.sh rewrites cloned hooks so an instance touches its own
directory instead of the base app's. Its sed matched ${containers_dir}<type>,
which this sweep just replaced with $(appDir <type>) — so it would have
silently stopped redirecting, and an instance would have written to the
original's files (the adguard auth adapter case its own comment warns
about). Now matches both appDir forms, verified against bare, quoted,
unrelated-app, legacy and prose cases.
peer_shell/peer_pull streamed and extracted relative to the primary root.
Both now use the app's own root, and peer_shell keeps a single-root
fallback since it runs as a restricted SSH shell with no LibrePortal env.
Also fixes a pre-existing bug found on the way: webui_app_config.sh
tested "$containers_dir/frontend/data/last_update", one level short of the
real tree under the libreportal app dir, so the WebUI refresh trigger
after a config update has never once fired.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
233 lines
11 KiB
Bash
233 lines
11 KiB
Bash
#!/bin/bash
|
|
|
|
# Upgrade verification — "did that rung actually land?"
|
|
# ---------------------------------------------------------------------------
|
|
# A stepped upgrade is only as safe as its verification. Stepping 31 -> 32 -> 33
|
|
# is arithmetic; knowing that 32 FINISHED before touching 33 is the whole ball
|
|
# game, because the dangerous state is invisible from the outside: Nextcloud
|
|
# runs its migration on boot and can sit in maintenance mode, or fail halfway,
|
|
# while Docker cheerfully reports the container healthy. Advance a rung then and
|
|
# you have skipped a migration on live data.
|
|
#
|
|
# So "the container is up" is explicitly NOT accepted as proof for an app that
|
|
# declares a verifier. Contract:
|
|
#
|
|
# <app>_upgrade_verify <app> <expected-tag> <deadline-epoch> -> 0 = verified
|
|
#
|
|
# It must poll until the deadline and return 0 ONLY when it can positively
|
|
# confirm the app is serving at the expected version with no migration
|
|
# outstanding. Any other outcome — unhealthy, indeterminate, timed out — must
|
|
# return non-zero. Uncertainty is a failure here, not a maybe: the engine
|
|
# aborts and restores rather than guessing, which is the only honest reading
|
|
# when the alternative risks someone's data.
|
|
#
|
|
# Apps with no verifier fall back to updaterVerifyGeneric, which is deliberately
|
|
# conservative and is NOT sufficient for a stepped upgrade — the engine refuses
|
|
# to ladder an app that has not declared a real one.
|
|
|
|
# Container name for an app's ANCHOR service — the one whose image carries the
|
|
# app's own version. Read from the compose rather than assumed, because
|
|
# "<app>-service" is a convention and not every app follows it: matrix names its
|
|
# anchor service matrix-synapse and stoat names its api, so the assumption
|
|
# inspected a container that does not exist, found no state at all, and the
|
|
# generic verifier could only ever time out on exactly the stateful apps that
|
|
# most need verifying. Falls back to the convention when there is no compose.
|
|
_updaterPrimaryContainer() {
|
|
local app="$1"
|
|
local compose="$(appDir "$app")/docker-compose.yml"
|
|
local fallback; fallback="$(printf '%s-service' "${app//_/-}")"
|
|
[ -f "$compose" ] || { printf '%s' "$fallback"; return 0; }
|
|
|
|
local up; up="$(printf '%s' "$app" | tr '[:lower:]' '[:upper:]')"
|
|
# The service block that owns the bare <APP>_VERSION_TAG sentinel, then that
|
|
# block's container_name -- compose's own answer for what the container is
|
|
# called. \047 is an apostrophe: the program is single-quoted, so it cannot
|
|
# contain one literally.
|
|
local name
|
|
name="$(awk -v key="#LIBREPORTAL|${up}_VERSION_TAG|" '
|
|
/^[[:space:]]{2,4}[a-zA-Z0-9_-]+:[[:space:]]*(#|$)/ {
|
|
if (found) { if (cname != "") print cname; else print svc; exit }
|
|
s=$1; sub(/:.*/,"",s); gsub(/[[:space:]]/,"",s)
|
|
svc=s; cname=""
|
|
}
|
|
/^[[:space:]]*container_name:/ { cname=$2; gsub(/["\047]/,"",cname) }
|
|
index($0,key) { found=1 }
|
|
END { if (found) { if (cname != "") print cname; else print svc } }
|
|
' "$compose" 2>/dev/null | head -1 | tr -d '[:space:]')"
|
|
|
|
[ -n "$name" ] && printf '%s' "$name" || printf '%s' "$fallback"
|
|
}
|
|
|
|
# Generic health: running, not restarting, healthcheck (if any) reporting
|
|
# healthy, and STILL true after a settle period — a crash-loop looks perfect
|
|
# in the instant between restarts. Good enough for a single in-place update,
|
|
# never good enough to justify climbing another rung.
|
|
updaterVerifyGeneric() {
|
|
local app="$1" deadline="${3:-$(( $(date +%s) + 120 ))}"
|
|
local c; c="$(_updaterPrimaryContainer "$app")"
|
|
local stable_needed=3 stable=0
|
|
|
|
while [ "$(date +%s)" -lt "$deadline" ]; do
|
|
local state health restarts
|
|
state="$(dockerCommandRun "docker inspect --format '{{.State.Status}}' $c" 2>/dev/null | tr -d '\r')"
|
|
health="$(dockerCommandRun "docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' $c" 2>/dev/null | tr -d '\r')"
|
|
restarts="$(dockerCommandRun "docker inspect --format '{{.RestartCount}}' $c" 2>/dev/null | tr -d '\r')"
|
|
|
|
if [ "$state" = "running" ] && { [ "$health" = "healthy" ] || [ "$health" = "none" ]; }; then
|
|
stable=$((stable + 1))
|
|
[ -z "${_uv_restarts:-}" ] && _uv_restarts="$restarts"
|
|
# A restart during the settle window means it is looping, not up.
|
|
[ "$restarts" != "$_uv_restarts" ] && stable=0 && _uv_restarts="$restarts"
|
|
(( stable >= stable_needed )) && { unset _uv_restarts; return 0; }
|
|
else
|
|
stable=0
|
|
fi
|
|
sleep 5
|
|
done
|
|
unset _uv_restarts
|
|
return 1
|
|
}
|
|
|
|
# Does this app ship a real verifier? The stepped engine requires one.
|
|
#
|
|
# "Not currently defined" is NOT the same as "not shipped", and conflating the
|
|
# two is the most expensive mistake this file can make. The CLI runs lazy
|
|
# (LP_LAZY=1), where the container scan is skipped entirely and every function
|
|
# has to arrive through function_manifest.sh — a BUILD-time artifact. Any app
|
|
# that came into existence after that build has no stub, so its verifier reads
|
|
# as absent while sitting on disk two directories away. That is exactly every
|
|
# multi-instance clone, and the consequences both point the wrong way: GATE 1
|
|
# refuses an upgrade for an app that is fully verifiable, and the auto-ladder's
|
|
# `updaterHasVerifier || continue` drops it in silence, for good.
|
|
#
|
|
# So consult the disk before answering no, and source what is actually there —
|
|
# the same file the eager loader would have picked up. Cheap (one stat for the
|
|
# common case) and self-healing regardless of how stale the manifest is, which
|
|
# matters because a LibrePortal self-update overwrites the manifest with the
|
|
# shipped copy and drops every instance entry again.
|
|
updaterHasVerifier() {
|
|
local app="$1"
|
|
declare -F "${app}_upgrade_verify" >/dev/null 2>&1 && return 0
|
|
|
|
local hooks="${install_containers_dir%/}/$app/scripts/${app}_upgrade_hooks.sh"
|
|
[ -f "$hooks" ] || return 1
|
|
source "$hooks" 2>/dev/null || return 1
|
|
declare -F "${app}_upgrade_verify" >/dev/null 2>&1
|
|
}
|
|
|
|
# updaterVerifyUpgrade <app> <expected-tag> [timeout-secs]
|
|
# Dispatches to the app's verifier, falling back to the generic check. Returns
|
|
# 0 only on positive confirmation.
|
|
updaterVerifyUpgrade() {
|
|
local app="$1" expected="$2" timeout="${3:-600}"
|
|
local deadline=$(( $(date +%s) + timeout ))
|
|
|
|
if updaterHasVerifier "$app"; then
|
|
isNotice "Verifying $app is serving $expected (up to ${timeout}s)…"
|
|
if "${app}_upgrade_verify" "$app" "$expected" "$deadline"; then
|
|
isSuccessful "$app verified at $expected."
|
|
return 0
|
|
fi
|
|
isError "$app did NOT verify at $expected — treating as failed."
|
|
return 1
|
|
fi
|
|
|
|
isNotice "$app has no upgrade verifier; using the generic health check."
|
|
updaterVerifyGeneric "$app" "$expected" "$deadline"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Shared HTTP version verification.
|
|
#
|
|
# The strongest thing a verifier can say is "the app itself reports the version
|
|
# we asked for, and kept reporting it." Most apps expose that over HTTP, so the
|
|
# per-app hook becomes three facts — port, path, where the version lives — and
|
|
# the polling, extraction, comparison and stability rules live here once.
|
|
#
|
|
# Probed from the HOST against the container's PUBLISHED port, deliberately, not
|
|
# via `docker exec … curl`: half these images ship no curl at all (mattermost is
|
|
# one), so exec-based probes are a coin flip on the vendor's base image. The
|
|
# host has curl, and a published port is something LibrePortal already
|
|
# guarantees for every app with a web interface.
|
|
|
|
# First published host port for a container's internal port. Empty if unmapped.
|
|
_updaterPublishedPortFor() {
|
|
local c="$1" internal="$2"
|
|
dockerCommandRun "docker port $c $internal" 2>/dev/null \
|
|
| tr -d '\r' | grep -oE '[0-9]+$' | head -1
|
|
}
|
|
|
|
# Pull a version out of an HTTP response. "json:<key>" reads a top-level string
|
|
# field; "header:<Name>" reads a response header (case-insensitively).
|
|
_updaterExtractVersion() {
|
|
local resp="$1" spec="$2"
|
|
case "$spec" in
|
|
json:*)
|
|
local k="${spec#json:}"
|
|
printf '%s' "$resp" | tr -d '\r' \
|
|
| grep -oE "\"$k\"[[:space:]]*:[[:space:]]*\"[^\"]*\"" | head -1 \
|
|
| sed -E 's/.*:[[:space:]]*"([^"]*)"/\1/'
|
|
;;
|
|
header:*)
|
|
local h="${spec#header:}"
|
|
printf '%s' "$resp" | tr -d '\r' | grep -i "^${h}:" | head -1 \
|
|
| sed -E 's/^[^:]*:[[:space:]]*//'
|
|
;;
|
|
esac
|
|
}
|
|
|
|
# Do a tag and a self-reported version agree on every component BOTH of them
|
|
# state? Tags and running versions are rarely the same precision:
|
|
# v1.158.0 vs 1.158.0 -> yes (the v is noise)
|
|
# 11.9 vs 11.9.1 -> yes (the tag is a line; the build is more precise)
|
|
# 8.7.0 vs 8.7 -> yes (the app reports less precision than the tag)
|
|
# 11.9 vs 11.10 -> NO
|
|
# Comparing only the shared prefix is what makes one rule work for all of them;
|
|
# demanding string equality would fail every app above except Synapse.
|
|
_updaterVersionAgrees() {
|
|
local a b
|
|
a="$(printf '%s' "$1" | grep -oE '[0-9]+' | tr '\n' ' ')"
|
|
b="$(printf '%s' "$2" | grep -oE '[0-9]+' | tr '\n' ' ')"
|
|
local -a A=($a) B=($b)
|
|
local n=${#A[@]}; [ ${#B[@]} -lt "$n" ] && n=${#B[@]}
|
|
[ "$n" -gt 0 ] || return 1
|
|
local i
|
|
for ((i=0; i<n; i++)); do
|
|
[ "$((10#${A[i]}))" -eq "$((10#${B[i]}))" ] || return 1
|
|
done
|
|
return 0
|
|
}
|
|
|
|
# updaterVerifyHttpVersion <app> <expected-tag> <deadline> <internal-port> <path> <extract>
|
|
# 0 only when the app reports a version agreeing with the tag, three polls
|
|
# running. Three because one lucky answer during a rolling restart proves
|
|
# nothing — the old container can still be serving while the new one boots.
|
|
updaterVerifyHttpVersion() {
|
|
local app="$1" expected="$2" deadline="$3" iport="$4" path="$5" extract="$6"
|
|
local c; c="$(_updaterPrimaryContainer "$app")"
|
|
local port; port="$(_updaterPublishedPortFor "$c" "$iport")"
|
|
if [ -z "$port" ]; then
|
|
isError "$app: container $c publishes no host port for $iport — cannot verify."
|
|
return 1
|
|
fi
|
|
|
|
local stable=0 need=3 last="" resp ver
|
|
while [ "$(date +%s)" -lt "$deadline" ]; do
|
|
resp="$(curl -fsS -i --max-time 6 "http://127.0.0.1:${port}${path}" 2>/dev/null)"
|
|
ver="$(_updaterExtractVersion "$resp" "$extract")"
|
|
last="reported=${ver:-none}"
|
|
if [ -n "$ver" ] && _updaterVersionAgrees "$expected" "$ver"; then
|
|
stable=$((stable + 1))
|
|
if (( stable >= need )); then
|
|
isSuccessful "$app reports $ver, agreeing with $expected, and held it."
|
|
return 0
|
|
fi
|
|
else
|
|
stable=0
|
|
fi
|
|
sleep 5
|
|
done
|
|
isError "$app did not report $expected before the deadline.${last:+ Last probe: $last}"
|
|
return 1
|
|
}
|