diff --git a/containers/matrix/scripts/matrix_upgrade_hooks.sh b/containers/matrix/scripts/matrix_upgrade_hooks.sh new file mode 100644 index 0000000..90f5e86 --- /dev/null +++ b/containers/matrix/scripts/matrix_upgrade_hooks.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +# Matrix (Synapse) upgrade verification. +# --------------------------------------------------------------------------- +# Synapse runs database schema migrations on boot and does not serve until they +# finish, so "answering with the new version" is genuine evidence the rung +# landed rather than merely that a process started. +# +# /_synapse/admin/v1/server_version needs no authentication and returns the +# running version outright: {"server_version":"1.158.0"}. The tag carries a +# leading v (v1.158.0); the shared comparison comes down to numbers, so that +# difference does not matter. + +# matrix_upgrade_verify +matrix_upgrade_verify() { + updaterVerifyHttpVersion "$1" "$2" "$3" 8008 \ + "/_synapse/admin/v1/server_version" "json:server_version" +} diff --git a/containers/mattermost/scripts/mattermost_upgrade_hooks.sh b/containers/mattermost/scripts/mattermost_upgrade_hooks.sh new file mode 100644 index 0000000..2b1ba9b --- /dev/null +++ b/containers/mattermost/scripts/mattermost_upgrade_hooks.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +# Mattermost upgrade verification. +# --------------------------------------------------------------------------- +# Mattermost migrates its schema on boot and refuses to serve the API until it +# has finished, so a 200 from the ping endpoint carrying the new version is +# evidence the migration completed. +# +# /api/v4/system/ping is unauthenticated. The version is NOT in the body — it +# rides in the X-Version-Id header, as +# "11.9.1.311276....false": build version first, then metadata. The shared +# comparison reads leading numbers, so it matches the 11.9 tag against the +# 11.9.1 build without needing to know that layout. + +# mattermost_upgrade_verify +mattermost_upgrade_verify() { + updaterVerifyHttpVersion "$1" "$2" "$3" 8065 \ + "/api/v4/system/ping" "header:X-Version-Id" +} diff --git a/containers/rocketchat/scripts/rocketchat_upgrade_hooks.sh b/containers/rocketchat/scripts/rocketchat_upgrade_hooks.sh new file mode 100644 index 0000000..d6977a0 --- /dev/null +++ b/containers/rocketchat/scripts/rocketchat_upgrade_hooks.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +# Rocket.Chat upgrade verification. +# --------------------------------------------------------------------------- +# Rocket.Chat runs schema migrations on boot and refuses to start when the image +# is more than one major ahead of the database — the exact failure that once +# kept this app on manual updates. Confirming the version it actually serves is +# what makes stepping it safe. +# +# /api/info is unauthenticated and reports {"version":"8.7", …}. Note it reports +# a shorter version than the tag (8.7 for tag 8.7.0); the shared comparison only +# requires agreement on the components both sides state. + +# rocketchat_upgrade_verify +rocketchat_upgrade_verify() { + updaterVerifyHttpVersion "$1" "$2" "$3" 3000 \ + "/api/info" "json:version" +} diff --git a/scripts/cli/commands/updater/cli_updater_auto.sh b/scripts/cli/commands/updater/cli_updater_auto.sh index ef57814..de56613 100644 --- a/scripts/cli/commands/updater/cli_updater_auto.sh +++ b/scripts/cli/commands/updater/cli_updater_auto.sh @@ -226,6 +226,13 @@ updaterUpgradeAuto() dayf="$(_updaterAutoRungDay "$app")" [[ -f "$dayf" && "$(cat "$dayf" 2>/dev/null)" == "$today" ]] && continue + # No verifier, no automatic climb. GATE 1 in the engine would refuse + # this anyway, but refusing HERE means we never enqueue a task that + # exists only to fail: the user would get a failure notification every + # single day for an app that was never eligible. + declare -F updaterHasVerifier >/dev/null 2>&1 && \ + { updaterHasVerifier "$app" || continue; } + repo="$(updaterRepoTag "$image")"; repo="${repo%:*}" next="$(updaterNextRung "$channel" "$repo")" [[ -n "$next" ]] || continue diff --git a/scripts/cli/commands/updater/cli_updater_verify.sh b/scripts/cli/commands/updater/cli_updater_verify.sh index a7fdb5e..4c71dce 100644 --- a/scripts/cli/commands/updater/cli_updater_verify.sh +++ b/scripts/cli/commands/updater/cli_updater_verify.sh @@ -40,22 +40,20 @@ _updaterPrimaryContainer() { local up; up="$(printf '%s' "$app" | tr '[:lower:]' '[:upper:]')" # The service block that owns the bare _VERSION_TAG sentinel, then that - # block's container_name (compose's own answer for what the container is - # called). Service name is the fallback: compose defaults to it. + # block's container_name -- compose's own answer for what the container is + # called. \047 is an apostrophe: the program is single-quoted, so it cannot + # contain one literally. local name - name="$(awk -v key="#LIBREPORTAL|${up}_VERSION_TAG|" ''' + name="$(awk -v key="#LIBREPORTAL|${up}_VERSION_TAG|" ' /^[[:space:]]{2,4}[a-zA-Z0-9_-]+:[[:space:]]*(#|$)/ { + if (found) { if (cname != "") print cname; else print svc; exit } s=$1; sub(/:.*/,"",s); gsub(/[[:space:]]/,"",s) - if (found && cname=="") { print svc; exit } - if (found) exit svc=s; cname="" } + /^[[:space:]]*container_name:/ { cname=$2; gsub(/["\047]/,"",cname) } index($0,key) { found=1 } - found && /^[[:space:]]*container_name:/ { - cname=$2; gsub(/["']/,"",cname); print cname; exit - } - END { if (found && cname=="") print svc } - ''' "$compose" 2>/dev/null | head -1 | tr -d "[:space:]")" + END { if (found) { if (cname != "") print cname; else print svc } } + ' "$compose" 2>/dev/null | head -1 | tr -d '[:space:]')" [ -n "$name" ] && printf '%s' "$name" || printf '%s' "$fallback" } @@ -116,3 +114,98 @@ updaterVerifyUpgrade() { isNotice "$app has no upgrade verifier; using the generic health check." updaterVerifyGeneric "$app" "$expected" "$deadline" } + +# --------------------------------------------------------------------------- +# Shared HTTP version verification. +# +# The strongest thing a verifier can say is "the app itself reports the version +# we asked for, and kept reporting it." Most apps expose that over HTTP, so the +# per-app hook becomes three facts — port, path, where the version lives — and +# the polling, extraction, comparison and stability rules live here once. +# +# Probed from the HOST against the container's PUBLISHED port, deliberately, not +# via `docker exec … curl`: half these images ship no curl at all (mattermost is +# one), so exec-based probes are a coin flip on the vendor's base image. The +# host has curl, and a published port is something LibrePortal already +# guarantees for every app with a web interface. + +# First published host port for a container's internal port. Empty if unmapped. +_updaterPublishedPortFor() { + local c="$1" internal="$2" + dockerCommandRun "docker port $c $internal" 2>/dev/null \ + | tr -d '\r' | grep -oE '[0-9]+$' | head -1 +} + +# Pull a version out of an HTTP response. "json:" reads a top-level string +# field; "header:" reads a response header (case-insensitively). +_updaterExtractVersion() { + local resp="$1" spec="$2" + case "$spec" in + json:*) + local k="${spec#json:}" + printf '%s' "$resp" | tr -d '\r' \ + | grep -oE "\"$k\"[[:space:]]*:[[:space:]]*\"[^\"]*\"" | head -1 \ + | sed -E 's/.*:[[:space:]]*"([^"]*)"/\1/' + ;; + header:*) + local h="${spec#header:}" + printf '%s' "$resp" | tr -d '\r' | grep -i "^${h}:" | head -1 \ + | sed -E 's/^[^:]*:[[:space:]]*//' + ;; + esac +} + +# Do a tag and a self-reported version agree on every component BOTH of them +# state? Tags and running versions are rarely the same precision: +# v1.158.0 vs 1.158.0 -> yes (the v is noise) +# 11.9 vs 11.9.1 -> yes (the tag is a line; the build is more precise) +# 8.7.0 vs 8.7 -> yes (the app reports less precision than the tag) +# 11.9 vs 11.10 -> NO +# Comparing only the shared prefix is what makes one rule work for all of them; +# demanding string equality would fail every app above except Synapse. +_updaterVersionAgrees() { + local a b + a="$(printf '%s' "$1" | grep -oE '[0-9]+' | tr '\n' ' ')" + b="$(printf '%s' "$2" | grep -oE '[0-9]+' | tr '\n' ' ')" + local -a A=($a) B=($b) + local n=${#A[@]}; [ ${#B[@]} -lt "$n" ] && n=${#B[@]} + [ "$n" -gt 0 ] || return 1 + local i + for ((i=0; i +# 0 only when the app reports a version agreeing with the tag, three polls +# running. Three because one lucky answer during a rolling restart proves +# nothing — the old container can still be serving while the new one boots. +updaterVerifyHttpVersion() { + local app="$1" expected="$2" deadline="$3" iport="$4" path="$5" extract="$6" + local c; c="$(_updaterPrimaryContainer "$app")" + local port; port="$(_updaterPublishedPortFor "$c" "$iport")" + if [ -z "$port" ]; then + isError "$app: container $c publishes no host port for $iport — cannot verify." + return 1 + fi + + local stable=0 need=3 last="" resp ver + while [ "$(date +%s)" -lt "$deadline" ]; do + resp="$(curl -fsS -i --max-time 6 "http://127.0.0.1:${port}${path}" 2>/dev/null)" + ver="$(_updaterExtractVersion "$resp" "$extract")" + last="reported=${ver:-none}" + if [ -n "$ver" ] && _updaterVersionAgrees "$expected" "$ver"; then + stable=$((stable + 1)) + if (( stable >= need )); then + isSuccessful "$app reports $ver, agreeing with $expected, and held it." + return 0 + fi + else + stable=0 + fi + sleep 5 + done + isError "$app did not report $expected before the deadline.${last:+ Last probe: $last}" + return 1 +} diff --git a/scripts/source/files/arrays/function_manifest.sh b/scripts/source/files/arrays/function_manifest.sh index d08376d..698312e 100644 --- a/scripts/source/files/arrays/function_manifest.sh +++ b/scripts/source/files/arrays/function_manifest.sh @@ -714,8 +714,10 @@ declare -gA LP_FN_MAP=( [matrix_install_pre]="matrix/scripts/matrix_install_hooks.sh" [_matrixPublicBaseUrl]="matrix/scripts/matrix_install_hooks.sh" [_matrixServerName]="matrix/scripts/matrix_install_hooks.sh" + [matrix_upgrade_verify]="matrix/scripts/matrix_upgrade_hooks.sh" [_matrixUserId]="matrix/scripts/matrix_auth.sh" [mattermostToolsMenu]="menu/tools/manage_mattermost.sh" + [mattermost_upgrade_verify]="mattermost/scripts/mattermost_upgrade_hooks.sh" [maybeRegenPoll]="task/crontab_task_processor.sh" [menuContinue]="menu/message/continue.sh" [menuLoginRequired]="menu/message/login.sh" @@ -897,6 +899,7 @@ declare -gA LP_FN_MAP=( [_rocketchatLogin]="rocketchat/scripts/rocketchat_auth.sh" [_rocketchatOk]="rocketchat/scripts/rocketchat_auth.sh" [_rocketchatPassword]="rocketchat/scripts/rocketchat_auth.sh" + [rocketchat_upgrade_verify]="rocketchat/scripts/rocketchat_upgrade_hooks.sh" [_rocketchatUserId]="rocketchat/scripts/rocketchat_auth.sh" [_rocketchatUsernameOf]="rocketchat/scripts/rocketchat_auth.sh" [runAppCfg]="docker/command/run_privileged.sh" @@ -1039,6 +1042,7 @@ declare -gA LP_FN_MAP=( [updaterComposePull]="cli/commands/updater/cli_updater_commands.sh" [updaterCurrentTag]="cli/commands/updater/cli_updater_upgrade.sh" [updaterDisplayVersion]="webui/data/generators/updater/webui_updater_scan.sh" + [_updaterExtractVersion]="cli/commands/updater/cli_updater_verify.sh" [updaterHasVerifier]="cli/commands/updater/cli_updater_verify.sh" [updaterInspectLocal]="webui/data/generators/updater/webui_updater_scan.sh" [updaterInWindow]="cli/commands/updater/cli_updater_auto.sh" @@ -1050,6 +1054,7 @@ declare -gA LP_FN_MAP=( [updaterNextRung]="cli/commands/updater/cli_updater_ladder.sh" [_updaterPrimaryContainer]="cli/commands/updater/cli_updater_verify.sh" [updaterPrimaryImage]="webui/data/generators/updater/webui_updater_scan.sh" + [_updaterPublishedPortFor]="cli/commands/updater/cli_updater_verify.sh" [updaterRecordHistory]="cli/commands/updater/cli_updater_commands.sh" [updaterRefDigest]="cli/commands/updater/cli_updater_commands.sh" [updaterRegistryDigest]="webui/data/generators/updater/webui_updater_scan.sh" @@ -1073,7 +1078,9 @@ declare -gA LP_FN_MAP=( [_updaterUpgradePruneImages]="cli/commands/updater/cli_updater_upgrade.sh" [_updaterUpgradeRollbackStep]="cli/commands/updater/cli_updater_upgrade.sh" [updaterVerifyGeneric]="cli/commands/updater/cli_updater_verify.sh" + [updaterVerifyHttpVersion]="cli/commands/updater/cli_updater_verify.sh" [updaterVerifyUpgrade]="cli/commands/updater/cli_updater_verify.sh" + [_updaterVersionAgrees]="cli/commands/updater/cli_updater_verify.sh" [updaterVersionLadder]="cli/commands/updater/cli_updater_ladder.sh" [updateTaskFields]="task/crontab_task_processor.sh" [_upReportComposeFailure]="docker/app/compose/up_app.sh" @@ -1870,8 +1877,10 @@ declare -gA LP_FN_ROOT=( [matrix_install_pre]="containers" [_matrixPublicBaseUrl]="containers" [_matrixServerName]="containers" + [matrix_upgrade_verify]="containers" [_matrixUserId]="containers" [mattermostToolsMenu]="scripts" + [mattermost_upgrade_verify]="containers" [maybeRegenPoll]="scripts" [menuContinue]="scripts" [menuLoginRequired]="scripts" @@ -2053,6 +2062,7 @@ declare -gA LP_FN_ROOT=( [_rocketchatLogin]="containers" [_rocketchatOk]="containers" [_rocketchatPassword]="containers" + [rocketchat_upgrade_verify]="containers" [_rocketchatUserId]="containers" [_rocketchatUsernameOf]="containers" [runAppCfg]="scripts" @@ -2195,6 +2205,7 @@ declare -gA LP_FN_ROOT=( [updaterComposePull]="scripts" [updaterCurrentTag]="scripts" [updaterDisplayVersion]="scripts" + [_updaterExtractVersion]="scripts" [updaterHasVerifier]="scripts" [updaterInspectLocal]="scripts" [updaterInWindow]="scripts" @@ -2206,6 +2217,7 @@ declare -gA LP_FN_ROOT=( [updaterNextRung]="scripts" [_updaterPrimaryContainer]="scripts" [updaterPrimaryImage]="scripts" + [_updaterPublishedPortFor]="scripts" [updaterRecordHistory]="scripts" [updaterRefDigest]="scripts" [updaterRegistryDigest]="scripts" @@ -2229,7 +2241,9 @@ declare -gA LP_FN_ROOT=( [_updaterUpgradePruneImages]="scripts" [_updaterUpgradeRollbackStep]="scripts" [updaterVerifyGeneric]="scripts" + [updaterVerifyHttpVersion]="scripts" [updaterVerifyUpgrade]="scripts" + [_updaterVersionAgrees]="scripts" [updaterVersionLadder]="scripts" [updateTaskFields]="scripts" [_upReportComposeFailure]="scripts" @@ -3062,8 +3076,10 @@ matrix_install_post_start() { unset -f matrix_install_post_start; __lpAutoload " matrix_install_pre() { unset -f matrix_install_pre; __lpAutoload "${install_containers_dir}matrix/scripts/matrix_install_hooks.sh"; matrix_install_pre "$@"; } _matrixPublicBaseUrl() { unset -f _matrixPublicBaseUrl; __lpAutoload "${install_containers_dir}matrix/scripts/matrix_install_hooks.sh"; _matrixPublicBaseUrl "$@"; } _matrixServerName() { unset -f _matrixServerName; __lpAutoload "${install_containers_dir}matrix/scripts/matrix_install_hooks.sh"; _matrixServerName "$@"; } +matrix_upgrade_verify() { unset -f matrix_upgrade_verify; __lpAutoload "${install_containers_dir}matrix/scripts/matrix_upgrade_hooks.sh"; matrix_upgrade_verify "$@"; } _matrixUserId() { unset -f _matrixUserId; __lpAutoload "${install_containers_dir}matrix/scripts/matrix_auth.sh"; _matrixUserId "$@"; } mattermostToolsMenu() { unset -f mattermostToolsMenu; __lpAutoload "${install_scripts_dir}menu/tools/manage_mattermost.sh"; mattermostToolsMenu "$@"; } +mattermost_upgrade_verify() { unset -f mattermost_upgrade_verify; __lpAutoload "${install_containers_dir}mattermost/scripts/mattermost_upgrade_hooks.sh"; mattermost_upgrade_verify "$@"; } maybeRegenPoll() { unset -f maybeRegenPoll; __lpAutoload "${install_scripts_dir}task/crontab_task_processor.sh"; maybeRegenPoll "$@"; } menuContinue() { unset -f menuContinue; __lpAutoload "${install_scripts_dir}menu/message/continue.sh"; menuContinue "$@"; } menuLoginRequired() { unset -f menuLoginRequired; __lpAutoload "${install_scripts_dir}menu/message/login.sh"; menuLoginRequired "$@"; } @@ -3245,6 +3261,7 @@ rocketchat_install_post_start() { unset -f rocketchat_install_post_start; __lpAu _rocketchatLogin() { unset -f _rocketchatLogin; __lpAutoload "${install_containers_dir}rocketchat/scripts/rocketchat_auth.sh"; _rocketchatLogin "$@"; } _rocketchatOk() { unset -f _rocketchatOk; __lpAutoload "${install_containers_dir}rocketchat/scripts/rocketchat_auth.sh"; _rocketchatOk "$@"; } _rocketchatPassword() { unset -f _rocketchatPassword; __lpAutoload "${install_containers_dir}rocketchat/scripts/rocketchat_auth.sh"; _rocketchatPassword "$@"; } +rocketchat_upgrade_verify() { unset -f rocketchat_upgrade_verify; __lpAutoload "${install_containers_dir}rocketchat/scripts/rocketchat_upgrade_hooks.sh"; rocketchat_upgrade_verify "$@"; } _rocketchatUserId() { unset -f _rocketchatUserId; __lpAutoload "${install_containers_dir}rocketchat/scripts/rocketchat_auth.sh"; _rocketchatUserId "$@"; } _rocketchatUsernameOf() { unset -f _rocketchatUsernameOf; __lpAutoload "${install_containers_dir}rocketchat/scripts/rocketchat_auth.sh"; _rocketchatUsernameOf "$@"; } runAppCfg() { unset -f runAppCfg; __lpAutoload "${install_scripts_dir}docker/command/run_privileged.sh"; runAppCfg "$@"; } @@ -3387,6 +3404,7 @@ _updaterCleanImageRef() { unset -f _updaterCleanImageRef; __lpAutoload "${instal updaterComposePull() { unset -f updaterComposePull; __lpAutoload "${install_scripts_dir}cli/commands/updater/cli_updater_commands.sh"; updaterComposePull "$@"; } updaterCurrentTag() { unset -f updaterCurrentTag; __lpAutoload "${install_scripts_dir}cli/commands/updater/cli_updater_upgrade.sh"; updaterCurrentTag "$@"; } updaterDisplayVersion() { unset -f updaterDisplayVersion; __lpAutoload "${install_scripts_dir}webui/data/generators/updater/webui_updater_scan.sh"; updaterDisplayVersion "$@"; } +_updaterExtractVersion() { unset -f _updaterExtractVersion; __lpAutoload "${install_scripts_dir}cli/commands/updater/cli_updater_verify.sh"; _updaterExtractVersion "$@"; } updaterHasVerifier() { unset -f updaterHasVerifier; __lpAutoload "${install_scripts_dir}cli/commands/updater/cli_updater_verify.sh"; updaterHasVerifier "$@"; } updaterInspectLocal() { unset -f updaterInspectLocal; __lpAutoload "${install_scripts_dir}webui/data/generators/updater/webui_updater_scan.sh"; updaterInspectLocal "$@"; } updaterInWindow() { unset -f updaterInWindow; __lpAutoload "${install_scripts_dir}cli/commands/updater/cli_updater_auto.sh"; updaterInWindow "$@"; } @@ -3398,6 +3416,7 @@ updaterNewerVersionTag() { unset -f updaterNewerVersionTag; __lpAutoload "${inst updaterNextRung() { unset -f updaterNextRung; __lpAutoload "${install_scripts_dir}cli/commands/updater/cli_updater_ladder.sh"; updaterNextRung "$@"; } _updaterPrimaryContainer() { unset -f _updaterPrimaryContainer; __lpAutoload "${install_scripts_dir}cli/commands/updater/cli_updater_verify.sh"; _updaterPrimaryContainer "$@"; } updaterPrimaryImage() { unset -f updaterPrimaryImage; __lpAutoload "${install_scripts_dir}webui/data/generators/updater/webui_updater_scan.sh"; updaterPrimaryImage "$@"; } +_updaterPublishedPortFor() { unset -f _updaterPublishedPortFor; __lpAutoload "${install_scripts_dir}cli/commands/updater/cli_updater_verify.sh"; _updaterPublishedPortFor "$@"; } updaterRecordHistory() { unset -f updaterRecordHistory; __lpAutoload "${install_scripts_dir}cli/commands/updater/cli_updater_commands.sh"; updaterRecordHistory "$@"; } updaterRefDigest() { unset -f updaterRefDigest; __lpAutoload "${install_scripts_dir}cli/commands/updater/cli_updater_commands.sh"; updaterRefDigest "$@"; } updaterRegistryDigest() { unset -f updaterRegistryDigest; __lpAutoload "${install_scripts_dir}webui/data/generators/updater/webui_updater_scan.sh"; updaterRegistryDigest "$@"; } @@ -3421,7 +3440,9 @@ _updaterUpgradeGenDir() { unset -f _updaterUpgradeGenDir; __lpAutoload "${instal _updaterUpgradePruneImages() { unset -f _updaterUpgradePruneImages; __lpAutoload "${install_scripts_dir}cli/commands/updater/cli_updater_upgrade.sh"; _updaterUpgradePruneImages "$@"; } _updaterUpgradeRollbackStep() { unset -f _updaterUpgradeRollbackStep; __lpAutoload "${install_scripts_dir}cli/commands/updater/cli_updater_upgrade.sh"; _updaterUpgradeRollbackStep "$@"; } updaterVerifyGeneric() { unset -f updaterVerifyGeneric; __lpAutoload "${install_scripts_dir}cli/commands/updater/cli_updater_verify.sh"; updaterVerifyGeneric "$@"; } +updaterVerifyHttpVersion() { unset -f updaterVerifyHttpVersion; __lpAutoload "${install_scripts_dir}cli/commands/updater/cli_updater_verify.sh"; updaterVerifyHttpVersion "$@"; } updaterVerifyUpgrade() { unset -f updaterVerifyUpgrade; __lpAutoload "${install_scripts_dir}cli/commands/updater/cli_updater_verify.sh"; updaterVerifyUpgrade "$@"; } +_updaterVersionAgrees() { unset -f _updaterVersionAgrees; __lpAutoload "${install_scripts_dir}cli/commands/updater/cli_updater_verify.sh"; _updaterVersionAgrees "$@"; } updaterVersionLadder() { unset -f updaterVersionLadder; __lpAutoload "${install_scripts_dir}cli/commands/updater/cli_updater_ladder.sh"; updaterVersionLadder "$@"; } updateTaskFields() { unset -f updateTaskFields; __lpAutoload "${install_scripts_dir}task/crontab_task_processor.sh"; updateTaskFields "$@"; } _upReportComposeFailure() { unset -f _upReportComposeFailure; __lpAutoload "${install_scripts_dir}docker/app/compose/up_app.sh"; _upReportComposeFailure "$@"; }