diff --git a/containers/stalwart/scripts/stalwart_upgrade_hooks.sh b/containers/stalwart/scripts/stalwart_upgrade_hooks.sh index f1d490e..e534472 100644 --- a/containers/stalwart/scripts/stalwart_upgrade_hooks.sh +++ b/containers/stalwart/scripts/stalwart_upgrade_hooks.sh @@ -18,13 +18,21 @@ # the ladder is only ever as strong as the check underneath it, and pretending # otherwise is how a half-migrated app advances a rung. # -# Readiness is also necessary but NOT sufficient. Since v0.16 the WebUI is not -# in the Docker image: the server downloads the admin console from GitHub, and -# a fresh image after an upgrade means fetching it again. With no outbound -# HTTPS at that moment the fetch fails, /healthz/ready still answers 200 (the -# mail server genuinely is serving) and the admin is handed a "successful" -# upgrade whose control panel 404s. So readiness is the gate, and the admin -# console is checked behind it. +# Readiness is still the only thing that GATES the upgrade. Behind it sits a +# second, advisory check. Since v0.16 the WebUI is not in the Docker image: the +# server downloads the admin console from GitHub, and a fresh image after an +# upgrade means fetching it again. With no outbound HTTPS at that moment the +# fetch fails, /healthz/ready still answers 200 (the mail server genuinely is +# serving) and the admin is handed a "successful" upgrade whose control panel +# 404s with nothing to explain why. +# +# That is reported loudly but is NOT failed on, deliberately. The engine's +# response to a failed verify is to abort and restore, and restoring cannot put +# back a bundle that was never downloaded — it would roll a perfectly good mail +# server back a version to fix a missing web page, then hit the same empty +# GitHub fetch on the next attempt. The upgrade did land; a piece of it needs a +# network path the box did not have. Saying exactly that beats both a silent +# pass and a pointless rollback. # stalwart_upgrade_admin_ui_code # HTTP status of the admin console, empty if the container did not answer. @@ -34,8 +42,8 @@ stalwart_upgrade_admin_ui_code() { } # stalwart_upgrade_verify -# 0 only when /healthz/ready answers 200, keeps answering it, and the admin -# console is actually being served. +# 0 only when /healthz/ready answers 200 and keeps answering it. The admin +# console is reported on afterwards but cannot change the result. stalwart_upgrade_verify() { local app="$1" expected="$2" deadline="$3" local stable=0 stable_needed=3 last="" @@ -51,9 +59,9 @@ stalwart_upgrade_verify() { # Ready must HOLD: a server that flaps ready/not-ready is mid-restart, # and one lucky 200 is not evidence the upgrade settled. if (( stable >= stable_needed )); then - isNotice "Stalwart is ready (readiness probe stable) after moving to $expected." - stalwart_upgrade_verify_admin_ui "$expected" "$deadline" - return $? + isSuccessful "Stalwart is ready (readiness probe stable) after moving to $expected." + stalwart_upgrade_check_admin_ui "$expected" "$deadline" + return 0 fi else stable=0 @@ -65,41 +73,54 @@ stalwart_upgrade_verify() { return 1 } -# stalwart_upgrade_verify_admin_ui -# 0 only when /admin is served. Polled rather than probed once, because the -# WebUI fetch runs behind the server coming up: readiness can go stable while -# the bundle is still being pulled, and failing on that first 404 would abort -# an upgrade that was only a few seconds from finishing. -stalwart_upgrade_verify_admin_ui() { +# stalwart_upgrade_check_admin_ui +# Reports whether /admin is being served. ALWAYS returns 0 — named "check" and +# not "verify" so nobody wires it into the gate later by mistake; see the note +# at the top of this file for why a missing console must not trigger a restore. +# +# Polled rather than probed once, because the WebUI fetch runs behind the server +# coming up: readiness can go stable while the bundle is still being pulled, and +# a single early probe would cry 404 at an install that was seconds from fine. +# The window is short and capped by the caller's deadline — the upgrade result +# is already decided by this point, so there is nothing to be gained by holding +# the run open for the full remaining timeout waiting on a web asset. +stalwart_upgrade_check_admin_ui() { local expected="$1" deadline="$2" + local grace=$(( $(date +%s) + 60 )) local code="" - while [ "$(date +%s)" -lt "$deadline" ]; do + (( grace > deadline )) && grace="$deadline" + + while [ "$(date +%s)" -lt "$grace" ]; do code="$(stalwart_upgrade_admin_ui_code)" if [ -n "$code" ] && [ "$code" != "404" ]; then - isSuccessful "Stalwart is ready and serving its admin console after moving to $expected." + isSuccessful "Stalwart's admin console is being served (/admin=$code)." return 0 fi sleep 5 done - # The mail server is up — this is the console alone, so say so. An admin who - # reads this as data loss will roll back and re-hit the same missing bundle. if [ -z "$code" ]; then # No status at all is a different fault from a 404: the probe never # reached the container. Named separately so nobody goes hunting a # firewall rule for what is actually an exec/curl problem. - isError "Stalwart reported ready at $expected, but its admin console could not be probed." - isNotice " No response from 'docker exec stalwart-service curl … /admin' — check the" - isNotice " container is running and that curl exists inside the image." - return 1 + isError "The admin console could not be probed — check it by hand before relying on it." + isNotice " No response from 'docker exec stalwart-service curl … /admin'. The upgrade" + isNotice " to $expected itself is fine: the readiness probe above passed." + return 0 fi - isError "Stalwart is serving mail at $expected, but its admin console is missing (/admin=$code)." - isNotice " The WebUI is not bundled in the image: Stalwart downloads it from" - isNotice " https://github.com/stalwartlabs/webui/releases/latest when it starts on" - isNotice " a new image. That download did not complete, so /admin and /account will" - isNotice " 404 until it does. Allow outbound HTTPS to github.com from this host and" - isNotice " restart the container, then re-run the upgrade." - return 1 + # Loud, because a 404 admin panel is exactly the kind of thing that gets + # discovered weeks later — but explicitly NOT a failure, so the wording has + # to carry that or the next person reads it as a broken upgrade. + isError "The admin console is MISSING (/admin=$code) — the upgrade to $expected still succeeded." + isNotice " Stalwart does not bundle the WebUI: it downloads the console from" + isNotice " https://github.com/stalwartlabs/webui/releases/latest when it starts on a new" + isNotice " image. That download did not complete, so /admin and /account will 404 until" + isNotice " it does. Mail delivery is unaffected — only the web interface." + isNotice " Fix: allow outbound HTTPS to github.com from this host, then restart the" + isNotice " container: docker restart stalwart-service" + isNotice " Do NOT roll back — the previous version fetches the same bundle from the" + isNotice " same place and will land in the same state." + return 0 } diff --git a/scripts/source/files/arrays/function_manifest.sh b/scripts/source/files/arrays/function_manifest.sh index dab7cd8..c0ca018 100644 --- a/scripts/source/files/arrays/function_manifest.sh +++ b/scripts/source/files/arrays/function_manifest.sh @@ -865,8 +865,8 @@ declare -gA LP_FN_MAP=( [stalwart_install_message_data]="stalwart/scripts/stalwart_install_hooks.sh" [stalwart_install_post_start]="stalwart/scripts/stalwart_install_hooks.sh" [stalwart_upgrade_admin_ui_code]="stalwart/scripts/stalwart_upgrade_hooks.sh" + [stalwart_upgrade_check_admin_ui]="stalwart/scripts/stalwart_upgrade_hooks.sh" [stalwart_upgrade_verify]="stalwart/scripts/stalwart_upgrade_hooks.sh" - [stalwart_upgrade_verify_admin_ui]="stalwart/scripts/stalwart_upgrade_hooks.sh" [startInstall]="start/start_install.sh" [startLoad]="start/start_load.sh" [startOther]="start/start_other.sh" @@ -1903,8 +1903,8 @@ declare -gA LP_FN_ROOT=( [stalwart_install_message_data]="containers" [stalwart_install_post_start]="containers" [stalwart_upgrade_admin_ui_code]="containers" + [stalwart_upgrade_check_admin_ui]="containers" [stalwart_upgrade_verify]="containers" - [stalwart_upgrade_verify_admin_ui]="containers" [startInstall]="scripts" [startLoad]="scripts" [startOther]="scripts" @@ -2975,8 +2975,8 @@ sshRemote() { unset -f sshRemote; __lpAutoload "${install_scripts_dir}network/ss stalwart_install_message_data() { unset -f stalwart_install_message_data; __lpAutoload "${install_containers_dir}stalwart/scripts/stalwart_install_hooks.sh"; stalwart_install_message_data "$@"; } stalwart_install_post_start() { unset -f stalwart_install_post_start; __lpAutoload "${install_containers_dir}stalwart/scripts/stalwart_install_hooks.sh"; stalwart_install_post_start "$@"; } stalwart_upgrade_admin_ui_code() { unset -f stalwart_upgrade_admin_ui_code; __lpAutoload "${install_containers_dir}stalwart/scripts/stalwart_upgrade_hooks.sh"; stalwart_upgrade_admin_ui_code "$@"; } +stalwart_upgrade_check_admin_ui() { unset -f stalwart_upgrade_check_admin_ui; __lpAutoload "${install_containers_dir}stalwart/scripts/stalwart_upgrade_hooks.sh"; stalwart_upgrade_check_admin_ui "$@"; } stalwart_upgrade_verify() { unset -f stalwart_upgrade_verify; __lpAutoload "${install_containers_dir}stalwart/scripts/stalwart_upgrade_hooks.sh"; stalwart_upgrade_verify "$@"; } -stalwart_upgrade_verify_admin_ui() { unset -f stalwart_upgrade_verify_admin_ui; __lpAutoload "${install_containers_dir}stalwart/scripts/stalwart_upgrade_hooks.sh"; stalwart_upgrade_verify_admin_ui "$@"; } startInstall() { unset -f startInstall; __lpAutoload "${install_scripts_dir}start/start_install.sh"; startInstall "$@"; } startLoad() { unset -f startLoad; __lpAutoload "${install_scripts_dir}start/start_load.sh"; startLoad "$@"; } startOther() { unset -f startOther; __lpAutoload "${install_scripts_dir}start/start_other.sh"; startOther "$@"; }