stalwart: check the admin console, not just readiness

Stalwart v0.16 does not ship the WebUI in its Docker image — the admin
console is fetched from GitHub on first start. With no outbound HTTPS at
that moment the fetch fails silently: /healthz/ready still answers 200
because the mail server genuinely is serving, so both the installer and
the upgrade verifier reported success while /admin and /account 404'd
with nothing to explain why.

Install hook now probes /admin after the port-25 and PTR checks and, on
404, names the GitHub download as the cause rather than emitting a
generic failure. Upgrade verifier treats stable readiness as necessary
but not sufficient and confirms /admin before returning 0; the console
is polled under the same deadline because the bundle download runs
behind the server coming up, and failing on the first 404 would abort an
upgrade that was seconds from finishing.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

containers/stalwart/scripts/stalwart_install_hooks.sh

@@ -62,7 +62,32 @@ stalwart_install_post_start()
         fi
     fi
 
-    # ---- 3. The records the admin must add themselves --------------------
+    # ---- 3. Is the admin console actually there? -------------------------
+    # Stalwart v0.16 does not ship the WebUI inside the Docker image: the admin
+    # console is a single-page app the server fetches from GitHub on first
+    # start. If this host had no outbound HTTPS at that moment the download
+    # silently fails, the server still comes up healthy, and /admin and
+    # /account answer 404 forever with nothing to explain why. Checking it here
+    # turns "the panel is broken" into a one-line, fixable cause.
+    isNotice "Checking the admin console (WebUI) responds…"
+    local admin_code
+    admin_code=$(runFileOp docker exec stalwart-service curl -fsS -o /dev/null -w '%{http_code}' \
+        --max-time 5 http://localhost:8080/admin 2>/dev/null | tr -d '\r')
+    if [[ "$admin_code" == "404" ]]; then
+        isError "The admin console is missing (/admin returns 404)."
+        isNotice "  Stalwart does not bundle the WebUI — it downloads it from"
+        isNotice "  https://github.com/stalwartlabs/webui/releases/latest on first start."
+        isNotice "  That download failed, so /admin and /account will 404 until it succeeds."
+        isNotice "  Allow outbound HTTPS to github.com from this host, then restart the"
+        isNotice "  container: docker restart stalwart-service"
+        isNotice "  The mail server itself is unaffected — only the web interface is."
+    elif [[ -z "$admin_code" ]]; then
+        isError "Could not probe the admin console (no response from the container)."
+    else
+        isSuccessful "Admin console is being served (HTTP $admin_code)."
+    fi
+
+    # ---- 4. The records the admin must add themselves --------------------
     # Printed with real values so they can be pasted at the registrar. DKIM is
     # deliberately NOT guessed here: Stalwart generates the keypair on first
     # run, and the public key must be copied from its admin UI.
@@ -78,7 +103,7 @@ stalwart_install_post_start()
     echo "                                    generated the key; it is not known until then)"
     echo ""
 
-    # ---- 4. Where to go next ---------------------------------------------
+    # ---- 5. Where to go next ---------------------------------------------
     if [[ -n "$admin_port" ]]; then
         isNotice "Finish setup in the admin interface:"
         [[ -n "$public_ip_v4" ]] && echo "    http://$public_ip_v4:$admin_port/"

containers/stalwart/scripts/stalwart_upgrade_hooks.sh

@@ -17,9 +17,25 @@
 # rather than "running exactly $expected". Stated plainly instead of implied —
 # the ladder is only ever as strong as the check underneath it, and pretending
 # otherwise is how a half-migrated app advances a rung.
+#
+# Readiness is also necessary but NOT sufficient. Since v0.16 the WebUI is not
+# in the Docker image: the server downloads the admin console from GitHub, and
+# a fresh image after an upgrade means fetching it again. With no outbound
+# HTTPS at that moment the fetch fails, /healthz/ready still answers 200 (the
+# mail server genuinely is serving) and the admin is handed a "successful"
+# upgrade whose control panel 404s. So readiness is the gate, and the admin
+# console is checked behind it.
+
+# stalwart_upgrade_admin_ui_code
+# HTTP status of the admin console, empty if the container did not answer.
+stalwart_upgrade_admin_ui_code() {
+    runFileOp docker exec stalwart-service curl -fsS -o /dev/null -w '%{http_code}' \
+        --max-time 5 http://localhost:8080/admin 2>/dev/null | tr -d '\r'
+}
 
 # stalwart_upgrade_verify <app> <expected-tag> <deadline-epoch>
-# 0 only when /healthz/ready answers 200 and keeps answering it.
+# 0 only when /healthz/ready answers 200, keeps answering it, and the admin
+# console is actually being served.
 stalwart_upgrade_verify() {
     local app="$1" expected="$2" deadline="$3"
     local stable=0 stable_needed=3 last=""
@@ -35,8 +51,9 @@ stalwart_upgrade_verify() {
             # Ready must HOLD: a server that flaps ready/not-ready is mid-restart,
             # and one lucky 200 is not evidence the upgrade settled.
             if (( stable >= stable_needed )); then
-                isSuccessful "Stalwart is ready (readiness probe stable) after moving to $expected."
+                isNotice "Stalwart is ready (readiness probe stable) after moving to $expected."
-                return 0
+                stalwart_upgrade_verify_admin_ui "$expected" "$deadline"
+                return $?
             fi
         else
             stable=0
@@ -47,3 +64,42 @@ stalwart_upgrade_verify() {
     isError "Stalwart did not report ready for $expected before the deadline.${last:+ Last probe: $last}"
     return 1
 }
+
+# stalwart_upgrade_verify_admin_ui <expected-tag> <deadline-epoch>
+# 0 only when /admin is served. Polled rather than probed once, because the
+# WebUI fetch runs behind the server coming up: readiness can go stable while
+# the bundle is still being pulled, and failing on that first 404 would abort
+# an upgrade that was only a few seconds from finishing.
+stalwart_upgrade_verify_admin_ui() {
+    local expected="$1" deadline="$2"
+    local code=""
+
+    while [ "$(date +%s)" -lt "$deadline" ]; do
+        code="$(stalwart_upgrade_admin_ui_code)"
+        if [ -n "$code" ] && [ "$code" != "404" ]; then
+            isSuccessful "Stalwart is ready and serving its admin console after moving to $expected."
+            return 0
+        fi
+        sleep 5
+    done
+
+    # The mail server is up — this is the console alone, so say so. An admin who
+    # reads this as data loss will roll back and re-hit the same missing bundle.
+    if [ -z "$code" ]; then
+        # No status at all is a different fault from a 404: the probe never
+        # reached the container. Named separately so nobody goes hunting a
+        # firewall rule for what is actually an exec/curl problem.
+        isError "Stalwart reported ready at $expected, but its admin console could not be probed."
+        isNotice "  No response from 'docker exec stalwart-service curl … /admin' — check the"
+        isNotice "  container is running and that curl exists inside the image."
+        return 1
+    fi
+
+    isError "Stalwart is serving mail at $expected, but its admin console is missing (/admin=$code)."
+    isNotice "  The WebUI is not bundled in the image: Stalwart downloads it from"
+    isNotice "  https://github.com/stalwartlabs/webui/releases/latest when it starts on"
+    isNotice "  a new image. That download did not complete, so /admin and /account will"
+    isNotice "  404 until it does. Allow outbound HTTPS to github.com from this host and"
+    isNotice "  restart the container, then re-run the upgrade."
+    return 1
+}

scripts/source/files/arrays/function_manifest.sh

@@ -864,7 +864,9 @@ declare -gA LP_FN_MAP=(
     [sshRemote]="network/ssh/ssh.sh"
     [stalwart_install_message_data]="stalwart/scripts/stalwart_install_hooks.sh"
     [stalwart_install_post_start]="stalwart/scripts/stalwart_install_hooks.sh"
+    [stalwart_upgrade_admin_ui_code]="stalwart/scripts/stalwart_upgrade_hooks.sh"
     [stalwart_upgrade_verify]="stalwart/scripts/stalwart_upgrade_hooks.sh"
+    [stalwart_upgrade_verify_admin_ui]="stalwart/scripts/stalwart_upgrade_hooks.sh"
     [startInstall]="start/start_install.sh"
     [startLoad]="start/start_load.sh"
     [startOther]="start/start_other.sh"
@@ -1900,7 +1902,9 @@ declare -gA LP_FN_ROOT=(
     [sshRemote]="scripts"
     [stalwart_install_message_data]="containers"
     [stalwart_install_post_start]="containers"
+    [stalwart_upgrade_admin_ui_code]="containers"
     [stalwart_upgrade_verify]="containers"
+    [stalwart_upgrade_verify_admin_ui]="containers"
     [startInstall]="scripts"
     [startLoad]="scripts"
     [startOther]="scripts"
@@ -2970,7 +2974,9 @@ sourceBackupLocations() { unset -f sourceBackupLocations; __lpAutoload "${instal
 sshRemote() { unset -f sshRemote; __lpAutoload "${install_scripts_dir}network/ssh/ssh.sh"; sshRemote "$@"; }
 stalwart_install_message_data() { unset -f stalwart_install_message_data; __lpAutoload "${install_containers_dir}stalwart/scripts/stalwart_install_hooks.sh"; stalwart_install_message_data "$@"; }
 stalwart_install_post_start() { unset -f stalwart_install_post_start; __lpAutoload "${install_containers_dir}stalwart/scripts/stalwart_install_hooks.sh"; stalwart_install_post_start "$@"; }
+stalwart_upgrade_admin_ui_code() { unset -f stalwart_upgrade_admin_ui_code; __lpAutoload "${install_containers_dir}stalwart/scripts/stalwart_upgrade_hooks.sh"; stalwart_upgrade_admin_ui_code "$@"; }
 stalwart_upgrade_verify() { unset -f stalwart_upgrade_verify; __lpAutoload "${install_containers_dir}stalwart/scripts/stalwart_upgrade_hooks.sh"; stalwart_upgrade_verify "$@"; }
+stalwart_upgrade_verify_admin_ui() { unset -f stalwart_upgrade_verify_admin_ui; __lpAutoload "${install_containers_dir}stalwart/scripts/stalwart_upgrade_hooks.sh"; stalwart_upgrade_verify_admin_ui "$@"; }
 startInstall() { unset -f startInstall; __lpAutoload "${install_scripts_dir}start/start_install.sh"; startInstall "$@"; }
 startLoad() { unset -f startLoad; __lpAutoload "${install_scripts_dir}start/start_load.sh"; startLoad "$@"; }
 startOther() { unset -f startOther; __lpAutoload "${install_scripts_dir}start/start_other.sh"; startOther "$@"; }