#!/bin/bash # Stalwart upgrade verifier. # --------------------------------------------------------------------------- # Stalwart publishes Kubernetes-style probes on its admin port (8080): # GET /healthz/live — the process is alive and not deadlocked # GET /healthz/ready — dependencies initialised, config loaded, accepting traffic # https://stalw.art/docs/http/overview/ # # Readiness is the one that matters after a version move: it only answers 200 # once the storage backend is open and the config has loaded, which is exactly # the window where a schema change would otherwise go unnoticed. Liveness alone # would pass on a process that is up but unable to serve. # # Deliberately weaker than the Nextcloud verifier: Stalwart's probes confirm the # server is serving, but do not report a version, so this asserts readiness # rather than "running exactly $expected". Stated plainly instead of implied — # the ladder is only ever as strong as the check underneath it, and pretending # otherwise is how a half-migrated app advances a rung. # # Readiness is also necessary but NOT sufficient. Since v0.16 the WebUI is not # in the Docker image: the server downloads the admin console from GitHub, and # a fresh image after an upgrade means fetching it again. With no outbound # HTTPS at that moment the fetch fails, /healthz/ready still answers 200 (the # mail server genuinely is serving) and the admin is handed a "successful" # upgrade whose control panel 404s. So readiness is the gate, and the admin # console is checked behind it. # stalwart_upgrade_admin_ui_code # HTTP status of the admin console, empty if the container did not answer. stalwart_upgrade_admin_ui_code() { runFileOp docker exec stalwart-service curl -fsS -o /dev/null -w '%{http_code}' \ --max-time 5 http://localhost:8080/admin 2>/dev/null | tr -d '\r' } # stalwart_upgrade_verify # 0 only when /healthz/ready answers 200, keeps answering it, and the admin # console is actually being served. stalwart_upgrade_verify() { local app="$1" expected="$2" deadline="$3" local stable=0 stable_needed=3 last="" while [ "$(date +%s)" -lt "$deadline" ]; do local code code="$(runFileOp docker exec stalwart-service curl -fsS -o /dev/null -w '%{http_code}' \ --max-time 5 http://localhost:8080/healthz/ready 2>/dev/null | tr -d '\r')" last="healthz/ready=${code:-none}" if [ "$code" = "200" ]; then stable=$((stable + 1)) # Ready must HOLD: a server that flaps ready/not-ready is mid-restart, # and one lucky 200 is not evidence the upgrade settled. if (( stable >= stable_needed )); then isNotice "Stalwart is ready (readiness probe stable) after moving to $expected." stalwart_upgrade_verify_admin_ui "$expected" "$deadline" return $? fi else stable=0 fi sleep 5 done isError "Stalwart did not report ready for $expected before the deadline.${last:+ Last probe: $last}" return 1 } # stalwart_upgrade_verify_admin_ui # 0 only when /admin is served. Polled rather than probed once, because the # WebUI fetch runs behind the server coming up: readiness can go stable while # the bundle is still being pulled, and failing on that first 404 would abort # an upgrade that was only a few seconds from finishing. stalwart_upgrade_verify_admin_ui() { local expected="$1" deadline="$2" local code="" while [ "$(date +%s)" -lt "$deadline" ]; do code="$(stalwart_upgrade_admin_ui_code)" if [ -n "$code" ] && [ "$code" != "404" ]; then isSuccessful "Stalwart is ready and serving its admin console after moving to $expected." return 0 fi sleep 5 done # The mail server is up — this is the console alone, so say so. An admin who # reads this as data loss will roll back and re-hit the same missing bundle. if [ -z "$code" ]; then # No status at all is a different fault from a 404: the probe never # reached the container. Named separately so nobody goes hunting a # firewall rule for what is actually an exec/curl problem. isError "Stalwart reported ready at $expected, but its admin console could not be probed." isNotice " No response from 'docker exec stalwart-service curl … /admin' — check the" isNotice " container is running and that curl exists inside the image." return 1 fi isError "Stalwart is serving mail at $expected, but its admin console is missing (/admin=$code)." isNotice " The WebUI is not bundled in the image: Stalwart downloads it from" isNotice " https://github.com/stalwartlabs/webui/releases/latest when it starts on" isNotice " a new image. That download did not complete, so /admin and /account will" isNotice " 404 until it does. Allow outbound HTTPS to github.com from this host and" isNotice " restart the container, then re-run the upgrade." return 1 }