LibrePortal/scripts/cli/commands/updater/cli_updater_verify.sh

#!/bin/bash

# Upgrade verification — "did that rung actually land?"
# ---------------------------------------------------------------------------
# A stepped upgrade is only as safe as its verification. Stepping 31 -> 32 -> 33
# is arithmetic; knowing that 32 FINISHED before touching 33 is the whole ball
# game, because the dangerous state is invisible from the outside: Nextcloud
# runs its migration on boot and can sit in maintenance mode, or fail halfway,
# while Docker cheerfully reports the container healthy. Advance a rung then and
# you have skipped a migration on live data.
#
# So "the container is up" is explicitly NOT accepted as proof for an app that
# declares a verifier. Contract:
#
#   <app>_upgrade_verify <app> <expected-tag> <deadline-epoch>   -> 0 = verified
#
# It must poll until the deadline and return 0 ONLY when it can positively
# confirm the app is serving at the expected version with no migration
# outstanding. Any other outcome — unhealthy, indeterminate, timed out — must
# return non-zero. Uncertainty is a failure here, not a maybe: the engine
# aborts and restores rather than guessing, which is the only honest reading
# when the alternative risks someone's data.
#
# Apps with no verifier fall back to updaterVerifyGeneric, which is deliberately
# conservative and is NOT sufficient for a stepped upgrade — the engine refuses
# to ladder an app that has not declared a real one.

# Container name for an app's ANCHOR service — the one whose image carries the
# app's own version. Read from the compose rather than assumed, because
# "<app>-service" is a convention and not every app follows it: matrix names its
# anchor service matrix-synapse and stoat names its api, so the assumption
# inspected a container that does not exist, found no state at all, and the
# generic verifier could only ever time out on exactly the stateful apps that
# most need verifying. Falls back to the convention when there is no compose.
_updaterPrimaryContainer() {
    local app="$1"
    local compose="${containers_dir%/}/$app/docker-compose.yml"
    local fallback; fallback="$(printf '%s-service' "${app//_/-}")"
    [ -f "$compose" ] || { printf '%s' "$fallback"; return 0; }

    local up; up="$(printf '%s' "$app" | tr '[:lower:]' '[:upper:]')"
    # The service block that owns the bare <APP>_VERSION_TAG sentinel, then that
    # block's container_name -- compose's own answer for what the container is
    # called. \047 is an apostrophe: the program is single-quoted, so it cannot
    # contain one literally.
    local name
    name="$(awk -v key="#LIBREPORTAL|${up}_VERSION_TAG|" '
        /^[[:space:]]{2,4}[a-zA-Z0-9_-]+:[[:space:]]*(#|$)/ {
            if (found) { if (cname != "") print cname; else print svc; exit }
            s=$1; sub(/:.*/,"",s); gsub(/[[:space:]]/,"",s)
            svc=s; cname=""
        }
        /^[[:space:]]*container_name:/ { cname=$2; gsub(/["\047]/,"",cname) }
        index($0,key) { found=1 }
        END { if (found) { if (cname != "") print cname; else print svc } }
    ' "$compose" 2>/dev/null | head -1 | tr -d '[:space:]')"

    [ -n "$name" ] && printf '%s' "$name" || printf '%s' "$fallback"
}

# Generic health: running, not restarting, healthcheck (if any) reporting
# healthy, and STILL true after a settle period — a crash-loop looks perfect
# in the instant between restarts. Good enough for a single in-place update,
# never good enough to justify climbing another rung.
updaterVerifyGeneric() {
    local app="$1" deadline="${3:-$(( $(date +%s) + 120 ))}"
    local c; c="$(_updaterPrimaryContainer "$app")"
    local stable_needed=3 stable=0

    while [ "$(date +%s)" -lt "$deadline" ]; do
        local state health restarts
        state="$(dockerCommandRun "docker inspect --format '{{.State.Status}}' $c" 2>/dev/null | tr -d '\r')"
        health="$(dockerCommandRun "docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' $c" 2>/dev/null | tr -d '\r')"
        restarts="$(dockerCommandRun "docker inspect --format '{{.RestartCount}}' $c" 2>/dev/null | tr -d '\r')"

        if [ "$state" = "running" ] && { [ "$health" = "healthy" ] || [ "$health" = "none" ]; }; then
            stable=$((stable + 1))
            [ -z "${_uv_restarts:-}" ] && _uv_restarts="$restarts"
            # A restart during the settle window means it is looping, not up.
            [ "$restarts" != "$_uv_restarts" ] && stable=0 && _uv_restarts="$restarts"
            (( stable >= stable_needed )) && { unset _uv_restarts; return 0; }
        else
            stable=0
        fi
        sleep 5
    done
    unset _uv_restarts
    return 1
}

# Does this app ship a real verifier? The stepped engine requires one.
#
# "Not currently defined" is NOT the same as "not shipped", and conflating the
# two is the most expensive mistake this file can make. The CLI runs lazy
# (LP_LAZY=1), where the container scan is skipped entirely and every function
# has to arrive through function_manifest.sh — a BUILD-time artifact. Any app
# that came into existence after that build has no stub, so its verifier reads
# as absent while sitting on disk two directories away. That is exactly every
# multi-instance clone, and the consequences both point the wrong way: GATE 1
# refuses an upgrade for an app that is fully verifiable, and the auto-ladder's
# `updaterHasVerifier || continue` drops it in silence, for good.
#
# So consult the disk before answering no, and source what is actually there —
# the same file the eager loader would have picked up. Cheap (one stat for the
# common case) and self-healing regardless of how stale the manifest is, which
# matters because a LibrePortal self-update overwrites the manifest with the
# shipped copy and drops every instance entry again.
updaterHasVerifier() {
    local app="$1"
    declare -F "${app}_upgrade_verify" >/dev/null 2>&1 && return 0

    local hooks="${install_containers_dir%/}/$app/scripts/${app}_upgrade_hooks.sh"
    [ -f "$hooks" ] || return 1
    source "$hooks" 2>/dev/null || return 1
    declare -F "${app}_upgrade_verify" >/dev/null 2>&1
}

# updaterVerifyUpgrade <app> <expected-tag> [timeout-secs]
# Dispatches to the app's verifier, falling back to the generic check. Returns
# 0 only on positive confirmation.
updaterVerifyUpgrade() {
    local app="$1" expected="$2" timeout="${3:-600}"
    local deadline=$(( $(date +%s) + timeout ))

    if updaterHasVerifier "$app"; then
        isNotice "Verifying $app is serving $expected (up to ${timeout}s)…"
        if "${app}_upgrade_verify" "$app" "$expected" "$deadline"; then
            isSuccessful "$app verified at $expected."
            return 0
        fi
        isError "$app did NOT verify at $expected — treating as failed."
        return 1
    fi

    isNotice "$app has no upgrade verifier; using the generic health check."
    updaterVerifyGeneric "$app" "$expected" "$deadline"
}

# ---------------------------------------------------------------------------
# Shared HTTP version verification.
#
# The strongest thing a verifier can say is "the app itself reports the version
# we asked for, and kept reporting it." Most apps expose that over HTTP, so the
# per-app hook becomes three facts — port, path, where the version lives — and
# the polling, extraction, comparison and stability rules live here once.
#
# Probed from the HOST against the container's PUBLISHED port, deliberately, not
# via `docker exec … curl`: half these images ship no curl at all (mattermost is
# one), so exec-based probes are a coin flip on the vendor's base image. The
# host has curl, and a published port is something LibrePortal already
# guarantees for every app with a web interface.

# First published host port for a container's internal port. Empty if unmapped.
_updaterPublishedPortFor() {
    local c="$1" internal="$2"
    dockerCommandRun "docker port $c $internal" 2>/dev/null \
        | tr -d '\r' | grep -oE '[0-9]+$' | head -1
}

# Pull a version out of an HTTP response. "json:<key>" reads a top-level string
# field; "header:<Name>" reads a response header (case-insensitively).
_updaterExtractVersion() {
    local resp="$1" spec="$2"
    case "$spec" in
        json:*)
            local k="${spec#json:}"
            printf '%s' "$resp" | tr -d '\r' \
                | grep -oE "\"$k\"[[:space:]]*:[[:space:]]*\"[^\"]*\"" | head -1 \
                | sed -E 's/.*:[[:space:]]*"([^"]*)"/\1/'
            ;;
        header:*)
            local h="${spec#header:}"
            printf '%s' "$resp" | tr -d '\r' | grep -i "^${h}:" | head -1 \
                | sed -E 's/^[^:]*:[[:space:]]*//'
            ;;
    esac
}

# Do a tag and a self-reported version agree on every component BOTH of them
# state? Tags and running versions are rarely the same precision:
#   v1.158.0 vs 1.158.0 -> yes (the v is noise)
#   11.9     vs 11.9.1  -> yes (the tag is a line; the build is more precise)
#   8.7.0    vs 8.7     -> yes (the app reports less precision than the tag)
#   11.9     vs 11.10   -> NO
# Comparing only the shared prefix is what makes one rule work for all of them;
# demanding string equality would fail every app above except Synapse.
_updaterVersionAgrees() {
    local a b
    a="$(printf '%s' "$1" | grep -oE '[0-9]+' | tr '\n' ' ')"
    b="$(printf '%s' "$2" | grep -oE '[0-9]+' | tr '\n' ' ')"
    local -a A=($a) B=($b)
    local n=${#A[@]}; [ ${#B[@]} -lt "$n" ] && n=${#B[@]}
    [ "$n" -gt 0 ] || return 1
    local i
    for ((i=0; i<n; i++)); do
        [ "$((10#${A[i]}))" -eq "$((10#${B[i]}))" ] || return 1
    done
    return 0
}

# updaterVerifyHttpVersion <app> <expected-tag> <deadline> <internal-port> <path> <extract>
# 0 only when the app reports a version agreeing with the tag, three polls
# running. Three because one lucky answer during a rolling restart proves
# nothing — the old container can still be serving while the new one boots.
updaterVerifyHttpVersion() {
    local app="$1" expected="$2" deadline="$3" iport="$4" path="$5" extract="$6"
    local c; c="$(_updaterPrimaryContainer "$app")"
    local port; port="$(_updaterPublishedPortFor "$c" "$iport")"
    if [ -z "$port" ]; then
        isError "$app: container $c publishes no host port for $iport — cannot verify."
        return 1
    fi

    local stable=0 need=3 last="" resp ver
    while [ "$(date +%s)" -lt "$deadline" ]; do
        resp="$(curl -fsS -i --max-time 6 "http://127.0.0.1:${port}${path}" 2>/dev/null)"
        ver="$(_updaterExtractVersion "$resp" "$extract")"
        last="reported=${ver:-none}"
        if [ -n "$ver" ] && _updaterVersionAgrees "$expected" "$ver"; then
            stable=$((stable + 1))
            if (( stable >= need )); then
                isSuccessful "$app reports $ver, agreeing with $expected, and held it."
                return 0
            fi
        else
            stable=0
        fi
        sleep 5
    done
    isError "$app did not report $expected before the deadline.${last:+ Last probe: $last}"
    return 1
}