diff --git a/containers/mastodon/scripts/mastodon_upgrade_hooks.sh b/containers/mastodon/scripts/mastodon_upgrade_hooks.sh new file mode 100644 index 0000000..5812f57 --- /dev/null +++ b/containers/mastodon/scripts/mastodon_upgrade_hooks.sh @@ -0,0 +1,54 @@ +#!/bin/bash + +# Mastodon upgrade verifier. +# --------------------------------------------------------------------------- +# Mastodon runs Rails migrations on boot and expects releases to be applied in +# order. The web process answers /health once Puma is listening — which happens +# BEFORE migrations necessarily finish — so /health alone is not proof. The +# authoritative check is that no migration is still pending, which +# `rails db:migrate:status` reports (any row marked "down" means outstanding). +# +# Version is confirmed from the instance API rather than the image tag, so we +# are reading what the app says about itself, not what we asked for. + +# mastodon_upgrade_verify +# 0 only when the app serves, reports the expected version, and has no +# migration outstanding. +mastodon_upgrade_verify() { + local app="$1" expected="$2" deadline="$3" + # "v4.6" -> "4.6"; the API reports "4.6.5", so this is a prefix comparison. + local want; want="$(printf '%s' "$expected" | sed 's/^v//')" + + local last="" + while [ "$(date +%s)" -lt "$deadline" ]; do + # 1. Serving at all? + local health + health="$(runFileOp docker exec mastodon-service curl -fsS --max-time 5 \ + http://localhost:3000/health 2>/dev/null | tr -d '\r\n')" + + if [ -n "$health" ]; then + # 2. Migrations finished? Any "down" row means Rails still has work. + local pending + pending="$(runFileOp docker exec mastodon-service bin/rails db:migrate:status 2>/dev/null \ + | awk '$1 == "down" { n++ } END { print n+0 }')" + + # 3. Which version does it actually report? + local ver + ver="$(runFileOp docker exec mastodon-service curl -fsS --max-time 5 \ + http://localhost:3000/api/v1/instance 2>/dev/null \ + | grep -oE '"version":"[^"]+"' | head -1 | cut -d'"' -f4)" + + last="health=$health pendingMigrations=${pending:-?} version=${ver:-?}" + + if [ "${pending:-1}" = "0" ] && [ -n "$ver" ] && [[ "$ver" == "$want"* ]]; then + isSuccessful "Mastodon reports $ver with no pending migrations." + return 0 + fi + isNotice "Mastodon not ready yet: $last" + fi + sleep 10 + done + + isError "Mastodon did not reach a verified state for $expected before the deadline.${last:+ Last status: $last}" + return 1 +} diff --git a/containers/nextcloud/scripts/nextcloud_upgrade_hooks.sh b/containers/nextcloud/scripts/nextcloud_upgrade_hooks.sh new file mode 100644 index 0000000..7bea749 --- /dev/null +++ b/containers/nextcloud/scripts/nextcloud_upgrade_hooks.sh @@ -0,0 +1,64 @@ +#!/bin/bash + +# Nextcloud upgrade verifier. +# --------------------------------------------------------------------------- +# Nextcloud is the reason stepped upgrades exist here: it refuses to cross more +# than one major ("Updates between multiple major versions and downgrades are +# unsupported") and simply will not start. So the ladder must be certain each +# major finished before starting the next. +# +# `occ status` is the authoritative answer and reports exactly the three facts +# that matter — whether the instance is installed, whether it is still in +# maintenance mode, and whether a database upgrade is outstanding — plus the +# version actually running. Container health tells you none of that: during the +# post-upgrade migration the container is up and healthy while the instance is +# in maintenance mode with the schema half-converted. +# +# Runs through the app's own occ wrapper idiom (docker exec -u www-data), so it +# inherits whatever the rest of the Nextcloud tooling already relies on. + +# nextcloud_upgrade_verify +# 0 only when the instance is installed, out of maintenance, has no pending DB +# upgrade, and reports the major version the tag asked for. +nextcloud_upgrade_verify() { + local app="$1" expected="$2" deadline="$3" + # "34-fpm-alpine" -> 34. The tag's leading number IS the major; occ reports + # a full version (34.0.2.1) and only the major is comparable. + local want_major; want_major="$(printf '%s' "$expected" | grep -oE '^[0-9]+')" + + local last="" + while [ "$(date +%s)" -lt "$deadline" ]; do + local out + out="$(runFileOp docker exec -u www-data nextcloud-service php occ status --output=json 2>/dev/null | tr -d '\r')" + + if [ -n "$out" ] && printf '%s' "$out" | grep -q '"installed"'; then + local installed maint needs_db ver major + if command -v jq >/dev/null 2>&1; then + installed="$(printf '%s' "$out" | jq -r '.installed // false' 2>/dev/null)" + maint="$(printf '%s' "$out" | jq -r '.maintenance // false' 2>/dev/null)" + needs_db="$(printf '%s' "$out" | jq -r '.needsDbUpgrade // false' 2>/dev/null)" + ver="$(printf '%s' "$out" | jq -r '.versionstring // ""' 2>/dev/null)" + else + installed="$(printf '%s' "$out" | grep -o '"installed":[^,}]*' | cut -d: -f2 | tr -d ' "')" + maint="$(printf '%s' "$out" | grep -o '"maintenance":[^,}]*' | cut -d: -f2 | tr -d ' "')" + needs_db="$(printf '%s' "$out" | grep -o '"needsDbUpgrade":[^,}]*' | cut -d: -f2 | tr -d ' "')" + ver="$(printf '%s' "$out" | grep -o '"versionstring":"[^"]*"' | cut -d'"' -f4)" + fi + major="$(printf '%s' "$ver" | cut -d. -f1)" + last="installed=$installed maintenance=$maint needsDbUpgrade=$needs_db version=$ver" + + if [ "$installed" = "true" ] && [ "$maint" = "false" ] && [ "$needs_db" = "false" ] \ + && [ -n "$want_major" ] && [ "$major" = "$want_major" ]; then + isSuccessful "Nextcloud reports $ver, out of maintenance, no pending DB upgrade." + return 0 + fi + # Maintenance mode mid-migration is EXPECTED and not a failure — + # keep waiting. Only the deadline ends this. + isNotice "Nextcloud not ready yet: $last" + fi + sleep 10 + done + + isError "Nextcloud did not reach a verified state for $expected before the deadline.${last:+ Last status: $last}" + return 1 +} diff --git a/containers/stalwart/scripts/stalwart_upgrade_hooks.sh b/containers/stalwart/scripts/stalwart_upgrade_hooks.sh new file mode 100644 index 0000000..bb77889 --- /dev/null +++ b/containers/stalwart/scripts/stalwart_upgrade_hooks.sh @@ -0,0 +1,49 @@ +#!/bin/bash + +# Stalwart upgrade verifier. +# --------------------------------------------------------------------------- +# Stalwart publishes Kubernetes-style probes on its admin port (8080): +# GET /healthz/live — the process is alive and not deadlocked +# GET /healthz/ready — dependencies initialised, config loaded, accepting traffic +# https://stalw.art/docs/http/overview/ +# +# Readiness is the one that matters after a version move: it only answers 200 +# once the storage backend is open and the config has loaded, which is exactly +# the window where a schema change would otherwise go unnoticed. Liveness alone +# would pass on a process that is up but unable to serve. +# +# Deliberately weaker than the Nextcloud verifier: Stalwart's probes confirm the +# server is serving, but do not report a version, so this asserts readiness +# rather than "running exactly $expected". Stated plainly instead of implied — +# the ladder is only ever as strong as the check underneath it, and pretending +# otherwise is how a half-migrated app advances a rung. + +# stalwart_upgrade_verify +# 0 only when /healthz/ready answers 200 and keeps answering it. +stalwart_upgrade_verify() { + local app="$1" expected="$2" deadline="$3" + local stable=0 stable_needed=3 last="" + + while [ "$(date +%s)" -lt "$deadline" ]; do + local code + code="$(runFileOp docker exec stalwart-service curl -fsS -o /dev/null -w '%{http_code}' \ + --max-time 5 http://localhost:8080/healthz/ready 2>/dev/null | tr -d '\r')" + last="healthz/ready=${code:-none}" + + if [ "$code" = "200" ]; then + stable=$((stable + 1)) + # Ready must HOLD: a server that flaps ready/not-ready is mid-restart, + # and one lucky 200 is not evidence the upgrade settled. + if (( stable >= stable_needed )); then + isSuccessful "Stalwart is ready (readiness probe stable) after moving to $expected." + return 0 + fi + else + stable=0 + fi + sleep 5 + done + + isError "Stalwart did not report ready for $expected before the deadline.${last:+ Last probe: $last}" + return 1 +} diff --git a/scripts/cli/commands/updater/cli_updater_verify.sh b/scripts/cli/commands/updater/cli_updater_verify.sh new file mode 100644 index 0000000..96f6454 --- /dev/null +++ b/scripts/cli/commands/updater/cli_updater_verify.sh @@ -0,0 +1,86 @@ +#!/bin/bash + +# Upgrade verification — "did that rung actually land?" +# --------------------------------------------------------------------------- +# A stepped upgrade is only as safe as its verification. Stepping 31 -> 32 -> 33 +# is arithmetic; knowing that 32 FINISHED before touching 33 is the whole ball +# game, because the dangerous state is invisible from the outside: Nextcloud +# runs its migration on boot and can sit in maintenance mode, or fail halfway, +# while Docker cheerfully reports the container healthy. Advance a rung then and +# you have skipped a migration on live data. +# +# So "the container is up" is explicitly NOT accepted as proof for an app that +# declares a verifier. Contract: +# +# _upgrade_verify -> 0 = verified +# +# It must poll until the deadline and return 0 ONLY when it can positively +# confirm the app is serving at the expected version with no migration +# outstanding. Any other outcome — unhealthy, indeterminate, timed out — must +# return non-zero. Uncertainty is a failure here, not a maybe: the engine +# aborts and restores rather than guessing, which is the only honest reading +# when the alternative risks someone's data. +# +# Apps with no verifier fall back to updaterVerifyGeneric, which is deliberately +# conservative and is NOT sufficient for a stepped upgrade — the engine refuses +# to ladder an app that has not declared a real one. + +# Container name for an app's primary service, matching the compose convention. +_updaterPrimaryContainer() { printf '%s-service' "${1//_/-}"; } + +# Generic health: running, not restarting, healthcheck (if any) reporting +# healthy, and STILL true after a settle period — a crash-loop looks perfect +# in the instant between restarts. Good enough for a single in-place update, +# never good enough to justify climbing another rung. +updaterVerifyGeneric() { + local app="$1" deadline="${3:-$(( $(date +%s) + 120 ))}" + local c; c="$(_updaterPrimaryContainer "$app")" + local stable_needed=3 stable=0 + + while [ "$(date +%s)" -lt "$deadline" ]; do + local state health restarts + state="$(dockerCommandRun "docker inspect --format '{{.State.Status}}' $c" 2>/dev/null | tr -d '\r')" + health="$(dockerCommandRun "docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' $c" 2>/dev/null | tr -d '\r')" + restarts="$(dockerCommandRun "docker inspect --format '{{.RestartCount}}' $c" 2>/dev/null | tr -d '\r')" + + if [ "$state" = "running" ] && { [ "$health" = "healthy" ] || [ "$health" = "none" ]; }; then + stable=$((stable + 1)) + [ -z "${_uv_restarts:-}" ] && _uv_restarts="$restarts" + # A restart during the settle window means it is looping, not up. + [ "$restarts" != "$_uv_restarts" ] && stable=0 && _uv_restarts="$restarts" + (( stable >= stable_needed )) && { unset _uv_restarts; return 0; } + else + stable=0 + fi + sleep 5 + done + unset _uv_restarts + return 1 +} + +# Does this app ship a real verifier? The stepped engine requires one. +updaterHasVerifier() { + local app="$1" + declare -F "${app}_upgrade_verify" >/dev/null 2>&1 +} + +# updaterVerifyUpgrade [timeout-secs] +# Dispatches to the app's verifier, falling back to the generic check. Returns +# 0 only on positive confirmation. +updaterVerifyUpgrade() { + local app="$1" expected="$2" timeout="${3:-600}" + local deadline=$(( $(date +%s) + timeout )) + + if updaterHasVerifier "$app"; then + isNotice "Verifying $app is serving $expected (up to ${timeout}s)…" + if "${app}_upgrade_verify" "$app" "$expected" "$deadline"; then + isSuccessful "$app verified at $expected." + return 0 + fi + isError "$app did NOT verify at $expected — treating as failed." + return 1 + fi + + isNotice "$app has no upgrade verifier; using the generic health check." + updaterVerifyGeneric "$app" "$expected" "$deadline" +} diff --git a/scripts/source/files/arrays/files_cli.sh b/scripts/source/files/arrays/files_cli.sh index 92b3cca..5c6b234 100755 --- a/scripts/source/files/arrays/files_cli.sh +++ b/scripts/source/files/arrays/files_cli.sh @@ -55,6 +55,7 @@ cli_scripts=( "cli/commands/updater/cli_updater_commands.sh" "cli/commands/updater/cli_updater_header.sh" "cli/commands/updater/cli_updater_ladder.sh" + "cli/commands/updater/cli_updater_verify.sh" "cli/commands/validation/cli_validation_commands.sh" "cli/commands/validation/cli_validation_header.sh" "cli/commands/verify/cli_verify_commands.sh" diff --git a/scripts/source/files/arrays/function_manifest.sh b/scripts/source/files/arrays/function_manifest.sh index 20ed085..eec278e 100644 --- a/scripts/source/files/arrays/function_manifest.sh +++ b/scripts/source/files/arrays/function_manifest.sh @@ -658,6 +658,7 @@ declare -gA LP_FN_MAP=( [manifestReadFromSnapshot]="backup/manifest/manifest_read.sh" [manifestRemove]="backup/manifest/manifest_write.sh" [manifestWrite]="backup/manifest/manifest_write.sh" + [mastodon_upgrade_verify]="mastodon/scripts/mastodon_upgrade_hooks.sh" [mattermostToolsMenu]="menu/tools/manage_mattermost.sh" [maybeRegenPoll]="task/crontab_task_processor.sh" [menuContinue]="menu/message/continue.sh" @@ -701,6 +702,7 @@ declare -gA LP_FN_MAP=( [networkScanConflicts]="docker/network/network_conflicts.sh" [_nextcloudOcc]="nextcloud/scripts/nextcloud_auth.sh" [_nextcloudOccWithPass]="nextcloud/scripts/nextcloud_auth.sh" + [nextcloud_upgrade_verify]="nextcloud/scripts/nextcloud_upgrade_hooks.sh" [onlyoffice_install_message_data]="onlyoffice/scripts/onlyoffice_install_hooks.sh" [openFifoReader]="task/crontab_task_processor.sh" [owncloud_install_post_compose]="owncloud/scripts/owncloud_install_hooks.sh" @@ -874,6 +876,7 @@ declare -gA LP_FN_MAP=( [sshRemote]="network/ssh/ssh.sh" [stalwart_install_message_data]="stalwart/scripts/stalwart_install_hooks.sh" [stalwart_install_post_start]="stalwart/scripts/stalwart_install_hooks.sh" + [stalwart_upgrade_verify]="stalwart/scripts/stalwart_upgrade_hooks.sh" [startInstall]="start/start_install.sh" [startLoad]="start/start_load.sh" [startOther]="start/start_other.sh" @@ -935,11 +938,13 @@ declare -gA LP_FN_MAP=( [_updaterCleanImageRef]="webui/data/generators/updater/webui_updater_scan.sh" [updaterComposePull]="cli/commands/updater/cli_updater_commands.sh" [updaterDisplayVersion]="webui/data/generators/updater/webui_updater_scan.sh" + [updaterHasVerifier]="cli/commands/updater/cli_updater_verify.sh" [updaterInspectLocal]="webui/data/generators/updater/webui_updater_scan.sh" [updaterInWindow]="cli/commands/updater/cli_updater_auto.sh" [updaterLadderSummary]="cli/commands/updater/cli_updater_ladder.sh" [updaterLastUpdateFrom]="cli/commands/updater/cli_updater_commands.sh" [updaterNewerVersionTag]="webui/data/generators/updater/webui_updater_scan.sh" + [_updaterPrimaryContainer]="cli/commands/updater/cli_updater_verify.sh" [updaterPrimaryImage]="webui/data/generators/updater/webui_updater_scan.sh" [updaterRecordHistory]="cli/commands/updater/cli_updater_commands.sh" [updaterRefDigest]="cli/commands/updater/cli_updater_commands.sh" @@ -955,6 +960,8 @@ declare -gA LP_FN_MAP=( [updaterTagOf]="webui/data/generators/updater/webui_updater_scan.sh" [updaterTagShape]="webui/data/generators/updater/webui_updater_scan.sh" [updaterTagSortKey]="cli/commands/updater/cli_updater_ladder.sh" + [updaterVerifyGeneric]="cli/commands/updater/cli_updater_verify.sh" + [updaterVerifyUpgrade]="cli/commands/updater/cli_updater_verify.sh" [updaterVersionLadder]="cli/commands/updater/cli_updater_ladder.sh" [updateTaskFields]="task/crontab_task_processor.sh" [_upReportComposeFailure]="docker/app/compose/up_app.sh" @@ -1691,6 +1698,7 @@ declare -gA LP_FN_ROOT=( [manifestReadFromSnapshot]="scripts" [manifestRemove]="scripts" [manifestWrite]="scripts" + [mastodon_upgrade_verify]="containers" [mattermostToolsMenu]="scripts" [maybeRegenPoll]="scripts" [menuContinue]="scripts" @@ -1734,6 +1742,7 @@ declare -gA LP_FN_ROOT=( [networkScanConflicts]="scripts" [_nextcloudOcc]="containers" [_nextcloudOccWithPass]="containers" + [nextcloud_upgrade_verify]="containers" [onlyoffice_install_message_data]="containers" [openFifoReader]="scripts" [owncloud_install_post_compose]="containers" @@ -1907,6 +1916,7 @@ declare -gA LP_FN_ROOT=( [sshRemote]="scripts" [stalwart_install_message_data]="containers" [stalwart_install_post_start]="containers" + [stalwart_upgrade_verify]="containers" [startInstall]="scripts" [startLoad]="scripts" [startOther]="scripts" @@ -1968,11 +1978,13 @@ declare -gA LP_FN_ROOT=( [_updaterCleanImageRef]="scripts" [updaterComposePull]="scripts" [updaterDisplayVersion]="scripts" + [updaterHasVerifier]="scripts" [updaterInspectLocal]="scripts" [updaterInWindow]="scripts" [updaterLadderSummary]="scripts" [updaterLastUpdateFrom]="scripts" [updaterNewerVersionTag]="scripts" + [_updaterPrimaryContainer]="scripts" [updaterPrimaryImage]="scripts" [updaterRecordHistory]="scripts" [updaterRefDigest]="scripts" @@ -1988,6 +2000,8 @@ declare -gA LP_FN_ROOT=( [updaterTagOf]="scripts" [updaterTagShape]="scripts" [updaterTagSortKey]="scripts" + [updaterVerifyGeneric]="scripts" + [updaterVerifyUpgrade]="scripts" [updaterVersionLadder]="scripts" [updateTaskFields]="scripts" [_upReportComposeFailure]="scripts" @@ -2086,6 +2100,7 @@ LP_EAGER_FILES=( "scripts:docker/install/rootless/rootless_apparmor.sh" "scripts:docker/type_switcher/swap_docker_type.sh" "scripts:migrate/migrate_url_rewrite.sh" + "containers:nextcloud/scripts/nextcloud_upgrade_hooks.sh" "scripts:setup/setup_lock.sh" "scripts:source/artifacts.sh" "scripts:task/crontab_check_processor.sh" @@ -2757,6 +2772,7 @@ manifestReadField() { unset -f manifestReadField; __lpAutoload "${install_script manifestReadFromSnapshot() { unset -f manifestReadFromSnapshot; __lpAutoload "${install_scripts_dir}backup/manifest/manifest_read.sh"; manifestReadFromSnapshot "$@"; } manifestRemove() { unset -f manifestRemove; __lpAutoload "${install_scripts_dir}backup/manifest/manifest_write.sh"; manifestRemove "$@"; } manifestWrite() { unset -f manifestWrite; __lpAutoload "${install_scripts_dir}backup/manifest/manifest_write.sh"; manifestWrite "$@"; } +mastodon_upgrade_verify() { unset -f mastodon_upgrade_verify; __lpAutoload "${install_containers_dir}mastodon/scripts/mastodon_upgrade_hooks.sh"; mastodon_upgrade_verify "$@"; } mattermostToolsMenu() { unset -f mattermostToolsMenu; __lpAutoload "${install_scripts_dir}menu/tools/manage_mattermost.sh"; mattermostToolsMenu "$@"; } maybeRegenPoll() { unset -f maybeRegenPoll; __lpAutoload "${install_scripts_dir}task/crontab_task_processor.sh"; maybeRegenPoll "$@"; } menuContinue() { unset -f menuContinue; __lpAutoload "${install_scripts_dir}menu/message/continue.sh"; menuContinue "$@"; } @@ -2800,6 +2816,7 @@ networkRedetectMtu() { unset -f networkRedetectMtu; __lpAutoload "${install_scri networkScanConflicts() { unset -f networkScanConflicts; __lpAutoload "${install_scripts_dir}docker/network/network_conflicts.sh"; networkScanConflicts "$@"; } _nextcloudOcc() { unset -f _nextcloudOcc; __lpAutoload "${install_containers_dir}nextcloud/scripts/nextcloud_auth.sh"; _nextcloudOcc "$@"; } _nextcloudOccWithPass() { unset -f _nextcloudOccWithPass; __lpAutoload "${install_containers_dir}nextcloud/scripts/nextcloud_auth.sh"; _nextcloudOccWithPass "$@"; } +nextcloud_upgrade_verify() { unset -f nextcloud_upgrade_verify; __lpAutoload "${install_containers_dir}nextcloud/scripts/nextcloud_upgrade_hooks.sh"; nextcloud_upgrade_verify "$@"; } onlyoffice_install_message_data() { unset -f onlyoffice_install_message_data; __lpAutoload "${install_containers_dir}onlyoffice/scripts/onlyoffice_install_hooks.sh"; onlyoffice_install_message_data "$@"; } openFifoReader() { unset -f openFifoReader; __lpAutoload "${install_scripts_dir}task/crontab_task_processor.sh"; openFifoReader "$@"; } owncloud_install_post_compose() { unset -f owncloud_install_post_compose; __lpAutoload "${install_containers_dir}owncloud/scripts/owncloud_install_hooks.sh"; owncloud_install_post_compose "$@"; } @@ -2973,6 +2990,7 @@ sourceBackupLocations() { unset -f sourceBackupLocations; __lpAutoload "${instal sshRemote() { unset -f sshRemote; __lpAutoload "${install_scripts_dir}network/ssh/ssh.sh"; sshRemote "$@"; } stalwart_install_message_data() { unset -f stalwart_install_message_data; __lpAutoload "${install_containers_dir}stalwart/scripts/stalwart_install_hooks.sh"; stalwart_install_message_data "$@"; } stalwart_install_post_start() { unset -f stalwart_install_post_start; __lpAutoload "${install_containers_dir}stalwart/scripts/stalwart_install_hooks.sh"; stalwart_install_post_start "$@"; } +stalwart_upgrade_verify() { unset -f stalwart_upgrade_verify; __lpAutoload "${install_containers_dir}stalwart/scripts/stalwart_upgrade_hooks.sh"; stalwart_upgrade_verify "$@"; } startInstall() { unset -f startInstall; __lpAutoload "${install_scripts_dir}start/start_install.sh"; startInstall "$@"; } startLoad() { unset -f startLoad; __lpAutoload "${install_scripts_dir}start/start_load.sh"; startLoad "$@"; } startOther() { unset -f startOther; __lpAutoload "${install_scripts_dir}start/start_other.sh"; startOther "$@"; } @@ -3034,11 +3052,13 @@ updaterClassifyTag() { unset -f updaterClassifyTag; __lpAutoload "${install_scri _updaterCleanImageRef() { unset -f _updaterCleanImageRef; __lpAutoload "${install_scripts_dir}webui/data/generators/updater/webui_updater_scan.sh"; _updaterCleanImageRef "$@"; } updaterComposePull() { unset -f updaterComposePull; __lpAutoload "${install_scripts_dir}cli/commands/updater/cli_updater_commands.sh"; updaterComposePull "$@"; } updaterDisplayVersion() { unset -f updaterDisplayVersion; __lpAutoload "${install_scripts_dir}webui/data/generators/updater/webui_updater_scan.sh"; updaterDisplayVersion "$@"; } +updaterHasVerifier() { unset -f updaterHasVerifier; __lpAutoload "${install_scripts_dir}cli/commands/updater/cli_updater_verify.sh"; updaterHasVerifier "$@"; } updaterInspectLocal() { unset -f updaterInspectLocal; __lpAutoload "${install_scripts_dir}webui/data/generators/updater/webui_updater_scan.sh"; updaterInspectLocal "$@"; } updaterInWindow() { unset -f updaterInWindow; __lpAutoload "${install_scripts_dir}cli/commands/updater/cli_updater_auto.sh"; updaterInWindow "$@"; } updaterLadderSummary() { unset -f updaterLadderSummary; __lpAutoload "${install_scripts_dir}cli/commands/updater/cli_updater_ladder.sh"; updaterLadderSummary "$@"; } updaterLastUpdateFrom() { unset -f updaterLastUpdateFrom; __lpAutoload "${install_scripts_dir}cli/commands/updater/cli_updater_commands.sh"; updaterLastUpdateFrom "$@"; } updaterNewerVersionTag() { unset -f updaterNewerVersionTag; __lpAutoload "${install_scripts_dir}webui/data/generators/updater/webui_updater_scan.sh"; updaterNewerVersionTag "$@"; } +_updaterPrimaryContainer() { unset -f _updaterPrimaryContainer; __lpAutoload "${install_scripts_dir}cli/commands/updater/cli_updater_verify.sh"; _updaterPrimaryContainer "$@"; } updaterPrimaryImage() { unset -f updaterPrimaryImage; __lpAutoload "${install_scripts_dir}webui/data/generators/updater/webui_updater_scan.sh"; updaterPrimaryImage "$@"; } updaterRecordHistory() { unset -f updaterRecordHistory; __lpAutoload "${install_scripts_dir}cli/commands/updater/cli_updater_commands.sh"; updaterRecordHistory "$@"; } updaterRefDigest() { unset -f updaterRefDigest; __lpAutoload "${install_scripts_dir}cli/commands/updater/cli_updater_commands.sh"; updaterRefDigest "$@"; } @@ -3054,6 +3074,8 @@ updaterTagNums() { unset -f updaterTagNums; __lpAutoload "${install_scripts_dir} updaterTagOf() { unset -f updaterTagOf; __lpAutoload "${install_scripts_dir}webui/data/generators/updater/webui_updater_scan.sh"; updaterTagOf "$@"; } updaterTagShape() { unset -f updaterTagShape; __lpAutoload "${install_scripts_dir}webui/data/generators/updater/webui_updater_scan.sh"; updaterTagShape "$@"; } updaterTagSortKey() { unset -f updaterTagSortKey; __lpAutoload "${install_scripts_dir}cli/commands/updater/cli_updater_ladder.sh"; updaterTagSortKey "$@"; } +updaterVerifyGeneric() { unset -f updaterVerifyGeneric; __lpAutoload "${install_scripts_dir}cli/commands/updater/cli_updater_verify.sh"; updaterVerifyGeneric "$@"; } +updaterVerifyUpgrade() { unset -f updaterVerifyUpgrade; __lpAutoload "${install_scripts_dir}cli/commands/updater/cli_updater_verify.sh"; updaterVerifyUpgrade "$@"; } updaterVersionLadder() { unset -f updaterVersionLadder; __lpAutoload "${install_scripts_dir}cli/commands/updater/cli_updater_ladder.sh"; updaterVersionLadder "$@"; } updateTaskFields() { unset -f updateTaskFields; __lpAutoload "${install_scripts_dir}task/crontab_task_processor.sh"; updateTaskFields "$@"; } _upReportComposeFailure() { unset -f _upReportComposeFailure; __lpAutoload "${install_scripts_dir}docker/app/compose/up_app.sh"; _upReportComposeFailure "$@"; }