config: carry newly-added app options into existing installs

An app's deployed config is written once, on first install, and never
touched again — dockerConfigSetupToContainer copies only when the file is
absent, precisely so an update can never overwrite values someone has
edited. Right default, unchosen consequence: an app that gains a CFG_
option in a new release has it on every fresh install and on no existing
one.

The failure is silent, which is the worst part. Nothing errors. The key
reads as empty and whatever depends on it quietly does something else.

Two halves, because there were two gaps. Per-app, when a config is set up,
options present in the template and missing from the deployed file are
appended with their comment blocks — the comment is the only explanation
of a new option that exists, and a bare key at the end of a documented file
is not actionable. And a sweep across every installed app after an update,
because an update redeploys LibrePortal itself and nothing else, so without
it a new option would reach an app only when someone next reinstalled it —
which, for an app that is working, may be never.

Existing values are never touched, and keys the deployed file has but the
template no longer does are left alone: a removed option is usually a
rename, and deleting someone's value is not recoverable. Deliberately not a
regenerate-from-template, which would place new keys in their proper
section and refresh the docs, but would put a whole-file rewrite of every
app config in the path of every app action — appending cannot lose a line.

Backfilled RANDOMIZED* defaults are generated in both paths. A placeholder
left in place would otherwise be a credential identical on every install
that took the upgrade.

The sweep is driven from the template directory, not the container one:
under rootless the container tree is drwxr-x--x and owned by the docker
user, so the manager can traverse it but not list it, and a glob there
expands to nothing — the sweep would report success having examined no apps.

Run against this install it found real drift beyond the test fixtures:
mattermost was missing CFG_MATTERMOST_ADMIN_PASSWORD, whose own comment
notes that without it the password-reset tool has nothing to write to, and
speedtest was missing its password key entirely.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

scripts/config/docker/config_backfill_keys.sh

@@ -0,0 +1,154 @@
+#!/bin/bash
+
+# Bring config options added to an app's template into an already-deployed copy.
+#
+# The deployed config at /docker/containers/<app>/<app>.config is written once,
+# on first install, and never touched again — dockerConfigSetupToContainer only
+# copies when the file is absent, precisely so a LibrePortal update can never
+# overwrite the values someone has edited. That is the right default, but it has
+# a consequence nobody chose: an app that gains a CFG_ option in a new release
+# has that option on every FRESH install and on no EXISTING one.
+#
+# The failure is silent, which is the worst part. Nothing errors. The new key
+# simply reads as empty, and whatever depends on it does something quietly
+# different — a default it did not mean to take, or a value it needed and did
+# not get. It only surfaces as "why does this work on my other box".
+#
+# So: copy across keys the template has and the deployed file does not, and
+# nothing else.
+#
+# Deliberately NOT a merge or a regenerate. Rebuilding the deployed file from
+# the template would place new keys in their proper section and refresh the
+# documentation with them, which is genuinely nicer to read — but it would put a
+# whole-file rewrite of every app's config in the path of every app action, and
+# the worst case of a bug there is silently corrupting settings across the whole
+# install. Appending cannot lose an existing line. That trade is not close.
+#
+# Existing keys are never touched, and keys the deployed file has but the
+# template no longer does are left exactly where they are: a removed option is
+# usually a rename, and deleting the user's value is not recoverable.
+
+configBackfillMissingKeys()
+{
+    local app_name="$1"
+    local template="$2"
+    local deployed="$3"
+    local silent_flag="${4:-silent}"
+
+    [[ -f "$template" ]] || return 0
+    [[ -f "$deployed" ]] || return 0
+
+    # Same file, or a fresh copy of it — nothing can be missing.
+    runFileOp cmp -s "$template" "$deployed" && return 0
+
+    local template_keys deployed_keys missing
+    template_keys=$(grep -oE '^CFG_[A-Z0-9_]+=' "$template" 2>/dev/null | sed 's/=$//' | sort -u)
+    deployed_keys=$(runFileOp grep -oE '^CFG_[A-Z0-9_]+=' "$deployed" 2>/dev/null | sed 's/=$//' | sort -u)
+
+    [[ -z "$template_keys" ]] && return 0
+
+    missing=$(comm -23 <(printf '%s\n' "$template_keys") <(printf '%s\n' "$deployed_keys"))
+    [[ -z "$missing" ]] && return 0
+
+    # Carry each key's comment block over with it. A bare `CFG_X=value` appended
+    # to the end of a heavily-documented file is an option nobody can act on;
+    # the comment above it in the template is the only explanation that exists.
+    local missing_csv
+    missing_csv=$(printf '%s\n' "$missing" | paste -sd, -)
+
+    local block
+    block=$(awk -v keys="$missing_csv" '
+        BEGIN { n = split(keys, K, ","); for (i = 1; i <= n; i++) want[K[i]] = 1 }
+        # Accumulate the contiguous comment block sitting directly above a key.
+        /^[[:space:]]*#/          { buf = buf $0 "\n"; next }
+        # A blank line ends a block — the comments above it belong to something else.
+        /^[[:space:]]*$/          { buf = ""; next }
+        /^CFG_[A-Z0-9_]+=/ {
+            k = $0; sub(/=.*/, "", k)
+            if (k in want) printf "%s%s\n\n", buf, $0
+            buf = ""
+            next
+        }
+        { buf = "" }
+    ' "$template")
+
+    [[ -z "$block" ]] && return 0
+
+    {
+        printf '\n#\n'
+        printf '# =============================================================================\n'
+        printf '# ADDED BY A LIBREPORTAL UPDATE\n'
+        printf '# =============================================================================\n'
+        printf '# These options did not exist when this app was installed. They are set to the\n'
+        printf '# defaults shipped with the new version — review them, they are yours to change.\n'
+        printf '#\n'
+        printf '%s\n' "$block"
+    } | runFileWrite -a "$deployed"
+
+    local count
+    count=$(printf '%s\n' "$missing" | grep -c .)
+    # Counter rather than a return code: the callers use this as a bare
+    # statement, and a non-zero "nothing to do" would read as a failure to any
+    # of them running under errexit.
+    LP_BACKFILL_ADDED=$(( ${LP_BACKFILL_ADDED:-0} + count ))
+
+    if [[ "$silent_flag" == "loud" ]]; then
+        isSuccessful "Added $count new config option(s) to $app_name:"
+        printf '%s\n' "$missing" | sed 's/^/    /'
+    else
+        isNotice "Added $count new config option(s) to $app_name's config."
+    fi
+
+    return 0
+}
+
+# Sweep every installed app after a LibrePortal update.
+#
+# The per-app backfill above only fires when an app's config is set up, which
+# happens on install — and an update installs exactly one app, LibrePortal
+# itself. So without this sweep, an option added to some app in a new release
+# reaches that app only when someone next reinstalls it, which for a working app
+# may be never. Since the whole point is that new options arrive on upgrade,
+# upgrade is where this has to run.
+configBackfillAllApps()
+{
+    local dir app template deployed
+    LP_BACKFILL_ADDED=0
+
+    # Driven from the template directory, not from the deployed one. Under
+    # rootless the container tree is mode drwxr-x--x and owned by the docker
+    # install user, so the manager running this can traverse it but cannot LIST
+    # it — a glob over it silently expands to nothing and the sweep would report
+    # success having examined no apps at all. The template dir is manager-owned
+    # and readable, and it defines the same set of apps; whether each one is
+    # actually installed is then just "does its deployed config exist", which is
+    # a traverse, not a list.
+    for dir in "$install_containers_dir"*/; do
+        [[ -d "$dir" ]] || continue
+        app="${dir%/}"; app="${app##*/}"
+        [[ "$app" == "template" ]] && continue
+
+        template="${dir}${app}.config"
+        deployed="${containers_dir}${app}/${app}.config"
+        [[ -f "$template" ]] || continue
+        runFileOp test -f "$deployed" || continue
+
+        configBackfillMissingKeys "$app" "$template" "$deployed" "silent"
+
+        # A backfilled key whose default is a RANDOMIZED* placeholder has to be
+        # given a real value here — nothing else will run over this file until
+        # the app is next installed, and a placeholder left in place is a
+        # credential that is identical on every install that took this upgrade.
+        if runFileOp grep -qE 'RANDOMIZED(PASSWORD|USERNAME|BCRYPTPASSWORD|HEX|VAPID|APPKEY)[0-9]*' "$deployed" 2>/dev/null; then
+            scanFileForRandomPasswordKeysUsers "$deployed"
+        fi
+    done
+
+    if (( ${LP_BACKFILL_ADDED:-0} > 0 )); then
+        isSuccessful "Carried $LP_BACKFILL_ADDED new config option(s) into your existing apps."
+        isNotice "  They are set to the defaults shipped with this version — review them"
+        isNotice "  in each app's Config tab."
+    fi
+
+    return 0
+}

scripts/config/docker/docker_config_to_container.sh

@@ -50,6 +50,16 @@ dockerConfigSetupToContainer()
             isNotice "Copying config file to '$target_path/$config_file'..."
         fi
         copyFile "$silent_flag" "$source_file" "$target_path/$config_file" $sudo_user_name | runInstallWrite -a "$logs_dir/$docker_log_file" 2>&1
+    else
+        # The file already exists, so the copy above was skipped — which is what
+        # protects the user's edited values from every subsequent update. The
+        # cost is that options ADDED to the template since this app was installed
+        # would never arrive. Carry those across; existing values are untouched.
+        #
+        # Runs before the RANDOMIZED* pass below on purpose: a newly-added key
+        # whose default is a RANDOMIZEDPASSWORD<n> placeholder gets a real
+        # generated value from that pass, exactly as it would on a fresh install.
+        configBackfillMissingKeys "$app_name" "$source_file" "$target_path/$config_file" "$silent_flag"
     fi
 
     if [[ -n "$config_overrides" ]]; then

scripts/source/files/arrays/files_config.sh

@@ -18,6 +18,7 @@ config_scripts=(
     "config/core/config_update_option.sh"
     "config/core/variables/config_missing_variables.sh"
     "config/core/variables/config_scan_variables.sh"
+    "config/docker/config_backfill_keys.sh"
     "config/docker/docker_compose_menu.sh"
     "config/docker/docker_config_setup_data.sh"
     "config/docker/docker_config_to_container.sh"

scripts/source/files/arrays/function_manifest.sh

@@ -370,6 +370,8 @@ declare -gA LP_FN_MAP=(
     [cliUpdateCommands]="cli/cli_update.sh"
     [cliWebuiLoginReset]="cli/commands/webui/cli_webui_commands.sh"
     [completeMessage]="menu/message/complete.sh"
+    [configBackfillAllApps]="config/docker/config_backfill_keys.sh"
+    [configBackfillMissingKeys]="config/docker/config_backfill_keys.sh"
     [configSetupFileWithData]="config/core/config_file_setup_data.sh"
     [configUpdateBatch]="config/config_update.sh"
     [containsElement]="function/validation/email.sh"
@@ -929,6 +931,7 @@ declare -gA LP_FN_MAP=(
     [sshRemote]="network/ssh/ssh.sh"
     [stalwart_apply_port_access]="stalwart/scripts/stalwart_install_hooks.sh"
     [stalwart_cli]="stalwart/scripts/stalwart_install_hooks.sh"
+    [stalwart_http_code]="stalwart/scripts/stalwart_install_hooks.sh"
     [stalwart_install_dns_provider]="stalwart/scripts/stalwart_install_hooks.sh"
     [stalwart_install_first_mailbox]="stalwart/scripts/stalwart_install_hooks.sh"
     [stalwart_install_message_data]="stalwart/scripts/stalwart_install_hooks.sh"
@@ -1021,6 +1024,8 @@ declare -gA LP_FN_MAP=(
     [updaterInWindow]="cli/commands/updater/cli_updater_auto.sh"
     [updaterLadderSummary]="cli/commands/updater/cli_updater_ladder.sh"
     [updaterLastUpdateFrom]="cli/commands/updater/cli_updater_commands.sh"
+    [updaterNewerVersionByList]="webui/data/generators/updater/webui_updater_scan.sh"
+    [updaterNewerVersionByProbe]="webui/data/generators/updater/webui_updater_scan.sh"
     [updaterNewerVersionTag]="webui/data/generators/updater/webui_updater_scan.sh"
     [_updaterPrimaryContainer]="cli/commands/updater/cli_updater_verify.sh"
     [updaterPrimaryImage]="webui/data/generators/updater/webui_updater_scan.sh"
@@ -1032,6 +1037,7 @@ declare -gA LP_FN_MAP=(
     [updaterRollbackApp]="cli/commands/updater/cli_updater_commands.sh"
     [updaterSetAnchorRef]="cli/commands/updater/cli_updater_commands.sh"
     [updaterSetAnchorVersion]="cli/commands/updater/cli_updater_upgrade.sh"
+    [updaterTagBumpAt]="webui/data/generators/updater/webui_updater_scan.sh"
     [updaterTagExists]="cli/commands/updater/cli_updater_ladder.sh"
     [updaterTagGreater]="webui/data/generators/updater/webui_updater_scan.sh"
     [updaterTagIncrement]="cli/commands/updater/cli_updater_ladder.sh"
@@ -1495,6 +1501,8 @@ declare -gA LP_FN_ROOT=(
     [cliUpdateCommands]="scripts"
     [cliWebuiLoginReset]="scripts"
     [completeMessage]="scripts"
+    [configBackfillAllApps]="scripts"
+    [configBackfillMissingKeys]="scripts"
     [configSetupFileWithData]="scripts"
     [configUpdateBatch]="scripts"
     [containsElement]="scripts"
@@ -2054,6 +2062,7 @@ declare -gA LP_FN_ROOT=(
     [sshRemote]="scripts"
     [stalwart_apply_port_access]="containers"
     [stalwart_cli]="containers"
+    [stalwart_http_code]="containers"
     [stalwart_install_dns_provider]="containers"
     [stalwart_install_first_mailbox]="containers"
     [stalwart_install_message_data]="containers"
@@ -2146,6 +2155,8 @@ declare -gA LP_FN_ROOT=(
     [updaterInWindow]="scripts"
     [updaterLadderSummary]="scripts"
     [updaterLastUpdateFrom]="scripts"
+    [updaterNewerVersionByList]="scripts"
+    [updaterNewerVersionByProbe]="scripts"
     [updaterNewerVersionTag]="scripts"
     [_updaterPrimaryContainer]="scripts"
     [updaterPrimaryImage]="scripts"
@@ -2157,6 +2168,7 @@ declare -gA LP_FN_ROOT=(
     [updaterRollbackApp]="scripts"
     [updaterSetAnchorRef]="scripts"
     [updaterSetAnchorVersion]="scripts"
+    [updaterTagBumpAt]="scripts"
     [updaterTagExists]="scripts"
     [updaterTagGreater]="scripts"
     [updaterTagIncrement]="scripts"
@@ -2655,6 +2667,8 @@ cliTaskRun() { unset -f cliTaskRun; __lpAutoload "${install_scripts_dir}cli/task
 cliUpdateCommands() { unset -f cliUpdateCommands; __lpAutoload "${install_scripts_dir}cli/cli_update.sh"; cliUpdateCommands "$@"; }
 cliWebuiLoginReset() { unset -f cliWebuiLoginReset; __lpAutoload "${install_scripts_dir}cli/commands/webui/cli_webui_commands.sh"; cliWebuiLoginReset "$@"; }
 completeMessage() { unset -f completeMessage; __lpAutoload "${install_scripts_dir}menu/message/complete.sh"; completeMessage "$@"; }
+configBackfillAllApps() { unset -f configBackfillAllApps; __lpAutoload "${install_scripts_dir}config/docker/config_backfill_keys.sh"; configBackfillAllApps "$@"; }
+configBackfillMissingKeys() { unset -f configBackfillMissingKeys; __lpAutoload "${install_scripts_dir}config/docker/config_backfill_keys.sh"; configBackfillMissingKeys "$@"; }
 configSetupFileWithData() { unset -f configSetupFileWithData; __lpAutoload "${install_scripts_dir}config/core/config_file_setup_data.sh"; configSetupFileWithData "$@"; }
 configUpdateBatch() { unset -f configUpdateBatch; __lpAutoload "${install_scripts_dir}config/config_update.sh"; configUpdateBatch "$@"; }
 containsElement() { unset -f containsElement; __lpAutoload "${install_scripts_dir}function/validation/email.sh"; containsElement "$@"; }
@@ -3214,6 +3228,7 @@ sourceBackupLocations() { unset -f sourceBackupLocations; __lpAutoload "${instal
 sshRemote() { unset -f sshRemote; __lpAutoload "${install_scripts_dir}network/ssh/ssh.sh"; sshRemote "$@"; }
 stalwart_apply_port_access() { unset -f stalwart_apply_port_access; __lpAutoload "${install_containers_dir}stalwart/scripts/stalwart_install_hooks.sh"; stalwart_apply_port_access "$@"; }
 stalwart_cli() { unset -f stalwart_cli; __lpAutoload "${install_containers_dir}stalwart/scripts/stalwart_install_hooks.sh"; stalwart_cli "$@"; }
+stalwart_http_code() { unset -f stalwart_http_code; __lpAutoload "${install_containers_dir}stalwart/scripts/stalwart_install_hooks.sh"; stalwart_http_code "$@"; }
 stalwart_install_dns_provider() { unset -f stalwart_install_dns_provider; __lpAutoload "${install_containers_dir}stalwart/scripts/stalwart_install_hooks.sh"; stalwart_install_dns_provider "$@"; }
 stalwart_install_first_mailbox() { unset -f stalwart_install_first_mailbox; __lpAutoload "${install_containers_dir}stalwart/scripts/stalwart_install_hooks.sh"; stalwart_install_first_mailbox "$@"; }
 stalwart_install_message_data() { unset -f stalwart_install_message_data; __lpAutoload "${install_containers_dir}stalwart/scripts/stalwart_install_hooks.sh"; stalwart_install_message_data "$@"; }
@@ -3306,6 +3321,8 @@ updaterInspectLocal() { unset -f updaterInspectLocal; __lpAutoload "${install_sc
 updaterInWindow() { unset -f updaterInWindow; __lpAutoload "${install_scripts_dir}cli/commands/updater/cli_updater_auto.sh"; updaterInWindow "$@"; }
 updaterLadderSummary() { unset -f updaterLadderSummary; __lpAutoload "${install_scripts_dir}cli/commands/updater/cli_updater_ladder.sh"; updaterLadderSummary "$@"; }
 updaterLastUpdateFrom() { unset -f updaterLastUpdateFrom; __lpAutoload "${install_scripts_dir}cli/commands/updater/cli_updater_commands.sh"; updaterLastUpdateFrom "$@"; }
+updaterNewerVersionByList() { unset -f updaterNewerVersionByList; __lpAutoload "${install_scripts_dir}webui/data/generators/updater/webui_updater_scan.sh"; updaterNewerVersionByList "$@"; }
+updaterNewerVersionByProbe() { unset -f updaterNewerVersionByProbe; __lpAutoload "${install_scripts_dir}webui/data/generators/updater/webui_updater_scan.sh"; updaterNewerVersionByProbe "$@"; }
 updaterNewerVersionTag() { unset -f updaterNewerVersionTag; __lpAutoload "${install_scripts_dir}webui/data/generators/updater/webui_updater_scan.sh"; updaterNewerVersionTag "$@"; }
 _updaterPrimaryContainer() { unset -f _updaterPrimaryContainer; __lpAutoload "${install_scripts_dir}cli/commands/updater/cli_updater_verify.sh"; _updaterPrimaryContainer "$@"; }
 updaterPrimaryImage() { unset -f updaterPrimaryImage; __lpAutoload "${install_scripts_dir}webui/data/generators/updater/webui_updater_scan.sh"; updaterPrimaryImage "$@"; }
@@ -3317,6 +3334,7 @@ updaterRepoTag() { unset -f updaterRepoTag; __lpAutoload "${install_scripts_dir}
 updaterRollbackApp() { unset -f updaterRollbackApp; __lpAutoload "${install_scripts_dir}cli/commands/updater/cli_updater_commands.sh"; updaterRollbackApp "$@"; }
 updaterSetAnchorRef() { unset -f updaterSetAnchorRef; __lpAutoload "${install_scripts_dir}cli/commands/updater/cli_updater_commands.sh"; updaterSetAnchorRef "$@"; }
 updaterSetAnchorVersion() { unset -f updaterSetAnchorVersion; __lpAutoload "${install_scripts_dir}cli/commands/updater/cli_updater_upgrade.sh"; updaterSetAnchorVersion "$@"; }
+updaterTagBumpAt() { unset -f updaterTagBumpAt; __lpAutoload "${install_scripts_dir}webui/data/generators/updater/webui_updater_scan.sh"; updaterTagBumpAt "$@"; }
 updaterTagExists() { unset -f updaterTagExists; __lpAutoload "${install_scripts_dir}cli/commands/updater/cli_updater_ladder.sh"; updaterTagExists "$@"; }
 updaterTagGreater() { unset -f updaterTagGreater; __lpAutoload "${install_scripts_dir}webui/data/generators/updater/webui_updater_scan.sh"; updaterTagGreater "$@"; }
 updaterTagIncrement() { unset -f updaterTagIncrement; __lpAutoload "${install_scripts_dir}cli/commands/updater/cli_updater_ladder.sh"; updaterTagIncrement "$@"; }

scripts/update/check_update.sh

@@ -63,6 +63,9 @@ webuiRunUpdate()
         lpFetchRelease "$lat" || { isError "Release fetch failed — install unchanged."; return 1; }
         isNotice "Redeploying LibrePortal with the new version..."
         dockerInstallApp "libreportal"
+        # Options added to any app in this release reach only fresh installs
+        # otherwise — this redeploys LibrePortal, not the other 30-odd apps.
+        configBackfillAllApps
         WEBUI_UPDATER_FORCE=1 webuiLibrePortalUpdate
         webuiSystemUpdateCheck "force"
         webuiSystemVerify "force"
@@ -114,6 +117,10 @@ webuiRunUpdate()
     isNotice "Redeploying LibrePortal with the new version..."
     dockerInstallApp "libreportal"
 
+    # Same as the release branch above: carry newly-added config options into
+    # apps that are already installed, since nothing else will.
+    configBackfillAllApps
+
     # Regenerate WebUI data (new version, configs, etc.) and clear the
     # out-of-date flag.
     WEBUI_UPDATER_FORCE=1 webuiLibrePortalUpdate