diff --git a/scripts/config/docker/config_backfill_keys.sh b/scripts/config/docker/config_backfill_keys.sh new file mode 100644 index 0000000..b0c5c72 --- /dev/null +++ b/scripts/config/docker/config_backfill_keys.sh @@ -0,0 +1,154 @@ +#!/bin/bash + +# Bring config options added to an app's template into an already-deployed copy. +# +# The deployed config at /docker/containers//.config is written once, +# on first install, and never touched again — dockerConfigSetupToContainer only +# copies when the file is absent, precisely so a LibrePortal update can never +# overwrite the values someone has edited. That is the right default, but it has +# a consequence nobody chose: an app that gains a CFG_ option in a new release +# has that option on every FRESH install and on no EXISTING one. +# +# The failure is silent, which is the worst part. Nothing errors. The new key +# simply reads as empty, and whatever depends on it does something quietly +# different — a default it did not mean to take, or a value it needed and did +# not get. It only surfaces as "why does this work on my other box". +# +# So: copy across keys the template has and the deployed file does not, and +# nothing else. +# +# Deliberately NOT a merge or a regenerate. Rebuilding the deployed file from +# the template would place new keys in their proper section and refresh the +# documentation with them, which is genuinely nicer to read — but it would put a +# whole-file rewrite of every app's config in the path of every app action, and +# the worst case of a bug there is silently corrupting settings across the whole +# install. Appending cannot lose an existing line. That trade is not close. +# +# Existing keys are never touched, and keys the deployed file has but the +# template no longer does are left exactly where they are: a removed option is +# usually a rename, and deleting the user's value is not recoverable. + +configBackfillMissingKeys() +{ + local app_name="$1" + local template="$2" + local deployed="$3" + local silent_flag="${4:-silent}" + + [[ -f "$template" ]] || return 0 + [[ -f "$deployed" ]] || return 0 + + # Same file, or a fresh copy of it — nothing can be missing. + runFileOp cmp -s "$template" "$deployed" && return 0 + + local template_keys deployed_keys missing + template_keys=$(grep -oE '^CFG_[A-Z0-9_]+=' "$template" 2>/dev/null | sed 's/=$//' | sort -u) + deployed_keys=$(runFileOp grep -oE '^CFG_[A-Z0-9_]+=' "$deployed" 2>/dev/null | sed 's/=$//' | sort -u) + + [[ -z "$template_keys" ]] && return 0 + + missing=$(comm -23 <(printf '%s\n' "$template_keys") <(printf '%s\n' "$deployed_keys")) + [[ -z "$missing" ]] && return 0 + + # Carry each key's comment block over with it. A bare `CFG_X=value` appended + # to the end of a heavily-documented file is an option nobody can act on; + # the comment above it in the template is the only explanation that exists. + local missing_csv + missing_csv=$(printf '%s\n' "$missing" | paste -sd, -) + + local block + block=$(awk -v keys="$missing_csv" ' + BEGIN { n = split(keys, K, ","); for (i = 1; i <= n; i++) want[K[i]] = 1 } + # Accumulate the contiguous comment block sitting directly above a key. + /^[[:space:]]*#/ { buf = buf $0 "\n"; next } + # A blank line ends a block — the comments above it belong to something else. + /^[[:space:]]*$/ { buf = ""; next } + /^CFG_[A-Z0-9_]+=/ { + k = $0; sub(/=.*/, "", k) + if (k in want) printf "%s%s\n\n", buf, $0 + buf = "" + next + } + { buf = "" } + ' "$template") + + [[ -z "$block" ]] && return 0 + + { + printf '\n#\n' + printf '# =============================================================================\n' + printf '# ADDED BY A LIBREPORTAL UPDATE\n' + printf '# =============================================================================\n' + printf '# These options did not exist when this app was installed. They are set to the\n' + printf '# defaults shipped with the new version — review them, they are yours to change.\n' + printf '#\n' + printf '%s\n' "$block" + } | runFileWrite -a "$deployed" + + local count + count=$(printf '%s\n' "$missing" | grep -c .) + # Counter rather than a return code: the callers use this as a bare + # statement, and a non-zero "nothing to do" would read as a failure to any + # of them running under errexit. + LP_BACKFILL_ADDED=$(( ${LP_BACKFILL_ADDED:-0} + count )) + + if [[ "$silent_flag" == "loud" ]]; then + isSuccessful "Added $count new config option(s) to $app_name:" + printf '%s\n' "$missing" | sed 's/^/ /' + else + isNotice "Added $count new config option(s) to $app_name's config." + fi + + return 0 +} + +# Sweep every installed app after a LibrePortal update. +# +# The per-app backfill above only fires when an app's config is set up, which +# happens on install — and an update installs exactly one app, LibrePortal +# itself. So without this sweep, an option added to some app in a new release +# reaches that app only when someone next reinstalls it, which for a working app +# may be never. Since the whole point is that new options arrive on upgrade, +# upgrade is where this has to run. +configBackfillAllApps() +{ + local dir app template deployed + LP_BACKFILL_ADDED=0 + + # Driven from the template directory, not from the deployed one. Under + # rootless the container tree is mode drwxr-x--x and owned by the docker + # install user, so the manager running this can traverse it but cannot LIST + # it — a glob over it silently expands to nothing and the sweep would report + # success having examined no apps at all. The template dir is manager-owned + # and readable, and it defines the same set of apps; whether each one is + # actually installed is then just "does its deployed config exist", which is + # a traverse, not a list. + for dir in "$install_containers_dir"*/; do + [[ -d "$dir" ]] || continue + app="${dir%/}"; app="${app##*/}" + [[ "$app" == "template" ]] && continue + + template="${dir}${app}.config" + deployed="${containers_dir}${app}/${app}.config" + [[ -f "$template" ]] || continue + runFileOp test -f "$deployed" || continue + + configBackfillMissingKeys "$app" "$template" "$deployed" "silent" + + # A backfilled key whose default is a RANDOMIZED* placeholder has to be + # given a real value here — nothing else will run over this file until + # the app is next installed, and a placeholder left in place is a + # credential that is identical on every install that took this upgrade. + if runFileOp grep -qE 'RANDOMIZED(PASSWORD|USERNAME|BCRYPTPASSWORD|HEX|VAPID|APPKEY)[0-9]*' "$deployed" 2>/dev/null; then + scanFileForRandomPasswordKeysUsers "$deployed" + fi + done + + if (( ${LP_BACKFILL_ADDED:-0} > 0 )); then + isSuccessful "Carried $LP_BACKFILL_ADDED new config option(s) into your existing apps." + isNotice " They are set to the defaults shipped with this version — review them" + isNotice " in each app's Config tab." + fi + + return 0 +} diff --git a/scripts/config/docker/docker_config_to_container.sh b/scripts/config/docker/docker_config_to_container.sh index 7098dfe..8321be7 100755 --- a/scripts/config/docker/docker_config_to_container.sh +++ b/scripts/config/docker/docker_config_to_container.sh @@ -50,6 +50,16 @@ dockerConfigSetupToContainer() isNotice "Copying config file to '$target_path/$config_file'..." fi copyFile "$silent_flag" "$source_file" "$target_path/$config_file" $sudo_user_name | runInstallWrite -a "$logs_dir/$docker_log_file" 2>&1 + else + # The file already exists, so the copy above was skipped — which is what + # protects the user's edited values from every subsequent update. The + # cost is that options ADDED to the template since this app was installed + # would never arrive. Carry those across; existing values are untouched. + # + # Runs before the RANDOMIZED* pass below on purpose: a newly-added key + # whose default is a RANDOMIZEDPASSWORD placeholder gets a real + # generated value from that pass, exactly as it would on a fresh install. + configBackfillMissingKeys "$app_name" "$source_file" "$target_path/$config_file" "$silent_flag" fi if [[ -n "$config_overrides" ]]; then diff --git a/scripts/source/files/arrays/files_config.sh b/scripts/source/files/arrays/files_config.sh index c309e41..834871b 100755 --- a/scripts/source/files/arrays/files_config.sh +++ b/scripts/source/files/arrays/files_config.sh @@ -18,6 +18,7 @@ config_scripts=( "config/core/config_update_option.sh" "config/core/variables/config_missing_variables.sh" "config/core/variables/config_scan_variables.sh" + "config/docker/config_backfill_keys.sh" "config/docker/docker_compose_menu.sh" "config/docker/docker_config_setup_data.sh" "config/docker/docker_config_to_container.sh" diff --git a/scripts/source/files/arrays/function_manifest.sh b/scripts/source/files/arrays/function_manifest.sh index 4187e54..6f327a3 100644 --- a/scripts/source/files/arrays/function_manifest.sh +++ b/scripts/source/files/arrays/function_manifest.sh @@ -370,6 +370,8 @@ declare -gA LP_FN_MAP=( [cliUpdateCommands]="cli/cli_update.sh" [cliWebuiLoginReset]="cli/commands/webui/cli_webui_commands.sh" [completeMessage]="menu/message/complete.sh" + [configBackfillAllApps]="config/docker/config_backfill_keys.sh" + [configBackfillMissingKeys]="config/docker/config_backfill_keys.sh" [configSetupFileWithData]="config/core/config_file_setup_data.sh" [configUpdateBatch]="config/config_update.sh" [containsElement]="function/validation/email.sh" @@ -929,6 +931,7 @@ declare -gA LP_FN_MAP=( [sshRemote]="network/ssh/ssh.sh" [stalwart_apply_port_access]="stalwart/scripts/stalwart_install_hooks.sh" [stalwart_cli]="stalwart/scripts/stalwart_install_hooks.sh" + [stalwart_http_code]="stalwart/scripts/stalwart_install_hooks.sh" [stalwart_install_dns_provider]="stalwart/scripts/stalwart_install_hooks.sh" [stalwart_install_first_mailbox]="stalwart/scripts/stalwart_install_hooks.sh" [stalwart_install_message_data]="stalwart/scripts/stalwart_install_hooks.sh" @@ -1021,6 +1024,8 @@ declare -gA LP_FN_MAP=( [updaterInWindow]="cli/commands/updater/cli_updater_auto.sh" [updaterLadderSummary]="cli/commands/updater/cli_updater_ladder.sh" [updaterLastUpdateFrom]="cli/commands/updater/cli_updater_commands.sh" + [updaterNewerVersionByList]="webui/data/generators/updater/webui_updater_scan.sh" + [updaterNewerVersionByProbe]="webui/data/generators/updater/webui_updater_scan.sh" [updaterNewerVersionTag]="webui/data/generators/updater/webui_updater_scan.sh" [_updaterPrimaryContainer]="cli/commands/updater/cli_updater_verify.sh" [updaterPrimaryImage]="webui/data/generators/updater/webui_updater_scan.sh" @@ -1032,6 +1037,7 @@ declare -gA LP_FN_MAP=( [updaterRollbackApp]="cli/commands/updater/cli_updater_commands.sh" [updaterSetAnchorRef]="cli/commands/updater/cli_updater_commands.sh" [updaterSetAnchorVersion]="cli/commands/updater/cli_updater_upgrade.sh" + [updaterTagBumpAt]="webui/data/generators/updater/webui_updater_scan.sh" [updaterTagExists]="cli/commands/updater/cli_updater_ladder.sh" [updaterTagGreater]="webui/data/generators/updater/webui_updater_scan.sh" [updaterTagIncrement]="cli/commands/updater/cli_updater_ladder.sh" @@ -1495,6 +1501,8 @@ declare -gA LP_FN_ROOT=( [cliUpdateCommands]="scripts" [cliWebuiLoginReset]="scripts" [completeMessage]="scripts" + [configBackfillAllApps]="scripts" + [configBackfillMissingKeys]="scripts" [configSetupFileWithData]="scripts" [configUpdateBatch]="scripts" [containsElement]="scripts" @@ -2054,6 +2062,7 @@ declare -gA LP_FN_ROOT=( [sshRemote]="scripts" [stalwart_apply_port_access]="containers" [stalwart_cli]="containers" + [stalwart_http_code]="containers" [stalwart_install_dns_provider]="containers" [stalwart_install_first_mailbox]="containers" [stalwart_install_message_data]="containers" @@ -2146,6 +2155,8 @@ declare -gA LP_FN_ROOT=( [updaterInWindow]="scripts" [updaterLadderSummary]="scripts" [updaterLastUpdateFrom]="scripts" + [updaterNewerVersionByList]="scripts" + [updaterNewerVersionByProbe]="scripts" [updaterNewerVersionTag]="scripts" [_updaterPrimaryContainer]="scripts" [updaterPrimaryImage]="scripts" @@ -2157,6 +2168,7 @@ declare -gA LP_FN_ROOT=( [updaterRollbackApp]="scripts" [updaterSetAnchorRef]="scripts" [updaterSetAnchorVersion]="scripts" + [updaterTagBumpAt]="scripts" [updaterTagExists]="scripts" [updaterTagGreater]="scripts" [updaterTagIncrement]="scripts" @@ -2655,6 +2667,8 @@ cliTaskRun() { unset -f cliTaskRun; __lpAutoload "${install_scripts_dir}cli/task cliUpdateCommands() { unset -f cliUpdateCommands; __lpAutoload "${install_scripts_dir}cli/cli_update.sh"; cliUpdateCommands "$@"; } cliWebuiLoginReset() { unset -f cliWebuiLoginReset; __lpAutoload "${install_scripts_dir}cli/commands/webui/cli_webui_commands.sh"; cliWebuiLoginReset "$@"; } completeMessage() { unset -f completeMessage; __lpAutoload "${install_scripts_dir}menu/message/complete.sh"; completeMessage "$@"; } +configBackfillAllApps() { unset -f configBackfillAllApps; __lpAutoload "${install_scripts_dir}config/docker/config_backfill_keys.sh"; configBackfillAllApps "$@"; } +configBackfillMissingKeys() { unset -f configBackfillMissingKeys; __lpAutoload "${install_scripts_dir}config/docker/config_backfill_keys.sh"; configBackfillMissingKeys "$@"; } configSetupFileWithData() { unset -f configSetupFileWithData; __lpAutoload "${install_scripts_dir}config/core/config_file_setup_data.sh"; configSetupFileWithData "$@"; } configUpdateBatch() { unset -f configUpdateBatch; __lpAutoload "${install_scripts_dir}config/config_update.sh"; configUpdateBatch "$@"; } containsElement() { unset -f containsElement; __lpAutoload "${install_scripts_dir}function/validation/email.sh"; containsElement "$@"; } @@ -3214,6 +3228,7 @@ sourceBackupLocations() { unset -f sourceBackupLocations; __lpAutoload "${instal sshRemote() { unset -f sshRemote; __lpAutoload "${install_scripts_dir}network/ssh/ssh.sh"; sshRemote "$@"; } stalwart_apply_port_access() { unset -f stalwart_apply_port_access; __lpAutoload "${install_containers_dir}stalwart/scripts/stalwart_install_hooks.sh"; stalwart_apply_port_access "$@"; } stalwart_cli() { unset -f stalwart_cli; __lpAutoload "${install_containers_dir}stalwart/scripts/stalwart_install_hooks.sh"; stalwart_cli "$@"; } +stalwart_http_code() { unset -f stalwart_http_code; __lpAutoload "${install_containers_dir}stalwart/scripts/stalwart_install_hooks.sh"; stalwart_http_code "$@"; } stalwart_install_dns_provider() { unset -f stalwart_install_dns_provider; __lpAutoload "${install_containers_dir}stalwart/scripts/stalwart_install_hooks.sh"; stalwart_install_dns_provider "$@"; } stalwart_install_first_mailbox() { unset -f stalwart_install_first_mailbox; __lpAutoload "${install_containers_dir}stalwart/scripts/stalwart_install_hooks.sh"; stalwart_install_first_mailbox "$@"; } stalwart_install_message_data() { unset -f stalwart_install_message_data; __lpAutoload "${install_containers_dir}stalwart/scripts/stalwart_install_hooks.sh"; stalwart_install_message_data "$@"; } @@ -3306,6 +3321,8 @@ updaterInspectLocal() { unset -f updaterInspectLocal; __lpAutoload "${install_sc updaterInWindow() { unset -f updaterInWindow; __lpAutoload "${install_scripts_dir}cli/commands/updater/cli_updater_auto.sh"; updaterInWindow "$@"; } updaterLadderSummary() { unset -f updaterLadderSummary; __lpAutoload "${install_scripts_dir}cli/commands/updater/cli_updater_ladder.sh"; updaterLadderSummary "$@"; } updaterLastUpdateFrom() { unset -f updaterLastUpdateFrom; __lpAutoload "${install_scripts_dir}cli/commands/updater/cli_updater_commands.sh"; updaterLastUpdateFrom "$@"; } +updaterNewerVersionByList() { unset -f updaterNewerVersionByList; __lpAutoload "${install_scripts_dir}webui/data/generators/updater/webui_updater_scan.sh"; updaterNewerVersionByList "$@"; } +updaterNewerVersionByProbe() { unset -f updaterNewerVersionByProbe; __lpAutoload "${install_scripts_dir}webui/data/generators/updater/webui_updater_scan.sh"; updaterNewerVersionByProbe "$@"; } updaterNewerVersionTag() { unset -f updaterNewerVersionTag; __lpAutoload "${install_scripts_dir}webui/data/generators/updater/webui_updater_scan.sh"; updaterNewerVersionTag "$@"; } _updaterPrimaryContainer() { unset -f _updaterPrimaryContainer; __lpAutoload "${install_scripts_dir}cli/commands/updater/cli_updater_verify.sh"; _updaterPrimaryContainer "$@"; } updaterPrimaryImage() { unset -f updaterPrimaryImage; __lpAutoload "${install_scripts_dir}webui/data/generators/updater/webui_updater_scan.sh"; updaterPrimaryImage "$@"; } @@ -3317,6 +3334,7 @@ updaterRepoTag() { unset -f updaterRepoTag; __lpAutoload "${install_scripts_dir} updaterRollbackApp() { unset -f updaterRollbackApp; __lpAutoload "${install_scripts_dir}cli/commands/updater/cli_updater_commands.sh"; updaterRollbackApp "$@"; } updaterSetAnchorRef() { unset -f updaterSetAnchorRef; __lpAutoload "${install_scripts_dir}cli/commands/updater/cli_updater_commands.sh"; updaterSetAnchorRef "$@"; } updaterSetAnchorVersion() { unset -f updaterSetAnchorVersion; __lpAutoload "${install_scripts_dir}cli/commands/updater/cli_updater_upgrade.sh"; updaterSetAnchorVersion "$@"; } +updaterTagBumpAt() { unset -f updaterTagBumpAt; __lpAutoload "${install_scripts_dir}webui/data/generators/updater/webui_updater_scan.sh"; updaterTagBumpAt "$@"; } updaterTagExists() { unset -f updaterTagExists; __lpAutoload "${install_scripts_dir}cli/commands/updater/cli_updater_ladder.sh"; updaterTagExists "$@"; } updaterTagGreater() { unset -f updaterTagGreater; __lpAutoload "${install_scripts_dir}webui/data/generators/updater/webui_updater_scan.sh"; updaterTagGreater "$@"; } updaterTagIncrement() { unset -f updaterTagIncrement; __lpAutoload "${install_scripts_dir}cli/commands/updater/cli_updater_ladder.sh"; updaterTagIncrement "$@"; } diff --git a/scripts/update/check_update.sh b/scripts/update/check_update.sh index 994fb66..9d99a32 100755 --- a/scripts/update/check_update.sh +++ b/scripts/update/check_update.sh @@ -63,6 +63,9 @@ webuiRunUpdate() lpFetchRelease "$lat" || { isError "Release fetch failed — install unchanged."; return 1; } isNotice "Redeploying LibrePortal with the new version..." dockerInstallApp "libreportal" + # Options added to any app in this release reach only fresh installs + # otherwise — this redeploys LibrePortal, not the other 30-odd apps. + configBackfillAllApps WEBUI_UPDATER_FORCE=1 webuiLibrePortalUpdate webuiSystemUpdateCheck "force" webuiSystemVerify "force" @@ -114,6 +117,10 @@ webuiRunUpdate() isNotice "Redeploying LibrePortal with the new version..." dockerInstallApp "libreportal" + # Same as the release branch above: carry newly-added config options into + # apps that are already installed, since nothing else will. + configBackfillAllApps + # Regenerate WebUI data (new version, configs, etc.) and clear the # out-of-date flag. WEBUI_UPDATER_FORCE=1 webuiLibrePortalUpdate