From 0e98988fcb5175d1c4dc646f41ff6ef56610fb35 Mon Sep 17 00:00:00 2001 From: librelad Date: Fri, 21 Aug 2026 00:49:51 +0100 Subject: [PATCH] =?UTF-8?q?fix(boot):=20reconcile=20apps=20at=20startup=20?= =?UTF-8?q?=E2=80=94=20unless-stopped=20loses=20a=20shutdown=20race?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Prometheus kept being found stopped after boots, always Exited(0), always alone. The journal settles it: both stops sit seconds before a host shutdown boundary — container stopped 05:45:22, boot ended 05:45:30; stopped 04:41:59, boot ended 04:42:05. This is a laptop-class host that gets shut down, and under ROOTLESS docker the containers are ordinary processes in the user session, torn down by systemd in parallel with dockerd's own exit. That parallelism is the race. An app that handles SIGTERM promptly exits while dockerd is still alive to record "stopped" — and unless-stopped then means what it says: not restarted at the next boot. Apps that exit slower, or die only when dockerd does, are recorded as running and come back. Prometheus loses reliably because it is the best-behaved process on the box ("See you next time!"), but which app loses is a scheduling accident — changing Prometheus's restart policy would treat the sample, not the race. So an @reboot crontab entry now waits for the rootless daemon (up to five minutes, then gives up rather than hang) and `compose up -d`s every installed app via the existing dockerComposeUpAllApps. Idempotent: running apps see no diff, stopped ones start, ordering is compose's problem. Registered through crontabRefresh like the other entries, and installed on this box. The accepted trade, stated rather than hidden: an app deliberately stopped before a reboot comes back after it. On a self-hosting box "the fleet is up after boot" is the promise unless-stopped was already trying to make; a stop that must survive reboots is what uninstall is for. Verified by direct execution: daemon answered immediately, all installed apps reconciled, running containers untouched. Co-Authored-By: Claude Opus 5 --- scripts/crontab/crontab_refresh.sh | 1 + .../system/crontab_boot_app_reconcile.sh | 80 +++++++++++++++++++ .../crontab_setup_system_info_updater.sh | 16 ++++ .../source/files/arrays/function_manifest.sh | 4 + 4 files changed, 101 insertions(+) create mode 100644 scripts/crontab/system/crontab_boot_app_reconcile.sh diff --git a/scripts/crontab/crontab_refresh.sh b/scripts/crontab/crontab_refresh.sh index 6115c73..7bcbcab 100755 --- a/scripts/crontab/crontab_refresh.sh +++ b/scripts/crontab/crontab_refresh.sh @@ -13,6 +13,7 @@ crontabRefresh() #crontabSetupTaskProcessor # Switched to Systemd crontabSetupSystemInfoUpdater; + crontabSetupBootAppReconcile; crontabSetupCheckProcessor; isSuccessful "Crontab refreshed successfully" diff --git a/scripts/crontab/system/crontab_boot_app_reconcile.sh b/scripts/crontab/system/crontab_boot_app_reconcile.sh new file mode 100644 index 0000000..bf3095c --- /dev/null +++ b/scripts/crontab/system/crontab_boot_app_reconcile.sh @@ -0,0 +1,80 @@ +#!/bin/bash + +# Boot-time app reconcile — bring every installed app's containers back up. +# --------------------------------------------------------------------------- +# WHY THIS EXISTS. Every app runs with restart:unless-stopped, and on a machine +# that shuts down (this is a laptop-class host, not a rack server) that policy +# holds a race it can lose. Under ROOTLESS docker the containers are ordinary +# processes in the user's session, and at shutdown systemd tears that session +# down in parallel with dockerd's own exit. An app that handles SIGTERM +# promptly — Prometheus is the best-behaved process on the box — exits while +# dockerd is still alive to record "stopped", and unless-stopped then means +# exactly what it says: not restarted at the next boot. Apps that exit slower, +# or only die when dockerd itself does, are recorded as running and come back. +# +# Observed twice, same victim both times: host shutdown 05:45:30, Prometheus +# stopped 05:45:22; host shutdown 04:42:05, Prometheus stopped 04:41:59. Eight +# and six seconds ahead of the teardown — first over the line, only loser. +# Which app loses is a scheduling accident; the graceful ones are simply the +# most likely, so "fix Prometheus's policy" would treat the sample, not the +# race. +# +# So at boot, once the rootless daemon answers, `compose up -d` every installed +# app. That is idempotent: running apps are untouched (compose sees no diff), +# stopped ones start, and dependency order is compose's problem, not ours. +# +# The one behavioural trade: an app someone deliberately stopped BEFORE +# rebooting comes back after the reboot. That is accepted — on a self-hosting +# box "the fleet is up after boot" is the promise the restart policy was +# already trying to make, and a stop that must survive reboots is what +# uninstall (or disabling the app) is for. + +script_boot_flag="$1" + +# Only run when executed directly, not when sourced (mirrors the task +# processor's guard so sourcing this file for its functions stays side-effect +# free). +if [[ "$script_boot_flag" == "start_script" ]]; then + +# --- Bootstrap: cron runs this standalone, same dance as the task processor -- +LP_SELF_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" 2>/dev/null && pwd)" +LP_SCRIPTS="${install_scripts_dir:-$(cd "$LP_SELF_DIR/../../.." 2>/dev/null && pwd)/scripts/}" +[[ -f "${LP_SCRIPTS}source/paths.sh" ]] && source "${LP_SCRIPTS}source/paths.sh" +LP_SCRIPTS="${install_scripts_dir:-$LP_SCRIPTS}" +LP_DOCKER_CFG="${configs_dir:-/libreportal-system/configs/}general/general_docker_install" +[[ -f "$LP_DOCKER_CFG" ]] && \ + eval "$(grep -E '^CFG_DOCKER_INSTALL_(TYPE|USER)=' "$LP_DOCKER_CFG" | sed 's/[[:space:]]*#.*//')" +: "${sudo_user_name:=libreportal}" +: "${containers_dir:=/libreportal-containers/}" +: "${docker_dir:=/libreportal-system}" +for _lp_f in docker/command/run_privileged.sh \ + docker/command/docker_run_install.sh \ + checks/requirements/check_install_type.sh \ + source/files/arrays/function_manifest.sh; do + [[ -f "${LP_SCRIPTS}${_lp_f}" ]] && source "${LP_SCRIPTS}${_lp_f}" +done +command -v resolveDockerInstallUser >/dev/null 2>&1 && resolveDockerInstallUser + +# Minimal logging shims when the full CLI helpers are not loaded. +command -v isNotice >/dev/null 2>&1 || isNotice() { echo "[boot-reconcile] $*"; } +command -v isSuccessful >/dev/null 2>&1 || isSuccessful() { echo "[boot-reconcile] $*"; } +command -v isError >/dev/null 2>&1 || isError() { echo "[boot-reconcile] $*" >&2; } + +# --- Wait for the rootless daemon; it starts with the user session and can be +# a while behind cron's @reboot. Give up after 5 minutes rather than hang a +# boot-scoped job forever — the next manual `libreportal start` still works. +_lp_waited=0 +until dockerCommandRun "docker info" >/dev/null 2>&1; do + sleep 5 + _lp_waited=$(( _lp_waited + 5 )) + if (( _lp_waited >= 300 )); then + isError "Docker daemon not up after ${_lp_waited}s — skipping boot app reconcile." + exit 0 + fi +done + +isNotice "Docker up after ~${_lp_waited}s — reconciling installed apps." +declare -F dockerComposeUpAllApps >/dev/null 2>&1 && dockerComposeUpAllApps >> "${docker_dir}/logs/boot_reconcile.log" 2>&1 +isSuccessful "Boot app reconcile finished." + +fi diff --git a/scripts/crontab/system/crontab_setup_system_info_updater.sh b/scripts/crontab/system/crontab_setup_system_info_updater.sh index 579730d..cfafa9f 100755 --- a/scripts/crontab/system/crontab_setup_system_info_updater.sh +++ b/scripts/crontab/system/crontab_setup_system_info_updater.sh @@ -13,3 +13,19 @@ crontabSetupSystemInfoUpdater() isSuccessful "System info updater added to crontab (every 1 minute)." fi } + + +# @reboot: bring every installed app back up once the rootless daemon answers. +# Exists because restart:unless-stopped loses a shutdown race under rootless +# docker — see crontab_boot_app_reconcile.sh for the full account. +crontabSetupBootAppReconcile() +{ + local cronEntry="@reboot ${install_scripts_dir}crontab/system/crontab_boot_app_reconcile.sh start_script >/dev/null 2>&1" + + if runAsManager crontab -l 2>/dev/null | grep -q "crontab_boot_app_reconcile.sh"; then + isNotice "Boot app reconcile already in crontab" + else + (runAsManager crontab -l 2>/dev/null; echo "$cronEntry") | runAsManager crontab - + isSuccessful "Boot app reconcile added to crontab (@reboot)." + fi +} diff --git a/scripts/source/files/arrays/function_manifest.sh b/scripts/source/files/arrays/function_manifest.sh index c59df68..e376820 100644 --- a/scripts/source/files/arrays/function_manifest.sh +++ b/scripts/source/files/arrays/function_manifest.sh @@ -394,6 +394,7 @@ declare -gA LP_FN_MAP=( [crontabRefresh]="crontab/crontab_refresh.sh" [crontabSetup]="crontab/crontab_setup.sh" [crontabSetupBackupScheduler]="crontab/app/crontab_backup_scheduler.sh" + [crontabSetupBootAppReconcile]="crontab/system/crontab_setup_system_info_updater.sh" [crontabSetupCheckProcessor]="task/crontab_setup_check_processor.sh" [crontabSetupSystemInfoUpdater]="crontab/system/crontab_setup_system_info_updater.sh" [crontabSetupTaskProcessor]="task/crontab_setup_task_processor.sh" @@ -1572,6 +1573,7 @@ declare -gA LP_FN_ROOT=( [crontabRefresh]="scripts" [crontabSetup]="scripts" [crontabSetupBackupScheduler]="scripts" + [crontabSetupBootAppReconcile]="scripts" [crontabSetupCheckProcessor]="scripts" [crontabSetupSystemInfoUpdater]="scripts" [crontabSetupTaskProcessor]="scripts" @@ -2373,6 +2375,7 @@ LP_EAGER_FILES=( "scripts:backup/db/backup_db.sh" "scripts:backup/files/backup_files.sh" "scripts:catalog/catalog_sources.sh" + "scripts:crontab/system/crontab_boot_app_reconcile.sh" "scripts:docker/install/rootless/rootless_apparmor.sh" "scripts:docker/type_switcher/swap_docker_type.sh" "containers:matrix/scripts/matrix_auth.sh" @@ -2786,6 +2789,7 @@ crontabClear() { unset -f crontabClear; __lpAutoload "${install_scripts_dir}cron crontabRefresh() { unset -f crontabRefresh; __lpAutoload "${install_scripts_dir}crontab/crontab_refresh.sh"; crontabRefresh "$@"; } crontabSetup() { unset -f crontabSetup; __lpAutoload "${install_scripts_dir}crontab/crontab_setup.sh"; crontabSetup "$@"; } crontabSetupBackupScheduler() { unset -f crontabSetupBackupScheduler; __lpAutoload "${install_scripts_dir}crontab/app/crontab_backup_scheduler.sh"; crontabSetupBackupScheduler "$@"; } +crontabSetupBootAppReconcile() { unset -f crontabSetupBootAppReconcile; __lpAutoload "${install_scripts_dir}crontab/system/crontab_setup_system_info_updater.sh"; crontabSetupBootAppReconcile "$@"; } crontabSetupCheckProcessor() { unset -f crontabSetupCheckProcessor; __lpAutoload "${install_scripts_dir}task/crontab_setup_check_processor.sh"; crontabSetupCheckProcessor "$@"; } crontabSetupSystemInfoUpdater() { unset -f crontabSetupSystemInfoUpdater; __lpAutoload "${install_scripts_dir}crontab/system/crontab_setup_system_info_updater.sh"; crontabSetupSystemInfoUpdater "$@"; } crontabSetupTaskProcessor() { unset -f crontabSetupTaskProcessor; __lpAutoload "${install_scripts_dir}task/crontab_setup_task_processor.sh"; crontabSetupTaskProcessor "$@"; }