diff --git a/scripts/crontab/crontab_refresh.sh b/scripts/crontab/crontab_refresh.sh index 6115c73..7bcbcab 100755 --- a/scripts/crontab/crontab_refresh.sh +++ b/scripts/crontab/crontab_refresh.sh @@ -13,6 +13,7 @@ crontabRefresh() #crontabSetupTaskProcessor # Switched to Systemd crontabSetupSystemInfoUpdater; + crontabSetupBootAppReconcile; crontabSetupCheckProcessor; isSuccessful "Crontab refreshed successfully" diff --git a/scripts/crontab/system/crontab_boot_app_reconcile.sh b/scripts/crontab/system/crontab_boot_app_reconcile.sh new file mode 100644 index 0000000..bf3095c --- /dev/null +++ b/scripts/crontab/system/crontab_boot_app_reconcile.sh @@ -0,0 +1,80 @@ +#!/bin/bash + +# Boot-time app reconcile — bring every installed app's containers back up. +# --------------------------------------------------------------------------- +# WHY THIS EXISTS. Every app runs with restart:unless-stopped, and on a machine +# that shuts down (this is a laptop-class host, not a rack server) that policy +# holds a race it can lose. Under ROOTLESS docker the containers are ordinary +# processes in the user's session, and at shutdown systemd tears that session +# down in parallel with dockerd's own exit. An app that handles SIGTERM +# promptly — Prometheus is the best-behaved process on the box — exits while +# dockerd is still alive to record "stopped", and unless-stopped then means +# exactly what it says: not restarted at the next boot. Apps that exit slower, +# or only die when dockerd itself does, are recorded as running and come back. +# +# Observed twice, same victim both times: host shutdown 05:45:30, Prometheus +# stopped 05:45:22; host shutdown 04:42:05, Prometheus stopped 04:41:59. Eight +# and six seconds ahead of the teardown — first over the line, only loser. +# Which app loses is a scheduling accident; the graceful ones are simply the +# most likely, so "fix Prometheus's policy" would treat the sample, not the +# race. +# +# So at boot, once the rootless daemon answers, `compose up -d` every installed +# app. That is idempotent: running apps are untouched (compose sees no diff), +# stopped ones start, and dependency order is compose's problem, not ours. +# +# The one behavioural trade: an app someone deliberately stopped BEFORE +# rebooting comes back after the reboot. That is accepted — on a self-hosting +# box "the fleet is up after boot" is the promise the restart policy was +# already trying to make, and a stop that must survive reboots is what +# uninstall (or disabling the app) is for. + +script_boot_flag="$1" + +# Only run when executed directly, not when sourced (mirrors the task +# processor's guard so sourcing this file for its functions stays side-effect +# free). +if [[ "$script_boot_flag" == "start_script" ]]; then + +# --- Bootstrap: cron runs this standalone, same dance as the task processor -- +LP_SELF_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" 2>/dev/null && pwd)" +LP_SCRIPTS="${install_scripts_dir:-$(cd "$LP_SELF_DIR/../../.." 2>/dev/null && pwd)/scripts/}" +[[ -f "${LP_SCRIPTS}source/paths.sh" ]] && source "${LP_SCRIPTS}source/paths.sh" +LP_SCRIPTS="${install_scripts_dir:-$LP_SCRIPTS}" +LP_DOCKER_CFG="${configs_dir:-/libreportal-system/configs/}general/general_docker_install" +[[ -f "$LP_DOCKER_CFG" ]] && \ + eval "$(grep -E '^CFG_DOCKER_INSTALL_(TYPE|USER)=' "$LP_DOCKER_CFG" | sed 's/[[:space:]]*#.*//')" +: "${sudo_user_name:=libreportal}" +: "${containers_dir:=/libreportal-containers/}" +: "${docker_dir:=/libreportal-system}" +for _lp_f in docker/command/run_privileged.sh \ + docker/command/docker_run_install.sh \ + checks/requirements/check_install_type.sh \ + source/files/arrays/function_manifest.sh; do + [[ -f "${LP_SCRIPTS}${_lp_f}" ]] && source "${LP_SCRIPTS}${_lp_f}" +done +command -v resolveDockerInstallUser >/dev/null 2>&1 && resolveDockerInstallUser + +# Minimal logging shims when the full CLI helpers are not loaded. +command -v isNotice >/dev/null 2>&1 || isNotice() { echo "[boot-reconcile] $*"; } +command -v isSuccessful >/dev/null 2>&1 || isSuccessful() { echo "[boot-reconcile] $*"; } +command -v isError >/dev/null 2>&1 || isError() { echo "[boot-reconcile] $*" >&2; } + +# --- Wait for the rootless daemon; it starts with the user session and can be +# a while behind cron's @reboot. Give up after 5 minutes rather than hang a +# boot-scoped job forever — the next manual `libreportal start` still works. +_lp_waited=0 +until dockerCommandRun "docker info" >/dev/null 2>&1; do + sleep 5 + _lp_waited=$(( _lp_waited + 5 )) + if (( _lp_waited >= 300 )); then + isError "Docker daemon not up after ${_lp_waited}s — skipping boot app reconcile." + exit 0 + fi +done + +isNotice "Docker up after ~${_lp_waited}s — reconciling installed apps." +declare -F dockerComposeUpAllApps >/dev/null 2>&1 && dockerComposeUpAllApps >> "${docker_dir}/logs/boot_reconcile.log" 2>&1 +isSuccessful "Boot app reconcile finished." + +fi diff --git a/scripts/crontab/system/crontab_setup_system_info_updater.sh b/scripts/crontab/system/crontab_setup_system_info_updater.sh index 579730d..cfafa9f 100755 --- a/scripts/crontab/system/crontab_setup_system_info_updater.sh +++ b/scripts/crontab/system/crontab_setup_system_info_updater.sh @@ -13,3 +13,19 @@ crontabSetupSystemInfoUpdater() isSuccessful "System info updater added to crontab (every 1 minute)." fi } + + +# @reboot: bring every installed app back up once the rootless daemon answers. +# Exists because restart:unless-stopped loses a shutdown race under rootless +# docker — see crontab_boot_app_reconcile.sh for the full account. +crontabSetupBootAppReconcile() +{ + local cronEntry="@reboot ${install_scripts_dir}crontab/system/crontab_boot_app_reconcile.sh start_script >/dev/null 2>&1" + + if runAsManager crontab -l 2>/dev/null | grep -q "crontab_boot_app_reconcile.sh"; then + isNotice "Boot app reconcile already in crontab" + else + (runAsManager crontab -l 2>/dev/null; echo "$cronEntry") | runAsManager crontab - + isSuccessful "Boot app reconcile added to crontab (@reboot)." + fi +} diff --git a/scripts/source/files/arrays/function_manifest.sh b/scripts/source/files/arrays/function_manifest.sh index c59df68..e376820 100644 --- a/scripts/source/files/arrays/function_manifest.sh +++ b/scripts/source/files/arrays/function_manifest.sh @@ -394,6 +394,7 @@ declare -gA LP_FN_MAP=( [crontabRefresh]="crontab/crontab_refresh.sh" [crontabSetup]="crontab/crontab_setup.sh" [crontabSetupBackupScheduler]="crontab/app/crontab_backup_scheduler.sh" + [crontabSetupBootAppReconcile]="crontab/system/crontab_setup_system_info_updater.sh" [crontabSetupCheckProcessor]="task/crontab_setup_check_processor.sh" [crontabSetupSystemInfoUpdater]="crontab/system/crontab_setup_system_info_updater.sh" [crontabSetupTaskProcessor]="task/crontab_setup_task_processor.sh" @@ -1572,6 +1573,7 @@ declare -gA LP_FN_ROOT=( [crontabRefresh]="scripts" [crontabSetup]="scripts" [crontabSetupBackupScheduler]="scripts" + [crontabSetupBootAppReconcile]="scripts" [crontabSetupCheckProcessor]="scripts" [crontabSetupSystemInfoUpdater]="scripts" [crontabSetupTaskProcessor]="scripts" @@ -2373,6 +2375,7 @@ LP_EAGER_FILES=( "scripts:backup/db/backup_db.sh" "scripts:backup/files/backup_files.sh" "scripts:catalog/catalog_sources.sh" + "scripts:crontab/system/crontab_boot_app_reconcile.sh" "scripts:docker/install/rootless/rootless_apparmor.sh" "scripts:docker/type_switcher/swap_docker_type.sh" "containers:matrix/scripts/matrix_auth.sh" @@ -2786,6 +2789,7 @@ crontabClear() { unset -f crontabClear; __lpAutoload "${install_scripts_dir}cron crontabRefresh() { unset -f crontabRefresh; __lpAutoload "${install_scripts_dir}crontab/crontab_refresh.sh"; crontabRefresh "$@"; } crontabSetup() { unset -f crontabSetup; __lpAutoload "${install_scripts_dir}crontab/crontab_setup.sh"; crontabSetup "$@"; } crontabSetupBackupScheduler() { unset -f crontabSetupBackupScheduler; __lpAutoload "${install_scripts_dir}crontab/app/crontab_backup_scheduler.sh"; crontabSetupBackupScheduler "$@"; } +crontabSetupBootAppReconcile() { unset -f crontabSetupBootAppReconcile; __lpAutoload "${install_scripts_dir}crontab/system/crontab_setup_system_info_updater.sh"; crontabSetupBootAppReconcile "$@"; } crontabSetupCheckProcessor() { unset -f crontabSetupCheckProcessor; __lpAutoload "${install_scripts_dir}task/crontab_setup_check_processor.sh"; crontabSetupCheckProcessor "$@"; } crontabSetupSystemInfoUpdater() { unset -f crontabSetupSystemInfoUpdater; __lpAutoload "${install_scripts_dir}crontab/system/crontab_setup_system_info_updater.sh"; crontabSetupSystemInfoUpdater "$@"; } crontabSetupTaskProcessor() { unset -f crontabSetupTaskProcessor; __lpAutoload "${install_scripts_dir}task/crontab_setup_task_processor.sh"; crontabSetupTaskProcessor "$@"; }