From ffc7801762b51ae2a2c9a0498a05e9e39cc12963 Mon Sep 17 00:00:00 2001 From: librelad Date: Mon, 6 Jul 2026 20:38:11 +0100 Subject: [PATCH 1/3] fix(network): auto-detect path MTU so large image pulls don't stall with EOF MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CFG_NETWORK_MTU=1500 was baked blindly into the rootless daemon uplink (DOCKERD_ROOTLESS_ROOTLESSKIT_MTU) and every app's container network. On links whose real path MTU is below 1500 (Qubes/NAT/VPN with PMTU discovery blocked), image manifests + tiny images pull fine but large image LAYERS stall and reset mid-blob with "httpReadSeeker: ... EOF" — apps silently fail to install. Probed here: path MTU ~1328, Docker at 1500 → black hole. - New scripts/network/network_mtu.sh: networkDetectMtu (don't-fragment ICMP ladder → largest standard MTU that gets through, 1500 when ICMP gives no signal) + networkEffectiveMtu (CFG_NETWORK_MTU: a number is verbatim, "auto" probes once and caches $docker_dir/.network_mtu) + networkRedetectMtu. - CFG_NETWORK_MTU default 1500 -> auto. Rootless setup now writes the resolved MTU into the override (and re-detects per install); the per-app NETWORK_MTU_TAG uses the resolved value too. Explicit numbers still win. Verified on this box: auto -> 1300; alpine (multi-MB) + the full ~100MB navidrome image now pull to completion where they previously EOF'd. Applied live (override 1300, config=auto, cache=1300, rootless docker restarted). Signed-off-by: librelad Co-Authored-By: Claude Opus 4.8 (1M context) --- configs/network/network_docker | 2 +- .../config/docker/docker_config_setup_data.sh | 2 +- .../install/rootless/rootless_docker.sh | 8 ++- scripts/network/network_mtu.sh | 68 +++++++++++++++++++ scripts/source/files/arrays/files_network.sh | 1 + .../source/files/arrays/function_manifest.sh | 12 ++++ 6 files changed, 90 insertions(+), 3 deletions(-) create mode 100644 scripts/network/network_mtu.sh diff --git a/configs/network/network_docker b/configs/network/network_docker index cedf78c..a3d9b30 100755 --- a/configs/network/network_docker +++ b/configs/network/network_docker @@ -4,7 +4,7 @@ CFG_NETWORK_NAME=vpn # Network Name - Docker network name for container communication CFG_NETWORK_SUBNET=10.100.0.0/16 # Network Subnet - Subnet range for Docker network -CFG_NETWORK_MTU=1500 # Network MTU - Maximum transmission unit for network packets +CFG_NETWORK_MTU=auto # Network MTU - Maximum transmission unit for network packets. 'auto' probes the real path MTU and clamps to a safe value (fixes large image pulls stalling with EOF on Qubes/NAT/VPN links where the path MTU is below 1500); or set an explicit number like 1500 or 1280. **ADVANCED** CFG_REQUIREMENT_DOCKER_NETWORK=true # Docker Network - Create and manage the Docker network for container communication CFG_REQUIREMENT_DOCKER_NETWORK_PRUNE=true # Network Cleanup - Automatically prune unused Docker networks CFG_REQUIREMENT_DOCKER_SWITCHER=true # Docker Switcher - Install the Docker version switching utility diff --git a/scripts/config/docker/docker_config_setup_data.sh b/scripts/config/docker/docker_config_setup_data.sh index b086792..e280751 100755 --- a/scripts/config/docker/docker_config_setup_data.sh +++ b/scripts/config/docker/docker_config_setup_data.sh @@ -93,7 +93,7 @@ dockerConfigSetupFileWithData() tagsManagerUpdateUniversalTag "$full_file_path" "DOCKER_NETWORK_TAG" "$CFG_NETWORK_NAME" tagsManagerUpdateUniversalTag "$full_file_path" "PUBLIC_IP_TAG" "$public_ip_v4" tagsManagerUpdateUniversalTag "$full_file_path" "NETWORK_SUBNET_TAG" "$CFG_NETWORK_SUBNET" - tagsManagerUpdateUniversalTag "$full_file_path" "NETWORK_MTU_TAG" "$CFG_NETWORK_MTU" + tagsManagerUpdateUniversalTag "$full_file_path" "NETWORK_MTU_TAG" "$(networkEffectiveMtu)" ipUpdateComposeTags "$app_name" "$full_file_path" portUpdateComposeTags "$app_name" "$full_file_path" tagsProcessorTrustedDomains "$full_file_path" diff --git a/scripts/docker/install/rootless/rootless_docker.sh b/scripts/docker/install/rootless/rootless_docker.sh index ecdbf65..3b11fac 100755 --- a/scripts/docker/install/rootless/rootless_docker.sh +++ b/scripts/docker/install/rootless/rootless_docker.sh @@ -144,11 +144,17 @@ EOF local result; result=$(sudo touch $override_conf_file) checkSuccess "Create the override.conf in docker.service.d" + # Resolve CFG_NETWORK_MTU to a concrete, path-safe value (auto = probe). + # This is the MTU the daemon PULLS over — a blind 1500 on a lower-path-MTU + # link (Qubes/NAT/VPN) black-holes large image-layer downloads (EOF). + local resolved_mtu; resolved_mtu="$(networkRedetectMtu)" + isNotice "Rootless uplink MTU: $resolved_mtu (CFG_NETWORK_MTU=${CFG_NETWORK_MTU:-auto})" + sudo bash -c "cat < '$override_conf_file' [Service] Environment='DOCKERD_ROOTLESS_ROOTLESSKIT_NET=$rootless_net' Environment='DOCKERD_ROOTLESS_ROOTLESSKIT_PORT_DRIVER=$rootless_port_driver' -Environment='DOCKERD_ROOTLESS_ROOTLESSKIT_MTU=$CFG_NETWORK_MTU' +Environment='DOCKERD_ROOTLESS_ROOTLESSKIT_MTU=$resolved_mtu' EOL" local result; result=$(sudo chown $CFG_DOCKER_INSTALL_USER:$CFG_DOCKER_INSTALL_USER $override_conf_file) diff --git a/scripts/network/network_mtu.sh b/scripts/network/network_mtu.sh new file mode 100644 index 0000000..61942e0 --- /dev/null +++ b/scripts/network/network_mtu.sh @@ -0,0 +1,68 @@ +#!/bin/bash +# +# Network MTU resolution. +# --------------------------------------------------------------------------- +# CFG_NETWORK_MTU drives BOTH the rootless daemon's uplink MTU +# (DOCKERD_ROOTLESS_ROOTLESSKIT_MTU — how image PULLS egress) and every app's +# container-network MTU (the NETWORK_MTU_TAG baked into each compose). A blind +# 1500 breaks on links whose real path MTU is lower (Qubes/NAT/VPN with PMTU +# discovery blocked): small transfers (image manifests, tiny images) succeed but +# large image-layer downloads stall and reset with "EOF" mid-blob. +# +# CFG_NETWORK_MTU accepts: +# — use exactly (explicit override). +# auto — probe the path MTU once, cache a safe value (default). +# +# The probe is a don't-fragment ICMP ladder. If ICMP is fully filtered (no probe +# gets through at all) we can't tell a clean link from a constrained one, so we +# assume the standard 1500 rather than needlessly shrinking every network — set +# an explicit number for that case. + +networkMtuCacheFile() { echo "${docker_dir%/}/.network_mtu"; } + +# Largest standard MTU whose don't-fragment probe reaches the internet. +# Echoes a number; 1500 when no signal (ping missing / all probes filtered). +networkDetectMtu() { + local host="${1:-1.1.1.1}" m best="" + command -v ping >/dev/null 2>&1 || { echo 1500; return 0; } + for m in 1500 1492 1400 1300 1280; do + # payload = mtu - 28 (IPv4 + ICMP headers) + if ping -M do -s $((m - 28)) -c1 -W2 "$host" >/dev/null 2>&1; then + best="$m"; break + fi + done + echo "${best:-1500}" +} + +# The concrete MTU to bake into the daemon + composes. Resolves CFG_NETWORK_MTU: +# a number is used verbatim; "auto"/empty is probed once and cached so we don't +# re-ping on every app install. +networkEffectiveMtu() { + local want="${CFG_NETWORK_MTU:-auto}" + if [[ "$want" =~ ^[0-9]+$ ]]; then echo "$want"; return 0; fi + + local cache cached + cache="$(networkMtuCacheFile)" + cached="$(cat "$cache" 2>/dev/null | tr -dc '0-9')" + if [[ -n "$cached" ]]; then echo "$cached"; return 0; fi + + local detected; detected="$(networkDetectMtu)" + if declare -F runInstallWrite >/dev/null 2>&1; then + printf '%s\n' "$detected" | runInstallWrite "$cache" 2>/dev/null || true + else + printf '%s\n' "$detected" > "$cache" 2>/dev/null || true + fi + echo "$detected" +} + +# Force a fresh probe (clears the cache), echoing the new value. The rootless +# setup calls this so each (re)install re-detects the current link. +networkRedetectMtu() { + local cache; cache="$(networkMtuCacheFile)" + if declare -F runInstallOp >/dev/null 2>&1; then + runInstallOp rm -f "$cache" 2>/dev/null || true + else + rm -f "$cache" 2>/dev/null || true + fi + networkEffectiveMtu +} diff --git a/scripts/source/files/arrays/files_network.sh b/scripts/source/files/arrays/files_network.sh index b1532f1..ce81d9e 100755 --- a/scripts/source/files/arrays/files_network.sh +++ b/scripts/source/files/arrays/files_network.sh @@ -28,6 +28,7 @@ network_scripts=( "network/ip/ip_remove_from_db.sh" "network/ip/ip_replace_tags.sh" "network/monitoring/monitoring.sh" + "network/network_mtu.sh" "network/ports/allocation/port_allocate.sh" "network/ports/allocation/port_store_mapping.sh" "network/ports/allocation/port_update_compose_tags.sh" diff --git a/scripts/source/files/arrays/function_manifest.sh b/scripts/source/files/arrays/function_manifest.sh index 114a472..0e61df8 100644 --- a/scripts/source/files/arrays/function_manifest.sh +++ b/scripts/source/files/arrays/function_manifest.sh @@ -681,7 +681,11 @@ declare -gA LP_FN_MAP=( [monitoringToggleAppConfig]="network/monitoring/monitoring.sh" [moveFile]="function/file/move_file.sh" [_netServiceIsRouted]="docker/network/network_conflicts.sh" + [networkDetectMtu]="network/network_mtu.sh" + [networkEffectiveMtu]="network/network_mtu.sh" [networkHealConflicts]="docker/network/network_heal.sh" + [networkMtuCacheFile]="network/network_mtu.sh" + [networkRedetectMtu]="network/network_mtu.sh" [networkScanConflicts]="docker/network/network_conflicts.sh" [_nextcloudOcc]="nextcloud/scripts/nextcloud_auth.sh" [_nextcloudOccWithPass]="nextcloud/scripts/nextcloud_auth.sh" @@ -1661,7 +1665,11 @@ declare -gA LP_FN_ROOT=( [monitoringToggleAppConfig]="scripts" [moveFile]="scripts" [_netServiceIsRouted]="scripts" + [networkDetectMtu]="scripts" + [networkEffectiveMtu]="scripts" [networkHealConflicts]="scripts" + [networkMtuCacheFile]="scripts" + [networkRedetectMtu]="scripts" [networkScanConflicts]="scripts" [_nextcloudOcc]="containers" [_nextcloudOccWithPass]="containers" @@ -2662,7 +2670,11 @@ monitoringResolveScrapeTags() { source "${install_scripts_dir}network/monitoring monitoringToggleAppConfig() { source "${install_scripts_dir}network/monitoring/monitoring.sh"; monitoringToggleAppConfig "$@"; } moveFile() { source "${install_scripts_dir}function/file/move_file.sh"; moveFile "$@"; } _netServiceIsRouted() { source "${install_scripts_dir}docker/network/network_conflicts.sh"; _netServiceIsRouted "$@"; } +networkDetectMtu() { source "${install_scripts_dir}network/network_mtu.sh"; networkDetectMtu "$@"; } +networkEffectiveMtu() { source "${install_scripts_dir}network/network_mtu.sh"; networkEffectiveMtu "$@"; } networkHealConflicts() { source "${install_scripts_dir}docker/network/network_heal.sh"; networkHealConflicts "$@"; } +networkMtuCacheFile() { source "${install_scripts_dir}network/network_mtu.sh"; networkMtuCacheFile "$@"; } +networkRedetectMtu() { source "${install_scripts_dir}network/network_mtu.sh"; networkRedetectMtu "$@"; } networkScanConflicts() { source "${install_scripts_dir}docker/network/network_conflicts.sh"; networkScanConflicts "$@"; } _nextcloudOcc() { source "${install_containers_dir}nextcloud/scripts/nextcloud_auth.sh"; _nextcloudOcc "$@"; } _nextcloudOccWithPass() { source "${install_containers_dir}nextcloud/scripts/nextcloud_auth.sh"; _nextcloudOccWithPass "$@"; } From c92fcc9773c04e2ea9297f42ed4882ded8ef1b66 Mon Sep 17 00:00:00 2001 From: librelad Date: Mon, 6 Jul 2026 20:43:56 +0100 Subject: [PATCH 2/3] fix(install): don't record an app as installed when its container never started MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An image-pull failure (e.g. the MTU-EOF black hole) left apps marked installed+active with no container: the compose `up -d` failure was invisible because (a) checkSuccess read $? after `_rc=$?`, always printing "✓ Started", and (b) the exit code is dropped across dockerComposeUpdateAndStartApp → dockerComposeUpdate → dockerComposeRestartAfterUpdate, and installApp never checked it — so post-install integrations ran and set status=1 regardless. - up_app.sh: restore $? to the compose exit before checkSuccess (both rootless and rooted) so a failed `up -d` is reported as an error + logged, not "✓". - app_install.sh: after `up`, reality-gate on the app's compose project having at least one container (ps -a, so a slow-to-start container still counts). If none exists, print a clear failure, skip _appPostStartIntegrations (which is what records the app + sets status=1), and return non-zero. Verified: the gate query passes for a running app (libreportal) and refuses apps with no container (navidrome/ipinfo after their pulls EOF'd). Signed-off-by: librelad Co-Authored-By: Claude Opus 4.8 (1M context) --- scripts/app/install/app_install.sh | 15 +++++++++++++++ scripts/docker/app/compose/up_app.sh | 13 +++++++++---- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/scripts/app/install/app_install.sh b/scripts/app/install/app_install.sh index e15c182..e2a5d9f 100644 --- a/scripts/app/install/app_install.sh +++ b/scripts/app/install/app_install.sh @@ -160,6 +160,21 @@ installApp() dockerComposeUpdateAndStartApp "$app_name" install _appCallHook "${app_slug}_install_post_start" "$app_name" + # Reality gate: after `up`, the app MUST have at least one container + # (its compose project == the app dir name). An image-pull failure + # creates none, yet the deep compose call chain swallows that error — + # without this the app is recorded as installed+active with nothing + # running (the silent-fail seen when a low path-MTU black-holed pulls). + # ps -a (not just running) so a created-but-slow-to-start container + # still counts; only a total absence is treated as failure. + if declare -F dockerCommandRun >/dev/null 2>&1 \ + && ! dockerCommandRun "docker ps -a --filter label=com.docker.compose.project=$app_name --format '{{.Names}}' 2>/dev/null" 2>/dev/null | grep -q '[^[:space:]]'; then + isError "$app_name did not start — no container was created (usually the image failed to pull). NOT recording it as installed." + isNotice "Fix the cause (often image-registry connectivity / MTU), then re-run: libreportal app install $app_name" + eval "$app_slug=n" + return 1 + fi + ((menu_number++)) echo "" echo "---- $menu_number. Running post-install integrations." diff --git a/scripts/docker/app/compose/up_app.sh b/scripts/docker/app/compose/up_app.sh index e222b09..6436abd 100755 --- a/scripts/docker/app/compose/up_app.sh +++ b/scripts/docker/app/compose/up_app.sh @@ -122,12 +122,17 @@ dockerComposeUp() fi if [[ $CFG_DOCKER_INSTALL_TYPE == "rootless" ]]; then isNotice "Starting container for $app_name, this may take a while..." - local result; result=$(dockerCommandRunInstallUser "cd $containers_dir$app_name && COMPOSE_PROGRESS=plain docker compose $setup_compose up $_compose_quiet $_compose_build_flag -d"); _rc=$? - checkSuccess "Started container for $app_name" + local result; result=$(dockerCommandRunInstallUser "cd $containers_dir$app_name && COMPOSE_PROGRESS=plain docker compose $setup_compose up $_compose_quiet $_compose_build_flag -d") + _rc=$? + # Restore $? to the compose exit code — a bare `checkSuccess` + # after `_rc=$?` would read the assignment's 0 and always print + # success, hiding a failed `up -d` (e.g. an image pull EOF). + ( exit "$_rc" ); checkSuccess "Started container for $app_name" elif [[ $CFG_DOCKER_INSTALL_TYPE == "rooted" ]]; then isNotice "Starting container for $app_name, this may take a while..." - local result; result=$(cd "$containers_dir$app_name" && COMPOSE_PROGRESS=plain docker compose $setup_compose up $_compose_quiet $_compose_build_flag -d); _rc=$? - checkSuccess "Started container for $app_name" + local result; result=$(cd "$containers_dir$app_name" && COMPOSE_PROGRESS=plain docker compose $setup_compose up $_compose_quiet $_compose_build_flag -d) + _rc=$? + ( exit "$_rc" ); checkSuccess "Started container for $app_name" fi # Used for the CLI dockertype switcher. else From f2207448b55bcd13f4d779bf95f84ab1571761b5 Mon Sep 17 00:00:00 2001 From: librelad Date: Mon, 6 Jul 2026 20:45:32 +0100 Subject: [PATCH 3/3] style(install): trim the new MTU/gate messages to one short line each MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per feedback — shorten the install-failure error (drop the extra hint line) and the rootless MTU notice. Signed-off-by: librelad Co-Authored-By: Claude Opus 4.8 (1M context) --- scripts/app/install/app_install.sh | 3 +-- scripts/docker/install/rootless/rootless_docker.sh | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/scripts/app/install/app_install.sh b/scripts/app/install/app_install.sh index e2a5d9f..189df50 100644 --- a/scripts/app/install/app_install.sh +++ b/scripts/app/install/app_install.sh @@ -169,8 +169,7 @@ installApp() # still counts; only a total absence is treated as failure. if declare -F dockerCommandRun >/dev/null 2>&1 \ && ! dockerCommandRun "docker ps -a --filter label=com.docker.compose.project=$app_name --format '{{.Names}}' 2>/dev/null" 2>/dev/null | grep -q '[^[:space:]]'; then - isError "$app_name did not start — no container was created (usually the image failed to pull). NOT recording it as installed." - isNotice "Fix the cause (often image-registry connectivity / MTU), then re-run: libreportal app install $app_name" + isError "$app_name: no container started (image pull failed?) — not installed." eval "$app_slug=n" return 1 fi diff --git a/scripts/docker/install/rootless/rootless_docker.sh b/scripts/docker/install/rootless/rootless_docker.sh index 3b11fac..da427e6 100755 --- a/scripts/docker/install/rootless/rootless_docker.sh +++ b/scripts/docker/install/rootless/rootless_docker.sh @@ -148,7 +148,7 @@ EOF # This is the MTU the daemon PULLS over — a blind 1500 on a lower-path-MTU # link (Qubes/NAT/VPN) black-holes large image-layer downloads (EOF). local resolved_mtu; resolved_mtu="$(networkRedetectMtu)" - isNotice "Rootless uplink MTU: $resolved_mtu (CFG_NETWORK_MTU=${CFG_NETWORK_MTU:-auto})" + isNotice "Rootless uplink MTU: $resolved_mtu" sudo bash -c "cat < '$override_conf_file' [Service]