From ffc7801762b51ae2a2c9a0498a05e9e39cc12963 Mon Sep 17 00:00:00 2001 From: librelad Date: Mon, 6 Jul 2026 20:38:11 +0100 Subject: [PATCH] fix(network): auto-detect path MTU so large image pulls don't stall with EOF MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CFG_NETWORK_MTU=1500 was baked blindly into the rootless daemon uplink (DOCKERD_ROOTLESS_ROOTLESSKIT_MTU) and every app's container network. On links whose real path MTU is below 1500 (Qubes/NAT/VPN with PMTU discovery blocked), image manifests + tiny images pull fine but large image LAYERS stall and reset mid-blob with "httpReadSeeker: ... EOF" — apps silently fail to install. Probed here: path MTU ~1328, Docker at 1500 → black hole. - New scripts/network/network_mtu.sh: networkDetectMtu (don't-fragment ICMP ladder → largest standard MTU that gets through, 1500 when ICMP gives no signal) + networkEffectiveMtu (CFG_NETWORK_MTU: a number is verbatim, "auto" probes once and caches $docker_dir/.network_mtu) + networkRedetectMtu. - CFG_NETWORK_MTU default 1500 -> auto. Rootless setup now writes the resolved MTU into the override (and re-detects per install); the per-app NETWORK_MTU_TAG uses the resolved value too. Explicit numbers still win. Verified on this box: auto -> 1300; alpine (multi-MB) + the full ~100MB navidrome image now pull to completion where they previously EOF'd. Applied live (override 1300, config=auto, cache=1300, rootless docker restarted). Signed-off-by: librelad Co-Authored-By: Claude Opus 4.8 (1M context) --- configs/network/network_docker | 2 +- .../config/docker/docker_config_setup_data.sh | 2 +- .../install/rootless/rootless_docker.sh | 8 ++- scripts/network/network_mtu.sh | 68 +++++++++++++++++++ scripts/source/files/arrays/files_network.sh | 1 + .../source/files/arrays/function_manifest.sh | 12 ++++ 6 files changed, 90 insertions(+), 3 deletions(-) create mode 100644 scripts/network/network_mtu.sh diff --git a/configs/network/network_docker b/configs/network/network_docker index cedf78c..a3d9b30 100755 --- a/configs/network/network_docker +++ b/configs/network/network_docker @@ -4,7 +4,7 @@ CFG_NETWORK_NAME=vpn # Network Name - Docker network name for container communication CFG_NETWORK_SUBNET=10.100.0.0/16 # Network Subnet - Subnet range for Docker network -CFG_NETWORK_MTU=1500 # Network MTU - Maximum transmission unit for network packets +CFG_NETWORK_MTU=auto # Network MTU - Maximum transmission unit for network packets. 'auto' probes the real path MTU and clamps to a safe value (fixes large image pulls stalling with EOF on Qubes/NAT/VPN links where the path MTU is below 1500); or set an explicit number like 1500 or 1280. **ADVANCED** CFG_REQUIREMENT_DOCKER_NETWORK=true # Docker Network - Create and manage the Docker network for container communication CFG_REQUIREMENT_DOCKER_NETWORK_PRUNE=true # Network Cleanup - Automatically prune unused Docker networks CFG_REQUIREMENT_DOCKER_SWITCHER=true # Docker Switcher - Install the Docker version switching utility diff --git a/scripts/config/docker/docker_config_setup_data.sh b/scripts/config/docker/docker_config_setup_data.sh index b086792..e280751 100755 --- a/scripts/config/docker/docker_config_setup_data.sh +++ b/scripts/config/docker/docker_config_setup_data.sh @@ -93,7 +93,7 @@ dockerConfigSetupFileWithData() tagsManagerUpdateUniversalTag "$full_file_path" "DOCKER_NETWORK_TAG" "$CFG_NETWORK_NAME" tagsManagerUpdateUniversalTag "$full_file_path" "PUBLIC_IP_TAG" "$public_ip_v4" tagsManagerUpdateUniversalTag "$full_file_path" "NETWORK_SUBNET_TAG" "$CFG_NETWORK_SUBNET" - tagsManagerUpdateUniversalTag "$full_file_path" "NETWORK_MTU_TAG" "$CFG_NETWORK_MTU" + tagsManagerUpdateUniversalTag "$full_file_path" "NETWORK_MTU_TAG" "$(networkEffectiveMtu)" ipUpdateComposeTags "$app_name" "$full_file_path" portUpdateComposeTags "$app_name" "$full_file_path" tagsProcessorTrustedDomains "$full_file_path" diff --git a/scripts/docker/install/rootless/rootless_docker.sh b/scripts/docker/install/rootless/rootless_docker.sh index ecdbf65..3b11fac 100755 --- a/scripts/docker/install/rootless/rootless_docker.sh +++ b/scripts/docker/install/rootless/rootless_docker.sh @@ -144,11 +144,17 @@ EOF local result; result=$(sudo touch $override_conf_file) checkSuccess "Create the override.conf in docker.service.d" + # Resolve CFG_NETWORK_MTU to a concrete, path-safe value (auto = probe). + # This is the MTU the daemon PULLS over — a blind 1500 on a lower-path-MTU + # link (Qubes/NAT/VPN) black-holes large image-layer downloads (EOF). + local resolved_mtu; resolved_mtu="$(networkRedetectMtu)" + isNotice "Rootless uplink MTU: $resolved_mtu (CFG_NETWORK_MTU=${CFG_NETWORK_MTU:-auto})" + sudo bash -c "cat < '$override_conf_file' [Service] Environment='DOCKERD_ROOTLESS_ROOTLESSKIT_NET=$rootless_net' Environment='DOCKERD_ROOTLESS_ROOTLESSKIT_PORT_DRIVER=$rootless_port_driver' -Environment='DOCKERD_ROOTLESS_ROOTLESSKIT_MTU=$CFG_NETWORK_MTU' +Environment='DOCKERD_ROOTLESS_ROOTLESSKIT_MTU=$resolved_mtu' EOL" local result; result=$(sudo chown $CFG_DOCKER_INSTALL_USER:$CFG_DOCKER_INSTALL_USER $override_conf_file) diff --git a/scripts/network/network_mtu.sh b/scripts/network/network_mtu.sh new file mode 100644 index 0000000..61942e0 --- /dev/null +++ b/scripts/network/network_mtu.sh @@ -0,0 +1,68 @@ +#!/bin/bash +# +# Network MTU resolution. +# --------------------------------------------------------------------------- +# CFG_NETWORK_MTU drives BOTH the rootless daemon's uplink MTU +# (DOCKERD_ROOTLESS_ROOTLESSKIT_MTU — how image PULLS egress) and every app's +# container-network MTU (the NETWORK_MTU_TAG baked into each compose). A blind +# 1500 breaks on links whose real path MTU is lower (Qubes/NAT/VPN with PMTU +# discovery blocked): small transfers (image manifests, tiny images) succeed but +# large image-layer downloads stall and reset with "EOF" mid-blob. +# +# CFG_NETWORK_MTU accepts: +# — use exactly (explicit override). +# auto — probe the path MTU once, cache a safe value (default). +# +# The probe is a don't-fragment ICMP ladder. If ICMP is fully filtered (no probe +# gets through at all) we can't tell a clean link from a constrained one, so we +# assume the standard 1500 rather than needlessly shrinking every network — set +# an explicit number for that case. + +networkMtuCacheFile() { echo "${docker_dir%/}/.network_mtu"; } + +# Largest standard MTU whose don't-fragment probe reaches the internet. +# Echoes a number; 1500 when no signal (ping missing / all probes filtered). +networkDetectMtu() { + local host="${1:-1.1.1.1}" m best="" + command -v ping >/dev/null 2>&1 || { echo 1500; return 0; } + for m in 1500 1492 1400 1300 1280; do + # payload = mtu - 28 (IPv4 + ICMP headers) + if ping -M do -s $((m - 28)) -c1 -W2 "$host" >/dev/null 2>&1; then + best="$m"; break + fi + done + echo "${best:-1500}" +} + +# The concrete MTU to bake into the daemon + composes. Resolves CFG_NETWORK_MTU: +# a number is used verbatim; "auto"/empty is probed once and cached so we don't +# re-ping on every app install. +networkEffectiveMtu() { + local want="${CFG_NETWORK_MTU:-auto}" + if [[ "$want" =~ ^[0-9]+$ ]]; then echo "$want"; return 0; fi + + local cache cached + cache="$(networkMtuCacheFile)" + cached="$(cat "$cache" 2>/dev/null | tr -dc '0-9')" + if [[ -n "$cached" ]]; then echo "$cached"; return 0; fi + + local detected; detected="$(networkDetectMtu)" + if declare -F runInstallWrite >/dev/null 2>&1; then + printf '%s\n' "$detected" | runInstallWrite "$cache" 2>/dev/null || true + else + printf '%s\n' "$detected" > "$cache" 2>/dev/null || true + fi + echo "$detected" +} + +# Force a fresh probe (clears the cache), echoing the new value. The rootless +# setup calls this so each (re)install re-detects the current link. +networkRedetectMtu() { + local cache; cache="$(networkMtuCacheFile)" + if declare -F runInstallOp >/dev/null 2>&1; then + runInstallOp rm -f "$cache" 2>/dev/null || true + else + rm -f "$cache" 2>/dev/null || true + fi + networkEffectiveMtu +} diff --git a/scripts/source/files/arrays/files_network.sh b/scripts/source/files/arrays/files_network.sh index b1532f1..ce81d9e 100755 --- a/scripts/source/files/arrays/files_network.sh +++ b/scripts/source/files/arrays/files_network.sh @@ -28,6 +28,7 @@ network_scripts=( "network/ip/ip_remove_from_db.sh" "network/ip/ip_replace_tags.sh" "network/monitoring/monitoring.sh" + "network/network_mtu.sh" "network/ports/allocation/port_allocate.sh" "network/ports/allocation/port_store_mapping.sh" "network/ports/allocation/port_update_compose_tags.sh" diff --git a/scripts/source/files/arrays/function_manifest.sh b/scripts/source/files/arrays/function_manifest.sh index 114a472..0e61df8 100644 --- a/scripts/source/files/arrays/function_manifest.sh +++ b/scripts/source/files/arrays/function_manifest.sh @@ -681,7 +681,11 @@ declare -gA LP_FN_MAP=( [monitoringToggleAppConfig]="network/monitoring/monitoring.sh" [moveFile]="function/file/move_file.sh" [_netServiceIsRouted]="docker/network/network_conflicts.sh" + [networkDetectMtu]="network/network_mtu.sh" + [networkEffectiveMtu]="network/network_mtu.sh" [networkHealConflicts]="docker/network/network_heal.sh" + [networkMtuCacheFile]="network/network_mtu.sh" + [networkRedetectMtu]="network/network_mtu.sh" [networkScanConflicts]="docker/network/network_conflicts.sh" [_nextcloudOcc]="nextcloud/scripts/nextcloud_auth.sh" [_nextcloudOccWithPass]="nextcloud/scripts/nextcloud_auth.sh" @@ -1661,7 +1665,11 @@ declare -gA LP_FN_ROOT=( [monitoringToggleAppConfig]="scripts" [moveFile]="scripts" [_netServiceIsRouted]="scripts" + [networkDetectMtu]="scripts" + [networkEffectiveMtu]="scripts" [networkHealConflicts]="scripts" + [networkMtuCacheFile]="scripts" + [networkRedetectMtu]="scripts" [networkScanConflicts]="scripts" [_nextcloudOcc]="containers" [_nextcloudOccWithPass]="containers" @@ -2662,7 +2670,11 @@ monitoringResolveScrapeTags() { source "${install_scripts_dir}network/monitoring monitoringToggleAppConfig() { source "${install_scripts_dir}network/monitoring/monitoring.sh"; monitoringToggleAppConfig "$@"; } moveFile() { source "${install_scripts_dir}function/file/move_file.sh"; moveFile "$@"; } _netServiceIsRouted() { source "${install_scripts_dir}docker/network/network_conflicts.sh"; _netServiceIsRouted "$@"; } +networkDetectMtu() { source "${install_scripts_dir}network/network_mtu.sh"; networkDetectMtu "$@"; } +networkEffectiveMtu() { source "${install_scripts_dir}network/network_mtu.sh"; networkEffectiveMtu "$@"; } networkHealConflicts() { source "${install_scripts_dir}docker/network/network_heal.sh"; networkHealConflicts "$@"; } +networkMtuCacheFile() { source "${install_scripts_dir}network/network_mtu.sh"; networkMtuCacheFile "$@"; } +networkRedetectMtu() { source "${install_scripts_dir}network/network_mtu.sh"; networkRedetectMtu "$@"; } networkScanConflicts() { source "${install_scripts_dir}docker/network/network_conflicts.sh"; networkScanConflicts "$@"; } _nextcloudOcc() { source "${install_containers_dir}nextcloud/scripts/nextcloud_auth.sh"; _nextcloudOcc "$@"; } _nextcloudOccWithPass() { source "${install_containers_dir}nextcloud/scripts/nextcloud_auth.sh"; _nextcloudOccWithPass "$@"; }