fix(tasks): stop the lazy-autoload stub from looping forever + add processor failsafes

Root cause of the "task loops eternally" bug: the lazy-autoload stub was

    fn() { source "$file"; fn "$@"; }

If the source fails — the real case: an app-install/deploy rsync briefly
removes-then-replaces a generator file while a setup task calls it — `fn` is
never redefined, so `fn "$@"` re-invokes the *stub*, which sources the (still
missing) file, which re-invokes the stub… A single setup finalize recursed
12,050 levels, flooding the task log and taking 82s before it happened to
recover when the file reappeared. A permanently-missing file would never
recover.

Fix (root cause): drop the stub before sourcing —

    fn() { unset -f fn; source "$file"; fn "$@"; }

so a failed source degrades to one "command not found" (rc 127) instead of
unbounded recursion. Regenerated function_manifest.sh (975 stubs, reformat
only — no function-set change).

Failsafes on the task processor (defence in depth, per request):
- FUNCNEST cap (TASK_FUNCNEST_MAX, default 1000) inside the task's eval
  subshell — any runaway recursion now aborts in milliseconds instead of
  spamming the log until the stack/disk gives out.
- Wall-clock cap (TASK_MAX_RUNTIME_SECS, default 7200s, 0=off) — the heartbeat
  watcher TERM→KILLs a task's process group once exceeded and marks it failed
  (distinct from a user cancel via a .timeout marker). Generous so real long
  installs/backups/migrations finish.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Signed-off-by: librelad <librelad@digitalangels.vip>

scripts/source/files/generate_function_manifest.sh

@@ -11,7 +11,11 @@
 #   LP_EAGER_FILES=( "rel/path1.sh" "rel/path2.sh" ... )
 #
 # LP_FN_MAP is what the lazy loader uses to install autoload stubs:
-#   funcname() { source "$install_scripts_dir${LP_FN_MAP[funcname]}"; funcname "$@"; }
+#   funcname() { unset -f funcname; source "$install_scripts_dir${LP_FN_MAP[funcname]}"; funcname "$@"; }
+# The `unset -f` FIRST is a hard failsafe: it drops the stub before sourcing, so
+# if the source fails (file transiently missing mid-deploy, renamed, etc.) the
+# following `funcname "$@"` hits "command not found" (rc 127) ONCE instead of
+# re-invoking the still-installed stub forever (infinite recursion / log flood).
 #
 # LP_EAGER_FILES are files with side effects at source time (set vars, run
 # commands, etc.) that the lazy loader MUST source unconditionally — skipping
@@ -271,9 +275,11 @@ fi
     done < <(printf '%s\n' "${eager_files[@]}" | sort -u)
     printf ')\n\n'
 
-    printf '# Autoload stubs — one per public function. First call sources the\n'
+    printf '# Autoload stubs — one per public function. First call unsets the\n'
-    printf '# real file (which redefines this stub with the real body), then\n'
+    printf '# stub, sources the real file (which redefines the function), then\n'
-    printf '# re-invokes. Sourced inline instead of eval-in-loop because bash\n'
+    printf '# re-invokes. The `unset -f` first means a failed source degrades to a\n'
+    printf '# single "command not found" instead of recursing on the stub forever.\n'
+    printf '# Sourced inline instead of eval-in-loop because bash\n'
     printf '# parses one large file faster than it evals snippets at startup.\n'
     while IFS= read -r name; do
         root="${fn_to_root[$name]}"
@@ -281,8 +287,8 @@ fi
             containers) base_var='install_containers_dir' ;;
             *)          base_var='install_scripts_dir' ;;
         esac
-        printf '%s() { source "${%s}%s"; %s "$@"; }\n' \
+        printf '%s() { unset -f %s; source "${%s}%s"; %s "$@"; }\n' \
-            "$name" "$base_var" "${fn_to_file[$name]}" "$name"
+            "$name" "$name" "$base_var" "${fn_to_file[$name]}" "$name"
     done < <(printf '%s\n' "${!fn_to_file[@]}" | sort)
 } > "$OUTPUT"
 

scripts/task/crontab_task_processor.sh

@@ -72,6 +72,22 @@ HEARTBEAT_INTERVAL=5
 HEARTBEAT_STALE_SECS=60
 IDLE_POLL_SECS=3    # max time we'll block on FIFO before re-scanning anyway
 
+# --- Runaway-task failsafes -------------------------------------------------
+# A task must never be able to run forever (infinite loop, runaway recursion, a
+# wedged command) and take the box down with it. Two independent bounds:
+#   * FUNCNEST caps shell function-nesting depth inside the task's eval
+#     subshell, so runaway recursion (e.g. a lazy-autoload stub whose source
+#     file went missing) aborts in milliseconds instead of flooding the log
+#     until the stack or disk gives out. 1000 is far above any legitimate call
+#     chain yet far below a recursion bug's blow-up (the one that prompted this
+#     hit 12,050 levels).
+#   * TASK_MAX_RUNTIME_SECS is a hard wall-clock cap: the heartbeat watcher
+#     TERM→KILLs the task's process group once it's exceeded, and the task is
+#     marked failed (not cancelled). Generous by default so real long installs/
+#     backups/migrations finish; set 0 to disable.
+TASK_FUNCNEST_MAX="${TASK_FUNCNEST_MAX:-1000}"
+TASK_MAX_RUNTIME_SECS="${TASK_MAX_RUNTIME_SECS:-7200}"
+
 # Periodic self-heal poll: on idle ticks, ask the CLI to rebuild the WebUI data if
 # an app folder/config/tool was dropped in out-of-band (drag-drop, marketplace).
 # `libreportal regen webui` self-gates (only does work when sources are newer than
@@ -253,6 +269,7 @@ runTask() {
   local taskId; taskId=$(basename "$taskFile" .json)
   local logFile="$TASK_DIR/${taskId}.log"
   local cancelMarker="$TASK_DIR/${taskId}.cancel"
+  local timeoutMarker="$TASK_DIR/${taskId}.timeout"
 
   local command; command=$(readTaskField "$taskFile" command)
   if [[ -z "$command" || "$command" == "null" ]]; then
@@ -267,8 +284,9 @@ runTask() {
 
   logInfo "Task $taskId starting: $command"
 
-  # Any pre-existing cancel marker is stale; remove it.
+  # Any pre-existing cancel/timeout marker is stale; remove it.
   [[ -f "$cancelMarker" ]] && runFileOp rm -f "$cancelMarker"
+  [[ -f "$timeoutMarker" ]] && runFileOp rm -f "$timeoutMarker"
 
   # Mark running with initial heartbeat.
   local now; now=$(date -Iseconds)
@@ -310,8 +328,14 @@ runTask() {
   #     only terminates the subshell — the daemon keeps running.
   # `set -m` gives the backgrounded subshell its own process group so we can
   # kill -TERM the whole tree on cancel via `kill -TERM -$cmdPid`.
+  # FUNCNEST inside the subshell so runaway recursion in the eval'd command
+  # aborts itself rather than looping until the box dies. Only set when >0 so
+  # TASK_FUNCNEST_MAX=0 leaves bash's default (unlimited) in place.
+  local taskStart; taskStart=$(date +%s)
   set -m
-  ( cd "$HOME" 2>/dev/null || cd /; eval "$command" ) </dev/null >>"$logFile" 2>&1 &
+  ( cd "$HOME" 2>/dev/null || cd /
+    (( TASK_FUNCNEST_MAX > 0 )) && FUNCNEST="$TASK_FUNCNEST_MAX"
+    eval "$command" ) </dev/null >>"$logFile" 2>&1 &
   local cmdPid=$!
   set +m
 
@@ -346,6 +370,16 @@ runTask() {
           runFileOp rm -f "$cancelMarker"
           exit 0
         fi
+        # Runaway wall-clock failsafe. Drop a marker (so runTask reports this as
+        # a timeout-failure, not a user cancel) then TERM→KILL the whole group.
+        if (( TASK_MAX_RUNTIME_SECS > 0 )) && (( $(date +%s) - taskStart >= TASK_MAX_RUNTIME_SECS )); then
+          logError "Task $taskId exceeded ${TASK_MAX_RUNTIME_SECS}s runtime cap; terminating (loop/hang failsafe)."
+          runFileOp touch "$timeoutMarker" 2>/dev/null
+          kill -TERM "-$cmdPid" 2>/dev/null
+          sleep 5
+          kill -0 "$cmdPid" 2>/dev/null && kill -KILL "-$cmdPid" 2>/dev/null
+          exit 0
+        fi
         sleep 1
       done
     done
@@ -366,7 +400,11 @@ runTask() {
   local finalStatus="completed"
   local errorMessage=""
 
-  if [[ -f "$cancelMarker" ]] || [[ $rc -eq 143 ]] || [[ $rc -eq 137 ]]; then
+  if [[ -f "$timeoutMarker" ]]; then
+    finalStatus="failed"
+    errorMessage="Task exceeded the ${TASK_MAX_RUNTIME_SECS}s runtime cap and was terminated (loop/hang failsafe)."
+    runFileOp rm -f "$timeoutMarker" 2>/dev/null
+  elif [[ -f "$cancelMarker" ]] || [[ $rc -eq 143 ]] || [[ $rc -eq 137 ]]; then
     finalStatus="cancelled"
     errorMessage="Cancelled by user."
     runFileOp rm -f "$cancelMarker" 2>/dev/null