feat(regen): unified regeneration front door + self-heal poll

Add `lpRegen` (scripts/webui/webui_regen.sh) — one entry point that rebuilds the
file-derived artifacts whose sources changed, so callers don't have to know which
generator owns what. Self-heal is a cheap `find -newer` mtime compare (no watcher
/ daemon): a stage runs only when a source is newer than its artifact, or --force.

- `libreportal regen [all|webui|arrays] [--force]` CLI command (new category).
- Task processor idle tick runs a throttled `regen webui` poll, so an app dropped
  in out-of-band (drag-drop / marketplace) appears on its own — no manual command,
  no inotify (works on the relocatable/external-drive roots where inotify can't).
- make_release.sh guards against shipping stale source arrays (regenerate; abort
  if the committed tree was out of date), killing the "forgot generate_arrays" bug
  class at the build boundary.
- Document the front door in DEVELOPMENT.md.

webui scope rebuilds from containers/<app>/{*.config,tools/*.tools.json}; arrays
scope from scripts/** (a dev/build concern — a no-op on a normal install). Gate
logic verified in a sandbox (clean/config-newer/tools-newer/force/missing).

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
Signed-off-by: librelad <librelad@digitalangels.vip>

docs/DEVELOPMENT.md

@@ -182,7 +182,13 @@ is a dev-only escape hatch.
     `.tools.json`. So dropping the folder onto an install is all it takes — no
     central edits, no array regen.
 - **New runtime script?** Add it under `scripts/<area>/…` and run
-  `scripts/source/files/generate_arrays.sh run` so it's sourced (build/standalone
+  `scripts/source/files/generate_arrays.sh run` (or `libreportal regen arrays`) so
-  tooling under `scripts/release` and `scripts/system` is intentionally excluded).
+  it's sourced (build/standalone tooling under `scripts/release` and
+  `scripts/system` is intentionally excluded). `make_release` refuses to build if
+  the committed arrays are stale, so a release can't ship a mismatch.
+- **Regenerating generated data:** `libreportal regen [all|webui|arrays] [--force]`
+  is the one front door — it rebuilds only what's stale (a `find -newer` check, no
+  watcher). The task processor runs `regen webui` on its idle tick, so a dropped-in
+  app (drag-drop / marketplace) appears on its own without a manual command.
 - **Don't** make the OS footprint (`/etc/*`, `/usr/local/*`) relocatable — it's
   fixed by design for the privilege model.

scripts/cli/commands/regen/cli_regen_commands.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+# Regen Commands Handler
+# Front door to lpRegen — rebuilds generated data whose sources changed.
+
+cliHandleRegenCommands()
+{
+    local scope="${initial_command2:-all}"
+    local opt="$initial_command3"
+
+    # Allow `regen --force` (force in the scope slot) as well as `regen all --force`.
+    local force=""
+    if [[ "$scope" == "--force" || "$scope" == "force" || "$scope" == "-f" ]]; then
+        force="--force"; scope="all"
+    fi
+    [[ "$opt" == "--force" || "$opt" == "force" || "$opt" == "-f" ]] && force="--force"
+
+    case "$scope" in
+        all|webui|arrays)
+            lpRegen "$scope" $force
+            ;;
+        help|"")
+            cliShowRegenHelp
+            ;;
+        *)
+            isNotice "Invalid regen scope: $scope"
+            cliShowRegenHelp
+            ;;
+    esac
+}

scripts/cli/commands/regen/cli_regen_header.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+# Regen Commands Header
+# Shows available regen commands and help information
+
+cliShowRegenHelp()
+{
+    echo ""
+    echo "Available Regen Commands:"
+    echo ""
+    echo "  libreportal regen [scope] [--force]      - Rebuild generated data whose sources changed"
+    echo "    all                                   - webui + arrays (default)"
+    echo "    webui                                 - apps.json / apps-tools.json from container configs"
+    echo "    arrays                                - the static source arrays from scripts/** (dev)"
+    echo "    --force                               - rebuild even if nothing looks stale"
+    echo ""
+    echo "Self-heal: each scope only rebuilds when a source file is newer than its"
+    echo "generated artifact, so it is cheap to run any time. The task processor also"
+    echo "runs 'regen webui' periodically, so a dropped-in app appears on its own."
+    echo ""
+    echo "Examples:"
+    echo "  libreportal regen                        - rebuild anything stale"
+    echo "  libreportal regen webui                  - just the WebUI data"
+    echo "  libreportal regen --force                - force a full rebuild"
+    echo ""
+}

scripts/crontab/task/crontab_task_processor.sh

@@ -70,6 +70,14 @@ HEARTBEAT_INTERVAL=5
 HEARTBEAT_STALE_SECS=60
 IDLE_POLL_SECS=3    # max time we'll block on FIFO before re-scanning anyway
 
+# Periodic self-heal poll: on idle ticks, ask the CLI to rebuild the WebUI data if
+# an app folder/config/tool was dropped in out-of-band (drag-drop, marketplace).
+# `libreportal regen webui` self-gates (only does work when sources are newer than
+# the generated artifacts), so this is the "watcher" without a watcher — throttled
+# here so we don't spawn the CLI on every 3s idle tick.
+REGEN_POLL_INTERVAL="${REGEN_POLL_INTERVAL:-60}"
+REGEN_POLL_STAMP="$TASK_DIR/.regen_poll_at"
+
 # ============================================================================
 #  LOGGING
 # ============================================================================
@@ -417,6 +425,22 @@ cleanupZeroByteFiles() {
   done
 }
 
+# ============================================================================
+#  PERIODIC SELF-HEAL POLL
+# ============================================================================
+# Throttled to REGEN_POLL_INTERVAL. The CLI's `regen webui` only rebuilds when a
+# container config/tool is newer than the generated data, so this is a no-op in
+# the common case — it just means a dropped-in app shows up without anyone running
+# a command.
+maybeRegenPoll() {
+  local now last=0
+  now=$(date +%s)
+  [[ -f "$REGEN_POLL_STAMP" ]] && last=$(stat -c %Y "$REGEN_POLL_STAMP" 2>/dev/null || echo 0)
+  (( now - last < REGEN_POLL_INTERVAL )) && return 0
+  : > "$REGEN_POLL_STAMP" 2>/dev/null || true
+  command -v libreportal >/dev/null 2>&1 && libreportal regen webui >/dev/null 2>&1 || true
+}
+
 # ============================================================================
 #  MAIN LOOP
 # ============================================================================
@@ -474,6 +498,7 @@ mainLoop() {
       recoverOrphans
       dispatchPending
       cleanupZeroByteFiles
+      maybeRegenPoll
     fi
   done
 }

scripts/release/make_release.sh

@@ -31,6 +31,19 @@ VERSION="$(tr -d ' \t\n\r' < VERSION 2>/dev/null || true)"
 FOOTPRINT_VERSION="$(grep -oE '^footprint_version=[0-9]+' init.sh | head -1 | cut -d= -f2)"
 [[ -n "$FOOTPRINT_VERSION" ]] || FOOTPRINT_VERSION=0
 
+# Guard: a release must never ship stale source arrays. Regenerate them; if that
+# changes anything not committed, the tree was stale — abort and tell the dev to
+# commit, so `git archive` (committed files only) can't bake in a mismatch. Only
+# enforced when building HEAD (an explicit old tag/ref is taken as-is).
+if [[ "$REF" == "HEAD" && -x scripts/source/files/generate_arrays.sh ]]; then
+    scripts/source/files/generate_arrays.sh run >/dev/null 2>&1 || true
+    if ! git diff --quiet -- scripts/source/files/arrays 2>/dev/null; then
+        echo "make_release: source arrays were stale — regenerated them now." >&2
+        echo "  Commit the changes under scripts/source/files/arrays/ and re-run." >&2
+        exit 1
+    fi
+fi
+
 TARBALL="libreportal-${VERSION}.tar.gz"
 PREFIX="libreportal-${VERSION}/"          # top-level dir inside the tarball
 OUT="$REPO_ROOT/dist/$CHANNEL"

scripts/source/files/arrays/files_cli.sh

@@ -24,6 +24,8 @@ cli_scripts=(
     "cli/commands/install/cli_install_header.sh"
     "cli/commands/ip/cli_ip_commands.sh"
     "cli/commands/ip/cli_ip_header.sh"
+    "cli/commands/regen/cli_regen_commands.sh"
+    "cli/commands/regen/cli_regen_header.sh"
     "cli/commands/reset/cli_reset_commands.sh"
     "cli/commands/reset/cli_reset_header.sh"
     "cli/commands/restore/cli_restore_commands.sh"

scripts/source/files/arrays/files_webui.sh

@@ -47,6 +47,7 @@ webui_scripts=(
     "webui/webui_install_app.sh"
     "webui/webui_install_image.sh"
     "webui/webui_install_systemd.sh"
+    "webui/webui_regen.sh"
     "webui/webui_updater.sh"
 
 )

scripts/webui/webui_regen.sh

@@ -0,0 +1,82 @@
+#!/bin/bash
+
+# Unified regeneration front door.
+#
+# One entry point that rebuilds the file-derived artifacts whose SOURCES changed,
+# so callers (the CLI, the task-processor poll, a deploy) don't have to know which
+# generator owns what. Runs as the manager and writes only into the data dirs — no
+# privilege needed.
+#
+#   lpRegen [scope] [--force]
+#     scope: all (default) | webui | arrays
+#
+# Self-heal: a stage runs only when a source file is newer than its artifact (or
+# the artifact is missing), unless --force. The check is a cheap `find -newer`
+# mtime compare — there is no watcher/daemon. The natural triggers (install,
+# config change, deploy, the periodic task-processor poll) call this; it no-ops
+# when nothing is stale, so it is safe to call often.
+#
+#   webui  — apps.json / apps-tools.json etc. from containers/<app>/{*.config,tools/*.tools.json}
+#   arrays — the static files_*.sh source arrays from scripts/** (a dev/build concern;
+#            a normal install never adds core scripts, so this is a no-op there)
+
+# Is $artifact stale relative to the given find expression? Returns 0 (stale) when
+# the artifact is missing or any matched source file is newer than it.
+_lpRegenStale() {
+    local artifact="$1"; shift
+    [[ -f "$artifact" ]] || return 0
+    find "$@" -newer "$artifact" -print -quit 2>/dev/null | grep -q .
+}
+
+lpRegenWebui() {
+    local force="$1"
+    local gen="${containers_dir}libreportal/frontend/data/apps/generated"
+    local apps_json="$gen/apps.json"
+    local tools_json="$gen/apps-tools.json"
+
+    if [[ "$force" == "force" ]] \
+       || _lpRegenStale "$apps_json"  "$install_containers_dir" -maxdepth 2 -name '*.config' \
+       || _lpRegenStale "$tools_json" "$install_containers_dir" -maxdepth 3 -path '*/tools/*.tools.json'; then
+        # Sources changed (e.g. an app folder was dropped in) — do the full,
+        # debounced refresh so the new app appears everywhere. Force past the
+        # updater's own debounce: we have already established there is real work.
+        WEBUI_UPDATER_FORCE=1 webuiLibrePortalUpdate
+        return $?
+    fi
+    return 0
+}
+
+lpRegenArrays() {
+    local force="$1"
+    local arrays_dir="${install_scripts_dir}source/files/arrays"
+    local gen_script="${install_scripts_dir}source/files/generate_arrays.sh"
+    local newest_array
+    newest_array="$(ls -t "$arrays_dir"/files_*.sh 2>/dev/null | head -1)"
+
+    if [[ "$force" == "force" ]] || [[ -z "$newest_array" ]] \
+       || find "$install_scripts_dir" -name '*.sh' -newer "$newest_array" -print -quit 2>/dev/null | grep -q .; then
+        [[ -f "$gen_script" ]] && "$gen_script" run
+        return $?
+    fi
+    return 0
+}
+
+lpRegen() {
+    local scope="all" force=""
+    local a
+    for a in "$@"; do
+        case "$a" in
+            --force|-f|force) force="force" ;;
+            all|webui|arrays) scope="$a" ;;
+            *) isNotice "Unknown regen argument: $a (use: all|webui|arrays [--force])" ;;
+        esac
+    done
+
+    local rc=0
+    case "$scope" in
+        arrays) lpRegenArrays "$force" || rc=$? ;;
+        webui)  lpRegenWebui  "$force" || rc=$? ;;
+        all)    lpRegenArrays "$force" || rc=$?; lpRegenWebui "$force" || rc=$? ;;
+    esac
+    return $rc
+}