From 899e04bcd3c0ac1747b0949aa69b5c975af43b3e Mon Sep 17 00:00:00 2001 From: librelad Date: Mon, 25 May 2026 23:20:02 +0100 Subject: [PATCH] feat(regen): unified regeneration front door + self-heal poll MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add `lpRegen` (scripts/webui/webui_regen.sh) — one entry point that rebuilds the file-derived artifacts whose sources changed, so callers don't have to know which generator owns what. Self-heal is a cheap `find -newer` mtime compare (no watcher / daemon): a stage runs only when a source is newer than its artifact, or --force. - `libreportal regen [all|webui|arrays] [--force]` CLI command (new category). - Task processor idle tick runs a throttled `regen webui` poll, so an app dropped in out-of-band (drag-drop / marketplace) appears on its own — no manual command, no inotify (works on the relocatable/external-drive roots where inotify can't). - make_release.sh guards against shipping stale source arrays (regenerate; abort if the committed tree was out of date), killing the "forgot generate_arrays" bug class at the build boundary. - Document the front door in DEVELOPMENT.md. webui scope rebuilds from containers//{*.config,tools/*.tools.json}; arrays scope from scripts/** (a dev/build concern — a no-op on a normal install). Gate logic verified in a sandbox (clean/config-newer/tools-newer/force/missing). Co-Authored-By: Claude Opus 4.7 Signed-off-by: librelad --- docs/DEVELOPMENT.md | 10 ++- .../cli/commands/regen/cli_regen_commands.sh | 30 +++++++ .../cli/commands/regen/cli_regen_header.sh | 26 ++++++ .../crontab/task/crontab_task_processor.sh | 25 ++++++ scripts/release/make_release.sh | 13 +++ scripts/source/files/arrays/files_cli.sh | 2 + scripts/source/files/arrays/files_webui.sh | 1 + scripts/webui/webui_regen.sh | 82 +++++++++++++++++++ 8 files changed, 187 insertions(+), 2 deletions(-) create mode 100644 scripts/cli/commands/regen/cli_regen_commands.sh create mode 100644 scripts/cli/commands/regen/cli_regen_header.sh create mode 100644 scripts/webui/webui_regen.sh diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md index 97c527d..86aaae6 100644 --- a/docs/DEVELOPMENT.md +++ b/docs/DEVELOPMENT.md @@ -182,7 +182,13 @@ is a dev-only escape hatch. `.tools.json`. So dropping the folder onto an install is all it takes — no central edits, no array regen. - **New runtime script?** Add it under `scripts//…` and run - `scripts/source/files/generate_arrays.sh run` so it's sourced (build/standalone - tooling under `scripts/release` and `scripts/system` is intentionally excluded). + `scripts/source/files/generate_arrays.sh run` (or `libreportal regen arrays`) so + it's sourced (build/standalone tooling under `scripts/release` and + `scripts/system` is intentionally excluded). `make_release` refuses to build if + the committed arrays are stale, so a release can't ship a mismatch. +- **Regenerating generated data:** `libreportal regen [all|webui|arrays] [--force]` + is the one front door — it rebuilds only what's stale (a `find -newer` check, no + watcher). The task processor runs `regen webui` on its idle tick, so a dropped-in + app (drag-drop / marketplace) appears on its own without a manual command. - **Don't** make the OS footprint (`/etc/*`, `/usr/local/*`) relocatable — it's fixed by design for the privilege model. diff --git a/scripts/cli/commands/regen/cli_regen_commands.sh b/scripts/cli/commands/regen/cli_regen_commands.sh new file mode 100644 index 0000000..3078c49 --- /dev/null +++ b/scripts/cli/commands/regen/cli_regen_commands.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +# Regen Commands Handler +# Front door to lpRegen — rebuilds generated data whose sources changed. + +cliHandleRegenCommands() +{ + local scope="${initial_command2:-all}" + local opt="$initial_command3" + + # Allow `regen --force` (force in the scope slot) as well as `regen all --force`. + local force="" + if [[ "$scope" == "--force" || "$scope" == "force" || "$scope" == "-f" ]]; then + force="--force"; scope="all" + fi + [[ "$opt" == "--force" || "$opt" == "force" || "$opt" == "-f" ]] && force="--force" + + case "$scope" in + all|webui|arrays) + lpRegen "$scope" $force + ;; + help|"") + cliShowRegenHelp + ;; + *) + isNotice "Invalid regen scope: $scope" + cliShowRegenHelp + ;; + esac +} diff --git a/scripts/cli/commands/regen/cli_regen_header.sh b/scripts/cli/commands/regen/cli_regen_header.sh new file mode 100644 index 0000000..9375f26 --- /dev/null +++ b/scripts/cli/commands/regen/cli_regen_header.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +# Regen Commands Header +# Shows available regen commands and help information + +cliShowRegenHelp() +{ + echo "" + echo "Available Regen Commands:" + echo "" + echo " libreportal regen [scope] [--force] - Rebuild generated data whose sources changed" + echo " all - webui + arrays (default)" + echo " webui - apps.json / apps-tools.json from container configs" + echo " arrays - the static source arrays from scripts/** (dev)" + echo " --force - rebuild even if nothing looks stale" + echo "" + echo "Self-heal: each scope only rebuilds when a source file is newer than its" + echo "generated artifact, so it is cheap to run any time. The task processor also" + echo "runs 'regen webui' periodically, so a dropped-in app appears on its own." + echo "" + echo "Examples:" + echo " libreportal regen - rebuild anything stale" + echo " libreportal regen webui - just the WebUI data" + echo " libreportal regen --force - force a full rebuild" + echo "" +} diff --git a/scripts/crontab/task/crontab_task_processor.sh b/scripts/crontab/task/crontab_task_processor.sh index b970308..892efb6 100755 --- a/scripts/crontab/task/crontab_task_processor.sh +++ b/scripts/crontab/task/crontab_task_processor.sh @@ -70,6 +70,14 @@ HEARTBEAT_INTERVAL=5 HEARTBEAT_STALE_SECS=60 IDLE_POLL_SECS=3 # max time we'll block on FIFO before re-scanning anyway +# Periodic self-heal poll: on idle ticks, ask the CLI to rebuild the WebUI data if +# an app folder/config/tool was dropped in out-of-band (drag-drop, marketplace). +# `libreportal regen webui` self-gates (only does work when sources are newer than +# the generated artifacts), so this is the "watcher" without a watcher — throttled +# here so we don't spawn the CLI on every 3s idle tick. +REGEN_POLL_INTERVAL="${REGEN_POLL_INTERVAL:-60}" +REGEN_POLL_STAMP="$TASK_DIR/.regen_poll_at" + # ============================================================================ # LOGGING # ============================================================================ @@ -417,6 +425,22 @@ cleanupZeroByteFiles() { done } +# ============================================================================ +# PERIODIC SELF-HEAL POLL +# ============================================================================ +# Throttled to REGEN_POLL_INTERVAL. The CLI's `regen webui` only rebuilds when a +# container config/tool is newer than the generated data, so this is a no-op in +# the common case — it just means a dropped-in app shows up without anyone running +# a command. +maybeRegenPoll() { + local now last=0 + now=$(date +%s) + [[ -f "$REGEN_POLL_STAMP" ]] && last=$(stat -c %Y "$REGEN_POLL_STAMP" 2>/dev/null || echo 0) + (( now - last < REGEN_POLL_INTERVAL )) && return 0 + : > "$REGEN_POLL_STAMP" 2>/dev/null || true + command -v libreportal >/dev/null 2>&1 && libreportal regen webui >/dev/null 2>&1 || true +} + # ============================================================================ # MAIN LOOP # ============================================================================ @@ -474,6 +498,7 @@ mainLoop() { recoverOrphans dispatchPending cleanupZeroByteFiles + maybeRegenPoll fi done } diff --git a/scripts/release/make_release.sh b/scripts/release/make_release.sh index 0ca5d84..ceca668 100644 --- a/scripts/release/make_release.sh +++ b/scripts/release/make_release.sh @@ -31,6 +31,19 @@ VERSION="$(tr -d ' \t\n\r' < VERSION 2>/dev/null || true)" FOOTPRINT_VERSION="$(grep -oE '^footprint_version=[0-9]+' init.sh | head -1 | cut -d= -f2)" [[ -n "$FOOTPRINT_VERSION" ]] || FOOTPRINT_VERSION=0 +# Guard: a release must never ship stale source arrays. Regenerate them; if that +# changes anything not committed, the tree was stale — abort and tell the dev to +# commit, so `git archive` (committed files only) can't bake in a mismatch. Only +# enforced when building HEAD (an explicit old tag/ref is taken as-is). +if [[ "$REF" == "HEAD" && -x scripts/source/files/generate_arrays.sh ]]; then + scripts/source/files/generate_arrays.sh run >/dev/null 2>&1 || true + if ! git diff --quiet -- scripts/source/files/arrays 2>/dev/null; then + echo "make_release: source arrays were stale — regenerated them now." >&2 + echo " Commit the changes under scripts/source/files/arrays/ and re-run." >&2 + exit 1 + fi +fi + TARBALL="libreportal-${VERSION}.tar.gz" PREFIX="libreportal-${VERSION}/" # top-level dir inside the tarball OUT="$REPO_ROOT/dist/$CHANNEL" diff --git a/scripts/source/files/arrays/files_cli.sh b/scripts/source/files/arrays/files_cli.sh index 7ae26dc..3df7b1f 100755 --- a/scripts/source/files/arrays/files_cli.sh +++ b/scripts/source/files/arrays/files_cli.sh @@ -24,6 +24,8 @@ cli_scripts=( "cli/commands/install/cli_install_header.sh" "cli/commands/ip/cli_ip_commands.sh" "cli/commands/ip/cli_ip_header.sh" + "cli/commands/regen/cli_regen_commands.sh" + "cli/commands/regen/cli_regen_header.sh" "cli/commands/reset/cli_reset_commands.sh" "cli/commands/reset/cli_reset_header.sh" "cli/commands/restore/cli_restore_commands.sh" diff --git a/scripts/source/files/arrays/files_webui.sh b/scripts/source/files/arrays/files_webui.sh index 74dc93f..86f31fd 100755 --- a/scripts/source/files/arrays/files_webui.sh +++ b/scripts/source/files/arrays/files_webui.sh @@ -47,6 +47,7 @@ webui_scripts=( "webui/webui_install_app.sh" "webui/webui_install_image.sh" "webui/webui_install_systemd.sh" + "webui/webui_regen.sh" "webui/webui_updater.sh" ) diff --git a/scripts/webui/webui_regen.sh b/scripts/webui/webui_regen.sh new file mode 100644 index 0000000..6a90b0c --- /dev/null +++ b/scripts/webui/webui_regen.sh @@ -0,0 +1,82 @@ +#!/bin/bash + +# Unified regeneration front door. +# +# One entry point that rebuilds the file-derived artifacts whose SOURCES changed, +# so callers (the CLI, the task-processor poll, a deploy) don't have to know which +# generator owns what. Runs as the manager and writes only into the data dirs — no +# privilege needed. +# +# lpRegen [scope] [--force] +# scope: all (default) | webui | arrays +# +# Self-heal: a stage runs only when a source file is newer than its artifact (or +# the artifact is missing), unless --force. The check is a cheap `find -newer` +# mtime compare — there is no watcher/daemon. The natural triggers (install, +# config change, deploy, the periodic task-processor poll) call this; it no-ops +# when nothing is stale, so it is safe to call often. +# +# webui — apps.json / apps-tools.json etc. from containers//{*.config,tools/*.tools.json} +# arrays — the static files_*.sh source arrays from scripts/** (a dev/build concern; +# a normal install never adds core scripts, so this is a no-op there) + +# Is $artifact stale relative to the given find expression? Returns 0 (stale) when +# the artifact is missing or any matched source file is newer than it. +_lpRegenStale() { + local artifact="$1"; shift + [[ -f "$artifact" ]] || return 0 + find "$@" -newer "$artifact" -print -quit 2>/dev/null | grep -q . +} + +lpRegenWebui() { + local force="$1" + local gen="${containers_dir}libreportal/frontend/data/apps/generated" + local apps_json="$gen/apps.json" + local tools_json="$gen/apps-tools.json" + + if [[ "$force" == "force" ]] \ + || _lpRegenStale "$apps_json" "$install_containers_dir" -maxdepth 2 -name '*.config' \ + || _lpRegenStale "$tools_json" "$install_containers_dir" -maxdepth 3 -path '*/tools/*.tools.json'; then + # Sources changed (e.g. an app folder was dropped in) — do the full, + # debounced refresh so the new app appears everywhere. Force past the + # updater's own debounce: we have already established there is real work. + WEBUI_UPDATER_FORCE=1 webuiLibrePortalUpdate + return $? + fi + return 0 +} + +lpRegenArrays() { + local force="$1" + local arrays_dir="${install_scripts_dir}source/files/arrays" + local gen_script="${install_scripts_dir}source/files/generate_arrays.sh" + local newest_array + newest_array="$(ls -t "$arrays_dir"/files_*.sh 2>/dev/null | head -1)" + + if [[ "$force" == "force" ]] || [[ -z "$newest_array" ]] \ + || find "$install_scripts_dir" -name '*.sh' -newer "$newest_array" -print -quit 2>/dev/null | grep -q .; then + [[ -f "$gen_script" ]] && "$gen_script" run + return $? + fi + return 0 +} + +lpRegen() { + local scope="all" force="" + local a + for a in "$@"; do + case "$a" in + --force|-f|force) force="force" ;; + all|webui|arrays) scope="$a" ;; + *) isNotice "Unknown regen argument: $a (use: all|webui|arrays [--force])" ;; + esac + done + + local rc=0 + case "$scope" in + arrays) lpRegenArrays "$force" || rc=$? ;; + webui) lpRegenWebui "$force" || rc=$? ;; + all) lpRegenArrays "$force" || rc=$?; lpRegenWebui "$force" || rc=$? ;; + esac + return $rc +}