From ef60cce98af075729c25710568a524c9e4d6982d Mon Sep 17 00:00:00 2001 From: librelad Date: Thu, 27 Aug 2026 09:08:41 +0100 Subject: [PATCH] =?UTF-8?q?feat(app):=20portable=20.lpapp=20export=20and?= =?UTF-8?q?=20import=20=E2=80=94=20phase=204?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `libreportal app export ` writes one app to a single file; `libreportal app import ` installs it here. This is the thing the original request described as "upload or navigate to the backup file" — a restic repository is not a file, but the want behind the phrasing is real. The format is deliberately boring: gzipped tar of the app directory with its .libreportal-manifest.json at the root. That manifest already records size, images, volumes, databases and storage location, so import reuses the phase-3 checks for free — refusing an app this version no longer ships, or one that will not fit, before unpacking anything. Export stops the app first. A tar of a running Postgres is a corrupt Postgres, and a file that looks fine until you restore it is worse than a refusal. tar runs as the owning user with --numeric-owner so container sub-UIDs survive the round trip instead of being remapped through this machine's /etc/passwd. Import re-runs the normal install pipeline after unpacking, because the compose still carries the SOURCE machine's ports, IPs and domains — that pipeline is what re-allocates them here, and migrateUrlRewrite fixes the host-bound CFG_* fields. Documented throughout as a courier format, not a backup: no history, no retention, no encryption. Importing under a different name is refused outright rather than half-working — the CFG__* namespace and compose identities would all need rewriting, and `instance create` already answers "a second copy". Fixes a bug this surfaced: _appDirIntended did an indirect expansion on CFG__STORAGE without checking can be a variable name, so a hyphenated or mistyped app name emitted "invalid variable name" and then reported the misleading "storage location is not mounted" for an app that simply did not exist. Co-Authored-By: Claude Opus 5 --- .../components/tasks/js/tasks-format.js | 3 + .../frontend/core/tasks/js/task-commands.js | 4 + docs/roadmap/first-run-restore.md | 14 +- scripts/app/app_portable.sh | 196 ++++++++++++++++++ scripts/cli/commands/app/cli_app_commands.sh | 24 +++ scripts/cli/commands/app/cli_app_header.sh | 7 + scripts/source/files/arrays/files_app.sh | 1 + .../source/files/arrays/function_manifest.sh | 13 ++ scripts/source/paths.sh | 12 +- 9 files changed, 269 insertions(+), 5 deletions(-) create mode 100644 scripts/app/app_portable.sh diff --git a/containers/libreportal/frontend/components/tasks/js/tasks-format.js b/containers/libreportal/frontend/components/tasks/js/tasks-format.js index 0231850..53401be 100644 --- a/containers/libreportal/frontend/components/tasks/js/tasks-format.js +++ b/containers/libreportal/frontend/components/tasks/js/tasks-format.js @@ -70,6 +70,8 @@ Object.assign(TasksManager.prototype, { // fallback below would render "Move Application" and silently drop the // one detail that matters about the task. { match: /^libreportal app move (\S+) (\S+)/, title: (m) => `${displayName(m[1])} - Move to ${m[2]}` }, + // Import names the FILE, not an app — the app is whatever the file holds. + { match: /^libreportal app import (\S+)/, title: (m) => `Import ${String(m[1]).split('/').pop()}` }, // -- Instances --------------------------------------------------------- // Named off the TYPE, not the new slug: at create time the instance does @@ -167,6 +169,7 @@ Object.assign(TasksManager.prototype, { 'delete': 'Delete Backup', 'backup': 'Backup Application', 'move': 'Move Application', + 'export': 'Export Application', // Compose verbs the WebUI dispatches. Without these the generic // " Application" fallback rendered "Up Application" and // "Down Application", which read as broken English rather than a task. diff --git a/containers/libreportal/frontend/core/tasks/js/task-commands.js b/containers/libreportal/frontend/core/tasks/js/task-commands.js index acf528b..a8e22db 100755 --- a/containers/libreportal/frontend/core/tasks/js/task-commands.js +++ b/containers/libreportal/frontend/core/tasks/js/task-commands.js @@ -17,6 +17,8 @@ class TaskCommands { backup: 'libreportal app backup {appName}', // Storage location move — see scripts/storage/storage_move.sh. move: 'libreportal app move {appName} {location}', + // Portable single-file import; {file} is a path, not an app name. + import: 'libreportal app import {file}', status: 'libreportal app status {appName}', // Multi-instance (✅ IMPLEMENTED) — provision another isolated copy of a @@ -54,6 +56,7 @@ class TaskCommands { stop: 'implemented', backup: 'implemented', move: 'implemented', + import: 'implemented', status: 'implemented', up: 'implemented', down: 'implemented', @@ -150,6 +153,7 @@ class TaskCommands { stop: ['appName'], backup: ['appName'], move: ['appName', 'location'], + import: ['file'], status: ['appName'], up: ['appName'], down: ['appName'], diff --git a/docs/roadmap/first-run-restore.md b/docs/roadmap/first-run-restore.md index 06a6498..dc7c407 100644 --- a/docs/roadmap/first-run-restore.md +++ b/docs/roadmap/first-run-restore.md @@ -1,6 +1,6 @@ # LibrePortal — First-run: New Install or Restore (Roadmap / Proposal) -**Status:** Phases 1–2 **built**; 3–4 specified below. · **Audience:** us, future-self · **Scope:** make "I'm rebuilding my server" a first-class path at first run, not a CLI expedition · **Origin:** "on the first install/setup we need 2 option blocks (New Install and Restore from Backup)" (2026-08-27) +**Status:** Phases 1–4 **built**. · **Audience:** us, future-self · **Scope:** make "I'm rebuilding my server" a first-class path at first run, not a CLI expedition · **Origin:** "on the first install/setup we need 2 option blocks (New Install and Restore from Backup)" (2026-08-27) --- @@ -184,12 +184,20 @@ change — and the docs should say so plainly so nobody uses it as their backup. |---|---| | **1** ✅ | Backup destination step in the WebUI wizard (§5) — the *new setup* half | | **2** ✅ | Two installer paths: New setup / Restore from backup, through connect → discover → system config → apps | -| **3** | Preflight reconciliation report in the installer (§3) | -| **4** | `app export` / `app import`, and a `.lpapp` option in the installer's restore path (§4) | +| **3** ✅ | Preflight reconciliation report in the installer (§3) | +| **4** ✅ | `app export` / `app import` (§7). The installer's `.lpapp` option is still open — see §9.5 | ## 9. Open questions 1. **Does the restore branch also restore the system config's *identity*** — install name, domains, WebUI credentials? Restoring the WebUI login means the user logs into the new box with the old password, which is probably what they expect, but it is a surprise if not stated. 2. **Partial restore of a host** — pick apps individually (already supported by `restoreFirstRunBulk`'s signature) or all-or-nothing at first run? 3. **What if the backup is newer than this LibrePortal version?** The manifest records the commit; refusing is safer than guessing, but it strands someone whose only copy is newer. +5. **Should the installer's restore path accept a `.lpapp` too?** `app import` + exists, so the third answer to "where is your backup?" is a small addition — + but a single app file is a thin thing to rebuild a *server* from, and + offering it beside a repository may imply more than it delivers. +6. **Import under a different name** is refused today: the app's `CFG__*` + namespace and its compose identities (container names, Traefik routers, + backup labels) would all need rewriting. `instance create` already solves + "a second copy", so this may never be worth building. 4. **Where does the repository password go once entered** — straight into the location config it will restore over, or held only in memory until the system config lands and then reconciled? diff --git a/scripts/app/app_portable.sh b/scripts/app/app_portable.sh new file mode 100644 index 0000000..f770b4b --- /dev/null +++ b/scripts/app/app_portable.sh @@ -0,0 +1,196 @@ +#!/bin/bash + +# Portable per-app export/import — the "one file you can hand around". +# +# libreportal app export [file] -> -.lpapp +# libreportal app import [newname] +# +# WHAT THIS IS NOT +# +# Not a backup. A restic/borg/kopia repository gives you deduplication, history, +# retention and encryption; this gives you one tarball of one app at one moment. +# It exists because "email me that app" and "keep a copy before I break this" are +# real jobs that a repository answers badly — and because a single file is what +# people picture when they say "the backup file". +# +# The format is deliberately boring: gzipped tar of the app directory, with its +# .libreportal-manifest.json at the root. That manifest already records the +# compose hash, images, volumes, size, databases and storage location, so import +# gets the same reconciliation the restore preflight does, for free. +# +# The app is stopped for the duration of an export. A tar of a running Postgres +# is a corrupt Postgres, and a file that looks fine until you restore it is +# worse than a refusal. + +lpAppExtension="lpapp" + +appExport() +{ + local app="$1" + local out="$2" + + if [[ -z "$app" ]]; then + isError "Usage: app export [file]" + return 1 + fi + + local dir + if ! dir=$(appDir "$app"); then + # appDir only refuses for an unmounted location; anything else is the + # app simply not being here, and saying "not mounted" for a typo sends + # people looking at their disks. + if [[ -d "$(primaryRoot)/$app" ]]; then + isError "$app is on a storage location that is not mounted." + else + isError "$app is not installed." + fi + return 1 + fi + if [[ ! -d "$dir" ]]; then + isError "$app is not installed." + return 1 + fi + + [[ -z "$out" ]] && out="$(pwd)/${app}-$(date +%Y%m%d-%H%M%S).${lpAppExtension}" + case "$out" in */) out="${out}${app}-$(date +%Y%m%d-%H%M%S).${lpAppExtension}" ;; esac + if [[ -e "$out" ]]; then + isError "'$out' already exists — refusing to overwrite it." + return 1 + fi + + isHeader "Export $app" + + # Refresh the manifest first: it is what makes the file self-describing, and + # a stale one would describe a different app than the tar contains. + declare -f manifestWrite >/dev/null 2>&1 && manifestWrite "$app" >/dev/null 2>&1 + + local was_running=0 + if declare -f dockerComposeDown >/dev/null 2>&1; then + if runFileOp docker compose -f "$dir/docker-compose.yml" ps -q 2>/dev/null | grep -q .; then + was_running=1 + fi + isNotice "Stopping $app — a copy taken while it runs can be corrupt." + dockerComposeDown "$app" >/dev/null 2>&1 || true + fi + + isNotice "Writing $out" + # Run as the owner: app data holds container sub-UIDs the manager cannot + # read. --numeric-owner so those uids survive the round trip rather than + # being remapped through this machine's /etc/passwd. + if ! runFileOp tar --numeric-owner -C "${dir%/*}" -czf "$out" "$app" 2>/dev/null; then + isError "Export failed." + (( was_running )) && dockerComposeUp "$app" >/dev/null 2>&1 + return 1 + fi + runFileOp chmod 0640 "$out" 2>/dev/null + + if (( was_running )); then + isNotice "Starting $app again." + dockerComposeUp "$app" >/dev/null 2>&1 || true + fi + + local size; size=$(du -h "$out" 2>/dev/null | awk '{print $1}') + isSuccessful "Exported $app to $out (${size:-?})" + isNotice "This is a copy, not a backup — no history, no retention, and not encrypted." + echo "$out" +} + +# Read the manifest out of an export without unpacking the whole thing. +appImportManifest() +{ + local file="$1" + tar -xzOf "$file" --wildcards '*/.libreportal-manifest.json' 2>/dev/null | head -c 65536 +} + +# The app name an export contains, taken from the tar's top-level directory +# rather than the filename — the filename is whatever someone renamed it to. +appImportName() +{ + local file="$1" + tar -tzf "$file" 2>/dev/null | head -1 | cut -d/ -f1 +} + +appImport() +{ + local file="$1" + local as_name="$2" + + if [[ -z "$file" || ! -f "$file" ]]; then + isError "Usage: app import [new_name]" + return 1 + fi + + isHeader "Import $file" + + local app + app=$(appImportName "$file") + if [[ -z "$app" || ! "$app" =~ ^[A-Za-z0-9._-]+$ ]]; then + isError "'$file' does not look like a LibrePortal export." + return 1 + fi + [[ -n "$as_name" ]] && { + isError "Importing under a different name is not supported yet — the app's config namespace (CFG_${app^^}_*) and compose identities would all need rewriting. Use 'libreportal instance create' for a second copy." + return 1 + } + + # --- the same checks the restore preflight makes ------------------------- + local manifest size_bytes loc + manifest=$(appImportManifest "$file") + size_bytes=$(printf '%s' "$manifest" | grep -o '"size_bytes":[0-9]*' | head -1 | cut -d: -f2) + loc=$(printf '%s' "$manifest" | grep -o '"location":"[^"]*"' | head -1 | cut -d'"' -f4) + + if [[ ! -f "${install_containers_dir%/}/$app/$app.config" ]]; then + isError "This version of LibrePortal does not ship '$app' — it would restore into something that cannot start." + return 1 + fi + + local dir + if ! dir=$(appDir "$app"); then + isError "The storage location for '$app' is not mounted." + return 1 + fi + if [[ -d "$dir" ]]; then + isError "'$app' is already installed at $dir — uninstall it first, or export it before overwriting." + return 1 + fi + + if [[ -n "$loc" && "$loc" != "default" && "$loc" != "primary" ]] \ + && ! storageLocationPath "$loc" >/dev/null 2>&1; then + isNotice "Came from storage location '$loc', which this machine does not have — using $(storageLocationName "${dir%/*}")." + fi + + if [[ -n "$size_bytes" && "$size_bytes" =~ ^[0-9]+$ ]]; then + local need_kb=$(( size_bytes / 1024 )) avail_kb + avail_kb=$(df -Pk "${dir%/*}" 2>/dev/null | awk 'NR==2 {print $4}') + if [[ -n "$avail_kb" ]] && (( avail_kb < need_kb )); then + isError "Needs $(( need_kb / 1048576 ))G, only $(( avail_kb / 1048576 ))G free at ${dir%/*}." + return 1 + fi + fi + + # --- unpack --------------------------------------------------------------- + isNotice "Unpacking into $dir" + runFileOp mkdir -p "${dir%/*}" + if ! runFileOp tar --numeric-owner -xzf "$file" -C "${dir%/*}" 2>/dev/null; then + isError "Unpack failed — removing the partial directory." + runFileOp rm -rf "$dir" + return 1 + fi + + # Ownership, then the normal install pipeline: the compose still carries the + # SOURCE machine's ports, IPs and domains, and re-running the pipeline is + # what re-allocates them for this one. + declare -f runOwnership >/dev/null 2>&1 && runOwnership app-perms >/dev/null 2>&1 + + isNotice "Wiring $app into this machine (ports, IPs, domains)." + dockerConfigSetupToContainer "silent" "$app" + initializeAppVariables "$app" + declare -f migrateUrlRewrite >/dev/null 2>&1 && migrateUrlRewrite "$app" >/dev/null 2>&1 + dockerComposeUpdateAndStartApp "$app" install + dockerComposeUp "$app" + + declare -f databaseInstallApp >/dev/null 2>&1 && databaseInstallApp "$app" + + isSuccessful "$app imported and started" + return 0 +} diff --git a/scripts/cli/commands/app/cli_app_commands.sh b/scripts/cli/commands/app/cli_app_commands.sh index d5a9f26..e28928c 100755 --- a/scripts/cli/commands/app/cli_app_commands.sh +++ b/scripts/cli/commands/app/cli_app_commands.sh @@ -158,6 +158,30 @@ cliHandleAppCommands() fi ;; + "export") + # A copy you can hand around — not a backup. Runs inline: it stops + # the app, and a task row that silently stopped something would be + # worse than watching it happen. + if [[ -z "$app_name" ]]; then + isNotice "Usage: app export [file]" + cliShowAppHelp + else + appExport "$app_name" "$initial_command4" + fi + ;; + + "import") + # Here $app_name is the FILE, since there is no app yet. + if [[ -z "$app_name" ]]; then + isNotice "Usage: app import " + cliShowAppHelp + elif [[ "$LIBREPORTAL_TASK_EXEC" == "1" ]]; then + appImport "$app_name" "$initial_command4" + else + cliTaskRun "libreportal app import $app_name" "import" "libreportal" + fi + ;; + "move") # `app move ` — relocate an app's data to another # storage location. Long-running and it stops the app, so it goes diff --git a/scripts/cli/commands/app/cli_app_header.sh b/scripts/cli/commands/app/cli_app_header.sh index 1a71c34..eca7af5 100755 --- a/scripts/cli/commands/app/cli_app_header.sh +++ b/scripts/cli/commands/app/cli_app_header.sh @@ -20,6 +20,13 @@ cliShowAppHelp() echo " catalog (marketplace). It then installs like any" echo " other app. Browse: libreportal artifact index" echo " libreportal app uninstall [name*] - Uninstall the specified app" + echo " libreportal app export [name*] [file] - Write one app to a single .lpapp file" + echo " (stops it briefly). A copy you can hand" + echo " around — not a backup: no history, no" + echo " retention, not encrypted." + echo " libreportal app import [file*] - Install an app from a .lpapp file," + echo " re-wiring its ports, IPs and domains" + echo " for this machine." echo " libreportal app move [name*] [location*] - Move the app's data to another storage" echo " location. Stops the app, snapshots it," echo " copies, verifies, then removes the source." diff --git a/scripts/source/files/arrays/files_app.sh b/scripts/source/files/arrays/files_app.sh index cf64447..7452b58 100755 --- a/scripts/source/files/arrays/files_app.sh +++ b/scripts/source/files/arrays/files_app.sh @@ -5,6 +5,7 @@ app_scripts=( "app/app_get_key_data.sh" + "app/app_portable.sh" "app/app_scan_available.sh" "app/app_status.sh" "app/app_update_specifics.sh" diff --git a/scripts/source/files/arrays/function_manifest.sh b/scripts/source/files/arrays/function_manifest.sh index bfa886e..432f66e 100644 --- a/scripts/source/files/arrays/function_manifest.sh +++ b/scripts/source/files/arrays/function_manifest.sh @@ -38,6 +38,7 @@ declare -gA LP_FN_MAP=( [appCrowdSecVerifyFirewall]="crowdsec/scripts/crowdsec_verify_firewall.sh" [appDashyManageShortcuts]="dashy/tools/dashy_manage_shortcuts.sh" [appDashyUpdateConf]="dashy/scripts/dashy_update_conf.sh" + [appExport]="app/app_portable.sh" [appGetKeyData]="app/app_get_key_data.sh" [appGiteaCreateAccount]="gitea/tools/gitea_create_account.sh" [appGiteaDeleteUser]="gitea/tools/gitea_delete_user.sh" @@ -46,6 +47,9 @@ declare -gA LP_FN_MAP=( [appGiteaSetAdmin]="gitea/tools/gitea_set_admin.sh" [appGluetunRecreateRouted]="gluetun/scripts/gluetun_recreate_routed.sh" [appGluetunRefreshProviders]="gluetun/tools/gluetun_refresh_providers.sh" + [appImport]="app/app_portable.sh" + [appImportManifest]="app/app_portable.sh" + [appImportName]="app/app_portable.sh" [appInstallCheckRequirements]="checks/requirements/check_app_install.sh" [appInstallMenu]="menu/menu_app_install.sh" [appInvidiousCreateAccount]="invidious/tools/invidious_create_account.sh" @@ -1266,6 +1270,7 @@ declare -gA LP_FN_ROOT=( [appCrowdSecVerifyFirewall]="containers" [appDashyManageShortcuts]="containers" [appDashyUpdateConf]="containers" + [appExport]="scripts" [appGetKeyData]="scripts" [appGiteaCreateAccount]="containers" [appGiteaDeleteUser]="containers" @@ -1274,6 +1279,9 @@ declare -gA LP_FN_ROOT=( [appGiteaSetAdmin]="containers" [appGluetunRecreateRouted]="containers" [appGluetunRefreshProviders]="containers" + [appImport]="scripts" + [appImportManifest]="scripts" + [appImportName]="scripts" [appInstallCheckRequirements]="scripts" [appInstallMenu]="scripts" [appInvidiousCreateAccount]="containers" @@ -2470,6 +2478,7 @@ declare -gA LP_FN_ROOT=( # loader picks the right base dir; existing entries without a prefix # (pre-Phase-5 manifests) default to scripts/. LP_EAGER_FILES=( + "scripts:app/app_portable.sh" "scripts:backup/db/backup_db.sh" "scripts:backup/files/backup_files.sh" "scripts:catalog/catalog_sources.sh" @@ -2531,6 +2540,7 @@ appCrowdSecUpdate() { unset -f appCrowdSecUpdate; __lpAutoload "${install_contai appCrowdSecVerifyFirewall() { unset -f appCrowdSecVerifyFirewall; __lpAutoload "${install_containers_dir}crowdsec/scripts/crowdsec_verify_firewall.sh"; appCrowdSecVerifyFirewall "$@"; } appDashyManageShortcuts() { unset -f appDashyManageShortcuts; __lpAutoload "${install_containers_dir}dashy/tools/dashy_manage_shortcuts.sh"; appDashyManageShortcuts "$@"; } appDashyUpdateConf() { unset -f appDashyUpdateConf; __lpAutoload "${install_containers_dir}dashy/scripts/dashy_update_conf.sh"; appDashyUpdateConf "$@"; } +appExport() { unset -f appExport; __lpAutoload "${install_scripts_dir}app/app_portable.sh"; appExport "$@"; } appGetKeyData() { unset -f appGetKeyData; __lpAutoload "${install_scripts_dir}app/app_get_key_data.sh"; appGetKeyData "$@"; } appGiteaCreateAccount() { unset -f appGiteaCreateAccount; __lpAutoload "${install_containers_dir}gitea/tools/gitea_create_account.sh"; appGiteaCreateAccount "$@"; } appGiteaDeleteUser() { unset -f appGiteaDeleteUser; __lpAutoload "${install_containers_dir}gitea/tools/gitea_delete_user.sh"; appGiteaDeleteUser "$@"; } @@ -2539,6 +2549,9 @@ appGiteaResetPassword() { unset -f appGiteaResetPassword; __lpAutoload "${instal appGiteaSetAdmin() { unset -f appGiteaSetAdmin; __lpAutoload "${install_containers_dir}gitea/tools/gitea_set_admin.sh"; appGiteaSetAdmin "$@"; } appGluetunRecreateRouted() { unset -f appGluetunRecreateRouted; __lpAutoload "${install_containers_dir}gluetun/scripts/gluetun_recreate_routed.sh"; appGluetunRecreateRouted "$@"; } appGluetunRefreshProviders() { unset -f appGluetunRefreshProviders; __lpAutoload "${install_containers_dir}gluetun/tools/gluetun_refresh_providers.sh"; appGluetunRefreshProviders "$@"; } +appImport() { unset -f appImport; __lpAutoload "${install_scripts_dir}app/app_portable.sh"; appImport "$@"; } +appImportManifest() { unset -f appImportManifest; __lpAutoload "${install_scripts_dir}app/app_portable.sh"; appImportManifest "$@"; } +appImportName() { unset -f appImportName; __lpAutoload "${install_scripts_dir}app/app_portable.sh"; appImportName "$@"; } appInstallCheckRequirements() { unset -f appInstallCheckRequirements; __lpAutoload "${install_scripts_dir}checks/requirements/check_app_install.sh"; appInstallCheckRequirements "$@"; } appInstallMenu() { unset -f appInstallMenu; __lpAutoload "${install_scripts_dir}menu/menu_app_install.sh"; appInstallMenu "$@"; } appInvidiousCreateAccount() { unset -f appInvidiousCreateAccount; __lpAutoload "${install_containers_dir}invidious/tools/invidious_create_account.sh"; appInvidiousCreateAccount "$@"; } diff --git a/scripts/source/paths.sh b/scripts/source/paths.sh index 433f01c..d7790b2 100644 --- a/scripts/source/paths.sh +++ b/scripts/source/paths.sh @@ -356,8 +356,16 @@ storageIndexRemove() _appDirIntended() { local slug="$1" - local key="CFG_${slug^^}_STORAGE" - local want="${!key:-}" + # Only look up CFG__STORAGE when can actually BE a variable + # name. A slug with a hyphen (a typo, or a name from an export file) makes + # the indirect expansion below emit "invalid variable name" to stderr and + # return non-zero, which surfaced as a misleading "storage location is not + # mounted" for an app that simply does not exist. + local key="" want="" + if [[ "$slug" =~ ^[A-Za-z_][A-Za-z0-9_]*$ ]]; then + key="CFG_${slug^^}_STORAGE" + want="${!key:-}" + fi # Three states, and the distinction matters: # this app goes there, whatever the global default says