From 90a20e9e17f0433c24fd4ed6e6d8253c733ab831 Mon Sep 17 00:00:00 2001 From: librelad Date: Mon, 6 Jul 2026 19:39:21 +0100 Subject: [PATCH] =?UTF-8?q?feat(catalog):=20multiple=20catalog=20sources?= =?UTF-8?q?=20=E2=80=94=20backend=20model=20+=20multi-source=20scan=20+=20?= =?UTF-8?q?CLI?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit First slice of multi-catalog ("taps") support. Today the App Center browses one catalog (get.libreportal.org). This adds an ordered list of catalog sources and teaches the browse scan to merge them. - New scripts/catalog/catalog_sources.sh — the source list in its OWN file ($configs_dir/catalog/sources.json). Source #1 is ALWAYS the official catalog, synthesized live from CFG_RELEASE_BASE_URL/CHANNEL (can't be edited/removed, always pinned on top). Extra sources are stored as a small JSON array and are UNVERIFIED (trust=community). Helpers: catalogSourcesJson / catalogEnabledSources (priority order) / catalogSourceAdd|Remove|Toggle|List / catalogFetchCommunityIndex. - webui_registry_scan.sh now walks catalogEnabledSources: the OFFICIAL source is still signature-verified (lpFetchIndexInto, unchanged trust path); third-party sources are fetched unverified. Apps are merged by slug into one card carrying a sources[] array in priority order (highest first = default). Trust/verified are taken from the SOURCE, never the artifact's self-claim, so a community index can't promote itself to "official". Icons still mirror same-origin from the official index only. registry_catalog.json gains top-level sources[] + per-app sources[]; the old source{} object + signed/serial are kept for back-compat. - New `libreportal catalog source list|add|remove|enable|disable` + `catalog refresh` CLI (dynamic-routed). Mutations go through the task system (cliTaskRun "…" "catalog"), never a new mutating API. Scope firewall: this governs APP BROWSE + ADD only. LibrePortal's own updates and hotfixes still resolve from the official CFG_RELEASE_BASE_URL alone — lpFetchIndex is untouched, so a third-party catalog can never become a system-update channel. Next: `app add --source`, then the WebUI (domains-style block source manager + Add-dialog source picker with the Official badge / unverified warning). Signed-off-by: librelad Co-Authored-By: Claude Opus 4.8 (1M context) --- scripts/catalog/catalog_sources.sh | 145 ++++++++++++++++++ .../commands/catalog/cli_catalog_commands.sh | 70 +++++++++ .../commands/catalog/cli_catalog_header.sh | 24 +++ scripts/source/files/arrays/files_catalog.sh | 9 ++ scripts/source/files/arrays/files_cli.sh | 2 + scripts/source/files/arrays/files_source.sh | 1 + .../source/files/arrays/function_manifest.sh | 39 +++++ .../generators/apps/webui_registry_scan.sh | 143 +++++++++++------ 8 files changed, 390 insertions(+), 43 deletions(-) create mode 100644 scripts/catalog/catalog_sources.sh create mode 100644 scripts/cli/commands/catalog/cli_catalog_commands.sh create mode 100644 scripts/cli/commands/catalog/cli_catalog_header.sh create mode 100644 scripts/source/files/arrays/files_catalog.sh diff --git a/scripts/catalog/catalog_sources.sh b/scripts/catalog/catalog_sources.sh new file mode 100644 index 0000000..6ee3a23 --- /dev/null +++ b/scripts/catalog/catalog_sources.sh @@ -0,0 +1,145 @@ +#!/bin/bash +# +# Catalog sources — the ordered list of app-catalog URLs the App Center browses. +# --------------------------------------------------------------------------- +# Source #1 is ALWAYS the official LibrePortal catalog, synthesized live from +# CFG_RELEASE_BASE_URL / CFG_RELEASE_CHANNEL so it can't be edited or removed and +# always tracks the release host. Extra (third-party) sources are stored as a +# small JSON array in their OWN file under the live configs dir and are treated +# as UNVERIFIED (the operator added the URL themselves). +# +# Priority = list order: the official catalog is first, then extras in the order +# they were added. When the same app slug is offered by more than one source the +# highest-priority source that has it is the default (the Add dialog can override). +# +# IMPORTANT SCOPE: this list governs APP BROWSE + APP ADD only. LibrePortal's own +# updates and hotfixes still resolve from the official CFG_RELEASE_BASE_URL alone +# (lpFetchIndex is untouched) — a third-party catalog can never become a channel +# for system updates. + +# Its own file, next to the other live config (manager-owned control plane). +catalogSourcesFile() { echo "${configs_dir%/}/catalog/sources.json"; } + +# The extra (non-official) sources as a JSON array; [] when none/absent/invalid. +catalogExtraSourcesJson() { + local f; f="$(catalogSourcesFile)" + command -v jq >/dev/null 2>&1 || { echo '[]'; return 0; } + if [[ -s "$f" ]] && jq empty "$f" 2>/dev/null; then + jq -c 'if type=="array" then . else [] end' "$f" 2>/dev/null || echo '[]' + else + echo '[]' + fi +} + +# The full ordered source list as a JSON array. Each entry: +# { idx, name, base, channel, enabled, official, trust } +# idx 1 = official (synthesized from CFG); 2..N = extras in file order. +catalogSourcesJson() { + local off_base off_channel extras + off_base="$(lpReleaseBaseUrl)"; off_base="${off_base%/}" + off_channel="$(lpReleaseChannel)" + extras="$(catalogExtraSourcesJson)" + jq -cn --arg base "$off_base" --arg channel "$off_channel" --argjson extras "$extras" ' + [ { idx: 1, name: "LibrePortal Official", base: $base, channel: $channel, + enabled: true, official: true, trust: "official" } ] + + ( $extras | to_entries | map( + { idx: (.key + 2), + name: (.value.name // .value.base), + base: (.value.base // ""), + channel: (.value.channel // "stable"), + enabled: (if (.value | has("enabled")) then (.value.enabled == true) else true end), + official: false, + trust: "community" } ) )' +} + +# One ENABLED source per line (compact JSON), in priority order — for `while read`. +catalogEnabledSources() { catalogSourcesJson | jq -c '.[] | select(.enabled)'; } + +# Human-readable listing for the CLI. +catalogSourceList() { + catalogSourcesJson | jq -r '.[] | + "[\(.idx)] \(.name) — \(.base)/\(.channel) " + + "(\(if .official then "official ✓" else "unverified" end), " + + "\(if .enabled then "on" else "off" end))"' +} + +# Fetch a THIRD-PARTY catalog index. Deliberately does NOT verify against the +# official key and does NOT touch the shared anti-rollback serial — community +# sources are UNVERIFIED (their payloads are still sha256-pinned at add time by +# the pin carried in this very index). Echoes the index JSON; non-zero on failure. +catalogFetchCommunityIndex() { + local base="$1" channel="$2" tmp idx json + base="${base%/}" + [[ -n "$base" && -n "$channel" ]] || return 1 + command -v jq >/dev/null 2>&1 || return 1 + [[ -n "$(_lpFetchTool)" ]] || return 1 + tmp="$(mktemp -d)"; idx="$tmp/index.json" + if ! _lpDownload "$base/$channel/index.json" "$idx" 2>/dev/null; then rm -rf "$tmp"; return 1; fi + json="$(cat "$idx")"; rm -rf "$tmp" + jq empty <<<"$json" 2>/dev/null || return 1 + printf '%s' "$json" +} + +# --- Mutations (manager-owned file; write via the manager funnel) ----------- + +# catalogSourceAdd [name] [channel] +catalogSourceAdd() { + local url="$1" name="$2" channel="${3:-stable}" + if [[ ! "$url" =~ ^https?://[^[:space:]/]+ ]]; then + isError "catalog: source URL must be http(s)://…"; return 1 + fi + url="${url%/}" + [[ -z "$name" ]] && name="$url" + # Refuse a duplicate base+channel (including the official one). + if catalogSourcesJson | jq -e --arg b "$url" --arg c "$channel" 'any(.[]; .base==$b and .channel==$c)' >/dev/null 2>&1; then + isNotice "catalog: '$url' ($channel) is already a source."; return 0 + fi + local f dir extras new + f="$(catalogSourcesFile)"; dir="$(dirname "$f")" + runInstallOp mkdir -p "$dir" 2>/dev/null || true + extras="$(catalogExtraSourcesJson)" + new="$(jq -c --arg n "$name" --arg b "$url" --arg c "$channel" \ + '. + [ { name: $n, base: $b, channel: $c, enabled: true } ]' <<<"$extras")" || { + isError "catalog: failed to build the new source list."; return 1; } + printf '%s\n' "$new" | runInstallWrite "$f" + isSuccessful "catalog: added source '$name' ($url/$channel)." +} + +# catalogSourceRemove (idx 1 = official, cannot be removed) +catalogSourceRemove() { + local idx="$1" + [[ "$idx" =~ ^[0-9]+$ ]] || { isError "catalog: must be a number."; return 1; } + (( idx == 1 )) && { isError "catalog: source 1 is the official catalog and can't be removed."; return 1; } + local f extras count pos new + f="$(catalogSourcesFile)"; extras="$(catalogExtraSourcesJson)" + count="$(jq 'length' <<<"$extras")" + pos=$(( idx - 2 )) + if (( pos < 0 || pos >= count )); then isError "catalog: no source with idx $idx."; return 1; fi + new="$(jq -c --argjson i "$pos" 'del(.[$i])' <<<"$extras")" + printf '%s\n' "$new" | runInstallWrite "$f" + isSuccessful "catalog: removed source $idx." +} + +# catalogSourceToggle +catalogSourceToggle() { + local idx="$1" on="$2" + [[ "$idx" =~ ^[0-9]+$ ]] || { isError "catalog: must be a number."; return 1; } + (( idx == 1 )) && { isError "catalog: the official catalog is always enabled."; return 1; } + [[ "$on" == "true" || "$on" == "false" ]] || { isError "catalog: toggle value must be true|false."; return 1; } + local f extras count pos new + f="$(catalogSourcesFile)"; extras="$(catalogExtraSourcesJson)" + count="$(jq 'length' <<<"$extras")" + pos=$(( idx - 2 )) + if (( pos < 0 || pos >= count )); then isError "catalog: no source with idx $idx."; return 1; fi + new="$(jq -c --argjson i "$pos" --argjson v "$on" '.[$i].enabled=$v' <<<"$extras")" + printf '%s\n' "$new" | runInstallWrite "$f" + isSuccessful "catalog: source $idx $([[ "$on" == "true" ]] && echo enabled || echo disabled)." +} + +# Re-run the App Center browse scan (so a source change reflects immediately +# instead of waiting for the next `updater check`). +catalogRefresh() { + declare -F webuiRegistryCatalogScan >/dev/null 2>&1 || \ + source "${install_scripts_dir}webui/data/generators/apps/webui_registry_scan.sh" 2>/dev/null + declare -F webuiRegistryCatalogScan >/dev/null 2>&1 && webuiRegistryCatalogScan +} diff --git a/scripts/cli/commands/catalog/cli_catalog_commands.sh b/scripts/cli/commands/catalog/cli_catalog_commands.sh new file mode 100644 index 0000000..1aecfe6 --- /dev/null +++ b/scripts/cli/commands/catalog/cli_catalog_commands.sh @@ -0,0 +1,70 @@ +#!/bin/bash + +cliHandleCatalogCommands() +{ + local sub="$initial_command2" # source | refresh | help + local action="$initial_command3" # list | add | remove | enable | disable + local a1="$initial_command4" + local a2="$initial_command5" + local a3="$initial_command6" + + # Read-side + model helpers (lazy loader may not have sourced these yet). + declare -F lpReleaseBaseUrl >/dev/null 2>&1 || source "$install_scripts_dir/source/fetch.sh" 2>/dev/null + declare -F catalogSourcesJson >/dev/null 2>&1 || source "$install_scripts_dir/catalog/catalog_sources.sh" 2>/dev/null + + case "$sub" in + source|sources) + case "$action" in + ""|list) + catalogSourceList + ;; + add) + [[ -z "$a1" ]] && { isNotice "Usage: catalog source add [name] [channel]"; return; } + if [[ "$LIBREPORTAL_TASK_EXEC" == "1" ]]; then + catalogSourceAdd "$a1" "$a2" "$a3" && catalogRefresh + else + cliTaskRun "libreportal catalog source add $a1 $a2 $a3" "catalog" + fi + ;; + remove) + [[ -z "$a1" ]] && { isNotice "Usage: catalog source remove "; return; } + if [[ "$LIBREPORTAL_TASK_EXEC" == "1" ]]; then + catalogSourceRemove "$a1" && catalogRefresh + else + cliTaskRun "libreportal catalog source remove $a1" "catalog" + fi + ;; + enable) + [[ -z "$a1" ]] && { isNotice "Usage: catalog source enable "; return; } + if [[ "$LIBREPORTAL_TASK_EXEC" == "1" ]]; then + catalogSourceToggle "$a1" true && catalogRefresh + else + cliTaskRun "libreportal catalog source enable $a1" "catalog" + fi + ;; + disable) + [[ -z "$a1" ]] && { isNotice "Usage: catalog source disable "; return; } + if [[ "$LIBREPORTAL_TASK_EXEC" == "1" ]]; then + catalogSourceToggle "$a1" false && catalogRefresh + else + cliTaskRun "libreportal catalog source disable $a1" "catalog" + fi + ;; + *) + isNotice "Invalid catalog source action: $action" + cliShowCatalogHelp + ;; + esac + ;; + refresh) + catalogRefresh + ;; + ""|help) + cliShowCatalogHelp + ;; + *) + isNotice "Unknown catalog subcommand: $sub" + cliShowCatalogHelp + ;; + esac +} diff --git a/scripts/cli/commands/catalog/cli_catalog_header.sh b/scripts/cli/commands/catalog/cli_catalog_header.sh new file mode 100644 index 0000000..b3d50cf --- /dev/null +++ b/scripts/cli/commands/catalog/cli_catalog_header.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +cliShowCatalogHelp() +{ + echo "" + echo "LibrePortal App Catalog — manage the sources the App Center browses." + echo "" + echo "catalog source list" + echo " List catalog sources in priority order (source 1 = official)." + echo "" + echo "catalog source add [name] [channel]" + echo " Add a third-party catalog. channel defaults to 'stable'. Third-party" + echo " sources are UNVERIFIED — you are trusting that host." + echo "" + echo "catalog source remove " + echo " Remove a third-party catalog by index (the official source is fixed)." + echo "" + echo "catalog source enable | disable " + echo " Toggle a third-party catalog on/off without removing it." + echo "" + echo "catalog refresh" + echo " Re-scan all enabled sources and rebuild the App Center browse data." + echo "" +} diff --git a/scripts/source/files/arrays/files_catalog.sh b/scripts/source/files/arrays/files_catalog.sh new file mode 100644 index 0000000..dedb41d --- /dev/null +++ b/scripts/source/files/arrays/files_catalog.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +# This file is auto-generated by generate_arrays.sh +# Do not edit manually - run './scripts/source/files/generate_arrays.sh run' to regenerate + +catalog_scripts=( + "catalog/catalog_sources.sh" + +) diff --git a/scripts/source/files/arrays/files_cli.sh b/scripts/source/files/arrays/files_cli.sh index bc38f54..95949fe 100755 --- a/scripts/source/files/arrays/files_cli.sh +++ b/scripts/source/files/arrays/files_cli.sh @@ -15,6 +15,8 @@ cli_scripts=( "cli/commands/artifact/cli_artifact_header.sh" "cli/commands/backup/cli_backup_commands.sh" "cli/commands/backup/cli_backup_header.sh" + "cli/commands/catalog/cli_catalog_commands.sh" + "cli/commands/catalog/cli_catalog_header.sh" "cli/commands/config/cli_config_commands.sh" "cli/commands/config/cli_config_header.sh" "cli/commands/debug/cli_debug_commands.sh" diff --git a/scripts/source/files/arrays/files_source.sh b/scripts/source/files/arrays/files_source.sh index 55b1206..b3149f8 100755 --- a/scripts/source/files/arrays/files_source.sh +++ b/scripts/source/files/arrays/files_source.sh @@ -8,6 +8,7 @@ source_scripts=( "source/fetch.sh" "source/files/arrays/files_app.sh" "source/files/arrays/files_backup.sh" + "source/files/arrays/files_catalog.sh" "source/files/arrays/files_checks.sh" "source/files/arrays/files_cli.sh" "source/files/arrays/files_config.sh" diff --git a/scripts/source/files/arrays/function_manifest.sh b/scripts/source/files/arrays/function_manifest.sh index a32f7bd..114a472 100644 --- a/scripts/source/files/arrays/function_manifest.sh +++ b/scripts/source/files/arrays/function_manifest.sh @@ -238,6 +238,17 @@ declare -gA LP_FN_MAP=( [borgSnapshotListFiles]="backup/engine/borg_snapshots.sh" [borgSnapshotsJson]="backup/engine/borg_snapshots.sh" [borgSystemSnapshotsJson]="backup/engine/borg_snapshots.sh" + [catalogEnabledSources]="catalog/catalog_sources.sh" + [catalogExtraSourcesJson]="catalog/catalog_sources.sh" + [catalogFetchCommunityIndex]="catalog/catalog_sources.sh" + [catalogRefresh]="catalog/catalog_sources.sh" + [_catalogRowsFrom]="webui/data/generators/apps/webui_registry_scan.sh" + [catalogSourceAdd]="catalog/catalog_sources.sh" + [catalogSourceList]="catalog/catalog_sources.sh" + [catalogSourceRemove]="catalog/catalog_sources.sh" + [catalogSourcesFile]="catalog/catalog_sources.sh" + [catalogSourcesJson]="catalog/catalog_sources.sh" + [catalogSourceToggle]="catalog/catalog_sources.sh" [changeRootOwnedFile]="function/permission/ownership/root_file.sh" [changeUserGroupOnFolder]="function/permission/ownership/folder_group.sh" [checkApplicationsConfigFilesMissingVariables]="config/application/application_missing_variables.sh" @@ -279,6 +290,7 @@ declare -gA LP_FN_MAP=( [cliHandleAppCommands]="cli/commands/app/cli_app_commands.sh" [cliHandleArtifactCommands]="cli/commands/artifact/cli_artifact_commands.sh" [cliHandleBackupCommands]="cli/commands/backup/cli_backup_commands.sh" + [cliHandleCatalogCommands]="cli/commands/catalog/cli_catalog_commands.sh" [cliHandleConfigCommands]="cli/commands/config/cli_config_commands.sh" [cliHandleDebugCommands]="cli/commands/debug/cli_debug_commands.sh" [cliHandleDockertypeCommands]="cli/commands/dockertype/cli_dockertype_commands.sh" @@ -304,6 +316,7 @@ declare -gA LP_FN_MAP=( [cliShowAppHelp]="cli/commands/app/cli_app_header.sh" [cliShowArtifactHelp]="cli/commands/artifact/cli_artifact_header.sh" [cliShowBackupHelp]="cli/commands/backup/cli_backup_header.sh" + [cliShowCatalogHelp]="cli/commands/catalog/cli_catalog_header.sh" [cliShowConfigHelp]="cli/commands/config/cli_config_header.sh" [cliShowDebugHelp]="cli/commands/debug/cli_debug_header.sh" [cliShowDockertypeHelp]="cli/commands/dockertype/cli_dockertype_header.sh" @@ -1205,6 +1218,17 @@ declare -gA LP_FN_ROOT=( [borgSnapshotListFiles]="scripts" [borgSnapshotsJson]="scripts" [borgSystemSnapshotsJson]="scripts" + [catalogEnabledSources]="scripts" + [catalogExtraSourcesJson]="scripts" + [catalogFetchCommunityIndex]="scripts" + [catalogRefresh]="scripts" + [_catalogRowsFrom]="scripts" + [catalogSourceAdd]="scripts" + [catalogSourceList]="scripts" + [catalogSourceRemove]="scripts" + [catalogSourcesFile]="scripts" + [catalogSourcesJson]="scripts" + [catalogSourceToggle]="scripts" [changeRootOwnedFile]="scripts" [changeUserGroupOnFolder]="scripts" [checkApplicationsConfigFilesMissingVariables]="scripts" @@ -1246,6 +1270,7 @@ declare -gA LP_FN_ROOT=( [cliHandleAppCommands]="scripts" [cliHandleArtifactCommands]="scripts" [cliHandleBackupCommands]="scripts" + [cliHandleCatalogCommands]="scripts" [cliHandleConfigCommands]="scripts" [cliHandleDebugCommands]="scripts" [cliHandleDockertypeCommands]="scripts" @@ -1271,6 +1296,7 @@ declare -gA LP_FN_ROOT=( [cliShowAppHelp]="scripts" [cliShowArtifactHelp]="scripts" [cliShowBackupHelp]="scripts" + [cliShowCatalogHelp]="scripts" [cliShowConfigHelp]="scripts" [cliShowDebugHelp]="scripts" [cliShowDockertypeHelp]="scripts" @@ -2193,6 +2219,17 @@ borgSnapshotLatestId() { source "${install_scripts_dir}backup/engine/borg_snapsh borgSnapshotListFiles() { source "${install_scripts_dir}backup/engine/borg_snapshots.sh"; borgSnapshotListFiles "$@"; } borgSnapshotsJson() { source "${install_scripts_dir}backup/engine/borg_snapshots.sh"; borgSnapshotsJson "$@"; } borgSystemSnapshotsJson() { source "${install_scripts_dir}backup/engine/borg_snapshots.sh"; borgSystemSnapshotsJson "$@"; } +catalogEnabledSources() { source "${install_scripts_dir}catalog/catalog_sources.sh"; catalogEnabledSources "$@"; } +catalogExtraSourcesJson() { source "${install_scripts_dir}catalog/catalog_sources.sh"; catalogExtraSourcesJson "$@"; } +catalogFetchCommunityIndex() { source "${install_scripts_dir}catalog/catalog_sources.sh"; catalogFetchCommunityIndex "$@"; } +catalogRefresh() { source "${install_scripts_dir}catalog/catalog_sources.sh"; catalogRefresh "$@"; } +_catalogRowsFrom() { source "${install_scripts_dir}webui/data/generators/apps/webui_registry_scan.sh"; _catalogRowsFrom "$@"; } +catalogSourceAdd() { source "${install_scripts_dir}catalog/catalog_sources.sh"; catalogSourceAdd "$@"; } +catalogSourceList() { source "${install_scripts_dir}catalog/catalog_sources.sh"; catalogSourceList "$@"; } +catalogSourceRemove() { source "${install_scripts_dir}catalog/catalog_sources.sh"; catalogSourceRemove "$@"; } +catalogSourcesFile() { source "${install_scripts_dir}catalog/catalog_sources.sh"; catalogSourcesFile "$@"; } +catalogSourcesJson() { source "${install_scripts_dir}catalog/catalog_sources.sh"; catalogSourcesJson "$@"; } +catalogSourceToggle() { source "${install_scripts_dir}catalog/catalog_sources.sh"; catalogSourceToggle "$@"; } changeRootOwnedFile() { source "${install_scripts_dir}function/permission/ownership/root_file.sh"; changeRootOwnedFile "$@"; } changeUserGroupOnFolder() { source "${install_scripts_dir}function/permission/ownership/folder_group.sh"; changeUserGroupOnFolder "$@"; } checkApplicationsConfigFilesMissingVariables() { source "${install_scripts_dir}config/application/application_missing_variables.sh"; checkApplicationsConfigFilesMissingVariables "$@"; } @@ -2234,6 +2271,7 @@ cliFirewallHeader() { source "${install_scripts_dir}cli/commands/firewall/cli_fi cliHandleAppCommands() { source "${install_scripts_dir}cli/commands/app/cli_app_commands.sh"; cliHandleAppCommands "$@"; } cliHandleArtifactCommands() { source "${install_scripts_dir}cli/commands/artifact/cli_artifact_commands.sh"; cliHandleArtifactCommands "$@"; } cliHandleBackupCommands() { source "${install_scripts_dir}cli/commands/backup/cli_backup_commands.sh"; cliHandleBackupCommands "$@"; } +cliHandleCatalogCommands() { source "${install_scripts_dir}cli/commands/catalog/cli_catalog_commands.sh"; cliHandleCatalogCommands "$@"; } cliHandleConfigCommands() { source "${install_scripts_dir}cli/commands/config/cli_config_commands.sh"; cliHandleConfigCommands "$@"; } cliHandleDebugCommands() { source "${install_scripts_dir}cli/commands/debug/cli_debug_commands.sh"; cliHandleDebugCommands "$@"; } cliHandleDockertypeCommands() { source "${install_scripts_dir}cli/commands/dockertype/cli_dockertype_commands.sh"; cliHandleDockertypeCommands "$@"; } @@ -2259,6 +2297,7 @@ cliRunVerify() { source "${install_scripts_dir}cli/commands/verify/cli_verify_co cliShowAppHelp() { source "${install_scripts_dir}cli/commands/app/cli_app_header.sh"; cliShowAppHelp "$@"; } cliShowArtifactHelp() { source "${install_scripts_dir}cli/commands/artifact/cli_artifact_header.sh"; cliShowArtifactHelp "$@"; } cliShowBackupHelp() { source "${install_scripts_dir}cli/commands/backup/cli_backup_header.sh"; cliShowBackupHelp "$@"; } +cliShowCatalogHelp() { source "${install_scripts_dir}cli/commands/catalog/cli_catalog_header.sh"; cliShowCatalogHelp "$@"; } cliShowConfigHelp() { source "${install_scripts_dir}cli/commands/config/cli_config_header.sh"; cliShowConfigHelp "$@"; } cliShowDebugHelp() { source "${install_scripts_dir}cli/commands/debug/cli_debug_header.sh"; cliShowDebugHelp "$@"; } cliShowDockertypeHelp() { source "${install_scripts_dir}cli/commands/dockertype/cli_dockertype_header.sh"; cliShowDockertypeHelp "$@"; } diff --git a/scripts/webui/data/generators/apps/webui_registry_scan.sh b/scripts/webui/data/generators/apps/webui_registry_scan.sh index bf3edd1..c9e896b 100644 --- a/scripts/webui/data/generators/apps/webui_registry_scan.sh +++ b/scripts/webui/data/generators/apps/webui_registry_scan.sh @@ -1,20 +1,46 @@ #!/bin/bash -# WebUI registry-catalog data generator (the marketplace browse data) +# WebUI registry-catalog data generator (the App Center browse data) # --------------------------------------------------------------------------- # Writes the read-only JSON the App Center merges into its grid as # "Available — Add" cards: # frontend/data/apps/generated/registry_catalog.json # -# It fetches + verifies the signed artifact index (lpFetchIndexInto), selects -# the type:"app" / payload.kind:"bundle" envelopes, annotates each with -# `defined` (a definition exists in the install tree) and `installed` (a live -# dir exists), and mirrors each catalog icon into -# frontend/core/icons/apps/registry/. -# only when it matches the sha256 pin in the signed index — the browser is -# never pointed at a remote host (same-origin only). Written ATOMICALLY -# (temp -> validate -> one runFileWrite); on ANY fetch/verify failure the -# prior file is KEPT. Run by `libreportal updater check`. +# MULTI-SOURCE: it walks the ordered catalog source list (catalogEnabledSources +# — official #1, then any operator-added third-party catalogs). The OFFICIAL +# source is fetched + signature-verified exactly as before (lpFetchIndexInto); +# third-party sources are fetched UNVERIFIED (catalogFetchCommunityIndex) and +# every app they offer is tagged trust="community"/verified=false so the UI can +# show an "unverified source" caution. Apps are merged by slug: an app offered +# by several catalogs carries a `sources[]` array in priority order (highest +# first = the default the Add dialog pre-selects). +# +# Icons are mirrored same-origin ONLY from the official index (sha256-pinned, so +# a tampered/oversized icon is skipped) — third-party catalogs get the default +# icon for now (never point the browser at an untrusted remote host). +# +# Written ATOMICALLY (temp -> validate -> one runFileWrite); on a FAILED OFFICIAL +# fetch the prior file is KEPT untouched. Run by `libreportal updater check` and +# on demand by `catalogRefresh` after a source is added/removed. + +# Rows (one object per app-offering) from a single source's verified index JSON. +# Trust/verified come from the SOURCE, never the artifact's self-claimed .trust — +# a community index can't promote itself to "official". +_catalogRowsFrom() { + local index="$1" sidx="$2" sname="$3" base="$4" channel="$5" strust="$6" verified="$7" icons="$8" + printf '%s' "$index" | jq -c \ + --argjson sidx "$sidx" --arg sname "$sname" --arg base "$base" --arg channel "$channel" \ + --arg strust "$strust" --arg verified "$verified" --argjson icons "$icons" ' + [ .artifacts[]? | select(.type=="app" and .payload.kind=="bundle") + | (.applies_when.app // "") as $slug | select($slug != "") + | { app: $slug, id, version: (.version // 1), title: (.title // $slug), why: (.why // ""), + publisher: (.publisher // ""), trust: $strust, + category: (.meta.category // ""), description: (.meta.description // .why // ""), + long_description: (.meta.long_description // ""), + icon: ($icons[$slug] // null), + src_idx: $sidx, src_name: $sname, base: $base, channel: $channel, + verified: ($verified == "true") } ]' 2>/dev/null +} webuiRegistryCatalogScan() { local out_dir="${containers_dir%/}/libreportal/frontend/data/apps/generated" @@ -24,25 +50,31 @@ webuiRegistryCatalogScan() { local max_icon=262144 runFileOp mkdir -p "$out_dir" 2>/dev/null || true - # Lazy-loader gap: ensure the read primitives are present. + # Lazy-loader gap: ensure the read primitives + source list are present. if ! declare -F lpFetchIndex >/dev/null 2>&1; then source "$install_scripts_dir/source/fetch.sh" 2>/dev/null source "$install_scripts_dir/source/artifacts.sh" 2>/dev/null fi + declare -F catalogEnabledSources >/dev/null 2>&1 || \ + source "$install_scripts_dir/catalog/catalog_sources.sh" 2>/dev/null if ! command -v jq >/dev/null 2>&1; then isNotice "webuiRegistryCatalogScan: jq not available — keeping the prior registry_catalog.json." return 0 fi + # --- OFFICIAL index first (verified; its failure keeps the prior file) --- local index if ! lpFetchIndexInto index; then - isNotice "webuiRegistryCatalogScan: no verified index available — keeping the prior file." + isNotice "webuiRegistryCatalogScan: no verified official index — keeping the prior file." return 0 fi - local signed="false"; [[ "$LP_INDEX_SIGSTATE" == "verified" ]] && signed="true" - local serial now + local verified_official="false"; [[ "$LP_INDEX_SIGSTATE" == "verified" ]] && verified_official="true" + local signed="$verified_official" + local serial now off_base off_channel serial="$(_lpJsonNum "$index" index_serial)" now="$(date -Iseconds 2>/dev/null || date)" + off_base="$(lpReleaseBaseUrl)"; off_base="${off_base%/}" + off_channel="$(lpReleaseChannel)" # Defined apps = definition dirs in the install tree; installed = live dirs. local defined="[]" installed="[]" d @@ -53,11 +85,9 @@ webuiRegistryCatalogScan() { installed="$(for d in "$containers_dir"/*/; do [[ -d "$d" ]] && basename "$d"; done | jq -R . | jq -cs .)" fi - # Mirror the pinned catalog icons locally (slug -> local web path). An icon - # is used only if its bytes match the sha256 the signed index pins — a - # tampered or oversized icon is simply skipped, never served. - local icons_map="{}" base rows - base="$(lpReleaseBaseUrl)" + # Mirror the official catalog's pinned icons locally (slug -> web path). An + # icon is used only if its bytes match the sha256 the SIGNED index pins. + local icons_map="{}" rows rows="$(printf '%s' "$index" | jq -r ' .artifacts[]? | select(.type=="app" and .payload.kind=="bundle") | select((.meta.icon // "") != "" and (.meta.icon_sha256 // "") != "") @@ -74,7 +104,7 @@ webuiRegistryCatalogScan() { icons_map="$(jq -c --arg s "$slug" --arg p "$icon_web/$slug.$ext" '.[$s]=$p' <<<"$icons_map")" continue fi - url="$icon"; case "$url" in http*://*) : ;; *) url="$base/$url" ;; esac + url="$icon"; case "$url" in http*://*) : ;; *) url="$off_base/$url" ;; esac tmpf="$(mktemp)" if _lpDownload "$url" "$tmpf" 2>/dev/null; then got="$(_lpSha256 "$tmpf")" @@ -89,30 +119,57 @@ webuiRegistryCatalogScan() { done <<<"$rows" fi + # --- Walk every enabled source, collect rows + per-source meta ----------- + local all_rows="[]" sources_meta="[]" src + local s_idx s_name s_base s_chan s_official s_index s_verified s_trust s_icons s_rows + while IFS= read -r src; do + [[ -z "$src" ]] && continue + s_idx="$(jq -r '.idx' <<<"$src")" + s_name="$(jq -r '.name' <<<"$src")" + s_base="$(jq -r '.base' <<<"$src")"; s_base="${s_base%/}" + s_chan="$(jq -r '.channel' <<<"$src")" + s_official="$(jq -r '.official' <<<"$src")" + if [[ "$s_official" == "true" ]]; then + s_index="$index"; s_verified="$verified_official"; s_trust="official"; s_icons="$icons_map" + else + if ! s_index="$(catalogFetchCommunityIndex "$s_base" "$s_chan")"; then + isNotice "webuiRegistryCatalogScan: source '$s_name' ($s_base/$s_chan) unreachable — skipping." + continue + fi + s_verified="false"; s_trust="community"; s_icons="{}" + fi + s_rows="$(_catalogRowsFrom "$s_index" "$s_idx" "$s_name" "$s_base" "$s_chan" "$s_trust" "$s_verified" "$s_icons")" + [[ -z "$s_rows" ]] && s_rows="[]" + all_rows="$(jq -cn --argjson a "$all_rows" --argjson b "$s_rows" '$a + $b' 2>/dev/null || printf '%s' "$all_rows")" + sources_meta="$(jq -cn --argjson a "$sources_meta" --argjson s "$src" --arg v "$s_verified" \ + '$a + [ $s + { verified: ($v == "true") } ]' 2>/dev/null || printf '%s' "$sources_meta")" + done < <(catalogEnabledSources) + + # --- Merge rows by slug: one card per app, sources[] in priority order --- + local apps + apps="$(jq -cn --argjson rows "$all_rows" --argjson defined "$defined" --argjson installed "$installed" ' + $rows | group_by(.app) | map( + .[0] as $top + | { id: $top.id, app: $top.app, version: $top.version, title: $top.title, why: $top.why, + publisher: $top.publisher, trust: $top.trust, + category: $top.category, description: $top.description, long_description: $top.long_description, + icon: $top.icon, + defined: (($defined | index($top.app)) != null), + installed: (($installed | index($top.app)) != null), + sources: ( map({ src_idx, src_name, id, version, trust, verified, base, channel }) ) } )' 2>/dev/null)" + [[ -z "$apps" ]] && apps="[]" + local tmp; tmp="$(mktemp)" - printf '%s' "$index" | jq \ + jq -n \ --arg now "$now" --arg signed "$signed" --arg serial "${serial:-0}" \ - --arg src_base "$base" --arg src_channel "$(lpReleaseChannel)" \ - --argjson defined "$defined" --argjson installed "$installed" --argjson icons "$icons_map" ' + --arg ob "$off_base" --arg oc "$off_channel" \ + --argjson sources "$sources_meta" --argjson apps "$apps" ' { generated_at: $now, - signed: ($signed=="true"), - serial: ($serial|tonumber? // 0), - source: { base: $src_base, channel: $src_channel }, - apps: [ .artifacts[]? | select(.type=="app" and .payload.kind=="bundle") - | (.applies_when.app // "") as $slug | select($slug != "") - | { id, - app: $slug, - version: (.version // 1), - title: (.title // $slug), - why: (.why // ""), - publisher: (.publisher // ""), - trust: (.trust // "official"), - category: (.meta.category // ""), - description: (.meta.description // .why // ""), - long_description: (.meta.long_description // ""), - icon: ($icons[$slug] // null), - defined: (($defined | index($slug)) != null), - installed: (($installed | index($slug)) != null) } ] }' > "$tmp" 2>/dev/null + signed: ($signed == "true"), + serial: ($serial | tonumber? // 0), + source: { base: $ob, channel: $oc }, + sources: $sources, + apps: $apps }' > "$tmp" 2>/dev/null if ! jq empty "$tmp" 2>/dev/null; then isNotice "webuiRegistryCatalogScan: generated JSON was invalid — keeping the prior file." @@ -121,6 +178,6 @@ webuiRegistryCatalogScan() { runFileWrite "$out" < "$tmp" rm -f "$tmp" runFileOp chown "$docker_install_user":"$docker_install_user" "$out" 2>/dev/null || true - local n; n="$(jq '.apps | length' "$out" 2>/dev/null || echo '?')" - isSuccessful "Registry catalog refreshed ($n app(s), serial=${serial:-?}, signed=$signed)." + local n ns; n="$(jq '.apps | length' "$out" 2>/dev/null || echo '?')"; ns="$(jq '.sources | length' "$out" 2>/dev/null || echo '?')" + isSuccessful "Registry catalog refreshed ($n app(s) across $ns source(s), serial=${serial:-?}, signed=$signed)." }