From abf3a65c88f67846e16c505b4d9ce8baedda446b Mon Sep 17 00:00:00 2001 From: librelad Date: Mon, 6 Jul 2026 22:01:22 +0100 Subject: [PATCH] refactor(catalog): move sources into a config file (CFG_CATALOG_1..9), generate from it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per the intended model: catalog sources should live in the config place (like domains), and registry_catalog.json should be a purely GENERATED artifact derived from them — not the source of truth. Replaces the earlier $docker_dir/catalog/sources.json store. - New configs/general/general_catalogs — CFG_CATALOG_1..9, one catalog base URL per slot ("url" or "url|channel"), domains-style. Official stays pinned as source #1 (derived from CFG_RELEASE_BASE_URL, not listed here). Slot N → source idx N+1 (stable id for the Add picker / `app add --source`). - catalog_sources.sh now reads/writes those CFG vars (via updateConfigOption) instead of a JSON file; dropped catalogSourcesFile + the enable/disable toggle (presence = enabled; remove = clear the slot). - configUpdateBatch regenerates registry_catalog.json when a CFG_CATALOG_* key changed — so pressing Save in the WebUI rebuilds the browse data. - webuiRegistryCatalogScan is unchanged (still iterates catalogEnabledSources). Verified: CFG_CATALOG_1/2 → sources at idx 2/3, empty slots skipped, url|channel parsed, official pinned at idx 1. Signed-off-by: librelad Co-Authored-By: Claude Opus 4.8 (1M context) --- configs/general/.category | 2 +- configs/general/general_catalogs | 17 ++ scripts/catalog/catalog_sources.sh | 151 ++++++++---------- .../commands/catalog/cli_catalog_commands.sh | 24 +-- .../commands/catalog/cli_catalog_header.sh | 9 +- scripts/config/config_update.sh | 11 ++ .../source/files/arrays/function_manifest.sh | 7 +- 7 files changed, 102 insertions(+), 119 deletions(-) create mode 100644 configs/general/general_catalogs diff --git a/configs/general/.category b/configs/general/.category index 81beef1..6e488a0 100755 --- a/configs/general/.category +++ b/configs/general/.category @@ -2,4 +2,4 @@ TITLE=General DESCRIPTION=Basic system settings and identification ICON=general ORDER=1 -SUBCATEGORY_ORDER=general_basic,general_mail,general_install,general_docker_install,general_terminal,general_libreportal +SUBCATEGORY_ORDER=general_basic,general_mail,general_install,general_docker_install,general_terminal,general_catalogs,general_libreportal diff --git a/configs/general/general_catalogs b/configs/general/general_catalogs new file mode 100644 index 0000000..95f3c47 --- /dev/null +++ b/configs/general/general_catalogs @@ -0,0 +1,17 @@ +# ================================================================================ +# App Catalogs - Extra app-catalog sources the App Center browses +# ================================================================================ +# Each slot is a catalog base URL served with a signed //index.json +# (channel defaults to stable; append | for another). Sources are tried +# in priority order 1..9. The official catalog (from the Release Host) is always +# source #1 and is NOT listed here. Third-party catalogs are UNVERIFIED — you are +# trusting that host. Edit these from the App Center's Catalog Sources block. +CFG_CATALOG_1= # Catalog 1 - Extra catalog base URL, e.g. https://catalog.example.org +CFG_CATALOG_2= # Catalog 2 - Extra catalog base URL +CFG_CATALOG_3= # Catalog 3 - Extra catalog base URL +CFG_CATALOG_4= # Catalog 4 - Extra catalog base URL +CFG_CATALOG_5= # Catalog 5 - Extra catalog base URL +CFG_CATALOG_6= # Catalog 6 - Extra catalog base URL +CFG_CATALOG_7= # Catalog 7 - Extra catalog base URL +CFG_CATALOG_8= # Catalog 8 - Extra catalog base URL +CFG_CATALOG_9= # Catalog 9 - Extra catalog base URL diff --git a/scripts/catalog/catalog_sources.sh b/scripts/catalog/catalog_sources.sh index 86e476f..1bd91dc 100644 --- a/scripts/catalog/catalog_sources.sh +++ b/scripts/catalog/catalog_sources.sh @@ -2,40 +2,45 @@ # # Catalog sources — the ordered list of app-catalog URLs the App Center browses. # --------------------------------------------------------------------------- -# Source #1 is ALWAYS the official LibrePortal catalog, synthesized live from -# CFG_RELEASE_BASE_URL / CFG_RELEASE_CHANNEL so it can't be edited or removed and -# always tracks the release host. Extra (third-party) sources are stored as a -# small JSON array in their OWN file under the live configs dir and are treated -# as UNVERIFIED (the operator added the URL themselves). +# SOURCE OF TRUTH = the CFG_CATALOG_1..9 config vars in configs/general/ +# general_catalogs (edited from the WebUI Catalog Sources block, like domains). +# registry_catalog.json is a GENERATED artifact derived from these + the official +# source — rebuilt by webuiRegistryCatalogScan on config save + `updater check`. # -# Priority = list order: the official catalog is first, then extras in the order -# they were added. When the same app slug is offered by more than one source the -# highest-priority source that has it is the default (the Add dialog can override). +# Source #1 is ALWAYS the official catalog, synthesized live from +# CFG_RELEASE_BASE_URL / CFG_RELEASE_CHANNEL (pinned, NOT in general_catalogs). +# Each extra slot CFG_CATALOG_ maps to source idx (a stable id the Add +# dialog / `app add --source` can reference) and is UNVERIFIED third-party. # -# IMPORTANT SCOPE: this list governs APP BROWSE + APP ADD only. LibrePortal's own -# updates and hotfixes still resolve from the official CFG_RELEASE_BASE_URL alone -# (lpFetchIndex is untouched) — a third-party catalog can never become a channel -# for system updates. +# Priority = official first, then slots 1..9 ascending; the highest-priority +# source offering an app is the default (Add dialog can pick another). +# +# SCOPE: browse + add only. System updates/hotfixes still resolve from the +# official CFG_RELEASE_BASE_URL alone (lpFetchIndex untouched) — a third-party +# catalog can never become a system-update channel. -# Its own file in the manager's persistent state root ($docker_dir — beside the -# apps database), NOT under configs/ (every file there is treated as a sourced -# .config, so a JSON there would be executed as bash at startup). -catalogSourcesFile() { echo "${docker_dir%/}/catalog/sources.json"; } +CATALOG_MAX_SLOTS=9 -# The extra (non-official) sources as a JSON array; [] when none/absent/invalid. +# Extra (third-party) sources as a JSON array [{idx,name,base,channel}] in slot +# order, skipping empty slots. A slot value is "url" or "url|channel". catalogExtraSourcesJson() { - local f; f="$(catalogSourcesFile)" command -v jq >/dev/null 2>&1 || { echo '[]'; return 0; } - if [[ -s "$f" ]] && jq empty "$f" 2>/dev/null; then - jq -c 'if type=="array" then . else [] end' "$f" 2>/dev/null || echo '[]' - else - echo '[]' - fi + local arr='[]' n var val url channel host + for (( n=1; n<=CATALOG_MAX_SLOTS; n++ )); do + var="CFG_CATALOG_${n}"; val="${!var:-}" + [[ -z "$val" ]] && continue + url="${val%%|*}" + channel="stable"; [[ "$val" == *"|"* ]] && channel="${val##*|}" + [[ "$url" =~ ^https?://[^[:space:]/]+ ]] || continue + url="${url%/}" + host="${url#*://}"; host="${host%%/*}" + arr="$(jq -c --argjson i "$(( n + 1 ))" --arg name "$host" --arg base "$url" --arg ch "$channel" \ + '. + [ { idx:$i, name:$name, base:$base, channel:$ch } ]' <<<"$arr")" + done + printf '%s' "$arr" } -# The full ordered source list as a JSON array. Each entry: -# { idx, name, base, channel, enabled, official, trust } -# idx 1 = official (synthesized from CFG); 2..N = extras in file order. +# The full ordered source list as a JSON array. Each: {idx,name,base,channel,official,trust}. catalogSourcesJson() { local off_base off_channel extras off_base="$(lpReleaseBaseUrl)"; off_base="${off_base%/}" @@ -43,32 +48,24 @@ catalogSourcesJson() { extras="$(catalogExtraSourcesJson)" jq -cn --arg base "$off_base" --arg channel "$off_channel" --argjson extras "$extras" ' [ { idx: 1, name: "LibrePortal Official", base: $base, channel: $channel, - enabled: true, official: true, trust: "official" } ] - + ( $extras | to_entries | map( - { idx: (.key + 2), - name: (.value.name // .value.base), - base: (.value.base // ""), - channel: (.value.channel // "stable"), - enabled: (if (.value | has("enabled")) then (.value.enabled == true) else true end), - official: false, - trust: "community" } ) )' + official: true, trust: "official" } ] + + ( $extras | map( . + { official: false, trust: "community" } ) )' } -# One ENABLED source per line (compact JSON), in priority order — for `while read`. -catalogEnabledSources() { catalogSourcesJson | jq -c '.[] | select(.enabled)'; } +# One source per line (compact JSON), priority order — for `while read`. Every +# configured source is active (empty slots are skipped upstream). +catalogEnabledSources() { catalogSourcesJson | jq -c '.[]'; } # Human-readable listing for the CLI. catalogSourceList() { catalogSourcesJson | jq -r '.[] | - "[\(.idx)] \(.name) — \(.base)/\(.channel) " + - "(\(if .official then "official ✓" else "unverified" end), " + - "\(if .enabled then "on" else "off" end))"' + "[\(.idx)] \(.name) — \(.base)/\(.channel) (\(if .official then "official ✓" else "unverified" end))"' } -# Fetch a THIRD-PARTY catalog index. Deliberately does NOT verify against the -# official key and does NOT touch the shared anti-rollback serial — community -# sources are UNVERIFIED (their payloads are still sha256-pinned at add time by -# the pin carried in this very index). Echoes the index JSON; non-zero on failure. +# Fetch a THIRD-PARTY catalog index. Deliberately NOT verified against the +# official key and does NOT touch the anti-rollback serial — community sources +# are UNVERIFIED (their payloads are still sha256-pinned at add time by the pin +# carried in this index). Echoes the index JSON; non-zero on failure. catalogFetchCommunityIndex() { local base="$1" channel="$2" tmp idx json base="${base%/}" @@ -82,64 +79,46 @@ catalogFetchCommunityIndex() { printf '%s' "$json" } -# --- Mutations (manager-owned file; write via the manager funnel) ----------- +# --- Mutations: write the CFG_CATALOG_ vars via the config system --------- -# catalogSourceAdd [name] [channel] +# catalogSourceAdd [channel] — into the first free slot. catalogSourceAdd() { - local url="$1" name="$2" channel="${3:-stable}" + local url="$1" channel="$2" if [[ ! "$url" =~ ^https?://[^[:space:]/]+ ]]; then isError "catalog: source URL must be http(s)://…"; return 1 fi url="${url%/}" - [[ -z "$name" ]] && name="$url" - # Refuse a duplicate base+channel (including the official one). - if catalogSourcesJson | jq -e --arg b "$url" --arg c "$channel" 'any(.[]; .base==$b and .channel==$c)' >/dev/null 2>&1; then - isNotice "catalog: '$url' ($channel) is already a source."; return 0 + local stored="$url"; [[ -n "$channel" && "$channel" != "stable" ]] && stored="$url|$channel" + if catalogSourcesJson | jq -e --arg b "$url" 'any(.[]; .base==$b)' >/dev/null 2>&1; then + isNotice "catalog: '$url' is already a source."; return 0 fi - local f dir extras new - f="$(catalogSourcesFile)"; dir="$(dirname "$f")" - runInstallOp mkdir -p "$dir" 2>/dev/null || true - extras="$(catalogExtraSourcesJson)" - new="$(jq -c --arg n "$name" --arg b "$url" --arg c "$channel" \ - '. + [ { name: $n, base: $b, channel: $c, enabled: true } ]' <<<"$extras")" || { - isError "catalog: failed to build the new source list."; return 1; } - printf '%s\n' "$new" | runInstallWrite "$f" - isSuccessful "catalog: added source '$name' ($url/$channel)." + local n var + for (( n=1; n<=CATALOG_MAX_SLOTS; n++ )); do + var="CFG_CATALOG_${n}" + if [[ -z "${!var:-}" ]]; then + updateConfigOption "$var" "$stored" >/dev/null || { isError "catalog: failed to write $var."; return 1; } + isSuccessful "catalog: added source '$url' (slot $n)." + catalogRefresh + return 0 + fi + done + isError "catalog: all $CATALOG_MAX_SLOTS catalog slots are full — remove one first."; return 1 } -# catalogSourceRemove (idx 1 = official, cannot be removed) +# catalogSourceRemove — idx 1 = official (fixed); idx N+1 = CFG_CATALOG_N. catalogSourceRemove() { local idx="$1" [[ "$idx" =~ ^[0-9]+$ ]] || { isError "catalog: must be a number."; return 1; } (( idx == 1 )) && { isError "catalog: source 1 is the official catalog and can't be removed."; return 1; } - local f extras count pos new - f="$(catalogSourcesFile)"; extras="$(catalogExtraSourcesJson)" - count="$(jq 'length' <<<"$extras")" - pos=$(( idx - 2 )) - if (( pos < 0 || pos >= count )); then isError "catalog: no source with idx $idx."; return 1; fi - new="$(jq -c --argjson i "$pos" 'del(.[$i])' <<<"$extras")" - printf '%s\n' "$new" | runInstallWrite "$f" + local slot=$(( idx - 1 )) var="CFG_CATALOG_$(( idx - 1 ))" + (( slot < 1 || slot > CATALOG_MAX_SLOTS )) && { isError "catalog: no source with idx $idx."; return 1; } + [[ -z "${!var:-}" ]] && { isError "catalog: no source with idx $idx."; return 1; } + updateConfigOption "$var" "" >/dev/null || { isError "catalog: failed to clear $var."; return 1; } isSuccessful "catalog: removed source $idx." + catalogRefresh } -# catalogSourceToggle -catalogSourceToggle() { - local idx="$1" on="$2" - [[ "$idx" =~ ^[0-9]+$ ]] || { isError "catalog: must be a number."; return 1; } - (( idx == 1 )) && { isError "catalog: the official catalog is always enabled."; return 1; } - [[ "$on" == "true" || "$on" == "false" ]] || { isError "catalog: toggle value must be true|false."; return 1; } - local f extras count pos new - f="$(catalogSourcesFile)"; extras="$(catalogExtraSourcesJson)" - count="$(jq 'length' <<<"$extras")" - pos=$(( idx - 2 )) - if (( pos < 0 || pos >= count )); then isError "catalog: no source with idx $idx."; return 1; fi - new="$(jq -c --argjson i "$pos" --argjson v "$on" '.[$i].enabled=$v' <<<"$extras")" - printf '%s\n' "$new" | runInstallWrite "$f" - isSuccessful "catalog: source $idx $([[ "$on" == "true" ]] && echo enabled || echo disabled)." -} - -# Re-run the App Center browse scan (so a source change reflects immediately -# instead of waiting for the next `updater check`). +# Re-run the App Center browse scan so a source change reflects immediately. catalogRefresh() { declare -F webuiRegistryCatalogScan >/dev/null 2>&1 || \ source "${install_scripts_dir}webui/data/generators/apps/webui_registry_scan.sh" 2>/dev/null diff --git a/scripts/cli/commands/catalog/cli_catalog_commands.sh b/scripts/cli/commands/catalog/cli_catalog_commands.sh index 1aecfe6..0626752 100644 --- a/scripts/cli/commands/catalog/cli_catalog_commands.sh +++ b/scripts/cli/commands/catalog/cli_catalog_commands.sh @@ -19,37 +19,21 @@ cliHandleCatalogCommands() catalogSourceList ;; add) - [[ -z "$a1" ]] && { isNotice "Usage: catalog source add [name] [channel]"; return; } + [[ -z "$a1" ]] && { isNotice "Usage: catalog source add [channel]"; return; } if [[ "$LIBREPORTAL_TASK_EXEC" == "1" ]]; then - catalogSourceAdd "$a1" "$a2" "$a3" && catalogRefresh + catalogSourceAdd "$a1" "$a2" else - cliTaskRun "libreportal catalog source add $a1 $a2 $a3" "catalog" + cliTaskRun "libreportal catalog source add $a1 $a2" "catalog" fi ;; remove) [[ -z "$a1" ]] && { isNotice "Usage: catalog source remove "; return; } if [[ "$LIBREPORTAL_TASK_EXEC" == "1" ]]; then - catalogSourceRemove "$a1" && catalogRefresh + catalogSourceRemove "$a1" else cliTaskRun "libreportal catalog source remove $a1" "catalog" fi ;; - enable) - [[ -z "$a1" ]] && { isNotice "Usage: catalog source enable "; return; } - if [[ "$LIBREPORTAL_TASK_EXEC" == "1" ]]; then - catalogSourceToggle "$a1" true && catalogRefresh - else - cliTaskRun "libreportal catalog source enable $a1" "catalog" - fi - ;; - disable) - [[ -z "$a1" ]] && { isNotice "Usage: catalog source disable "; return; } - if [[ "$LIBREPORTAL_TASK_EXEC" == "1" ]]; then - catalogSourceToggle "$a1" false && catalogRefresh - else - cliTaskRun "libreportal catalog source disable $a1" "catalog" - fi - ;; *) isNotice "Invalid catalog source action: $action" cliShowCatalogHelp diff --git a/scripts/cli/commands/catalog/cli_catalog_header.sh b/scripts/cli/commands/catalog/cli_catalog_header.sh index b3d50cf..1807472 100644 --- a/scripts/cli/commands/catalog/cli_catalog_header.sh +++ b/scripts/cli/commands/catalog/cli_catalog_header.sh @@ -8,16 +8,13 @@ cliShowCatalogHelp() echo "catalog source list" echo " List catalog sources in priority order (source 1 = official)." echo "" - echo "catalog source add [name] [channel]" - echo " Add a third-party catalog. channel defaults to 'stable'. Third-party" - echo " sources are UNVERIFIED — you are trusting that host." + echo "catalog source add [channel]" + echo " Add a third-party catalog (into the next free CFG_CATALOG slot)." + echo " channel defaults to 'stable'. Third-party sources are UNVERIFIED." echo "" echo "catalog source remove " echo " Remove a third-party catalog by index (the official source is fixed)." echo "" - echo "catalog source enable | disable " - echo " Toggle a third-party catalog on/off without removing it." - echo "" echo "catalog refresh" echo " Re-scan all enabled sources and rebuild the App Center browse data." echo "" diff --git a/scripts/config/config_update.sh b/scripts/config/config_update.sh index 440f421..6aa2e17 100755 --- a/scripts/config/config_update.sh +++ b/scripts/config/config_update.sh @@ -14,6 +14,7 @@ configUpdateBatch() local applied=0 local failed=0 + local catalog_changed=false IFS='|' read -ra pairs <<< "$encoded_pairs" for pair in "${pairs[@]}"; do @@ -21,6 +22,7 @@ configUpdateBatch() if [[ "$pair" =~ ^(CFG_[A-Z0-9_]+)=(.*)$ ]]; then local key="${BASH_REMATCH[1]}" local value="${BASH_REMATCH[2]//%7C/|}" + [[ "$key" == CFG_CATALOG_* ]] && catalog_changed=true if updateConfigOption "$key" "$value"; then ((applied++)) else @@ -50,6 +52,15 @@ configUpdateBatch() isSuccessful "System scan completed." fi + # Catalog sources are the source of truth for the App Center browse data; + # rebuild the generated registry_catalog.json when a CFG_CATALOG_* changed. + if [[ "$catalog_changed" == true ]] && declare -f catalogRefresh >/dev/null 2>&1; then + echo "" + echo "---- Refreshing catalog sources..." + echo "" + catalogRefresh + fi + echo "" isSuccessful "Configuration update complete." } diff --git a/scripts/source/files/arrays/function_manifest.sh b/scripts/source/files/arrays/function_manifest.sh index 809a38a..f6006b2 100644 --- a/scripts/source/files/arrays/function_manifest.sh +++ b/scripts/source/files/arrays/function_manifest.sh @@ -246,9 +246,7 @@ declare -gA LP_FN_MAP=( [catalogSourceAdd]="catalog/catalog_sources.sh" [catalogSourceList]="catalog/catalog_sources.sh" [catalogSourceRemove]="catalog/catalog_sources.sh" - [catalogSourcesFile]="catalog/catalog_sources.sh" [catalogSourcesJson]="catalog/catalog_sources.sh" - [catalogSourceToggle]="catalog/catalog_sources.sh" [changeRootOwnedFile]="function/permission/ownership/root_file.sh" [changeUserGroupOnFolder]="function/permission/ownership/folder_group.sh" [checkApplicationsConfigFilesMissingVariables]="config/application/application_missing_variables.sh" @@ -1230,9 +1228,7 @@ declare -gA LP_FN_ROOT=( [catalogSourceAdd]="scripts" [catalogSourceList]="scripts" [catalogSourceRemove]="scripts" - [catalogSourcesFile]="scripts" [catalogSourcesJson]="scripts" - [catalogSourceToggle]="scripts" [changeRootOwnedFile]="scripts" [changeUserGroupOnFolder]="scripts" [checkApplicationsConfigFilesMissingVariables]="scripts" @@ -1984,6 +1980,7 @@ declare -gA LP_FN_ROOT=( LP_EAGER_FILES=( "scripts:backup/db/backup_db.sh" "scripts:backup/files/backup_files.sh" + "scripts:catalog/catalog_sources.sh" "scripts:docker/install/rootless/rootless_apparmor.sh" "scripts:docker/type_switcher/swap_docker_type.sh" "scripts:migrate/migrate_url_rewrite.sh" @@ -2237,9 +2234,7 @@ _catalogRowsFrom() { unset -f _catalogRowsFrom; source "${install_scripts_dir}we catalogSourceAdd() { unset -f catalogSourceAdd; source "${install_scripts_dir}catalog/catalog_sources.sh"; catalogSourceAdd "$@"; } catalogSourceList() { unset -f catalogSourceList; source "${install_scripts_dir}catalog/catalog_sources.sh"; catalogSourceList "$@"; } catalogSourceRemove() { unset -f catalogSourceRemove; source "${install_scripts_dir}catalog/catalog_sources.sh"; catalogSourceRemove "$@"; } -catalogSourcesFile() { unset -f catalogSourcesFile; source "${install_scripts_dir}catalog/catalog_sources.sh"; catalogSourcesFile "$@"; } catalogSourcesJson() { unset -f catalogSourcesJson; source "${install_scripts_dir}catalog/catalog_sources.sh"; catalogSourcesJson "$@"; } -catalogSourceToggle() { unset -f catalogSourceToggle; source "${install_scripts_dir}catalog/catalog_sources.sh"; catalogSourceToggle "$@"; } changeRootOwnedFile() { unset -f changeRootOwnedFile; source "${install_scripts_dir}function/permission/ownership/root_file.sh"; changeRootOwnedFile "$@"; } changeUserGroupOnFolder() { unset -f changeUserGroupOnFolder; source "${install_scripts_dir}function/permission/ownership/folder_group.sh"; changeUserGroupOnFolder "$@"; } checkApplicationsConfigFilesMissingVariables() { unset -f checkApplicationsConfigFilesMissingVariables; source "${install_scripts_dir}config/application/application_missing_variables.sh"; checkApplicationsConfigFilesMissingVariables "$@"; }