refactor(catalog): move sources into a config file (CFG_CATALOG_1..9), generate from it

Per the intended model: catalog sources should live in the config place (like
domains), and registry_catalog.json should be a purely GENERATED artifact
derived from them — not the source of truth. Replaces the earlier
$docker_dir/catalog/sources.json store.

- New configs/general/general_catalogs — CFG_CATALOG_1..9, one catalog base URL
  per slot ("url" or "url|channel"), domains-style. Official stays pinned as
  source #1 (derived from CFG_RELEASE_BASE_URL, not listed here). Slot N → source
  idx N+1 (stable id for the Add picker / `app add --source`).
- catalog_sources.sh now reads/writes those CFG vars (via updateConfigOption)
  instead of a JSON file; dropped catalogSourcesFile + the enable/disable toggle
  (presence = enabled; remove = clear the slot).
- configUpdateBatch regenerates registry_catalog.json when a CFG_CATALOG_* key
  changed — so pressing Save in the WebUI rebuilds the browse data.
- webuiRegistryCatalogScan is unchanged (still iterates catalogEnabledSources).

Verified: CFG_CATALOG_1/2 → sources at idx 2/3, empty slots skipped, url|channel
parsed, official pinned at idx 1.

Signed-off-by: librelad <librelad@digitalangels.vip>
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
librelad 2026-07-06 22:01:22 +01:00
parent 2ab1aa3850
commit abf3a65c88
7 changed files with 102 additions and 119 deletions

View File

@ -2,4 +2,4 @@ TITLE=General
DESCRIPTION=Basic system settings and identification
ICON=general
ORDER=1
SUBCATEGORY_ORDER=general_basic,general_mail,general_install,general_docker_install,general_terminal,general_libreportal
SUBCATEGORY_ORDER=general_basic,general_mail,general_install,general_docker_install,general_terminal,general_catalogs,general_libreportal

View File

@ -0,0 +1,17 @@
# ================================================================================
# App Catalogs - Extra app-catalog sources the App Center browses
# ================================================================================
# Each slot is a catalog base URL served with a signed /<channel>/index.json
# (channel defaults to stable; append |<channel> for another). Sources are tried
# in priority order 1..9. The official catalog (from the Release Host) is always
# source #1 and is NOT listed here. Third-party catalogs are UNVERIFIED — you are
# trusting that host. Edit these from the App Center's Catalog Sources block.
CFG_CATALOG_1= # Catalog 1 - Extra catalog base URL, e.g. https://catalog.example.org
CFG_CATALOG_2= # Catalog 2 - Extra catalog base URL
CFG_CATALOG_3= # Catalog 3 - Extra catalog base URL
CFG_CATALOG_4= # Catalog 4 - Extra catalog base URL
CFG_CATALOG_5= # Catalog 5 - Extra catalog base URL
CFG_CATALOG_6= # Catalog 6 - Extra catalog base URL
CFG_CATALOG_7= # Catalog 7 - Extra catalog base URL
CFG_CATALOG_8= # Catalog 8 - Extra catalog base URL
CFG_CATALOG_9= # Catalog 9 - Extra catalog base URL

View File

@ -2,40 +2,45 @@
#
# Catalog sources — the ordered list of app-catalog URLs the App Center browses.
# ---------------------------------------------------------------------------
# Source #1 is ALWAYS the official LibrePortal catalog, synthesized live from
# CFG_RELEASE_BASE_URL / CFG_RELEASE_CHANNEL so it can't be edited or removed and
# always tracks the release host. Extra (third-party) sources are stored as a
# small JSON array in their OWN file under the live configs dir and are treated
# as UNVERIFIED (the operator added the URL themselves).
# SOURCE OF TRUTH = the CFG_CATALOG_1..9 config vars in configs/general/
# general_catalogs (edited from the WebUI Catalog Sources block, like domains).
# registry_catalog.json is a GENERATED artifact derived from these + the official
# source — rebuilt by webuiRegistryCatalogScan on config save + `updater check`.
#
# Priority = list order: the official catalog is first, then extras in the order
# they were added. When the same app slug is offered by more than one source the
# highest-priority source that has it is the default (the Add dialog can override).
# Source #1 is ALWAYS the official catalog, synthesized live from
# CFG_RELEASE_BASE_URL / CFG_RELEASE_CHANNEL (pinned, NOT in general_catalogs).
# Each extra slot CFG_CATALOG_<N> maps to source idx <N+1> (a stable id the Add
# dialog / `app add --source` can reference) and is UNVERIFIED third-party.
#
# IMPORTANT SCOPE: this list governs APP BROWSE + APP ADD only. LibrePortal's own
# updates and hotfixes still resolve from the official CFG_RELEASE_BASE_URL alone
# (lpFetchIndex is untouched) — a third-party catalog can never become a channel
# for system updates.
# Priority = official first, then slots 1..9 ascending; the highest-priority
# source offering an app is the default (Add dialog can pick another).
#
# SCOPE: browse + add only. System updates/hotfixes still resolve from the
# official CFG_RELEASE_BASE_URL alone (lpFetchIndex untouched) — a third-party
# catalog can never become a system-update channel.
# Its own file in the manager's persistent state root ($docker_dir — beside the
# apps database), NOT under configs/ (every file there is treated as a sourced
# .config, so a JSON there would be executed as bash at startup).
catalogSourcesFile() { echo "${docker_dir%/}/catalog/sources.json"; }
CATALOG_MAX_SLOTS=9
# The extra (non-official) sources as a JSON array; [] when none/absent/invalid.
# Extra (third-party) sources as a JSON array [{idx,name,base,channel}] in slot
# order, skipping empty slots. A slot value is "url" or "url|channel".
catalogExtraSourcesJson() {
local f; f="$(catalogSourcesFile)"
command -v jq >/dev/null 2>&1 || { echo '[]'; return 0; }
if [[ -s "$f" ]] && jq empty "$f" 2>/dev/null; then
jq -c 'if type=="array" then . else [] end' "$f" 2>/dev/null || echo '[]'
else
echo '[]'
fi
local arr='[]' n var val url channel host
for (( n=1; n<=CATALOG_MAX_SLOTS; n++ )); do
var="CFG_CATALOG_${n}"; val="${!var:-}"
[[ -z "$val" ]] && continue
url="${val%%|*}"
channel="stable"; [[ "$val" == *"|"* ]] && channel="${val##*|}"
[[ "$url" =~ ^https?://[^[:space:]/]+ ]] || continue
url="${url%/}"
host="${url#*://}"; host="${host%%/*}"
arr="$(jq -c --argjson i "$(( n + 1 ))" --arg name "$host" --arg base "$url" --arg ch "$channel" \
'. + [ { idx:$i, name:$name, base:$base, channel:$ch } ]' <<<"$arr")"
done
printf '%s' "$arr"
}
# The full ordered source list as a JSON array. Each entry:
# { idx, name, base, channel, enabled, official, trust }
# idx 1 = official (synthesized from CFG); 2..N = extras in file order.
# The full ordered source list as a JSON array. Each: {idx,name,base,channel,official,trust}.
catalogSourcesJson() {
local off_base off_channel extras
off_base="$(lpReleaseBaseUrl)"; off_base="${off_base%/}"
@ -43,32 +48,24 @@ catalogSourcesJson() {
extras="$(catalogExtraSourcesJson)"
jq -cn --arg base "$off_base" --arg channel "$off_channel" --argjson extras "$extras" '
[ { idx: 1, name: "LibrePortal Official", base: $base, channel: $channel,
enabled: true, official: true, trust: "official" } ]
+ ( $extras | to_entries | map(
{ idx: (.key + 2),
name: (.value.name // .value.base),
base: (.value.base // ""),
channel: (.value.channel // "stable"),
enabled: (if (.value | has("enabled")) then (.value.enabled == true) else true end),
official: false,
trust: "community" } ) )'
official: true, trust: "official" } ]
+ ( $extras | map( . + { official: false, trust: "community" } ) )'
}
# One ENABLED source per line (compact JSON), in priority order — for `while read`.
catalogEnabledSources() { catalogSourcesJson | jq -c '.[] | select(.enabled)'; }
# One source per line (compact JSON), priority order — for `while read`. Every
# configured source is active (empty slots are skipped upstream).
catalogEnabledSources() { catalogSourcesJson | jq -c '.[]'; }
# Human-readable listing for the CLI.
catalogSourceList() {
catalogSourcesJson | jq -r '.[] |
"[\(.idx)] \(.name) — \(.base)/\(.channel) " +
"(\(if .official then "official ✓" else "unverified" end), " +
"\(if .enabled then "on" else "off" end))"'
"[\(.idx)] \(.name) — \(.base)/\(.channel) (\(if .official then "official ✓" else "unverified" end))"'
}
# Fetch a THIRD-PARTY catalog index. Deliberately does NOT verify against the
# official key and does NOT touch the shared anti-rollback serial — community
# sources are UNVERIFIED (their payloads are still sha256-pinned at add time by
# the pin carried in this very index). Echoes the index JSON; non-zero on failure.
# Fetch a THIRD-PARTY catalog index. Deliberately NOT verified against the
# official key and does NOT touch the anti-rollback serial — community sources
# are UNVERIFIED (their payloads are still sha256-pinned at add time by the pin
# carried in this index). Echoes the index JSON; non-zero on failure.
catalogFetchCommunityIndex() {
local base="$1" channel="$2" tmp idx json
base="${base%/}"
@ -82,64 +79,46 @@ catalogFetchCommunityIndex() {
printf '%s' "$json"
}
# --- Mutations (manager-owned file; write via the manager funnel) -----------
# --- Mutations: write the CFG_CATALOG_<N> vars via the config system ---------
# catalogSourceAdd <url> [name] [channel]
# catalogSourceAdd <url> [channel] — into the first free slot.
catalogSourceAdd() {
local url="$1" name="$2" channel="${3:-stable}"
local url="$1" channel="$2"
if [[ ! "$url" =~ ^https?://[^[:space:]/]+ ]]; then
isError "catalog: source URL must be http(s)://…"; return 1
fi
url="${url%/}"
[[ -z "$name" ]] && name="$url"
# Refuse a duplicate base+channel (including the official one).
if catalogSourcesJson | jq -e --arg b "$url" --arg c "$channel" 'any(.[]; .base==$b and .channel==$c)' >/dev/null 2>&1; then
isNotice "catalog: '$url' ($channel) is already a source."; return 0
local stored="$url"; [[ -n "$channel" && "$channel" != "stable" ]] && stored="$url|$channel"
if catalogSourcesJson | jq -e --arg b "$url" 'any(.[]; .base==$b)' >/dev/null 2>&1; then
isNotice "catalog: '$url' is already a source."; return 0
fi
local f dir extras new
f="$(catalogSourcesFile)"; dir="$(dirname "$f")"
runInstallOp mkdir -p "$dir" 2>/dev/null || true
extras="$(catalogExtraSourcesJson)"
new="$(jq -c --arg n "$name" --arg b "$url" --arg c "$channel" \
'. + [ { name: $n, base: $b, channel: $c, enabled: true } ]' <<<"$extras")" || {
isError "catalog: failed to build the new source list."; return 1; }
printf '%s\n' "$new" | runInstallWrite "$f"
isSuccessful "catalog: added source '$name' ($url/$channel)."
local n var
for (( n=1; n<=CATALOG_MAX_SLOTS; n++ )); do
var="CFG_CATALOG_${n}"
if [[ -z "${!var:-}" ]]; then
updateConfigOption "$var" "$stored" >/dev/null || { isError "catalog: failed to write $var."; return 1; }
isSuccessful "catalog: added source '$url' (slot $n)."
catalogRefresh
return 0
fi
done
isError "catalog: all $CATALOG_MAX_SLOTS catalog slots are full — remove one first."; return 1
}
# catalogSourceRemove <idx> (idx 1 = official, cannot be removed)
# catalogSourceRemove <idx> — idx 1 = official (fixed); idx N+1 = CFG_CATALOG_N.
catalogSourceRemove() {
local idx="$1"
[[ "$idx" =~ ^[0-9]+$ ]] || { isError "catalog: <idx> must be a number."; return 1; }
(( idx == 1 )) && { isError "catalog: source 1 is the official catalog and can't be removed."; return 1; }
local f extras count pos new
f="$(catalogSourcesFile)"; extras="$(catalogExtraSourcesJson)"
count="$(jq 'length' <<<"$extras")"
pos=$(( idx - 2 ))
if (( pos < 0 || pos >= count )); then isError "catalog: no source with idx $idx."; return 1; fi
new="$(jq -c --argjson i "$pos" 'del(.[$i])' <<<"$extras")"
printf '%s\n' "$new" | runInstallWrite "$f"
local slot=$(( idx - 1 )) var="CFG_CATALOG_$(( idx - 1 ))"
(( slot < 1 || slot > CATALOG_MAX_SLOTS )) && { isError "catalog: no source with idx $idx."; return 1; }
[[ -z "${!var:-}" ]] && { isError "catalog: no source with idx $idx."; return 1; }
updateConfigOption "$var" "" >/dev/null || { isError "catalog: failed to clear $var."; return 1; }
isSuccessful "catalog: removed source $idx."
catalogRefresh
}
# catalogSourceToggle <idx> <true|false>
catalogSourceToggle() {
local idx="$1" on="$2"
[[ "$idx" =~ ^[0-9]+$ ]] || { isError "catalog: <idx> must be a number."; return 1; }
(( idx == 1 )) && { isError "catalog: the official catalog is always enabled."; return 1; }
[[ "$on" == "true" || "$on" == "false" ]] || { isError "catalog: toggle value must be true|false."; return 1; }
local f extras count pos new
f="$(catalogSourcesFile)"; extras="$(catalogExtraSourcesJson)"
count="$(jq 'length' <<<"$extras")"
pos=$(( idx - 2 ))
if (( pos < 0 || pos >= count )); then isError "catalog: no source with idx $idx."; return 1; fi
new="$(jq -c --argjson i "$pos" --argjson v "$on" '.[$i].enabled=$v' <<<"$extras")"
printf '%s\n' "$new" | runInstallWrite "$f"
isSuccessful "catalog: source $idx $([[ "$on" == "true" ]] && echo enabled || echo disabled)."
}
# Re-run the App Center browse scan (so a source change reflects immediately
# instead of waiting for the next `updater check`).
# Re-run the App Center browse scan so a source change reflects immediately.
catalogRefresh() {
declare -F webuiRegistryCatalogScan >/dev/null 2>&1 || \
source "${install_scripts_dir}webui/data/generators/apps/webui_registry_scan.sh" 2>/dev/null

View File

@ -19,37 +19,21 @@ cliHandleCatalogCommands()
catalogSourceList
;;
add)
[[ -z "$a1" ]] && { isNotice "Usage: catalog source add <url> [name] [channel]"; return; }
[[ -z "$a1" ]] && { isNotice "Usage: catalog source add <url> [channel]"; return; }
if [[ "$LIBREPORTAL_TASK_EXEC" == "1" ]]; then
catalogSourceAdd "$a1" "$a2" "$a3" && catalogRefresh
catalogSourceAdd "$a1" "$a2"
else
cliTaskRun "libreportal catalog source add $a1 $a2 $a3" "catalog"
cliTaskRun "libreportal catalog source add $a1 $a2" "catalog"
fi
;;
remove)
[[ -z "$a1" ]] && { isNotice "Usage: catalog source remove <idx>"; return; }
if [[ "$LIBREPORTAL_TASK_EXEC" == "1" ]]; then
catalogSourceRemove "$a1" && catalogRefresh
catalogSourceRemove "$a1"
else
cliTaskRun "libreportal catalog source remove $a1" "catalog"
fi
;;
enable)
[[ -z "$a1" ]] && { isNotice "Usage: catalog source enable <idx>"; return; }
if [[ "$LIBREPORTAL_TASK_EXEC" == "1" ]]; then
catalogSourceToggle "$a1" true && catalogRefresh
else
cliTaskRun "libreportal catalog source enable $a1" "catalog"
fi
;;
disable)
[[ -z "$a1" ]] && { isNotice "Usage: catalog source disable <idx>"; return; }
if [[ "$LIBREPORTAL_TASK_EXEC" == "1" ]]; then
catalogSourceToggle "$a1" false && catalogRefresh
else
cliTaskRun "libreportal catalog source disable $a1" "catalog"
fi
;;
*)
isNotice "Invalid catalog source action: $action"
cliShowCatalogHelp

View File

@ -8,16 +8,13 @@ cliShowCatalogHelp()
echo "catalog source list"
echo " List catalog sources in priority order (source 1 = official)."
echo ""
echo "catalog source add <url> [name] [channel]"
echo " Add a third-party catalog. channel defaults to 'stable'. Third-party"
echo " sources are UNVERIFIED — you are trusting that host."
echo "catalog source add <url> [channel]"
echo " Add a third-party catalog (into the next free CFG_CATALOG slot)."
echo " channel defaults to 'stable'. Third-party sources are UNVERIFIED."
echo ""
echo "catalog source remove <idx>"
echo " Remove a third-party catalog by index (the official source is fixed)."
echo ""
echo "catalog source enable <idx> | disable <idx>"
echo " Toggle a third-party catalog on/off without removing it."
echo ""
echo "catalog refresh"
echo " Re-scan all enabled sources and rebuild the App Center browse data."
echo ""

View File

@ -14,6 +14,7 @@ configUpdateBatch()
local applied=0
local failed=0
local catalog_changed=false
IFS='|' read -ra pairs <<< "$encoded_pairs"
for pair in "${pairs[@]}"; do
@ -21,6 +22,7 @@ configUpdateBatch()
if [[ "$pair" =~ ^(CFG_[A-Z0-9_]+)=(.*)$ ]]; then
local key="${BASH_REMATCH[1]}"
local value="${BASH_REMATCH[2]//%7C/|}"
[[ "$key" == CFG_CATALOG_* ]] && catalog_changed=true
if updateConfigOption "$key" "$value"; then
((applied++))
else
@ -50,6 +52,15 @@ configUpdateBatch()
isSuccessful "System scan completed."
fi
# Catalog sources are the source of truth for the App Center browse data;
# rebuild the generated registry_catalog.json when a CFG_CATALOG_* changed.
if [[ "$catalog_changed" == true ]] && declare -f catalogRefresh >/dev/null 2>&1; then
echo ""
echo "---- Refreshing catalog sources..."
echo ""
catalogRefresh
fi
echo ""
isSuccessful "Configuration update complete."
}

View File

@ -246,9 +246,7 @@ declare -gA LP_FN_MAP=(
[catalogSourceAdd]="catalog/catalog_sources.sh"
[catalogSourceList]="catalog/catalog_sources.sh"
[catalogSourceRemove]="catalog/catalog_sources.sh"
[catalogSourcesFile]="catalog/catalog_sources.sh"
[catalogSourcesJson]="catalog/catalog_sources.sh"
[catalogSourceToggle]="catalog/catalog_sources.sh"
[changeRootOwnedFile]="function/permission/ownership/root_file.sh"
[changeUserGroupOnFolder]="function/permission/ownership/folder_group.sh"
[checkApplicationsConfigFilesMissingVariables]="config/application/application_missing_variables.sh"
@ -1230,9 +1228,7 @@ declare -gA LP_FN_ROOT=(
[catalogSourceAdd]="scripts"
[catalogSourceList]="scripts"
[catalogSourceRemove]="scripts"
[catalogSourcesFile]="scripts"
[catalogSourcesJson]="scripts"
[catalogSourceToggle]="scripts"
[changeRootOwnedFile]="scripts"
[changeUserGroupOnFolder]="scripts"
[checkApplicationsConfigFilesMissingVariables]="scripts"
@ -1984,6 +1980,7 @@ declare -gA LP_FN_ROOT=(
LP_EAGER_FILES=(
"scripts:backup/db/backup_db.sh"
"scripts:backup/files/backup_files.sh"
"scripts:catalog/catalog_sources.sh"
"scripts:docker/install/rootless/rootless_apparmor.sh"
"scripts:docker/type_switcher/swap_docker_type.sh"
"scripts:migrate/migrate_url_rewrite.sh"
@ -2237,9 +2234,7 @@ _catalogRowsFrom() { unset -f _catalogRowsFrom; source "${install_scripts_dir}we
catalogSourceAdd() { unset -f catalogSourceAdd; source "${install_scripts_dir}catalog/catalog_sources.sh"; catalogSourceAdd "$@"; }
catalogSourceList() { unset -f catalogSourceList; source "${install_scripts_dir}catalog/catalog_sources.sh"; catalogSourceList "$@"; }
catalogSourceRemove() { unset -f catalogSourceRemove; source "${install_scripts_dir}catalog/catalog_sources.sh"; catalogSourceRemove "$@"; }
catalogSourcesFile() { unset -f catalogSourcesFile; source "${install_scripts_dir}catalog/catalog_sources.sh"; catalogSourcesFile "$@"; }
catalogSourcesJson() { unset -f catalogSourcesJson; source "${install_scripts_dir}catalog/catalog_sources.sh"; catalogSourcesJson "$@"; }
catalogSourceToggle() { unset -f catalogSourceToggle; source "${install_scripts_dir}catalog/catalog_sources.sh"; catalogSourceToggle "$@"; }
changeRootOwnedFile() { unset -f changeRootOwnedFile; source "${install_scripts_dir}function/permission/ownership/root_file.sh"; changeRootOwnedFile "$@"; }
changeUserGroupOnFolder() { unset -f changeUserGroupOnFolder; source "${install_scripts_dir}function/permission/ownership/folder_group.sh"; changeUserGroupOnFolder "$@"; }
checkApplicationsConfigFilesMissingVariables() { unset -f checkApplicationsConfigFilesMissingVariables; source "${install_scripts_dir}config/application/application_missing_variables.sh"; checkApplicationsConfigFilesMissingVariables "$@"; }