diff --git a/scripts/cli/commands/updater/cli_updater_ladder.sh b/scripts/cli/commands/updater/cli_updater_ladder.sh index 514e356..601d8b7 100644 --- a/scripts/cli/commands/updater/cli_updater_ladder.sh +++ b/scripts/cli/commands/updater/cli_updater_ladder.sh @@ -28,6 +28,86 @@ updaterTagSortKey() { printf '%s' "$1" | grep -oE '[0-9]+' | awk '{ printf "%06d.", $0 }' } +# --------------------------------------------------------------------------- +# Generic OCI registry access, for everything that is not Docker Hub. +# +# Hub keeps its own path below: hub.docker.com/v2/repositories is a SEPARATE +# rate-limit pool, whereas registry-1.docker.io manifest reads count against the +# anonymous PULL limit (100/hour). A ladder probes a tag per rung, so routing +# Hub through the OCI API would spend the budget the updater needs for pulls. +# +# Everything else speaks the standard Distribution API and needs only the token +# dance: ask, read the WWW-Authenticate challenge, fetch a token from the realm +# it names, retry. That is all "we cannot probe ghcr.io" ever amounted to — +# ghcr.io, lscr.io (which points its realm at ghcr.io) and quay.io all answer +# anonymously for public images. Until this existed those apps were reported as +# up to date having never been looked at, which is the wrong kind of quiet. +# --------------------------------------------------------------------------- + +# Split an image ref into registry + repository. Sets _oci_reg / _oci_repo. +# A first segment containing a dot or a colon, or "localhost", is a host; +# anything else is Hub, where a bare name means the library/ namespace. +_updaterOciSplit() { + local ref="${1%%:*}" + ref="${ref#docker.io/}"; ref="${ref#index.docker.io/}" + case "$ref" in + localhost/*|*.*/*|*:*/*) _oci_reg="${ref%%/*}"; _oci_repo="${ref#*/}" ;; + */*) _oci_reg="registry-1.docker.io"; _oci_repo="$ref" ;; + *) _oci_reg="registry-1.docker.io"; _oci_repo="library/$ref" ;; + esac +} + +# Bearer token for , or nothing when the registry does not +# challenge (quay.io serves public repos unauthenticated). +_updaterOciToken() { + local reg="$1" repo="$2" ch realm service scope + ch="$(curl -sS -o /dev/null -D - --connect-timeout 5 --max-time 12 \ + "https://${reg}/v2/${repo}/tags/list?n=1" 2>/dev/null \ + | tr -d '\r' | grep -i '^www-authenticate:' | head -1)" + [ -n "$ch" ] || return 0 + realm="$(printf '%s' "$ch" | grep -o 'realm="[^"]*"' | head -1 | cut -d'"' -f2)" + service="$(printf '%s' "$ch" | grep -o 'service="[^"]*"' | head -1 | cut -d'"' -f2)" + scope="$(printf '%s' "$ch" | grep -o 'scope="[^"]*"' | head -1 | cut -d'"' -f2)" + [ -n "$realm" ] || return 0 + [ -n "$scope" ] || scope="repository:${repo}:pull" + curl -fsS --connect-timeout 5 --max-time 12 \ + "${realm}?service=${service}&scope=${scope}" 2>/dev/null \ + | grep -oE '"(access_)?token"[[:space:]]*:[[:space:]]*"[^"]+"' | head -1 \ + | sed -E 's/.*"([^"]+)"$/\1/' +} + +# Does exist in ? A manifest HEAD, so nothing is downloaded. +updaterOciTagExists() { + local ref="$1" tag="$2" + [ -n "$tag" ] || return 1 + command -v curl >/dev/null 2>&1 || return 1 + _updaterOciSplit "$ref" + local tok; tok="$(_updaterOciToken "$_oci_reg" "$_oci_repo")" + local code + code="$(curl -sS -o /dev/null -w '%{http_code}' -I --connect-timeout 5 --max-time 12 \ + ${tok:+-H "Authorization: Bearer $tok"} \ + -H 'Accept: application/vnd.oci.image.index.v1+json' \ + -H 'Accept: application/vnd.docker.distribution.manifest.list.v2+json' \ + -H 'Accept: application/vnd.docker.distribution.manifest.v2+json' \ + "https://${_oci_reg}/v2/${_oci_repo}/manifests/${tag}" 2>/dev/null)" + [ "$code" = "200" ] +} + +# Tags for , one per line. NOTE the ordering differs from Hub's: +# /v2/.../tags/list is lexical, not newest-first, and large repos cap the page — +# so a listing can legitimately miss the newest release. That is survivable +# because newer-version discovery falls back to probing, which has no window. +updaterOciTagList() { + command -v curl >/dev/null 2>&1 || return 0 + _updaterOciSplit "$1" + local tok; tok="$(_updaterOciToken "$_oci_reg" "$_oci_repo")" + curl -fsSL --connect-timeout 5 --max-time 20 \ + ${tok:+-H "Authorization: Bearer $tok"} \ + "https://${_oci_reg}/v2/${_oci_repo}/tags/list?n=1000" 2>/dev/null \ + | sed -E 's/.*"tags"[[:space:]]*:[[:space:]]*\[//; s/\].*//' \ + | tr ',' '\n' | grep -oE '"[^"]*"' | tr -d '"' | grep -v '^$' +} + # Does this exact tag exist? One cheap lookup, and the ONLY reliable way to ask. # Listing cannot answer it: Docker Hub pages at 100 and orders by recency, so an # older intermediate rung falls off the end — mastodon's v4.3 exists but is @@ -37,7 +117,9 @@ updaterTagSortKey() { updaterTagExists() { local repo="${1%%:*}" tag="$2" repo="${repo#docker.io/}"; repo="${repo#index.docker.io/}" # docker.io/ IS Hub - case "$repo" in *.*/*|localhost/*) return 1 ;; esac # non-Hub: unknown + # Non-Hub: the standard OCI API rather than a shrug. This is what makes + # ghcr.io / lscr.io / quay.io apps laddered and version-checked at all. + case "$repo" in *.*/*|localhost/*) updaterOciTagExists "$1" "$tag"; return $? ;; esac case "$repo" in */*) : ;; *) repo="library/$repo" ;; esac command -v curl >/dev/null 2>&1 || return 1 local code diff --git a/scripts/source/files/arrays/function_manifest.sh b/scripts/source/files/arrays/function_manifest.sh index b9f7903..8c763be 100644 --- a/scripts/source/files/arrays/function_manifest.sh +++ b/scripts/source/files/arrays/function_manifest.sh @@ -829,6 +829,7 @@ declare -gA LP_FN_MAP=( [reconcileConfigFile]="config/core/variables/config_scan_variables.sh" [reconcileContainersTopOwnership]="function/permission/libreportal_folders.sh" [reconcileDockerOwnership]="function/permission/libreportal_folders.sh" + [_reconcilePortColumns]="config/core/variables/config_scan_variables.sh" [_reconcileSplitValueComment]="config/core/variables/config_scan_variables.sh" [reconcileWebuiDirOwnership]="function/permission/libreportal_folders.sh" [recoverOrphans]="task/crontab_task_processor.sh" @@ -1061,6 +1062,10 @@ declare -gA LP_FN_MAP=( [updaterNewerVersionByProbe]="webui/data/generators/updater/webui_updater_scan.sh" [updaterNewerVersionTag]="webui/data/generators/updater/webui_updater_scan.sh" [updaterNextRung]="cli/commands/updater/cli_updater_ladder.sh" + [_updaterOciSplit]="cli/commands/updater/cli_updater_ladder.sh" + [updaterOciTagExists]="cli/commands/updater/cli_updater_ladder.sh" + [updaterOciTagList]="cli/commands/updater/cli_updater_ladder.sh" + [_updaterOciToken]="cli/commands/updater/cli_updater_ladder.sh" [_updaterPrimaryContainer]="cli/commands/updater/cli_updater_verify.sh" [updaterPrimaryImage]="webui/data/generators/updater/webui_updater_scan.sh" [_updaterPublishedPortFor]="cli/commands/updater/cli_updater_verify.sh" @@ -2001,6 +2006,7 @@ declare -gA LP_FN_ROOT=( [reconcileConfigFile]="scripts" [reconcileContainersTopOwnership]="scripts" [reconcileDockerOwnership]="scripts" + [_reconcilePortColumns]="scripts" [_reconcileSplitValueComment]="scripts" [reconcileWebuiDirOwnership]="scripts" [recoverOrphans]="scripts" @@ -2233,6 +2239,10 @@ declare -gA LP_FN_ROOT=( [updaterNewerVersionByProbe]="scripts" [updaterNewerVersionTag]="scripts" [updaterNextRung]="scripts" + [_updaterOciSplit]="scripts" + [updaterOciTagExists]="scripts" + [updaterOciTagList]="scripts" + [_updaterOciToken]="scripts" [_updaterPrimaryContainer]="scripts" [updaterPrimaryImage]="scripts" [_updaterPublishedPortFor]="scripts" @@ -3209,6 +3219,7 @@ reclaimDockerSpace() { unset -f reclaimDockerSpace; __lpAutoload "${install_scri reconcileConfigFile() { unset -f reconcileConfigFile; __lpAutoload "${install_scripts_dir}config/core/variables/config_scan_variables.sh"; reconcileConfigFile "$@"; } reconcileContainersTopOwnership() { unset -f reconcileContainersTopOwnership; __lpAutoload "${install_scripts_dir}function/permission/libreportal_folders.sh"; reconcileContainersTopOwnership "$@"; } reconcileDockerOwnership() { unset -f reconcileDockerOwnership; __lpAutoload "${install_scripts_dir}function/permission/libreportal_folders.sh"; reconcileDockerOwnership "$@"; } +_reconcilePortColumns() { unset -f _reconcilePortColumns; __lpAutoload "${install_scripts_dir}config/core/variables/config_scan_variables.sh"; _reconcilePortColumns "$@"; } _reconcileSplitValueComment() { unset -f _reconcileSplitValueComment; __lpAutoload "${install_scripts_dir}config/core/variables/config_scan_variables.sh"; _reconcileSplitValueComment "$@"; } reconcileWebuiDirOwnership() { unset -f reconcileWebuiDirOwnership; __lpAutoload "${install_scripts_dir}function/permission/libreportal_folders.sh"; reconcileWebuiDirOwnership "$@"; } recoverOrphans() { unset -f recoverOrphans; __lpAutoload "${install_scripts_dir}task/crontab_task_processor.sh"; recoverOrphans "$@"; } @@ -3441,6 +3452,10 @@ updaterNewerVersionByList() { unset -f updaterNewerVersionByList; __lpAutoload " updaterNewerVersionByProbe() { unset -f updaterNewerVersionByProbe; __lpAutoload "${install_scripts_dir}webui/data/generators/updater/webui_updater_scan.sh"; updaterNewerVersionByProbe "$@"; } updaterNewerVersionTag() { unset -f updaterNewerVersionTag; __lpAutoload "${install_scripts_dir}webui/data/generators/updater/webui_updater_scan.sh"; updaterNewerVersionTag "$@"; } updaterNextRung() { unset -f updaterNextRung; __lpAutoload "${install_scripts_dir}cli/commands/updater/cli_updater_ladder.sh"; updaterNextRung "$@"; } +_updaterOciSplit() { unset -f _updaterOciSplit; __lpAutoload "${install_scripts_dir}cli/commands/updater/cli_updater_ladder.sh"; _updaterOciSplit "$@"; } +updaterOciTagExists() { unset -f updaterOciTagExists; __lpAutoload "${install_scripts_dir}cli/commands/updater/cli_updater_ladder.sh"; updaterOciTagExists "$@"; } +updaterOciTagList() { unset -f updaterOciTagList; __lpAutoload "${install_scripts_dir}cli/commands/updater/cli_updater_ladder.sh"; updaterOciTagList "$@"; } +_updaterOciToken() { unset -f _updaterOciToken; __lpAutoload "${install_scripts_dir}cli/commands/updater/cli_updater_ladder.sh"; _updaterOciToken "$@"; } _updaterPrimaryContainer() { unset -f _updaterPrimaryContainer; __lpAutoload "${install_scripts_dir}cli/commands/updater/cli_updater_verify.sh"; _updaterPrimaryContainer "$@"; } updaterPrimaryImage() { unset -f updaterPrimaryImage; __lpAutoload "${install_scripts_dir}webui/data/generators/updater/webui_updater_scan.sh"; updaterPrimaryImage "$@"; } _updaterPublishedPortFor() { unset -f _updaterPublishedPortFor; __lpAutoload "${install_scripts_dir}cli/commands/updater/cli_updater_verify.sh"; _updaterPublishedPortFor "$@"; } diff --git a/scripts/webui/data/generators/updater/webui_updater_scan.sh b/scripts/webui/data/generators/updater/webui_updater_scan.sh index a22f2fc..35f3d3d 100644 --- a/scripts/webui/data/generators/updater/webui_updater_scan.sh +++ b/scripts/webui/data/generators/updater/webui_updater_scan.sh @@ -135,12 +135,18 @@ updaterRegistryDigest() { # on their own, so there is nothing to discover. # --------------------------------------------------------------------------- -# Tag list for a repo. Docker Hub only, deliberately: every pinned app in the -# catalogue lives there (nextcloud, stalwartlabs, tootsuite), the endpoint needs -# no auth for public repos, and the generic OCI /v2/tags/list wants a per- -# registry token dance. Anything else returns nothing and the feature simply -# stays quiet for that app rather than guessing. 100 newest tags is plenty: -# they are returned newest-first and we only care about ones ABOVE the current. +# Tag list for a repo, from whichever registry hosts it. +# +# Docker Hub keeps its own endpoint because it is strictly better for this job: +# hub.docker.com returns tags NEWEST-FIRST, so the 100 it pages are the 100 that +# matter, and it draws on a different rate limit from the pull budget. 32 of the +# catalogue's 37 anchors live there. +# +# The rest (ghcr.io, quay.io, lscr.io) go through the standard OCI tags/list via +# the ladder's helpers. That listing is lexical rather than newest-first and big +# repos cap the page, so it is a weaker signal — but the alternative was +# returning nothing, which had those apps reported as up to date having never +# been asked. Probing backfills whatever the listing cannot see. updaterRegistryTags() { local repo="${1%%:*}" # strip any :tag # docker.io/ IS Docker Hub, just spelled out — several apps write it that @@ -148,8 +154,20 @@ updaterRegistryTags() { # host check stops those being mistaken for a third-party registry and # silently skipped. repo="${repo#docker.io/}"; repo="${repo#index.docker.io/}" + # Non-Hub goes through the standard OCI tags/list instead of returning + # nothing. Its ordering is lexical rather than newest-first and big repos + # cap the page, so this listing is a weaker signal than Hub's — but a weaker + # signal beats the silence that had ghcr.io/quay.io/lscr.io apps reported as + # current without ever being asked. Probing covers what the listing misses. case "$repo" in - *.*/*|localhost/*) return 0 ;; # a real registry host — not Hub + *.*/*|localhost/*) + if ! declare -F updaterOciTagList >/dev/null 2>&1; then + [ -f "$install_scripts_dir/cli/commands/updater/cli_updater_ladder.sh" ] \ + && source "$install_scripts_dir/cli/commands/updater/cli_updater_ladder.sh" 2>/dev/null + fi + declare -F updaterOciTagList >/dev/null 2>&1 && updaterOciTagList "$1" + return 0 + ;; esac case "$repo" in */*) : ;; *) repo="library/$repo" ;; esac # official images command -v curl >/dev/null 2>&1 || return 0