diff --git a/.github/scripts/download_packages.sh b/.github/scripts/download_packages.sh index 1e22606..9496060 100755 --- a/.github/scripts/download_packages.sh +++ b/.github/scripts/download_packages.sh @@ -73,26 +73,165 @@ generate_hashes() { echo "Downloading packages from $REPO releases" -if [ "$DOCUMENTDB_VERSION" = "latest" ]; then - if release=$(curl -fqs "https://api.github.com/repos/${REPO}/releases" | python3 -c "import sys, json; releases = json.load(sys.stdin); print(json.dumps(releases[0])) if releases else sys.exit(1)") - then - tag="$(echo "$release" | python3 -c "import sys, json; print(json.load(sys.stdin)['tag_name'])")" - echo "Using latest release: $tag" - else - echo "Error: Could not fetch latest release information" - exit 1 - fi -else - tag="$DOCUMENTDB_VERSION" - if ! release=$(curl -fqs "https://api.github.com/repos/${REPO}/releases/tags/$tag") - then - echo "Error: Version $tag not found in releases" - exit 1 - fi - echo "Using specified release: $tag" +# --------------------------------------------------------------------------- +# Release selection +# +# The primary release supplies the site's release-info.json and is the version +# users are told about. But a release only ships the distributions that were +# in its own build matrix: v0.116-0, for example, ships Tier-1 (ubuntu24 + +# rhel9) only, while v0.114-0 shipped seven distributions. Rebuilding the +# repository from the primary release alone would therefore DELETE the +# deb11/deb12/deb13/ubuntu22 components and the whole rhel8 repository from +# documentdb.io, and every host already pointed at one of them would start +# failing `apt update` with "Component 'ubuntu22' is not defined". That is a +# client-visible outage, not a cosmetic regression. +# +# So the repository is built additively: the primary release fills every +# distribution it ships, then progressively older releases are consulted ONLY +# to fill distributions still missing. A distribution is claimed by the newest +# release that ships it and is never overwritten by an older one. Once a +# release ships every distribution again, the older ones stop contributing +# by themselves - no cleanup required. +# --------------------------------------------------------------------------- +MAX_RELEASES="${MAX_RELEASES:-8}" + +RELEASES_JSON=$(mktemp) +if ! curl -fqs "https://api.github.com/repos/${REPO}/releases?per_page=100" > "$RELEASES_JSON"; then + echo "Error: Could not fetch release list" + exit 1 fi +# Ordered list of tags to consider, newest first. Drafts and prereleases are +# skipped: they are not what a repository-backed `apt install` should serve. +TAG_LIST=$(DOCUMENTDB_VERSION="$DOCUMENTDB_VERSION" python3 - "$RELEASES_JSON" <<'PY' +import json, os, sys + +releases = json.load(open(sys.argv[1])) +published = [r for r in releases if not r.get("draft") and not r.get("prerelease")] +if not published: + sys.exit("Error: no published releases found") + +requested = os.environ.get("DOCUMENTDB_VERSION", "latest") +if requested != "latest": + # The API returns releases newest-first. + index = next((i for i, r in enumerate(published) + if r["tag_name"] == requested), None) + if index is None: + sys.exit(f"Error: Version {requested} not found in releases") + # A pin means "serve this version". Only the pinned release and OLDER ones + # may contribute: pulling gap-fillers from NEWER releases would defeat the + # pin, and worse, it would mix releases that depend on each other. Pinning + # to a release that predates the multi-package layout would otherwise add a + # newer `documentdb` meta package whose `documentdb-N (>= X)` dependency the + # pinned extension cannot satisfy - an unsatisfiable repository. + ordered = published[index:] +else: + ordered = published + +print("\n".join(r["tag_name"] for r in ordered)) +PY +) + +PRIMARY_TAG=$(printf '%s\n' "$TAG_LIST" | head -n 1) +echo "Primary release: $PRIMARY_TAG" + +# Packages already placed, keyed by "pool|name|arch". A newer release always +# wins; an older release contributes only packages the newer ones do not +# provide at all. That is what keeps `postgresql-16-documentdb` alive on +# ubuntu24/rhel9 after v0.116-0 narrowed Tier 1 to PostgreSQL 17/18, instead of +# silently deleting a package the install docs still tell people to use. +SEEN_FILE=$(mktemp) + +is_claimed() { case " $1 " in *" $2 "*) return 0 ;; *) return 1 ;; esac; } + +# "name|arch" identity of a package file, independent of its version. +# postgresql-16-documentdb_0.114-0_amd64.deb -> postgresql-16-documentdb|amd64 +# documentdb_0.116.0_all.deb -> documentdb|all +# postgresql16-documentdb-0.114.0-1.el8.x86_64.rpm -> postgresql16-documentdb|x86_64 +# documentdb-17-0.116.0-1.noarch.rpm -> documentdb-17|noarch +package_identity() { + local f="$1" + case "$f" in + *.deb) + printf '%s|%s' "${f%%_*}" "$(printf '%s' "$f" | sed -E 's/.*_([^_]+)\.deb$/\1/')" + ;; + *.rpm) + local base="${f%.rpm}" + local arch="${base##*.}" + local nvr="${base%.*}" + # Drop the trailing VERSION and RELEASE fields to leave the package name. + printf '%s|%s' "$(printf '%s' "$nvr" | sed -E 's/-[^-]+-[^-]+$//')" "$arch" + ;; + esac +} + +claim_package() { + # claim_package -> 0 when this is a new package for the pool + local key="$1|$(package_identity "$2")" + if grep -qxF "$key" "$SEEN_FILE" 2>/dev/null; then + return 1 + fi + printf '%s\n' "$key" >> "$SEEN_FILE" + return 0 +} + +# Map an asset filename to its APT component, or empty when it is not a +# distribution-prefixed .deb. Any DocumentDB package for that distribution +# matches - the extension, the gateway, the tools, documentdb-common, the +# per-major stand-alone and the meta package - because the v0.116-0 packaging +# redesign ships all of them and an extension-only filter would silently drop +# every package that makes `apt install documentdb` resolve. +deb_component_for() { + case "$1" in + *dbgsym*) echo "" ;; + deb11-*.deb) echo "deb11" ;; + deb12-*.deb) echo "deb12" ;; + deb13-*.deb) echo "deb13" ;; + ubuntu22.04-*.deb) echo "ubuntu22" ;; + ubuntu24.04-*.deb) echo "ubuntu24" ;; + *) echo "" ;; + esac +} + +deb_pool_for() { + case "$1" in + deb11) echo "$DEB_POOL_DEB11" ;; + deb12) echo "$DEB_POOL_DEB12" ;; + deb13) echo "$DEB_POOL_DEB13" ;; + ubuntu22) echo "$DEB_POOL_UBUNTU22" ;; + ubuntu24) echo "$DEB_POOL_UBUNTU24" ;; + esac +} + +# RPM naming is less uniform than DEB. The extension RPMs carry a distro +# prefix (rhel9-...), the gateway carries a dist tag (....el9.x86_64.rpm), and +# the meta / per-major / common / tools RPMs are noarch with NO dist tag at +# all, so they cannot be routed by name. Those EL-agnostic packages are placed +# in exactly the pools this release populated via its prefixed assets - never +# into a pool served by a different release, where their >= dependencies on a +# same-version documentdb-N would be unsatisfiable. +rpm_pool_for() { + case "$1" in + *debuginfo*|*debugsource*) echo "" ;; + rhel8-*.rpm) echo "rhel8" ;; + rhel9-*.rpm) echo "rhel9" ;; + *.el8.*.rpm) echo "rhel8" ;; + *.el9.*.rpm) echo "rhel9" ;; + *) echo "" ;; + esac +} + mkdir -p out/packages + +for tag in $TAG_LIST; do + MAX_RELEASES=$((MAX_RELEASES - 1)) + [ "$MAX_RELEASES" -lt 0 ] && break + + if ! release=$(curl -fqs "https://api.github.com/repos/${REPO}/releases/tags/$tag"); then + echo "::warning::Could not fetch release $tag, skipping" + continue + fi + ASSETS_FILE=$(mktemp) echo "$release" | python3 -c " import sys, json @@ -101,73 +240,88 @@ for asset in data.get('assets', []): print(f\"{asset['name']}|{asset['browser_download_url']}\") " > "$ASSETS_FILE" - # Process each asset - while IFS='|' read -r filename download_url - do - if [ -z "$filename" ]; then - continue - fi - - if [[ "$filename" == *.deb ]]; then - wget -q -P out/packages "$download_url" - - if [[ "$filename" =~ ^deb11-postgresql-[0-9]+-documentdb.*\.deb$ ]]; then - GOT_DEB=1 - mkdir -p "$DEB_POOL_DEB11" - clean_name=$(echo "$filename" | sed 's/^deb11-//') - cp "out/packages/$filename" "$DEB_POOL_DEB11/$clean_name" - sign_deb_package "$DEB_POOL_DEB11/$clean_name" - elif [[ "$filename" =~ ^deb12-postgresql-[0-9]+-documentdb.*\.deb$ ]]; then - GOT_DEB=1 - mkdir -p "$DEB_POOL_DEB12" - clean_name=$(echo "$filename" | sed 's/^deb12-//') - cp "out/packages/$filename" "$DEB_POOL_DEB12/$clean_name" - sign_deb_package "$DEB_POOL_DEB12/$clean_name" - elif [[ "$filename" =~ ^deb13-postgresql-[0-9]+-documentdb.*\.deb$ ]]; then - GOT_DEB=1 - mkdir -p "$DEB_POOL_DEB13" - clean_name=$(echo "$filename" | sed 's/^deb13-//') - cp "out/packages/$filename" "$DEB_POOL_DEB13/$clean_name" - sign_deb_package "$DEB_POOL_DEB13/$clean_name" - elif [[ "$filename" =~ ^ubuntu22\.04-postgresql-[0-9]+-documentdb.*\.deb$ ]]; then - GOT_DEB=1 - mkdir -p "$DEB_POOL_UBUNTU22" - clean_name=$(echo "$filename" | sed 's/^ubuntu22\.04-//') - cp "out/packages/$filename" "$DEB_POOL_UBUNTU22/$clean_name" - sign_deb_package "$DEB_POOL_UBUNTU22/$clean_name" - elif [[ "$filename" =~ ^ubuntu24\.04-postgresql-[0-9]+-documentdb.*\.deb$ ]]; then + # First pass: which RPM pools does this release populate? Needed to route the + # EL-agnostic noarch packages, which carry no distro hint in their names. + serve_rpm="" + while IFS='|' read -r filename _; do + [ -z "$filename" ] && continue + case "$filename" in + *.rpm) + pool=$(rpm_pool_for "$filename") + if [ -n "$pool" ] && ! is_claimed "$serve_rpm" "$pool"; then + serve_rpm="$serve_rpm $pool" + fi ;; + esac + done < "$ASSETS_FILE" + + added=0 + while IFS='|' read -r filename download_url; do + [ -z "$filename" ] && continue + + case "$filename" in + *.deb) + comp=$(deb_component_for "$filename") + [ -z "$comp" ] && continue + claim_package "$comp" "$filename" || continue + wget -q -P out/packages "$download_url" || { echo "::warning::download failed: $filename"; continue; } GOT_DEB=1 - mkdir -p "$DEB_POOL_UBUNTU24" - clean_name=$(echo "$filename" | sed 's/^ubuntu24\.04-//') - cp "out/packages/$filename" "$DEB_POOL_UBUNTU24/$clean_name" - sign_deb_package "$DEB_POOL_UBUNTU24/$clean_name" - fi - elif [[ "$filename" == *.rpm ]]; then - echo "Processing RPM: $filename" - wget -q -P out/packages "$download_url" - - if [[ "$filename" =~ ^rhel8-postgresql[0-9]+-documentdb.*\.rpm$ ]]; then - GOT_RPM=1 - mkdir -p "$RPM_POOL_RHEL8" - clean_name=$(echo "$filename" | sed 's/^rhel8-//') - echo " Adding to RHEL 8: $filename -> $clean_name" - cp "out/packages/$filename" "$RPM_POOL_RHEL8/$clean_name" - elif [[ "$filename" =~ ^rhel9-postgresql[0-9]+-documentdb.*\.rpm$ ]]; then - GOT_RPM=1 - mkdir -p "$RPM_POOL_RHEL9" - clean_name=$(echo "$filename" | sed 's/^rhel9-//') - echo " Adding to RHEL 9: $filename -> $clean_name" - cp "out/packages/$filename" "$RPM_POOL_RHEL9/$clean_name" - else - echo " Skipping RPM (does not match patterns): $filename" - fi - else - wget -q -P out/packages "$download_url" - fi + pool=$(deb_pool_for "$comp") + mkdir -p "$pool" + # The distro prefix disambiguates release assets; it is not part of + # the package name and must not survive into the pool. + clean_name=$(echo "$filename" | sed -E 's/^(deb1[123]|ubuntu2[24]\.04)-//') + cp "out/packages/$filename" "$pool/$clean_name" + sign_deb_package "$pool/$clean_name" + added=$((added + 1)) ;; + + *.rpm) + pool_name=$(rpm_pool_for "$filename") + if [ -n "$pool_name" ]; then + targets="$pool_name" + else + case "$filename" in + *.noarch.rpm) targets="$serve_rpm" ;; + *) targets="" ;; + esac + fi + [ -z "$targets" ] && continue + + downloaded=0 + clean_name=$(echo "$filename" | sed -E 's/^rhel[89]-//') + for t in $targets; do + claim_package "$t" "$filename" || continue + case "$t" in + rhel8) dest="$RPM_POOL_RHEL8" ;; + rhel9) dest="$RPM_POOL_RHEL9" ;; + *) continue ;; + esac + if [ "$downloaded" -eq 0 ]; then + wget -q -P out/packages "$download_url" || { echo "::warning::download failed: $filename"; break; } + downloaded=1 + GOT_RPM=1 + fi + mkdir -p "$dest" + echo " Adding to ${t}: $filename -> $clean_name" + cp "out/packages/$filename" "$dest/$clean_name" + added=$((added + 1)) + done ;; + + *) + # Non-package assets (SHA256SUMS, manifest.txt, ...) are mirrored only + # for the primary release, which is what the site links to. + [ "$tag" = "$PRIMARY_TAG" ] && wget -q -P out/packages "$download_url" ;; + esac done < "$ASSETS_FILE" - -rm -f "$ASSETS_FILE" + echo "Release $tag contributed $added package file(s)" + rm -f "$ASSETS_FILE" +done + +rm -f "$RELEASES_JSON" "$SEEN_FILE" + +# release-info.json describes the primary release only: it is the "what is the +# current version" feed for the site, not an inventory of the pool. +release=$(curl -fqs "https://api.github.com/repos/${REPO}/releases/tags/${PRIMARY_TAG}") echo "$release" | python3 -c " import sys, json data = json.load(sys.stdin) diff --git a/.github/workflows/continuous-deployment.yml b/.github/workflows/continuous-deployment.yml index c0e86b5..27431c1 100644 --- a/.github/workflows/continuous-deployment.yml +++ b/.github/workflows/continuous-deployment.yml @@ -251,6 +251,101 @@ jobs: if not Path(f"{repomd}.asc").exists(): print(f"::warning::{repomd} was not signed") PY + # The metadata checks above prove the repository FILES exist. They do not + # prove the repository is INSTALLABLE, and that gap has already shipped a + # broken repository once: when v0.116-0 introduced the multi-package + # layout, the asset filter still matched only `postgresql-N-documentdb`, + # so 14 of 22 packages - including the `documentdb` meta package and + # everything it depends on - were silently dropped. Every file the + # verifier looks for was present and correct; `apt install documentdb` + # was simply unsatisfiable. + # + # So resolve the real thing against the repository we are about to + # publish. This is dependency RESOLUTION only (`apt-get -s`, `dnf + # --assumeno`): it reads the indexes and solves the graph without + # downloading ~500 MB of PostgreSQL and PostGIS, which keeps the check to + # a few seconds while still catching the entire "package missing from the + # pool / unsatisfiable dependency" class. + - name: Smoke test the generated repository (dependency resolution) + if: steps.features.outputs.packages == 'true' + run: | + set -euo pipefail + + # The pool is only meaningful if the meta package is actually in it. + # Fail loudly rather than "passing" a repository that has nothing to + # resolve. + if ! ls out/deb/pool/ubuntu24/documentdb_*_all.deb >/dev/null 2>&1; then + echo "::error::out/deb/pool/ubuntu24 has no documentdb meta package - the APT pool is incomplete" + exit 1 + fi + if ! ls out/rpm/rhel9/documentdb-*.noarch.rpm >/dev/null 2>&1; then + echo "::error::out/rpm/rhel9 has no documentdb meta package - the RPM pool is incomplete" + exit 1 + fi + + python3 -m http.server 8099 --directory out >/tmp/repo-http.log 2>&1 & + HTTP_PID=$! + trap 'kill "$HTTP_PID" 2>/dev/null || true' EXIT + for _ in $(seq 1 30); do + curl -fsS -o /dev/null "http://127.0.0.1:8099/deb/dists/stable/Release" && break + sleep 1 + done + + echo "::group::APT resolution (ubuntu24 component)" + # --network host so the container reaches the runner's http.server. + # [trusted=yes] because this local mirror is unsigned when the + # deployment runs without a signing key; signature verification is + # asserted separately by the "Verify generated package components" + # step above. + docker run --rm --network host ubuntu:24.04 bash -c ' + set -e + export DEBIAN_FRONTEND=noninteractive + apt-get update -qq >/dev/null + apt-get install -y -qq curl ca-certificates gnupg lsb-release >/dev/null + install -d /usr/share/postgresql-common/pgdg + curl -fsSL https://www.postgresql.org/media/keys/ACCC4CF8.asc \ + | gpg --dearmor -o /usr/share/postgresql-common/pgdg/apt.postgresql.org.gpg + echo "deb [signed-by=/usr/share/postgresql-common/pgdg/apt.postgresql.org.gpg] https://apt.postgresql.org/pub/repos/apt noble-pgdg main" \ + > /etc/apt/sources.list.d/pgdg.list + echo "deb [trusted=yes] http://127.0.0.1:8099/deb stable ubuntu24" \ + > /etc/apt/sources.list.d/documentdb.list + apt-get update -qq + # -s solves the dependency graph and exits non-zero if it cannot. + apt-get install -s documentdb > /tmp/sim.txt + # Resolving is necessary but not sufficient: assert the transaction + # actually pulls the stack, so a meta package that degenerated into + # an empty shell cannot pass. + for pkg in documentdb documentdb-common documentdb-gateway documentdb-postgresql-tools; do + grep -q "Inst $pkg " /tmp/sim.txt || { echo "APT plan is missing $pkg"; cat /tmp/sim.txt; exit 1; } + done + grep -qE "Inst postgresql-1[78]-documentdb " /tmp/sim.txt || { echo "APT plan pulls no extension package"; exit 1; } + echo "APT OK: $(grep -c "^Inst " /tmp/sim.txt) packages in the plan" + ' + echo "::endgroup::" + + echo "::group::DNF resolution (rhel9 pool)" + docker run --rm --network host rockylinux/rockylinux:9 bash -c ' + set -e + dnf install -y -q dnf-plugins-core >/dev/null 2>&1 + dnf install -y -q https://download.postgresql.org/pub/repos/yum/reporpms/EL-9-x86_64/pgdg-redhat-repo-latest.noarch.rpm >/dev/null 2>&1 + dnf install -y -q epel-release >/dev/null 2>&1 + dnf config-manager --set-enabled crb + dnf -qy module disable postgresql >/dev/null 2>&1 + printf "%s\n" "[documentdb]" "name=DocumentDB" "baseurl=http://127.0.0.1:8099/rpm/rhel9" "enabled=1" "gpgcheck=0" \ + > /etc/yum.repos.d/documentdb.repo + # --assumeno always exits non-zero (the answer is "no"), so judge the + # transaction it printed rather than the exit code. + dnf install --assumeno documentdb > /tmp/sim.txt 2>&1 || true + if grep -qE "^Error|nothing provides|Problem:" /tmp/sim.txt; then + echo "DNF could not resolve documentdb:"; cat /tmp/sim.txt; exit 1 + fi + for pkg in documentdb-common documentdb-gateway documentdb-postgresql-tools; do + grep -q "$pkg" /tmp/sim.txt || { echo "DNF plan is missing $pkg"; cat /tmp/sim.txt; exit 1; } + done + grep -qE "postgresql1[78]-documentdb" /tmp/sim.txt || { echo "DNF plan pulls no extension package"; exit 1; } + echo "DNF OK" + ' + echo "::endgroup::" - name: Upload artifact uses: actions/upload-pages-artifact@v5 with: diff --git a/PACKAGE-INSTALL.md b/PACKAGE-INSTALL.md index 878387c..7d0b248 100644 --- a/PACKAGE-INSTALL.md +++ b/PACKAGE-INSTALL.md @@ -1,53 +1,200 @@ # DocumentDB Package Installation -Repository-backed installation commands for the DocumentDB PostgreSQL extension package. +Repository-backed installation commands for DocumentDB. ## What is published -- Repository-backed extension packages are published for Ubuntu 22.04, Ubuntu 24.04, Debian 11, Debian 12, Debian 13, RHEL-compatible 8, and RHEL-compatible 9 targets. +Starting with **v0.116-0**, DocumentDB ships a multi-package layout with a setup wizard and +systemd integration, instead of just a bare PostgreSQL extension package. Not every +distribution has caught up to it yet, so the repository currently serves two shapes: + +| Distribution | Repository component | Packages available | +|---|---|---| +| Ubuntu 24.04 | `ubuntu24` | **Full stack** (v0.116-0): `documentdb` meta, `documentdb-N`, `documentdb-common`, `documentdb-gateway`, `documentdb-postgresql-tools`, plus the `postgresql-N-documentdb` extension | +| RHEL-compatible 9 | `rpm/rhel9` | **Full stack** (v0.116-0), same package set | +| Ubuntu 22.04, Debian 11/12/13, RHEL-compatible 8 | `ubuntu22`, `deb11`, `deb12`, `deb13`, `rpm/rhel8` | **Extension only** (v0.114-0): `postgresql-N-documentdb` | + - Both `amd64`/`x86_64` and `arm64`/`aarch64` variants are published. -- PostgreSQL package variants `16`, `17`, and `18` are published for the supported repository-backed combinations, with one exception: Debian 11 currently resolves PostgreSQL `16` and `17` only. -- Debian 13 `.deb` assets are published on GitHub Releases, and the APT repository now publishes a `deb13` component for repository-backed installs. -- The published package repository installs the PostgreSQL extension package. It does not currently publish a gateway package, setup helper, or systemd service in either the repository-backed install flow or GitHub Releases. +- The full stack is published for PostgreSQL **17** and **18**. The extension package alone is + additionally available for PostgreSQL **16** on every distribution. +- Debian 11 currently resolves PostgreSQL `16` and `17` only. + +> On the extension-only distributions there is still no packaged gateway, setup helper, or +> systemd service. To get a MongoDB-compatible endpoint there, follow +> [Extension-only hosts](#extension-only-hosts-run-the-gateway-from-source) below. ## Supported PostgreSQL Versions -- Ubuntu 22.04 / 24.04: 16, 17, 18 -- Debian 11: 16, 17 -- Debian 12: 16, 17, 18 -- Debian 13: 16, 17, 18 -- RHEL-compatible 8 / 9: 16, 17, 18 +- Ubuntu 24.04, RHEL-compatible 9: 16, 17, 18 (full stack on 17 and 18) +- Ubuntu 22.04: 16, 17, 18 (extension only) +- Debian 11: 16, 17 (extension only) +- Debian 12 / 13: 16, 17, 18 (extension only) +- RHEL-compatible 8: 16, 17, 18 (extension only) -## Repository-backed package installs +## Quickstart — Ubuntu 24.04 and RHEL 9 -These commands install the required PostgreSQL upstream repositories first, then add the DocumentDB package repository, and finally install the DocumentDB PostgreSQL extension package. +This is the recommended path. It installs the whole stack and brings up a working +wire-protocol endpoint. -> These commands assume a regular Linux host where you use `sudo`. If you are testing in a clean container that already runs as `root`, omit `sudo` from the package-install commands. -> -> On Debian and Ubuntu in a clean container, also run `export DEBIAN_FRONTEND=noninteractive` in the shell before the APT commands. Without it, `tzdata` (and a few other packages) prompt for input during `apt install` and the install hangs with no visible error. +> These commands assume a regular Linux host where you use `sudo`. In a clean container that +> already runs as `root`, omit `sudo`, and on Debian/Ubuntu also +> `export DEBIAN_FRONTEND=noninteractive` first, or `tzdata` will hang the install with an +> invisible prompt. -### Ubuntu 22.04 (Jammy) +### Ubuntu 24.04 (Noble) ```bash sudo apt update && \ sudo apt install -y curl ca-certificates gnupg && \ curl -fsSL https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo gpg --dearmor --yes -o /usr/share/keyrings/postgresql.gpg && \ -echo "deb [signed-by=/usr/share/keyrings/postgresql.gpg] https://apt.postgresql.org/pub/repos/apt jammy-pgdg main" | sudo tee /etc/apt/sources.list.d/pgdg.list >/dev/null && \ +echo "deb [signed-by=/usr/share/keyrings/postgresql.gpg] https://apt.postgresql.org/pub/repos/apt noble-pgdg main" | sudo tee /etc/apt/sources.list.d/pgdg.list >/dev/null && \ curl -fsSL https://documentdb.io/documentdb-archive-keyring.gpg | sudo gpg --dearmor --yes -o /usr/share/keyrings/documentdb-archive-keyring.gpg && \ -echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/documentdb-archive-keyring.gpg] https://documentdb.io/deb stable ubuntu22" | sudo tee /etc/apt/sources.list.d/documentdb.list >/dev/null && \ +echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/documentdb-archive-keyring.gpg] https://documentdb.io/deb stable ubuntu24" | sudo tee /etc/apt/sources.list.d/documentdb.list >/dev/null && \ sudo apt update && \ -sudo apt install -y postgresql-16-documentdb +sudo apt install -y documentdb ``` -### Ubuntu 24.04 (Noble) +### RHEL-compatible 9 + +`crb` is disabled by default and is **required**: PostGIS pulls in `gdal*-libs`, which needs +`libqhull_r.so.7`, and that library ships only in CRB. Without it `dnf install` fails with a +wall of GDAL candidate lines that never name the missing repository. + +```bash +sudo dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm && \ +sudo dnf install -y https://download.postgresql.org/pub/repos/yum/reporpms/EL-9-$(uname -m)/pgdg-redhat-repo-latest.noarch.rpm && \ +sudo dnf -qy module disable postgresql && \ +sudo dnf install -y dnf-plugins-core && \ +(sudo dnf config-manager --set-enabled crb || \ + sudo dnf config-manager --set-enabled codeready-builder-for-rhel-9-$(uname -m)-rpms) && \ +sudo rpm --import https://documentdb.io/documentdb-archive-keyring.gpg && \ +printf '%s\n' \ + '[documentdb]' \ + 'name=DocumentDB Repository' \ + 'baseurl=https://documentdb.io/rpm/rhel9' \ + 'enabled=1' \ + 'gpgcheck=1' \ + 'gpgkey=https://documentdb.io/documentdb-archive-keyring.gpg' | sudo tee /etc/yum.repos.d/documentdb.repo >/dev/null && \ +sudo dnf install -y documentdb +``` + +### Then set up and connect + +`documentdb-setup` **prompts for the admin password** interactively. For servers, CI or any +non-TTY context, pass it in instead with `--admin-password-file ` or +`--admin-password-stdin`, together with `--yes` — the bare command below will hang without a +terminal. + +```bash +# Runs initdb / CREATE EXTENSION / admin bootstrap, starts the gateway, and enables +# documentdb-local@.target so the stack survives reboot. +sudo documentdb-setup --admin-user admin + +# Unattended equivalent: +# printf '%s' "$ADMIN_PW" | sudo documentdb-setup --admin-user admin --admin-password-stdin --yes +``` + +`mongosh` is not shipped by these packages. Install it from the +[official instructions](https://www.mongodb.com/docs/mongodb-shell/install/), then: + +```bash +mongosh 'mongodb://admin:@127.0.0.1:10260/mydb?tls=true&tlsAllowInvalidCertificates=true' \ + --eval 'db.runCommand({ping: 1})' +``` + +A first database and collection are created on first write: + +```javascript +db.orders.insertOne({ item: "widget", qty: 5 }) +db.orders.find() +``` + +Other useful `documentdb-setup` flags: `--status`, `--print-config`, `--no-enable`. + +### ⚠️ Before you expose this to a network + +**The gateway listens on all interfaces (`0.0.0.0:10260` and `[::]:10260`) by default**, even +though the connect string above says `127.0.0.1`. On a cloud VM with an open security group, +the commands above stand up an internet-reachable endpoint protected only by the admin +password. The PostgreSQL instance behind it is *not* exposed — it stays on `127.0.0.1`. + +Before using this anywhere but a private machine: + +- **Restrict the listener** to loopback by setting `DOCUMENTDB_LISTEN_ADDR=127.0.0.1:10260` + in `/etc/documentdb/local//gateway.env`, then restarting the service. (Only loopback + hosts and the bare `:port` form are accepted; an arbitrary IP is rejected.) Otherwise + firewall port `10260` yourself. +- **Replace the auto-generated self-signed certificate.** `tlsAllowInvalidCertificates=true` + in the example disables certificate validation, so it gives you encryption without + authenticating the server. Point `DOCUMENTDB_TLS_CERT_FILE` / `DOCUMENTDB_TLS_KEY_FILE` at a + real certificate and drop that option. +- Use a strong admin password, and create per-application users rather than sharing `admin`. + +### Verify and operate + +```bash +sudo documentdb-setup --status # gateway listener, service states, resolved paths +documentdb-gateway --version # DocumentDB version (0.116.0) +dpkg -l | grep documentdb # or: rpm -qa | grep documentdb +``` + +> Do not use `db.version()` / `buildInfo` in `mongosh` to check the DocumentDB version — those +> report the **emulated MongoDB wire version** (e.g. `7.0.0`), not DocumentDB's. + +| Thing | Where | +|---|---| +| Gateway port | `10260` | +| PostgreSQL port | `9700 + ` (9718 for PG 18), loopback only | +| Gateway log | `/var/lib/documentdb-gateway/gateway.log` | +| PostgreSQL log | `/var/lib/documentdb-local//data/pglog.log` | +| Setup state / gateway env | `/etc/documentdb/local//setup.conf`, `.../gateway.env` | + +**Day 2** (units are templated per PostgreSQL major — substitute `18` as needed): + +```bash +sudo systemctl status documentdb-local@18.target +sudo systemctl restart documentdb-local@18.target +sudo systemctl stop documentdb-local@18.target +``` + +On hosts without systemd (containers, some dev images) the wizard starts the gateway directly +instead; re-run `documentdb-setup` to restart it. + +**Remove or reset:** + +```bash +sudo documentdb-setup --restore # detach the managed integration +sudo documentdb-local-reset --pg-version 18 --confirm-destroy # DESTROYS the data directory +sudo apt remove documentdb # or: sudo dnf remove documentdb +``` + +### What the packages are + +| Package | Role | +|---|---| +| `documentdb` (meta) + `documentdb-N` | Full stand-alone install; pins PostgreSQL major N + its extension and owns the systemd lifecycle. The meta package pins PG 18. | +| `postgresql-N-documentdb` | The extension for PostgreSQL major N (files only). | +| `documentdb-gateway` | Wire-protocol runtime (binary + systemd unit). | +| `documentdb-postgresql-tools` | Admin helpers: `documentdb-tune`, `documentdb-createcluster`, `documentdb-register-gateway`, `documentdb-gateway-admin`. | +| `documentdb-common` | Shared, PG-agnostic payload: `documentdb-setup`, systemd template units, sysusers.d/tmpfiles.d drop-ins, helper scripts, sample data. | + +To install the extension by itself on these distributions, use `postgresql-18-documentdb` +(APT) or `postgresql18-documentdb` (RPM) instead of the `documentdb` meta package. + +## Extension-only distributions + +Ubuntu 22.04, Debian 11/12/13 and RHEL-compatible 8 currently serve the extension package +only. The commands below add the DocumentDB repository and install it. + +### Ubuntu 22.04 (Jammy) ```bash sudo apt update && \ sudo apt install -y curl ca-certificates gnupg && \ curl -fsSL https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo gpg --dearmor --yes -o /usr/share/keyrings/postgresql.gpg && \ -echo "deb [signed-by=/usr/share/keyrings/postgresql.gpg] https://apt.postgresql.org/pub/repos/apt noble-pgdg main" | sudo tee /etc/apt/sources.list.d/pgdg.list >/dev/null && \ +echo "deb [signed-by=/usr/share/keyrings/postgresql.gpg] https://apt.postgresql.org/pub/repos/apt jammy-pgdg main" | sudo tee /etc/apt/sources.list.d/pgdg.list >/dev/null && \ curl -fsSL https://documentdb.io/documentdb-archive-keyring.gpg | sudo gpg --dearmor --yes -o /usr/share/keyrings/documentdb-archive-keyring.gpg && \ -echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/documentdb-archive-keyring.gpg] https://documentdb.io/deb stable ubuntu24" | sudo tee /etc/apt/sources.list.d/documentdb.list >/dev/null && \ +echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/documentdb-archive-keyring.gpg] https://documentdb.io/deb stable ubuntu22" | sudo tee /etc/apt/sources.list.d/documentdb.list >/dev/null && \ sudo apt update && \ sudo apt install -y postgresql-16-documentdb ``` @@ -98,8 +245,8 @@ sudo dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-8. sudo dnf install -y https://download.postgresql.org/pub/repos/yum/reporpms/EL-8-$(uname -m)/pgdg-redhat-repo-latest.noarch.rpm && \ sudo dnf -qy module disable postgresql && \ sudo dnf install -y dnf-plugins-core && \ -(sudo dnf config-manager --set-enabled crb || \ - sudo dnf config-manager --set-enabled powertools || \ +(sudo dnf config-manager --set-enabled powertools || \ + sudo dnf config-manager --set-enabled crb || \ sudo dnf config-manager --set-enabled codeready-builder-for-rhel-8-$(uname -m)-rpms) && \ sudo rpm --import https://documentdb.io/documentdb-archive-keyring.gpg && \ printf '%s\n' \ @@ -112,39 +259,52 @@ printf '%s\n' \ sudo dnf install -y postgresql16-documentdb ``` -### RHEL-compatible 9 - -```bash -sudo dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm && \ -sudo dnf install -y https://download.postgresql.org/pub/repos/yum/reporpms/EL-9-$(uname -m)/pgdg-redhat-repo-latest.noarch.rpm && \ -sudo dnf -qy module disable postgresql && \ -sudo dnf install -y dnf-plugins-core && \ -(sudo dnf config-manager --set-enabled crb || \ - sudo dnf config-manager --set-enabled powertools || \ - sudo dnf config-manager --set-enabled codeready-builder-for-rhel-9-$(uname -m)-rpms) && \ -sudo rpm --import https://documentdb.io/documentdb-archive-keyring.gpg && \ -printf '%s\n' \ - '[documentdb]' \ - 'name=DocumentDB Repository' \ - 'baseurl=https://documentdb.io/rpm/rhel9' \ - 'enabled=1' \ - 'gpgcheck=1' \ - 'gpgkey=https://documentdb.io/documentdb-archive-keyring.gpg' | sudo tee /etc/yum.repos.d/documentdb.repo >/dev/null && \ -sudo dnf install -y postgresql16-documentdb -``` - -## Installing PostgreSQL 17 or 18 instead +## Installing a different PostgreSQL major Swap the package name at the end of the command: - APT: `postgresql-17-documentdb` or `postgresql-18-documentdb` - RPM: `postgresql17-documentdb` or `postgresql18-documentdb` -> Debian 11 currently supports PostgreSQL `16` and `17` in the repository-backed install flow. PostgreSQL `18` on Debian 11 is blocked by the missing `postgresql-18-postgis-3` dependency in the upstream Bullseye packages. +> Debian 11 currently supports PostgreSQL `16` and `17` in the repository-backed install flow. +> PostgreSQL `18` on Debian 11 is blocked by the missing `postgresql-18-postgis-3` dependency +> in the upstream Bullseye packages. + +## Upgrading an existing install + +The package repository serves the newest build for each distribution, so once v0.116-0 is +published a host already running v0.114-0 will see it as an available upgrade. + +**A package upgrade only replaces files on disk.** It does not touch the SQL objects already +created in your databases, so after upgrading you must update the extensions in **every +database** that has DocumentDB installed: + +```sql +ALTER EXTENSION documentdb_core UPDATE; +ALTER EXTENSION documentdb UPDATE; +ALTER EXTENSION documentdb_extended_rum UPDATE; -- only if it is installed +``` + +PostgreSQL applies the intermediate upgrade scripts automatically, so 0.114-0 → 0.116-0 is +applied as 0.114-0 → 0.115-0 → 0.116-0 in one step. Confirm afterwards with: + +```sql +SELECT extname, extversion FROM pg_extension WHERE extname LIKE 'documentdb%'; +``` + +> **Pre-GA:** in-place upgrades are not yet a supported, fully tested path. Take a backup +> first, and prefer a clean install where you can. If you would rather not be offered the +> upgrade at all, pin the current version: +> +> ```bash +> sudo apt-mark hold postgresql-18-documentdb # APT +> sudo dnf install -y python3-dnf-plugin-versionlock && \ +> sudo dnf versionlock add postgresql18-documentdb # DNF +> ``` ## Version pinning -Run the repository setup for your distro first, then use these commands: +Run the repository setup for your distro first, then: ### APT @@ -160,9 +320,14 @@ dnf --showduplicates list postgresql16-documentdb sudo dnf install postgresql16-documentdb- ``` -## From package install to a local `mongosh` endpoint +## Extension-only hosts: run the gateway from source -The repository-backed install gives you the PostgreSQL extension package. To expose a local MongoDB-compatible endpoint on the same host, run PostgreSQL and the gateway from the source repository against the packaged extension files. +On Ubuntu 24.04 and RHEL 9, use `documentdb-setup` from the packaged stack above instead — +this section is only for distributions where the gateway is not packaged yet. + +The repository-backed install there gives you the PostgreSQL extension package. To expose a +local MongoDB-compatible endpoint on the same host, run PostgreSQL and the gateway from the +source repository against the packaged extension files. ### Prerequisites @@ -172,9 +337,12 @@ The repository-backed install gives you the PostgreSQL extension package. To exp - A current Rust toolchain via `rustup` - `mongosh` -> Run the PostgreSQL and gateway steps from an unprivileged user account, not `root`. PostgreSQL will not initialize as `root`. +> Run the PostgreSQL and gateway steps from an unprivileged user account, not `root`. +> PostgreSQL will not initialize as `root`. > -> If you are following these steps in a clean container that starts as `root`, finish the package-install commands as `root`, then switch to an unprivileged account such as `postgres` before you start PostgreSQL or the gateway. +> If you are following these steps in a clean container that starts as `root`, finish the +> package-install commands as `root`, then switch to an unprivileged account such as +> `postgres` before you start PostgreSQL or the gateway. ```bash # from a root shell inside the container @@ -198,13 +366,17 @@ curl https://sh.rustup.rs -sSf | sh -s -- -y . "$HOME/.cargo/env" ``` -In a clean container that starts as `root`, install the system packages above and install `mongosh` while you are still `root`. Then switch to the unprivileged user and run the `rustup` commands plus the remaining gateway steps from that user's shell. +In a clean container that starts as `root`, install the system packages above and install +`mongosh` while you are still `root`. Then switch to the unprivileged user and run the +`rustup` commands plus the remaining gateway steps from that user's shell. ### Example host flow -Replace `` with the PostgreSQL major version you installed from the package repository, such as `16`, `17`, or `18`. +Replace `` with the PostgreSQL major version you installed from the package +repository, such as `16`, `17`, or `18`. -If you do not already have `mongosh`, install it with the official MongoDB shell instructions for your distro before continuing: +If you do not already have `mongosh`, install it with the official MongoDB shell instructions +for your distro before continuing: - https://www.mongodb.com/docs/mongodb-shell/install/ @@ -247,22 +419,31 @@ mongosh localhost:10260 \ ## Direct downloads -GitHub Releases contains `.deb` and `.rpm` extension assets for every published combination, including Debian 13 release assets. It does not currently publish a gateway package. +GitHub Releases contains the `.deb` and `.rpm` assets for every published combination. Note +that a release only carries the distributions in its own build matrix — v0.116-0 ships +Ubuntu 24.04 and RHEL 9 — while the package repository additionally keeps the most recent +package for every other distribution, so nothing disappears when a release narrows its matrix. Examples: ```text -ubuntu22.04-postgresql-18-documentdb_0.114-0_amd64.deb +ubuntu24.04-documentdb_0.116.0_all.deb +ubuntu24.04-postgresql-18-documentdb_0.116-0_amd64.deb +rhel9-postgresql18-documentdb-0.116.0-1.el9.x86_64.rpm deb13-postgresql-18-documentdb_0.114-0_amd64.deb -rhel9-postgresql18-documentdb-0.114.0-1.el9.x86_64.rpm ``` +Because the packages depend on each other, installing a downloaded meta package on its own +fails with `Depends: documentdb-18 ... but it is not installable`. Pass the whole set to a +single command, or just use the repository-backed install above. + - GitHub Releases: https://github.com/documentdb/documentdb/releases - Release metadata: https://documentdb.io/packages/release-info.json ## Notes -- The APT repository currently publishes components for `ubuntu22`, `ubuntu24`, `deb11`, `deb12`, and `deb13`. +- The APT repository publishes components for `ubuntu22`, `ubuntu24`, `deb11`, `deb12`, and `deb13`; the RPM repositories are `rhel8` and `rhel9`. - Debian 11 PostgreSQL 18 assets exist, but the upstream Bullseye PostGIS dependency is not currently installable from PGDG. - The RPM flow depends on EPEL plus PostgreSQL's upstream RPM repository because DocumentDB depends on PostgreSQL, `pg_cron`, `pgvector`, PostGIS, and `rum` for PostgreSQL 16/17. - On Debian/Ubuntu, the distro-packaged `cargo` can be older than the current gateway workspace lockfile. `rustup` avoids that mismatch. +