From 31fa9508f953166ee0949e3930740a4a62a33b09 Mon Sep 17 00:00:00 2001 From: Keegan Smith Date: Wed, 9 Sep 2026 14:46:34 +0000 Subject: [PATCH] feat/packaging: build Sourcegraph Git with zlib-ng Object inflation is a significant part of Git runtime, and local benchmarking showed roughly 25% lower wall time with zlib-ng. Use its native API so it can coexist with ordinary zlib used by other dependencies, and link the pinned build statically to preserve relocatability. Share one checksum-verified dependency recipe across orb setup and Linux and macOS release builders, recording provenance and licensing in each archive. Keep plain orb make invocations on the same backend and verify that release artifacts neither acquire a dynamic zlib-ng dependency nor retain build-time macOS rpaths. Reserve sourcegraph/v2.55.0-3 for this follow-up so the stock-zlib v2.55.0-2 distribution can ship first. Sourcegraph Git distributions now use zlib-ng for faster object compression and decompression. Amp-Thread-ID: https://ampcode.com/threads/T-01a0863e-cefe-716b-a3d5-be8e3f40e19f Co-authored-by: Amp --- .agents/setup | 25 +++++++++++ .github/workflows/sourcegraph.yml | 12 ++--- .../sourcegraph/packaging/Dockerfile.linux | 7 ++- contrib/sourcegraph/packaging/README.md | 35 +++++++++------ contrib/sourcegraph/packaging/build-darwin.sh | 4 ++ .../packaging/build-from-source.sh | 19 +++++++- .../sourcegraph/packaging/build-zlib-ng.sh | 44 +++++++++++++++++++ contrib/sourcegraph/packaging/release.sh | 2 +- .../packaging/validate-darwin-archive.sh | 17 +++++++ .../packaging/validate-linux-archive.sh | 11 +++++ .../packaging/verify-darwin-dependencies.sh | 12 +++++ 11 files changed, 165 insertions(+), 23 deletions(-) create mode 100755 contrib/sourcegraph/packaging/build-zlib-ng.sh diff --git a/.agents/setup b/.agents/setup index c9a8918d0a9ac3..476cda6b2d1e42 100755 --- a/.agents/setup +++ b/.agents/setup @@ -1,9 +1,12 @@ #!/usr/bin/env bash set -euo pipefail +zlib_ng_prefix="$HOME/.cache/sourcegraph-git/zlib-ng" + packages=' apache2 cargo + curl cvs cvsps default-jre @@ -59,6 +62,28 @@ else echo "System packages are already installed." fi +start=$SECONDS +contrib/sourcegraph/packaging/build-zlib-ng.sh "$zlib_ng_prefix" +echo "zlib-ng setup completed in $((SECONDS - start))s." + +# Keep subsequent plain `make` invocations on the same backend. config.mak is +# intentionally ignored by Git and may also contain a developer's own flags. +config_tmp=$(mktemp) +trap 'rm -f "$config_tmp"' EXIT +if test -f config.mak +then + sed '/^# BEGIN orb zlib-ng$/,/^# END orb zlib-ng$/d' config.mak >"$config_tmp" +fi +{ + cat "$config_tmp" + echo '# BEGIN orb zlib-ng' + echo 'ZLIB_NG = YesPlease' + printf 'ZLIB_NG_PATH = %s\n' "$zlib_ng_prefix" + echo '# END orb zlib-ng' +} >config.mak +rm -f "$config_tmp" +trap - EXIT + start=$SECONDS echo "Building Git..." make -j"$(getconf _NPROCESSORS_ONLN)" all diff --git a/.github/workflows/sourcegraph.yml b/.github/workflows/sourcegraph.yml index 06bd04cd455f6a..10734944b0c0a8 100644 --- a/.github/workflows/sourcegraph.yml +++ b/.github/workflows/sourcegraph.yml @@ -85,13 +85,13 @@ jobs: - name: Validate archive run: | ./contrib/sourcegraph/packaging/validate-darwin-archive.sh \ - artifacts/git-sourcegraph-v2.55.0-2-darwin-arm64.tar.gz + artifacts/git-sourcegraph-v2.55.0-3-darwin-arm64.tar.gz - uses: actions/upload-artifact@v6 with: - name: git-sourcegraph-v2.55.0-2-darwin-arm64 + name: git-sourcegraph-v2.55.0-3-darwin-arm64 path: | - artifacts/git-sourcegraph-v2.55.0-2-darwin-arm64.tar.gz - artifacts/git-sourcegraph-v2.55.0-2-darwin-arm64.tar.gz.sha256 + artifacts/git-sourcegraph-v2.55.0-3-darwin-arm64.tar.gz + artifacts/git-sourcegraph-v2.55.0-3-darwin-arm64.tar.gz.sha256 if-no-files-found: error retention-days: 7 @@ -118,7 +118,7 @@ jobs: . contrib/sourcegraph/packaging/release.sh test "$SOURCE_TAG" = v2.55.0 test "$SOURCE_COMMIT" = e9019fcafe0040228b8631c30f97ae1adb61bcdc - test "$RELEASE_VERSION" = v2.55.0-2 - test "$GIT_VERSION" = 2.55.0.sourcegraph.2 + test "$RELEASE_VERSION" = v2.55.0-3 + test "$GIT_VERSION" = 2.55.0.sourcegraph.3 grep -F 'FROM debian:12@sha256:' contrib/sourcegraph/packaging/Dockerfile.linux } diff --git a/contrib/sourcegraph/packaging/Dockerfile.linux b/contrib/sourcegraph/packaging/Dockerfile.linux index f7e535c80ee932..76a9810498d96c 100644 --- a/contrib/sourcegraph/packaging/Dockerfile.linux +++ b/contrib/sourcegraph/packaging/Dockerfile.linux @@ -2,9 +2,14 @@ FROM debian:12@sha256:6ebd97fa83deb272194a2cf015b3d26a4d538e9ad3a7a79d544c8af5b0 RUN apt-get update && \ DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ - build-essential ca-certificates cargo file gettext libcurl4-openssl-dev \ + build-essential ca-certificates cargo curl file gettext libcurl4-openssl-dev \ libexpat1-dev libpcre2-dev libssl-dev patchelf perl tcl tk zlib1g-dev && \ rm -rf /var/lib/apt/lists/* +ENV ZLIB_NG_PREFIX=/opt/zlib-ng + +COPY contrib/sourcegraph/packaging/build-zlib-ng.sh /tmp/build-zlib-ng.sh +RUN /tmp/build-zlib-ng.sh "$ZLIB_NG_PREFIX" && rm /tmp/build-zlib-ng.sh + WORKDIR /src ENTRYPOINT ["contrib/sourcegraph/packaging/build-from-source.sh", "linux-amd64"] diff --git a/contrib/sourcegraph/packaging/README.md b/contrib/sourcegraph/packaging/README.md index 0dfab09e479e1b..1281e58d295728 100644 --- a/contrib/sourcegraph/packaging/README.md +++ b/contrib/sourcegraph/packaging/README.md @@ -6,15 +6,16 @@ export the exact upstream-compatible `v2.55.0` source at commit `e9019fcafe0040228b8631c30f97ae1adb61bcdc`, regardless of the branch from which the packaging script runs. -The next immutable downstream release is `sourcegraph/v2.55.0-2`, whose Git -binary reports `2.55.0.sourcegraph.2`. It consists of exactly these files: +The next immutable downstream release is `sourcegraph/v2.55.0-3`, whose Git +binary reports `2.55.0.sourcegraph.3`. It follows the stock-zlib +`sourcegraph/v2.55.0-2` release and consists of exactly these files: -* `git-sourcegraph-v2.55.0-2-linux-amd64.tar.gz` -* `git-sourcegraph-v2.55.0-2-linux-amd64.tar.gz.sha256` -* `git-sourcegraph-v2.55.0-2-darwin-arm64.tar.gz` -* `git-sourcegraph-v2.55.0-2-darwin-arm64.tar.gz.sha256` +* `git-sourcegraph-v2.55.0-3-linux-amd64.tar.gz` +* `git-sourcegraph-v2.55.0-3-linux-amd64.tar.gz.sha256` +* `git-sourcegraph-v2.55.0-3-darwin-arm64.tar.gz` +* `git-sourcegraph-v2.55.0-3-darwin-arm64.tar.gz.sha256` -The existing `sourcegraph/v2.55.0-1` tag and assets remain immutable. +Existing downstream tags and assets remain immutable. Each archive has one `git-sourcegraph/` root. Stripping that directory exposes `bin/`, `libexec/`, `share/`, optional `lib/`, and `BUILD-INFO`. The latter @@ -37,13 +38,16 @@ Install Docker, then run: ```console ./contrib/sourcegraph/packaging/build-linux.sh ./contrib/sourcegraph/packaging/validate-linux-archive.sh \ - artifacts/git-sourcegraph-v2.55.0-2-linux-amd64.tar.gz + artifacts/git-sourcegraph-v2.55.0-3-linux-amd64.tar.gz ``` The builder image starts from Debian 12 at a pinned multi-platform image digest. `BUILD-INFO` captures the selected amd64 image's installed package versions. The archive bundles the non-glibc dynamic dependency closure and uses relative ELF RPATHs; glibc itself remains at Debian 12's 2.36 baseline. +Git links a checksum-pinned zlib-ng 2.3.3 static library built with its native +API, allowing its optimized implementation to coexist with ordinary zlib used +by other dependencies without adding a runtime library requirement. Installed executables are stripped without removing features. Git's Rust components remain enabled and are built with Debian's Rust toolchain. The build container uses the invoking user's numeric UID and GID so bind-mount @@ -62,7 +66,7 @@ xcode-select --install # if the tools are not already installed brew install rust gnu-tar ./contrib/sourcegraph/packaging/build-darwin.sh ./contrib/sourcegraph/packaging/validate-darwin-archive.sh \ - artifacts/git-sourcegraph-v2.55.0-2-darwin-arm64.tar.gz + artifacts/git-sourcegraph-v2.55.0-3-darwin-arm64.tar.gz ``` The `Darwin ARM64 archive` job runs the same commands on GitHub's macOS 15 @@ -80,6 +84,8 @@ it statically for the same target. Git uses the macOS SDK's curl, iconv, and system libraries. Localization is disabled because macOS has no system libintl and linking an incidental Homebrew gettext would make the archive depend on the build machine. Git still includes its English fallthrough messages. +The builder also downloads and statically links the same checksum-pinned +zlib-ng 2.3.3 native library used by the Linux archive. The full install includes `git-credential-osxkeychain`, rejects non-system Mach-O dependencies (including `/opt/homebrew` and build paths), verifies @@ -104,12 +110,13 @@ binary metadata, not a promise of runtime support for that older macOS release. ## License notices -Both archives include Git's `COPYING` under `LICENSES/`. Darwin also includes -the pinned PCRE2 source's `LICENCE`. Linux includes Debian's copyright notice -for every package whose shared library is copied into `lib/`, while +Both archives include Git's `COPYING` and zlib-ng's `LICENSE.md` under +`LICENSES/`. Darwin also includes the pinned PCRE2 source's `LICENCE`. Linux +includes Debian's copyright notice for every package whose shared library is +copied into `lib/`, while `BUNDLED-LIBRARIES` records each library's exact binary and source package versions and a Debian source-retrieval link. References to Debian's `/usr/share/common-licenses` resolve within `LICENSES/debian/common-licenses`. System libraries referenced by the Darwin archive are not redistributed. -Release notes should link the exact Git and PCRE2 sources; checksum sidecars -are checksums, not signatures. +Release notes should link the exact Git, PCRE2, and zlib-ng sources; checksum +sidecars are checksums, not signatures. diff --git a/contrib/sourcegraph/packaging/build-darwin.sh b/contrib/sourcegraph/packaging/build-darwin.sh index 71b85f8f83ec07..92bc61f109491b 100755 --- a/contrib/sourcegraph/packaging/build-darwin.sh +++ b/contrib/sourcegraph/packaging/build-darwin.sh @@ -88,12 +88,16 @@ pcre_prefix="$work/pcre2-install" /usr/bin/make install ) +zlib_ng_prefix="$work/zlib-ng-install" +MAKE=/usr/bin/make contrib/sourcegraph/packaging/build-zlib-ng.sh "$zlib_ng_prefix" + ( cd "$work" SOURCE_DATE_EPOCH=$(git -C "$root" show -s --format=%ct "$SOURCE_COMMIT") \ RECIPE_COMMIT="$RECIPE_COMMIT" \ PCRE2_PREFIX="$pcre_prefix" \ PCRE2_LICENSE="$work/pcre2-$PCRE2_VERSION/LICENCE.md" \ + ZLIB_NG_PREFIX="$zlib_ng_prefix" \ SOURCE_BUILD_ROOT="$work" \ contrib/sourcegraph/packaging/build-from-source.sh darwin-arm64 "$output" ) diff --git a/contrib/sourcegraph/packaging/build-from-source.sh b/contrib/sourcegraph/packaging/build-from-source.sh index 325ca31157f07a..74e686fb3e4689 100755 --- a/contrib/sourcegraph/packaging/build-from-source.sh +++ b/contrib/sourcegraph/packaging/build-from-source.sh @@ -8,6 +8,7 @@ platform=${1:?usage: build-from-source.sh PLATFORM [OUTPUT]} output=${2:-/out} : "${RECIPE_COMMIT:?RECIPE_COMMIT is required}" : "${SOURCE_DATE_EPOCH:?SOURCE_DATE_EPOCH is required}" +: "${ZLIB_NG_PREFIX:?ZLIB_NG_PREFIX is required}" stage=$(mktemp -d) trap 'rm -rf "$stage"' EXIT HUP INT TERM @@ -15,7 +16,9 @@ trap 'rm -rf "$stage"' EXIT HUP INT TERM case "$platform" in linux-amd64) test "$(uname -s)-$(uname -m)" = Linux-x86_64 - make_options='RUNTIME_PREFIX=YesPlease USE_LIBPCRE2=YesPlease INSTALL_STRIP=-s NO_INSTALL_HARDLINKS=YesPlease' + make_options="RUNTIME_PREFIX=YesPlease USE_LIBPCRE2=YesPlease \ + ZLIB_NG=YesPlease ZLIB_NG_PATH=$ZLIB_NG_PREFIX \ + INSTALL_STRIP=-s NO_INSTALL_HARDLINKS=YesPlease" ;; darwin-arm64) test "$(uname -s)-$(uname -m)" = Darwin-arm64 @@ -31,11 +34,21 @@ case "$platform" in cp "$pcre_prefix/lib/libpcre2-8.a" "$static_pcre/lib/" make_options="RUNTIME_PREFIX=YesPlease USE_LIBPCRE2=YesPlease \ LIBPCREDIR=$static_pcre ICONVDIR=/usr \ + ZLIB_NG=YesPlease ZLIB_NG_PATH=$ZLIB_NG_PREFIX \ + CC_LD_DYNPATH=-L \ INSTALL_STRIP=-s NO_GETTEXT=YesPlease NO_INSTALL_HARDLINKS=YesPlease \ USE_HOMEBREW_LIBICONV= NEEDS_GOOD_LIBICONV=" ;; *) echo "error: unsupported platform: $platform" >&2; exit 1 ;; esac +test -f "$ZLIB_NG_PREFIX/include/zlib-ng.h" +test -f "$ZLIB_NG_PREFIX/lib/libz-ng.a" +test -f "$ZLIB_NG_PREFIX/LICENSE.md" +test -f "$ZLIB_NG_PREFIX/SOURCE-INFO" +. "$ZLIB_NG_PREFIX/SOURCE-INFO" +: "${zlib_ng_version:?zlib_ng_version is required}" +: "${zlib_ng_source_commit:?zlib_ng_source_commit is required}" +: "${zlib_ng_source_sha256:?zlib_ng_source_sha256 is required}" prefix="$stage/git-sourcegraph" mkdir -p "$prefix" "$output" @@ -56,6 +69,7 @@ $prefix/bin/git version --build-options | mkdir "$prefix/LICENSES" cp COPYING "$prefix/LICENSES/Git-COPYING" +cp "$ZLIB_NG_PREFIX/LICENSE.md" "$prefix/LICENSES/Zlib-ng-LICENSE" if test "$platform" = darwin-arm64 then @@ -90,6 +104,9 @@ fi echo "cc_version=$(cc --version | head -1)" echo "rustc_version=$(rustc --version)" echo "cargo_version=$(cargo --version)" + echo "zlib_ng_version=$zlib_ng_version" + echo "zlib_ng_source_commit=$zlib_ng_source_commit" + echo "zlib_ng_source_sha256=$zlib_ng_source_sha256" if command -v dpkg-query >/dev/null 2>&1 then echo 'builder_image=debian:12@sha256:6ebd97fa83deb272194a2cf015b3d26a4d538e9ad3a7a79d544c8af5b0a01443' diff --git a/contrib/sourcegraph/packaging/build-zlib-ng.sh b/contrib/sourcegraph/packaging/build-zlib-ng.sh new file mode 100755 index 00000000000000..2422f5562d7c6c --- /dev/null +++ b/contrib/sourcegraph/packaging/build-zlib-ng.sh @@ -0,0 +1,44 @@ +#!/bin/sh +set -eu + +prefix=${1:?usage: build-zlib-ng.sh PREFIX} +version=2.3.3 +source_commit=12731092979c6d07f42da27da673a9f6c7b13586 +source_sha256=a0d2a5d122c84b56a793a1553a9c3327fb2eb7469bf7a86b79e3c7be5d92e8d6 + +work=$(mktemp -d) +trap 'rm -rf "$work"' EXIT HUP INT TERM +cat >"$work/SOURCE-INFO" <&2 echo failed >>"$status_file" fi + otool -l "$executable" | awk ' + $1 == "cmd" && $2 == "LC_RPATH" { found = 1; next } + found && $1 == "path" { print $2; found = 0 } + ' | while IFS= read -r rpath + do + case "$rpath" in + /System/Library/*|/usr/lib/*) ;; + *) echo "error: non-system rpath in $executable: $rpath" >&2; echo failed >>"$status_file" ;; + esac + done otool -L "$executable" | tail -n +2 | awk '{ print $1 }' | while IFS= read -r library do case "$library" in diff --git a/contrib/sourcegraph/packaging/validate-linux-archive.sh b/contrib/sourcegraph/packaging/validate-linux-archive.sh index 21f98c1060acf7..2788562bf30c16 100755 --- a/contrib/sourcegraph/packaging/validate-linux-archive.sh +++ b/contrib/sourcegraph/packaging/validate-linux-archive.sh @@ -17,15 +17,22 @@ do tar -xzf "$archive" -C "$work/$location" --strip-components=1 git="$work/$location/bin/git" test -s "$work/$location/LICENSES/Git-COPYING" + test -s "$work/$location/LICENSES/Zlib-ng-LICENSE" test -s "$work/$location/BUNDLED-LIBRARIES" test "$($git --version)" = "git version $GIT_VERSION" $git version --build-options | grep -F "built from commit: $SOURCE_COMMIT" + $git version --build-options | grep -Fx 'zlib-ng: 2.3.3' grep -Fx "release_version=$RELEASE_VERSION" "$work/$location/BUILD-INFO" grep -Fx "upstream_version=$UPSTREAM_VERSION" "$work/$location/BUILD-INFO" grep -Fx "release_revision=$RELEASE_REVISION" "$work/$location/BUILD-INFO" grep -Fx "source_tag=$SOURCE_TAG" "$work/$location/BUILD-INFO" grep -Fx "source_commit=$SOURCE_COMMIT" "$work/$location/BUILD-INFO" grep -Ex 'recipe_commit=[0-9a-f]{40}' "$work/$location/BUILD-INFO" + grep -Fx 'zlib_ng_version=2.3.3' "$work/$location/BUILD-INFO" + grep -Fx 'zlib_ng_source_commit=12731092979c6d07f42da27da673a9f6c7b13586' \ + "$work/$location/BUILD-INFO" + grep -Fx 'zlib_ng_source_sha256=a0d2a5d122c84b56a793a1553a9c3327fb2eb7469bf7a86b79e3c7be5d92e8d6' \ + "$work/$location/BUILD-INFO" test "$($git --exec-path)" = "$work/$location/libexec/git-core" test "$($git --html-path)" = "$work/$location/share/doc/git-doc" @@ -41,6 +48,10 @@ do rm -rf "$work/repository" done +# zlib-ng is linked statically so the relocatable archive does not gain a new +# runtime dependency or risk loading a host-provided version. +! ldd "$work/second/moved/prefix/bin/git" | grep -q 'libz-ng' + # Every bundled shared library identifies its exact Debian binary/source # package, source retrieval location, and included copyright notice. prefix="$work/second/moved/prefix" diff --git a/contrib/sourcegraph/packaging/verify-darwin-dependencies.sh b/contrib/sourcegraph/packaging/verify-darwin-dependencies.sh index 259e3be3be33c2..bd19eb4927a112 100755 --- a/contrib/sourcegraph/packaging/verify-darwin-dependencies.sh +++ b/contrib/sourcegraph/packaging/verify-darwin-dependencies.sh @@ -6,6 +6,8 @@ prefix=${1:?usage: verify-darwin-dependencies.sh PREFIX} : "${SOURCE_BUILD_ROOT:?SOURCE_BUILD_ROOT is required}" test -s "$prefix/LICENSES/Git-COPYING" test -s "$prefix/LICENSES/PCRE2-LICENCE" +test -s "$prefix/LICENSES/Zlib-ng-LICENSE" +"$prefix/bin/git" version --build-options | grep -Fx 'zlib-ng: 2.3.3' status_file=$(mktemp) trap 'rm -f "$status_file"' EXIT HUP INT TERM find "$prefix/bin" "$prefix/libexec" -type f -perm -111 | while IFS= read -r executable @@ -29,6 +31,16 @@ do echo failed >>"$status_file" fi done + otool -l "$executable" | awk ' + $1 == "cmd" && $2 == "LC_RPATH" { found = 1; next } + found && $1 == "path" { print $2; found = 0 } + ' | while IFS= read -r rpath + do + case "$rpath" in + /System/Library/*|/usr/lib/*) ;; + *) echo "error: non-system rpath in $executable: $rpath" >&2; echo failed >>"$status_file" ;; + esac + done otool -L "$executable" | tail -n +2 | awk '{ print $1 }' | while IFS= read -r library do case "$library" in