bundle/artifacts: parallel gzip (pgzip) for the tgz artifact builder

NOTICE

@@ -131,6 +131,10 @@ jackc/pgx - https://github.com/jackc/pgx
 Copyright (c) 2013-2021 Jack Christensen
 License - https://github.com/jackc/pgx/blob/master/LICENSE
 
+klauspost/pgzip - https://github.com/klauspost/pgzip
+Copyright (c) 2014 Klaus Post
+License - https://github.com/klauspost/pgzip/blob/master/LICENSE
+
 charmbracelet/bubbles - https://github.com/charmbracelet/bubbles
 Copyright (c) 2020-2025 Charmbracelet, Inc
 License - https://github.com/charmbracelet/bubbles/blob/master/LICENSE

bundle/artifacts/tarball.go

@@ -2,7 +2,6 @@ package artifacts
 
 import (
 	"archive/tar"
-	"compress/gzip"
 	"context"
 	"errors"
 	"fmt"
@@ -11,6 +10,7 @@ import (
 	"os/exec"
 	"path"
 	"path/filepath"
+	"runtime"
 	"slices"
 	"strings"
 	"time"
@@ -21,12 +21,32 @@ import (
 	"github.com/databricks/cli/libs/fileset"
 	libsync "github.com/databricks/cli/libs/sync"
 	"github.com/databricks/cli/libs/vfs"
+	"github.com/klauspost/pgzip"
 )
 
 // tarballEpoch stamps every entry so the archive is reproducible: identical contents
 // produce identical bytes regardless of file mtimes.
 var tarballEpoch = time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC)
 
+// pgzipBlockSize is the input chunk pgzip compresses per block. gzip's single-threaded
+// compression dominates packaging time on a large tree, so we use klauspost/pgzip to
+// spread it across cores. We keep the default compression level (matching the old
+// single-threaded gzip, so tarball sizes don't change) and pin the block size: pgzip
+// only parallelizes *across* blocks, so a fixed block size keeps the compressed byte
+// stream identical regardless of how many cores the build host has — preserving the
+// reproducibility contract above. This is pgzip's own default block size (1 MiB).
+const pgzipBlockSize = 1 << 20
+
+// newParallelGzip returns a parallel gzip writer with a deterministic block size. Callers
+// must Close it to flush the trailer.
+func newParallelGzip(w io.Writer) (*pgzip.Writer, error) {
+	gzw := pgzip.NewWriter(w)
+	if err := gzw.SetConcurrency(pgzipBlockSize, runtime.GOMAXPROCS(0)); err != nil {
+		return nil, err
+	}
+	return gzw, nil
+}
+
 // buildTarballArtifact produces the gzipped tarball for a `type: tgz` artifact that
 // DABs builds itself (no user `build` command). Archive entries are the packed files
 // named relative to the artifact's `path`. With `git` set the tarball snapshots that
@@ -123,7 +143,10 @@ func tarballFromInclude(ctx context.Context, b *bundle.Bundle, a *config.Artifac
 		return strings.Compare(x.Relative, y.Relative)
 	})
 
-	gzw := gzip.NewWriter(w)
+	gzw, err := newParallelGzip(w)
+	if err != nil {
+		return err
+	}
 	tw := tar.NewWriter(gzw)
 	for _, file := range list {
 		if err := addFileToTarball(tw, b.SyncRoot, relBase, file); err != nil {
@@ -157,19 +180,26 @@ func tarballFromGit(ctx context.Context, b *bundle.Bundle, a *config.Artifact, w
 		// The tree at <ref>:<path>, so entries come out relative to `path`.
 		treeish = ref + ":" + relBase
 	}
-	args := []string{"-C", b.SyncRootPath, "archive", "--format=tar.gz", treeish}
+	// Ask git for an uncompressed tar and gzip it ourselves with pgzip: git's
+	// --format=tar.gz gzip is single-threaded, whereas pgzip spreads it across cores.
+	args := []string{"-C", b.SyncRootPath, "archive", "--format=tar", treeish}
 	if len(a.Include) > 0 {
 		args = append(args, "--")
 		args = append(args, a.Include...)
 	}
+	gzw, err := newParallelGzip(w)
+	if err != nil {
+		return err
+	}
 	cmd := exec.CommandContext(ctx, "git", args...)
-	cmd.Stdout = w
+	cmd.Stdout = gzw
 	var stderr strings.Builder
 	cmd.Stderr = &stderr
 	if err := cmd.Run(); err != nil {
+		gzw.Close()
 		return fmt.Errorf("git archive %s: %w: %s", treeish, err, stderr.String())
 	}
-	return nil
+	return gzw.Close()
 }
 
 // addFileToTarball writes f (a sync-root-relative file) to the archive under a name

bundle/artifacts/tarball_test.go

@@ -77,3 +77,30 @@ func TestTarballFromGitRequiresRef(t *testing.T) {
 	err := tarballFromGit(t.Context(), b, a, io.Discard)
 	require.ErrorContains(t, err, "git.commit or git.branch")
 }
+
+// The tarball is content-addressed on upload, so the same tree must always compress to
+// the same bytes. We now gzip with parallel pgzip; this guards that its output stays
+// deterministic (pgzip only parallelizes across fixed-size blocks, so core count doesn't
+// leak into the bytes) and that it decompresses cleanly.
+func TestTarballFromGitIsReproducible(t *testing.T) {
+	repo := t.TempDir()
+	runGit(t, repo, "init", "-q")
+	runGit(t, repo, "config", "user.email", "t@example.com")
+	runGit(t, repo, "config", "user.name", "t")
+	runGit(t, repo, "config", "core.autocrlf", "false")
+	require.NoError(t, os.WriteFile(filepath.Join(repo, "train.py"), []byte("print('x')\n"), 0o644))
+	require.NoError(t, os.WriteFile(filepath.Join(repo, "big.txt"), bytes.Repeat([]byte("compress me\n"), 200000), 0o644))
+	runGit(t, repo, "add", "-A")
+	runGit(t, repo, "commit", "-qm", "init")
+
+	b := &bundle.Bundle{SyncRootPath: repo}
+	a := &config.Artifact{Path: repo, Git: &config.ArtifactGit{Commit: "HEAD"}}
+
+	var buf1, buf2 bytes.Buffer
+	require.NoError(t, tarballFromGit(t.Context(), b, a, &buf1))
+	require.NoError(t, tarballFromGit(t.Context(), b, a, &buf2))
+
+	assert.Equal(t, buf1.Bytes(), buf2.Bytes(), "identical tree must compress to identical bytes")
+	// Round-trips through gzip and carries the packed file.
+	assert.Equal(t, "print('x')", strings.TrimSpace(tarEntries(t, buf1.Bytes())["train.py"]))
+}

go.mod

@@ -23,6 +23,7 @@ require (
 	github.com/hashicorp/terraform-json v0.28.0 // MPL-2.0
 	github.com/hexops/gotextdiff v1.0.3 // BSD-3-Clause
 	github.com/jackc/pgx/v5 v5.10.0 // MIT
+	github.com/klauspost/pgzip v1.2.6 // MIT
 	github.com/mattn/go-isatty v0.0.24 // MIT
 	github.com/muesli/termenv v0.16.0 // MIT
 	github.com/palantir/pkg/yamlpatch v1.5.0 // BSD-3-Clause
@@ -87,6 +88,7 @@ require (
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/jackc/pgpassfile v1.0.0 // indirect
 	github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
+	github.com/klauspost/compress v1.19.2 // indirect
 	github.com/lucasb-eyer/go-colorful v1.4.0 // indirect
 	github.com/mattn/go-colorable v0.1.14 // indirect
 	github.com/mattn/go-localereader v0.0.1 // indirect

go.sum

@@ -156,8 +156,12 @@ github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOl
 github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo=
 github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4=
 github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
+github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8=
+github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
 github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y=
 github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
+github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU=
+github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
 github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=