diff --git a/NOTICE b/NOTICE index 8ef4a7a1bd0..4b9c9991f20 100644 --- a/NOTICE +++ b/NOTICE @@ -131,6 +131,10 @@ jackc/pgx - https://github.com/jackc/pgx Copyright (c) 2013-2021 Jack Christensen License - https://github.com/jackc/pgx/blob/master/LICENSE +klauspost/pgzip - https://github.com/klauspost/pgzip +Copyright (c) 2014 Klaus Post +License - https://github.com/klauspost/pgzip/blob/master/LICENSE + charmbracelet/bubbles - https://github.com/charmbracelet/bubbles Copyright (c) 2020-2025 Charmbracelet, Inc License - https://github.com/charmbracelet/bubbles/blob/master/LICENSE diff --git a/bundle/artifacts/tarball.go b/bundle/artifacts/tarball.go index 26e34efe8af..85c0124220b 100644 --- a/bundle/artifacts/tarball.go +++ b/bundle/artifacts/tarball.go @@ -2,7 +2,6 @@ package artifacts import ( "archive/tar" - "compress/gzip" "context" "errors" "fmt" @@ -11,6 +10,7 @@ import ( "os/exec" "path" "path/filepath" + "runtime" "slices" "strings" "time" @@ -21,12 +21,32 @@ import ( "github.com/databricks/cli/libs/fileset" libsync "github.com/databricks/cli/libs/sync" "github.com/databricks/cli/libs/vfs" + "github.com/klauspost/pgzip" ) // tarballEpoch stamps every entry so the archive is reproducible: identical contents // produce identical bytes regardless of file mtimes. var tarballEpoch = time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC) +// pgzipBlockSize is the input chunk pgzip compresses per block. gzip's single-threaded +// compression dominates packaging time on a large tree, so we use klauspost/pgzip to +// spread it across cores. We keep the default compression level (matching the old +// single-threaded gzip, so tarball sizes don't change) and pin the block size: pgzip +// only parallelizes *across* blocks, so a fixed block size keeps the compressed byte +// stream identical regardless of how many cores the build host has — preserving the +// reproducibility contract above. This is pgzip's own default block size (1 MiB). +const pgzipBlockSize = 1 << 20 + +// newParallelGzip returns a parallel gzip writer with a deterministic block size. Callers +// must Close it to flush the trailer. +func newParallelGzip(w io.Writer) (*pgzip.Writer, error) { + gzw := pgzip.NewWriter(w) + if err := gzw.SetConcurrency(pgzipBlockSize, runtime.GOMAXPROCS(0)); err != nil { + return nil, err + } + return gzw, nil +} + // buildTarballArtifact produces the gzipped tarball for a `type: tgz` artifact that // DABs builds itself (no user `build` command). Archive entries are the packed files // named relative to the artifact's `path`. With `git` set the tarball snapshots that @@ -123,7 +143,10 @@ func tarballFromInclude(ctx context.Context, b *bundle.Bundle, a *config.Artifac return strings.Compare(x.Relative, y.Relative) }) - gzw := gzip.NewWriter(w) + gzw, err := newParallelGzip(w) + if err != nil { + return err + } tw := tar.NewWriter(gzw) for _, file := range list { if err := addFileToTarball(tw, b.SyncRoot, relBase, file); err != nil { @@ -157,19 +180,26 @@ func tarballFromGit(ctx context.Context, b *bundle.Bundle, a *config.Artifact, w // The tree at :, so entries come out relative to `path`. treeish = ref + ":" + relBase } - args := []string{"-C", b.SyncRootPath, "archive", "--format=tar.gz", treeish} + // Ask git for an uncompressed tar and gzip it ourselves with pgzip: git's + // --format=tar.gz gzip is single-threaded, whereas pgzip spreads it across cores. + args := []string{"-C", b.SyncRootPath, "archive", "--format=tar", treeish} if len(a.Include) > 0 { args = append(args, "--") args = append(args, a.Include...) } + gzw, err := newParallelGzip(w) + if err != nil { + return err + } cmd := exec.CommandContext(ctx, "git", args...) - cmd.Stdout = w + cmd.Stdout = gzw var stderr strings.Builder cmd.Stderr = &stderr if err := cmd.Run(); err != nil { + gzw.Close() return fmt.Errorf("git archive %s: %w: %s", treeish, err, stderr.String()) } - return nil + return gzw.Close() } // addFileToTarball writes f (a sync-root-relative file) to the archive under a name diff --git a/bundle/artifacts/tarball_test.go b/bundle/artifacts/tarball_test.go index 40febce099a..6da9a91d9e2 100644 --- a/bundle/artifacts/tarball_test.go +++ b/bundle/artifacts/tarball_test.go @@ -77,3 +77,30 @@ func TestTarballFromGitRequiresRef(t *testing.T) { err := tarballFromGit(t.Context(), b, a, io.Discard) require.ErrorContains(t, err, "git.commit or git.branch") } + +// The tarball is content-addressed on upload, so the same tree must always compress to +// the same bytes. We now gzip with parallel pgzip; this guards that its output stays +// deterministic (pgzip only parallelizes across fixed-size blocks, so core count doesn't +// leak into the bytes) and that it decompresses cleanly. +func TestTarballFromGitIsReproducible(t *testing.T) { + repo := t.TempDir() + runGit(t, repo, "init", "-q") + runGit(t, repo, "config", "user.email", "t@example.com") + runGit(t, repo, "config", "user.name", "t") + runGit(t, repo, "config", "core.autocrlf", "false") + require.NoError(t, os.WriteFile(filepath.Join(repo, "train.py"), []byte("print('x')\n"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(repo, "big.txt"), bytes.Repeat([]byte("compress me\n"), 200000), 0o644)) + runGit(t, repo, "add", "-A") + runGit(t, repo, "commit", "-qm", "init") + + b := &bundle.Bundle{SyncRootPath: repo} + a := &config.Artifact{Path: repo, Git: &config.ArtifactGit{Commit: "HEAD"}} + + var buf1, buf2 bytes.Buffer + require.NoError(t, tarballFromGit(t.Context(), b, a, &buf1)) + require.NoError(t, tarballFromGit(t.Context(), b, a, &buf2)) + + assert.Equal(t, buf1.Bytes(), buf2.Bytes(), "identical tree must compress to identical bytes") + // Round-trips through gzip and carries the packed file. + assert.Equal(t, "print('x')", strings.TrimSpace(tarEntries(t, buf1.Bytes())["train.py"])) +} diff --git a/go.mod b/go.mod index d57fab795a1..34f6ef38720 100644 --- a/go.mod +++ b/go.mod @@ -23,6 +23,7 @@ require ( github.com/hashicorp/terraform-json v0.28.0 // MPL-2.0 github.com/hexops/gotextdiff v1.0.3 // BSD-3-Clause github.com/jackc/pgx/v5 v5.10.0 // MIT + github.com/klauspost/pgzip v1.2.6 // MIT github.com/mattn/go-isatty v0.0.24 // MIT github.com/muesli/termenv v0.16.0 // MIT github.com/palantir/pkg/yamlpatch v1.5.0 // BSD-3-Clause @@ -87,6 +88,7 @@ require ( github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/klauspost/compress v1.19.2 // indirect github.com/lucasb-eyer/go-colorful v1.4.0 // indirect github.com/mattn/go-colorable v0.1.14 // indirect github.com/mattn/go-localereader v0.0.1 // indirect diff --git a/go.sum b/go.sum index 0c274225dfa..e06a6c8a10a 100644 --- a/go.sum +++ b/go.sum @@ -156,8 +156,12 @@ github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOl github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= +github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8= +github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU= +github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=