From fbe71c9e2670377e6d7fe24f2f0ee19de079ee54 Mon Sep 17 00:00:00 2001 From: "ben.hansen" Date: Tue, 8 Sep 2026 14:35:53 +0000 Subject: [PATCH 1/3] experimental/air: parallel gzip for the plain_tar snapshot packer createPlainTarball shelled out to `tar -czf`, whose gzip is single-threaded and dominates packaging time for a large code_source tree. Pipe `tar -cf -` through klauspost/pgzip instead, spreading compression across cores. Measured ~4x on universe/research (2456 ms -> 609 ms) and ~18x on the 470 MiB gzip step alone; the output is an ordinary gzip stream. Level is BestSpeed since the uploaded size does not matter for this workflow, only latency. Compressing outside tar also passes no archive path to tar, which removes the Windows colon-in-path workaround (bare `-f` basename + -C) the -czf form needed. Co-authored-by: Isaac --- NOTICE | 4 +++ experimental/air/cmd/snapshot_package.go | 41 ++++++++++++++++-------- go.mod | 2 ++ go.sum | 4 +++ 4 files changed, 38 insertions(+), 13 deletions(-) diff --git a/NOTICE b/NOTICE index 8ef4a7a1bd0..4b9c9991f20 100644 --- a/NOTICE +++ b/NOTICE @@ -131,6 +131,10 @@ jackc/pgx - https://github.com/jackc/pgx Copyright (c) 2013-2021 Jack Christensen License - https://github.com/jackc/pgx/blob/master/LICENSE +klauspost/pgzip - https://github.com/klauspost/pgzip +Copyright (c) 2014 Klaus Post +License - https://github.com/klauspost/pgzip/blob/master/LICENSE + charmbracelet/bubbles - https://github.com/charmbracelet/bubbles Copyright (c) 2020-2025 Charmbracelet, Inc License - https://github.com/charmbracelet/bubbles/blob/master/LICENSE diff --git a/experimental/air/cmd/snapshot_package.go b/experimental/air/cmd/snapshot_package.go index c22f7dd61af..a3ac56d6131 100644 --- a/experimental/air/cmd/snapshot_package.go +++ b/experimental/air/cmd/snapshot_package.go @@ -9,6 +9,8 @@ import ( "os/exec" "path/filepath" "strings" + + "github.com/klauspost/pgzip" ) // Tar builders ported from cli/utils/snapshot.py. Both shell out (git archive / tar) @@ -41,47 +43,60 @@ func createGitArchiveSnapshot(ctx context.Context, git gitRepo, commitSHA, outpu // excluded; a .gitignore at repoPath is honored. func createPlainTarball(ctx context.Context, repoPath, outputTarball string, includePaths []string, isGitRepo bool) error { dirName := filepath.Base(repoPath) - // Absolute so it resolves correctly regardless of tar's working dir (set below). + // Absolute so it resolves correctly regardless of tar's working dir. parent, err := filepath.Abs(filepath.Dir(repoPath)) if err != nil { return err } - // Pass the archive path relative to its own directory (run tar there), never a - // full path: on Windows an absolute path like `C:\out\x.tar.gz` makes tar read - // the `C:` as a remote host ("Cannot connect to C:"), since tar treats a colon - // in the -f arg as host:path. A bare basename with -C avoids that on GNU tar and - // bsdtar alike. - outDirAbs, err := filepath.Abs(filepath.Dir(outputTarball)) + files, err := snapshotFiles(ctx, repoPath, includePaths, isGitRepo) if err != nil { return err } - outName := filepath.Base(outputTarball) - files, err := snapshotFiles(ctx, repoPath, includePaths, isGitRepo) + out, err := os.Create(outputTarball) + if err != nil { + return fmt.Errorf("failed to create tarball: %w", err) + } + + // tar writes the uncompressed archive to stdout (`-cf -`) and we gzip it here with + // klauspost/pgzip instead of tar's built-in -z: tar's gzip is single-threaded and + // dominates packaging time on a large tree, whereas pgzip spreads it across cores + // (measured ~18x faster on a ~470 MiB archive) while still emitting an ordinary gzip + // stream. BestSpeed because the uploaded size does not matter for this workflow, only + // latency. Compressing outside tar also passes no archive path to tar, sidestepping + // the Windows colon-in-path issue a `-f ` argument otherwise hits (tar reads the + // `C:` in `C:\out\x.tar.gz` as a remote host). + gz, err := pgzip.NewWriterLevel(out, pgzip.BestSpeed) if err != nil { + out.Close() return err } - args := []string{"-czf", outName, "-C", parent, "--null", "--no-recursion", "-T", "-"} + args := []string{"-cf", "-", "-C", parent, "--null", "--no-recursion", "-T", "-"} cmd := exec.CommandContext(ctx, "tar", args...) - // Run tar in the output directory so the bare -f basename lands there. - cmd.Dir = outDirAbs var stdin bytes.Buffer for _, file := range files { stdin.WriteString(filepath.ToSlash(filepath.Join(dirName, file))) stdin.WriteByte(0) } cmd.Stdin = &stdin + cmd.Stdout = gz var stderr bytes.Buffer cmd.Stderr = &stderr if err := cmd.Run(); err != nil { + gz.Close() + out.Close() if msg := strings.TrimSpace(stderr.String()); msg != "" { return fmt.Errorf("failed to create plain tarball: %w: %s", err, msg) } return fmt.Errorf("failed to create plain tarball: %w", err) } - return nil + if err := gz.Close(); err != nil { + out.Close() + return fmt.Errorf("failed to finalize gzip: %w", err) + } + return out.Close() } func snapshotFiles(ctx context.Context, repoPath string, includePaths []string, isGitRepo bool) ([]string, error) { diff --git a/go.mod b/go.mod index d57fab795a1..34f6ef38720 100644 --- a/go.mod +++ b/go.mod @@ -23,6 +23,7 @@ require ( github.com/hashicorp/terraform-json v0.28.0 // MPL-2.0 github.com/hexops/gotextdiff v1.0.3 // BSD-3-Clause github.com/jackc/pgx/v5 v5.10.0 // MIT + github.com/klauspost/pgzip v1.2.6 // MIT github.com/mattn/go-isatty v0.0.24 // MIT github.com/muesli/termenv v0.16.0 // MIT github.com/palantir/pkg/yamlpatch v1.5.0 // BSD-3-Clause @@ -87,6 +88,7 @@ require ( github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/klauspost/compress v1.19.2 // indirect github.com/lucasb-eyer/go-colorful v1.4.0 // indirect github.com/mattn/go-colorable v0.1.14 // indirect github.com/mattn/go-localereader v0.0.1 // indirect diff --git a/go.sum b/go.sum index 0c274225dfa..e06a6c8a10a 100644 --- a/go.sum +++ b/go.sum @@ -156,8 +156,12 @@ github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOl github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= +github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8= +github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU= +github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= From e3b164d42506e50f5036efec141a836cd5ece503 Mon Sep 17 00:00:00 2001 From: "ben.hansen" Date: Tue, 8 Sep 2026 17:08:02 +0000 Subject: [PATCH 2/3] experimental/air: use default gzip level instead of BestSpeed Now that gzip is parallel the compression level is nearly free, so trade a little CPU for a smaller upload -- the plain_tar archive is re-uploaded on every run. DefaultCompression matches the old `tar -czf` size at ~18x the speed (research: 716 ms / 24 MB, vs BestSpeed 609 ms / 27 MB). Level 9 buys ~1% fewer bytes for ~2x the time, so 6 is the knee. Co-authored-by: Isaac --- experimental/air/cmd/snapshot_package.go | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/experimental/air/cmd/snapshot_package.go b/experimental/air/cmd/snapshot_package.go index a3ac56d6131..1e7adadc19f 100644 --- a/experimental/air/cmd/snapshot_package.go +++ b/experimental/air/cmd/snapshot_package.go @@ -61,13 +61,15 @@ func createPlainTarball(ctx context.Context, repoPath, outputTarball string, inc // tar writes the uncompressed archive to stdout (`-cf -`) and we gzip it here with // klauspost/pgzip instead of tar's built-in -z: tar's gzip is single-threaded and - // dominates packaging time on a large tree, whereas pgzip spreads it across cores - // (measured ~18x faster on a ~470 MiB archive) while still emitting an ordinary gzip - // stream. BestSpeed because the uploaded size does not matter for this workflow, only - // latency. Compressing outside tar also passes no archive path to tar, sidestepping - // the Windows colon-in-path issue a `-f ` argument otherwise hits (tar reads the - // `C:` in `C:\out\x.tar.gz` as a remote host). - gz, err := pgzip.NewWriterLevel(out, pgzip.BestSpeed) + // dominates packaging time on a large tree, whereas pgzip spreads the same + // compression across cores (~18x faster on a ~470 MiB archive). We keep the default + // level rather than BestSpeed: the archive is re-uploaded on every run, so its size + // matters, and now that compression is parallel a normal level costs only a few + // hundred ms more for ~15-18% fewer bytes (and matches the old `tar -czf` size); + // level 9 buys almost nothing beyond that for ~2x the time. Compressing outside tar + // also passes no archive path to tar, sidestepping the Windows colon-in-path issue a + // `-f ` argument otherwise hits (tar reads the `C:` in `C:\out\x` as a host). + gz, err := pgzip.NewWriterLevel(out, pgzip.DefaultCompression) if err != nil { out.Close() return err From 8155ccfd55f291b226f9a4d53457cd3b76a46053 Mon Sep 17 00:00:00 2001 From: "ben.hansen" Date: Tue, 8 Sep 2026 21:24:20 +0000 Subject: [PATCH 3/3] experimental/air: trim the pgzip comment per review Vincent found the pack-step comment too dense. Cut it from the compression-level essay (that rationale lives in the PR description and commit message) down to the two non-obvious whys: parallel gzip vs tar's single-threaded -z, and compressing outside tar to avoid the Windows colon-in-path issue. Co-authored-by: Isaac --- experimental/air/cmd/snapshot_package.go | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/experimental/air/cmd/snapshot_package.go b/experimental/air/cmd/snapshot_package.go index 1e7adadc19f..a2203c24970 100644 --- a/experimental/air/cmd/snapshot_package.go +++ b/experimental/air/cmd/snapshot_package.go @@ -59,16 +59,11 @@ func createPlainTarball(ctx context.Context, repoPath, outputTarball string, inc return fmt.Errorf("failed to create tarball: %w", err) } - // tar writes the uncompressed archive to stdout (`-cf -`) and we gzip it here with - // klauspost/pgzip instead of tar's built-in -z: tar's gzip is single-threaded and - // dominates packaging time on a large tree, whereas pgzip spreads the same - // compression across cores (~18x faster on a ~470 MiB archive). We keep the default - // level rather than BestSpeed: the archive is re-uploaded on every run, so its size - // matters, and now that compression is parallel a normal level costs only a few - // hundred ms more for ~15-18% fewer bytes (and matches the old `tar -czf` size); - // level 9 buys almost nothing beyond that for ~2x the time. Compressing outside tar - // also passes no archive path to tar, sidestepping the Windows colon-in-path issue a - // `-f ` argument otherwise hits (tar reads the `C:` in `C:\out\x` as a host). + // tar writes the uncompressed archive to stdout and we gzip it with klauspost/pgzip + // rather than tar's single-threaded -z, which dominates packaging time on a large + // tree (pgzip spreads the same compression across cores). Compressing outside tar + // also keeps any archive path out of tar's args, avoiding the Windows colon-in-path + // issue the `-f ` form otherwise hits. gz, err := pgzip.NewWriterLevel(out, pgzip.DefaultCompression) if err != nil { out.Close()