diff --git a/NOTICE b/NOTICE index 8ef4a7a1bd..4b9c9991f2 100644 --- a/NOTICE +++ b/NOTICE @@ -131,6 +131,10 @@ jackc/pgx - https://github.com/jackc/pgx Copyright (c) 2013-2021 Jack Christensen License - https://github.com/jackc/pgx/blob/master/LICENSE +klauspost/pgzip - https://github.com/klauspost/pgzip +Copyright (c) 2014 Klaus Post +License - https://github.com/klauspost/pgzip/blob/master/LICENSE + charmbracelet/bubbles - https://github.com/charmbracelet/bubbles Copyright (c) 2020-2025 Charmbracelet, Inc License - https://github.com/charmbracelet/bubbles/blob/master/LICENSE diff --git a/experimental/air/cmd/snapshot_package.go b/experimental/air/cmd/snapshot_package.go index c22f7dd61a..a2203c2497 100644 --- a/experimental/air/cmd/snapshot_package.go +++ b/experimental/air/cmd/snapshot_package.go @@ -9,6 +9,8 @@ import ( "os/exec" "path/filepath" "strings" + + "github.com/klauspost/pgzip" ) // Tar builders ported from cli/utils/snapshot.py. Both shell out (git archive / tar) @@ -41,47 +43,57 @@ func createGitArchiveSnapshot(ctx context.Context, git gitRepo, commitSHA, outpu // excluded; a .gitignore at repoPath is honored. func createPlainTarball(ctx context.Context, repoPath, outputTarball string, includePaths []string, isGitRepo bool) error { dirName := filepath.Base(repoPath) - // Absolute so it resolves correctly regardless of tar's working dir (set below). + // Absolute so it resolves correctly regardless of tar's working dir. parent, err := filepath.Abs(filepath.Dir(repoPath)) if err != nil { return err } - // Pass the archive path relative to its own directory (run tar there), never a - // full path: on Windows an absolute path like `C:\out\x.tar.gz` makes tar read - // the `C:` as a remote host ("Cannot connect to C:"), since tar treats a colon - // in the -f arg as host:path. A bare basename with -C avoids that on GNU tar and - // bsdtar alike. - outDirAbs, err := filepath.Abs(filepath.Dir(outputTarball)) + files, err := snapshotFiles(ctx, repoPath, includePaths, isGitRepo) if err != nil { return err } - outName := filepath.Base(outputTarball) - files, err := snapshotFiles(ctx, repoPath, includePaths, isGitRepo) + out, err := os.Create(outputTarball) + if err != nil { + return fmt.Errorf("failed to create tarball: %w", err) + } + + // tar writes the uncompressed archive to stdout and we gzip it with klauspost/pgzip + // rather than tar's single-threaded -z, which dominates packaging time on a large + // tree (pgzip spreads the same compression across cores). Compressing outside tar + // also keeps any archive path out of tar's args, avoiding the Windows colon-in-path + // issue the `-f ` form otherwise hits. + gz, err := pgzip.NewWriterLevel(out, pgzip.DefaultCompression) if err != nil { + out.Close() return err } - args := []string{"-czf", outName, "-C", parent, "--null", "--no-recursion", "-T", "-"} + args := []string{"-cf", "-", "-C", parent, "--null", "--no-recursion", "-T", "-"} cmd := exec.CommandContext(ctx, "tar", args...) - // Run tar in the output directory so the bare -f basename lands there. - cmd.Dir = outDirAbs var stdin bytes.Buffer for _, file := range files { stdin.WriteString(filepath.ToSlash(filepath.Join(dirName, file))) stdin.WriteByte(0) } cmd.Stdin = &stdin + cmd.Stdout = gz var stderr bytes.Buffer cmd.Stderr = &stderr if err := cmd.Run(); err != nil { + gz.Close() + out.Close() if msg := strings.TrimSpace(stderr.String()); msg != "" { return fmt.Errorf("failed to create plain tarball: %w: %s", err, msg) } return fmt.Errorf("failed to create plain tarball: %w", err) } - return nil + if err := gz.Close(); err != nil { + out.Close() + return fmt.Errorf("failed to finalize gzip: %w", err) + } + return out.Close() } func snapshotFiles(ctx context.Context, repoPath string, includePaths []string, isGitRepo bool) ([]string, error) { diff --git a/go.mod b/go.mod index d57fab795a..34f6ef3872 100644 --- a/go.mod +++ b/go.mod @@ -23,6 +23,7 @@ require ( github.com/hashicorp/terraform-json v0.28.0 // MPL-2.0 github.com/hexops/gotextdiff v1.0.3 // BSD-3-Clause github.com/jackc/pgx/v5 v5.10.0 // MIT + github.com/klauspost/pgzip v1.2.6 // MIT github.com/mattn/go-isatty v0.0.24 // MIT github.com/muesli/termenv v0.16.0 // MIT github.com/palantir/pkg/yamlpatch v1.5.0 // BSD-3-Clause @@ -87,6 +88,7 @@ require ( github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/klauspost/compress v1.19.2 // indirect github.com/lucasb-eyer/go-colorful v1.4.0 // indirect github.com/mattn/go-colorable v0.1.14 // indirect github.com/mattn/go-localereader v0.0.1 // indirect diff --git a/go.sum b/go.sum index 0c274225df..e06a6c8a10 100644 --- a/go.sum +++ b/go.sum @@ -156,8 +156,12 @@ github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOl github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= +github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8= +github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU= +github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=