diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 2b327d8e..7f4d927f 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -34,7 +34,7 @@ jobs: - name: Find tests not covered by any named group id: find-uncovered run: | - # Combined regex of every pattern used across the 13 named matrix groups. + # Combined regex of every pattern used across the 14 named matrix groups. # Any test whose name does NOT match this will land in the catch-all group. # Built via concatenation so every line stays at ≥10-space YAML indentation. CP="TestCreateScan|TestScanCreate|TestScansE2E|TestFastScan" @@ -57,6 +57,7 @@ jobs: CP="${CP}|TestGetLearnMore|TestImport|TestGetTenant|TestMaskSecrets|TestFailedMask" CP="${CP}|TestScaRemediation|TestKicsRemediation|TestTelemetry|Test_Handle|TestChat" CP="${CP}|TestIntegrationScaResolver" + CP="${CP}|TestExcludeGitFolder" COVERED_PATTERNS="${CP}" ALL_TESTS=$(grep -rh "^func Test" test/integration/*_test.go \ @@ -85,7 +86,7 @@ jobs: fi # ───────────────────────────────────────────────────────────────────────────── - # Job B: Run each test group in parallel across 14 matrix entries. + # Job B: Run each test group in parallel across 15 matrix entries. # The 14th entry (uncovered) is a dynamic catch-all driven by Job A. # ───────────────────────────────────────────────────────────────────────────── integration-tests: @@ -200,7 +201,15 @@ jobs: needs_precommit: "false" run_cleandata: "true" - # 14 ── Catch-All (dynamic; pattern injected at runtime from Job A output) + # 14 ── Git Folder Exclusion & Contributors CSV (integration tests for --exclude-git-folder flag with CSV/JSON generation) + - name: git-folder-exclude + label: "Git Folder Exclusion & CSV" + run_pattern: "TestExcludeGitFolder" + timeout: "30m" + needs_precommit: "false" + run_cleandata: "false" + + # 15 ── Catch-All (dynamic; pattern injected at runtime from Job A output) - name: uncovered label: "Catch-All (Uncovered)" run_pattern: "__UNCOVERED__" diff --git a/go.mod b/go.mod index c1949950..fafb62f0 100644 --- a/go.mod +++ b/go.mod @@ -40,7 +40,7 @@ require ( ) require ( - cyphar.com/go-pathrs v0.2.4 // indirect + cyphar.com/go-pathrs v0.2.5 // indirect github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 // indirect github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20240914100643-eb91380d8434 // indirect github.com/Masterminds/semver v1.5.0 // indirect @@ -55,7 +55,7 @@ require ( github.com/godbus/dbus/v5 v5.2.2 // indirect github.com/golang/snappy v1.0.0 // indirect github.com/google/jsonschema-go v0.4.3 // indirect - github.com/klauspost/cpuid/v2 v2.3.0 // indirect + github.com/klauspost/cpuid/v2 v2.4.0 // indirect github.com/knqyf263/go-rpmdb v0.1.1 // indirect github.com/mitchellh/hashstructure/v2 v2.0.2 // indirect github.com/mitchellh/mapstructure v1.5.1-0.20220423092549-19e70c243037 // indirect @@ -86,7 +86,7 @@ require ( github.com/Masterminds/squirrel v1.5.4 // indirect github.com/Microsoft/go-winio v0.6.3-0.20251027160822-ad3df93bed29 // indirect github.com/Microsoft/hcsshim v0.15.0-rc.3 // indirect - github.com/ProtonMail/go-crypto v1.4.0 // indirect + github.com/ProtonMail/go-crypto v1.4.1 // indirect github.com/acobaugh/osrelease v0.1.0 // indirect github.com/adrg/xdg v0.5.3 // indirect github.com/agext/levenshtein v1.2.3 // indirect @@ -119,7 +119,7 @@ require ( github.com/charmbracelet/x/ansi v0.11.6 // indirect github.com/charmbracelet/x/cellbuf v0.0.15 // indirect github.com/charmbracelet/x/term v0.2.2 // indirect - github.com/cloudflare/circl v1.6.3 // indirect + github.com/cloudflare/circl v1.6.5 // indirect github.com/containerd/cgroups/v3 v3.1.3 // indirect github.com/containerd/containerd v1.7.35 // indirect github.com/containerd/containerd/api v1.10.0 // indirect @@ -131,7 +131,7 @@ require ( github.com/containerd/platforms v1.0.0-rc.4 // indirect github.com/containerd/ttrpc v1.2.8 // indirect github.com/containerd/typeurl/v2 v2.3.0 // indirect - github.com/cyphar/filepath-securejoin v0.6.1 // indirect + github.com/cyphar/filepath-securejoin v0.7.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/deitch/magic v0.0.0-20240306090643-c67ab88f10cb // indirect github.com/distribution/reference v0.6.0 // indirect @@ -158,8 +158,8 @@ require ( github.com/gitleaks/go-gitdiff v0.9.1 // indirect github.com/go-errors/errors v1.5.1 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect - github.com/go-git/go-billy/v5 v5.9.0 // indirect - github.com/go-git/go-git/v5 v5.19.2 // indirect + github.com/go-git/go-billy/v5 v5.9.1 // indirect + github.com/go-git/go-git/v5 v5.19.2 github.com/go-gorp/gorp/v3 v3.1.0 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect @@ -199,7 +199,7 @@ require ( github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/kastenhq/goversion v0.0.0-20230811215019-93b2f8823953 // indirect - github.com/kevinburke/ssh_config v1.2.0 // indirect + github.com/kevinburke/ssh_config v1.6.0 // indirect github.com/klauspost/compress v1.18.6 // indirect github.com/klauspost/pgzip v1.2.6 // indirect github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect @@ -258,7 +258,7 @@ require ( github.com/shirou/gopsutil v3.21.11+incompatible // indirect github.com/shopspring/decimal v1.4.0 // indirect github.com/sirupsen/logrus v1.9.4 // indirect - github.com/skeema/knownhosts v1.3.1 // indirect + github.com/skeema/knownhosts v1.3.2 // indirect github.com/slack-go/slack v0.23.1 // indirect github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect github.com/spdx/gordf v0.0.0-20250128162952-000978ccd6fb // indirect diff --git a/go.sum b/go.sum index 88edeeb9..46539859 100644 --- a/go.sum +++ b/go.sum @@ -45,8 +45,8 @@ cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0Zeo cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= -cyphar.com/go-pathrs v0.2.4 h1:iD/mge36swa1UFKdINkr1Frkpp6wZsy3YYEildj9cLY= -cyphar.com/go-pathrs v0.2.4/go.mod h1:y8f1EMG7r+hCuFf/rXsKqMJrJAUoADZGNh5/vZPKcGc= +cyphar.com/go-pathrs v0.2.5 h1:SnX9FBvnoyn3lUs1dkMgZ52bAETpirNu3FTRh5HlRik= +cyphar.com/go-pathrs v0.2.5/go.mod h1:y8f1EMG7r+hCuFf/rXsKqMJrJAUoADZGNh5/vZPKcGc= dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8= dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= @@ -110,8 +110,8 @@ github.com/Microsoft/hcsshim v0.15.0-rc.3/go.mod h1:VhDiwXgb8cEJxO9H57YL4NNIYqvZ github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/OneOfOne/xxhash v1.2.8 h1:31czK/TI9sNkxIKfaUfGlU47BAxQ0ztGgd9vPyqimf8= github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= -github.com/ProtonMail/go-crypto v1.4.0 h1:Zq/pbM3F5DFgJiMouxEdSVY44MVoQNEKp5d5QxIQceQ= -github.com/ProtonMail/go-crypto v1.4.0/go.mod h1:e1OaTyu5SYVrO9gKOEhTc+5UcXtTUa+P3uLudwcgPqo= +github.com/ProtonMail/go-crypto v1.4.1 h1:9RfcZHqEQUvP8RzecWEUafnZVtEvrBVL9BiF67IQOfM= +github.com/ProtonMail/go-crypto v1.4.1/go.mod h1:e1OaTyu5SYVrO9gKOEhTc+5UcXtTUa+P3uLudwcgPqo= github.com/acobaugh/osrelease v0.1.0 h1:Yb59HQDGGNhCj4suHaFQQfBps5wyoKLSSX/J/+UifRE= github.com/acobaugh/osrelease v0.1.0/go.mod h1:4bFEs0MtgHNHBrmHCt67gNisnabCRAlzdVasCEGHTWY= github.com/adrg/xdg v0.5.3 h1:xRnxJXne7+oWDatRhR1JLnvuccuIeCoBu2rtuLqQB78= @@ -241,8 +241,8 @@ github.com/clipperhouse/displaywidth v0.10.0 h1:GhBG8WuerxjFQQYeuZAeVTuyxuX+Urai github.com/clipperhouse/displaywidth v0.10.0/go.mod h1:XqJajYsaiEwkxOj4bowCTMcT1SgvHo9flfF3jQasdbs= github.com/clipperhouse/uax29/v2 v2.6.0 h1:z0cDbUV+aPASdFb2/ndFnS9ts/WNXgTNNGFoKXuhpos= github.com/clipperhouse/uax29/v2 v2.6.0/go.mod h1:Wn1g7MK6OoeDT0vL+Q0SQLDz/KpfsVRgg6W7ihQeh4g= -github.com/cloudflare/circl v1.6.3 h1:9GPOhQGF9MCYUeXyMYlqTR6a5gTrgR/fBLXvUgtVcg8= -github.com/cloudflare/circl v1.6.3/go.mod h1:2eXP6Qfat4O/Yhh8BznvKnJ+uzEoTQ6jVKJRn81BiS4= +github.com/cloudflare/circl v1.6.5 h1:O64F26HEqNhznd/hrC5KZXVKYuKM2rx4deZDTc4ihQA= +github.com/cloudflare/circl v1.6.5/go.mod h1:h5LNyxAc5nTue9DS5jT+48en2PSDYt3zdGnz5OstK6c= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= @@ -284,8 +284,8 @@ github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46t github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= -github.com/cyphar/filepath-securejoin v0.6.1 h1:5CeZ1jPXEiYt3+Z6zqprSAgSWiggmpVyciv8syjIpVE= -github.com/cyphar/filepath-securejoin v0.6.1/go.mod h1:A8hd4EnAeyujCJRrICiOWqjS1AX0a9kM5XL+NwKoYSc= +github.com/cyphar/filepath-securejoin v0.7.0 h1:s0Y3ITPy6sQn5xt54DuYvTF8hu134ooYLUb58DX/HjE= +github.com/cyphar/filepath-securejoin v0.7.0/go.mod h1:ymLGms/u3BYaviIiuKFnUx8EkQEZeK6cInNoAPJA3o4= github.com/danieljoos/wincred v1.2.3 h1:v7dZC2x32Ut3nEfRH+vhoZGvN72+dQ/snVXo/vMFLdQ= github.com/danieljoos/wincred v1.2.3/go.mod h1:6qqX0WNrS4RzPZ1tnroDzq9kY3fu1KwE7MRLQK4X0bs= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -391,8 +391,8 @@ github.com/go-errors/errors v1.5.1 h1:ZwEMSLRCapFLflTpT7NKaAc7ukJ8ZPEjzlxt8rPN8b github.com/go-errors/errors v1.5.1/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI= github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic= -github.com/go-git/go-billy/v5 v5.9.0 h1:jItGXszUDRtR/AlferWPTMN4j38BQ88XnXKbilmmBPA= -github.com/go-git/go-billy/v5 v5.9.0/go.mod h1:jCnQMLj9eUgGU7+ludSTYoZL/GGmii14RxKFj7ROgHw= +github.com/go-git/go-billy/v5 v5.9.1 h1:8U73XiOTfINdItHVa6z4Gv7ToObcZ6grkqQbLryLCdA= +github.com/go-git/go-billy/v5 v5.9.1/go.mod h1:ExsU+jcGwXTBOnyilvAnEM1wug1IxHr4yP2ZXsNRtV0= github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMje31YglSBqCdIqdhKBW8lokaMrL3uTkpGYlE2OOT4= github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399/go.mod h1:1OCfN199q1Jm3HZlxleg+Dw/mwps2Wbk9frAWm+4FII= github.com/go-git/go-git/v5 v5.19.2 h1:wkfn7vOlUBu8ivAWKBWisTiwJK4jYHzTF8Ndv1LyGqY= @@ -655,16 +655,16 @@ github.com/jsumners/go-getport v1.0.0/go.mod h1:KpeJgwNSkpuXuoGhJ2Hgl5QJqWbLG1m0 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kastenhq/goversion v0.0.0-20230811215019-93b2f8823953 h1:WdAeg/imY2JFPc/9CST4bZ80nNJbiBFCAdSZCSgrS5Y= github.com/kastenhq/goversion v0.0.0-20230811215019-93b2f8823953/go.mod h1:6o+UrvuZWc4UTyBhQf0LGjW9Ld7qJxLz/OqvSOWWlEc= -github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= -github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= +github.com/kevinburke/ssh_config v1.6.0 h1:J1FBfmuVosPHf5GRdltRLhPJtJpTlMdKTBjRgTaQBFY= +github.com/kevinburke/ssh_config v1.6.0/go.mod h1:q2RIzfka+BXARoNexmF9gkxEX7DmvbW9P4hIVx2Kg4M= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao= github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= -github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= -github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV913Dw= +github.com/klauspost/cpuid/v2 v2.4.0/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU= github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU= github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/knqyf263/go-rpmdb v0.1.1 h1:oh68mTCvp1XzxdU7EfafcWzzfstUZAEa3MW0IJye584= @@ -928,8 +928,8 @@ github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6Mwd github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w= github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g= -github.com/skeema/knownhosts v1.3.1 h1:X2osQ+RAjK76shCbvhHHHVl3ZlgDm8apHEHFqRjnBY8= -github.com/skeema/knownhosts v1.3.1/go.mod h1:r7KTdC8l4uxWRyK2TpQZ/1o5HaSzh06ePQNxPwTcfiY= +github.com/skeema/knownhosts v1.3.2 h1:EDL9mgf4NzwMXCTfaxSD/o/a5fxDw/xL9nkU28JjdBg= +github.com/skeema/knownhosts v1.3.2/go.mod h1:bEg3iQAuw+jyiw+484wwFJoKSLwcfd7fqRy+N0QTiow= github.com/slack-go/slack v0.23.1 h1:ZS5B96wxxYQRwvJ3/vJFtqtUZi3tXhsZCyT44Nv7M80= github.com/slack-go/slack v0.23.1/go.mod h1:H0yR/YBuRJ39RkE+JpV/d/oEsbanzTRowR82bCN0cEs= github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 h1:+jumHNA0Wrelhe64i8F6HNlS8pkoyMv5sreGx2Ry5Rw= diff --git a/internal/commands/gitmetadata.go b/internal/commands/gitmetadata.go new file mode 100644 index 00000000..e958778d --- /dev/null +++ b/internal/commands/gitmetadata.go @@ -0,0 +1,569 @@ +package commands + +import ( + "bytes" + "encoding/csv" + "encoding/json" + "fmt" + "net/http" + "os" + "os/exec" + "path/filepath" + "sort" + "strings" + "time" + + gogit "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/object" + "github.com/pkg/errors" + + "github.com/checkmarx/ast-cli/internal/logger" +) + +const ( + // CheckmarxFolderName is the folder where CLI-generated metadata files are stored. + CheckmarxFolderName = ".checkmarx" + // ContributorsFileName is the filename for the CSV with contributor information. + ContributorsFileName = "contributors.csv" + // MetadataFileName is the filename for the JSON metadata file. + MetadataFileName = "metadata.json" + + commitHistoryWindow = 90 * 24 * time.Hour + defaultRemoteName = "origin" + contributorsSizeLimit = 1 * 1024 * 1024 // repostore ignores contributors.csv above this size + generatedFilePerm = 0o644 + generatedDirPerm = 0o755 + csvFieldCount = 4 + urlSchemeParts = 2 // parts when splitting by "://" + pathParts = 2 // parts when splitting by "/" + sshSplitParts = 2 // parts when splitting SSH URL by "@" or ":" +) + +// contributorsMetadata mirrors repostore metadata structure; omits branchName per tech design. +type contributorsMetadata struct { + RepositoryURL string `json:"repositoryUrl"` + LastCommitHash string `json:"lastCommitHash"` + LastCommitDate string `json:"lastCommitDate"` + CommitsCount int `json:"commitsCount"` +} + +// GenerateAndWrite creates contributors.csv (private only) and metadata.json under .checkmarx/; errors should be logged but not fail scan. +func GenerateAndWrite(repoPath string, isPrivateRepo bool) error { + repo, err := gogit.PlainOpen(repoPath) + if err != nil { + // Fallback to system git if go-git fails (e.g., unsupported git extensions like worktreeConfig). + if strings.Contains(err.Error(), "does not support extension") { + logger.PrintfIfVerbose("go-git cannot open repository (unsupported git extension), falling back to system git") + return generateViaSystemGit(repoPath, isPrivateRepo) + } + return errors.Wrap(err, "could not open local git repository") + } + + headCommit, err := resolveHeadCommit(repo) + if err != nil { + return errors.Wrap(err, "could not resolve HEAD commit") + } + + since := time.Now().Add(-commitHistoryWindow) + commits, err := commitsSince(repo, since) + if err != nil { + return errors.Wrap(err, "could not read commit history") + } + + // Generate CSV only for private repos + var csvData []byte + if isPrivateRepo { + csvData, err = buildContributorsCSV(commits) + if err != nil { + return errors.Wrap(err, "could not build contributors.csv") + } + if len(csvData) > contributorsSizeLimit { + logger.PrintfIfVerbose("contributors.csv is %d bytes, over the 1MB size repostore accepts", len(csvData)) + } + } + + // Always generate metadata.json (all repos) + metadataData, err := buildMetadataJSON(remoteURL(repo), headCommit, len(commits)) + if err != nil { + return errors.Wrap(err, "could not build metadata.json") + } + + return writeGeneratedFilesConditional(repoPath, csvData, metadataData, isPrivateRepo) +} + +// resolveHeadCommit resolves HEAD to its full commit object, not just the hash. +func resolveHeadCommit(repo *gogit.Repository) (*object.Commit, error) { + head, err := repo.Head() + if err != nil { + return nil, err + } + return repo.CommitObject(head.Hash()) +} + +// commitsSince returns commits reachable from HEAD no older than since, sorted newest-first for dedup. +func commitsSince(repo *gogit.Repository, since time.Time) ([]*object.Commit, error) { + sinceCopy := since + iter, err := repo.Log(&gogit.LogOptions{Since: &sinceCopy}) + if err != nil { + return nil, err + } + defer iter.Close() + + var commits []*object.Commit + err = iter.ForEach(func(c *object.Commit) error { + commits = append(commits, c) + return nil + }) + if err != nil { + return nil, err + } + + sort.Slice(commits, func(i, j int) bool { + return commits[i].Author.When.After(commits[j].Author.When) + }) + return commits, nil +} + +// dedupByEmail keeps most recent commit per unique email to keep contributors.csv small. +func dedupByEmail(commits []*object.Commit) []*object.Commit { + seen := make(map[string]bool, len(commits)) + deduped := make([]*object.Commit, 0, len(commits)) + for _, c := range commits { + email := strings.ToLower(strings.TrimSpace(c.Author.Email)) + if seen[email] { + continue + } + seen[email] = true + deduped = append(deduped, c) + } + return deduped +} + +// buildContributorsCSV writes one row per unique email: date, hash, email, username. No header row. +func buildContributorsCSV(commits []*object.Commit) ([]byte, error) { + var buf strings.Builder + writer := csv.NewWriter(&buf) + + for _, c := range dedupByEmail(commits) { + row := []string{ + c.Author.When.Format(time.RFC3339), + c.Hash.String(), + c.Author.Email, + c.Author.Name, + } + if err := writer.Write(row); err != nil { + return nil, err + } + } + + writer.Flush() + if err := writer.Error(); err != nil { + return nil, err + } + return []byte(buf.String()), nil +} + +func buildMetadataJSON(repositoryURL string, headCommit *object.Commit, commitsCount int) ([]byte, error) { + metadata := contributorsMetadata{ + RepositoryURL: repositoryURL, + LastCommitHash: headCommit.Hash.String(), + LastCommitDate: headCommit.Author.When.Format(time.RFC3339), + CommitsCount: commitsCount, + } + return json.MarshalIndent(metadata, "", " ") +} + +// remoteURL returns the "origin" remote's first URL, or "" if there is none (e.g. a bare local clone). +func remoteURL(repo *gogit.Repository) string { + remote, err := repo.Remote(defaultRemoteName) + if err != nil { + return "" + } + urls := remote.Config().URLs + if len(urls) == 0 { + return "" + } + return urls[0] +} + +// generateViaSystemGit extracts repo info via system git when go-git fails (e.g., unsupported extensions). +func generateViaSystemGit(repoPath string, isPrivateRepo bool) error { + remoteURL, err := gitCommand(repoPath, "config", "--get", "remote.origin.url") + if err != nil { + return errors.Wrap(err, "could not get remote URL via system git") + } + + headHash, err := gitCommand(repoPath, "rev-parse", "HEAD") + if err != nil { + return errors.Wrap(err, "could not get HEAD commit via system git") + } + + since := time.Now().Add(-commitHistoryWindow).Format("2006-01-02") + logOutput, err := gitCommand(repoPath, "log", "--since="+since, "--pretty=format:%aI%x1f%H%x1f%ae%x1f%an") + if err != nil { + return errors.Wrap(err, "could not get commit log via system git") + } + + commits := parseGitLogOutput(logOutput) + if len(commits) == 0 { + commits = []map[string]string{} // empty list for builds with no commits in 90 days + } + + // Extract the actual last commit date (newest commit is first in array) + lastCommitDateStr := "" + if len(commits) > 0 { + lastCommitDateStr = commits[0]["date"] + } + + // Generate CSV only for private repos + var csvData []byte + if isPrivateRepo { + csvData, err = buildContributorsCSV(convertToCoreCommits(commits)) + if err != nil { + return errors.Wrap(err, "could not build contributors.csv") + } + if len(csvData) > contributorsSizeLimit { + logger.PrintfIfVerbose("contributors.csv is %d bytes, over the 1MB size repostore accepts", len(csvData)) + } + } + + // Always generate metadata.json + metadataData, err := buildMetadataJSONFromSystem(remoteURL, headHash, len(commits), lastCommitDateStr) + if err != nil { + return errors.Wrap(err, "could not build metadata.json") + } + + return writeGeneratedFilesConditional(repoPath, csvData, metadataData, isPrivateRepo) +} + +// gitCommand runs a git command in the given repo directory and returns trimmed output. +func gitCommand(repoPath string, args ...string) (string, error) { + cmd := exec.Command("git", args...) + cmd.Dir = repoPath + var out bytes.Buffer + cmd.Stdout = &out + cmd.Stderr = &out + if err := cmd.Run(); err != nil { + return "", errors.Wrapf(err, "git command failed: %s", out.String()) + } + return strings.TrimSpace(out.String()), nil +} + +// parseGitLogOutput parses git log output (%ai%H%ae%an) and returns commits newest-first. +func parseGitLogOutput(logOutput string) []map[string]string { + if logOutput == "" { + return nil + } + + lines := strings.Split(logOutput, "\n") + commits := make([]map[string]string, 0, len(lines)) + + for _, line := range lines { + if line == "" { + continue + } + parts := strings.Split(line, "\x1f") + if len(parts) != csvFieldCount { + continue + } + commits = append(commits, map[string]string{ + "date": parts[0], + "hash": parts[1], + "email": parts[2], + "name": parts[3], + }) + } + + // Git log returns oldest-first; reverse to newest-first for dedup consistency. + for i, j := 0, len(commits)-1; i < j; i, j = i+1, j-1 { + commits[i], commits[j] = commits[j], commits[i] + } + + return commits +} + +// convertToCoreCommits converts system-git commit maps to go-git Commit objects for CSV building. +func convertToCoreCommits(commits []map[string]string) []*object.Commit { + result := make([]*object.Commit, 0, len(commits)) + for _, c := range commits { + // Parse ISO8601 commit date; fall back to zero time on error. + commitTime, _ := time.Parse(time.RFC3339, c["date"]) + // Parse the commit hash; fall back to zero hash on error. + hash := plumbing.NewHash(c["hash"]) + commit := &object.Commit{ + Hash: hash, + Author: object.Signature{ + Email: c["email"], + Name: c["name"], + When: commitTime, + }, + } + result = append(result, commit) + } + return result +} + +// buildMetadataJSONFromSystem builds metadata.json using data from system git. +func buildMetadataJSONFromSystem(remoteURL, headHash string, commitCount int, lastCommitDate string) ([]byte, error) { + metadata := contributorsMetadata{ + RepositoryURL: remoteURL, + LastCommitHash: headHash, + LastCommitDate: lastCommitDate, + CommitsCount: commitCount, + } + return json.Marshal(metadata) +} + +// writeGeneratedFilesConditional always writes metadata.json; only writes contributors.csv for private repos. +func writeGeneratedFilesConditional(repoPath string, csvData, metadataData []byte, isPrivateRepo bool) error { + checkmarxDir := filepath.Join(repoPath, CheckmarxFolderName) + if err := os.MkdirAll(checkmarxDir, generatedDirPerm); err != nil { + return errors.Wrap(err, "could not create .checkmarx directory") + } + + // ALWAYS write metadata.json (all repos) + if err := os.WriteFile(filepath.Join(checkmarxDir, MetadataFileName), metadataData, generatedFilePerm); err != nil { + return errors.Wrap(err, "could not write "+MetadataFileName) + } + + // ONLY write contributors.csv for private repos + if isPrivateRepo { + if err := os.WriteFile(filepath.Join(checkmarxDir, ContributorsFileName), csvData, generatedFilePerm); err != nil { + return errors.Wrap(err, "could not write "+ContributorsFileName) + } + } + + return nil +} + +// detectRepositoryPrivacy determines if repo is private/public; conservatively defaults to private for unknown repos. +func detectRepositoryPrivacy(repoPath string) bool { + // Method 1: Try go-git + repo, err := gogit.PlainOpen(repoPath) + if err == nil { + return isPrivateByURL(remoteURL(repo)) + } + + // Method 2: Try system git + remoteURL, err := gitCommand(repoPath, "config", "--get", "remote.origin.url") + if err == nil && remoteURL != "" { + return isPrivateByURL(remoteURL) + } + + // Method 3: Check if contributors.csv exists (was private) + checkmarxDir := filepath.Join(repoPath, CheckmarxFolderName) + csvPath := filepath.Join(checkmarxDir, ContributorsFileName) + if fileExists(csvPath) { + return true + } + + // Default: Conservative (treat as private) + return true +} + +// isPrivateByURL detects repository privacy via public APIs (GitHub/GitLab/Bitbucket/Azure); defaults to private on any error. +func isPrivateByURL(remoteURL string) bool { + if remoteURL == "" { + return true // Local-only repo → private + } + + urlLower := strings.ToLower(remoteURL) + + // Route to appropriate API handler based on platform + switch { + case strings.Contains(urlLower, "github.com"): + return isPrivateGitHub(remoteURL) + case strings.Contains(urlLower, "gitlab"): + return isPrivateGitLab(remoteURL) + case strings.Contains(urlLower, "bitbucket"): + return isPrivateBitbucket(remoteURL) + case strings.Contains(urlLower, "dev.azure.com") || strings.Contains(urlLower, "visualstudio.com"): + return isPrivateAzureDevOps(remoteURL) + default: + return true // Unknown platform → conservative: default to PRIVATE + } +} + +// isPrivateGitHub checks GitHub repo privacy via direct HTTP URL (HTTP 200 = Public, else = Private). +func isPrivateGitHub(repoURL string) bool { + owner, repo := extractGitHubOwnerRepo(repoURL) + if owner == "" || repo == "" { + return true + } + + directURL := fmt.Sprintf("https://github.com/%s/%s", owner, repo) + isPublic := isRepoPublic(directURL) + return !isPublic +} + +// isPrivateGitLab checks GitLab repo privacy via direct HTTP URL (HTTP 200 = Public, else = Private). +func isPrivateGitLab(repoURL string) bool { + groupPath, projectName, host := extractGitLabGroupProject(repoURL) + if groupPath == "" || projectName == "" { + return true + } + + if host == "" { + host = "gitlab.com" + } + + directURL := fmt.Sprintf("https://%s/%s/%s", host, groupPath, projectName) + isPublic := isRepoPublic(directURL) + return !isPublic +} + +// isPrivateBitbucket checks Bitbucket repo privacy via direct HTTP URL (HTTP 200 = Public, else = Private). +func isPrivateBitbucket(repoURL string) bool { + workspace, repo := extractBitbucketWorkspaceRepo(repoURL) + if workspace == "" || repo == "" { + return true + } + + directURL := fmt.Sprintf("https://bitbucket.org/%s/%s", workspace, repo) + isPublic := isRepoPublic(directURL) + return !isPublic +} + +// isPrivateAzureDevOps checks Azure DevOps repo privacy via public API (no auth required). +func isPrivateAzureDevOps(repoURL string) bool { + org, repo := extractAzureDevOpsOrgRepo(repoURL) + if org == "" || repo == "" { + return true + } + + directURL := fmt.Sprintf("https://dev.azure.com/%s/_git/%s", org, repo) + isPublic := isRepoPublic(directURL) + return !isPublic +} + +// isRepoPublic checks if repository is publicly accessible (HTTP 200 = public, else = private). +func isRepoPublic(repoURL string) bool { + client := &http.Client{Timeout: 5 * time.Second} + resp, err := client.Get(repoURL) + if err != nil { + logger.PrintIfVerbose(fmt.Sprintf("Repository accessibility check failed for %s: %v, treating as PRIVATE", repoURL, err)) + return false + } + defer func() { + _ = resp.Body.Close() + }() + + isPublic := resp.StatusCode == http.StatusOK + if !isPublic { + logger.PrintIfVerbose(fmt.Sprintf("Repository URL %s returned status %d, treating as PRIVATE", repoURL, resp.StatusCode)) + } + return isPublic +} + +// normalizeSSHURL converts SSH git URLs to HTTPS format for extraction. +// Handles formats like: +// - git@github.com:owner/repo.git → https://github.com/owner/repo.git +// - git@gitlab.com:group/project.git → https://gitlab.com/group/project.git +// - ssh://git@github.com/owner/repo.git → https://github.com/owner/repo.git +func normalizeSSHURL(repoURL string) string { + if !strings.Contains(repoURL, "://") && strings.Contains(repoURL, "@") && strings.Contains(repoURL, ":") { + // Handle git@host:path format + // git@github.com:owner/repo.git → https://github.com/owner/repo.git + parts := strings.SplitN(repoURL, "@", sshSplitParts) + if len(parts) == sshSplitParts { + hostAndPath := strings.SplitN(parts[1], ":", sshSplitParts) + if len(hostAndPath) == sshSplitParts { + return "https://" + hostAndPath[0] + "/" + hostAndPath[1] + } + } + } else if strings.HasPrefix(repoURL, "ssh://") { + // Handle ssh://git@host/path format → https://host/path + sshURL := strings.TrimPrefix(repoURL, "ssh://") + if strings.Contains(sshURL, "@") { + parts := strings.SplitN(sshURL, "@", sshSplitParts) + if len(parts) == sshSplitParts { + return "https://" + parts[1] + } + } + } + return repoURL +} + +// Extract owner/repo from GitHub URLs: https://github.com/owner/repo or owner/repo +func extractGitHubOwnerRepo(repoURL string) (owner, repo string) { + repoURL = normalizeSSHURL(repoURL) + url := strings.TrimSuffix(repoURL, ".git") + parts := strings.FieldsFunc(url, func(r rune) bool { return r == '/' }) + if len(parts) >= pathParts { + return parts[len(parts)-pathParts], parts[len(parts)-1] + } + return "", "" +} + +// Extract group/project from GitLab URLs: https://gitlab.com/group/project or group/project +func extractGitLabGroupProject(repoURL string) (group, project, host string) { + repoURL = normalizeSSHURL(repoURL) + url := strings.TrimSuffix(repoURL, ".git") + + // Extract host if present + if strings.Contains(url, "://") { + parts := strings.SplitN(url, "://", urlSchemeParts) + hostAndPath := strings.SplitN(parts[1], "/", pathParts) + if len(hostAndPath) == pathParts { + host = hostAndPath[0] + url = hostAndPath[1] + } + } + + parts := strings.FieldsFunc(url, func(r rune) bool { return r == '/' }) + if len(parts) >= pathParts { + group = parts[0] + project = parts[1] + return + } + return "", "", host +} + +// Extract workspace/repo from Bitbucket URLs: https://bitbucket.org/workspace/repo +func extractBitbucketWorkspaceRepo(repoURL string) (workspace, repo string) { + repoURL = normalizeSSHURL(repoURL) + url := strings.TrimSuffix(repoURL, ".git") + parts := strings.FieldsFunc(url, func(r rune) bool { return r == '/' }) + if len(parts) >= pathParts { + return parts[len(parts)-pathParts], parts[len(parts)-1] + } + return "", "" +} + +// Extract org/repo from Azure DevOps URLs: https://dev.azure.com/org/_git/repo or https://ssh.dev.azure.com/v3/org/project/repo +func extractAzureDevOpsOrgRepo(repoURL string) (org, repo string) { + repoURL = normalizeSSHURL(repoURL) + url := strings.TrimSuffix(repoURL, ".git") + if strings.Contains(url, "dev.azure.com") { + parts := strings.Split(url, "/") + for i, part := range parts { + if part == "dev.azure.com" && i+1 < len(parts) { + org := parts[i+1] + // Find _git segment (HTTPS format) + for j := i + 2; j < len(parts); j++ { + if parts[j] == "_git" && j+1 < len(parts) { + repo := parts[j+1] + return org, repo + } + } + } else if part == "ssh.dev.azure.com" && i+1 < len(parts) && parts[i+1] == "v3" { + // SSH format: ssh.dev.azure.com/v3/org/project/repo + if i+3 < len(parts) { + org := parts[i+2] + if i+4 < len(parts) { + repo := parts[i+4] + return org, repo + } + } + } + } + } + return "", "" +} + +// fileExists checks if a file exists at the given path. +func fileExists(path string) bool { + _, err := os.Stat(path) + return err == nil +} diff --git a/internal/commands/gitmetadata_test.go b/internal/commands/gitmetadata_test.go new file mode 100644 index 00000000..91372d19 --- /dev/null +++ b/internal/commands/gitmetadata_test.go @@ -0,0 +1,981 @@ +package commands + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "testing" + "time" + + gogit "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/config" + "github.com/go-git/go-git/v5/plumbing/object" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// testCommit describes one fixture commit. AllowEmptyCommits means fixtures +// don't need real file content. +type testCommit struct { + email string + name string + when time.Time +} + +func newGitMetadataTestRepo(t *testing.T, remoteURL string, commits []testCommit) string { + t.Helper() + repoPath := t.TempDir() + + repo, err := gogit.PlainInit(repoPath, false) + require.NoError(t, err) + + if remoteURL != "" { + _, err = repo.CreateRemote(&config.RemoteConfig{ + Name: "origin", + URLs: []string{remoteURL}, + }) + require.NoError(t, err) + } + + worktree, err := repo.Worktree() + require.NoError(t, err) + + // Commits must be created oldest-first so the last one ends up as HEAD. + for _, c := range commits { + sig := &object.Signature{Name: c.name, Email: c.email, When: c.when} + _, err := worktree.Commit("test commit", &gogit.CommitOptions{ + Author: sig, + AllowEmptyCommits: true, + }) + require.NoError(t, err) + } + + return repoPath +} + +func readGitMetadataFiles(t *testing.T, repoPath string) (csvContent string, metadata contributorsMetadata) { + t.Helper() + csvBytes, err := os.ReadFile(filepath.Join(repoPath, CheckmarxFolderName, ContributorsFileName)) + require.NoError(t, err) + + metadataBytes, err := os.ReadFile(filepath.Join(repoPath, CheckmarxFolderName, MetadataFileName)) + require.NoError(t, err) + require.NoError(t, json.Unmarshal(metadataBytes, &metadata)) + + return string(csvBytes), metadata +} + +func TestGenerateAndWrite_Success(t *testing.T) { + now := time.Now() + repoPath := newGitMetadataTestRepo(t, "https://example.com/org/repo.git", []testCommit{ + {email: "alice@example.com", name: "Alice", when: now.Add(-10 * 24 * time.Hour)}, + {email: "bob@example.com", name: "Bob", when: now.Add(-5 * 24 * time.Hour)}, + {email: "alice@example.com", name: "Alice", when: now.Add(-1 * time.Hour)}, + }) + + require.NoError(t, GenerateAndWrite(repoPath, true)) + + csvContent, metadata := readGitMetadataFiles(t, repoPath) + + lines := strings.Split(strings.TrimRight(csvContent, "\n"), "\n") + assert.Len(t, lines, 2, "expected one row per unique email, most recent commit only") + + assert.Equal(t, "https://example.com/org/repo.git", metadata.RepositoryURL) + assert.Equal(t, 3, metadata.CommitsCount, "commitsCount should count every commit in the window, before dedup") + assert.NotEmpty(t, metadata.LastCommitHash) + assert.NotEmpty(t, metadata.LastCommitDate) +} + +func TestGenerateAndWrite_MetadataJSONFieldNames(t *testing.T) { + repoPath := newGitMetadataTestRepo(t, "https://example.com/org/repo.git", []testCommit{ + {email: "alice@example.com", name: "Alice", when: time.Now()}, + }) + require.NoError(t, GenerateAndWrite(repoPath, true)) + + raw, err := os.ReadFile(filepath.Join(repoPath, CheckmarxFolderName, MetadataFileName)) + require.NoError(t, err) + + var asMap map[string]interface{} + require.NoError(t, json.Unmarshal(raw, &asMap)) + + assert.Contains(t, asMap, "repositoryUrl") + assert.Contains(t, asMap, "lastCommitHash") + assert.Contains(t, asMap, "lastCommitDate") + assert.Contains(t, asMap, "commitsCount") + assert.NotContains(t, asMap, "branchName", "branchName is intentionally omitted, see tech design open questions") + assert.NotContains(t, asMap, "commitHash", "field is named lastCommitHash, not commitHash") +} + +func TestGenerateAndWrite_NoHeaderRow(t *testing.T) { + repoPath := newGitMetadataTestRepo(t, "", []testCommit{ + {email: "alice@example.com", name: "Alice", when: time.Now()}, + }) + + require.NoError(t, GenerateAndWrite(repoPath, true)) + + csvContent, _ := readGitMetadataFiles(t, repoPath) + firstLine := strings.SplitN(csvContent, ",", 2)[0] + + // A header would read "date" or similar; a real row starts with an RFC3339 + // timestamp, which always begins with a 4-digit year. + _, err := time.Parse(time.RFC3339, firstLine) + assert.NoError(t, err, "first line should be a data row (RFC3339 date), not a header") +} + +func TestGenerateAndWrite_CSVColumnOrder(t *testing.T) { + now := time.Now() + repoPath := newGitMetadataTestRepo(t, "", []testCommit{ + {email: "alice@example.com", name: "Alice Example", when: now}, + }) + require.NoError(t, GenerateAndWrite(repoPath, true)) + + csvContent, metadata := readGitMetadataFiles(t, repoPath) + fields := strings.Split(strings.TrimRight(csvContent, "\n"), ",") + require.Len(t, fields, 4) + + assert.Equal(t, now.Format(time.RFC3339), fields[0], "field 1 must be the commit date") + assert.Equal(t, metadata.LastCommitHash, fields[1], "field 2 must be the commit hash") + assert.Equal(t, "alice@example.com", fields[2], "field 3 must be the email") + assert.Equal(t, "Alice Example", fields[3], "field 4 must be the username") +} + +func TestGenerateAndWrite_DedupKeepsMostRecentPerEmail(t *testing.T) { + now := time.Now() + olderTime := now.Add(-20 * 24 * time.Hour) + newerTime := now.Add(-1 * 24 * time.Hour) + + repoPath := newGitMetadataTestRepo(t, "", []testCommit{ + {email: "alice@example.com", name: "Alice Old", when: olderTime}, + {email: "alice@example.com", name: "Alice New", when: newerTime}, + }) + + require.NoError(t, GenerateAndWrite(repoPath, true)) + + csvContent, _ := readGitMetadataFiles(t, repoPath) + lines := strings.Split(strings.TrimRight(csvContent, "\n"), "\n") + require.Len(t, lines, 1) + assert.Contains(t, lines[0], newerTime.Format(time.RFC3339)) + assert.Contains(t, lines[0], "Alice New") + assert.NotContains(t, lines[0], "Alice Old") +} + +func TestGenerateAndWrite_DedupIsCaseInsensitiveOnEmail(t *testing.T) { + now := time.Now() + repoPath := newGitMetadataTestRepo(t, "", []testCommit{ + {email: "Alice@Example.com", name: "Alice Mixed Case", when: now.Add(-2 * 24 * time.Hour)}, + {email: "alice@example.com", name: "Alice Lower Case", when: now.Add(-1 * time.Hour)}, + }) + + require.NoError(t, GenerateAndWrite(repoPath, true)) + + csvContent, metadata := readGitMetadataFiles(t, repoPath) + lines := strings.Split(strings.TrimRight(csvContent, "\n"), "\n") + assert.Len(t, lines, 1, "differently-cased emails for the same person should dedup to one row") + assert.Equal(t, 2, metadata.CommitsCount, "commitsCount still counts both raw commits") +} + +func TestGenerateAndWrite_ManyUniqueEmailsAllKept(t *testing.T) { + now := time.Now() + var commits []testCommit + for i := 0; i < 5; i++ { + commits = append(commits, testCommit{ + email: strings.Repeat("u", 1) + string(rune('a'+i)) + "@example.com", + name: "User", + when: now.Add(-time.Duration(i) * time.Hour), + }) + } + repoPath := newGitMetadataTestRepo(t, "", commits) + require.NoError(t, GenerateAndWrite(repoPath, true)) + + csvContent, metadata := readGitMetadataFiles(t, repoPath) + lines := strings.Split(strings.TrimRight(csvContent, "\n"), "\n") + assert.Len(t, lines, 5, "each unique email should get its own row") + assert.Equal(t, 5, metadata.CommitsCount) +} + +func TestGenerateAndWrite_90DayBoundary(t *testing.T) { + now := time.Now() + repoPath := newGitMetadataTestRepo(t, "", []testCommit{ + {email: "old@example.com", name: "TooOld", when: now.Add(-91 * 24 * time.Hour)}, + {email: "recent@example.com", name: "Recent", when: now.Add(-89 * 24 * time.Hour)}, + }) + + require.NoError(t, GenerateAndWrite(repoPath, true)) + + csvContent, metadata := readGitMetadataFiles(t, repoPath) + assert.Contains(t, csvContent, "recent@example.com") + assert.NotContains(t, csvContent, "old@example.com") + assert.Equal(t, 1, metadata.CommitsCount) +} + +func TestGenerateAndWrite_AllCommitsOutsideWindow(t *testing.T) { + now := time.Now() + repoPath := newGitMetadataTestRepo(t, "", []testCommit{ + {email: "old@example.com", name: "TooOld", when: now.Add(-200 * 24 * time.Hour)}, + }) + + require.NoError(t, GenerateAndWrite(repoPath, true)) + + csvContent, metadata := readGitMetadataFiles(t, repoPath) + assert.Empty(t, csvContent, "no commits in the window means an empty (but present) CSV") + assert.Equal(t, 0, metadata.CommitsCount) + assert.NotEmpty(t, metadata.LastCommitHash, "HEAD info is unconditional, independent of the 90-day window") +} + +func TestGenerateAndWrite_NoRemote(t *testing.T) { + repoPath := newGitMetadataTestRepo(t, "", []testCommit{ + {email: "alice@example.com", name: "Alice", when: time.Now()}, + }) + + require.NoError(t, GenerateAndWrite(repoPath, true)) + + _, metadata := readGitMetadataFiles(t, repoPath) + assert.Empty(t, metadata.RepositoryURL) +} + +func TestGenerateAndWrite_SSHRemoteURL(t *testing.T) { + repoPath := newGitMetadataTestRepo(t, "git@github.com:org/repo.git", []testCommit{ + {email: "alice@example.com", name: "Alice", when: time.Now()}, + }) + + require.NoError(t, GenerateAndWrite(repoPath, true)) + + _, metadata := readGitMetadataFiles(t, repoPath) + assert.Equal(t, "git@github.com:org/repo.git", metadata.RepositoryURL) +} + +func TestGenerateAndWrite_ReplacesStaleFiles(t *testing.T) { + repoPath := newGitMetadataTestRepo(t, "", []testCommit{ + {email: "alice@example.com", name: "Alice", when: time.Now()}, + }) + + require.NoError(t, GenerateAndWrite(repoPath, true)) + _, firstMetadata := readGitMetadataFiles(t, repoPath) + + repo, err := gogit.PlainOpen(repoPath) + require.NoError(t, err) + worktree, err := repo.Worktree() + require.NoError(t, err) + _, err = worktree.Commit("second commit", &gogit.CommitOptions{ + Author: &object.Signature{Name: "Bob", Email: "bob@example.com", When: time.Now()}, + AllowEmptyCommits: true, + }) + require.NoError(t, err) + + require.NoError(t, GenerateAndWrite(repoPath, true)) + csvContent, secondMetadata := readGitMetadataFiles(t, repoPath) + + assert.NotEqual(t, firstMetadata.LastCommitHash, secondMetadata.LastCommitHash, "second run should overwrite metadata.json with fresh data") + assert.Contains(t, csvContent, "bob@example.com") + assert.Contains(t, csvContent, "alice@example.com") +} + +func TestGenerateAndWrite_RunTwiceIdenticalStateProducesIdenticalOutput(t *testing.T) { + repoPath := newGitMetadataTestRepo(t, "https://example.com/repo.git", []testCommit{ + {email: "alice@example.com", name: "Alice", when: time.Now()}, + }) + + require.NoError(t, GenerateAndWrite(repoPath, true)) + firstCSV, firstMetadata := readGitMetadataFiles(t, repoPath) + + require.NoError(t, GenerateAndWrite(repoPath, true)) + secondCSV, secondMetadata := readGitMetadataFiles(t, repoPath) + + assert.Equal(t, firstCSV, secondCSV) + assert.Equal(t, firstMetadata, secondMetadata) +} + +func TestGenerateAndWrite_NotAGitRepository(t *testing.T) { + dir := t.TempDir() + err := GenerateAndWrite(dir, true) + assert.Error(t, err) +} + +func TestGenerateAndWrite_NoCommits(t *testing.T) { + dir := t.TempDir() + _, err := gogit.PlainInit(dir, false) + require.NoError(t, err) + + err = GenerateAndWrite(dir, true) + assert.Error(t, err, "an empty repo has no HEAD to resolve") +} + +func TestGenerateAndWrite_NonExistentPath(t *testing.T) { + err := GenerateAndWrite(filepath.Join(t.TempDir(), "does-not-exist"), true) + assert.Error(t, err) +} + +// TestBuildContributorsCSV_ExceedsSizeLimitStillSucceeds exercises buildContributorsCSV +// directly with synthetic in-memory commits (no real git repo), since creating enough +// real commits to exceed the 1MB warning threshold would make the test very slow. +func TestBuildContributorsCSV_ExceedsSizeLimitStillSucceeds(t *testing.T) { + now := time.Now() + commits := make([]*object.Commit, 0, 20000) + for i := 0; i < 20000; i++ { + commits = append(commits, &object.Commit{ + Author: object.Signature{ + Name: "User", + Email: "user" + strconv.Itoa(i) + "@example.com", + When: now.Add(-time.Duration(i) * time.Second), + }, + }) + } + + csvData, err := buildContributorsCSV(commits) + require.NoError(t, err, "exceeding the size warning threshold must not fail generation") + assert.Greater(t, len(csvData), contributorsSizeLimit) + + lines := strings.Split(strings.TrimRight(string(csvData), "\n"), "\n") + assert.Len(t, lines, len(commits), "all unique emails should be kept as rows") +} + +func TestBuildContributorsCSV_EmptyInput(t *testing.T) { + csvData, err := buildContributorsCSV(nil) + require.NoError(t, err) + assert.Empty(t, csvData) +} + +func TestDedupByEmail_EmptyInput(t *testing.T) { + assert.Empty(t, dedupByEmail(nil)) +} + +func TestDedupByEmail_PreservesFirstOccurrenceOrder(t *testing.T) { + now := time.Now() + commits := []*object.Commit{ + {Author: object.Signature{Email: "a@example.com", When: now}}, + {Author: object.Signature{Email: "b@example.com", When: now}}, + {Author: object.Signature{Email: "a@example.com", When: now}}, // duplicate, later in slice + {Author: object.Signature{Email: "c@example.com", When: now}}, + } + + deduped := dedupByEmail(commits) + require.Len(t, deduped, 3) + assert.Equal(t, "a@example.com", deduped[0].Author.Email) + assert.Equal(t, "b@example.com", deduped[1].Author.Email) + assert.Equal(t, "c@example.com", deduped[2].Author.Email) +} + +func TestBuildMetadataJSON_Structure(t *testing.T) { + headCommit := &object.Commit{ + Author: object.Signature{When: time.Date(2025, 9, 30, 10, 35, 5, 0, time.UTC)}, + } + + data, err := buildMetadataJSON("https://example.com/repo.git", headCommit, 42) + require.NoError(t, err) + + var asMap map[string]interface{} + require.NoError(t, json.Unmarshal(data, &asMap)) + assert.Equal(t, "https://example.com/repo.git", asMap["repositoryUrl"]) + assert.Equal(t, headCommit.Hash.String(), asMap["lastCommitHash"]) + assert.Equal(t, "2025-09-30T10:35:05Z", asMap["lastCommitDate"]) + assert.InDelta(t, 42, asMap["commitsCount"], 0) +} + +func TestCommitsSince_SortsNewestFirst(t *testing.T) { + now := time.Now() + // Committed out of chronological order to verify commitsSince re-sorts them. + repoPath := newGitMetadataTestRepo(t, "", []testCommit{ + {email: "a@example.com", name: "A", when: now.Add(-5 * 24 * time.Hour)}, + {email: "b@example.com", name: "B", when: now.Add(-1 * 24 * time.Hour)}, + {email: "c@example.com", name: "C", when: now.Add(-10 * 24 * time.Hour)}, + }) + + repo, err := gogit.PlainOpen(repoPath) + require.NoError(t, err) + + commits, err := commitsSince(repo, now.Add(-30*24*time.Hour)) + require.NoError(t, err) + require.Len(t, commits, 3) + + assert.True(t, commits[0].Author.When.After(commits[1].Author.When)) + assert.True(t, commits[1].Author.When.After(commits[2].Author.When)) +} + +func TestRemoteURL_ReturnsFirstURL(t *testing.T) { + repoPath := newGitMetadataTestRepo(t, "https://example.com/first.git", nil) + repo, err := gogit.PlainOpen(repoPath) + require.NoError(t, err) + + assert.Equal(t, "https://example.com/first.git", remoteURL(repo)) +} + +func TestRemoteURL_NoRemoteConfigured(t *testing.T) { + repoPath := newGitMetadataTestRepo(t, "", nil) + repo, err := gogit.PlainOpen(repoPath) + require.NoError(t, err) + + assert.Empty(t, remoteURL(repo)) +} + +func TestParseGitLogOutput_BasicParsing(t *testing.T) { + logOutput := "2026-08-19T18:24:26+05:30\x1fabc123def456789abc123def456789abc12345\x1fuser@example.com\x1fJohn Doe\n" + + "2026-08-13T12:58:25+03:00\x1fdef456789abc123def456789abc123def45678\x1fjane@example.com\x1fJane Smith" + + commits := parseGitLogOutput(logOutput) + require.Len(t, commits, 2) + + assert.Equal(t, "2026-08-13T12:58:25+03:00", commits[0]["date"]) + assert.Equal(t, "def456789abc123def456789abc123def45678", commits[0]["hash"]) + assert.Equal(t, "jane@example.com", commits[0]["email"]) + assert.Equal(t, "Jane Smith", commits[0]["name"]) + + assert.Equal(t, "2026-08-19T18:24:26+05:30", commits[1]["date"]) + assert.Equal(t, "abc123def456789abc123def456789abc12345", commits[1]["hash"]) + assert.Equal(t, "user@example.com", commits[1]["email"]) + assert.Equal(t, "John Doe", commits[1]["name"]) +} + +func TestParseGitLogOutput_RevertsToNewestFirst(t *testing.T) { + logOutput := "2026-07-01T10:00:00+00:00\x1f1111111111111111111111111111111111111111\x1fold@example.com\x1fOld User\n" + + "2026-07-15T15:00:00+00:00\x1f2222222222222222222222222222222222222222\x1fmid@example.com\x1fMid User\n" + + "2026-08-19T20:00:00+00:00\x1f3333333333333333333333333333333333333333\x1fnew@example.com\x1fNew User" + + commits := parseGitLogOutput(logOutput) + require.Len(t, commits, 3) + + assert.Equal(t, "2026-08-19T20:00:00+00:00", commits[0]["date"]) + assert.Equal(t, "2026-07-15T15:00:00+00:00", commits[1]["date"]) + assert.Equal(t, "2026-07-01T10:00:00+00:00", commits[2]["date"]) +} + +func TestParseGitLogOutput_EmptyInput(t *testing.T) { + commits := parseGitLogOutput("") + assert.Nil(t, commits) +} + +func TestParseGitLogOutput_SkipsMalformedLines(t *testing.T) { + logOutput := "2026-08-19T18:24:26+05:30\x1fabc123def456789abc123def456789abc12345\x1fuser@example.com\x1fJohn Doe\n" + + "malformed line\n" + + "2026-08-13T12:58:25+03:00\x1fdef456789abc123def456789abc123def45678\x1fjane@example.com\x1fJane Smith\n" + + "\n" + + "another bad line" + + commits := parseGitLogOutput(logOutput) + require.Len(t, commits, 2) + assert.Equal(t, "John Doe", commits[1]["name"]) + assert.Equal(t, "Jane Smith", commits[0]["name"]) +} + +func TestConvertToCoreCommits_BasicConversion(t *testing.T) { + now := time.Now() + commitMaps := []map[string]string{ + { + "date": now.Format(time.RFC3339), + "hash": "abc123def456789abcdef456789abcdef1234567", + "email": "user@example.com", + "name": "Test User", + }, + } + + commits := convertToCoreCommits(commitMaps) + require.Len(t, commits, 1) + + c := commits[0] + assert.Equal(t, "abc123def456789abcdef456789abcdef1234567", c.Hash.String()) + assert.Equal(t, "user@example.com", c.Author.Email) + assert.Equal(t, "Test User", c.Author.Name) + assert.WithinDuration(t, now, c.Author.When, 1*time.Second) +} + +func TestConvertToCoreCommits_PreservesOrder(t *testing.T) { + commitMaps := []map[string]string{ + { + "date": "2026-08-19T18:24:26+05:30", + "hash": "1111111111111111111111111111111111111111", + "email": "first@example.com", + "name": "First", + }, + { + "date": "2026-08-13T12:58:25+03:00", + "hash": "2222222222222222222222222222222222222222", + "email": "second@example.com", + "name": "Second", + }, + } + + commits := convertToCoreCommits(commitMaps) + require.Len(t, commits, 2) + + assert.Equal(t, "1111111111111111111111111111111111111111", commits[0].Hash.String()) + assert.Equal(t, "2222222222222222222222222222222222222222", commits[1].Hash.String()) +} + +func TestConvertToCoreCommits_RFC3339DateParsing(t *testing.T) { + commitMaps := []map[string]string{ + { + "date": "2026-08-19T18:24:26+05:30", + "hash": "abc123", + "email": "user@example.com", + "name": "User", + }, + } + + commits := convertToCoreCommits(commitMaps) + require.Len(t, commits, 1) + + // Verify the date was parsed correctly (RFC3339 with timezone) + assert.Equal(t, 2026, commits[0].Author.When.Year()) + assert.Equal(t, time.August, commits[0].Author.When.Month()) + assert.Equal(t, 19, commits[0].Author.When.Day()) +} + +func TestBuildMetadataJSONFromSystem_Structure(t *testing.T) { + data, err := buildMetadataJSONFromSystem("https://github.com/example/repo.git", "abc123def456", 42, "2026-08-19T18:24:26Z") + require.NoError(t, err) + + var asMap map[string]interface{} + require.NoError(t, json.Unmarshal(data, &asMap)) + + assert.Equal(t, "https://github.com/example/repo.git", asMap["repositoryUrl"]) + assert.Equal(t, "abc123def456", asMap["lastCommitHash"]) + assert.InDelta(t, 42, asMap["commitsCount"], 0) + assert.NotEmpty(t, asMap["lastCommitDate"]) + + // Verify lastCommitDate is valid RFC3339 + _, err = time.Parse(time.RFC3339, asMap["lastCommitDate"].(string)) + require.NoError(t, err) +} + +func TestBuildMetadataJSONFromSystem_EmptyRepository(t *testing.T) { + data, err := buildMetadataJSONFromSystem("", "", 0, "") + require.NoError(t, err) + + var asMap map[string]interface{} + require.NoError(t, json.Unmarshal(data, &asMap)) + + assert.Equal(t, "", asMap["repositoryUrl"]) + assert.Equal(t, "", asMap["lastCommitHash"]) + assert.InDelta(t, 0, asMap["commitsCount"], 0) +} + +func TestGitCommand_Success(t *testing.T) { + // Create a real git repo for this test + repoPath := t.TempDir() + + // Initialize a git repository using system git + cmd := exec.Command("git", "init") + cmd.Dir = repoPath + require.NoError(t, cmd.Run()) + + // Configure git user for this test repo (required in CI) + cmd = exec.Command("git", "config", "user.name", "Test User") + cmd.Dir = repoPath + require.NoError(t, cmd.Run()) + + // Test git config command + output, err := gitCommand(repoPath, "config", "--get", "user.name") + assert.NoError(t, err) + assert.Equal(t, "Test User", output) +} + +func TestGitCommand_InvalidRepo(t *testing.T) { + invalidPath := t.TempDir() + // This directory is not a git repo + + _, err := gitCommand(invalidPath, "log", "--oneline") + assert.Error(t, err) + assert.Contains(t, err.Error(), "git command failed") +} + +// Tests for URL extraction functions + +func TestExtractGitHubOwnerRepo(t *testing.T) { + tests := []struct { + name string + url string + wantOwner string + wantRepo string + }{ + { + name: "HTTPS GitHub URL", + url: "https://github.com/checkmarx/ast-cli", + wantOwner: "checkmarx", + wantRepo: "ast-cli", + }, + { + name: "HTTPS GitHub URL with .git suffix", + url: "https://github.com/checkmarx/ast-cli.git", + wantOwner: "checkmarx", + wantRepo: "ast-cli", + }, + { + name: "Short format GitHub URL", + url: "checkmarx/ast-cli", + wantOwner: "checkmarx", + wantRepo: "ast-cli", + }, + { + name: "Invalid format - too few parts", + url: "invalid", + wantOwner: "", + wantRepo: "", + }, + { + name: "Empty URL", + url: "", + wantOwner: "", + wantRepo: "", + }, + { + name: "SSH GitHub URL", + url: "git@github.com:checkmarx/ast-cli.git", + wantOwner: "checkmarx", + wantRepo: "ast-cli", + }, + { + name: "SSH GitHub URL without .git", + url: "git@github.com:checkmarx/ast-cli", + wantOwner: "checkmarx", + wantRepo: "ast-cli", + }, + { + name: "SSH protocol GitHub URL", + url: "ssh://git@github.com/checkmarx/ast-cli.git", + wantOwner: "checkmarx", + wantRepo: "ast-cli", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + owner, repo := extractGitHubOwnerRepo(tt.url) + assert.Equal(t, tt.wantOwner, owner, "owner mismatch") + assert.Equal(t, tt.wantRepo, repo, "repo mismatch") + }) + } +} + +func TestExtractGitLabGroupProject(t *testing.T) { + tests := []struct { + name string + url string + wantGroup string + wantRepo string + wantHost string + }{ + { + name: "HTTPS GitLab URL", + url: "https://gitlab.com/checkmarx/ast-cli", + wantGroup: "checkmarx", + wantRepo: "ast-cli", + wantHost: "gitlab.com", + }, + { + name: "HTTPS GitLab URL with .git suffix", + url: "https://gitlab.com/checkmarx/ast-cli.git", + wantGroup: "checkmarx", + wantRepo: "ast-cli", + wantHost: "gitlab.com", + }, + { + name: "Self-hosted GitLab", + url: "https://gitlab.internal.com/checkmarx/ast-cli", + wantGroup: "checkmarx", + wantRepo: "ast-cli", + wantHost: "gitlab.internal.com", + }, + { + name: "SSH GitLab URL", + url: "git@gitlab.com:checkmarx/ast-cli.git", + wantGroup: "checkmarx", + wantRepo: "ast-cli", + wantHost: "gitlab.com", + }, + { + name: "SSH self-hosted GitLab URL", + url: "git@gitlab.internal.com:checkmarx/ast-cli.git", + wantGroup: "checkmarx", + wantRepo: "ast-cli", + wantHost: "gitlab.internal.com", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + group, project, host := extractGitLabGroupProject(tt.url) + assert.Equal(t, tt.wantGroup, group, "group mismatch") + assert.Equal(t, tt.wantRepo, project, "project mismatch") + assert.Equal(t, tt.wantHost, host, "host mismatch") + }) + } +} + +func TestExtractBitbucketWorkspaceRepo(t *testing.T) { + tests := []struct { + name string + url string + wantWorkspace string + wantRepo string + }{ + { + name: "HTTPS Bitbucket URL", + url: "https://bitbucket.org/checkmarx/ast-cli", + wantWorkspace: "checkmarx", + wantRepo: "ast-cli", + }, + { + name: "HTTPS Bitbucket URL with .git suffix", + url: "https://bitbucket.org/checkmarx/ast-cli.git", + wantWorkspace: "checkmarx", + wantRepo: "ast-cli", + }, + { + name: "SSH Bitbucket URL", + url: "git@bitbucket.org:checkmarx/ast-cli.git", + wantWorkspace: "checkmarx", + wantRepo: "ast-cli", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + workspace, repo := extractBitbucketWorkspaceRepo(tt.url) + assert.Equal(t, tt.wantWorkspace, workspace, "workspace mismatch") + assert.Equal(t, tt.wantRepo, repo, "repo mismatch") + }) + } +} + +func TestExtractAzureDevOpsOrgRepo(t *testing.T) { + tests := []struct { + name string + url string + wantOrg string + wantRepo string + }{ + { + name: "HTTPS Azure DevOps URL with _git", + url: "https://dev.azure.com/checkmarx/project/_git/ast-cli", + wantOrg: "checkmarx", + wantRepo: "ast-cli", + }, + { + name: "HTTPS Azure DevOps URL with .git suffix", + url: "https://dev.azure.com/checkmarx/project/_git/ast-cli.git", + wantOrg: "checkmarx", + wantRepo: "ast-cli", + }, + { + name: "SSH Azure DevOps URL", + url: "git@ssh.dev.azure.com:v3/checkmarx/project/ast-cli.git", + wantOrg: "checkmarx", + wantRepo: "ast-cli", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + org, repo := extractAzureDevOpsOrgRepo(tt.url) + assert.Equal(t, tt.wantOrg, org, "org mismatch") + assert.Equal(t, tt.wantRepo, repo, "repo mismatch") + }) + } +} + +// Tests for privacy detection with mocked HTTP responses + +func TestFileExists(t *testing.T) { + tests := []struct { + name string + setup func(t *testing.T) string + wantTrue bool + }{ + { + name: "file exists", + setup: func(t *testing.T) string { + f, err := os.CreateTemp(t.TempDir(), "test") + require.NoError(t, err) + path := f.Name() + require.NoError(t, f.Close()) + return path + }, + wantTrue: true, + }, + { + name: "file does not exist", + setup: func(t *testing.T) string { + return "/nonexistent/path/that/does/not/exist" + }, + wantTrue: false, + }, + { + name: "directory exists", + setup: func(t *testing.T) string { + return t.TempDir() + }, + wantTrue: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + path := tt.setup(t) + exists := fileExists(path) + assert.Equal(t, tt.wantTrue, exists) + }) + } +} + +func TestPrivacyDetectionWithMockedHTTP(t *testing.T) { + t.Run("detectRepositoryPrivacy returns private for empty path", func(t *testing.T) { + tmpDir := t.TempDir() + // Empty directory with no .git or .checkmarx files + isPrivate := detectRepositoryPrivacy(tmpDir) + assert.True(t, isPrivate, "empty directory should default to private") + }) + + t.Run("detectRepositoryPrivacy returns private when checking nonexistent path", func(t *testing.T) { + isPrivate := detectRepositoryPrivacy("/nonexistent/path/that/does/not/exist") + assert.True(t, isPrivate, "nonexistent path should default to private") + }) + + t.Run("isPrivateByURL returns private for empty URL", func(t *testing.T) { + isPrivate := isPrivateByURL("") + assert.True(t, isPrivate, "empty URL should be private") + }) + + t.Run("isPrivateByURL routes unknown platforms to private", func(t *testing.T) { + isPrivate := isPrivateByURL("https://unknown-git-hosting.com/team/repo") + assert.True(t, isPrivate, "unknown platform should default to private") + }) +} + +func TestIsRepoPublicWithMockedServer(t *testing.T) { + t.Run("public repository returns true when HTTP 200", func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + + // Test with mocked server URL + isPublic := isRepoPublic(server.URL) + assert.True(t, isPublic, "HTTP 200 should indicate public") + }) + + t.Run("private repository returns false when HTTP 404", func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotFound) + })) + defer server.Close() + + isPublic := isRepoPublic(server.URL) + assert.False(t, isPublic, "HTTP 404 should indicate not public (private)") + }) + + t.Run("empty URL returns false (not public/private)", func(t *testing.T) { + isPublic := isRepoPublic("") + assert.False(t, isPublic, "empty URL should not be public") + }) + + t.Run("malformed URL returns false (not public/private)", func(t *testing.T) { + isPublic := isRepoPublic("://invalid") + assert.False(t, isPublic, "malformed URL should not be public") + }) +} + +func TestPlatformSpecificPrivacyDetection(t *testing.T) { + t.Run("isPrivateGitHub returns private when extraction fails", func(t *testing.T) { + // Invalid URL that won't extract properly - returns early without HTTP call + isPrivate := isPrivateGitHub("invalid") + assert.True(t, isPrivate, "invalid URL should be private") + }) + + t.Run("isPrivateGitHub returns private for empty owner", func(t *testing.T) { + // URL format that extracts to empty owner + isPrivate := isPrivateGitHub("") + assert.True(t, isPrivate, "empty URL should be private") + }) + + t.Run("isPrivateGitLab returns private when extraction fails", func(t *testing.T) { + // Invalid URL that won't extract properly + isPrivate := isPrivateGitLab("invalid") + assert.True(t, isPrivate, "invalid URL should be private") + }) + + t.Run("isPrivateGitLab returns private for empty group", func(t *testing.T) { + // URL format that extracts to empty group + isPrivate := isPrivateGitLab("") + assert.True(t, isPrivate, "empty URL should be private") + }) + + t.Run("isPrivateBitbucket returns private when extraction fails", func(t *testing.T) { + // Invalid URL that won't extract properly + isPrivate := isPrivateBitbucket("invalid") + assert.True(t, isPrivate, "invalid URL should be private") + }) + + t.Run("isPrivateAzureDevOps returns private when extraction fails", func(t *testing.T) { + // Invalid URL that won't extract properly + isPrivate := isPrivateAzureDevOps("invalid") + assert.True(t, isPrivate, "invalid URL should be private") + }) +} + +func TestGenerateAndWrite_WithCommits(t *testing.T) { + t.Run("creates metadata with commit count", func(t *testing.T) { + repoPath := t.TempDir() + repo, err := gogit.PlainInit(repoPath, false) + require.NoError(t, err) + + _, err = repo.CreateRemote(&config.RemoteConfig{ + Name: "origin", + URLs: []string{"https://github.com/test/repo.git"}, + }) + require.NoError(t, err) + + worktree, err := repo.Worktree() + require.NoError(t, err) + + sig := &object.Signature{Name: "User", Email: "user@example.com", When: time.Now()} + _, err = worktree.Commit("commit1", &gogit.CommitOptions{ + Author: sig, + AllowEmptyCommits: true, + }) + require.NoError(t, err) + + err = GenerateAndWrite(repoPath, true) + assert.NoError(t, err) + + metadataPath := filepath.Join(repoPath, CheckmarxFolderName, MetadataFileName) + data, err := os.ReadFile(metadataPath) + require.NoError(t, err) + + var metadata contributorsMetadata + err = json.Unmarshal(data, &metadata) + require.NoError(t, err) + assert.Equal(t, 1, metadata.CommitsCount) + }) + + t.Run("creates files for private repo with commit", func(t *testing.T) { + repoPath := t.TempDir() + repo, err := gogit.PlainInit(repoPath, false) + require.NoError(t, err) + + _, err = repo.CreateRemote(&config.RemoteConfig{ + Name: "origin", + URLs: []string{"https://github.com/private/repo.git"}, + }) + require.NoError(t, err) + + worktree, err := repo.Worktree() + require.NoError(t, err) + + sig := &object.Signature{Name: "Dev", Email: "dev@example.com", When: time.Now()} + _, err = worktree.Commit("work", &gogit.CommitOptions{ + Author: sig, + AllowEmptyCommits: true, + }) + require.NoError(t, err) + + err = GenerateAndWrite(repoPath, true) + assert.NoError(t, err) + + metadataPath := filepath.Join(repoPath, CheckmarxFolderName, MetadataFileName) + assert.True(t, fileExists(metadataPath)) + }) +} diff --git a/internal/commands/root.go b/internal/commands/root.go index 74602d61..3b968210 100644 --- a/internal/commands/root.go +++ b/internal/commands/root.go @@ -133,6 +133,9 @@ func NewAstCLI( return err } PrintConfiguration() + if viper.GetBool(params.InsecureFlag) { + fmt.Println("WARNING: --insecure flag is enabled. This disables SSL/TLS certificate verification. Do NOT use this flag in production unless your security team has explicitly evaluated and approved the risk.") + } err = configuration.LoadConfiguration() if err != nil { return err diff --git a/internal/commands/scan.go b/internal/commands/scan.go index 41b24008..d98ff8a6 100644 --- a/internal/commands/scan.go +++ b/internal/commands/scan.go @@ -60,6 +60,7 @@ const ( containerImagesFlagError = "--container-images flag error" git = "git" + gitFolderName = ".git" invalidSSHSource = "provided source does not need a key. Make sure you are defining the right source or remove the flag --ssh-key" errorUnzippingFile = "an error occurred while unzipping file. Reason: " containerRun = "run" @@ -928,6 +929,7 @@ func scanCreateSubCommand( createScanCmd.PersistentFlags().Bool(commonParams.GitIgnoreFileFilterFlag, false, commonParams.GitIgnoreFileFilterUsage) createScanCmd.PersistentFlags().StringSlice(commonParams.AntFilterFlag, []string{}, commonParams.AntFilterUsage) createScanCmd.PersistentFlags().Bool(commonParams.SkipDefaultFilterFlag, false, commonParams.SkipDefaultFilterFlagUsage) + createScanCmd.PersistentFlags().Bool(commonParams.ExcludeGitFolderFlag, false, commonParams.ExcludeGitFolderFlagUsage) return createScanCmd } @@ -1644,7 +1646,7 @@ func scanTypeEnabled(scanType string) bool { return false } -func compressFolder(sourceDir, filter, userIncludeFilter, scaResolver string, antMatcher filtering.Matcher, skipDefaultFilter bool) (string, error) { +func compressFolder(sourceDir, filter, userIncludeFilter, scaResolver string, antMatcher filtering.Matcher, skipDefaultFilter, includeGeneratedCsvJson, excludeGitFolder bool) (string, error) { scaToolPath := scaResolver outputFile, err := os.CreateTemp(os.TempDir(), "cx-*.zip") if err != nil { @@ -1672,7 +1674,7 @@ func compressFolder(sourceDir, filter, userIncludeFilter, scaResolver string, an } } else { // Add directory files normally - err = addDirFiles(zipWriter, "", sourceDir, getExcludeFilters(filter, skipDefaultFilter), getIncludeFilters(userIncludeFilter, skipDefaultFilter), antMatcher) + err = addDirFiles(zipWriter, "", sourceDir, getExcludeFilters(filter, skipDefaultFilter), getIncludeFilters(userIncludeFilter, skipDefaultFilter), antMatcher, excludeGitFolder) if err != nil { return "", err } @@ -1685,6 +1687,13 @@ func compressFolder(sourceDir, filter, userIncludeFilter, scaResolver string, an } } + // Add contributors.csv/metadata.json only if they were just freshly generated without error. + if includeGeneratedCsvJson { + if err := addGeneratedContributorsFiles(zipWriter, sourceDir); err != nil { + return "", err + } + } + // Close the file err = zipWriter.Close() if err != nil { @@ -1803,7 +1812,7 @@ func addDirFilesIgnoreFilter(zipWriter *zip.Writer, baseDir, parentDir string) e return nil } -func addDirFiles(zipWriter *zip.Writer, baseDir, parentDir string, filters, includeFilters []string, antMatcher filtering.Matcher) error { +func addDirFiles(zipWriter *zip.Writer, baseDir, parentDir string, filters, includeFilters []string, antMatcher filtering.Matcher, excludeGitFolder bool) error { fileEntries, err := os.ReadDir(parentDir) if err != nil { return err @@ -1816,7 +1825,7 @@ func addDirFiles(zipWriter *zip.Writer, baseDir, parentDir string, filters, incl } if util.IsDirOrSymLinkToDir(parentDir, fileInfo) { - err = handleDir(zipWriter, baseDir, parentDir, filters, includeFilters, fileInfo, antMatcher) + err = handleDir(zipWriter, baseDir, parentDir, filters, includeFilters, fileInfo, antMatcher, excludeGitFolder) } else { err = handleFile(zipWriter, baseDir, parentDir, filters, includeFilters, fileInfo, antMatcher) } @@ -1847,6 +1856,10 @@ func handleFile( } // relPath is forward-slash path from source root, used by antMatcher. relPath := filepath.ToSlash(baseDir + file.Name()) + if isGeneratedContributorsFile(relPath) { + logger.PrintIfVerbose("Excluded (added separately): " + fileName) + return nil + } if filterMatched(includeFilters, file.Name()) && filterMatched(filters, file.Name()) && !antMatcher.Excluded(relPath, false) { logger.PrintIfVerbose("Included: " + fileName) dat, err := ioutil.ReadFile(parentDir + file.Name()) @@ -1879,9 +1892,15 @@ func handleDir( includeFilters []string, file fs.FileInfo, antMatcher filtering.Matcher, + excludeGitFolder bool, ) error { // Check if folder belongs to the disabled exclusions if commonParams.DisabledExclusions[file.Name()] { + // Exclude .git folder when --exclude-git-folder flag is passed + if excludeGitFolder && file.Name() == gitFolderName { + logger.PrintIfVerbose("The folder " + file.Name() + " is being excluded (--exclude-git-folder flag passed)") + return nil + } logger.PrintIfVerbose("The folder " + file.Name() + " is being included") newParent, newBase := GetNewParentAndBase(parentDir, file, baseDir) return addDirFilesIgnoreFilter(zipWriter, newBase, newParent) @@ -1912,7 +1931,7 @@ func handleDir( } newParent, newBase := GetNewParentAndBase(parentDir, file, baseDir) - return addDirFiles(zipWriter, newBase, newParent, filters, includeFilters, antMatcher) + return addDirFiles(zipWriter, newBase, newParent, filters, includeFilters, antMatcher, excludeGitFolder) } func isDirFiltered(filename string, filters []string) (bool, error) { @@ -2139,6 +2158,8 @@ func getUploadURLFromSource(cmd *cobra.Command, uploadsWrapper wrappers.UploadsW scaResolverParams, scaResolver := getScaResolverFlags(cmd) isSbom, _ := cmd.PersistentFlags().GetBool(commonParams.SbomFlag) isGitIgnoreFilter, _ := cmd.Flags().GetBool(commonParams.GitIgnoreFileFilterFlag) + excludeGitFolder, _ := cmd.Flags().GetBool(commonParams.ExcludeGitFolderFlag) + contributorsCsvFlag, _ := wrappers.GetSpecificFeatureFlag(featureFlagsWrapper, wrappers.RepostoreCustomerContributorsCsvEnabled) // Build the Ant-style matcher from --file-filter-ext patterns. // Construction errors are surfaced immediately so the user gets clear @@ -2199,6 +2220,7 @@ func getUploadURLFromSource(cmd *cobra.Command, uploadsWrapper wrappers.UploadsW var errorUnzippingFile error userProvidedZip := len(zipFilePath) > 0 + contributorsCsvEnabled := (contributorsCsvFlag != nil && contributorsCsvFlag.Status) && !userProvidedZip // containerScanTriggered must stay in this condition: without it, a container scan // run with --containers-local-resolution and --skip-default-filter (and no @@ -2298,7 +2320,22 @@ func getUploadURLFromSource(cmd *cobra.Command, uploadsWrapper wrappers.UploadsW } } else { if !isSbom { - zipFilePath, dirPathErr = compressFolder(directoryPath, sourceDirFilter, userIncludeFilter, scaResolver, antMatcher, skipDefaultFilter) + // True only if contributors.csv/metadata.json were just generated successfully without error. + includeGeneratedCsvJson := false + if contributorsCsvEnabled { + isPrivate := detectRepositoryPrivacy(directoryPath) + if genErr := GenerateAndWrite(directoryPath, isPrivate); genErr != nil { + logger.PrintIfVerbose("Skipping contributors.csv/metadata.json generation: " + genErr.Error()) + } else { + includeGeneratedCsvJson = true + } + } + zipFilePath, dirPathErr = compressFolder(directoryPath, sourceDirFilter, userIncludeFilter, scaResolver, antMatcher, skipDefaultFilter, includeGeneratedCsvJson, excludeGitFolder) + + // Clean up generated contributors files after successful zip creation + if dirPathErr == nil && includeGeneratedCsvJson { + cleanGeneratedContributorsFiles(directoryPath) + } } // Clean up .checkmarx/containers directory after successful mixed scan (including containers) compression @@ -4267,7 +4304,7 @@ func hasGitRepository(source string) bool { } // Check if .git exists in the root directory - gitPath := filepath.Join(sourceTrimmed, ".git") + gitPath := filepath.Join(sourceTrimmed, gitFolderName) if _, err := os.Stat(gitPath); err == nil { return true } @@ -4283,7 +4320,7 @@ func searchGitInSubdirectories(sourcePath string) bool { if err != nil || found { return nil } - if info.IsDir() && info.Name() == ".git" { + if info.IsDir() && info.Name() == gitFolderName { found = true return filepath.SkipAll } @@ -4445,3 +4482,83 @@ func readGitIgnoreFromZip(zipPath string) ([]byte, error) { } return []byte(""), fmt.Errorf(".gitignore not found in zip: %s", zipPath) } + +// isGeneratedContributorsFile checks if file is contributors.csv or metadata.json to skip in zip walk. +func isGeneratedContributorsFile(relPath string) bool { + return relPath == CheckmarxFolderName+"/"+ContributorsFileName || + relPath == CheckmarxFolderName+"/"+MetadataFileName +} + +// addGeneratedContributorsFiles reads and writes contributors.csv/metadata.json to zip; missing files OK. +func addGeneratedContributorsFiles(zipWriter *zip.Writer, sourceDir string) error { + for _, fileName := range []string{ContributorsFileName, MetadataFileName} { + filePath := filepath.Join(sourceDir, CheckmarxFolderName, fileName) + dat, err := os.ReadFile(filePath) + if err != nil { + if os.IsNotExist(err) { + logger.PrintIfVerbose("Skipping " + fileName + ": not found under " + CheckmarxFolderName + "/") + continue + } + return err + } + + zipEntryName := CheckmarxFolderName + "/" + fileName + f, err := zipWriter.Create(zipEntryName) + if err != nil { + return err + } + if _, err := f.Write(dat); err != nil { + return err + } + logger.PrintIfVerbose("Included: " + zipEntryName) + } + return nil +} + +// cleanGeneratedContributorsFiles removes contributors.csv and metadata.json after zip creation. +// Removes both files if present (either or both may exist). Preserves .checkmarx folder if other files remain. +// Only deletes .checkmarx folder if it becomes completely empty after both files are removed. +func cleanGeneratedContributorsFiles(directoryPath string) { + checkmarxDir := filepath.Join(directoryPath, ".checkmarx") + if _, err := os.Stat(checkmarxDir); os.IsNotExist(err) { + return + } + + csvPath := filepath.Join(checkmarxDir, "contributors.csv") + jsonPath := filepath.Join(checkmarxDir, "metadata.json") + fileRemoved := false + + // Attempt to remove contributors.csv if it exists + if _, err := os.Stat(csvPath); err == nil { + if rmErr := os.Remove(csvPath); rmErr != nil { + logger.PrintIfVerbose(fmt.Sprintf("Warning: Failed to remove contributors.csv: %s", rmErr.Error())) + } else { + logger.PrintIfVerbose("Removed contributors.csv after zip creation") + fileRemoved = true + } + } + + // Attempt to remove metadata.json if it exists + if _, err := os.Stat(jsonPath); err == nil { + if rmErr := os.Remove(jsonPath); rmErr != nil { + logger.PrintIfVerbose(fmt.Sprintf("Warning: Failed to remove metadata.json: %s", rmErr.Error())) + } else { + logger.PrintIfVerbose("Removed metadata.json after zip creation") + fileRemoved = true + } + } + + // Only remove .checkmarx folder if it's empty after both contributor files removed (either or both may have existed) + if fileRemoved { + entries, err := os.ReadDir(checkmarxDir) + if err == nil && len(entries) == 0 { + if rmErr := os.Remove(checkmarxDir); rmErr != nil { + logger.PrintIfVerbose(fmt.Sprintf("Warning: Failed to remove empty .checkmarx directory: %s", rmErr.Error())) + return + } + logger.PrintIfVerbose("Removed empty .checkmarx directory (empty after removing contributor files)") + } else if err == nil && len(entries) > 0 { + logger.PrintIfVerbose(fmt.Sprintf("Kept .checkmarx directory (contains %d other file(s) e.g., containers/)", len(entries))) + } + } +} diff --git a/internal/commands/scan_test.go b/internal/commands/scan_test.go index 8b698800..850bf37e 100644 --- a/internal/commands/scan_test.go +++ b/internal/commands/scan_test.go @@ -5347,7 +5347,7 @@ func TestSbomFileExcludedFromZip_WithCustomOutputName(t *testing.T) { noopMatcher, matcherErr := filtering.NewAntMatcher(nil) assert.NilError(t, matcherErr) - zipPath, err := compressFolder(sbomTestSourceDir(projectDir), "", "", "", noopMatcher, false) + zipPath, err := compressFolder(sbomTestSourceDir(projectDir), "", "", "", noopMatcher, false, false, false) assert.NilError(t, err) defer func() { _ = os.Remove(zipPath) }() @@ -5378,7 +5378,7 @@ func TestDefaultSbomFileAlwaysExcludedFromZip(t *testing.T) { noopMatcher, matcherErr := filtering.NewAntMatcher(nil) assert.NilError(t, matcherErr) - zipPath, err := compressFolder(sbomTestSourceDir(projectDir), "", "", "", noopMatcher, false) + zipPath, err := compressFolder(sbomTestSourceDir(projectDir), "", "", "", noopMatcher, false, false, false) assert.NilError(t, err) defer func() { _ = os.Remove(zipPath) }() @@ -5411,7 +5411,7 @@ func TestSbomFileExcludedFromZip_InSubdirectory(t *testing.T) { noopMatcher, matcherErr := filtering.NewAntMatcher(nil) assert.NilError(t, matcherErr) - zipPath, err := compressFolder(sbomTestSourceDir(projectDir), "", "", "", noopMatcher, false) + zipPath, err := compressFolder(sbomTestSourceDir(projectDir), "", "", "", noopMatcher, false, false, false) assert.NilError(t, err) defer func() { _ = os.Remove(zipPath) }() @@ -5450,7 +5450,7 @@ func TestSbomFileExcludedFromZip_AbsoluteSubdirWithCustomName(t *testing.T) { noopMatcher, matcherErr := filtering.NewAntMatcher(nil) assert.NilError(t, matcherErr) - zipPath, err := compressFolder(sbomTestSourceDir(projectDir), "", "", "", noopMatcher, false) + zipPath, err := compressFolder(sbomTestSourceDir(projectDir), "", "", "", noopMatcher, false, false, false) assert.NilError(t, err) defer func() { _ = os.Remove(zipPath) }() @@ -5535,7 +5535,7 @@ func TestCompressFolder_DefaultBehaviorUnchanged(t *testing.T) { noopMatcher, matcherErr := filtering.NewAntMatcher(nil) assert.NilError(t, matcherErr) - zipPath, err := compressFolder(sbomTestSourceDir(projectDir), "", "", "", noopMatcher, false) + zipPath, err := compressFolder(sbomTestSourceDir(projectDir), "", "", "", noopMatcher, false, false, false) assert.NilError(t, err) defer func() { _ = os.Remove(zipPath) }() @@ -5556,7 +5556,7 @@ func TestCompressFolder_SkipDefaultFilter(t *testing.T) { noopMatcher, matcherErr := filtering.NewAntMatcher(nil) assert.NilError(t, matcherErr) - zipPath, err := compressFolder(sbomTestSourceDir(projectDir), "", "", "", noopMatcher, true) + zipPath, err := compressFolder(sbomTestSourceDir(projectDir), "", "", "", noopMatcher, true, false, false) assert.NilError(t, err) defer func() { _ = os.Remove(zipPath) }() @@ -5586,7 +5586,7 @@ func TestCompressFolder_SkipDefaultFilter_WithAntFilterExclude(t *testing.T) { antMatcher, matcherErr := filtering.NewAntMatcher([]string{"!excluded_by_ant/**"}) assert.NilError(t, matcherErr) - zipPath, err := compressFolder(sbomTestSourceDir(projectDir), "", "", "", antMatcher, true) + zipPath, err := compressFolder(sbomTestSourceDir(projectDir), "", "", "", antMatcher, true, false, false) assert.NilError(t, err) defer func() { _ = os.Remove(zipPath) }() @@ -5607,7 +5607,7 @@ func TestCompressFolder_SkipDefaultFilter_WithAntFilterIncludeOnly(t *testing.T) antMatcher, matcherErr := filtering.NewAntMatcher([]string{"**/*.customext"}) assert.NilError(t, matcherErr) - zipPath, err := compressFolder(sbomTestSourceDir(projectDir), "", "", "", antMatcher, true) + zipPath, err := compressFolder(sbomTestSourceDir(projectDir), "", "", "", antMatcher, true, false, false) assert.NilError(t, err) defer func() { _ = os.Remove(zipPath) }() @@ -5633,7 +5633,7 @@ func TestCompressFolder_DefaultFilters_WithAntFilter(t *testing.T) { antMatcher, matcherErr := filtering.NewAntMatcher([]string{"!keep_dir/**"}) assert.NilError(t, matcherErr) - zipPath, err := compressFolder(sbomTestSourceDir(projectDir), "", "", "", antMatcher, false) + zipPath, err := compressFolder(sbomTestSourceDir(projectDir), "", "", "", antMatcher, false, false, false) assert.NilError(t, err) defer func() { _ = os.Remove(zipPath) }() @@ -5642,3 +5642,336 @@ func TestCompressFolder_DefaultFilters_WithAntFilter(t *testing.T) { assert.Equal(t, false, zipContainsFile(t, zipPath, "lib.js")) assert.Equal(t, false, zipContainsFile(t, zipPath, "marker.go")) } + +// zipFileCount returns count of entries in zip with given filename to guard against duplicates. +func zipFileCount(t *testing.T, zipPath, filename string) int { + t.Helper() + r, err := zip.OpenReader(zipPath) + assert.NilError(t, err) + defer func() { _ = r.Close() }() + count := 0 + for _, f := range r.File { + if filepath.Base(f.Name) == filename || f.Name == filename { + count++ + } + } + return count +} + +func TestCompressFolder_GitExcluded_WhenContributorsCsvEnabled(t *testing.T) { + projectDir, err := os.MkdirTemp("", "contributors-csv-git-exclude-*") + assert.NilError(t, err) + defer func() { _ = os.RemoveAll(projectDir) }() + + assert.NilError(t, os.WriteFile(filepath.Join(projectDir, "main.go"), []byte("package main"), 0600)) + gitDir := filepath.Join(projectDir, ".git") + assert.NilError(t, os.MkdirAll(gitDir, 0700)) + assert.NilError(t, os.WriteFile(filepath.Join(gitDir, "HEAD"), []byte("ref: refs/heads/main"), 0600)) + + noopMatcher, matcherErr := filtering.NewAntMatcher(nil) + assert.NilError(t, matcherErr) + zipPath, err := compressFolder(sbomTestSourceDir(projectDir), "", "", "", noopMatcher, false, true, true) + assert.NilError(t, err) + defer func() { _ = os.Remove(zipPath) }() + + assert.Equal(t, true, zipContainsFile(t, zipPath, "main.go")) + assert.Equal(t, false, zipContainsFile(t, zipPath, "HEAD"), + ".git contents should be excluded when --exclude-git-folder flag is passed") +} + +// TestCompressFolder_GitIncluded_WhenContributorsCsvDisabled guards .git force-include when flag is off. +func TestCompressFolder_GitIncluded_WhenContributorsCsvDisabled(t *testing.T) { + projectDir, err := os.MkdirTemp("", "contributors-csv-git-include-*") + assert.NilError(t, err) + defer func() { _ = os.RemoveAll(projectDir) }() + + assert.NilError(t, os.WriteFile(filepath.Join(projectDir, "main.go"), []byte("package main"), 0600)) + gitDir := filepath.Join(projectDir, ".git") + assert.NilError(t, os.MkdirAll(gitDir, 0700)) + assert.NilError(t, os.WriteFile(filepath.Join(gitDir, "HEAD"), []byte("ref: refs/heads/main"), 0600)) + + noopMatcher, matcherErr := filtering.NewAntMatcher(nil) + assert.NilError(t, matcherErr) + zipPath, err := compressFolder(sbomTestSourceDir(projectDir), "", "", "", noopMatcher, false, false, false) + assert.NilError(t, err) + defer func() { _ = os.Remove(zipPath) }() + + assert.Equal(t, true, zipContainsFile(t, zipPath, "main.go")) + assert.Equal(t, true, zipContainsFile(t, zipPath, "HEAD"), + ".git contents must still be force-included when --exclude-git-folder flag is not passed") +} + +// TestCompressFolder_ContributorsFilesForceIncluded_WhenEnabled verifies both generated files land in zip once when enabled. +func TestCompressFolder_ContributorsFilesForceIncluded_WhenEnabled(t *testing.T) { + projectDir, err := os.MkdirTemp("", "contributors-csv-force-include-*") + assert.NilError(t, err) + defer func() { _ = os.RemoveAll(projectDir) }() + + assert.NilError(t, os.WriteFile(filepath.Join(projectDir, "main.go"), []byte("package main"), 0600)) + checkmarxDir := filepath.Join(projectDir, ".checkmarx") + assert.NilError(t, os.MkdirAll(checkmarxDir, 0700)) + assert.NilError(t, os.WriteFile(filepath.Join(checkmarxDir, "contributors.csv"), + []byte("2025-09-30T10:35:05+03:00,abc123,alice@example.com,Alice\n"), 0600)) + assert.NilError(t, os.WriteFile(filepath.Join(checkmarxDir, "metadata.json"), + []byte(`{"repositoryUrl":"https://example.com/repo.git"}`), 0600)) + + noopMatcher, matcherErr := filtering.NewAntMatcher(nil) + assert.NilError(t, matcherErr) + zipPath, err := compressFolder(sbomTestSourceDir(projectDir), "", "", "", noopMatcher, false, true, false) + assert.NilError(t, err) + defer func() { _ = os.Remove(zipPath) }() + + assert.Equal(t, true, zipContainsFile(t, zipPath, "main.go")) + assert.Equal(t, 1, zipFileCount(t, zipPath, "contributors.csv"), + "contributors.csv must be present exactly once, despite *.csv not being in the default include-filter allowlist") + assert.Equal(t, 1, zipFileCount(t, zipPath, "metadata.json"), + "metadata.json must be present exactly once (not duplicated by both the normal walk and the explicit add-back step)") +} + +// TestCompressFolder_ContributorsFilesNotIncluded_WhenDisabled guards .checkmarx files excluded when flag off. +func TestCompressFolder_ContributorsFilesNotIncluded_WhenDisabled(t *testing.T) { + projectDir, err := os.MkdirTemp("", "contributors-csv-disabled-*") + assert.NilError(t, err) + defer func() { _ = os.RemoveAll(projectDir) }() + + assert.NilError(t, os.WriteFile(filepath.Join(projectDir, "main.go"), []byte("package main"), 0600)) + checkmarxDir := filepath.Join(projectDir, ".checkmarx") + assert.NilError(t, os.MkdirAll(checkmarxDir, 0700)) + assert.NilError(t, os.WriteFile(filepath.Join(checkmarxDir, "contributors.csv"), + []byte("2025-09-30T10:35:05+03:00,abc123,alice@example.com,Alice\n"), 0600)) + + noopMatcher, matcherErr := filtering.NewAntMatcher(nil) + assert.NilError(t, matcherErr) + zipPath, err := compressFolder(sbomTestSourceDir(projectDir), "", "", "", noopMatcher, false, false, false) + assert.NilError(t, err) + defer func() { _ = os.Remove(zipPath) }() + + assert.Equal(t, true, zipContainsFile(t, zipPath, "main.go")) + assert.Equal(t, false, zipContainsFile(t, zipPath, "contributors.csv"), + "without the feature flag, contributors.csv should be dropped by the default include-filter allowlist, same as before this feature existed") +} + +// TestCompressFolder_StaleFilesNotIncluded_WhenGenerationFailedThisRun guards stale files not picked up if generation fails. +func TestCompressFolder_StaleFilesNotIncluded_WhenGenerationFailedThisRun(t *testing.T) { + projectDir, err := os.MkdirTemp("", "contributors-csv-stale-*") + assert.NilError(t, err) + defer func() { _ = os.RemoveAll(projectDir) }() + + assert.NilError(t, os.WriteFile(filepath.Join(projectDir, "main.go"), []byte("package main"), 0600)) + gitDir := filepath.Join(projectDir, ".git") + assert.NilError(t, os.MkdirAll(gitDir, 0700)) + assert.NilError(t, os.WriteFile(filepath.Join(gitDir, "HEAD"), []byte("ref: refs/heads/main"), 0600)) + + // Simulate leftover files from a previous successful run. + checkmarxDir := filepath.Join(projectDir, ".checkmarx") + assert.NilError(t, os.MkdirAll(checkmarxDir, 0700)) + assert.NilError(t, os.WriteFile(filepath.Join(checkmarxDir, "contributors.csv"), + []byte("2025-09-30T10:35:05+03:00,abc123,alice@example.com,Alice\n"), 0600)) + assert.NilError(t, os.WriteFile(filepath.Join(checkmarxDir, "metadata.json"), + []byte(`{"repositoryUrl":"https://example.com/repo.git"}`), 0600)) + + noopMatcher, matcherErr := filtering.NewAntMatcher(nil) + assert.NilError(t, matcherErr) + // excludeGitFolder=true (flag on) but includeGeneratedCsvJson=false (this run's + // generation failed) - the scenario this fix exists for. + zipPath, err := compressFolder(sbomTestSourceDir(projectDir), "", "", "", noopMatcher, false, false, true) + assert.NilError(t, err) + defer func() { _ = os.Remove(zipPath) }() + + assert.Equal(t, true, zipContainsFile(t, zipPath, "main.go")) + assert.Equal(t, false, zipContainsFile(t, zipPath, "HEAD"), + ".git should still be excluded - that decision is tied to the flag alone, not generation success") + assert.Equal(t, false, zipContainsFile(t, zipPath, "contributors.csv"), + "stale contributors.csv from a previous run must not be picked up when this run's generation failed") + assert.Equal(t, false, zipContainsFile(t, zipPath, "metadata.json"), + "stale metadata.json from a previous run must not be picked up when this run's generation failed") +} + +// TestIsGeneratedContributorsFile verifies correct identification of generated files +func TestIsGeneratedContributorsFile(t *testing.T) { + tests := []struct { + relPath string + expected bool + desc string + }{ + {".checkmarx/contributors.csv", true, "exact match for CSV file"}, + {".checkmarx/metadata.json", true, "exact match for JSON file"}, + {".checkmarx/other.txt", false, "non-generated file in .checkmarx"}, + {"contributors.csv", false, "CSV file not in .checkmarx"}, + {"metadata.json", false, "JSON file not in .checkmarx"}, + {".checkmarx/", false, "directory path"}, + {"", false, "empty path"}, + {".checkmarx/contributors.csv/", false, "trailing slash"}, + {"nested/.checkmarx/contributors.csv", false, "nested .checkmarx path"}, + } + + for _, tt := range tests { + t.Run(tt.desc, func(t *testing.T) { + result := isGeneratedContributorsFile(tt.relPath) + assert.Equal(t, tt.expected, result, "path: %s", tt.relPath) + }) + } +} + +// TestAddGeneratedContributorsFiles verifies files are added to zip correctly +func TestAddGeneratedContributorsFiles(t *testing.T) { + t.Run("both files exist and are added to zip", func(t *testing.T) { + sourceDir := t.TempDir() + checkmarxDir := filepath.Join(sourceDir, ".checkmarx") + assert.NilError(t, os.MkdirAll(checkmarxDir, 0700)) + + csvContent := []byte("2025-09-30T10:35:05Z,abc123,alice@example.com,Alice\n") + jsonContent := []byte(`{"repositoryUrl":"https://example.com/repo.git","commitsCount":5}`) + + assert.NilError(t, os.WriteFile(filepath.Join(checkmarxDir, "contributors.csv"), csvContent, 0600)) + assert.NilError(t, os.WriteFile(filepath.Join(checkmarxDir, "metadata.json"), jsonContent, 0600)) + + zipPath := filepath.Join(t.TempDir(), "test.zip") + zipFile, err := os.Create(zipPath) + assert.NilError(t, err) + defer func() { _ = zipFile.Close() }() + + zipWriter := zip.NewWriter(zipFile) + defer func() { _ = zipWriter.Close() }() + + err = addGeneratedContributorsFiles(zipWriter, sourceDir) + assert.NilError(t, err) + assert.NilError(t, zipWriter.Close()) + + assert.Equal(t, true, zipContainsFile(t, zipPath, ".checkmarx/contributors.csv")) + assert.Equal(t, true, zipContainsFile(t, zipPath, ".checkmarx/metadata.json")) + }) + + t.Run("only CSV file exists", func(t *testing.T) { + sourceDir := t.TempDir() + checkmarxDir := filepath.Join(sourceDir, ".checkmarx") + assert.NilError(t, os.MkdirAll(checkmarxDir, 0700)) + + csvContent := []byte("2025-09-30T10:35:05Z,abc123,alice@example.com,Alice\n") + assert.NilError(t, os.WriteFile(filepath.Join(checkmarxDir, "contributors.csv"), csvContent, 0600)) + + zipPath := filepath.Join(t.TempDir(), "test.zip") + zipFile, err := os.Create(zipPath) + assert.NilError(t, err) + defer func() { _ = zipFile.Close() }() + + zipWriter := zip.NewWriter(zipFile) + defer func() { _ = zipWriter.Close() }() + + err = addGeneratedContributorsFiles(zipWriter, sourceDir) + assert.NilError(t, err) + assert.NilError(t, zipWriter.Close()) + + assert.Equal(t, true, zipContainsFile(t, zipPath, ".checkmarx/contributors.csv")) + assert.Equal(t, false, zipContainsFile(t, zipPath, ".checkmarx/metadata.json")) + }) + + t.Run("no files exist should not error", func(t *testing.T) { + sourceDir := t.TempDir() + checkmarxDir := filepath.Join(sourceDir, ".checkmarx") + assert.NilError(t, os.MkdirAll(checkmarxDir, 0700)) + + zipPath := filepath.Join(t.TempDir(), "test.zip") + zipFile, err := os.Create(zipPath) + assert.NilError(t, err) + defer func() { _ = zipFile.Close() }() + + zipWriter := zip.NewWriter(zipFile) + defer func() { _ = zipWriter.Close() }() + + err = addGeneratedContributorsFiles(zipWriter, sourceDir) + assert.NilError(t, err, "should not error when files don't exist") + }) +} + +// TestCleanGeneratedContributorsFiles verifies cleanup removes files correctly +func TestCleanGeneratedContributorsFiles(t *testing.T) { + t.Run("removes both files and empty .checkmarx folder", func(t *testing.T) { + dirPath := t.TempDir() + checkmarxDir := filepath.Join(dirPath, ".checkmarx") + assert.NilError(t, os.MkdirAll(checkmarxDir, 0700)) + + csvPath := filepath.Join(checkmarxDir, "contributors.csv") + jsonPath := filepath.Join(checkmarxDir, "metadata.json") + assert.NilError(t, os.WriteFile(csvPath, []byte("data"), 0600)) + assert.NilError(t, os.WriteFile(jsonPath, []byte("data"), 0600)) + + cleanGeneratedContributorsFiles(dirPath) + + assert.Equal(t, false, fileExists(csvPath), "CSV should be removed") + assert.Equal(t, false, fileExists(jsonPath), "JSON should be removed") + assert.Equal(t, false, fileExists(checkmarxDir), ".checkmarx should be removed when empty") + }) + + t.Run("preserves .checkmarx folder if other files exist", func(t *testing.T) { + dirPath := t.TempDir() + checkmarxDir := filepath.Join(dirPath, ".checkmarx") + assert.NilError(t, os.MkdirAll(checkmarxDir, 0700)) + + csvPath := filepath.Join(checkmarxDir, "contributors.csv") + jsonPath := filepath.Join(checkmarxDir, "metadata.json") + otherPath := filepath.Join(checkmarxDir, "other.txt") + + assert.NilError(t, os.WriteFile(csvPath, []byte("data"), 0600)) + assert.NilError(t, os.WriteFile(jsonPath, []byte("data"), 0600)) + assert.NilError(t, os.WriteFile(otherPath, []byte("data"), 0600)) + + cleanGeneratedContributorsFiles(dirPath) + + assert.Equal(t, false, fileExists(csvPath), "CSV should be removed") + assert.Equal(t, false, fileExists(jsonPath), "JSON should be removed") + assert.Equal(t, true, fileExists(checkmarxDir), ".checkmarx should be preserved") + assert.Equal(t, true, fileExists(otherPath), "other files should be preserved") + }) + + t.Run("handles only CSV file existing", func(t *testing.T) { + dirPath := t.TempDir() + checkmarxDir := filepath.Join(dirPath, ".checkmarx") + assert.NilError(t, os.MkdirAll(checkmarxDir, 0700)) + + csvPath := filepath.Join(checkmarxDir, "contributors.csv") + assert.NilError(t, os.WriteFile(csvPath, []byte("data"), 0600)) + + cleanGeneratedContributorsFiles(dirPath) + + assert.Equal(t, false, fileExists(csvPath), "CSV should be removed") + assert.Equal(t, false, fileExists(checkmarxDir), ".checkmarx should be removed when empty") + }) + + t.Run("handles no .checkmarx folder gracefully", func(t *testing.T) { + dirPath := t.TempDir() + cleanGeneratedContributorsFiles(dirPath) + // Should not panic or error when .checkmarx doesn't exist + }) + + t.Run("handles missing files gracefully", func(t *testing.T) { + dirPath := t.TempDir() + checkmarxDir := filepath.Join(dirPath, ".checkmarx") + assert.NilError(t, os.MkdirAll(checkmarxDir, 0700)) + + // Should not panic or error when files don't exist + cleanGeneratedContributorsFiles(dirPath) + assert.Equal(t, true, fileExists(checkmarxDir), ".checkmarx should still exist") + }) + + t.Run("handles directory with other files - preserves non-generated files", func(t *testing.T) { + dirPath := t.TempDir() + checkmarxDir := filepath.Join(dirPath, ".checkmarx") + assert.NilError(t, os.MkdirAll(checkmarxDir, 0700)) + + csvPath := filepath.Join(checkmarxDir, "contributors.csv") + jsonPath := filepath.Join(checkmarxDir, "metadata.json") + otherPath := filepath.Join(checkmarxDir, "other.txt") + assert.NilError(t, os.WriteFile(csvPath, []byte("data"), 0600)) + assert.NilError(t, os.WriteFile(jsonPath, []byte("data"), 0600)) + assert.NilError(t, os.WriteFile(otherPath, []byte("data"), 0600)) + + // Remove generated files but keep directory and other files + cleanGeneratedContributorsFiles(dirPath) + + assert.Equal(t, false, fileExists(csvPath), "CSV should be removed") + assert.Equal(t, false, fileExists(jsonPath), "JSON should be removed") + assert.Equal(t, true, fileExists(checkmarxDir), ".checkmarx should still exist") + assert.Equal(t, true, fileExists(otherPath), "other files should be preserved") + }) +} diff --git a/internal/params/flags.go b/internal/params/flags.go index 0dd7e108..eae820f3 100644 --- a/internal/params/flags.go +++ b/internal/params/flags.go @@ -197,6 +197,8 @@ const ( LogFileConsoleUsage = "Saves logs to the specified file path as well as to the console" SkipDefaultFilterFlag = "skip-default-filter" SkipDefaultFilterFlagUsage = "Skip the default file filter." + ExcludeGitFolderFlag = "exclude-git-folder" + ExcludeGitFolderFlagUsage = "Exclude .git folder from scan source upload zip." GitIgnoreFileFilterFlag = "use-gitignore" GitIgnoreFileFilterUsage = "Exclude files and directories from the scan based on the patterns defined in the directory's .gitignore file" AntFilterFlag = "file-filter-ext" diff --git a/internal/wrappers/feature-flags.go b/internal/wrappers/feature-flags.go index ec0a31bd..c4008f64 100644 --- a/internal/wrappers/feature-flags.go +++ b/internal/wrappers/feature-flags.go @@ -22,6 +22,9 @@ const maxRetries = 3 const IncreaseFileUploadLimit = "INCREASE_FILE_UPLOAD_LIMIT" const ScaDeltaScanEnabled = "SCA_DELTASCAN_ENABLED" +// RepostoreCustomerContributorsCsvEnabled is the feature flag for generating contributors.csv and metadata.json. +const RepostoreCustomerContributorsCsvEnabled = "REPOSTORE_CUSTOMER_CONTRIBUTORS_CSV_ENABLED" + // AISupplyChainGAEnabled is the feature flag for AI Supply Chain Engine GA. const AISupplyChainGAEnabled = "AI_SUPPLY_CHAIN_ENGINE_GA_ENABLED" diff --git a/test/integration/exclude_git_test.go b/test/integration/exclude_git_test.go new file mode 100644 index 00000000..330020ec --- /dev/null +++ b/test/integration/exclude_git_test.go @@ -0,0 +1,110 @@ +//go:build integration + +package integration + +import ( + "bytes" + "log" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "testing" + + "github.com/checkmarx/ast-cli/internal/params" + "gotest.tools/assert" +) + +// Integration tests for --exclude-git-folder flag +// Clones a public repo once at package level and reuses for all tests + +var ( + clonedRepoPath string + cloneOnce sync.Once + cloneErr error +) + +// clonePublicRepoOnce clones the repo exactly once and caches the path +func clonePublicRepoOnce() string { + cloneOnce.Do(func() { + tempDir, err := os.MkdirTemp("", "ast-cli-git-test-*") + if err != nil { + cloneErr = err + return + } + + // Clone public repo: ast-vscode-extension + repoURL := "https://github.com/Checkmarx/ast-vscode-extension.git" + cmd := exec.Command("git", "clone", "--depth", "1", repoURL, tempDir) + if err := cmd.Run(); err != nil { + cloneErr = err + return + } + + // Verify .git folder exists in cloned repo + gitPath := filepath.Join(tempDir, ".git") + if _, err := os.Stat(gitPath); err != nil { + cloneErr = err + return + } + + clonedRepoPath = tempDir + }) + return clonedRepoPath +} + +func TestExcludeGitFolder_WithFlag(t *testing.T) { + repoPath := clonePublicRepoOnce() + assert.NilError(t, cloneErr, "Failed to clone public repository") + + args := []string{ + "scan", "create", + flag(params.ProjectName), getProjectNameForScanTests(), + flag(params.SourcesFlag), repoPath, + flag(params.ScanTypes), params.IacType, + flag(params.BranchFlag), "main", + flag(params.ExcludeGitFolderFlag), + flag(params.DebugFlag), + } + + var buf bytes.Buffer + log.SetOutput(&buf) + defer func() { + log.SetOutput(os.Stderr) + }() + err, _ := executeCommand(t, args...) + assert.NilError(t, err, "scan create with --exclude-git-folder should succeed") + + logText := buf.String() + assert.Assert(t, strings.Contains(logText, "The folder .git is being excluded"), "Expected .git exclusion message not found in logs") + assert.Assert(t, strings.Contains(logText, "--exclude-git-folder flag passed"), "Expected --exclude-git-folder flag confirmation message not found in logs") +} + +func TestExcludeGitFolder_IncludeCsvJson(t *testing.T) { + repoPath := clonePublicRepoOnce() + assert.NilError(t, cloneErr, "Failed to clone public repository") + + args := []string{ + "scan", "create", + flag(params.ProjectName), getProjectNameForScanTests(), + flag(params.SourcesFlag), repoPath, + flag(params.ScanTypes), params.IacType, + flag(params.BranchFlag), "main", + flag(params.ExcludeGitFolderFlag), + flag(params.DebugFlag), + } + + var buf bytes.Buffer + log.SetOutput(&buf) + defer func() { + log.SetOutput(os.Stderr) + }() + err, _ := executeCommand(t, args...) + assert.NilError(t, err, "scan create with --exclude-git-folder should succeed") + + logText := buf.String() + assert.Assert(t, strings.Contains(logText, "The folder .git is being excluded"), "Expected .git exclusion message not found in logs") + assert.Assert(t, strings.Contains(logText, "--exclude-git-folder flag passed"), "Expected --exclude-git-folder flag confirmation message not found in logs") + assert.Assert(t, strings.Contains(logText, "Included: .checkmarx/metadata.json"), "Included COntributor.csv and metadata.json") +}