From 76dca23506b5994ac0f550576325a8d6cfad4a03 Mon Sep 17 00:00:00 2001 From: frjcomp <107982661+frjcomp@users.noreply.github.com> Date: Wed, 18 Dec 2024 13:39:31 +0000 Subject: [PATCH 1/5] allow disabling the trufflehog verification --- src/pipeleak/cmd/scan.go | 1 + src/pipeleak/scanner/pipeline.go | 30 +++++++++++++++++------------- src/pipeleak/scanner/queue.go | 12 ++++++------ src/pipeleak/scanner/rules.go | 28 ++++++++++++++++++---------- 4 files changed, 42 insertions(+), 29 deletions(-) diff --git a/src/pipeleak/cmd/scan.go b/src/pipeleak/cmd/scan.go index 1571951..6aa7842 100644 --- a/src/pipeleak/cmd/scan.go +++ b/src/pipeleak/cmd/scan.go @@ -48,6 +48,7 @@ func NewScanCmd() *cobra.Command { scanCmd.PersistentFlags().StringVarP(&maxArtifactSize, "max-artifact-size", "", "500Mb", "Max file size of an artifact to be included in scanning. Larger files are skipped. Format: https://pkg.go.dev/github.com/docker/go-units#FromHumanSize") scanCmd.PersistentFlags().IntVarP(&options.MaxScanGoRoutines, "threads", "", 4, "Nr of threads used to scan") scanCmd.PersistentFlags().StringVarP(&options.QueueFolder, "queue", "q", "", "Relative folderpath where the queue files will be stored. Defaults to system tmp") + scanCmd.PersistentFlags().BoolVarP(&options.TruffleHogVerification, "truffleHogVerification", "", true, "Enable the TruffleHog credential verification, will actively test the found credentials and only report those. Disable with --truffleHogVerification=false") scanCmd.PersistentFlags().BoolVarP(&options.Verbose, "verbose", "v", false, "Verbose logging") diff --git a/src/pipeleak/scanner/pipeline.go b/src/pipeleak/scanner/pipeline.go index c359c7a..d98e9f4 100644 --- a/src/pipeleak/scanner/pipeline.go +++ b/src/pipeleak/scanner/pipeline.go @@ -24,19 +24,20 @@ var queueFileName string var queueDB *sql.DB type ScanOptions struct { - GitlabUrl string - GitlabApiToken string - GitlabCookie string - ProjectSearchQuery string - Artifacts bool - Owned bool - Member bool - JobLimit int - Verbose bool - ConfidenceFilter []string - MaxArtifactSize int64 - MaxScanGoRoutines int - QueueFolder string + GitlabUrl string + GitlabApiToken string + GitlabCookie string + ProjectSearchQuery string + Artifacts bool + Owned bool + Member bool + JobLimit int + Verbose bool + ConfidenceFilter []string + MaxArtifactSize int64 + MaxScanGoRoutines int + QueueFolder string + TruffleHogVerification bool } func ScanGitLabPipelines(options *ScanOptions) { @@ -50,6 +51,9 @@ func ScanGitLabPipelines(options *ScanOptions) { }) InitRules(options.ConfidenceFilter) + if options.TruffleHogVerification == false { + log.Info().Msg("TruffleHog verification is disabled") + } git, err := helper.GetGitlabClient(options.GitlabApiToken, options.GitlabUrl) if err != nil { diff --git a/src/pipeleak/scanner/queue.go b/src/pipeleak/scanner/queue.go index e7d3fc4..ced66b9 100644 --- a/src/pipeleak/scanner/queue.go +++ b/src/pipeleak/scanner/queue.go @@ -102,7 +102,7 @@ func analyzeJobTrace(git *gitlab.Client, item QueueItem, options *ScanOptions) { return } - findings := DetectHits(trace, options.MaxScanGoRoutines) + findings := DetectHits(trace, options.MaxScanGoRoutines, options.TruffleHogVerification) for _, finding := range findings { log.Warn().Str("confidence", finding.Pattern.Pattern.Confidence).Str("ruleName", finding.Pattern.Pattern.Name).Str("value", finding.Text).Str("url", item.Meta.JobWebUrl).Str("jobName", item.Meta.JobName).Msg("HIT") } @@ -140,9 +140,9 @@ func analyzeJobArtifact(git *gitlab.Client, item QueueItem, options *ScanOptions kind, _ := filetype.Match(content) // do not scan https://pkg.go.dev/github.com/h2non/filetype#readme-supported-types if kind == filetype.Unknown { - DetectFileHits(content, item.Meta.JobWebUrl, item.Meta.JobName, file.Name, "") + DetectFileHits(content, item.Meta.JobWebUrl, item.Meta.JobName, file.Name, "", options.TruffleHogVerification) } else if filetype.IsArchive(content) { - handleArchiveArtifact(file.Name, content, item.Meta.JobWebUrl, item.Meta.JobName) + handleArchiveArtifact(file.Name, content, item.Meta.JobWebUrl, item.Meta.JobName, options.TruffleHogVerification) } fc.Close() }) @@ -157,7 +157,7 @@ func analyzeDotenvArtifact(git *gitlab.Client, item QueueItem, options *ScanOpti return } - findings := DetectHits(dotenvText, options.MaxScanGoRoutines) + findings := DetectHits(dotenvText, options.MaxScanGoRoutines, options.TruffleHogVerification) for _, finding := range findings { artifactsBaseUrl, _ := url.JoinPath(item.Meta.JobWebUrl, "/-/artifacts") log.Warn().Str("confidence", finding.Pattern.Pattern.Confidence).Str("ruleName", finding.Pattern.Pattern.Name).Str("value", finding.Text).Str("artifactUrl", artifactsBaseUrl).Int("jobId", item.Meta.JobId).Str("jobName", item.Meta.JobName).Msg("HIT DOTENV: Check artifacts page which is the only place to download the dotenv file") @@ -286,7 +286,7 @@ func DownloadEnvArtifact(cookieVal string, gitlabUrl string, prjectPath string, // https://docs.gitlab.com/ee/ci/caching/#common-use-cases-for-caches var skippableDirectoryNames = []string{"node_modules", ".yarn", ".yarn-cache", ".npm", "venv", "vendor", ".go/pkg/mod/"} -func handleArchiveArtifact(archivefileName string, content []byte, jobWebUrl string, jobName string) { +func handleArchiveArtifact(archivefileName string, content []byte, jobWebUrl string, jobName string, enableTruffleHogVerification bool) { for _, skipKeyword := range skippableDirectoryNames { if strings.Contains(archivefileName, skipKeyword) { log.Debug().Str("file", archivefileName).Str("keyword", skipKeyword).Msg("Skipped archive due to blocklist entry") @@ -342,7 +342,7 @@ func handleArchiveArtifact(archivefileName string, content []byte, jobWebUrl str kind, _ := filetype.Match(fileBytes) if kind == filetype.Unknown { - DetectFileHits(fileBytes, jobWebUrl, jobName, path.Base(fPath), archivefileName) + DetectFileHits(fileBytes, jobWebUrl, jobName, path.Base(fPath), archivefileName, enableTruffleHogVerification) } } } diff --git a/src/pipeleak/scanner/rules.go b/src/pipeleak/scanner/rules.go index ba3e980..182140f 100644 --- a/src/pipeleak/scanner/rules.go +++ b/src/pipeleak/scanner/rules.go @@ -141,7 +141,7 @@ func AppendPipeleakRules(rules []PatternElement) []PatternElement { return slices.Concat(rules, customRules) } -func DetectHits(text []byte, maxThreads int) []Finding { +func DetectHits(text []byte, maxThreads int, enableTruffleHogVerification bool) []Finding { ctx := context.Background() group := parallel.Collect[[]Finding](parallel.Limited(ctx, maxThreads)) @@ -179,21 +179,29 @@ func DetectHits(text []byte, maxThreads int) []Finding { for _, detector := range defaults.DefaultDetectors() { trGroup.Go(func(ctx context.Context) ([]Finding, error) { findingsTr := []Finding{} - trHits, err := detector.FromData(ctx, true, text) + trHits, err := detector.FromData(ctx, enableTruffleHogVerification, text) if err != nil { log.Error().Msg("Truffelhog Detector Failed " + err.Error()) return []Finding{}, err } for _, result := range trHits { - // only report verified + secret := result.Raw + if len(result.RawV2) > 0 { + secret = result.RawV2 + } + finding := Finding{Pattern: PatternElement{Pattern: PatternPattern{Name: result.DetectorType.String(), Confidence: "high-verified"}}, Text: string(secret)} + + // if trufflehog verification is enalbed ONLY verified rules are reported if result.Verified { - secret := result.Raw - if len(result.RawV2) > 0 { - secret = result.Raw - } + findingsTr = append(findingsTr, finding) + } - findingsTr = append(findingsTr, Finding{Pattern: PatternElement{Pattern: PatternPattern{Name: result.DetectorType.String(), Confidence: "high-verified"}}, Text: string(secret)}) + // if trufflehog verification is disabled all rules are reported + if enableTruffleHogVerification == false { + // trufflehog itself does not have confidence information + finding.Pattern.Pattern.Confidence = "trufflehog-unverified" + findingsTr = append(findingsTr, finding) } } return findingsTr, nil @@ -231,9 +239,9 @@ func deduplicateFindings(totalFindings []Finding) []Finding { return dedupedFindings } -func DetectFileHits(content []byte, jobWebUrl string, jobName string, fileName string, archiveName string) { +func DetectFileHits(content []byte, jobWebUrl string, jobName string, fileName string, archiveName string, enableTruffleHogVerification bool) { // 1 goroutine to prevent maxThreads^2 which trashes memory - findings := DetectHits(content, 1) + findings := DetectHits(content, 1, enableTruffleHogVerification) for _, finding := range findings { baseLog := log.Warn().Str("confidence", finding.Pattern.Pattern.Confidence).Str("ruleName", finding.Pattern.Pattern.Name).Str("value", finding.Text).Str("url", jobWebUrl).Str("jobName", jobName).Str("file", fileName) if len(archiveName) > 0 { From abe32b6b7499f934b24a472ac378f7ffc9ccde95 Mon Sep 17 00:00:00 2001 From: frjcomp <107982661+frjcomp@users.noreply.github.com> Date: Wed, 18 Dec 2024 13:46:11 +0000 Subject: [PATCH 2/5] update --- src/pipeleak/go.mod | 6 +++--- src/pipeleak/go.sum | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/pipeleak/go.mod b/src/pipeleak/go.mod index 7d8071c..39c04c3 100644 --- a/src/pipeleak/go.mod +++ b/src/pipeleak/go.mod @@ -1,6 +1,6 @@ module github.com/CompassSecurity/pipeleak -go 1.23.1 +go 1.23.4 require ( atomicgo.dev/keyboard v0.2.9 @@ -12,7 +12,7 @@ require ( github.com/spf13/cobra v1.8.1 github.com/tidwall/gjson v1.18.0 github.com/trufflesecurity/trufflehog/v3 v3.86.1 - gitlab.com/gitlab-org/api/client-go v0.116.0 + gitlab.com/gitlab-org/api/client-go v0.117.0 gitlab.com/mitchenielsen/gitlab-ci-go v0.0.4 golift.io/xtractr v0.2.2 ) @@ -187,7 +187,7 @@ require ( golang.org/x/oauth2 v0.24.0 // indirect golang.org/x/sys v0.28.0 // indirect golang.org/x/time v0.8.0 // indirect - google.golang.org/protobuf v1.35.2 // indirect + google.golang.org/protobuf v1.36.0 // indirect gopkg.in/headzoo/surf.v1 v1.0.1 gopkg.in/yaml.v3 v3.0.1 ) diff --git a/src/pipeleak/go.sum b/src/pipeleak/go.sum index 3a01da9..3faf8c3 100644 --- a/src/pipeleak/go.sum +++ b/src/pipeleak/go.sum @@ -593,8 +593,8 @@ github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= -gitlab.com/gitlab-org/api/client-go v0.116.0 h1:Dy534gtZPMrnm3fAcmQRMadrcoUyFO4FQ4rXlSAdHAw= -gitlab.com/gitlab-org/api/client-go v0.116.0/go.mod h1:B29OfnZklmaoiR7uHANh9jTyfWEgmXvZLVEnosw2Dx0= +gitlab.com/gitlab-org/api/client-go v0.117.0 h1:HsbKxlTjVgfYmyCU+NRQk2G42RlMOKs6gF+/o0DL+TI= +gitlab.com/gitlab-org/api/client-go v0.117.0/go.mod h1:E+X2dndIYDuUfKVP0C3jhkWvTSE00BkLbCsXTY3edDo= gitlab.com/mitchenielsen/gitlab-ci-go v0.0.4 h1:0p3nMhTIkW03JJCBL8GhFTh40iDwON+uWM9eRcD4rvY= gitlab.com/mitchenielsen/gitlab-ci-go v0.0.4/go.mod h1:BOWEtlvgM79t1zjcAOER6o/KaftD4avnnShoqtTfGfw= go.einride.tech/aip v0.60.0 h1:h6bgabZ5BCfAptbGex8jbh3VvPBRLa6xq+pQ1CAjHYw= @@ -873,8 +873,8 @@ google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8 google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/grpc v1.67.2 h1:Lq11HW1nr5m4OYV+ZVy2BjOK78/zqnTx24vyDBP1JcQ= google.golang.org/grpc v1.67.2/go.mod h1:1gLDyUQU7CTLJI90u3nXZ9ekeghjeM7pTDZlqFNg2AA= -google.golang.org/protobuf v1.35.2 h1:8Ar7bF+apOIoThw1EdZl0p1oWvMqTHmpA2fRTyZO8io= -google.golang.org/protobuf v1.35.2/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +google.golang.org/protobuf v1.36.0 h1:mjIs9gYtt56AzC4ZaffQuh88TZurBGhIJMBZGSxNerQ= +google.golang.org/protobuf v1.36.0/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From f36be9d1c50a9ba13e7afdb6cc5b107a2416a592 Mon Sep 17 00:00:00 2001 From: frjcomp <107982661+frjcomp@users.noreply.github.com> Date: Wed, 18 Dec 2024 13:49:59 +0000 Subject: [PATCH 3/5] linter --- src/pipeleak/scanner/pipeline.go | 2 +- src/pipeleak/scanner/rules.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pipeleak/scanner/pipeline.go b/src/pipeleak/scanner/pipeline.go index d98e9f4..af677cb 100644 --- a/src/pipeleak/scanner/pipeline.go +++ b/src/pipeleak/scanner/pipeline.go @@ -51,7 +51,7 @@ func ScanGitLabPipelines(options *ScanOptions) { }) InitRules(options.ConfidenceFilter) - if options.TruffleHogVerification == false { + if !enableTruffleHogVerification { log.Info().Msg("TruffleHog verification is disabled") } diff --git a/src/pipeleak/scanner/rules.go b/src/pipeleak/scanner/rules.go index 182140f..27ce204 100644 --- a/src/pipeleak/scanner/rules.go +++ b/src/pipeleak/scanner/rules.go @@ -198,7 +198,7 @@ func DetectHits(text []byte, maxThreads int, enableTruffleHogVerification bool) } // if trufflehog verification is disabled all rules are reported - if enableTruffleHogVerification == false { + if !enableTruffleHogVerification { // trufflehog itself does not have confidence information finding.Pattern.Pattern.Confidence = "trufflehog-unverified" findingsTr = append(findingsTr, finding) From 946c85f569322ff984451d36b93eddc4bbece6f0 Mon Sep 17 00:00:00 2001 From: frjcomp <107982661+frjcomp@users.noreply.github.com> Date: Wed, 18 Dec 2024 13:54:11 +0000 Subject: [PATCH 4/5] linter again --- src/pipeleak/scanner/pipeline.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pipeleak/scanner/pipeline.go b/src/pipeleak/scanner/pipeline.go index af677cb..6c8775e 100644 --- a/src/pipeleak/scanner/pipeline.go +++ b/src/pipeleak/scanner/pipeline.go @@ -51,7 +51,7 @@ func ScanGitLabPipelines(options *ScanOptions) { }) InitRules(options.ConfidenceFilter) - if !enableTruffleHogVerification { + if !options.enableTruffleHogVerification { log.Info().Msg("TruffleHog verification is disabled") } From f37cd967f08eec264e1815e1695e3347be28b648 Mon Sep 17 00:00:00 2001 From: frjcomp <107982661+frjcomp@users.noreply.github.com> Date: Wed, 18 Dec 2024 13:58:20 +0000 Subject: [PATCH 5/5] linter again --- src/pipeleak/scanner/pipeline.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pipeleak/scanner/pipeline.go b/src/pipeleak/scanner/pipeline.go index 6c8775e..3ba6bd7 100644 --- a/src/pipeleak/scanner/pipeline.go +++ b/src/pipeleak/scanner/pipeline.go @@ -51,7 +51,7 @@ func ScanGitLabPipelines(options *ScanOptions) { }) InitRules(options.ConfidenceFilter) - if !options.enableTruffleHogVerification { + if !options.TruffleHogVerification { log.Info().Msg("TruffleHog verification is disabled") }