diff --git a/src/pipeleak/go.mod b/src/pipeleak/go.mod index 785e8da..cbbbfda 100644 --- a/src/pipeleak/go.mod +++ b/src/pipeleak/go.mod @@ -16,6 +16,7 @@ require ( ) require ( + github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/google/go-querystring v1.1.0 // indirect diff --git a/src/pipeleak/go.sum b/src/pipeleak/go.sum index 8bcf710..632d072 100644 --- a/src/pipeleak/go.sum +++ b/src/pipeleak/go.sum @@ -1,3 +1,5 @@ +github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8= +github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= diff --git a/src/pipeleak/scanner/gitlab.go b/src/pipeleak/scanner/gitlab.go index c27141b..2f9bbda 100644 --- a/src/pipeleak/scanner/gitlab.go +++ b/src/pipeleak/scanner/gitlab.go @@ -105,9 +105,14 @@ jobOut: func getJobTrace(git *gitlab.Client, project *gitlab.Project, job *gitlab.Job) { reader, _, err := git.Jobs.GetTraceFile(project.ID, job.ID) if err != nil { - log.Error().Msg(err.Error()) + log.Error().Msg("Failed fetching job trace with: " + err.Error()) + return + } + trace, err := io.ReadAll(reader) + if err != nil { + log.Error().Msg("Failed reading trace reader into byte array: " + err.Error()) + return } - trace := StreamToString(reader) findings := DetectHits(trace) for _, finding := range findings { @@ -126,24 +131,26 @@ func getJobArtifacts(git *gitlab.Client, project *gitlab.Project, job *gitlab.Jo zipListing, err := zip.NewReader(artifactsReader, artifactsReader.Size()) if err != nil { log.Warn().Msg("Unable to unzip artifacts for proj " + strconv.Itoa(project.ID) + " job " + strconv.Itoa(job.ID)) - + return } for _, file := range zipListing.File { fc, err := file.Open() if err != nil { log.Error().Msg("Unable to openRaw artifact zip file: " + err.Error()) + break } content, err := io.ReadAll(fc) if err != nil { log.Error().Msg("Unable to readAll artifact zip file: " + err.Error()) + break } kind, _ := filetype.Match(content) // do not scan https://pkg.go.dev/github.com/h2non/filetype#readme-supported-types if kind == filetype.Unknown { - findings := DetectHits(string(content)) + findings := DetectHits(content) for _, finding := range findings { log.Warn().Msg("HIT Artifact Confidence: " + finding.Pattern.Pattern.Confidence + " Name:" + finding.Pattern.Pattern.Name + " Value: " + finding.Text + " " + job.WebURL + " in file: " + file.Name) } @@ -179,20 +186,21 @@ func StreamToString(stream io.Reader) string { _, err := buf.ReadFrom(stream) if err != nil { log.Error().Msg("Unable to read job trace buffer: " + err.Error()) + return "" } return buf.String() } // .env artifacts are not accessible over the API thus we must use session cookie and use the UI path // however this is where the treasure is - my precious -func DownloadEnvArtifact(cookieVal string, gitlabUrl string, prjectPath string, jobId int) string { +func DownloadEnvArtifact(cookieVal string, gitlabUrl string, prjectPath string, jobId int) []byte { dotenvUrl, _ := url.JoinPath(gitlabUrl, prjectPath, "/-/jobs/", strconv.Itoa(jobId), "/artifacts/download") req, err := http.NewRequest("GET", dotenvUrl, nil) if err != nil { log.Debug().Msg(err.Error()) - return "" + return []byte{} } q := req.URL.Query() @@ -205,7 +213,7 @@ func DownloadEnvArtifact(cookieVal string, gitlabUrl string, prjectPath string, resp, err := client.Do(req) if err != nil { log.Debug().Msg("Failed requesting dotenv artifact with: " + err.Error()) - return "" + return []byte{} } defer resp.Body.Close() @@ -213,12 +221,12 @@ func DownloadEnvArtifact(cookieVal string, gitlabUrl string, prjectPath string, // means no dotenv exists if statCode == 404 { - return "" + return []byte{} } if statCode != 200 { log.Error().Msg("Invalid _gitlab_session detected, HTTP " + strconv.Itoa(statCode)) - return "" + return []byte{} } else { log.Debug().Msg("Checking .env.gz artifact") } @@ -229,16 +237,16 @@ func DownloadEnvArtifact(cookieVal string, gitlabUrl string, prjectPath string, gzreader, e1 := gzip.NewReader(reader) if e1 != nil { log.Debug().Msg(err.Error()) - return "" + return []byte{} } envText, err := io.ReadAll(gzreader) if err != nil { log.Debug().Msg(err.Error()) - return "" + return []byte{} } - return string(envText) + return envText } func SessionValid(gitlabUrl string, cookieVal string) { @@ -247,6 +255,7 @@ func SessionValid(gitlabUrl string, cookieVal string) { req, err := http.NewRequest("GET", gitlabSessionsUrl, nil) if err != nil { log.Fatal().Msg("Failed GitLab sessions request with: " + err.Error()) + return } req.AddCookie(&http.Cookie{Name: "_gitlab_session", Value: cookieVal}) client := &http.Client{} diff --git a/src/pipeleak/scanner/rules.go b/src/pipeleak/scanner/rules.go index c0aae78..047b41c 100644 --- a/src/pipeleak/scanner/rules.go +++ b/src/pipeleak/scanner/rules.go @@ -6,7 +6,9 @@ import ( "net/http" "os" "regexp" + "strings" + "github.com/acarl005/stripansi" "github.com/rs/zerolog/log" "gopkg.in/yaml.v3" ) @@ -87,21 +89,51 @@ func GetRules() []PatternElement { return secretsPatterns.Patterns } -func DetectHits(target string) []Finding { +func DetectHits(text []byte) []Finding { findings := []Finding{} for _, pattern := range GetRules() { m := regexp.MustCompile(pattern.Pattern.Regex) - res := m.FindString(target) - - // truncate output to max 1024 chars for output readability - if len(res) > 1024 { - res = res[0:1024] - } - - if res != "" { - findings = append(findings, Finding{Pattern: pattern, Text: res}) + hits := m.FindAllIndex(text, -1) + + for _, hit := range hits { + // truncate output to max 1024 chars for output readability + hitStr := extractHitWithSurroundingText(text, hit, 50) + hitStr = cleanHitLine(hitStr) + if len(hitStr) > 1024 { + hitStr = hitStr[0:1024] + } + + if hitStr != "" { + findings = append(findings, Finding{Pattern: pattern, Text: hitStr}) + } } } return findings } + +func extractHitWithSurroundingText(text []byte, hitIndex []int, additionalBytes int) string { + startIndex := hitIndex[0] + endIndex := hitIndex[1] + + extendedStartIndex := startIndex - additionalBytes + if extendedStartIndex < 0 { + startIndex = 0 + } else { + startIndex = extendedStartIndex + } + + extendedEndIndex := endIndex + additionalBytes + if extendedEndIndex > len(text) { + endIndex = len(text) + } else { + endIndex = extendedEndIndex + } + + return string(text[startIndex:endIndex]) +} + +func cleanHitLine(text string) string { + text = strings.ReplaceAll(text, "\n", " ") + return stripansi.Strip(text) +}