more output, refactored hit extraction

src/pipeleak/go.mod

@@ -16,6 +16,7 @@ require (
 )
 
 require (
+	github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/google/go-querystring v1.1.0 // indirect

src/pipeleak/go.sum

@@ -1,3 +1,5 @@
+github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8=
+github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo=
 github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
 github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=

src/pipeleak/scanner/gitlab.go

@@ -105,9 +105,14 @@ jobOut:
 func getJobTrace(git *gitlab.Client, project *gitlab.Project, job *gitlab.Job) {
 	reader, _, err := git.Jobs.GetTraceFile(project.ID, job.ID)
 	if err != nil {
-		log.Error().Msg(err.Error())
+		log.Error().Msg("Failed fetching job trace with: " + err.Error())
+		return
+	}
+	trace, err := io.ReadAll(reader)
+	if err != nil {
+		log.Error().Msg("Failed reading trace reader into byte array: " + err.Error())
+		return
 	}
-	trace := StreamToString(reader)
 	findings := DetectHits(trace)
 
 	for _, finding := range findings {
@@ -126,24 +131,26 @@ func getJobArtifacts(git *gitlab.Client, project *gitlab.Project, job *gitlab.Jo
 	zipListing, err := zip.NewReader(artifactsReader, artifactsReader.Size())
 	if err != nil {
 		log.Warn().Msg("Unable to unzip artifacts for proj " + strconv.Itoa(project.ID) + " job " + strconv.Itoa(job.ID))
-
+		return
 	}
 
 	for _, file := range zipListing.File {
 		fc, err := file.Open()
 		if err != nil {
 			log.Error().Msg("Unable to openRaw artifact zip file: " + err.Error())
+			break
 		}
 
 		content, err := io.ReadAll(fc)
 		if err != nil {
 			log.Error().Msg("Unable to readAll artifact zip file: " + err.Error())
+			break
 		}
 
 		kind, _ := filetype.Match(content)
 		// do not scan https://pkg.go.dev/github.com/h2non/filetype#readme-supported-types
 		if kind == filetype.Unknown {
-			findings := DetectHits(string(content))
+			findings := DetectHits(content)
 			for _, finding := range findings {
 				log.Warn().Msg("HIT Artifact Confidence: " + finding.Pattern.Pattern.Confidence + " Name:" + finding.Pattern.Pattern.Name + " Value: " + finding.Text + " " + job.WebURL + " in file: " + file.Name)
 			}
@@ -179,20 +186,21 @@ func StreamToString(stream io.Reader) string {
 	_, err := buf.ReadFrom(stream)
 	if err != nil {
 		log.Error().Msg("Unable to read job trace buffer: " + err.Error())
+		return ""
 	}
 	return buf.String()
 }
 
 // .env artifacts are not accessible over the API thus we must use session cookie and use the UI path
 // however this is where the treasure is - my precious
-func DownloadEnvArtifact(cookieVal string, gitlabUrl string, prjectPath string, jobId int) string {
+func DownloadEnvArtifact(cookieVal string, gitlabUrl string, prjectPath string, jobId int) []byte {
 
 	dotenvUrl, _ := url.JoinPath(gitlabUrl, prjectPath, "/-/jobs/", strconv.Itoa(jobId), "/artifacts/download")
 
 	req, err := http.NewRequest("GET", dotenvUrl, nil)
 	if err != nil {
 		log.Debug().Msg(err.Error())
-		return ""
+		return []byte{}
 	}
 
 	q := req.URL.Query()
@@ -205,20 +213,20 @@ func DownloadEnvArtifact(cookieVal string, gitlabUrl string, prjectPath string,
 	resp, err := client.Do(req)
 	if err != nil {
 		log.Debug().Msg("Failed requesting dotenv artifact with: " + err.Error())
-		return ""
+		return []byte{}
 	}
 	defer resp.Body.Close()
 
 	statCode := resp.StatusCode
 
 	// means no dotenv exists
 	if statCode == 404 {
-		return ""
+		return []byte{}
 	}
 
 	if statCode != 200 {
 		log.Error().Msg("Invalid _gitlab_session detected, HTTP " + strconv.Itoa(statCode))
-		return ""
+		return []byte{}
 	} else {
 		log.Debug().Msg("Checking .env.gz artifact")
 	}
@@ -229,16 +237,16 @@ func DownloadEnvArtifact(cookieVal string, gitlabUrl string, prjectPath string,
 	gzreader, e1 := gzip.NewReader(reader)
 	if e1 != nil {
 		log.Debug().Msg(err.Error())
-		return ""
+		return []byte{}
 	}
 
 	envText, err := io.ReadAll(gzreader)
 	if err != nil {
 		log.Debug().Msg(err.Error())
-		return ""
+		return []byte{}
 	}
 
-	return string(envText)
+	return envText
 }
 
 func SessionValid(gitlabUrl string, cookieVal string) {
@@ -247,6 +255,7 @@ func SessionValid(gitlabUrl string, cookieVal string) {
 	req, err := http.NewRequest("GET", gitlabSessionsUrl, nil)
 	if err != nil {
 		log.Fatal().Msg("Failed GitLab sessions request with: " + err.Error())
+		return
 	}
 	req.AddCookie(&http.Cookie{Name: "_gitlab_session", Value: cookieVal})
 	client := &http.Client{}

src/pipeleak/scanner/rules.go

@@ -6,7 +6,9 @@ import (
 	"net/http"
 	"os"
 	"regexp"
+	"strings"
 
+	"github.com/acarl005/stripansi"
 	"github.com/rs/zerolog/log"
 	"gopkg.in/yaml.v3"
 )
@@ -87,21 +89,51 @@ func GetRules() []PatternElement {
 	return secretsPatterns.Patterns
 }
 
-func DetectHits(target string) []Finding {
+func DetectHits(text []byte) []Finding {
 	findings := []Finding{}
 	for _, pattern := range GetRules() {
 		m := regexp.MustCompile(pattern.Pattern.Regex)
-		res := m.FindString(target)
-
-		// truncate output to max 1024 chars for output readability
-		if len(res) > 1024 {
-			res = res[0:1024]
-		}
-
-		if res != "" {
-			findings = append(findings, Finding{Pattern: pattern, Text: res})
+		hits := m.FindAllIndex(text, -1)
+
+		for _, hit := range hits {
+			// truncate output to max 1024 chars for output readability
+			hitStr := extractHitWithSurroundingText(text, hit, 50)
+			hitStr = cleanHitLine(hitStr)
+			if len(hitStr) > 1024 {
+				hitStr = hitStr[0:1024]
+			}
+
+			if hitStr != "" {
+				findings = append(findings, Finding{Pattern: pattern, Text: hitStr})
+			}
 		}
 	}
 
 	return findings
 }
+
+func extractHitWithSurroundingText(text []byte, hitIndex []int, additionalBytes int) string {
+	startIndex := hitIndex[0]
+	endIndex := hitIndex[1]
+
+	extendedStartIndex := startIndex - additionalBytes
+	if extendedStartIndex < 0 {
+		startIndex = 0
+	} else {
+		startIndex = extendedStartIndex
+	}
+
+	extendedEndIndex := endIndex + additionalBytes
+	if extendedEndIndex > len(text) {
+		endIndex = len(text)
+	} else {
+		endIndex = extendedEndIndex
+	}
+
+	return string(text[startIndex:endIndex])
+}
+
+func cleanHitLine(text string) string {
+	text = strings.ReplaceAll(text, "\n", " ")
+	return stripansi.Strip(text)
+}