limit nr of jobs

readme.md

@@ -36,6 +36,8 @@ Filter the scanned projects by using the `--search` flag and provide a search qu
 
 Filter the scanned projects by using the `--owned` flag to only process projects owned by you.
 
+Limit the scanned nr of jobs by using the `--job-limit` flag.
+
 ## Customizing Scan Rules
 
 When you run Pipeleak for the first time, it generates a `rules.yml` file based on [this repository](https://github.com/mazen160/secrets-patterns-db/blob/master/db/rules-stable.yml). You can customize your scan rules by modifying this file as needed.

src/pipeleak/cmd/scan.go

@@ -17,6 +17,7 @@ var (
 	projectSearchQuery string
 	artifacts          bool
 	owned              bool
+	jobLimit           int
 	verbose            bool
 )
 
@@ -45,6 +46,7 @@ func NewScanCmd() *cobra.Command {
 
 	scanCmd.PersistentFlags().BoolVarP(&artifacts, "artifacts", "a", false, "Scan Job Artifacts")
 	scanCmd.PersistentFlags().BoolVarP(&owned, "owned", "o", false, "Scan Onwed Projects Only")
+	scanCmd.PersistentFlags().IntVarP(&jobLimit, "job-limit", "j", 0, "Scan a max number of pipeline jobs - trade speed vs coverage. 0 scans all and is the default.")
 
 	scanCmd.PersistentFlags().BoolVarP(&verbose, "verbose", "v", false, "Verbose Logging")
 
@@ -60,7 +62,7 @@ func Scan(cmd *cobra.Command, args []string) {
 		os.Exit(1)
 	}
 
-	scanner.ScanGitLabPipelines(gitlabUrl, gitlabApiToken, gitlabCookie, artifacts, owned, projectSearchQuery)
+	scanner.ScanGitLabPipelines(gitlabUrl, gitlabApiToken, gitlabCookie, artifacts, owned, projectSearchQuery, jobLimit)
 }
 
 func setLogLevel() {

src/pipeleak/scanner/gitlab.go

@@ -14,7 +14,7 @@ import (
 	"github.com/xanzy/go-gitlab"
 )
 
-func ScanGitLabPipelines(gitlabUrl string, apiToken string, cookie string, scanArtifacts bool, scanOwnedOnly bool, query string) {
+func ScanGitLabPipelines(gitlabUrl string, apiToken string, cookie string, scanArtifacts bool, scanOwnedOnly bool, query string, jobLimit int) {
 	log.Info().Msg("Fetching projects")
 	git, err := gitlab.NewClient(apiToken, gitlab.WithBaseURL(gitlabUrl))
 	if err != nil {
@@ -49,7 +49,7 @@ func ScanGitLabPipelines(gitlabUrl string, apiToken string, cookie string, scanA
 
 		for _, project := range projects {
 			log.Debug().Msg("Scan Project jobs: " + project.Name)
-			getAllJobs(git, project, scanArtifacts, cookie, gitlabUrl)
+			getAllJobs(git, project, scanArtifacts, cookie, gitlabUrl, jobLimit)
 		}
 
 		if resp.NextPage == 0 {
@@ -60,7 +60,7 @@ func ScanGitLabPipelines(gitlabUrl string, apiToken string, cookie string, scanA
 	}
 }
 
-func getAllJobs(git *gitlab.Client, project *gitlab.Project, scanArtifacts bool, cookie string, gitlabUrl string) {
+func getAllJobs(git *gitlab.Client, project *gitlab.Project, scanArtifacts bool, cookie string, gitlabUrl string, jobLimit int) {
 
 	opts := &gitlab.ListJobsOptions{
 		ListOptions: gitlab.ListOptions{
@@ -69,6 +69,9 @@ func getAllJobs(git *gitlab.Client, project *gitlab.Project, scanArtifacts bool,
 		},
 	}
 
+	currentJobCtr := 0
+
+jobOut:
 	for {
 		jobs, resp, err := git.Jobs.ListProjectJobs(project.ID, opts)
 
@@ -77,11 +80,17 @@ func getAllJobs(git *gitlab.Client, project *gitlab.Project, scanArtifacts bool,
 		}
 
 		for _, job := range jobs {
+			currentJobCtr += 1
 			getJobTrace(git, project, job)
 
 			if scanArtifacts {
 				getJobArtifacts(git, project, job, cookie, gitlabUrl)
 			}
+
+			if jobLimit > 0 && currentJobCtr >= jobLimit {
+				log.Debug().Msg("Skipping jobs as job-limit is reached")
+				break jobOut
+			}
 		}
 
 		if resp.NextPage == 0 {
@@ -252,6 +261,6 @@ func SessionValid(gitlabUrl string, cookieVal string) {
 	if statCode != 200 {
 		log.Fatal().Msg("Negative _gitlab_session test, HTTP " + strconv.Itoa(statCode))
 	} else {
-		log.Info().Msg("Provided GitLab Session is valid")
+		log.Info().Msg("Provided GitLab session cookie is valid")
 	}
 }