From e66e8b845dd5f0c75499d985fa196d7459bddf3e Mon Sep 17 00:00:00 2001 From: Iain Lane Date: Wed, 8 May 2024 13:18:06 +0100 Subject: [PATCH] build-push-to-dockerhub: Generate SBOM & provenance attestations This generates an SBOM and provenance attestations. These are also stored in one of GitHub's SigStore instances, so the provenance of images built using this action can be verified. We delete the `syft-sbom-report` action. This wasn't a usable action (no `action.yaml`) anyway. What we do here is based on [a comment][comment] from one of the GitHub attestation folks. We only generate the provenance attestation with GitHub directly, and use Docker to generate the SBOM and another provenance attestation about the Dockerish bits of the build. The GitHub attestation acts as a signature over all of that, and should handle multi-arch manifests properly, which isn't possible with the documented GitHub way that exists currently. [comment]: https://github.com/actions/attest-sbom/issues/60#issuecomment-2108917313 --- .vscode/launch.json | 20 ++ actions/build-push-to-dockerhub/README.md | 6 +- actions/build-push-to-dockerhub/action.yaml | 18 +- .../clean-up-container-registry/action.yml | 116 +++++++ actions/clean-up-container-registry/go.mod | 30 ++ actions/clean-up-container-registry/go.sum | 70 ++++ actions/clean-up-container-registry/main.go | 317 ++++++++++++++++++ actions/clean-up-container-registry/tags.sh | 77 +++++ actions/push-to-gar-docker/action.yaml | 50 +-- actions/syft-sbom-report/README.md | 23 -- actions/syft-sbom-report/syft-sbom.yml | 15 - 11 files changed, 677 insertions(+), 65 deletions(-) create mode 100644 .vscode/launch.json create mode 100644 actions/clean-up-container-registry/action.yml create mode 100644 actions/clean-up-container-registry/go.mod create mode 100644 actions/clean-up-container-registry/go.sum create mode 100644 actions/clean-up-container-registry/main.go create mode 100755 actions/clean-up-container-registry/tags.sh delete mode 100644 actions/syft-sbom-report/README.md delete mode 100644 actions/syft-sbom-report/syft-sbom.yml diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 000000000..5f7bd918a --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,20 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "Launch Package", + "type": "go", + "request": "launch", + "mode": "auto", + "program": "${fileDirname}", + "args": [ + "--image-name", + "us.gcr.io/kubernetes-dev/infra-build", + "--dry-run" + ] + } + ] +} diff --git a/actions/build-push-to-dockerhub/README.md b/actions/build-push-to-dockerhub/README.md index 4827dbb89..50a369c3d 100644 --- a/actions/build-push-to-dockerhub/README.md +++ b/actions/build-push-to-dockerhub/README.md @@ -1,6 +1,9 @@ # build-push-to-dockerhub -This is a composite GitHub Action, used to build Docker images and push them to DockerHub. +This is a composite GitHub Action, used to build Docker images and push them to +DockerHub. A SBOM and provenenace attestation will be generated too, and pushed +to the registry and the GitHub signature store. + It uses `get-vault-secrets` action to get the DockerHub username and password from Vault. Example of how to use this action in a repository: @@ -13,6 +16,7 @@ on: permissions: contents: read id-token: write + attestations: write jobs: build: diff --git a/actions/build-push-to-dockerhub/action.yaml b/actions/build-push-to-dockerhub/action.yaml index 4205a4fd1..58edb2df0 100644 --- a/actions/build-push-to-dockerhub/action.yaml +++ b/actions/build-push-to-dockerhub/action.yaml @@ -98,15 +98,21 @@ runs: tags: ${{ inputs.tags }} - name: Build and push Docker image + id: build-push uses: docker/build-push-action@4a13e500e55cf31b7a5d59a38ab2040ab0f42f56 # v5.1.0 with: + attests: type=provenance,mode=max context: ${{ inputs.context }} + file: ${{ inputs.file }} + labels: ${{ steps.meta.outputs.labels }} platforms: ${{ inputs.platforms }} + sbom: true push: ${{ inputs.push }} tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - file: ${{ inputs.file }} - build-args: ${{ inputs.build-args }} - target: ${{ inputs.target }} - cache-from: ${{ inputs.cache-from }} - cache-to: ${{ inputs.cache-to }} + + - name: Attest build provenance + uses: actions/attest-build-provenance@5e9cb68e95676991667494a6a4e59b8a2f13e1d0 # v1.3.3 + with: + push-to-registry: ${{ inputs.push }} + subject-digest: ${{ steps.build-push.outputs.digest }} + subject-name: index.docker.io/${{ inputs.repository }} diff --git a/actions/clean-up-container-registry/action.yml b/actions/clean-up-container-registry/action.yml new file mode 100644 index 000000000..371ca3a3b --- /dev/null +++ b/actions/clean-up-container-registry/action.yml @@ -0,0 +1,116 @@ +name: Cleanup Docker Registry + +description: Clean up Docker images in a registry. + +inputs: + image_name: + description: "The name of the Docker image to clean up, e.g., `ghcr.io/myorg/myimage`." + required: true + + tag_filter: + description: "Glob pattern to filter tags. Defaults to *." + required: false + default: "*" + + exclude_tags: + description: "Comma-separated list of tags to exclude from deletion." + required: false + default: "" + + keep_latest: + description: "Number of latest images to keep." + required: true + + dry_run: + description: "Run the action in dry-run mode (true/false)." + required: false + default: "true" + + crane_version: + description: "The version of crane to use." + required: false + default: "v0.19.2" + +runs: + using: "composite" + steps: + - name: Fetch crane + id: fetch-crane + uses: robinraju/release-downloader@c39a3b234af58f0cf85888573d361fb6fa281534 # v1.10 + with: + extract: false + fileName: go-containerregistry_Linux_x86_64.tar.gz + releaseId: ${{ inputs.crane_version }} + repository: google/go-containerregistry + + - name: Download SLSA Verifier + uses: slsa-framework/slsa-verifier/actions/installer@eb7007070baa04976cb9e25a0d8034f8db030a86 #v2.5.1 + + - name: Verify Crane SLSA provenance + shell: sh + run: | + slsa-verifier \ + verify-artifact \ + "${{ fromJSON(steps.fetch-crane.outputs.downloaded_files)[0] }}" \ + --provenance-path provenance.intoto.jsonl \ + --source-uri github.com/google/go-containerregistry \ + --source-tag "${{ inputs.crane_version }}" + + - name: Extract crane + shell: sh + run: | + tar -xzvf go-containerregistry_Linux_x86_64.tar.gz crane + chmod +x crane + mv crane /usr/local/bin/crane + + - name: List images + shell: sh + id: list_images + run: | + image_name="${{ inputs.image_name }}" + tag_filter="${{ inputs.tag_filter }}" + exclude_tags="${{ inputs.exclude_tags }}" + keep_latest="${{ inputs.keep_latest }}" + dry_run="${{ inputs.dry_run }}" + + # Convert exclude_tags to an array + IFS=',' read -r -a exclude_array <<< "$exclude_tags" + + # List all image tags using crane + tags=$(crane ls $image_name) + + # Convert tags to an array + tags_array=($tags) + + # Filter tags by the glob pattern + filtered_tags=() + for tag in "${tags_array[@]}"; do + if [[ $tag == $tag_filter ]]; then + filtered_tags+=("$tag") + fi + done + + # Exclude specified tags + for exclude in "${exclude_array[@]}"; do + filtered_tags=("${filtered_tags[@]/$exclude}") + done + + # Sort tags using version sort + sorted_tags=$(printf "%s\n" "${filtered_tags[@]}" | sort -V) + + # Determine tags to remove + tags_to_remove=($(echo "$sorted_tags" | head -n -"$keep_latest")) + + if [ "$dry_run" == "true" ]; then + echo "Dry-run mode: the following tags would be removed:" + printf "%s\n" "${tags_to_remove[@]}" + else + echo "Removing the following tags:" + printf "%s\n" "${tags_to_remove[@]}" + for tag in "${tags_to_remove[@]}"; do + crane delete $image_name:$tag + done + fi + + # Output removed tags + echo "::set-output name=removed_tags::$(IFS=,; echo "${tags_to_remove[*]}")" diff --git a/actions/clean-up-container-registry/go.mod b/actions/clean-up-container-registry/go.mod new file mode 100644 index 000000000..874537eb3 --- /dev/null +++ b/actions/clean-up-container-registry/go.mod @@ -0,0 +1,30 @@ +module github.com/grafana/shared-workflows/actions/clean-up-container-registry + +go 1.22.4 + +require ( + github.com/gobwas/glob v0.2.3 + github.com/google/go-containerregistry v0.19.2 + github.com/urfave/cli/v2 v2.27.2 + golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 + golang.org/x/sync v0.7.0 +) + +require ( + github.com/containerd/stargz-snapshotter/estargz v0.14.3 // indirect + github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect + github.com/docker/cli v24.0.0+incompatible // indirect + github.com/docker/distribution v2.8.2+incompatible // indirect + github.com/docker/docker v24.0.0+incompatible // indirect + github.com/docker/docker-credential-helpers v0.7.0 // indirect + github.com/klauspost/compress v1.16.5 // indirect + github.com/mitchellh/go-homedir v1.1.0 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.1.0-rc3 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/russross/blackfriday/v2 v2.1.0 // indirect + github.com/sirupsen/logrus v1.9.1 // indirect + github.com/vbatts/tar-split v0.11.3 // indirect + github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 // indirect + golang.org/x/sys v0.15.0 // indirect +) diff --git a/actions/clean-up-container-registry/go.sum b/actions/clean-up-container-registry/go.sum new file mode 100644 index 000000000..397cb10a2 --- /dev/null +++ b/actions/clean-up-container-registry/go.sum @@ -0,0 +1,70 @@ +github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= +github.com/containerd/stargz-snapshotter/estargz v0.14.3 h1:OqlDCK3ZVUO6C3B/5FSkDwbkEETK84kQgEeFwDC+62k= +github.com/containerd/stargz-snapshotter/estargz v0.14.3/go.mod h1:KY//uOCIkSuNAHhJogcZtrNHdKrA99/FCCRjE3HD36o= +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lVfRxZq4= +github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/docker/cli v24.0.0+incompatible h1:0+1VshNwBQzQAx9lOl+OYCTCEAD8fKs/qeXMx3O0wqM= +github.com/docker/cli v24.0.0+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8= +github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= +github.com/docker/docker v24.0.0+incompatible h1:z4bf8HvONXX9Tde5lGBMQ7yCJgNahmJumdrStZAbeY4= +github.com/docker/docker v24.0.0+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker-credential-helpers v0.7.0 h1:xtCHsjxogADNZcdv1pKUHXryefjlVRqWqIhk/uXJp0A= +github.com/docker/docker-credential-helpers v0.7.0/go.mod h1:rETQfLdHNT3foU5kuNkFR1R1V12OJRRO5lzt2D1b5X0= +github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= +github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-containerregistry v0.19.2 h1:TannFKE1QSajsP6hPWb5oJNgKe1IKjHukIKDUmvsV6w= +github.com/google/go-containerregistry v0.19.2/go.mod h1:YCMFNQeeXeLF+dnhhWkqDItx/JSkH01j1Kis4PsjzFI= +github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI= +github.com/klauspost/compress v1.16.5/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.1.0-rc3 h1:fzg1mXZFj8YdPeNkRXMg+zb88BFV0Ys52cJydRwBkb8= +github.com/opencontainers/image-spec v1.1.0-rc3/go.mod h1:X4pATf0uXsnn3g5aiGIsVnJBR4mxhKzfwmvK/B2NTm8= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/sirupsen/logrus v1.9.1 h1:Ou41VVR3nMWWmTiEUnj0OlsgOSCUFgsPAOl6jRIcVtQ= +github.com/sirupsen/logrus v1.9.1/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/urfave/cli v1.22.12/go.mod h1:sSBEIC79qR6OvcmsD4U3KABeOTxDqQtdDnaFuUN30b8= +github.com/urfave/cli/v2 v2.27.2 h1:6e0H+AkS+zDckwPCUrZkKX38mRaau4nL2uipkJpbkcI= +github.com/urfave/cli/v2 v2.27.2/go.mod h1:g0+79LmHHATl7DAcHO99smiR/T7uGLw84w8Y42x+4eM= +github.com/vbatts/tar-split v0.11.3 h1:hLFqsOLQ1SsppQNTMpkpPXClLDfC2A3Zgy9OUU+RVck= +github.com/vbatts/tar-split v0.11.3/go.mod h1:9QlHN18E+fEH7RdG+QAJJcuya3rqT7eXSTY7wGrAokY= +github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 h1:+qGGcbkzsfDQNPPe9UDgpxAWQrhbbBXOYJFQDq/dtJw= +github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913/go.mod h1:4aEEwZQutDLsQv2Deui4iYQ6DWTxR14g6m8Wv88+Xqk= +golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 h1:yixxcjnhBmY0nkL253HFVIm0JsFHwrHdT3Yh6szTnfY= +golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8/go.mod h1:jj3sYF3dwk5D+ghuXyeI3r5MFf+NT2An6/9dOA95KSI= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220906165534-d0df966e6959/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0= +gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= diff --git a/actions/clean-up-container-registry/main.go b/actions/clean-up-container-registry/main.go new file mode 100644 index 000000000..c222b75c3 --- /dev/null +++ b/actions/clean-up-container-registry/main.go @@ -0,0 +1,317 @@ +package main + +import ( + "fmt" + "log" + "os" + "slices" + "strings" + "sync" + "time" + + "github.com/gobwas/glob" + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/urfave/cli/v2" + "golang.org/x/sync/errgroup" +) + +type refDate struct { + ref name.Reference + date time.Time +} + +type keepReason int + +const ( + keepReasonExcluded keepReason = iota + keepReasonLatest +) + +func (kr keepReason) String() string { + switch kr { + case keepReasonExcluded: + return "excluded by filter" + case keepReasonLatest: + return "new enough" + default: + return "Unknown" + } +} + +type keptTag struct { + refDate + keepReason +} + +type repoSearchResult struct { + tagsToKeep []keptTag + tagsToRemove []refDate +} + +type Config struct { + ImageRef string + ImageRepo name.Repository + ExcludeTags []glob.Glob + TagFilter []glob.Glob + KeepLatest int + DryRun bool +} + +func buildGlobs(filters []string) ([]glob.Glob, error) { + globs := make([]glob.Glob, 0, len(filters)) + for _, filter := range filters { + g, err := glob.Compile(filter) + if err != nil { + return nil, err + } + globs = append(globs, g) + } + return globs, nil +} + +func main() { + config := Config{} + + app := &cli.App{ + Name: "docker-registry-cleanup", + Usage: "Clean up Docker images in a registry", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "image-name", + Usage: "The name of the Docker image to clean up", + Required: true, + Destination: &config.ImageRef, + Action: func(c *cli.Context, imageName string) error { + repo, err := name.NewRepository(imageName) + if err != nil { + return err + } + + config.ImageRepo = repo + + return nil + }, + }, + &cli.StringSliceFlag{ + Name: "exclude-tags", + Usage: "Tags to exclude from deletion", + Action: func(c *cli.Context, excludeTags []string) error { + globs, err := buildGlobs(excludeTags) + if err != nil { + return err + } + + config.ExcludeTags = globs + return nil + }, + }, + &cli.StringSliceFlag{ + Name: "tag-filter", + Usage: "Glob pattern to filter tags", + Action: func(c *cli.Context, tagFilter []string) error { + globs, err := buildGlobs(tagFilter) + if err != nil { + return err + } + + config.TagFilter = globs + return nil + }, + }, + &cli.IntFlag{ + Name: "keep-latest", + Usage: "Number of latest images to keep", + Value: 6, + Destination: &config.KeepLatest, + }, + &cli.BoolFlag{ + Name: "dry-run", + Usage: "Run the action in dry-run mode", + Value: true, + Destination: &config.DryRun, + }, + }, + Action: func(_ *cli.Context) error { + return run(config) + }, + } + + err := app.Run(os.Args) + if err != nil { + log.Fatal(err) + } +} + +func run(config Config) error { + imageName := config.ImageRef + imageRepo := config.ImageRepo + + excludeTags := config.ExcludeTags + tagFilters := config.TagFilter + keepLatest := config.KeepLatest + dryRun := config.DryRun + + remoteTags, err := listRemoteTags(imageRepo) + if err != nil { + return fmt.Errorf("failed to list tags for %s: %v", imageName, err) + } + fmt.Printf("Found %d tags for %s\n", len(remoteTags), imageName) + + tags, err := getTagsToRemove(remoteTags, tagFilters, excludeTags, keepLatest, imageRepo) + if err != nil { + return fmt.Errorf("failed to determine tags to remove: %v", err) + } + + if dryRun { + fmt.Printf("Dry run mode enabled. Would have removed the following tags:\n%s", tags) + return nil + } + + err = removeTags(tags) + if err != nil { + return err + } + + return nil +} + +func listRemoteTags(imageRepo name.Repository) ([]string, error) { + return remote.List(imageRepo, remote.WithAuthFromKeychain(authn.DefaultKeychain)) +} + +func (rs repoSearchResult) String() string { + var sb strings.Builder + sb.WriteString("Tags to remove:\n") + + for _, tag := range rs.tagsToRemove { + sb.WriteString(fmt.Sprintf(" - %s\n", tag.ref.Name())) + } + + sb.WriteString("\nTags to keep:\n") + for _, tag := range rs.tagsToKeep { + sb.WriteString(fmt.Sprintf(" - %s (%s)\n", tag.ref.Name(), tag.keepReason)) + } + + return sb.String() +} + +func removeTags(tags repoSearchResult) error { + for _, tag := range tags.tagsToRemove { + if err := remote.Delete(tag.ref, remote.WithAuthFromKeychain(authn.DefaultKeychain)); err != nil { + return fmt.Errorf("failed to delete tag %s: %v", tag.ref.Name(), err) + } + + fmt.Printf("Deleted tag %s...\n", tag.ref.Name()) + } + return nil +} + +func getTagsToRemove(allTags []string, tagFilters []glob.Glob, excludes []glob.Glob, keepLatest int, imageRepo name.Repository) (repoSearchResult, error) { + var ( + tagsToConsiderRemoving []refDate + tagsKept []keptTag + mu sync.Mutex + ) + + eg := errgroup.Group{} + eg.SetLimit(100) + + for _, tag := range allTags { + eg.Go(func() error { + fmt.Printf("Processing tag %s...\n", tag) + ref, err := name.ParseReference(fmt.Sprintf("%s:%s", imageRepo, tag)) + if err != nil { + return err + } + + if !matchesFilters(tag, tagFilters, true) { + return nil + } + + creationDate, err := getCreationDate(ref) + if err != nil { + return fmt.Errorf("failed to get creation date for %s: %v", ref.Name(), err) + } + + mu.Lock() + defer mu.Unlock() + + if matchesFilters(tag, excludes, false) { + tagsKept = append(tagsKept, keptTag{ + refDate: refDate{ref, creationDate}, + keepReason: keepReasonExcluded, + }) + return nil + } + + tagsToConsiderRemoving = append(tagsToConsiderRemoving, refDate{ref, creationDate}) + return nil + }) + } + + if err := eg.Wait(); err != nil { + return repoSearchResult{}, err + } + + if len(tagsToConsiderRemoving) <= keepLatest { + keepLatest = len(tagsToConsiderRemoving) + } + + slices.SortFunc(tagsToConsiderRemoving, func(l, r refDate) int { + return l.date.Compare(r.date) * -1 + }) + + slices.SortFunc(tagsKept, func(l, r keptTag) int { + return l.refDate.date.Compare(r.refDate.date) * -1 + }) + + tagsToRemove := tagsToConsiderRemoving[keepLatest:] + for _, tag := range tagsToConsiderRemoving[:keepLatest] { + tagsKept = append(tagsKept, keptTag{refDate: tag, keepReason: keepReasonLatest}) + } + + return repoSearchResult{ + tagsToKeep: tagsKept, + tagsToRemove: tagsToRemove, + }, nil +} + +func matchesFilters(input string, filters []glob.Glob, matchesIfNoFilters bool) bool { + if len(filters) == 0 { + return matchesIfNoFilters + } + + for _, filter := range filters { + if filter.Match(input) { + return true + } + } + + return false +} + +func getCreationDate(ref name.Reference) (time.Time, error) { + img, err := getImage(ref) + if err != nil { + return time.Time{}, err + } + + configFile, err := img.ConfigFile() + if err != nil { + return time.Time{}, err + } + + return configFile.Created.Time, nil +} + +func getImage(ref name.Reference) (v1.Image, error) { + img, err := remote.Image(ref, remote.WithAuthFromKeychain(authn.DefaultKeychain)) + if err != nil { + return nil, err + } + + return img, nil +} diff --git a/actions/clean-up-container-registry/tags.sh b/actions/clean-up-container-registry/tags.sh new file mode 100755 index 000000000..f37541ec3 --- /dev/null +++ b/actions/clean-up-container-registry/tags.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash + +set -eu +set -o pipefail + +readonly image_name=us.gcr.io/kubernetes-dev/infra-build +readonly exclude_tags="2024-06-17-*" +readonly tag_filter= +readonly keep_latest=5 +readonly dry_run="true" + +# List all image tags using crane +mapfile -t tags < <(crane ls "${image_name}") + +# Convert exclude_tags to an array +mapfile -t exclude_array <<< "${exclude_tags}" + +declare -A tags_to_consider_removing +declare -A tags_kept + +# Filter tags by the glob pattern, then sort them +for tag in "${tags[@]}"; do + # We want glob patching, that's the point! + # shellcheck disable=SC2053 + if [[ -z "${tag_filter}" ]] || [[ ${tag} == ${tag_filter} ]]; then + full_image_reference="${image_name}:${tag}" + + creation_date=$(crane config "${full_image_reference}" | jq -r '.created') + + for exclude in "${exclude_array[@]}"; do + if [[ $tag == ${exclude} ]]; then + tags_kept[${tag}]="${creation_date}" + continue 2 + fi + done + tags_to_consider_removing[${tag}]="${creation_date}" + fi +done + +# Sort tags by creation date +mapfile -t tags_to_consider_removing < <( + for tag in "${!tags_to_consider_removing[@]}"; do + echo "${tags_to_consider_removing["${tag}"]} ${tag}" + done | sort -k2 -r | awk '{print $1}' +) + +# Sort tags to keep by creation date +mapfile -t tags_kept < <( + for tag in "${!tags_kept[@]}"; do + echo "${tags_kept["${tag}"]} ${tag}" + done | sort -k2 -r | awk '{print $1}' +) + +# Determine tags to keep and tags to remove +tags_to_keep=("${tags_kept[@]: -${keep_latest}}") +tags_to_remove=("${tags_to_consider_removing[@]:${keep_latest}}") + +if [ "$dry_run" == "true" ]; then + echo "Dry-run mode: the following tags would be removed:" + for tag in "${tags_to_remove[@]}"; do + echo "- $tag" + done + + echo "...and the following tags would be kept:" + for tag in "${tags_to_keep[@]}"; do + echo "- $tag" + done +else + echo "Removing the following tags:" + printf "%s\n" "${tags_to_remove[@]}" + for tag in "${tags_to_remove[@]}"; do + crane delete "${image_name}:${tag}" + done +fi + +# Output removed tags +echo "::set-output name=removed_tags::$(IFS=,; echo "${tags_to_remove[*]}")" diff --git a/actions/push-to-gar-docker/action.yaml b/actions/push-to-gar-docker/action.yaml index 9fa523fa9..9ff9ace88 100644 --- a/actions/push-to-gar-docker/action.yaml +++ b/actions/push-to-gar-docker/action.yaml @@ -104,10 +104,23 @@ runs: ref: ${{ env.action_ref }} path: shared-workflows - - name: Get repository name - id: get-repository-name + - name: Set up variables + id: set-repo-variables shell: bash run: | + case "${{ inputs.environment }}" in + dev) + PROJECT="dev" + ;; + prod) + PROJECT="prod" + ;; + *) + echo "Invalid environment. Valid environments: dev, prod" + exit 1 + ;; + esac + REPO_NAME="${{ inputs.repository_name }}" if [ -z "$REPO_NAME" ]; then REPO_NAME="$(echo "${{ github.repository }}" | awk -F'/' '{print $2}')" @@ -117,19 +130,7 @@ runs: fi echo "repo_name=${REPO_NAME}" >> "${GITHUB_OUTPUT}" - - name: Resolve GCP project - id: resolve-project - shell: bash - run: | - if [[ "${{ inputs.environment }}" == "dev" ]]; then - PROJECT="grafanalabs-dev" - elif [[ "${{ inputs.environment }}" == "prod" ]]; then - PROJECT="grafanalabs-global" - else - echo "Invalid environment. Valid environment variable inputs: dev, prod" - exit 1 - fi - echo "project=${PROJECT}" | tee -a ${GITHUB_OUTPUT} + echo "image=${{ inputs.registry }}/${{ steps.resolve-project.outputs.project }}/docker-${REPO_NAME}-${PROJECT}/${{ inputs.image_name }}" >> "${GITHUB_OUTPUT}" - name: Login to GAR uses: ./shared-workflows/actions/login-to-gar @@ -140,7 +141,7 @@ runs: id: meta uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v5.5.1 with: - images: "${{ inputs.registry }}/${{ steps.resolve-project.outputs.project }}/docker-${{ steps.get-repository-name.outputs.repo_name }}-${{ inputs.environment }}/${{ inputs.image_name }}" + images: ${{ steps.set-repo-variables.outputs.image }} tags: ${{ inputs.tags }} - name: Set up Docker Buildx @@ -152,13 +153,22 @@ runs: uses: docker/build-push-action@4a13e500e55cf31b7a5d59a38ab2040ab0f42f56 # v5.1.0 id: build with: - context: ${{ inputs.context }} + attests: type=provenance,mode=max build-args: ${{ inputs.build-args }} - push: ${{ inputs.push == 'true' }} - tags: ${{ steps.meta.outputs.tags }} + build-contexts: ${{ inputs.build-contexts }} cache-from: ${{ inputs.cache-from }} cache-to: ${{ inputs.cache-to }} + context: ${{ inputs.context }} file: ${{ inputs.file }} platforms: ${{ inputs.platforms }} + push: ${{ inputs.push == 'true' }} + sbom: true ssh: ${{ inputs.ssh }} - build-contexts: ${{ inputs.build-contexts }} + tags: ${{ steps.meta.outputs.tags }} + + - name: Attest build provenance + uses: actions/attest-build-provenance@5e9cb68e95676991667494a6a4e59b8a2f13e1d0 # v1.3.3 + with: + push-to-registry: ${{ inputs.push }} + subject-digest: ${{ steps.build-push.outputs.digest }} + subject-name: ${{ steps.set-repo-variables.outputs.image }} diff --git a/actions/syft-sbom-report/README.md b/actions/syft-sbom-report/README.md deleted file mode 100644 index 3d7a42cdd..000000000 --- a/actions/syft-sbom-report/README.md +++ /dev/null @@ -1,23 +0,0 @@ -# syft-sbom-report - -Generate an SPDX SBOM Report and attached to Release Artifcats on Release Publish - -Example workflow: - -```yaml -name: syft-sbom-ci -on: - release: - types: [published] - workflow_dispatch: -jobs: - syft-sbom: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Anchore SBOM Action - uses: anchore/sbom-action@v0.15.10 - with: - artifact-name: ${{ github.event.repository.name }}-spdx.json -``` diff --git a/actions/syft-sbom-report/syft-sbom.yml b/actions/syft-sbom-report/syft-sbom.yml deleted file mode 100644 index 298fdb066..000000000 --- a/actions/syft-sbom-report/syft-sbom.yml +++ /dev/null @@ -1,15 +0,0 @@ -name: syft-sbom-ci -on: - release: - types: [published] - workflow_dispatch: -jobs: - syft-sbom: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4.0.0 - - name: Anchore SBOM Action - uses: anchore/sbom-action@ab5d7b5f48981941c4c5d6bf33aeb98fe3bae38c # 0.15.10 - with: - artifact-name: ${{ github.event.repository.name }}-spdx.json