Automatically manage pre-release branches

.github/workflows/create-release-branch.yaml

@@ -8,6 +8,13 @@ on:
         required: false
         default: ""  # defaults to all matching branches (main and release branches)
         description: Run on which k8s-snap branches (space separated). If empty, it will run on all matching branches (main and release branches).
+  workflow_call:
+    inputs:
+      branches:
+        type: string
+        required: false
+        default: ""  # defaults to all matching branches (main and release branches)
+        description: Run on which k8s-snap branches (space separated). If empty, it will run on all matching branches (main and release branches).
 
 permissions:
   contents: read

.github/workflows/update-pre-release-branches.yaml

@@ -0,0 +1,81 @@
+name: Auto-update pre-release branches
+
+on:
+  pull_request:
+    paths:
+      - .github/workflows/update-pre-release-branches.yaml
+  schedule:
+    # Run 20 minutes after midnight, giving the k8s-snap nightly job
+    # enough time to pick up new k8s releases and setup the git branches.
+    - cron: "20 0 * * *"
+
+permissions:
+  contents: read
+
+jobs:
+  determine:
+    name: "Determine k8s pre-release"
+    runs-on: ubuntu-latest
+    outputs:
+      preRelease: ${{ steps.determine.outputs.preRelease }}
+      gitBranch: ${{ steps.determine.outputs.gitBranch }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          ssh-key: ${{ secrets.BOT_SSH_KEY }}
+      - name: Harden Runner
+        uses: step-security/harden-runner@v2
+        with:
+          egress-policy: audit
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+          cache: 'pip'
+      - name: Install Python dependencies
+        shell: bash
+        run: pip3 install -r ./scripts/requirements.txt
+      - name: Determine outstanding pre-release
+        id: determine
+        run: |
+          preRelease=`python3 ./scripts/k8s_release.py get_outstanding_prerelease`
+          echo "preRelease=$preRelease" >> "$GITHUB_OUTPUT"
+
+          if [[ -n "$preRelease" ]]; then
+            gitBranch=`python3 ./scripts/k8s_release.py get_prerelease_git_branch --prerelease $preRelease`
+          fi
+          echo "gitBranch=$gitBranch" >> "$GITHUB_OUTPUT"
+  handle-pre-release:
+    name: Handle pre-release ${{ needs.determine.outputs.preRelease }}
+    needs: [determine]
+    secrets: inherit
+    uses: ./.github/workflows/create-release-branch.yaml
+    if: ${{ needs.determine.outputs.preRelease }} != ''
+    with:
+      branches: ${{ needs.determine.outputs.gitBranch }}
+  clean-obsolete:
+    runs-on: ubuntu-latest
+    env:
+      CHARMCRAFT_AUTH : ${{ secrets.CHARMCRAFT_AUTH }}
+      LPCREDS_B64: ${{ secrets.LP_CREDS }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          ssh-key: ${{ secrets.BOT_SSH_KEY }}
+      - name: Harden Runner
+        uses: step-security/harden-runner@v2
+        with:
+          egress-policy: audit
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+          cache: 'pip'
+      - name: Install Python dependencies
+        shell: bash
+        run: pip3 install -r ./scripts/requirements.txt
+      - name: Clean obsolete branches
+        run: |
+          python3 ./scripts/k8s_release.py remove_obsolete_prereleases

scripts/ensure_snap_builds.py

@@ -49,7 +49,19 @@ def ensure_lp_recipe(
 
     recipe_name = util.recipe_name(flavour, ver, tip)
 
-    if tip:
+    if ver.prerelease:
+        if flavour != "classic":
+            raise Exception(
+                f"Unsupported pre-release flavour: {flavour}, only 'classic' "
+                "pre-releases are supported."
+            )
+        # Use a single branch for all pre-releases of a given risk level,
+        # e.g. v1.33.0-alpha.0 -> autoupdate/v1.33.0-alpha
+        prerelease = ver.prerelease.split(".")[0]
+        flavor_branch = (
+            f"autoupdate/v{ver.major}.{ver.minor}.{ver.patch}-{prerelease}"
+        )
+    elif tip:
         flavor_branch = "main" if flavour == "classic" else f"autoupdate/{flavour}"
     elif flavour == "classic":
         flavor_branch = f"release-{ver.major}.{ver.minor}"
@@ -101,6 +113,7 @@ def ensure_lp_recipe(
         LOG.info(" Creating LP recipe %s", recipe_name)
         params = dict(**manifest)
         params.pop("auto_build_channels")
+        LOG.info("Recipe manifest: %s", params)
         recipe = (not dry_run) and client.snaps.new(project=lp_project, **params)
 
     if recipe:
@@ -147,6 +160,12 @@ def prepare_track_builds(branch: str, args: argparse.Namespace):
         LOG.info("Current version detected %s", branch_ver)
         tip = branch == "main"
         for flavour in flavors:
+            if ver.prerelease and flavour != "classic":
+                LOG.info(
+                    f"Ignoring pre-release flavour: {flavour}, only 'classic' "
+                    "pre-releases are supported."
+                )
+                continue
             channels = ensure_snap_channels(flavour, ver, tip, args.dry_run)
             ensure_lp_recipe(flavour, ver, channels, tip, args.dry_run)
 

scripts/k8s_release.py

@@ -0,0 +1,140 @@
+#!/usr/bin/env python3
+
+import argparse
+import json
+import logging
+import re
+import subprocess
+from typing import List, Optional
+
+import requests
+from packaging.version import Version
+
+K8S_TAGS_URL = "https://api.github.com/repos/kubernetes/kubernetes/tags"
+EXEC_TIMEOUT = 60
+
+LOG = logging.getLogger(__name__)
+
+
+def _url_get(url: str) -> str:
+    r = requests.get(url, timeout=5)
+    r.raise_for_status()
+    return r.text
+
+
+def get_k8s_tags() -> List[str]:
+    """Retrieve semantically ordered k8s releases, newest to oldest."""
+    response = _url_get(K8S_TAGS_URL)
+    tags_json = json.loads(response)
+    if len(tags_json) == 0:
+        raise ValueError("No k8s tags retrieved.")
+    tag_names = [tag["name"] for tag in tags_json]
+    # Github already sorts the tags semantically but let's not rely on that.
+    tag_names.sort(key=lambda x: Version(x), reverse=True)
+    return tag_names
+
+
+# k8s release naming:
+# * alpha:  v{major}.{minor}.{patch}-alpha.{version}
+# * beta:   v{major}.{minor}.{patch}-beta.{version}
+# * rc:     v{major}.{minor}.{patch}-rc.{version}
+# * stable: v{major}.{minor}.{patch}
+def is_stable_release(release: str):
+    return "-" not in release
+
+
+def get_latest_stable() -> str:
+    k8s_tags = get_k8s_tags()
+    for tag in k8s_tags:
+        if is_stable_release(tag):
+            return tag
+    raise ValueError("Couldn't find stable release, received tags: %s" % k8s_tags)
+
+
+def get_latest_release() -> str:
+    k8s_tags = get_k8s_tags()
+    return k8s_tags[0]
+
+
+def get_outstanding_prerelease() -> Optional[str]:
+    latest_release = get_latest_release()
+    if not is_stable_release(latest_release):
+        return latest_release
+    # The latest release is a stable release, no outstanding pre-release.
+    return None
+
+
+def get_obsolete_prereleases() -> List[str]:
+    """Return obsolete K8s pre-releases.
+
+    We only keep the latest pre-release if there is no corresponding stable
+    release. All previous pre-releases are discarded.
+    """
+    k8s_tags = get_k8s_tags()
+    if not is_stable_release(k8s_tags[0]):
+        # Valid pre-release
+        k8s_tags = k8s_tags[1:]
+    # Discard all other pre-releases.
+    return [tag for tag in k8s_tags if not is_stable_release(tag)]
+
+
+def _exec(cmd: List[str], check=True, timeout=EXEC_TIMEOUT, cwd=None):
+    """Run the specified command and return the stdout/stderr output as a tuple."""
+    LOG.debug("Executing: %s, cwd: %s.", cmd, cwd)
+    proc = subprocess.run(
+        cmd, check=check, timeout=timeout, cwd=cwd, capture_output=True, text=True
+    )
+    return proc.stdout, proc.stderr
+
+
+def _branch_exists(
+    branch_name: str, remote=True, project_basedir: Optional[str] = None
+):
+    cmd = ["git", "branch"]
+    if remote:
+        cmd += ["-r"]
+
+    stdout, stderr = _exec(cmd, cwd=project_basedir)
+    return branch_name in stdout
+
+
+def get_prerelease_git_branch(prerelease: str):
+    """Retrieve the name of the k8s-snap git branch for a given k8s pre-release."""
+    prerelease_re = r"v\d+\.\d+\.\d-(?:alpha|beta|rc)\.\d+"
+    if not re.match(prerelease_re, prerelease):
+        raise ValueError("Unexpected k8s pre-release name: %s", prerelease)
+
+    # Use a single branch for all pre-releases of a given risk level,
+    # e.g. v1.33.0-alpha.0 -> autoupdate/v1.33.0-alpha
+    branch = f"autoupdate/{prerelease}"
+    return re.sub(r"(-[a-zA-Z]+)\.[0-9]+", r"\1", branch)
+
+
+def remove_obsolete_prereleases():
+    LOG.warning("TODO: not implemented.")
+
+
+if __name__ == "__main__":
+    logging.basicConfig(format="%(asctime)s %(message)s", level=logging.DEBUG)
+
+    parser = argparse.ArgumentParser()
+    subparsers = parser.add_subparsers(dest="subparser", required=True)
+
+    cmd = subparsers.add_parser("get_prerelease_git_branch")
+    cmd.add_argument(
+        "--prerelease",
+        dest="prerelease",
+        help="The upstream k8s pre-release.",
+    )
+
+    subparsers.add_parser("get_outstanding_prerelease")
+    subparsers.add_parser("remove_obsolete_prereleases")
+
+    kwargs = vars(parser.parse_args())
+    f = locals()[kwargs.pop("subparser")]
+    out = f(**kwargs)
+    if isinstance(out, (list, tuple)):
+        for item in out:
+            print(item)
+    else:
+        print(out or "")

scripts/promote_tracks.py

@@ -207,9 +207,14 @@ def sorter(info: Channel):
             chan_log.debug("Skipping promoting stable")
             continue
 
-        matched_pattern = next((pattern for pattern in ignored_tracks if re.fullmatch(pattern, track)), None)
+        matched_pattern = next(
+            (pattern for pattern in ignored_tracks if re.fullmatch(pattern, track)),
+            None,
+        )
         if matched_pattern:
-            chan_log.debug(f"Skipping ignored track '{track}' (matched pattern: '{matched_pattern}')")
+            chan_log.debug(
+                f"Skipping ignored track '{track}' (matched pattern: '{matched_pattern}')"
+            )
             continue
 
         if arch in ignored_arches:

scripts/requirements.txt

@@ -4,6 +4,8 @@ httplib2==0.22.0
 launchpadlib==1.11.0
 lazr-restfulclient==0.14.6
 lazr-uri==1.0.6
+packaging==24.2
+requests==2.32.3
 semver==3.0.2
 six==1.16.0
 tox==4.20.0

scripts/util/util.py

@@ -9,7 +9,9 @@
 LOG = logging.getLogger(__name__)
 SNAP_NAME: str = "k8s"
 SNAP_REPO: str = "https://github.com/canonical/k8s-snap.git/"
-TIP_BRANCH = re.compile(r"^(?:main)|^(?:release-\d+\.\d+)$")
+TIP_BRANCH = re.compile(
+    r"^(?:main)|^(?:release-\d+\.\d+)$|^(?:autoupdate\/v\d+\.\d+\.\d+-(?:alpha|beta|rc))$"
+)
 
 
 def flavors(dir: str) -> list[str]:

test_requirements.txt

@@ -1,2 +1,3 @@
 freezegun
 pytest
+types-requests

tests/unit/test_promote_tracks.py

@@ -130,6 +130,7 @@ def test_latest_track(risk, now):
         proposals = promote_tracks.create_proposal(args)
     assert proposals == [], "Latest track should not be promoted"
 
+
 @pytest.mark.parametrize(
     "track, ignored_patterns, expected_ignored",
     [
@@ -143,6 +144,6 @@ def test_ignored_tracks(track, ignored_patterns, expected_ignored):
     with _make_channel_map(track, "edge"):
         args.ignore_tracks = ignored_patterns
         proposals = promote_tracks.create_proposal(args)
-    assert (len(proposals) == 0) == expected_ignored, (
-        f"Track '{track}' should {'be ignored' if expected_ignored else 'not be ignored'}"
-        )
+    assert (
+        (len(proposals) == 0) == expected_ignored
+    ), f"Track '{track}' should {'be ignored' if expected_ignored else 'not be ignored'}"