run_tests.yml

name: Integration tests

on:
  pull_request:
    # Not running on "closed" - that is taken care of by "push" (if merged)
    types: [opened, synchronize, reopened]

# This cancels any previous job from the same PR if the PR has been updated.
# This cancel-in-progress only works per PR (thus, two different PRs wont be cancelled).
# Concurrency is not an issue because the self-hosted worker will anyways only run one
# job at a time from one repo.
concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  lint-test:
      name: Lint Helm chart
      runs-on: ubuntu-latest
      env:
        TEMPLATE_OUT_DIR: template_out
      steps:
        - name: Checkout main repo
          uses: actions/checkout@v4

        - run: |
            echo "GH ref:       ${{ github.ref }}"
            echo "GH ref_name:  ${{ github.ref_name }}"

        - name: Check copyright header
          run: ./.github/update_copyright.sh

        - name: Set up Helm
          uses: azure/setup-helm@v4
          with:
              version: '3.13.3'

        - run: helm lint --strict .
        - run: helm lint --strict --values values/dummy_lint.yaml .

        - name: Helm template
          if: always()
          run: |
            helm template . \
              --include-crds \
              --debug \
              --dry-run \
              --values values/dummy_lint.yaml \
              --output-dir $TEMPLATE_OUT_DIR

        - run: ls -l $TEMPLATE_OUT_DIR/**

        - uses: docker://ghcr.io/yannh/kubeconform:latest
          with:
            entrypoint: '/kubeconform'
            args: "-summary -strict -ignore-missing-schemas ${{ env.TEMPLATE_OUT_DIR }}"

        - name: Upload templates
          if: always()
          uses: actions/upload-artifact@v4
          with:
            name: templates
            path: ${{ env.TEMPLATE_OUT_DIR }}
            retention-days: 5

  benchmark-and-stability:
    needs: [lint-test]
    name: Benchmarking, scaling and stability
    if: github.repository == 'logicalclocks/rondb-helm'
    runs-on: [self-hosted, ARM64]
    env:
      K8S_NAMESPACE: rondb-helm-${{ github.run_id }}-${{ github.run_attempt }}
      RONDB_CLUSTER_NAME: my-rondb
    steps:
      - name: Checkout main repo
        uses: actions/checkout@v4

      - name: Check kubectl (should be Minikube)
        run: |
          kubectl version --client
          kubectl get nodes

      # Static CPU manager policy will only work if the FeatureGate is enabled
      - name: Create original RonDB cluster
        run: |
          helm install $RONDB_CLUSTER_NAME \
            --namespace=$K8S_NAMESPACE \
            --create-namespace \
            --values ./values/minikube/medium.yaml \
            --set benchmarking.enabled=true \
            --set staticCpuManagerPolicy=true \
            --set terminationGracePeriodSeconds=40 \
            --set meta.rdrs.statefulSet.endToEndTls.enabled=true \
            --set clusterSize.activeDataReplicas=1 \
            --set clusterSize.numNodeGroups=2 \
            --set clusterSize.minNumMySQLServers=1 \
            --set clusterSize.maxNumMySQLServers=2 \
            --set clusterSize.minNumRdrs=1 \
            --set clusterSize.maxNumRdrs=2 \
            .
          sleep 10

      - run: helm test -n $K8S_NAMESPACE $RONDB_CLUSTER_NAME --logs --filter name=generate-data

      - name: Waiting for benchmark job to complete
        run: bash .github/wait_job.sh $K8S_NAMESPACE benchs 400

      - name: Collect bench logs
        if: always()
        uses: ./.github/actions/collect_bench_logs
        with:
          namespace: ${{ env.K8S_NAMESPACE }}

      - name: Terminate RonDB data node
        run: kubectl -n $K8S_NAMESPACE delete pod node-group-0-0 --force

      - name: Test deploy stability
        shell: bash
        timeout-minutes: 6
        env:
          SLEEP_SECONDS: 10
          MIN_STABLE_MINUTES: 1
        run: bash .github/test_deploy_stability.sh
      - run: kubectl -n $K8S_NAMESPACE get pods

      # Check that data has not been lost
      - run: helm test -n $K8S_NAMESPACE $RONDB_CLUSTER_NAME --logs --filter name=verify-data
      - run: ./.github/delete_test_pods.sh $K8S_NAMESPACE

      # Avoid pro-longed stability check because of MySQLd HPA.
      # Can't scale down MySQLds because of config.ini.
      - name: Scale data node replicas
        run: |
          helm upgrade -i $RONDB_CLUSTER_NAME \
            --namespace=$K8S_NAMESPACE \
            --reuse-values \
            --set clusterSize.minNumMySQLServers=2 \
            --set clusterSize.activeDataReplicas=2 \
            .

      - name: Test deploy stability
        shell: bash
        timeout-minutes: 6
        env:
          SLEEP_SECONDS: 10
          MIN_STABLE_MINUTES: 1
        run: bash .github/test_deploy_stability.sh
      - run: kubectl -n $K8S_NAMESPACE get pods

      # Arbitrary ndbmtd change without changing the config.ini
      - name: Rolling upgrade of data nodes
        run: |
          helm upgrade -i $RONDB_CLUSTER_NAME \
            --namespace=$K8S_NAMESPACE \
            --reuse-values \
            --set meta.ndbmtd.statefulSet.podAnnotations.hello=world \
            .

      - name: Test deploy stability
        shell: bash
        timeout-minutes: 6
        env:
          SLEEP_SECONDS: 10
          MIN_STABLE_MINUTES: 1
        run: bash .github/test_deploy_stability.sh
      - run: kubectl -n $K8S_NAMESPACE get pods

      # Check again that data is still there
      - run: ./.github/delete_test_pods.sh $K8S_NAMESPACE
      - run: helm test -n $K8S_NAMESPACE $RONDB_CLUSTER_NAME --logs --filter name=verify-data

      - name: Collect logs
        if: always()
        uses: ./.github/actions/collect_logs
        with:
          namespace: ${{ env.K8S_NAMESPACE }}
          folder_name: k8s_logs

      - name: Remove cluster
        if: always()
        uses: ./.github/actions/remove_cluster
        timeout-minutes: 4
        with:
          namespace: ${{ env.K8S_NAMESPACE }}
          helm_chart: ${{ env.RONDB_CLUSTER_NAME }}

  test-lifecycle:
    needs: [benchmark-and-stability]
    name: Test Global Replication with backup/restore
    if: github.repository == 'logicalclocks/rondb-helm'
    runs-on: [self-hosted, ARM64]
    env:
      CLUSTER_A_NAME: cluster-a
      CLUSTER_B_NAME: cluster-b
      CLUSTER_C_NAME: cluster-c
      CLUSTER_D_NAME: cluster-d
    steps:
      - name: Checkout main repo
        uses: actions/checkout@v4

      - name: Check kubectl (should be Minikube)
        run: |
          kubectl version --client
          kubectl get nodes

      - name: Setup MinIO
        run: |
          ./test_scripts/setup_minio.sh

      # This script will also remove all clusters & namespaces
      - name: Run lifecycle test
        timeout-minutes: 20
        run: |
          ./test_scripts/lifecycle-test.sh \
            $CLUSTER_A_NAME \
            $CLUSTER_B_NAME \
            $CLUSTER_C_NAME \
            $CLUSTER_D_NAME

      - name: Collect logs
        if: failure()
        uses: ./.github/actions/collect_logs
        with:
          namespace: ${{ env.CLUSTER_A_NAME }}
          folder_name: ${{ env.CLUSTER_A_NAME }}

      - name: Remove cluster
        if: failure()
        uses: ./.github/actions/remove_cluster
        timeout-minutes: 4
        with:
          namespace: ${{ env.CLUSTER_A_NAME }}
          helm_chart: ${{ env.CLUSTER_A_NAME }}

      - name: Collect logs
        if: failure()
        uses: ./.github/actions/collect_logs
        with:
          namespace: ${{ env.CLUSTER_B_NAME }}
          folder_name: ${{ env.CLUSTER_B_NAME }}

      - name: Remove cluster
        if: failure()
        uses: ./.github/actions/remove_cluster
        timeout-minutes: 4
        with:
          namespace: ${{ env.CLUSTER_B_NAME }}
          helm_chart: ${{ env.CLUSTER_B_NAME }}

      - name: Collect logs
        if: failure()
        uses: ./.github/actions/collect_logs
        with:
          namespace: ${{ env.CLUSTER_C_NAME }}
          folder_name: ${{ env.CLUSTER_C_NAME }}

      - name: Remove cluster
        if: failure()
        uses: ./.github/actions/remove_cluster
        timeout-minutes: 4
        with:
          namespace: ${{ env.CLUSTER_C_NAME }}
          helm_chart: ${{ env.CLUSTER_C_NAME }}

      - name: Collect logs
        if: failure()
        uses: ./.github/actions/collect_logs
        with:
          namespace: ${{ env.CLUSTER_D_NAME }}
          folder_name: ${{ env.CLUSTER_D_NAME }}

      - name: Remove cluster
        if: failure()
        uses: ./.github/actions/remove_cluster
        timeout-minutes: 4
        with:
          namespace: ${{ env.CLUSTER_D_NAME }}
          helm_chart: ${{ env.CLUSTER_D_NAME }}

      - name: Delete MinIO tenant
        if: always()
        run: |
          source ./test_scripts/minio.env
          helm delete --namespace $MINIO_TENANT_NAMESPACE tenant

      - name: Delete MinIO namespace
        if: always()
        run: |
          source ./test_scripts/minio.env
          kubectl delete namespace $MINIO_TENANT_NAMESPACE --timeout=50s || true