From 4da52820220d0954f6dbaf31363776e94fae1e2d Mon Sep 17 00:00:00 2001 From: Amulyam24 Date: Tue, 8 Oct 2024 10:42:31 +0530 Subject: [PATCH] CI: check if PowerVS instances are cleaned up in workspace before creating a cluster --- test/e2e/e2e_test.go | 23 +++++++-- test/e2e/suite_test.go | 6 --- test/helpers/powervs.go | 109 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 128 insertions(+), 10 deletions(-) create mode 100644 test/helpers/powervs.go diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 56a5a4960..edf52def0 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -33,10 +33,19 @@ import ( "sigs.k8s.io/cluster-api/test/framework/clusterctl" "sigs.k8s.io/cluster-api/util" + "sigs.k8s.io/cluster-api-provider-ibmcloud/test/helpers" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) +const ( + // powervsRemediationFlavor represents the flavor of PowerVS cluster creation being tested. + powervsRemediationFlavor = "powervs-md-remediation" + kubernetesVersion = "KUBERNETES_VERSION" + serviceInstanceID = "IBMPOWERVS_SERVICE_INSTANCE_ID" +) + var _ = Describe("Workload cluster creation", func() { var ( ctx = context.TODO() @@ -55,7 +64,7 @@ var _ = Describe("Workload cluster creation", func() { Expect(bootstrapClusterProxy).ToNot(BeNil(), "Invalid argument. bootstrapClusterProxy can't be nil when calling %s spec", specName) Expect(os.MkdirAll(artifactFolder, 0750)).To(Succeed(), "Invalid argument. artifactFolder can't be created for %s spec", specName) - Expect(e2eConfig.Variables).To(HaveKey(KubernetesVersion)) + Expect(e2eConfig.Variables).To(HaveKey(kubernetesVersion)) clusterName = fmt.Sprintf("capibm-e2e-%s", util.RandomString(6)) @@ -70,6 +79,12 @@ var _ = Describe("Workload cluster creation", func() { // Path to the CNI file is defined in the config Expect(e2eConfig.Variables).To(HaveKey(capi_e2e.CNIPath), "Missing %s variable in the config", capi_e2e.CNIPath) cniPath = e2eConfig.GetVariable(capi_e2e.CNIPath) + + if flavor == powervsRemediationFlavor { + Expect(e2eConfig.Variables).To(HaveKey(serviceInstanceID)) + err := helpers.VerifyServerInstancesDeletion(e2eConfig.GetVariable(serviceInstanceID)) + Expect(err).To(BeNil()) + } }) AfterEach(func() { @@ -100,7 +115,7 @@ var _ = Describe("Workload cluster creation", func() { Flavor: flavor, Namespace: namespace.Name, ClusterName: clusterName, - KubernetesVersion: e2eConfig.GetVariable(KubernetesVersion), + KubernetesVersion: e2eConfig.GetVariable(kubernetesVersion), ControlPlaneMachineCount: ptr.To(int64(1)), WorkerMachineCount: ptr.To(int64(1)), }, @@ -121,7 +136,7 @@ var _ = Describe("Workload cluster creation", func() { Flavor: flavor, Namespace: namespace.Name, ClusterName: clusterName, - KubernetesVersion: e2eConfig.GetVariable(KubernetesVersion), + KubernetesVersion: e2eConfig.GetVariable(kubernetesVersion), ControlPlaneMachineCount: ptr.To(int64(1)), WorkerMachineCount: ptr.To(int64(3)), }, @@ -146,7 +161,7 @@ var _ = Describe("Workload cluster creation", func() { Flavor: flavor, Namespace: namespace.Name, ClusterName: clusterName, - KubernetesVersion: e2eConfig.GetVariable(KubernetesVersion), + KubernetesVersion: e2eConfig.GetVariable(kubernetesVersion), ControlPlaneMachineCount: ptr.To(int64(3)), WorkerMachineCount: ptr.To(int64(1)), }, diff --git a/test/e2e/suite_test.go b/test/e2e/suite_test.go index 07e617f46..2ed3a0ad3 100644 --- a/test/e2e/suite_test.go +++ b/test/e2e/suite_test.go @@ -43,12 +43,6 @@ import ( . "github.com/onsi/gomega" ) -const ( - KubernetesVersion = "KUBERNETES_VERSION" - CNIPath = "CNI" - CNIResources = "CNI_RESOURCES" -) - // Test suite flags. var ( // configPath is the path to the e2e config file. diff --git a/test/helpers/powervs.go b/test/helpers/powervs.go new file mode 100644 index 000000000..f1cc31a62 --- /dev/null +++ b/test/helpers/powervs.go @@ -0,0 +1,109 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package helpers + +import ( + "context" + "fmt" + "os" + "strings" + "time" + + "github.com/IBM-Cloud/power-go-client/clients/instance" + "github.com/IBM-Cloud/power-go-client/ibmpisession" + + "k8s.io/apimachinery/pkg/util/wait" + + "sigs.k8s.io/cluster-api-provider-ibmcloud/pkg/cloud/services/authenticator" + "sigs.k8s.io/cluster-api-provider-ibmcloud/pkg/cloud/services/utils" +) + +var ( + pollingInterval = time.Second * 30 + powerVSInstanceDeletionTimeout = time.Minute * 10 +) + +const ( + powervsInstanceDoesNotExist = "pvm-instance does not exist" + powervsInstanceNotFound = "could not be found" + powervsInstanceStateDeleting = "deleting" +) + +// VerifyServerInstancesDeletion checks if the virtual server instances are deleted +// before proceeding with the next cluster creation in the given PowerVS workspace. +func VerifyServerInstancesDeletion(serviceInstanceID string) error { + pclient, err := getPowerVSInstanceClient(serviceInstanceID) + if err != nil { + return err + } + + instances, err := pclient.GetAll() + if err != nil { + return err + } + + for _, ins := range instances.PvmInstances { + err = wait.PollUntilContextTimeout(context.Background(), pollingInterval, powerVSInstanceDeletionTimeout, false, func(_ context.Context) (done bool, err error) { + instance, err := pclient.Get(*ins.PvmInstanceID) + if err != nil { + if strings.Contains(err.Error(), powervsInstanceNotFound) || strings.Contains(err.Error(), powervsInstanceDoesNotExist) { + return true, nil + } + return false, err + } + + if instance.TaskState == powervsInstanceStateDeleting { + return false, nil + } + return false, nil + }) + if err != nil { + return err + } + } + return nil +} + +func getPowerVSInstanceClient(serviceInstanceID string) (*instance.IBMPIInstanceClient, error) { + auth, err := authenticator.GetAuthenticator() + if err != nil { + return nil, err + } + + zone := os.Getenv("IBMPOWERVS_ZONE") + if zone == "" { + return nil, fmt.Errorf("IBMPOWERVS_ZONE is not set") + } + + account, err := utils.GetAccount(auth) + if err != nil { + return nil, err + } + + piOptions := ibmpisession.IBMPIOptions{ + Authenticator: auth, + UserAccount: account, + Zone: zone, + Debug: true, + } + + session, err := ibmpisession.NewIBMPISession(&piOptions) + if err != nil { + return nil, err + } + return instance.NewIBMPIInstanceClient(context.Background(), session, serviceInstanceID), nil +}