test: Add scheduling suspension e2e

Signed-off-by: Monokaix <[email protected]>
karmada-io · Jan 2, 2025 · 6fdb137 · 6fdb137
1 parent a891673
commit 6fdb137
Show file tree

Hide file tree

Showing 4 changed files with 191 additions and 0 deletions.
diff --git a/test/e2e/framework/clusterresourcebinding.go b/test/e2e/framework/clusterresourcebinding.go
@@ -18,7 +18,9 @@ package framework
 
 import (
 	"context"
+	"fmt"
 
+	"github.com/onsi/ginkgo/v2"
 	"github.com/onsi/gomega"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
@@ -36,3 +38,11 @@ func WaitClusterResourceBindingFitWith(client karmada.Interface, name string, fi
 		return fit(clusterResourceBinding)
 	}, pollTimeout, pollInterval).Should(gomega.Equal(true))
 }
+
+// RemoveClusterResourceBinding delete ClusterResourceBinding with karmada client.
+func RemoveClusterResourceBinding(client karmada.Interface, name string) {
+	ginkgo.By(fmt.Sprintf("Removing ClusterResourceBinding(%s)", name), func() {
+		err := client.WorkV1alpha2().ClusterResourceBindings().Delete(context.TODO(), name, metav1.DeleteOptions{})
+		gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
+	})
+}
diff --git a/test/e2e/framework/resourcebinding.go b/test/e2e/framework/resourcebinding.go
@@ -80,3 +80,11 @@ func WaitGracefulEvictionTasksDone(client karmada.Interface, namespace, name str
 		}, pollTimeout, pollInterval).ShouldNot(gomega.HaveOccurred())
 	})
 }
+
+// RemoveResourceBinding delete ResourceBinding with karmada client.
+func RemoveResourceBinding(client karmada.Interface, namespace, name string) {
+	ginkgo.By(fmt.Sprintf("Removing ResourceBinding(%s/%s)", namespace, name), func() {
+		err := client.WorkV1alpha2().ResourceBindings(namespace).Delete(context.TODO(), name, metav1.DeleteOptions{})
+		gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
+	})
+}
diff --git a/test/e2e/scheduling_test.go b/test/e2e/scheduling_test.go
@@ -696,6 +696,130 @@ var _ = ginkgo.Describe("[JobReplicaScheduling] JobReplicaSchedulingStrategy tes
 	})
 })
 
+var _ = ginkgo.Describe("[Suspension] ResourceBinding testing", func() {
+	var deploymentName string
+	var rb *workv1alpha2.ResourceBinding
+
+	ginkgo.BeforeEach(func() {
+		// Create deployment.
+		deploymentName = deploymentNamePrefix + rand.String(RandomStrLength)
+		deployment := helper.NewDeployment(testNamespace, deploymentName)
+		framework.CreateDeployment(kubeClient, deployment)
+		_, err := kubeClient.AppsV1().Deployments(testNamespace).Get(context.TODO(), deploymentName, metav1.GetOptions{})
+		gomega.Expect(err).NotTo(gomega.HaveOccurred(), "Failed to get deployment")
+
+		// Create ResourceBinding.
+		rb = helper.NewResourceBinding(deploymentName, workv1alpha2.ObjectReference{
+			APIVersion: deployment.APIVersion,
+			Kind:       deployment.Kind,
+			Namespace:  testNamespace,
+			Name:       deploymentName,
+		}, &policyv1alpha1.Placement{
+			ReplicaScheduling: &policyv1alpha1.ReplicaSchedulingStrategy{
+				ReplicaSchedulingType:     policyv1alpha1.ReplicaSchedulingTypeDivided,
+				ReplicaDivisionPreference: policyv1alpha1.ReplicaDivisionPreferenceWeighted,
+			},
+		}, true)
+		rb, err = karmadaClient.WorkV1alpha2().ResourceBindings(testNamespace).Create(context.TODO(), rb, metav1.CreateOptions{})
+		gomega.Expect(err).NotTo(gomega.HaveOccurred(), "Failed to create ResourceBinding")
+	})
+
+	ginkgo.AfterEach(func() {
+		framework.RemoveResourceBinding(karmadaClient, testNamespace, rb.Name)
+		framework.RemoveDeployment(kubeClient, testNamespace, deploymentName)
+	})
+
+	ginkgo.It("should not be scheduled when suspended and should be scheduled after resumed for resourceBinding", func() {
+		ginkgo.By("Waiting for the ResourceBinding not to be scheduled when suspended", func() {
+			gomega.Eventually(func() bool {
+				latestRB, err := karmadaClient.WorkV1alpha2().ResourceBindings(testNamespace).Get(context.TODO(), rb.Name, metav1.GetOptions{})
+				gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
+				return meta.IsStatusConditionTrue(latestRB.Status.Conditions, workv1alpha2.Scheduled)
+			}, pollTimeout, pollInterval).Should(gomega.BeFalse(), "ResourceBinding should not be scheduled when suspended")
+		})
+
+		ginkgo.By("Resuming the ResourceBinding", func() {
+			gomega.Eventually(func() bool {
+				newResourceBinding, err := karmadaClient.WorkV1alpha2().ResourceBindings(testNamespace).Get(context.TODO(), rb.Name, metav1.GetOptions{})
+				gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
+				newResourceBinding.Spec.Suspension = nil
+				_, err = karmadaClient.WorkV1alpha2().ResourceBindings(testNamespace).Update(context.TODO(), newResourceBinding, metav1.UpdateOptions{})
+				return err == nil
+			}, pollTimeout, pollInterval).Should(gomega.BeTrue(), "ResourceBinding should can be resumed")
+		})
+
+		ginkgo.By("Waiting for the ResourceBinding to be scheduled after resumed", func() {
+			gomega.Eventually(func() bool {
+				latestRB, err := karmadaClient.WorkV1alpha2().ResourceBindings(testNamespace).Get(context.TODO(), rb.Name, metav1.GetOptions{})
+				gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
+				return meta.IsStatusConditionTrue(latestRB.Status.Conditions, workv1alpha2.Scheduled)
+			}, pollTimeout, pollInterval).Should(gomega.BeTrue(), "ResourceBinding should be scheduled after resumed")
+		})
+	})
+})
+
+var _ = ginkgo.Describe("[Suspension] ClusterResourceBinding testing", func() {
+	var configMapName string
+	var crb *workv1alpha2.ClusterResourceBinding
+
+	ginkgo.BeforeEach(func() {
+		// Create ConfigMap.
+		configMapName = configMapNamePrefix + rand.String(RandomStrLength)
+		configMap := helper.NewConfigMap(testNamespace, configMapName, make(map[string]string))
+		framework.CreateConfigMap(kubeClient, configMap)
+		_, err := kubeClient.CoreV1().ConfigMaps(testNamespace).Get(context.TODO(), configMapName, metav1.GetOptions{})
+		gomega.Expect(err).NotTo(gomega.HaveOccurred(), "Failed to get ConfigMap")
+
+		// Create ClusterResourceBinding.
+		targetMember := framework.ClusterNames()[0]
+		crb = helper.NewClusterResourceBinding(configMapName, workv1alpha2.ObjectReference{
+			APIVersion: configMap.APIVersion,
+			Kind:       configMap.Kind,
+			Namespace:  testNamespace,
+			Name:       configMapName,
+		}, &policyv1alpha1.Placement{
+			ClusterAffinity: &policyv1alpha1.ClusterAffinity{
+				ClusterNames: []string{targetMember},
+			},
+		}, true)
+		crb, err = karmadaClient.WorkV1alpha2().ClusterResourceBindings().Create(context.TODO(), crb, metav1.CreateOptions{})
+		gomega.Expect(err).NotTo(gomega.HaveOccurred(), "Failed to create ClusterResourceBinding")
+	})
+
+	ginkgo.AfterEach(func() {
+		framework.RemoveClusterResourceBinding(karmadaClient, crb.Name)
+		framework.RemoveConfigMap(kubeClient, testNamespace, configMapName)
+	})
+
+	ginkgo.It("should not be scheduled when suspended and should be scheduled after resumed for clusterResourceBinding", func() {
+		ginkgo.By("Waiting for the ClusterResourceBinding not to be scheduled when suspended", func() {
+			gomega.Eventually(func() bool {
+				latestCRB, err := karmadaClient.WorkV1alpha2().ClusterResourceBindings().Get(context.TODO(), crb.Name, metav1.GetOptions{})
+				gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
+				return meta.IsStatusConditionTrue(latestCRB.Status.Conditions, workv1alpha2.Scheduled)
+			}, pollTimeout, pollInterval).Should(gomega.BeFalse(), "ClusterResourceBinding should not be scheduled when suspended")
+		})
+
+		ginkgo.By("Resuming the ClusterResourceBinding", func() {
+			gomega.Eventually(func() bool {
+				newClusterResourceBinding, err := karmadaClient.WorkV1alpha2().ClusterResourceBindings().Get(context.TODO(), crb.Name, metav1.GetOptions{})
+				gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
+				newClusterResourceBinding.Spec.Suspension = nil
+				_, err = karmadaClient.WorkV1alpha2().ClusterResourceBindings().Update(context.TODO(), newClusterResourceBinding, metav1.UpdateOptions{})
+				return err == nil
+			}, pollTimeout, pollInterval).Should(gomega.BeTrue(), "ClusterResourceBinding should can be resumed")
+		})
+
+		ginkgo.By("Waiting for the ClusterResourceBinding to be scheduled after resumed", func() {
+			gomega.Eventually(func() bool {
+				latestCRB, err := karmadaClient.WorkV1alpha2().ClusterResourceBindings().Get(context.TODO(), crb.Name, metav1.GetOptions{})
+				gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
+				return meta.IsStatusConditionTrue(latestCRB.Status.Conditions, workv1alpha2.Scheduled)
+			}, pollTimeout, pollInterval).Should(gomega.BeTrue(), "ClusterResourceBinding should be scheduled after resumed")
+		})
+	})
+})
+
 // get the resource binding associated with the workload
 func getResourceBinding(workload interface{}) (*workv1alpha2.ResourceBinding, error) {
 	obj, err := utilhelper.ToUnstructured(workload)

diff --git a/test/helper/resource.go b/test/helper/resource.go
@@ -33,14 +33,17 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/intstr"
+	uuid "k8s.io/apimachinery/pkg/util/uuid"
 	"k8s.io/utils/ptr"
 
 	workloadv1alpha1 "github.com/karmada-io/karmada/examples/customresourceinterpreter/apis/workload/v1alpha1"
 	appsv1alpha1 "github.com/karmada-io/karmada/pkg/apis/apps/v1alpha1"
 	autoscalingv1alpha1 "github.com/karmada-io/karmada/pkg/apis/autoscaling/v1alpha1"
 	clusterv1alpha1 "github.com/karmada-io/karmada/pkg/apis/cluster/v1alpha1"
 	networkingv1alpha1 "github.com/karmada-io/karmada/pkg/apis/networking/v1alpha1"
+	policyv1alpha1 "github.com/karmada-io/karmada/pkg/apis/policy/v1alpha1"
 	workv1alpha1 "github.com/karmada-io/karmada/pkg/apis/work/v1alpha1"
+	workv1alpha2 "github.com/karmada-io/karmada/pkg/apis/work/v1alpha2"
 )
 
 // These are different resource units.
@@ -993,3 +996,49 @@ func NewWorkloadRebalancer(name string, objectReferences []appsv1alpha1.ObjectRe
 		},
 	}
 }
+
+// NewResourceBinding will build a new resourceBinding object.
+func NewResourceBinding(name string, resource workv1alpha2.ObjectReference, placement *policyv1alpha1.Placement, suspended bool) *workv1alpha2.ResourceBinding {
+	rb := &workv1alpha2.ResourceBinding{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       workv1alpha2.ResourceKindResourceBinding,
+			APIVersion: workv1alpha2.GroupVersion.Version,
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name: name,
+			Labels: map[string]string{
+				policyv1alpha1.PropagationPolicyPermanentIDLabel: string(uuid.NewUUID()),
+			},
+		},
+		Spec: workv1alpha2.ResourceBindingSpec{
+			SchedulerName: "default-scheduler",
+			Resource:      resource,
+			Placement:     placement,
+			Suspension:    &workv1alpha2.Suspension{Scheduling: &suspended},
+		},
+	}
+	return rb
+}
+
+// NewClusterResourceBinding will build a new resourceBinding object.
+func NewClusterResourceBinding(name string, resource workv1alpha2.ObjectReference, placement *policyv1alpha1.Placement, suspended bool) *workv1alpha2.ClusterResourceBinding {
+	crb := &workv1alpha2.ClusterResourceBinding{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       workv1alpha2.ResourceKindClusterResourceBinding,
+			APIVersion: workv1alpha2.GroupVersion.Version,
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name: name,
+			Labels: map[string]string{
+				policyv1alpha1.PropagationPolicyPermanentIDLabel: string(uuid.NewUUID()),
+			},
+		},
+		Spec: workv1alpha2.ResourceBindingSpec{
+			SchedulerName: "default-scheduler",
+			Resource:      resource,
+			Placement:     placement,
+			Suspension:    &workv1alpha2.Suspension{Scheduling: &suspended},
+		},
+	}
+	return crb
+}