update VolumeGroupSnapshot to v1beta1 (backport #5021)

build.env

@@ -30,7 +30,7 @@ GOLANGCI_VERSION=v1.57.2
 
 # external snapshotter version
 # Refer: https://github.com/kubernetes-csi/external-snapshotter/releases
-SNAPSHOT_VERSION=v8.0.1
+SNAPSHOT_VERSION=v8.2.0
 
 # "go test" configuration
 # set to stdout or html to enable coverage reporting, disabled by default
@@ -55,7 +55,7 @@ ROOK_CEPH_CLUSTER_IMAGE=quay.io/ceph/ceph:v19
 
 # CSI sidecar version
 CSI_ATTACHER_VERSION=v4.6.1
-CSI_SNAPSHOTTER_VERSION=v8.0.1
+CSI_SNAPSHOTTER_VERSION=v8.2.0
 CSI_RESIZER_VERSION=v1.11.1
 CSI_PROVISIONER_VERSION=v5.0.1
 CSI_NODE_DRIVER_REGISTRAR_VERSION=v2.11.1

charts/ceph-csi-cephfs/README.md

@@ -189,7 +189,7 @@ charts and their default values.
 | `provisioner.resizer.args.httpEndpointPort`        | Specifies http server port for diagnostics, health checks and metrics                                                                                    | `""`                                               |
 | `provisioner.resizer.extraArgs`                | Specifies extra arguments for the resizer sidecar                                                                                                    | `[]`                                               |
 | `provisioner.snapshotter.image.repository`     | Specifies the csi-snapshotter image repository URL                                                                                                   | `registry.k8s.io/sig-storage/csi-snapshotter`      |
-| `provisioner.snapshotter.image.tag`            | Specifies image tag                                                                                                                                  | `v8.0.1`                                           |
+| `provisioner.snapshotter.image.tag`            | Specifies image tag                                                                                                                                  | `v8.2.0`                                           |
 | `provisioner.snapshotter.image.pullPolicy`     | Specifies pull policy                                                                                                                                | `IfNotPresent`                                     |
 | `provisioner.snapshotter.args.enableVolumeGroupSnapshots`  | enables the creation of volume group snapshots                                                                                           | `false`                                            |
 | `provisioner.snapshotter.args.httpEndpointPort`    | Specifies http server port for diagnostics, health checks and metrics                                                                                    | `""`                                               |

charts/ceph-csi-cephfs/templates/provisioner-deployment.yaml

@@ -177,7 +177,7 @@ spec:
             - "--timeout={{ .Values.provisioner.timeout }}"
             - "--leader-election=true"
             - "--extra-create-metadata=true"
-            - "--enable-volume-group-snapshots={{.Values.provisioner.snapshotter.args.enableVolumeGroupSnapshots }}"
+            - "--feature-gates=CSIVolumeGroupSnapshot={{.Values.provisioner.snapshotter.args.enableVolumeGroupSnapshots }}"
 {{- if and .Values.provisioner.snapshotter.args .Values.provisioner.snapshotter.args.httpEndpointPort }}
             - "--http-endpoint=$(POD_IP):{{ .Values.provisioner.snapshotter.args.httpEndpointPort }}"
 {{- end }}

charts/ceph-csi-cephfs/values.yaml

@@ -264,7 +264,7 @@ provisioner:
   snapshotter:
     image:
       repository: registry.k8s.io/sig-storage/csi-snapshotter
-      tag: v8.0.1
+      tag: v8.2.0
       pullPolicy: IfNotPresent
     resources: {}
     args:

charts/ceph-csi-rbd/README.md

@@ -176,7 +176,7 @@ charts and their default values.
 | `provisioner.resizer.args.httpEndpointPort`        | Specifies http server port for diagnostics, health checks and metrics                                                                                    | `""`                                               |
 | `provisioner.resizer.extraArgs`                | Specifies extra arguments for the resizer sidecar                                                                                                    | `[]`                                               |
 | `provisioner.snapshotter.image.repository`     | Specifies the csi-snapshotter image repository URL                                                                                                   | `registry.k8s.io/sig-storage/csi-snapshotter`          |
-| `provisioner.snapshotter.image.tag`            | Specifies image tag                                                                                                                                  | `v8.0.1`                                           |
+| `provisioner.snapshotter.image.tag`            | Specifies image tag                                                                                                                                  | `v8.2.0`                                           |
 | `provisioner.snapshotter.image.pullPolicy`     | Specifies pull policy                                                                                                                                | `IfNotPresent`                                     |
 | `provisioner.snapshotter.args.enableVolumeGroupSnapshots` | enables the creation of volume group snapshots                                                                                            | `false`                                            |
 | `provisioner.snapshotter.args.httpEndpointPort`    | Specifies http server port for diagnostics, health checks and metrics                                                                                    | `""`                                               |

charts/ceph-csi-rbd/templates/provisioner-deployment.yaml

@@ -228,7 +228,7 @@ spec:
             - "--timeout={{ .Values.provisioner.timeout }}"
             - "--leader-election=true"
             - "--extra-create-metadata=true"
-            - "--enable-volume-group-snapshots={{.Values.provisioner.snapshotter.args.enableVolumeGroupSnapshots }}"
+            - "--feature-gates=CSIVolumeGroupSnapshot={{.Values.provisioner.snapshotter.args.enableVolumeGroupSnapshots }}"
 {{- if and .Values.provisioner.snapshotter.args .Values.provisioner.snapshotter.args.httpEndpointPort }}
             - "--http-endpoint=$(POD_IP):{{ .Values.provisioner.snapshotter.args.httpEndpointPort }}"
 {{- end }}

charts/ceph-csi-rbd/values.yaml

@@ -302,7 +302,7 @@ provisioner:
   snapshotter:
     image:
       repository: registry.k8s.io/sig-storage/csi-snapshotter
-      tag: v8.0.1
+      tag: v8.2.0
       pullPolicy: IfNotPresent
     resources: {}
     args:

deploy/cephfs/kubernetes/csi-cephfsplugin-provisioner.yaml

@@ -144,14 +144,14 @@ spec:
             - name: socket-dir
               mountPath: /csi
         - name: csi-snapshotter
-          image: registry.k8s.io/sig-storage/csi-snapshotter:v8.0.1
+          image: registry.k8s.io/sig-storage/csi-snapshotter:v8.2.0
           args:
             - "--csi-address=$(ADDRESS)"
             - "--v=1"
             - "--timeout=150s"
             - "--leader-election=true"
             - "--extra-create-metadata=true"
-            - "--enable-volume-group-snapshots=true"
+            - "--feature-gates=CSIVolumeGroupSnapshot=true"
             - "--http-endpoint=$(POD_IP):8092"
           env:
             - name: ADDRESS

deploy/nfs/kubernetes/csi-nfsplugin-provisioner.yaml

@@ -124,7 +124,7 @@ spec:
             - name: socket-dir
               mountPath: /csi
         - name: csi-snapshotter
-          image: registry.k8s.io/sig-storage/csi-snapshotter:v8.0.1
+          image: registry.k8s.io/sig-storage/csi-snapshotter:v8.2.0
           args:
             - "--csi-address=$(ADDRESS)"
             - "--v=1"

deploy/rbd/kubernetes/csi-rbdplugin-provisioner.yaml

@@ -133,14 +133,14 @@ spec:
             - name: socket-dir
               mountPath: /csi
         - name: csi-snapshotter
-          image: registry.k8s.io/sig-storage/csi-snapshotter:v8.0.1
+          image: registry.k8s.io/sig-storage/csi-snapshotter:v8.2.0
           args:
             - "--csi-address=$(ADDRESS)"
             - "--v=1"
             - "--timeout=150s"
             - "--leader-election=true"
             - "--extra-create-metadata=true"
-            - "--enable-volume-group-snapshots=true"
+            - "--feature-gates=CSIVolumeGroupSnapshot=true"
             - "--http-endpoint=$(POD_IP):8092"
           env:
             - name: ADDRESS

e2e/deployment.go

@@ -252,6 +252,9 @@ func (yrn *yamlResourceNamespaced) Do(action kubectlAction) error {
 
 	data = replaceLogLevelInTemplate(data)
 
+	// disable VGS alpha feature, TODO: remove this in next release (3.14.0)
+	data = disableVGSAlphaCLIArg(data)
+
 	if yrn.oneReplica {
 		data = oneReplicaDeployYaml(data)
 	}

e2e/utils.go

@@ -820,6 +820,10 @@ func checkMountOptions(pvcPath, appPath string, f *framework.Framework, mountFla
 	return err
 }
 
+func disableVGSAlphaCLIArg(template string) string {
+	return strings.ReplaceAll(template, "- \"--enable-volume-group-snapshots=true\"", "")
+}
+
 func addTopologyDomainsToDSYaml(template, labels string) string {
 	return strings.ReplaceAll(template, "# - \"--domainlabels=failure-domain/region,failure-domain/zone\"",
 		"- \"--domainlabels="+labels+"\"")

e2e/volumegroupsnapshot.go

@@ -18,9 +18,10 @@ package e2e
 
 import (
 	"context"
+	"crypto/sha256"
 	"fmt"
 
-	groupsnapapi "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumegroupsnapshot/v1alpha1"
+	groupsnapapi "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumegroupsnapshot/v1beta1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/kubernetes/test/e2e/framework"
 )
@@ -78,16 +79,32 @@ func (c *cephFSVolumeGroupSnapshot) ValidateResourcesForCreate(vgs *groupsnapapi
 		return fmt.Errorf("failed getting cephFS metadata pool name: %w", err)
 	}
 
-	sourcePVCCount := len(vgs.Status.PVCVolumeSnapshotRefList)
+	vgsc, err := c.groupclient.VolumeGroupSnapshotContents().Get(
+		ctx,
+		*vgs.Status.BoundVolumeGroupSnapshotContentName,
+		metav1.GetOptions{})
+	if err != nil {
+		return fmt.Errorf("failed to get VolumeGroupSnapshotContent: %w", err)
+	}
+
+	sourcePVCCount := len(vgsc.Status.VolumeSnapshotHandlePairList)
 	// we are creating clones for each source PVC
-	clonePVCCount := len(vgs.Status.PVCVolumeSnapshotRefList)
+	clonePVCCount := len(vgsc.Status.VolumeSnapshotHandlePairList)
 	totalPVCCount := sourcePVCCount + clonePVCCount
 	validateSubvolumeCount(c.framework, totalPVCCount, fileSystemName, subvolumegroup)
 
 	// we are creating 1 snapshot for each source PVC, validate the snapshot count
-	for _, pvcSnap := range vgs.Status.PVCVolumeSnapshotRefList {
+	for _, snapshot := range vgsc.Status.VolumeSnapshotHandlePairList {
+		volumeHandle := snapshot.VolumeHandle
+		volumeSnapshotName := fmt.Sprintf("snapshot-%x", sha256.Sum256([]byte(
+			string(vgsc.UID)+volumeHandle)))
+		volumeSnapshot, err := c.snapClient.VolumeSnapshots(vgs.Namespace).Get(ctx, volumeSnapshotName, metav1.GetOptions{})
+		if err != nil {
+			return fmt.Errorf("failed to get VolumeSnapshot: %w", err)
+		}
+		pvcName := *volumeSnapshot.Spec.Source.PersistentVolumeClaimName
 		pvc, err := c.framework.ClientSet.CoreV1().PersistentVolumeClaims(vgs.Namespace).Get(ctx,
-			pvcSnap.PersistentVolumeClaimRef.Name,
+			pvcName,
 			metav1.GetOptions{})
 		if err != nil {
 			return fmt.Errorf("failed to get PVC: %w", err)
@@ -165,8 +182,16 @@ func (rvgs *rbdVolumeGroupSnapshot) GetVolumeGroupSnapshotClass() (*groupsnapapi
 }
 
 func (rvgs *rbdVolumeGroupSnapshot) ValidateResourcesForCreate(vgs *groupsnapapi.VolumeGroupSnapshot) error {
-	sourcePVCCount := len(vgs.Status.PVCVolumeSnapshotRefList)
-	clonePVCCount := len(vgs.Status.PVCVolumeSnapshotRefList)
+	vgsc, err := rvgs.groupclient.VolumeGroupSnapshotContents().Get(
+		context.TODO(),
+		*vgs.Status.BoundVolumeGroupSnapshotContentName,
+		metav1.GetOptions{})
+	if err != nil {
+		return fmt.Errorf("failed to get VolumeGroupSnapshotContent: %w", err)
+	}
+
+	sourcePVCCount := len(vgsc.Status.VolumeSnapshotHandlePairList)
+	clonePVCCount := len(vgsc.Status.VolumeSnapshotHandlePairList)
 	totalPVCCount := sourcePVCCount + clonePVCCount
 
 	validateOmapCount(rvgs.framework, totalPVCCount, rbdType, defaultRBDPool, volumesType)

e2e/volumegroupsnapshot_base.go

@@ -18,12 +18,14 @@ package e2e
 
 import (
 	"context"
+	"crypto/sha256"
 	"fmt"
 	"time"
 
-	groupsnapapi "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumegroupsnapshot/v1alpha1"
+	groupsnapapi "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumegroupsnapshot/v1beta1"
 	snapapi "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1"
-	groupsnapclient "github.com/kubernetes-csi/external-snapshotter/client/v8/clientset/versioned/typed/volumegroupsnapshot/v1alpha1"
+	groupsnapclient "github.com/kubernetes-csi/external-snapshotter/client/v8/clientset/versioned/typed/volumegroupsnapshot/v1beta1"
+	snapclient "github.com/kubernetes-csi/external-snapshotter/client/v8/clientset/versioned/typed/volumesnapshot/v1"
 	v1 "k8s.io/api/core/v1"
 	apierrs "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/resource"
@@ -75,7 +77,8 @@ type VolumeGroupSnapshotter interface {
 type volumeGroupSnapshotterBase struct {
 	timeout          int
 	framework        *framework.Framework
-	groupclient      *groupsnapclient.GroupsnapshotV1alpha1Client
+	groupclient      *groupsnapclient.GroupsnapshotV1beta1Client
+	snapClient       *snapclient.SnapshotV1Client
 	storageClassName string
 	blockPVC         bool
 	totalPVCCount    int
@@ -96,9 +99,15 @@ func newVolumeGroupSnapshotBase(f *framework.Framework, namespace,
 		return nil, fmt.Errorf("error creating group snapshot client: %w", err)
 	}
 
+	s, err := snapclient.NewForConfig(config)
+	if err != nil {
+		return nil, fmt.Errorf("error creating snapshot client: %w", err)
+	}
+
 	return &volumeGroupSnapshotterBase{
 		framework:        f,
 		groupclient:      c,
+		snapClient:       s,
 		namespace:        namespace,
 		storageClassName: storageClass,
 		blockPVC:         blockPVC,
@@ -160,13 +169,27 @@ func (v *volumeGroupSnapshotterBase) DeletePVCs(pvcs []*v1.PersistentVolumeClaim
 func (v *volumeGroupSnapshotterBase) CreatePVCClones(
 	vgs *groupsnapapi.VolumeGroupSnapshot,
 ) ([]*v1.PersistentVolumeClaim, error) {
-	pvcSnapRef := vgs.Status.PVCVolumeSnapshotRefList
+	groupSnapshotContent, err := v.groupclient.VolumeGroupSnapshotContents().Get(
+		context.TODO(),
+		*vgs.Status.BoundVolumeGroupSnapshotContentName,
+		metav1.GetOptions{})
+	if err != nil {
+		return nil, fmt.Errorf("failed to get VolumeGroupSnapshotContent: %w", err)
+	}
 	namespace := vgs.Namespace
 	ctx := context.TODO()
-	pvcs := make([]*v1.PersistentVolumeClaim, len(pvcSnapRef))
-	for i, pvcSnap := range pvcSnapRef {
+	pvcs := make([]*v1.PersistentVolumeClaim, len(groupSnapshotContent.Status.VolumeSnapshotHandlePairList))
+	for i, snapshot := range groupSnapshotContent.Status.VolumeSnapshotHandlePairList {
+		volumeHandle := snapshot.VolumeHandle
+		volumeSnapshotName := fmt.Sprintf("snapshot-%x", sha256.Sum256([]byte(
+			string(groupSnapshotContent.UID)+volumeHandle)))
+		volumeSnapshot, err := v.snapClient.VolumeSnapshots(namespace).Get(ctx, volumeSnapshotName, metav1.GetOptions{})
+		if err != nil {
+			return nil, fmt.Errorf("failed to get VolumeSnapshot: %w", err)
+		}
+		pvcName := *volumeSnapshot.Spec.Source.PersistentVolumeClaimName
 		pvc, err := v.framework.ClientSet.CoreV1().PersistentVolumeClaims(namespace).Get(ctx,
-			pvcSnap.PersistentVolumeClaimRef.Name,
+			pvcName,
 			metav1.GetOptions{})
 		if err != nil {
 			return nil, fmt.Errorf("failed to get PVC: %w", err)
@@ -179,12 +202,11 @@ func (v *volumeGroupSnapshotterBase) CreatePVCClones(
 			Spec: *pvc.Spec.DeepCopy(),
 		}
 
-		snap := pvcSnap.VolumeSnapshotRef
 		apiGroup := snapapi.GroupName
 		pvcs[i].Spec.DataSource = &v1.TypedLocalObjectReference{
 			APIGroup: &apiGroup,
 			Kind:     "VolumeSnapshot",
-			Name:     snap.Name,
+			Name:     volumeSnapshot.Name,
 		}
 		pvcs[i].Spec.StorageClassName = &v.storageClassName
 		// cleanup the VolumeName as we are creating a new PVC
@@ -263,7 +285,7 @@ func (v *volumeGroupSnapshotterBase) DeletePods(pods []*v1.Pod) error {
 	return nil
 }
 
-func (v volumeGroupSnapshotterBase) CreateVolumeGroupSnapshotClass(
+func (v *volumeGroupSnapshotterBase) CreateVolumeGroupSnapshotClass(
 	groupSnapshotClass *groupsnapapi.VolumeGroupSnapshotClass,
 ) error {
 	return wait.PollUntilContextTimeout(
@@ -286,7 +308,7 @@ func (v volumeGroupSnapshotterBase) CreateVolumeGroupSnapshotClass(
 		})
 }
 
-func (v volumeGroupSnapshotterBase) CreateVolumeGroupSnapshot(name,
+func (v *volumeGroupSnapshotterBase) CreateVolumeGroupSnapshot(name,
 	volumeGroupSnapshotClassName string, labels map[string]string,
 ) (*groupsnapapi.VolumeGroupSnapshot, error) {
 	namespace := v.namespace
@@ -356,7 +378,7 @@ func (v volumeGroupSnapshotterBase) CreateVolumeGroupSnapshot(name,
 	return groupSnapshot, nil
 }
 
-func (v volumeGroupSnapshotterBase) DeleteVolumeGroupSnapshot(volumeGroupSnapshotName string) error {
+func (v *volumeGroupSnapshotterBase) DeleteVolumeGroupSnapshot(volumeGroupSnapshotName string) error {
 	namespace := v.namespace
 	ctx := context.TODO()
 	err := v.groupclient.VolumeGroupSnapshots(namespace).Delete(
@@ -395,7 +417,7 @@ func (v volumeGroupSnapshotterBase) DeleteVolumeGroupSnapshot(volumeGroupSnapsho
 		})
 }
 
-func (v volumeGroupSnapshotterBase) DeleteVolumeGroupSnapshotClass(groupSnapshotClassName string) error {
+func (v *volumeGroupSnapshotterBase) DeleteVolumeGroupSnapshotClass(groupSnapshotClassName string) error {
 	ctx := context.TODO()
 	err := v.groupclient.VolumeGroupSnapshotClasses().Delete(
 		ctx, groupSnapshotClassName, metav1.DeleteOptions{})

examples/cephfs/groupsnapshot.yaml

@@ -1,5 +1,5 @@
 ---
-apiVersion: groupsnapshot.storage.k8s.io/v1alpha1
+apiVersion: groupsnapshot.storage.k8s.io/v1beta1
 kind: VolumeGroupSnapshot
 metadata:
   name: new-groupsnapshot-demo-1

examples/cephfs/groupsnapshotclass.yaml

@@ -1,5 +1,5 @@
 ---
-apiVersion: groupsnapshot.storage.k8s.io/v1alpha1
+apiVersion: groupsnapshot.storage.k8s.io/v1beta1
 kind: VolumeGroupSnapshotClass
 metadata:
   name: csi-cephfsplugin-groupsnapclass

examples/rbd/groupsnapshot.yaml

@@ -1,5 +1,5 @@
 ---
-apiVersion: groupsnapshot.storage.k8s.io/v1alpha1
+apiVersion: groupsnapshot.storage.k8s.io/v1beta1
 kind: VolumeGroupSnapshot
 metadata:
   name: rbd-groupsnapshot

examples/rbd/groupsnapshotclass.yaml

@@ -1,5 +1,5 @@
 ---
-apiVersion: groupsnapshot.storage.k8s.io/v1alpha1
+apiVersion: groupsnapshot.storage.k8s.io/v1beta1
 kind: VolumeGroupSnapshotClass
 metadata:
   name: csi-rbdplugin-groupsnapclass

go.mod

@@ -18,7 +18,7 @@ require (
 	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
 	github.com/hashicorp/vault/api v1.15.0
 	github.com/kubernetes-csi/csi-lib-utils v0.19.0
-	github.com/kubernetes-csi/external-snapshotter/client/v8 v8.0.0
+	github.com/kubernetes-csi/external-snapshotter/client/v8 v8.2.0
 	github.com/libopenstorage/secrets v0.0.0-20231011182615-5f4b25ceede1
 	github.com/onsi/ginkgo/v2 v2.22.0
 	github.com/onsi/gomega v1.36.0

go.sum

@@ -1970,8 +1970,8 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/kubernetes-csi/csi-lib-utils v0.19.0 h1:3sT8mL9+St2acyrEtuR7CQ5L78GR4lgsb+sfon9tGfA=
 github.com/kubernetes-csi/csi-lib-utils v0.19.0/go.mod h1:lBuMKvoyd8c3EG+itmnVWApLDHnLkU7ibxxZSPuOw0M=
 github.com/kubernetes-csi/external-snapshotter/client/v4 v4.0.0/go.mod h1:YBCo4DoEeDndqvAn6eeu0vWM7QdXmHEeI9cFWplmBys=
-github.com/kubernetes-csi/external-snapshotter/client/v8 v8.0.0 h1:mjQG0Vakr2h246kEDR85U8y8ZhPgT3bguTCajRa/jaw=
-github.com/kubernetes-csi/external-snapshotter/client/v8 v8.0.0/go.mod h1:E3vdYxHj2C2q6qo8/Da4g7P+IcwqRZyy3gJBzYybV9Y=
+github.com/kubernetes-csi/external-snapshotter/client/v8 v8.2.0 h1:Q3jQ1NkFqv5o+F8dMmHd8SfEmlcwNeo1immFApntEwE=
+github.com/kubernetes-csi/external-snapshotter/client/v8 v8.2.0/go.mod h1:E3vdYxHj2C2q6qo8/Da4g7P+IcwqRZyy3gJBzYybV9Y=
 github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
 github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80/go.mod h1:imJHygn/1yfhB7XSJJKlFZKl/J+dCPAknuiaGOshXAs=