diff --git a/.golangci.yml b/.golangci.yml index f77b17e7c5..eedba53335 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -23,6 +23,9 @@ issues: - path: tests/e2e linters: - gosec + - path: tests/sanity + linters: + - gosec linters-settings: revive: rules: diff --git a/Makefile b/Makefile index c7da868d24..7f19cebaab 100644 --- a/Makefile +++ b/Makefile @@ -123,6 +123,10 @@ cluster/uninstall: bin/helm bin/aws ## E2E targets # Targets to run e2e tests +.PHONY: sanity +sanity: + go test -v -race ./tests/sanity/... + .PHONY: e2e/single-az e2e/single-az: bin/helm bin/ginkgo AWS_AVAILABILITY_ZONES=us-west-2a \ diff --git a/go.mod b/go.mod index 6a42cc3042..a847a9fefa 100644 --- a/go.mod +++ b/go.mod @@ -87,6 +87,7 @@ require ( github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.17.11 // indirect + github.com/kubernetes-csi/csi-test/v5 v5.3.1 github.com/kylelemons/godebug v1.1.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/moby/spdystream v0.5.0 // indirect diff --git a/go.sum b/go.sum index 2c692204da..5a2b06e2cc 100644 --- a/go.sum +++ b/go.sum @@ -227,6 +227,8 @@ github.com/kubernetes-csi/csi-proxy/client v1.1.3 h1:FdGU7NtxGhQX2wTfnuscmThG920 github.com/kubernetes-csi/csi-proxy/client v1.1.3/go.mod h1:SfK4HVKQdMH5KrffivddAWgX5hl3P5KmnuOTBbDNboU= github.com/kubernetes-csi/csi-proxy/v2 v2.0.0-alpha.1 h1:tVPvlL5N5X598hrO3g9rhyoi6h0LP4RpSJlGHItsbEE= github.com/kubernetes-csi/csi-proxy/v2 v2.0.0-alpha.1/go.mod h1:pacx+PW7lLlu6kAvpr8Lgq/5fdiAsKxOtXXFHMaLMb8= +github.com/kubernetes-csi/csi-test/v5 v5.3.1 h1:Wiukp1In+kif+BFo6q2ExjgB+MbrAz4jZWzGfijypuY= +github.com/kubernetes-csi/csi-test/v5 v5.3.1/go.mod h1:7hA2cSYJ6T8CraEZPA6zqkLZwemjBD54XAnPsPC3VpA= github.com/kubernetes-csi/external-snapshotter/client/v4 v4.2.0 h1:nHHjmvjitIiyPlUHk/ofpgvBcNcawJLtf4PYHORLjAA= github.com/kubernetes-csi/external-snapshotter/client/v4 v4.2.0/go.mod h1:YBCo4DoEeDndqvAn6eeu0vWM7QdXmHEeI9cFWplmBys= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= diff --git a/pkg/driver/controller_modify_volume.go b/pkg/driver/controller_modify_volume.go index 10f40b8c7b..1a1608094b 100644 --- a/pkg/driver/controller_modify_volume.go +++ b/pkg/driver/controller_modify_volume.go @@ -150,6 +150,8 @@ func executeModifyVolumeRequest(c cloud.Cloud) func(string, modifyVolumeRequest) if err != nil { if errors.Is(err, cloud.ErrInvalidArgument) { return 0, status.Errorf(codes.InvalidArgument, "Could not modify volume (invalid argument) %q: %v", volumeID, err) + } else if errors.Is(err, cloud.ErrNotFound) { + return 0, status.Errorf(codes.NotFound, "Could not modify volume (not found) %q: %v", volumeID, err) } return 0, status.Errorf(codes.Internal, "Could not modify volume %q: %v", volumeID, err) } else { diff --git a/pkg/driver/node.go b/pkg/driver/node.go index eccf4cf450..b3105e0c84 100644 --- a/pkg/driver/node.go +++ b/pkg/driver/node.go @@ -208,7 +208,7 @@ func (d *NodeService) NodeStageVolume(ctx context.Context, req *csi.NodeStageVol source, err := d.mounter.FindDevicePath(devicePath, volumeID, partition, d.metadata.GetRegion()) if err != nil { - return nil, status.Errorf(codes.Internal, "Failed to find device path %s. %v", devicePath, err) + return nil, status.Errorf(codes.NotFound, "Failed to find device path %s. %v", devicePath, err) } klog.V(4).InfoS("NodeStageVolume: find device path", "devicePath", devicePath, "source", source) @@ -388,7 +388,6 @@ func (d *NodeService) NodeExpandVolume(ctx context.Context, req *csi.NodeExpandV return &csi.NodeExpandVolumeResponse{CapacityBytes: bcap}, nil } } - deviceName, _, err := d.mounter.GetDeviceNameFromMount(volumePath) if err != nil { return nil, status.Errorf(codes.Internal, "failed to get device name from mount %s: %v", volumePath, err) @@ -396,7 +395,7 @@ func (d *NodeService) NodeExpandVolume(ctx context.Context, req *csi.NodeExpandV devicePath, err := d.mounter.FindDevicePath(deviceName, volumeID, "", d.metadata.GetRegion()) if err != nil { - return nil, status.Errorf(codes.Internal, "failed to find device path for device name %s for mount %s: %v", deviceName, req.GetVolumePath(), err) + return nil, status.Errorf(codes.NotFound, "failed to find device path for device name %s for mount %s: %v", deviceName, req.GetVolumePath(), err) } // TODO: lock per volume ID to have some idempotency @@ -627,7 +626,7 @@ func (d *NodeService) nodePublishVolumeForBlock(req *csi.NodePublishVolumeReques source, err := d.mounter.FindDevicePath(devicePath, volumeID, partition, d.metadata.GetRegion()) if err != nil { - return status.Errorf(codes.Internal, "Failed to find device path %s. %v", devicePath, err) + return status.Errorf(codes.NotFound, "Failed to find device path %s. %v", devicePath, err) } klog.V(4).InfoS("NodePublishVolume [block]: find device path", "devicePath", devicePath, "source", source) diff --git a/pkg/driver/node_test.go b/pkg/driver/node_test.go index 59f6ebab27..3d15923bb7 100644 --- a/pkg/driver/node_test.go +++ b/pkg/driver/node_test.go @@ -642,7 +642,7 @@ func TestNodeStageVolume(t *testing.T) { m.EXPECT().GetRegion().Return("us-west-2") return m }, - expectedErr: status.Errorf(codes.Internal, "Failed to find device path %s. %v", "/dev/xvdba", errors.New("find device path error")), + expectedErr: status.Errorf(codes.NotFound, "Failed to find device path %s. %v", "/dev/xvdba", errors.New("find device path error")), }, { name: "path_exists_error", @@ -1743,7 +1743,7 @@ func TestNodePublishVolume(t *testing.T) { m.EXPECT().GetRegion().Return("us-west-2") return m }, - expectedErr: status.Error(codes.Internal, "Failed to find device path /dev/xvdba. device path error"), + expectedErr: status.Error(codes.NotFound, "Failed to find device path /dev/xvdba. device path error"), }, } for _, tc := range testCases { @@ -2271,7 +2271,7 @@ func TestNodeExpandVolume(t *testing.T) { return m }, expectedResp: nil, - expectedErr: status.Error(codes.Internal, "failed to find device path for device name device-name for mount /volume/path: failed to find device path"), + expectedErr: status.Error(codes.NotFound, "failed to find device path for device name device-name for mount /volume/path: failed to find device path"), }, { name: "resize_error", diff --git a/tests/sanity/fake_sanity_cloud.go b/tests/sanity/fake_sanity_cloud.go new file mode 100644 index 0000000000..a694b3475d --- /dev/null +++ b/tests/sanity/fake_sanity_cloud.go @@ -0,0 +1,219 @@ +// Copyright 2024 The Kubernetes Authors. +// +// Licensed under the Apache License, Version 2.0 (the 'License'); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an 'AS IS' BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sanity + +import ( + "context" + "fmt" + "math/rand" + "strconv" + "time" + + "github.com/aws/aws-sdk-go-v2/service/ec2" + "github.com/aws/aws-sdk-go-v2/service/ec2/types" + "github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/cloud" + "github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/cloud/metadata" + "github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/util" +) + +var ( + disks = make(map[string]*cloud.Disk) + snapshots = make(map[string]*cloud.Snapshot) + snapshotNameToID = make(map[string]string) +) + +type FakeCloud struct { + fakeMetaData metadata.Metadata + mountPath string +} + +func newFakeCloud(fmd metadata.Metadata, mp string) *FakeCloud { + return &FakeCloud{ + fakeMetaData: fmd, + mountPath: mp, + } +} + +func (d *FakeCloud) CreateDisk(ctx context.Context, volumeID string, diskOptions *cloud.DiskOptions) (*cloud.Disk, error) { + for _, existingDisk := range disks { + if existingDisk.VolumeID == volumeID && existingDisk.CapacityGiB != util.BytesToGiB(diskOptions.CapacityBytes) { + return nil, cloud.ErrAlreadyExists + } + } + + if diskOptions.SnapshotID != "" { + if _, exists := snapshots[diskOptions.SnapshotID]; !exists { + return nil, cloud.ErrNotFound + } + newDisk := &cloud.Disk{ + SnapshotID: diskOptions.SnapshotID, + VolumeID: volumeID, + AvailabilityZone: diskOptions.AvailabilityZone, + CapacityGiB: util.BytesToGiB(diskOptions.CapacityBytes), + } + disks[volumeID] = newDisk + return newDisk, nil + } + + newDisk := &cloud.Disk{ + VolumeID: volumeID, + AvailabilityZone: diskOptions.AvailabilityZone, + CapacityGiB: util.BytesToGiB(diskOptions.CapacityBytes), + } + disks[volumeID] = newDisk + return newDisk, nil +} +func (d *FakeCloud) DeleteDisk(ctx context.Context, volumeID string) (bool, error) { + _, exists := disks[volumeID] + if !exists { + return false, cloud.ErrNotFound + } + delete(disks, volumeID) + return true, nil +} + +func (d *FakeCloud) GetDiskByID(ctx context.Context, volumeID string) (*cloud.Disk, error) { + disk, exists := disks[volumeID] + if !exists { + return nil, cloud.ErrNotFound + } + return disk, nil +} + +func (d *FakeCloud) CreateSnapshot(ctx context.Context, volumeID string, opts *cloud.SnapshotOptions) (*cloud.Snapshot, error) { + snapshotID := fmt.Sprintf("snapshot-%d", rand.New(rand.NewSource(time.Now().UnixNano())).Uint64()) + + _, exists := snapshots[snapshotID] + if exists { + return nil, cloud.ErrAlreadyExists + } + newSnapshot := &cloud.Snapshot{ + SnapshotID: snapshotID, + SourceVolumeID: volumeID, + CreationTime: time.Now(), + ReadyToUse: true, + } + snapshots[snapshotID] = newSnapshot + snapshotNameToID[opts.Tags["CSIVolumeSnapshotName"]] = snapshotID + return newSnapshot, nil +} + +func (d *FakeCloud) DeleteSnapshot(ctx context.Context, snapshotID string) (bool, error) { + if _, exists := snapshots[snapshotID]; !exists { + return false, cloud.ErrNotFound + } + for name, id := range snapshotNameToID { + if id == snapshotID { + delete(snapshotNameToID, name) + break + } + } + delete(snapshots, snapshotID) + return true, nil +} +func (d *FakeCloud) GetSnapshotByID(ctx context.Context, snapshotID string) (*cloud.Snapshot, error) { + snapshot, exists := snapshots[snapshotID] + if !exists { + return nil, cloud.ErrNotFound + } + return snapshot, nil +} + +func (d *FakeCloud) GetSnapshotByName(ctx context.Context, name string) (*cloud.Snapshot, error) { + if snapshotID, exists := snapshotNameToID[name]; exists { + return snapshots[snapshotID], nil + } + return nil, cloud.ErrNotFound +} + +func (d *FakeCloud) ListSnapshots(ctx context.Context, sourceVolumeID string, maxResults int32, nextToken string) (*cloud.ListSnapshotsResponse, error) { + var s []*cloud.Snapshot + startIndex := 0 + var err error + + if nextToken != "" { + startIndex, err = strconv.Atoi(nextToken) + if err != nil { + return nil, fmt.Errorf("invalid next token %s", nextToken) + } + } + var nextTokenStr string + count := 0 + for _, snap := range snapshots { + if snap.SourceVolumeID == sourceVolumeID || sourceVolumeID == "" { + if startIndex <= count { + s = append(s, snap) + if maxResults > 0 && int32(len(s)) >= maxResults { + nextTokenStr = strconv.Itoa(startIndex + int(maxResults)) + break + } + } + count++ + } + } + + return &cloud.ListSnapshotsResponse{ + Snapshots: s, + NextToken: nextTokenStr, + }, nil +} + +func (d *FakeCloud) AttachDisk(ctx context.Context, volumeID string, instanceID string) (string, error) { + _, diskExists := disks[volumeID] + if !diskExists || instanceID != d.fakeMetaData.InstanceID { + return "", cloud.ErrNotFound + } + return d.mountPath, nil +} + +func (d *FakeCloud) DetachDisk(ctx context.Context, volumeID string, instanceID string) error { + _, diskExists := disks[volumeID] + if !diskExists || instanceID != d.fakeMetaData.InstanceID { + return cloud.ErrNotFound + } + return nil +} + +func (d *FakeCloud) ResizeOrModifyDisk(ctx context.Context, volumeID string, newSizeBytes int64, modifyOptions *cloud.ModifyDiskOptions) (int32, error) { + disk, exists := disks[volumeID] + if !exists { + return 0, cloud.ErrNotFound + } + newSizeGiB := util.BytesToGiB(newSizeBytes) + disk.CapacityGiB = newSizeGiB + disks[volumeID] = disk + realSizeGiB := newSizeGiB + return realSizeGiB, nil +} + +func (d *FakeCloud) AvailabilityZones(ctx context.Context) (map[string]struct{}, error) { + return map[string]struct{}{}, nil +} + +func (d *FakeCloud) EnableFastSnapshotRestores(ctx context.Context, availabilityZones []string, snapshotID string) (*ec2.EnableFastSnapshotRestoresOutput, error) { + return &ec2.EnableFastSnapshotRestoresOutput{}, nil +} + +func (d *FakeCloud) GetDiskByName(ctx context.Context, name string, capacityBytes int64) (*cloud.Disk, error) { + return &cloud.Disk{}, nil +} + +func (d *FakeCloud) ModifyTags(ctx context.Context, volumeID string, tagOptions cloud.ModifyTagsOptions) error { + return nil +} + +func (d *FakeCloud) WaitForAttachmentState(ctx context.Context, volumeID, expectedState, expectedInstance, expectedDevice string, alreadyAssigned bool) (*types.VolumeAttachment, error) { + return &types.VolumeAttachment{}, nil +} diff --git a/tests/sanity/fake_sanity_mounter.go b/tests/sanity/fake_sanity_mounter.go new file mode 100644 index 0000000000..1bc6b06cb2 --- /dev/null +++ b/tests/sanity/fake_sanity_mounter.go @@ -0,0 +1,145 @@ +// Copyright 2024 The Kubernetes Authors. +// +// Licensed under the Apache License, Version 2.0 (the 'License'); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an 'AS IS' BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sanity + +import ( + "fmt" + "os" + + "github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/cloud" + "k8s.io/mount-utils" +) + +var mounts = make(map[string]string) + +type FakeMounter struct{} + +func NewFakeMounter() *FakeMounter { + return &FakeMounter{} +} + +func (m *FakeMounter) FindDevicePath(devicePath, volumeID, partition, region string) (string, error) { + if len(devicePath) == 0 { + return devicePath, cloud.ErrNotFound + } + return devicePath, nil +} + +func (m *FakeMounter) PreparePublishTarget(target string) error { + if err := m.MakeDir(target); err != nil { + return fmt.Errorf("could not create dir %q: %w", target, err) + } + return nil +} + +func (m *FakeMounter) IsBlockDevice(fullPath string) (bool, error) { + return false, nil +} + +func (m *FakeMounter) GetBlockSizeBytes(devicePath string) (int64, error) { + return 0, nil +} + +func (m *FakeMounter) GetDeviceNameFromMount(mountPath string) (string, int, error) { + return mounts[mountPath], 0, nil +} + +func (m *FakeMounter) IsCorruptedMnt(err error) bool { + return false +} + +func (m *FakeMounter) MakeFile(path string) error { + return nil +} + +func (m *FakeMounter) MakeDir(path string) error { + err := os.MkdirAll(path, os.FileMode(0755)) + if err != nil { + if !os.IsExist(err) { + return err + } + } + return nil +} + +func (m *FakeMounter) PathExists(path string) (bool, error) { + _, exists := mounts[path] + if !exists { + return false, nil + } + return true, nil +} + +func (m *FakeMounter) Resize(devicePath, deviceMountPath string) (bool, error) { + return false, nil +} + +func (m *FakeMounter) NeedResize(devicePath string, deviceMountPath string) (bool, error) { + return false, nil +} + +func (m *FakeMounter) Unpublish(path string) error { + return m.Unstage(path) +} + +func (m *FakeMounter) Unstage(path string) error { + err := os.RemoveAll(path) + return err +} + +func (m *FakeMounter) Mount(source string, target string, fstype string, options []string) error { + mounts[target] = source + return nil +} + +func (m *FakeMounter) CanSafelySkipMountPointCheck() bool { + return false +} + +func (m *FakeMounter) FormatAndMountSensitiveWithFormatOptions(source, target, fstype string, options, sensitiveOptions, formatOptions []string) error { + return nil +} + +func (m *FakeMounter) GetMountRefs(pathname string) ([]string, error) { + return nil, nil +} + +func (m *FakeMounter) IsLikelyNotMountPoint(file string) (bool, error) { + return true, nil +} + +func (m *FakeMounter) IsMountPoint(file string) (bool, error) { + return false, nil +} + +func (m *FakeMounter) List() ([]mount.MountPoint, error) { + return nil, nil +} + +func (m *FakeMounter) MountSensitive(source, target, fstype string, options, sensitiveOptions []string) error { + return nil +} + +func (m *FakeMounter) MountSensitiveWithoutSystemd(source, target, fstype string, options, sensitiveOptions []string) error { + return nil +} + +func (m *FakeMounter) MountSensitiveWithoutSystemdWithMountFlags(source, target, fstype string, options, sensitiveOptions, mountFlags []string) error { + return nil +} + +func (m *FakeMounter) Unmount(target string) error { + return nil +} diff --git a/tests/sanity/sanity_test.go b/tests/sanity/sanity_test.go new file mode 100644 index 0000000000..04557fa367 --- /dev/null +++ b/tests/sanity/sanity_test.go @@ -0,0 +1,126 @@ +// Copyright 2024 The Kubernetes Authors. +// +// Licensed under the Apache License, Version 2.0 (the 'License'); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an 'AS IS' BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build linux +// +build linux + +package sanity + +import ( + "fmt" + "os" + "path" + "testing" + + "github.com/aws/aws-sdk-go-v2/aws/arn" + "github.com/golang/mock/gomock" + csisanity "github.com/kubernetes-csi/csi-test/v5/pkg/sanity" + "github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/cloud/metadata" + d "github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/driver" + "github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/util" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" +) + +const ( + instanceID = "i-1234567890abcdef0" + region = "us-west-2" + availabilityZone = "us-west-2a" +) + +var ( + fakeMetaData = &metadata.Metadata{ + InstanceID: instanceID, + Region: region, + } + outpostArn = &arn.ARN{ + Partition: "aws", + Service: "outposts", + Region: "us-west-2", + AccountID: "123456789012", + Resource: "op-1234567890abcdef0", + } +) + +func TestSanity(t *testing.T) { + defer func() { + if r := recover(); r != nil { + t.Errorf("Test panicked: %v", r) + } + }() + + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + + tmpDir, err := os.MkdirTemp("", "csi-sanity-") + if err != nil { + t.Fatalf("Failed to create sanity temp working dir: %v", err) + } + defer os.RemoveAll(tmpDir) + + defer func() { + if err = os.RemoveAll(tmpDir); err != nil { + t.Fatalf("Failed to clean up sanity temp working dir %s: %v", tmpDir, err.Error()) + } + }() + + endpoint := fmt.Sprintf("unix:%s/csi.sock", tmpDir) + mountPath := path.Join(tmpDir, "mount") + stagePath := path.Join(tmpDir, "stage") + + fakeMounter, FakeCloud, fakeMetadataService, fakeKubernetesClient := createMockObjects(mockCtrl, mountPath) + + driverOptions := &d.Options{ + Mode: d.AllMode, + ModifyVolumeRequestHandlerTimeout: 60, + Endpoint: endpoint, + } + + drv, err := d.NewDriver(FakeCloud, driverOptions, fakeMounter, fakeMetadataService, fakeKubernetesClient) + if err != nil { + t.Fatalf("Failed to create fake driver: %v", err.Error()) + } + go func() { + if err := drv.Run(); err != nil { + panic(fmt.Sprintf("%v", err)) + } + }() + + config := csisanity.TestConfig{ + TargetPath: mountPath, + StagingPath: stagePath, + Address: endpoint, + DialOptions: []grpc.DialOption{grpc.WithTransportCredentials(insecure.NewCredentials())}, + IDGen: csisanity.DefaultIDGenerator{}, + TestVolumeSize: 10 * util.GiB, + TestVolumeAccessType: "mount", + TestVolumeMutableParameters: map[string]string{"iops": "3014", "throughput": "153"}, + TestVolumeParameters: map[string]string{"type": "gp3", "iops": "3000"}, + } + csisanity.Test(t, config) +} + +func createMockObjects(mockCtrl *gomock.Controller, mountPath string) (*FakeMounter, *FakeCloud, *metadata.MockMetadataService, *d.MockKubernetesClient) { + fakeMounter := NewFakeMounter() + fakeCloud := newFakeCloud(*fakeMetaData, mountPath) + fakeKubernetesClient := d.NewMockKubernetesClient(mockCtrl) + fakeMetadataService := metadata.NewMockMetadataService(mockCtrl) + + fakeMetadataService.EXPECT().GetAvailabilityZone().Return(availabilityZone).AnyTimes() + fakeMetadataService.EXPECT().GetOutpostArn().Return(*outpostArn).AnyTimes() + fakeMetadataService.EXPECT().GetInstanceID().Return(instanceID).AnyTimes() + fakeMetadataService.EXPECT().GetRegion().Return(region).AnyTimes() + + return fakeMounter, fakeCloud, fakeMetadataService, fakeKubernetesClient +}