From 39e98eacbc3006151807f5c48a5713e8019f501e Mon Sep 17 00:00:00 2001 From: Jian Wang Date: Thu, 17 Oct 2024 14:01:26 +0200 Subject: [PATCH] Allow to mark more IPs as internal IPs Signed-off-by: Jian Wang --- pkg/cloud-controller-manager/annotation.go | 2 + pkg/cloud-controller-manager/instance.go | 29 +- pkg/cloud-controller-manager/instance_test.go | 378 ++++++++++++++++++ 3 files changed, 408 insertions(+), 1 deletion(-) create mode 100644 pkg/cloud-controller-manager/instance_test.go diff --git a/pkg/cloud-controller-manager/annotation.go b/pkg/cloud-controller-manager/annotation.go index c29536bd..a8dd79dc 100644 --- a/pkg/cloud-controller-manager/annotation.go +++ b/pkg/cloud-controller-manager/annotation.go @@ -10,4 +10,6 @@ const ( KeyPrimaryService = prefix + "primary-service" KeyKubevipLoadBalancerIP = "kube-vip.io/loadbalancerIPs" + + KeyAdditionalInternalIPs = prefix + "additional-internal-ips" ) diff --git a/pkg/cloud-controller-manager/instance.go b/pkg/cloud-controller-manager/instance.go index 1cd33fdf..6659364f 100644 --- a/pkg/cloud-controller-manager/instance.go +++ b/pkg/cloud-controller-manager/instance.go @@ -2,10 +2,14 @@ package ccm import ( "context" + "encoding/json" + "fmt" "net" + "slices" "sync" ctlkubevirtv1 "github.com/harvester/harvester/pkg/generated/controllers/kubevirt.io/v1" + "github.com/sirupsen/logrus" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -86,6 +90,15 @@ func getNodeAddresses(node *v1.Node, vmi *kubevirtv1.VirtualMachineInstance) []v return nil } + aiIPs, err := getAdditionalInternalIPs(node) + if err != nil { + // if additional IPs are not correctly marked, only log an error, do not return this error + logrus.WithFields(logrus.Fields{ + "namespace": node.Namespace, + "name": node.Name, + }).Debugf("%s, skip it", err.Error()) + } + nodeAddresses := make([]v1.NodeAddress, 0, len(vmi.Spec.Networks)+1) for _, network := range vmi.Spec.Networks { @@ -95,7 +108,7 @@ func getNodeAddresses(node *v1.Node, vmi *kubevirtv1.VirtualMachineInstance) []v nodeAddr := v1.NodeAddress{ Address: networkInterface.IP, } - if networkInterface.IP == providedNodeIP { + if networkInterface.IP == providedNodeIP || (aiIPs != nil && slices.Contains(aiIPs, networkInterface.IP)) { nodeAddr.Type = v1.NodeInternalIP } else { nodeAddr.Type = v1.NodeExternalIP @@ -112,3 +125,17 @@ func getNodeAddresses(node *v1.Node, vmi *kubevirtv1.VirtualMachineInstance) []v return nodeAddresses } + +// User may want to mark some IPs of the node also as internal +func getAdditionalInternalIPs(node *v1.Node) ([]string, error) { + aiIPs, ok := node.Annotations[KeyAdditionalInternalIPs] + if !ok { + return nil, nil + } + var ips []string + err := json.Unmarshal([]byte(aiIPs), &ips) + if err != nil { + return nil, fmt.Errorf("failed to decode additional external IPs from %v: %w", aiIPs, err) + } + return ips, nil +} diff --git a/pkg/cloud-controller-manager/instance_test.go b/pkg/cloud-controller-manager/instance_test.go new file mode 100644 index 00000000..3c8e1a82 --- /dev/null +++ b/pkg/cloud-controller-manager/instance_test.go @@ -0,0 +1,378 @@ +package ccm + +import ( + "testing" + + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/cloud-provider/api" + kubevirtv1 "kubevirt.io/api/core/v1" +) + +const ( + testNamespace = "default" + nodeName = "test" + + networkDefault = "default" + network120 = "vlan120" + network130 = "vlan130" + + networkDefaultIP = "192.168.100.10" + network120IP = "192.168.120.10" + network130IP = "192.168.130.10" +) + +func Test_getNodeAddresses(t *testing.T) { + tests := []struct { + name string + node *v1.Node + vmi *kubevirtv1.VirtualMachineInstance + output []v1.NodeAddress + }{ + { + name: "1 internal and 2 external IPs", + node: &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: nodeName, + Annotations: map[string]string{ + api.AnnotationAlphaProvidedIPAddr: networkDefaultIP, + }, + }, + }, + vmi: &kubevirtv1.VirtualMachineInstance{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: testNamespace, + Name: nodeName, + }, + Spec: kubevirtv1.VirtualMachineInstanceSpec{ + Networks: []kubevirtv1.Network{ + { + Name: networkDefault, + }, + { + Name: network120, + }, + { + Name: network130, + }, + }, + }, + Status: kubevirtv1.VirtualMachineInstanceStatus{ + Interfaces: []kubevirtv1.VirtualMachineInstanceNetworkInterface{ + { + Name: networkDefault, + IP: networkDefaultIP, + }, + { + Name: network120, + IP: network120IP, + }, + { + Name: network130, + IP: network130IP, + }, + }, + }, + }, + output: []v1.NodeAddress{ + { + Type: v1.NodeInternalIP, + Address: networkDefaultIP, + }, + { + Type: v1.NodeExternalIP, + Address: network120IP, + }, + { + Type: v1.NodeExternalIP, + Address: network130IP, + }, + { + Type: v1.NodeHostName, + Address: nodeName, + }, + }, + }, + { + name: "1 internal and 2 external IPs, additional internal IPs do not match", + node: &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: nodeName, + Annotations: map[string]string{ + api.AnnotationAlphaProvidedIPAddr: networkDefaultIP, + KeyAdditionalInternalIPs: "[\"192.168.120.12\", \"192.168.120.11\"]", // match nothing + }, + }, + }, + vmi: &kubevirtv1.VirtualMachineInstance{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: testNamespace, + Name: nodeName, + }, + Spec: kubevirtv1.VirtualMachineInstanceSpec{ + Networks: []kubevirtv1.Network{ + { + Name: networkDefault, + }, + { + Name: network120, + }, + { + Name: network130, + }, + }, + }, + Status: kubevirtv1.VirtualMachineInstanceStatus{ + Interfaces: []kubevirtv1.VirtualMachineInstanceNetworkInterface{ + { + Name: networkDefault, + IP: networkDefaultIP, + }, + { + Name: network120, + IP: network120IP, + }, + { + Name: network130, + IP: network130IP, + }, + }, + }, + }, + output: []v1.NodeAddress{ + { + Type: v1.NodeInternalIP, + Address: networkDefaultIP, + }, + { + Type: v1.NodeExternalIP, + Address: network120IP, + }, + { + Type: v1.NodeExternalIP, + Address: network130IP, + }, + { + Type: v1.NodeHostName, + Address: nodeName, + }, + }, + }, + { + name: "1 internal and 2 external IPs, malformed annotations are skipped", + node: &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: nodeName, + Annotations: map[string]string{ + api.AnnotationAlphaProvidedIPAddr: networkDefaultIP, + KeyAdditionalInternalIPs: "192.168.120.11", // not a valid []string converted JSON + }, + }, + }, + vmi: &kubevirtv1.VirtualMachineInstance{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: testNamespace, + Name: nodeName, + }, + Spec: kubevirtv1.VirtualMachineInstanceSpec{ + Networks: []kubevirtv1.Network{ + { + Name: networkDefault, + }, + { + Name: network120, + }, + { + Name: network130, + }, + }, + }, + Status: kubevirtv1.VirtualMachineInstanceStatus{ + Interfaces: []kubevirtv1.VirtualMachineInstanceNetworkInterface{ + { + Name: networkDefault, + IP: networkDefaultIP, + }, + { + Name: network120, + IP: network120IP, + }, + { + Name: network130, + IP: network130IP, + }, + }, + }, + }, + output: []v1.NodeAddress{ + { + Type: v1.NodeInternalIP, + Address: networkDefaultIP, + }, + { + Type: v1.NodeExternalIP, + Address: network120IP, + }, + { + Type: v1.NodeExternalIP, + Address: network130IP, + }, + { + Type: v1.NodeHostName, + Address: nodeName, + }, + }, + }, + { + name: "2 internal and 1 external IPs", + node: &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: nodeName, + Annotations: map[string]string{ + api.AnnotationAlphaProvidedIPAddr: networkDefaultIP, + KeyAdditionalInternalIPs: "[\"192.168.120.10\", \"192.168.120.11\"]", + }, + }, + }, + vmi: &kubevirtv1.VirtualMachineInstance{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: testNamespace, + Name: nodeName, + }, + Spec: kubevirtv1.VirtualMachineInstanceSpec{ + Networks: []kubevirtv1.Network{ + { + Name: networkDefault, + }, + { + Name: network120, + }, + { + Name: network130, + }, + }, + }, + Status: kubevirtv1.VirtualMachineInstanceStatus{ + Interfaces: []kubevirtv1.VirtualMachineInstanceNetworkInterface{ + { + Name: networkDefault, + IP: networkDefaultIP, + }, + { + Name: network120, + IP: network120IP, + }, + { + Name: network130, + IP: network130IP, + }, + }, + }, + }, + output: []v1.NodeAddress{ + { + Type: v1.NodeInternalIP, + Address: networkDefaultIP, + }, + { + Type: v1.NodeInternalIP, + Address: network120IP, + }, + { + Type: v1.NodeExternalIP, + Address: network130IP, + }, + { + Type: v1.NodeHostName, + Address: nodeName, + }, + }, + }, + { + name: "3 internal and 0 external IPs", + node: &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: nodeName, + Annotations: map[string]string{ + api.AnnotationAlphaProvidedIPAddr: networkDefaultIP, + KeyAdditionalInternalIPs: "[\"192.168.120.10\", \"192.168.130.10\"]", + }, + }, + }, + vmi: &kubevirtv1.VirtualMachineInstance{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: testNamespace, + Name: nodeName, + }, + Spec: kubevirtv1.VirtualMachineInstanceSpec{ + Networks: []kubevirtv1.Network{ + { + Name: networkDefault, + }, + { + Name: network120, + }, + { + Name: network130, + }, + }, + }, + Status: kubevirtv1.VirtualMachineInstanceStatus{ + Interfaces: []kubevirtv1.VirtualMachineInstanceNetworkInterface{ + { + Name: networkDefault, + IP: networkDefaultIP, + }, + { + Name: network120, + IP: network120IP, + }, + { + Name: network130, + IP: network130IP, + }, + }, + }, + }, + output: []v1.NodeAddress{ + { + Type: v1.NodeInternalIP, + Address: networkDefaultIP, + }, + { + Type: v1.NodeInternalIP, + Address: network120IP, + }, + { + Type: v1.NodeInternalIP, + Address: network130IP, + }, + { + Type: v1.NodeHostName, + Address: nodeName, + }, + }, + }, + } + + checkOutputEqual := func(expected, output []v1.NodeAddress) bool { + if len(expected) != len(output) { + return false + } + for i := range expected { + if expected[i].Type != output[i].Type || expected[i].Address != output[i].Address { + return false + } + } + return true + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ips := getNodeAddresses(tt.node, tt.vmi) + if !checkOutputEqual(tt.output, ips) { + t.Errorf("case %v failed, expected output %+v, real output: %+v", tt.name, tt.output, ips) + } + }) + } +}