From 9edabc80edd521578d3f3efd30641791bfbc184c Mon Sep 17 00:00:00 2001 From: Nawaz Hussain Khazielakha Date: Thu, 14 Nov 2024 14:57:05 -0800 Subject: [PATCH 01/23] Update Tiltfile with AKS VNet peering and deletion logic --- Tiltfile | 96 ++++++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 87 insertions(+), 9 deletions(-) diff --git a/Tiltfile b/Tiltfile index aa9a4927a37..64c9da828a9 100644 --- a/Tiltfile +++ b/Tiltfile @@ -23,8 +23,8 @@ settings = { "capi_version": "v1.8.5", "caaph_version": "v0.2.5", "cert_manager_version": "v1.16.1", - "kubernetes_version": "v1.28.3", - "aks_kubernetes_version": "v1.28.3", + "kubernetes_version": "v1.28.15", + "aks_kubernetes_version": "v1.28.15", "flatcar_version": "3374.2.1", "azure_location": "eastus", "control_plane_machine_count": "1", @@ -51,6 +51,8 @@ if "default_registry" in settings: default_registry(settings.get("default_registry")) os_arch = str(local("go env GOARCH")).rstrip("\n") + +# TODO: no one is clearing MGMT_CLUSTER_NAME when using KIND, so this is always going to be true. Improve this logic. if "aks" in settings.get("kustomize_substitutions", {}).get("MGMT_CLUSTER_NAME", ""): print("Using AKS as management cluster, setting os_arch to amd64") os_arch = "amd64" @@ -118,7 +120,7 @@ def fixup_yaml_empty_arrays(yaml_str): return yaml_str.replace("storedVersions: null", "storedVersions: []") def validate_auth(): - substitutions = settings.get("kustomize_substitutions", {}) + substitutions = settings.get("kustomize_substitutions", {}) # all the env variables are exported here os.environ.update(substitutions) for sub in substitutions: if sub[-4:] == "_B64": @@ -212,10 +214,10 @@ def capz(): yaml = str(kustomizesub("./hack/observability")) # build an observable kind deployment by default # add extra_args if they are defined - if settings.get("extra_args"): - azure_extra_args = settings.get("extra_args").get("azure") + if settings.get("container_args"): + capz_container_args = settings.get("container_args").get("capz-controller-manager") yaml_dict = decode_yaml_stream(yaml) - append_arg_for_container_in_deployment(yaml_dict, "capz-controller-manager", "capz-system", "cluster-api-azure-controller", azure_extra_args) + append_arg_for_container_in_deployment(yaml_dict, "capz-controller-manager", "capz-system", "cluster-api-azure-controller", capz_container_args) yaml = str(encode_yaml_stream(yaml_dict)) yaml = fixup_yaml_empty_arrays(yaml) @@ -317,9 +319,15 @@ def flavors(): for template in template_list: deploy_worker_templates(template, substitutions) + delete_all_workload_clusters = kubectl_cmd + " delete clusters --all --wait=false" + + if "aks" in settings.get("kustomize_substitutions", {}).get("MGMT_CLUSTER_NAME", ""): + delete_all_workload_clusters += clear_aks_vnet_peerings() + + local_resource( name = "delete-all-workload-clusters", - cmd = kubectl_cmd + " delete clusters --all --wait=false", + cmd = ["sh", "-ec", delete_all_workload_clusters], auto_init = False, trigger_mode = TRIGGER_MODE_MANUAL, labels = ["flavors"], @@ -382,10 +390,19 @@ def deploy_worker_templates(template, substitutions): yaml = shlex.quote(yaml) flavor_name = os.path.basename(flavor) - flavor_cmd = "RANDOM=$(bash -c 'echo $RANDOM'); export CLUSTER_NAME=" + flavor.replace("windows", "win") + "-$RANDOM; make generate-flavors; echo " + yaml + "> ./.tiltbuild/" + flavor + "; cat ./.tiltbuild/" + flavor + " | " + envsubst_cmd + " | " + kubectl_cmd + " apply -f -; echo \"Cluster \'$CLUSTER_NAME\' created, don't forget to delete\"" + flavor_cmd = "RANDOM=$(bash -c 'echo $RANDOM')" + flavor_cmd += "; export CLUSTER_NAME=" + flavor.replace("windows", "win") + "-$RANDOM; echo " + yaml + "> ./.tiltbuild/" + flavor + "; cat ./.tiltbuild/" + flavor + " | " + envsubst_cmd + " | " + kubectl_cmd + " apply -f -" + flavor_cmd += "; echo \"Cluster \'$CLUSTER_NAME\' created, don't forget to delete\"" # wait for kubeconfig to be available - flavor_cmd += "; until " + kubectl_cmd + " get secret ${CLUSTER_NAME}-kubeconfig > /dev/null 2>&1; do sleep 5; done; " + kubectl_cmd + " get secret ${CLUSTER_NAME}-kubeconfig -o jsonpath={.data.value} | base64 --decode > ./${CLUSTER_NAME}.kubeconfig; chmod 600 ./${CLUSTER_NAME}.kubeconfig; until " + kubectl_cmd + " --kubeconfig=./${CLUSTER_NAME}.kubeconfig get nodes > /dev/null 2>&1; do sleep 5; done" + flavor_cmd += "; echo \"Waiting for kubeconfig to be available\"" + flavor_cmd += "; until " + kubectl_cmd + " get secret ${CLUSTER_NAME}-kubeconfig > /dev/null 2>&1; do sleep 5; done" + flavor_cmd += "; " + kubectl_cmd + " get secret ${CLUSTER_NAME}-kubeconfig -o jsonpath={.data.value} | base64 --decode > ./${CLUSTER_NAME}.kubeconfig" + flavor_cmd += "; chmod 600 ./${CLUSTER_NAME}.kubeconfig" + flavor_cmd += "; echo \"Kubeconfig for $CLUSTER_NAME created and saved in the local\"" + flavor_cmd += "; echo \"Waiting for $CLUSTER_NAME API Server to be accessible\"" + flavor_cmd += "; until " + kubectl_cmd + " --kubeconfig=./${CLUSTER_NAME}.kubeconfig get nodes > /dev/null 2>&1; do sleep 5; done" + flavor_cmd += "; echo \"API Server of $CLUSTER_NAME is accessible\"" # copy the kubeadm configmap to the calico-system namespace. # This is a workaround needed for the calico-node-windows daemonset to be able to run in the calico-system namespace. @@ -393,6 +410,10 @@ def deploy_worker_templates(template, substitutions): flavor_cmd += "; until " + kubectl_cmd + " --kubeconfig ./${CLUSTER_NAME}.kubeconfig get configmap kubeadm-config --namespace=kube-system > /dev/null 2>&1; do sleep 5; done" flavor_cmd += "; " + kubectl_cmd + " --kubeconfig ./${CLUSTER_NAME}.kubeconfig create namespace calico-system --dry-run=client -o yaml | " + kubectl_cmd + " --kubeconfig ./${CLUSTER_NAME}.kubeconfig apply -f -; " + kubectl_cmd + " --kubeconfig ./${CLUSTER_NAME}.kubeconfig get configmap kubeadm-config --namespace=kube-system -o yaml | sed 's/namespace: kube-system/namespace: calico-system/' | " + kubectl_cmd + " --kubeconfig ./${CLUSTER_NAME}.kubeconfig apply -f -" + # TODO: no one is clearing MGMT_CLUSTER_NAME when using KIND, so this is always going to be true. Improve this logic. + if "aks" in settings.get("kustomize_substitutions", {}).get("MGMT_CLUSTER_NAME", ""): + flavor_cmd += peer_vnets() + flavor_cmd += get_addons(flavor_name) local_resource( @@ -454,6 +475,63 @@ def waitforsystem(): local(kubectl_cmd + " wait --for=condition=ready --timeout=300s pod --all -n capi-kubeadm-control-plane-system") local(kubectl_cmd + " wait --for=condition=ready --timeout=300s pod --all -n capi-system") +def peer_vnets(): + # TODO: check for az cli to be installed in local + # wait for AKS VNet to be in the state created + peering_cmd = "; echo \"--------Peering VNETs--------\"" + peering_cmd += "; az network vnet wait --resource-group ${AKS_RESOURCE_GROUP} --name ${AKS_MGMT_VNET_NAME} --created --timeout 180" + peering_cmd += "; export MGMT_VNET_ID=$(az network vnet show --resource-group ${AKS_RESOURCE_GROUP} --name ${AKS_MGMT_VNET_NAME} --query id --output tsv)" + peering_cmd += "; echo \" 1/8 ${AKS_MGMT_VNET_NAME} found \"" + + # wait for workload VNet to be created + peering_cmd += "; az network vnet wait --resource-group ${CLUSTER_NAME} --name ${CLUSTER_NAME}-vnet --created --timeout 180" + peering_cmd += "; export WORKLOAD_VNET_ID=$(az network vnet show --resource-group ${CLUSTER_NAME} --name ${CLUSTER_NAME}-vnet --query id --output tsv)" + peering_cmd += "; echo \" 2/8 ${CLUSTER_NAME}-vnet found \"" + + # peer mgmt vnet + peering_cmd += "; az network vnet peering create --name mgmt-to-${CLUSTER_NAME} --resource-group ${AKS_RESOURCE_GROUP} --vnet-name ${AKS_MGMT_VNET_NAME} --remote-vnet \"${WORKLOAD_VNET_ID}\" --allow-vnet-access true --allow-forwarded-traffic true --only-show-errors --output none" + peering_cmd += "; az network vnet peering wait --name mgmt-to-${CLUSTER_NAME} --resource-group ${AKS_RESOURCE_GROUP} --vnet-name ${AKS_MGMT_VNET_NAME} --created --timeout 300 --only-show-errors --output none" + peering_cmd += "; echo \" 3/8 mgmt-to-${CLUSTER_NAME} peering created in ${AKS_MGMT_VNET_NAME}\"" + + # peer workload vnet + peering_cmd += "; az network vnet peering create --name ${CLUSTER_NAME}-to-mgmt --resource-group ${CLUSTER_NAME} --vnet-name ${CLUSTER_NAME}-vnet --remote-vnet \"${MGMT_VNET_ID}\" --allow-vnet-access true --allow-forwarded-traffic true --only-show-errors --output none" + peering_cmd += "; az network vnet peering wait --name ${CLUSTER_NAME}-to-mgmt --resource-group ${CLUSTER_NAME} --vnet-name ${CLUSTER_NAME}-vnet --created --timeout 300 --only-show-errors --output none" + peering_cmd += "; echo \" 4/8 ${CLUSTER_NAME}-to-mgmt peering created in ${CLUSTER_NAME}-vnet\"" + + # create private DNS zone + peering_cmd += "; az network private-dns zone create --resource-group ${CLUSTER_NAME} --name ${AZURE_LOCATION}.cloudapp.azure.com --only-show-errors --output none" + peering_cmd += "; az network private-dns zone wait --resource-group ${CLUSTER_NAME} --name ${AZURE_LOCATION}.cloudapp.azure.com --created --timeout 300 --only-show-errors --output none" + peering_cmd += "; echo \" 5/8 ${AZURE_LOCATION}.cloudapp.azure.com private DNS zone created in ${CLUSTER_NAME}\"" + + # link private DNS Zone to workload vnet + peering_cmd += "; az network private-dns link vnet create --resource-group ${CLUSTER_NAME} --zone-name ${AZURE_LOCATION}.cloudapp.azure.com --name ${CLUSTER_NAME}-to-mgmt --virtual-network \"${WORKLOAD_VNET_ID}\" --registration-enabled false --only-show-errors --output none" + peering_cmd += "; az network private-dns link vnet wait --resource-group ${CLUSTER_NAME} --zone-name ${AZURE_LOCATION}.cloudapp.azure.com --name ${CLUSTER_NAME}-to-mgmt --created --timeout 300 --only-show-errors --output none" + peering_cmd += "; echo \" 6/8 workload cluster vnet ${CLUSTER_NAME}-vnet linked with private DNS zone\"" + + # link private DNS Zone to mgmt vnet + peering_cmd += "; az network private-dns link vnet create --resource-group ${CLUSTER_NAME} --zone-name ${AZURE_LOCATION}.cloudapp.azure.com --name mgmt-to-${CLUSTER_NAME} --virtual-network \"${MGMT_VNET_ID}\" --registration-enabled false --only-show-errors --output none" + peering_cmd += "; az network private-dns link vnet wait --resource-group ${CLUSTER_NAME} --zone-name ${AZURE_LOCATION}.cloudapp.azure.com --name mgmt-to-${CLUSTER_NAME} --created --timeout 300 --only-show-errors --output none" + peering_cmd += "; echo \" 7/8 management cluster vnet ${AKS_MGMT_VNET_NAME} linked with private DNS zone\"" + + # create private DNS zone record + # TODO: 10.0.0.100 should be customizable + peering_cmd += "; az network private-dns record-set a add-record --resource-group ${CLUSTER_NAME} --zone-name ${AZURE_LOCATION}.cloudapp.azure.com --record-set-name ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX} --ipv4-address 10.0.0.100 --only-show-errors --output none" + peering_cmd += "; echo \" 8/8 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX} private DNS zone record created\n\"" + + return peering_cmd + +def clear_aks_vnet_peerings(): + delete_peering_cmd = "; echo \"--------Clearing AKS MGMT VNETs Peerings--------\"" + delete_peering_cmd += "; az network vnet wait --resource-group ${AKS_RESOURCE_GROUP} --name ${AKS_MGMT_VNET_NAME} --created --timeout 180" + delete_peering_cmd += "; echo \" ${AKS_MGMT_VNET_NAME} found \"" + + # List all peering names and store them in an array + delete_peering_cmd += "; PEERING_NAMES=$(az network vnet peering list --resource-group ${AKS_RESOURCE_GROUP} --vnet-name ${AKS_MGMT_VNET_NAME} --query \"[].name\" --output tsv)" + delete_peering_cmd += "; for PEERING_NAME in ${PEERING_NAMES[@]}; do echo \"Deleting peering: ${PEERING_NAME}\"; az network vnet peering delete --name ${PEERING_NAME} --resource-group ${AKS_RESOURCE_GROUP} --vnet-name ${AKS_MGMT_VNET_NAME}; done" + delete_peering_cmd += "; echo \"All VNETs Peerings deleted in ${AKS_MGMT_VNET_NAME}\"" + + return delete_peering_cmd + ############################## # Actual work happens here ############################## From e8c08d30e1f63024574ade98e6849c6540f1b520 Mon Sep 17 00:00:00 2001 From: Nawaz Hussain Khazielakha Date: Thu, 14 Nov 2024 14:59:02 -0800 Subject: [PATCH 02/23] update aks-as-mgmt scripts with VNet creation and all clusters deletion --- scripts/aks-as-mgmt.sh | 42 ++++++++++++++++++++++++++++++++---------- 1 file changed, 32 insertions(+), 10 deletions(-) diff --git a/scripts/aks-as-mgmt.sh b/scripts/aks-as-mgmt.sh index 1d97708c420..1fffb9a08bc 100755 --- a/scripts/aks-as-mgmt.sh +++ b/scripts/aks-as-mgmt.sh @@ -30,7 +30,7 @@ make --directory="${REPO_ROOT}" "${KUBECTL##*/}" "${AZWI##*/}" export MGMT_CLUSTER_NAME="${MGMT_CLUSTER_NAME:-aks-mgmt-capz-${RANDOM_SUFFIX}}" # management cluster name export AKS_RESOURCE_GROUP="${AKS_RESOURCE_GROUP:-aks-mgmt-capz-${RANDOM_SUFFIX}}" # resource group name export AKS_NODE_RESOURCE_GROUP="node-${AKS_RESOURCE_GROUP}" -export KUBERNETES_VERSION="${KUBERNETES_VERSION:-v1.30.2}" +export AKS_MGMT_KUBERNETES_VERSION="${AKS_MGMT_KUBERNETES_VERSION:-v1.30.2}" export AZURE_LOCATION="${AZURE_LOCATION:-westus2}" export AKS_NODE_VM_SIZE="${AKS_NODE_VM_SIZE:-"Standard_B2s"}" export AKS_NODE_COUNT="${AKS_NODE_COUNT:-1}" @@ -42,6 +42,13 @@ export AZWI_STORAGE_CONTAINER="\$web" export SERVICE_ACCOUNT_SIGNING_PUB_FILEPATH="${SERVICE_ACCOUNT_SIGNING_PUB_FILEPATH:-}" export SERVICE_ACCOUNT_SIGNING_KEY_FILEPATH="${SERVICE_ACCOUNT_SIGNING_KEY_FILEPATH:-}" export REGISTRY="${REGISTRY:-}" +export AKS_MGMT_VNET_NAME="${AKS_MGMT_VNET_NAME:-"aks-mgmt-vnet-${RANDOM_SUFFIX}"}" +export AKS_MGMT_VNET_CIDR="${AKS_MGMT_VNET_CIDR:-"20.255.0.0/16"}" +export AKS_MGMT_SERVICE_CIDR="${AKS_MGMT_SERVICE_CIDR:-"20.255.254.0/24"}" +export AKS_MGMT_DNS_SERVICE_IP="${AKS_MGMT_DNS_SERVICE_IP:-"20.255.254.100"}" +export AKS_MGMT_SUBNET_NAME="${AKS_MGMT_SUBNET_NAME:-"aks-mgmt-subnet-${RANDOM_SUFFIX}"}" +export AKS_MGMT_SUBNET_CIDR="${AKS_MGMT_SUBNET_CIDR:-"20.255.0.0/24"}" + export AZURE_SUBSCRIPTION_ID="${AZURE_SUBSCRIPTION_ID:-}" export AZURE_CLIENT_ID="${AZURE_CLIENT_ID:-}" @@ -63,7 +70,7 @@ main() { echo "MGMT_CLUSTER_NAME: $MGMT_CLUSTER_NAME" echo "AKS_RESOURCE_GROUP: $AKS_RESOURCE_GROUP" echo "AKS_NODE_RESOURCE_GROUP: $AKS_NODE_RESOURCE_GROUP" - echo "KUBERNETES_VERSION: $KUBERNETES_VERSION" + echo "AKS_MGMT_KUBERNETES_VERSION: $AKS_MGMT_KUBERNETES_VERSION" echo "AZURE_LOCATION: $AZURE_LOCATION" echo "AKS_NODE_VM_SIZE: $AKS_NODE_VM_SIZE" echo "AZURE_NODE_MACHINE_TYPE: $AZURE_NODE_MACHINE_TYPE" @@ -76,6 +83,12 @@ main() { echo "SERVICE_ACCOUNT_SIGNING_KEY_FILEPATH: $SERVICE_ACCOUNT_SIGNING_KEY_FILEPATH" echo "REGISTRY: $REGISTRY" echo "APISERVER_LB_DNS_SUFFIX: $APISERVER_LB_DNS_SUFFIX" + echo "AKS_MGMT_VNET_NAME: $AKS_MGMT_VNET_NAME" + echo "AKS_MGMT_VNET_CIDR: $AKS_MGMT_VNET_CIDR" + echo "AKS_MGMT_SERVICE_CIDR: $AKS_MGMT_SERVICE_CIDR" + echo "AKS_MGMT_DNS_SERVICE_IP: $AKS_MGMT_DNS_SERVICE_IP" + echo "AKS_MGMT_SUBNET_NAME: $AKS_MGMT_SUBNET_NAME" + echo "AKS_MGMT_SUBNET_CIDR: $AKS_MGMT_SUBNET_CIDR" echo "AZURE_SUBSCRIPTION_ID: $AZURE_SUBSCRIPTION_ID" echo "AZURE_CLIENT_ID: $AZURE_CLIENT_ID" @@ -102,6 +115,16 @@ create_aks_cluster() { --location "${AZURE_LOCATION}" \ --output none --only-show-errors \ --tags creationTimestamp="${TIMESTAMP}" jobName="${JOB_NAME}" buildProvenance="${BUILD_PROVENANCE}" + + echo "creating vnet for the resource group ${AKS_RESOURCE_GROUP}" + az network vnet create \ + --resource-group "${AKS_RESOURCE_GROUP}"\ + --name "${AKS_MGMT_VNET_NAME}" \ + --address-prefix "${AKS_MGMT_VNET_CIDR}" \ + --subnet-name "${AKS_MGMT_SUBNET_NAME}" \ + --subnet-prefix "${AKS_MGMT_SUBNET_CIDR}" \ + --output none --only-show-errors \ + --tags creationTimestamp="${TIMESTAMP}" jobName="${JOB_NAME}" buildProvenance="${BUILD_PROVENANCE}" fi aks_exists=$(az aks show --name "${MGMT_CLUSTER_NAME}" --resource-group "${AKS_RESOURCE_GROUP}" 2>&1 || true) # true because we want to continue if the command fails @@ -110,13 +133,16 @@ create_aks_cluster() { az aks create --name "${MGMT_CLUSTER_NAME}" \ --resource-group "${AKS_RESOURCE_GROUP}" \ --location "${AZURE_LOCATION}" \ - --kubernetes-version "${KUBERNETES_VERSION}" \ + --kubernetes-version "${AKS_MGMT_KUBERNETES_VERSION}" \ --node-count "${AKS_NODE_COUNT}" \ --node-vm-size "${AKS_NODE_VM_SIZE}" \ --node-resource-group "${AKS_NODE_RESOURCE_GROUP}" \ --vm-set-type VirtualMachineScaleSets \ --generate-ssh-keys \ --network-plugin azure \ + --vnet-subnet-id "/subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${AKS_RESOURCE_GROUP}/providers/Microsoft.Network/virtualNetworks/${AKS_MGMT_VNET_NAME}/subnets/${AKS_MGMT_SUBNET_NAME}" \ + --service-cidr "${AKS_MGMT_SERVICE_CIDR}" \ + --dns-service-ip "${AKS_MGMT_DNS_SERVICE_IP}" \ --tags creationTimestamp="${TIMESTAMP}" jobName="${JOB_NAME}" buildProvenance="${BUILD_PROVENANCE}" \ --output none --only-show-errors; elif echo "$aks_exists" | grep -q "${MGMT_CLUSTER_NAME}"; then @@ -127,6 +153,7 @@ create_aks_cluster() { fi # check and save kubeconfig + echo -e "\n" echo "saving credentials of cluster ${MGMT_CLUSTER_NAME} in ${REPO_ROOT}/${MGMT_CLUSTER_KUBECONFIG}" az aks get-credentials --name "${MGMT_CLUSTER_NAME}" --resource-group "${AKS_RESOURCE_GROUP}" \ --file "${REPO_ROOT}/${MGMT_CLUSTER_KUBECONFIG}" --only-show-errors @@ -179,15 +206,10 @@ create_aks_cluster() { set_env_varaibles(){ cat < tilt-settings-temp.yaml kustomize_substitutions: - MGMT_CLUSTER_NAME: "${MGMT_CLUSTER_NAME}" AKS_RESOURCE_GROUP: "${AKS_RESOURCE_GROUP}" AKS_NODE_RESOURCE_GROUP: "${AKS_NODE_RESOURCE_GROUP}" - MGMT_CLUSTER_KUBECONFIG: "${MGMT_CLUSTER_KUBECONFIG}" - AKS_MI_CLIENT_ID: "${AKS_MI_CLIENT_ID}" - AKS_MI_OBJECT_ID: "${AKS_MI_OBJECT_ID}" - AKS_MI_RESOURCE_ID: "${AKS_MI_RESOURCE_ID}" - MANAGED_IDENTITY_NAME: "${MANAGED_IDENTITY_NAME}" - MANAGED_IDENTITY_RG: "${MANAGED_IDENTITY_RG}" + AKS_MGMT_VNET_NAME: "${AKS_MGMT_VNET_NAME}" + MGMT_CLUSTER_NAME: "${MGMT_CLUSTER_NAME}" AZURE_CLIENT_ID_USER_ASSIGNED_IDENTITY: "${AKS_MI_CLIENT_ID}" CI_RG: "${MANAGED_IDENTITY_RG}" USER_IDENTITY: "${MANAGED_IDENTITY_NAME}" From 546030b9f143fb137d32f0dac492955fa188a288 Mon Sep 17 00:00:00 2001 From: Nawaz Hussain Khazielakha Date: Thu, 14 Nov 2024 15:33:46 -0800 Subject: [PATCH 03/23] update default template --- templates/cluster-template.yaml | 12 ++++++-- templates/flavors/default/kustomization.yaml | 4 +-- .../azure-cluster-vnet-subnet-cidrs.yaml | 28 +++++++++++++++++++ .../kubeadm-config-template-worker-node.yaml | 13 +++++++++ 4 files changed, 53 insertions(+), 4 deletions(-) create mode 100644 templates/flavors/default/patches/azure-cluster-vnet-subnet-cidrs.yaml create mode 100644 templates/flavors/default/patches/kubeadm-config-template-worker-node.yaml diff --git a/templates/cluster-template.yaml b/templates/cluster-template.yaml index a18b181f968..0426726c5a4 100644 --- a/templates/cluster-template.yaml +++ b/templates/cluster-template.yaml @@ -35,12 +35,20 @@ spec: publicIP: dnsName: ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com name: ${CLUSTER_NAME}-api-lb + - name: ${CLUSTER_NAME}-internal-lb-private-ip + privateIP: 10.0.0.100 subnets: - - name: control-plane-subnet + - cidrBlocks: + - 10.0.0.0/16 + name: control-plane-subnet role: control-plane - - name: node-subnet + - cidrBlocks: + - 10.1.0.0/16 + name: node-subnet role: node vnet: + cidrBlocks: + - 10.0.0.0/8 name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} subscriptionID: ${AZURE_SUBSCRIPTION_ID} diff --git a/templates/flavors/default/kustomization.yaml b/templates/flavors/default/kustomization.yaml index eb0e4743560..73db52cf938 100644 --- a/templates/flavors/default/kustomization.yaml +++ b/templates/flavors/default/kustomization.yaml @@ -8,8 +8,8 @@ resources: patches: - path: ../../azure-cluster-identity/azurecluster-identity-ref.yaml -- path: ../../internal-load-balancer/azure-cluster-frontend-ip.yaml -- path: ../../internal-load-balancer/kubeadm-config-template-worker-node.yaml +- path: patches/azure-cluster-vnet-subnet-cidrs.yaml +- path: patches/kubeadm-config-template-worker-node.yaml sortOptions: order: fifo diff --git a/templates/flavors/default/patches/azure-cluster-vnet-subnet-cidrs.yaml b/templates/flavors/default/patches/azure-cluster-vnet-subnet-cidrs.yaml new file mode 100644 index 00000000000..92e854c12eb --- /dev/null +++ b/templates/flavors/default/patches/azure-cluster-vnet-subnet-cidrs.yaml @@ -0,0 +1,28 @@ +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureCluster +metadata: + name: ${CLUSTER_NAME} +spec: + networkSpec: + apiServerLB: + # We pre-create this public IP and the DNS name to use it in the + # worker node's /etc/hosts. + frontendIPs: + - name: ${CLUSTER_NAME}-api-lb + publicIP: + name: ${CLUSTER_NAME}-api-lb + dnsName: ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com + - name: ${CLUSTER_NAME}-internal-lb-private-ip + privateIP: 10.0.0.100 + vnet: + cidrBlocks: + - 10.0.0.0/8 + subnets: + - name: control-plane-subnet + role: control-plane + cidrBlocks: + - 10.0.0.0/16 + - name: node-subnet + role: node + cidrBlocks: + - 10.1.0.0/16 diff --git a/templates/flavors/default/patches/kubeadm-config-template-worker-node.yaml b/templates/flavors/default/patches/kubeadm-config-template-worker-node.yaml new file mode 100644 index 00000000000..6f5dc9a772c --- /dev/null +++ b/templates/flavors/default/patches/kubeadm-config-template-worker-node.yaml @@ -0,0 +1,13 @@ +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: "${CLUSTER_NAME}-md-0" +spec: + template: + spec: + # /etc/hosts file is updated with a pre-created DNS name of the API server and internal load-balancer's IP. + # This custom DNS Resolution of the API server ensures that the worker nodes can reach the API server when + # the public IP of the API server is not accessible. + # 10.0.0.100 is the default IP that gets assigned to an internal load balancer. + preKubeadmCommands: + - echo '10.0.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' >> /etc/hosts From ec05d10d91c9eb8a7c215dbf0c142b6420ee5403 Mon Sep 17 00:00:00 2001 From: Nawaz Hussain Khazielakha Date: Thu, 14 Nov 2024 16:00:22 -0800 Subject: [PATCH 04/23] update default template --- templates/cluster-template.yaml | 6 +++--- .../default/patches/azure-cluster-vnet-subnet-cidrs.yaml | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/templates/cluster-template.yaml b/templates/cluster-template.yaml index 0426726c5a4..22a9af7a46c 100644 --- a/templates/cluster-template.yaml +++ b/templates/cluster-template.yaml @@ -39,16 +39,16 @@ spec: privateIP: 10.0.0.100 subnets: - cidrBlocks: - - 10.0.0.0/16 + - 10.0.0.0/24 name: control-plane-subnet role: control-plane - cidrBlocks: - - 10.1.0.0/16 + - 10.1.0.0/24 name: node-subnet role: node vnet: cidrBlocks: - - 10.0.0.0/8 + - 10.0.0.0/16 name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} subscriptionID: ${AZURE_SUBSCRIPTION_ID} diff --git a/templates/flavors/default/patches/azure-cluster-vnet-subnet-cidrs.yaml b/templates/flavors/default/patches/azure-cluster-vnet-subnet-cidrs.yaml index 92e854c12eb..c7f44d96a0c 100644 --- a/templates/flavors/default/patches/azure-cluster-vnet-subnet-cidrs.yaml +++ b/templates/flavors/default/patches/azure-cluster-vnet-subnet-cidrs.yaml @@ -16,13 +16,13 @@ spec: privateIP: 10.0.0.100 vnet: cidrBlocks: - - 10.0.0.0/8 + - 10.0.0.0/16 subnets: - name: control-plane-subnet role: control-plane cidrBlocks: - - 10.0.0.0/16 + - 10.0.0.0/24 - name: node-subnet role: node cidrBlocks: - - 10.1.0.0/16 + - 10.1.0.0/24 From d4fcec6b319767f870e9e3b3b131ad913bb548fe Mon Sep 17 00:00:00 2001 From: Nawaz Hussain Khazielakha Date: Thu, 14 Nov 2024 16:20:59 -0800 Subject: [PATCH 05/23] update default template --- templates/flavors/default/kustomization.yaml | 4 ++-- .../kubeadm-config-template-worker-node.yaml | 13 ------------- .../azure-cluster-cidrs-and-frontend-ips.yaml} | 0 .../azure-cluster-frontend-ip.yaml | 14 -------------- 4 files changed, 2 insertions(+), 29 deletions(-) delete mode 100644 templates/flavors/default/patches/kubeadm-config-template-worker-node.yaml rename templates/{flavors/default/patches/azure-cluster-vnet-subnet-cidrs.yaml => internal-load-balancer/azure-cluster-cidrs-and-frontend-ips.yaml} (100%) delete mode 100644 templates/internal-load-balancer/azure-cluster-frontend-ip.yaml diff --git a/templates/flavors/default/kustomization.yaml b/templates/flavors/default/kustomization.yaml index 73db52cf938..333ecdd55d6 100644 --- a/templates/flavors/default/kustomization.yaml +++ b/templates/flavors/default/kustomization.yaml @@ -8,8 +8,8 @@ resources: patches: - path: ../../azure-cluster-identity/azurecluster-identity-ref.yaml -- path: patches/azure-cluster-vnet-subnet-cidrs.yaml -- path: patches/kubeadm-config-template-worker-node.yaml +- path: ../../internal-load-balancer/azure-cluster-cidrs-and-frontend-ips.yaml +- path: ../../internal-load-balancer/kubeadm-config-template-worker-node.yaml sortOptions: order: fifo diff --git a/templates/flavors/default/patches/kubeadm-config-template-worker-node.yaml b/templates/flavors/default/patches/kubeadm-config-template-worker-node.yaml deleted file mode 100644 index 6f5dc9a772c..00000000000 --- a/templates/flavors/default/patches/kubeadm-config-template-worker-node.yaml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 -kind: KubeadmConfigTemplate -metadata: - name: "${CLUSTER_NAME}-md-0" -spec: - template: - spec: - # /etc/hosts file is updated with a pre-created DNS name of the API server and internal load-balancer's IP. - # This custom DNS Resolution of the API server ensures that the worker nodes can reach the API server when - # the public IP of the API server is not accessible. - # 10.0.0.100 is the default IP that gets assigned to an internal load balancer. - preKubeadmCommands: - - echo '10.0.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' >> /etc/hosts diff --git a/templates/flavors/default/patches/azure-cluster-vnet-subnet-cidrs.yaml b/templates/internal-load-balancer/azure-cluster-cidrs-and-frontend-ips.yaml similarity index 100% rename from templates/flavors/default/patches/azure-cluster-vnet-subnet-cidrs.yaml rename to templates/internal-load-balancer/azure-cluster-cidrs-and-frontend-ips.yaml diff --git a/templates/internal-load-balancer/azure-cluster-frontend-ip.yaml b/templates/internal-load-balancer/azure-cluster-frontend-ip.yaml deleted file mode 100644 index 45a584e4a02..00000000000 --- a/templates/internal-load-balancer/azure-cluster-frontend-ip.yaml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureCluster -metadata: - name: ${CLUSTER_NAME} -spec: - networkSpec: - apiServerLB: - # We pre-create this public IP and the DNS name to use it in the - # worker node's /etc/hosts. - frontendIPs: - - name: ${CLUSTER_NAME}-api-lb - publicIP: - name: ${CLUSTER_NAME}-api-lb - dnsName: ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com From d6155af78a7b7c66d14f5acbc333ccd55c751372 Mon Sep 17 00:00:00 2001 From: Nawaz Hussain Khazielakha Date: Thu, 14 Nov 2024 16:23:25 -0800 Subject: [PATCH 06/23] update default aad --- templates/cluster-template-aad.yaml | 14 +++++++++-- templates/flavors/aad/kustomization.yaml | 32 +++++++++++++++++++++++- 2 files changed, 43 insertions(+), 3 deletions(-) diff --git a/templates/cluster-template-aad.yaml b/templates/cluster-template-aad.yaml index 44a5d2b4455..1e95d8bbee9 100644 --- a/templates/cluster-template-aad.yaml +++ b/templates/cluster-template-aad.yaml @@ -35,12 +35,20 @@ spec: publicIP: dnsName: ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com name: ${CLUSTER_NAME}-api-lb + - name: ${CLUSTER_NAME}-internal-lb-private-ip + privateIP: 10.1.0.100 subnets: - - name: control-plane-subnet + - cidrBlocks: + - 10.1.0.0/24 + name: control-plane-subnet role: control-plane - - name: node-subnet + - cidrBlocks: + - 10.1.1.0/24 + name: node-subnet role: node vnet: + cidrBlocks: + - 10.1.0.0/16 name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} subscriptionID: ${AZURE_SUBSCRIPTION_ID} @@ -205,6 +213,8 @@ spec: preKubeadmCommands: - echo '10.0.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' >> /etc/hosts + - echo '10.1.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' + >> /etc/hosts --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 kind: AzureClusterIdentity diff --git a/templates/flavors/aad/kustomization.yaml b/templates/flavors/aad/kustomization.yaml index de60f8706b0..74f150305be 100644 --- a/templates/flavors/aad/kustomization.yaml +++ b/templates/flavors/aad/kustomization.yaml @@ -8,8 +8,38 @@ resources: patches: - path: patches/kubeadm-controlplane.yaml - path: ../../azure-cluster-identity/azurecluster-identity-ref.yaml -- path: ../../internal-load-balancer/azure-cluster-frontend-ip.yaml +- path: ../../internal-load-balancer/azure-cluster-cidrs-and-frontend-ips.yaml - path: ../../internal-load-balancer/kubeadm-config-template-worker-node.yaml +- target: + kind: KubeadmConfigTemplate + patch: |- + - op: add + path: /spec/template/spec/preKubeadmCommands/- + value: echo '10.1.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' >> /etc/hosts +- target: + kind: AzureCluster + patch: |- + - op: replace + path: /spec/networkSpec/apiServerLB/frontendIPs/1/privateIP + value: 10.1.0.100 +- target: + kind: AzureCluster + patch: |- + - op: replace + path: /spec/networkSpec/vnet/cidrBlocks/0 + value: 10.1.0.0/16 +- target: + kind: AzureCluster + patch: |- + - op: replace + path: /spec/networkSpec/subnets/0/cidrBlocks/0 + value: 10.1.0.0/24 +- target: + kind: AzureCluster + patch: |- + - op: replace + path: /spec/networkSpec/subnets/1/cidrBlocks/0 + value: 10.1.1.0/24 sortOptions: order: fifo From 56eb7995d6b5fa9d736fa3c7256c6aed6240c839 Mon Sep 17 00:00:00 2001 From: Nawaz Hussain Khazielakha Date: Thu, 14 Nov 2024 17:58:37 -0800 Subject: [PATCH 07/23] update default aad --- templates/cluster-template-aad.yaml | 2 -- templates/flavors/aad/kustomization.yaml | 5 +++++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/templates/cluster-template-aad.yaml b/templates/cluster-template-aad.yaml index 1e95d8bbee9..3019626550b 100644 --- a/templates/cluster-template-aad.yaml +++ b/templates/cluster-template-aad.yaml @@ -211,8 +211,6 @@ spec: cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: - - echo '10.0.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' - >> /etc/hosts - echo '10.1.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' >> /etc/hosts --- diff --git a/templates/flavors/aad/kustomization.yaml b/templates/flavors/aad/kustomization.yaml index 74f150305be..ae8b5daebbe 100644 --- a/templates/flavors/aad/kustomization.yaml +++ b/templates/flavors/aad/kustomization.yaml @@ -40,6 +40,11 @@ patches: - op: replace path: /spec/networkSpec/subnets/1/cidrBlocks/0 value: 10.1.1.0/24 +- target: + kind: KubeadmConfigTemplate + patch: |- + - op: remove + path: /spec/template/spec/preKubeadmCommands/0 sortOptions: order: fifo From b5ea502b707745340b033ff3683eeeccc9d7913e Mon Sep 17 00:00:00 2001 From: Nawaz Hussain Khazielakha Date: Thu, 14 Nov 2024 17:59:27 -0800 Subject: [PATCH 08/23] update azure-bastion templates --- templates/cluster-template-azure-bastion.yaml | 19 ++++++++-- .../flavors/azure-bastion/kustomization.yaml | 37 ++++++++++++++++++- .../azure-bastion/patches/azure-cluster.yaml | 5 ++- 3 files changed, 55 insertions(+), 6 deletions(-) diff --git a/templates/cluster-template-azure-bastion.yaml b/templates/cluster-template-azure-bastion.yaml index ba29dfeb37a..cc5b576a15b 100644 --- a/templates/cluster-template-azure-bastion.yaml +++ b/templates/cluster-template-azure-bastion.yaml @@ -24,7 +24,10 @@ metadata: namespace: default spec: bastionSpec: - azureBastion: {} + azureBastion: + subnet: + cidrBlocks: + - 10.2.3.0/24 identityRef: apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 kind: AzureClusterIdentity @@ -37,12 +40,20 @@ spec: publicIP: dnsName: ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com name: ${CLUSTER_NAME}-api-lb + - name: ${CLUSTER_NAME}-internal-lb-private-ip + privateIP: 10.2.0.100 subnets: - - name: control-plane-subnet + - cidrBlocks: + - 10.2.0.0/24 + name: control-plane-subnet role: control-plane - - name: node-subnet + - cidrBlocks: + - 10.2.1.0/24 + name: node-subnet role: node vnet: + cidrBlocks: + - 10.2.0.0/16 name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} subscriptionID: ${AZURE_SUBSCRIPTION_ID} @@ -198,7 +209,7 @@ spec: cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: - - echo '10.0.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' + - echo '10.2.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' >> /etc/hosts --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 diff --git a/templates/flavors/azure-bastion/kustomization.yaml b/templates/flavors/azure-bastion/kustomization.yaml index f39b9e3e32c..01ebddc55e5 100644 --- a/templates/flavors/azure-bastion/kustomization.yaml +++ b/templates/flavors/azure-bastion/kustomization.yaml @@ -8,8 +8,43 @@ resources: patches: - path: ../../azure-cluster-identity/azurecluster-identity-ref.yaml - path: patches/azure-cluster.yaml -- path: ../../internal-load-balancer/azure-cluster-frontend-ip.yaml +- path: ../../internal-load-balancer/azure-cluster-cidrs-and-frontend-ips.yaml - path: ../../internal-load-balancer/kubeadm-config-template-worker-node.yaml +- target: + kind: KubeadmConfigTemplate + patch: |- + - op: add + path: /spec/template/spec/preKubeadmCommands/- + value: echo '10.2.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' >> /etc/hosts +- target: + kind: AzureCluster + patch: |- + - op: replace + path: /spec/networkSpec/apiServerLB/frontendIPs/1/privateIP + value: 10.2.0.100 +- target: + kind: AzureCluster + patch: |- + - op: replace + path: /spec/networkSpec/vnet/cidrBlocks/0 + value: 10.2.0.0/16 +- target: + kind: AzureCluster + patch: |- + - op: replace + path: /spec/networkSpec/subnets/0/cidrBlocks/0 + value: 10.2.0.0/24 +- target: + kind: AzureCluster + patch: |- + - op: replace + path: /spec/networkSpec/subnets/1/cidrBlocks/0 + value: 10.2.1.0/24 +- target: + kind: KubeadmConfigTemplate + patch: |- + - op: remove + path: /spec/template/spec/preKubeadmCommands/0 sortOptions: order: fifo diff --git a/templates/flavors/azure-bastion/patches/azure-cluster.yaml b/templates/flavors/azure-bastion/patches/azure-cluster.yaml index 74d517df341..be525fe1036 100644 --- a/templates/flavors/azure-bastion/patches/azure-cluster.yaml +++ b/templates/flavors/azure-bastion/patches/azure-cluster.yaml @@ -5,4 +5,7 @@ metadata: name: ${CLUSTER_NAME} spec: bastionSpec: - azureBastion: {} + azureBastion: + subnet: + cidrBlocks: + - 10.2.3.0/24 From 7247a1d155d1735460198c8c45c513df32c6d07a Mon Sep 17 00:00:00 2001 From: Nawaz Hussain Khazielakha Date: Thu, 14 Nov 2024 18:00:27 -0800 Subject: [PATCH 09/23] update azure-cni-v1 templates --- templates/cluster-template-azure-cni-v1.yaml | 14 +++++-- .../flavors/azure-cni-v1/kustomization.yaml | 37 ++++++++++++++++++- .../cluster-template-prow-azure-cni-v1.yaml | 14 +++++-- 3 files changed, 58 insertions(+), 7 deletions(-) diff --git a/templates/cluster-template-azure-cni-v1.yaml b/templates/cluster-template-azure-cni-v1.yaml index 7704dcfc083..b2f2942f1e4 100644 --- a/templates/cluster-template-azure-cni-v1.yaml +++ b/templates/cluster-template-azure-cni-v1.yaml @@ -35,12 +35,20 @@ spec: publicIP: dnsName: ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com name: ${CLUSTER_NAME}-api-lb + - name: ${CLUSTER_NAME}-internal-lb-private-ip + privateIP: 10.3.0.100 subnets: - - name: control-plane-subnet + - cidrBlocks: + - 10.3.0.0/24 + name: control-plane-subnet role: control-plane - - name: node-subnet + - cidrBlocks: + - 10.3.1.0/24 + name: node-subnet role: node vnet: + cidrBlocks: + - 10.3.0.0/16 name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} subscriptionID: ${AZURE_SUBSCRIPTION_ID} @@ -205,7 +213,7 @@ spec: max-pods: "110" name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: - - echo '10.0.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' + - echo '10.3.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' >> /etc/hosts --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 diff --git a/templates/flavors/azure-cni-v1/kustomization.yaml b/templates/flavors/azure-cni-v1/kustomization.yaml index 43779e89117..277594f0f09 100644 --- a/templates/flavors/azure-cni-v1/kustomization.yaml +++ b/templates/flavors/azure-cni-v1/kustomization.yaml @@ -11,8 +11,43 @@ patches: - path: patches/azure-machine-template.yaml - path: patches/kubeadm-control-plane.yaml - path: patches/kubeadm-worker-node.yaml -- path: ../../internal-load-balancer/azure-cluster-frontend-ip.yaml +- path: ../../internal-load-balancer/azure-cluster-cidrs-and-frontend-ips.yaml - path: ../../internal-load-balancer/kubeadm-config-template-worker-node.yaml +- target: + kind: KubeadmConfigTemplate + patch: |- + - op: add + path: /spec/template/spec/preKubeadmCommands/- + value: echo '10.3.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' >> /etc/hosts +- target: + kind: AzureCluster + patch: |- + - op: replace + path: /spec/networkSpec/apiServerLB/frontendIPs/1/privateIP + value: 10.3.0.100 +- target: + kind: AzureCluster + patch: |- + - op: replace + path: /spec/networkSpec/vnet/cidrBlocks/0 + value: 10.3.0.0/16 +- target: + kind: AzureCluster + patch: |- + - op: replace + path: /spec/networkSpec/subnets/0/cidrBlocks/0 + value: 10.3.0.0/24 +- target: + kind: AzureCluster + patch: |- + - op: replace + path: /spec/networkSpec/subnets/1/cidrBlocks/0 + value: 10.3.1.0/24 +- target: + kind: KubeadmConfigTemplate + patch: |- + - op: remove + path: /spec/template/spec/preKubeadmCommands/0 sortOptions: order: fifo diff --git a/templates/test/ci/cluster-template-prow-azure-cni-v1.yaml b/templates/test/ci/cluster-template-prow-azure-cni-v1.yaml index 760210ba5a5..afdf70a91de 100644 --- a/templates/test/ci/cluster-template-prow-azure-cni-v1.yaml +++ b/templates/test/ci/cluster-template-prow-azure-cni-v1.yaml @@ -41,12 +41,20 @@ spec: publicIP: dnsName: ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com name: ${CLUSTER_NAME}-api-lb + - name: ${CLUSTER_NAME}-internal-lb-private-ip + privateIP: 10.3.0.100 subnets: - - name: control-plane-subnet + - cidrBlocks: + - 10.3.0.0/24 + name: control-plane-subnet role: control-plane - - name: node-subnet + - cidrBlocks: + - 10.3.1.0/24 + name: node-subnet role: node vnet: + cidrBlocks: + - 10.3.0.0/16 name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} subscriptionID: ${AZURE_SUBSCRIPTION_ID} @@ -212,7 +220,7 @@ spec: max-pods: "110" name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: - - echo '10.0.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' + - echo '10.3.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' >> /etc/hosts --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 From e6d0dda64c3d1ef5dea9046fdc67bb7ec8cc15d8 Mon Sep 17 00:00:00 2001 From: Nawaz Hussain Khazielakha Date: Thu, 14 Nov 2024 18:03:21 -0800 Subject: [PATCH 10/23] update edgezone templates --- templates/cluster-template-edgezone.yaml | 14 +++++-- templates/flavors/edgezone/kustomization.yaml | 37 ++++++++++++++++++- .../ci/cluster-template-prow-edgezone.yaml | 14 +++++-- 3 files changed, 58 insertions(+), 7 deletions(-) diff --git a/templates/cluster-template-edgezone.yaml b/templates/cluster-template-edgezone.yaml index fe5b94184d0..52cdd29d069 100644 --- a/templates/cluster-template-edgezone.yaml +++ b/templates/cluster-template-edgezone.yaml @@ -38,12 +38,20 @@ spec: publicIP: dnsName: ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com name: ${CLUSTER_NAME}-api-lb + - name: ${CLUSTER_NAME}-internal-lb-private-ip + privateIP: 10.4.0.100 subnets: - - name: control-plane-subnet + - cidrBlocks: + - 10.4.0.0/24 + name: control-plane-subnet role: control-plane - - name: node-subnet + - cidrBlocks: + - 10.4.1.0/24 + name: node-subnet role: node vnet: + cidrBlocks: + - 10.4.0.0/16 name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} subscriptionID: ${AZURE_SUBSCRIPTION_ID} @@ -199,7 +207,7 @@ spec: cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: - - echo '10.0.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' + - echo '10.4.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' >> /etc/hosts --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 diff --git a/templates/flavors/edgezone/kustomization.yaml b/templates/flavors/edgezone/kustomization.yaml index 5e44b5bc717..9eb72c2e5d6 100644 --- a/templates/flavors/edgezone/kustomization.yaml +++ b/templates/flavors/edgezone/kustomization.yaml @@ -10,8 +10,43 @@ patches: - path: ../../azure-cluster-identity/azurecluster-identity-ref.yaml - path: patches/azure-extendedlocation.yaml - path: patches/azure-remove-natgateway.yaml -- path: ../../internal-load-balancer/azure-cluster-frontend-ip.yaml +- path: ../../internal-load-balancer/azure-cluster-cidrs-and-frontend-ips.yaml - path: ../../internal-load-balancer/kubeadm-config-template-worker-node.yaml +- target: + kind: KubeadmConfigTemplate + patch: |- + - op: add + path: /spec/template/spec/preKubeadmCommands/- + value: echo '10.4.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' >> /etc/hosts +- target: + kind: AzureCluster + patch: |- + - op: replace + path: /spec/networkSpec/apiServerLB/frontendIPs/1/privateIP + value: 10.4.0.100 +- target: + kind: AzureCluster + patch: |- + - op: replace + path: /spec/networkSpec/vnet/cidrBlocks/0 + value: 10.4.0.0/16 +- target: + kind: AzureCluster + patch: |- + - op: replace + path: /spec/networkSpec/subnets/0/cidrBlocks/0 + value: 10.4.0.0/24 +- target: + kind: AzureCluster + patch: |- + - op: replace + path: /spec/networkSpec/subnets/1/cidrBlocks/0 + value: 10.4.1.0/24 +- target: + kind: KubeadmConfigTemplate + patch: |- + - op: remove + path: /spec/template/spec/preKubeadmCommands/0 sortOptions: order: fifo diff --git a/templates/test/ci/cluster-template-prow-edgezone.yaml b/templates/test/ci/cluster-template-prow-edgezone.yaml index ef1b007edf2..abf35ef1a4b 100644 --- a/templates/test/ci/cluster-template-prow-edgezone.yaml +++ b/templates/test/ci/cluster-template-prow-edgezone.yaml @@ -45,12 +45,20 @@ spec: publicIP: dnsName: ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com name: ${CLUSTER_NAME}-api-lb + - name: ${CLUSTER_NAME}-internal-lb-private-ip + privateIP: 10.4.0.100 subnets: - - name: control-plane-subnet + - cidrBlocks: + - 10.4.0.0/24 + name: control-plane-subnet role: control-plane - - name: node-subnet + - cidrBlocks: + - 10.4.1.0/24 + name: node-subnet role: node vnet: + cidrBlocks: + - 10.4.0.0/16 name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} subscriptionID: ${AZURE_SUBSCRIPTION_ID} @@ -215,7 +223,7 @@ spec: cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: - - echo '10.0.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' + - echo '10.4.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' >> /etc/hosts --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 From 8a8563da7788efc7fe698bad41b4ed910d029585 Mon Sep 17 00:00:00 2001 From: Nawaz Hussain Khazielakha Date: Thu, 14 Nov 2024 18:04:22 -0800 Subject: [PATCH 11/23] update ephemeral templates --- templates/cluster-template-ephemeral.yaml | 14 +++++-- .../flavors/ephemeral/kustomization.yaml | 37 ++++++++++++++++++- 2 files changed, 47 insertions(+), 4 deletions(-) diff --git a/templates/cluster-template-ephemeral.yaml b/templates/cluster-template-ephemeral.yaml index 8415a8f31b7..6712966dba4 100644 --- a/templates/cluster-template-ephemeral.yaml +++ b/templates/cluster-template-ephemeral.yaml @@ -35,12 +35,20 @@ spec: publicIP: dnsName: ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com name: ${CLUSTER_NAME}-api-lb + - name: ${CLUSTER_NAME}-internal-lb-private-ip + privateIP: 10.5.0.100 subnets: - - name: control-plane-subnet + - cidrBlocks: + - 10.5.0.0/24 + name: control-plane-subnet role: control-plane - - name: node-subnet + - cidrBlocks: + - 10.5.1.0/24 + name: node-subnet role: node vnet: + cidrBlocks: + - 10.5.0.0/16 name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} subscriptionID: ${AZURE_SUBSCRIPTION_ID} @@ -202,7 +210,7 @@ spec: cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: - - echo '10.0.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' + - echo '10.5.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' >> /etc/hosts --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 diff --git a/templates/flavors/ephemeral/kustomization.yaml b/templates/flavors/ephemeral/kustomization.yaml index b66c5670a19..b4f5196e128 100644 --- a/templates/flavors/ephemeral/kustomization.yaml +++ b/templates/flavors/ephemeral/kustomization.yaml @@ -20,8 +20,43 @@ patches: kind: AzureMachineTemplate name: .*-control-plane version: v1beta1 -- path: ../../internal-load-balancer/azure-cluster-frontend-ip.yaml +- path: ../../internal-load-balancer/azure-cluster-cidrs-and-frontend-ips.yaml - path: ../../internal-load-balancer/kubeadm-config-template-worker-node.yaml +- target: + kind: KubeadmConfigTemplate + patch: |- + - op: add + path: /spec/template/spec/preKubeadmCommands/- + value: echo '10.5.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' >> /etc/hosts +- target: + kind: AzureCluster + patch: |- + - op: replace + path: /spec/networkSpec/apiServerLB/frontendIPs/1/privateIP + value: 10.5.0.100 +- target: + kind: AzureCluster + patch: |- + - op: replace + path: /spec/networkSpec/vnet/cidrBlocks/0 + value: 10.5.0.0/16 +- target: + kind: AzureCluster + patch: |- + - op: replace + path: /spec/networkSpec/subnets/0/cidrBlocks/0 + value: 10.5.0.0/24 +- target: + kind: AzureCluster + patch: |- + - op: replace + path: /spec/networkSpec/subnets/1/cidrBlocks/0 + value: 10.5.1.0/24 +- target: + kind: KubeadmConfigTemplate + patch: |- + - op: remove + path: /spec/template/spec/preKubeadmCommands/0 sortOptions: order: fifo From e891464bfcbb9ddfb3b3e68c4006d9668f1aecaf Mon Sep 17 00:00:00 2001 From: Nawaz Hussain Khazielakha Date: Thu, 14 Nov 2024 18:05:55 -0800 Subject: [PATCH 12/23] update private templates --- templates/cluster-template-private.yaml | 17 ++++++-- templates/flavors/private/kustomization.yaml | 42 +++++++++++++++++++ .../ci/cluster-template-prow-private.yaml | 4 +- 3 files changed, 59 insertions(+), 4 deletions(-) diff --git a/templates/cluster-template-private.yaml b/templates/cluster-template-private.yaml index 5663fb7403d..ea096297823 100644 --- a/templates/cluster-template-private.yaml +++ b/templates/cluster-template-private.yaml @@ -32,6 +32,9 @@ spec: location: ${AZURE_LOCATION} networkSpec: apiServerLB: + frontendIPs: + - name: ${CLUSTER_NAME}-internal-lb-private-ip + privateIP: 10.9.0.100 name: ${CLUSTER_NAME}-internal-lb type: Internal controlPlaneOutboundLB: @@ -39,11 +42,17 @@ spec: nodeOutboundLB: frontendIPsCount: 1 subnets: - - name: control-plane-subnet + - cidrBlocks: + - 10.9.0.0/24 + name: control-plane-subnet role: control-plane - - name: node-subnet + - cidrBlocks: + - 10.9.1.0/24 + name: node-subnet role: node vnet: + cidrBlocks: + - 10.9.0.0/16 name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} subscriptionID: ${AZURE_SUBSCRIPTION_ID} @@ -203,7 +212,9 @@ spec: kubeletExtraArgs: cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' - preKubeadmCommands: [] + preKubeadmCommands: + - echo '10.9.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' + >> /etc/hosts --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 kind: AzureClusterIdentity diff --git a/templates/flavors/private/kustomization.yaml b/templates/flavors/private/kustomization.yaml index e8b87f0ff6b..0a49a8be7b9 100644 --- a/templates/flavors/private/kustomization.yaml +++ b/templates/flavors/private/kustomization.yaml @@ -11,6 +11,48 @@ patches: - path: patches/private-lb.yaml - path: patches/apiserver-host-dns.yaml - path: patches/azure-bastion.yaml +- path: ../../internal-load-balancer/azure-cluster-cidrs-and-frontend-ips.yaml +- path: ../../internal-load-balancer/kubeadm-config-template-worker-node.yaml +- target: + kind: KubeadmConfigTemplate + patch: |- + - op: add + path: /spec/template/spec/preKubeadmCommands/- + value: echo '10.9.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' >> /etc/hosts +- target: + kind: AzureCluster + patch: |- + - op: replace + path: /spec/networkSpec/apiServerLB/frontendIPs/1/privateIP + value: 10.9.0.100 +- target: + kind: AzureCluster + patch: |- + - op: replace + path: /spec/networkSpec/vnet/cidrBlocks/0 + value: 10.9.0.0/16 +- target: + kind: AzureCluster + patch: |- + - op: replace + path: /spec/networkSpec/subnets/0/cidrBlocks/0 + value: 10.9.0.0/24 +- target: + kind: AzureCluster + patch: |- + - op: replace + path: /spec/networkSpec/subnets/1/cidrBlocks/0 + value: 10.9.1.0/24 +- target: + kind: AzureCluster + patch: |- + - op: remove + path: /spec/networkSpec/apiServerLB/frontendIPs/0 +- target: + kind: KubeadmConfigTemplate + patch: |- + - op: remove + path: /spec/template/spec/preKubeadmCommands/0 sortOptions: order: fifo diff --git a/templates/test/ci/cluster-template-prow-private.yaml b/templates/test/ci/cluster-template-prow-private.yaml index 4f6613665e1..cdf5c19a249 100644 --- a/templates/test/ci/cluster-template-prow-private.yaml +++ b/templates/test/ci/cluster-template-prow-private.yaml @@ -236,7 +236,9 @@ spec: kubeletExtraArgs: cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' - preKubeadmCommands: [] + preKubeadmCommands: + - echo '10.9.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' + >> /etc/hosts --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 kind: AzureClusterIdentity From ef11e75cc017ae575747fbca93e9319edd0197ec Mon Sep 17 00:00:00 2001 From: Nawaz Hussain Khazielakha Date: Thu, 14 Nov 2024 18:57:57 -0800 Subject: [PATCH 13/23] update dual-stack templates --- templates/cluster-template-dual-stack.yaml | 14 +++++++------ .../flavors/dual-stack/kustomization.yaml | 11 ++++++++-- .../dual-stack/patches/dual-stack.yaml | 20 ++++++++++++++----- ...r-template-prow-ci-version-dual-stack.yaml | 20 ++++++++++++++----- .../ci/cluster-template-prow-dual-stack.yaml | 14 +++++++------ .../kustomization.yaml | 7 +++++++ 6 files changed, 62 insertions(+), 24 deletions(-) diff --git a/templates/cluster-template-dual-stack.yaml b/templates/cluster-template-dual-stack.yaml index fc890e60836..ce8ad77db4b 100644 --- a/templates/cluster-template-dual-stack.yaml +++ b/templates/cluster-template-dual-stack.yaml @@ -9,11 +9,11 @@ spec: clusterNetwork: pods: cidrBlocks: - - 10.244.0.0/16 + - 50.244.0.0/16 - 2001:1234:5678:9a40::/58 services: cidrBlocks: - - 10.0.0.0/16 + - 50.0.0.0/16 - fd00::/108 controlPlaneRef: apiVersion: controlplane.cluster.x-k8s.io/v1beta1 @@ -42,20 +42,22 @@ spec: publicIP: dnsName: ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com name: ${CLUSTER_NAME}-api-lb + - name: ${CLUSTER_NAME}-internal-lb-private-ip + privateIP: 50.0.0.100 subnets: - cidrBlocks: - - 10.0.0.0/16 + - 50.0.0.0/16 - 2001:1234:5678:9abc::/64 name: control-plane-subnet role: control-plane - cidrBlocks: - - 10.1.0.0/16 + - 50.1.0.0/16 - 2001:1234:5678:9abd::/64 name: node-subnet role: node vnet: cidrBlocks: - - 10.0.0.0/8 + - 50.0.0.0/8 - 2001:1234:5678:9a00::/56 name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} @@ -236,5 +238,5 @@ spec: cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: - - echo '10.0.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' + - echo '50.0.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' >> /etc/hosts diff --git a/templates/flavors/dual-stack/kustomization.yaml b/templates/flavors/dual-stack/kustomization.yaml index f8c00538723..0d3dae30fd6 100644 --- a/templates/flavors/dual-stack/kustomization.yaml +++ b/templates/flavors/dual-stack/kustomization.yaml @@ -10,8 +10,15 @@ patches: - path: patches/kubeadm-controlplane.yaml - path: patches/controlplane-azuremachinetemplate.yaml - path: ../../azure-cluster-identity/azurecluster-identity-ref.yaml -- path: ../../internal-load-balancer/azure-cluster-frontend-ip.yaml -- path: ../../internal-load-balancer/kubeadm-config-template-worker-node.yaml +- target: + kind: KubeadmConfigTemplate + patch: |- + - op: add + path: /spec/template/spec/preKubeadmCommands + value: [] + - op: add + path: /spec/template/spec/preKubeadmCommands/- + value: echo '50.0.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' >> /etc/hosts sortOptions: order: fifo diff --git a/templates/flavors/dual-stack/patches/dual-stack.yaml b/templates/flavors/dual-stack/patches/dual-stack.yaml index 95aca4fd28f..be768d802cd 100644 --- a/templates/flavors/dual-stack/patches/dual-stack.yaml +++ b/templates/flavors/dual-stack/patches/dual-stack.yaml @@ -11,11 +11,11 @@ spec: # this is a part of the virtual network IP range. # See https://docs.projectcalico.org/reference/public-cloud/azure cidrBlocks: - - "10.244.0.0/16" + - "50.244.0.0/16" - "2001:1234:5678:9a40::/58" services: cidrBlocks: - - "10.0.0.0/16" + - "50.0.0.0/16" - "fd00::/108" --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 @@ -24,18 +24,28 @@ metadata: name: ${CLUSTER_NAME} spec: networkSpec: + apiServerLB: + # We pre-create this public IP and the DNS name to use it in the + # worker node's /etc/hosts. + frontendIPs: + - name: ${CLUSTER_NAME}-api-lb + publicIP: + name: ${CLUSTER_NAME}-api-lb + dnsName: ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com + - name: ${CLUSTER_NAME}-internal-lb-private-ip + privateIP: 50.0.0.100 vnet: cidrBlocks: - - "10.0.0.0/8" + - "50.0.0.0/8" - "2001:1234:5678:9a00::/56" subnets: - name: control-plane-subnet role: control-plane cidrBlocks: - - "10.0.0.0/16" + - "50.0.0.0/16" - "2001:1234:5678:9abc::/64" - name: node-subnet role: node cidrBlocks: - - "10.1.0.0/16" + - "50.1.0.0/16" - "2001:1234:5678:9abd::/64" diff --git a/templates/test/ci/cluster-template-prow-ci-version-dual-stack.yaml b/templates/test/ci/cluster-template-prow-ci-version-dual-stack.yaml index 0a3d54c620f..451ce8a8451 100644 --- a/templates/test/ci/cluster-template-prow-ci-version-dual-stack.yaml +++ b/templates/test/ci/cluster-template-prow-ci-version-dual-stack.yaml @@ -14,11 +14,11 @@ spec: clusterNetwork: pods: cidrBlocks: - - 10.244.0.0/16 + - 50.244.0.0/16 - 2001:1234:5678:9a40::/58 services: cidrBlocks: - - 10.0.0.0/16 + - 50.0.0.0/16 - fd00::/108 controlPlaneRef: apiVersion: controlplane.cluster.x-k8s.io/v1beta1 @@ -45,20 +45,28 @@ spec: name: ${CLUSTER_IDENTITY_NAME} location: ${AZURE_LOCATION} networkSpec: + apiServerLB: + frontendIPs: + - name: ${CLUSTER_NAME}-api-lb + publicIP: + dnsName: ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com + name: ${CLUSTER_NAME}-api-lb + - name: ${CLUSTER_NAME}-internal-lb-private-ip + privateIP: 50.0.0.100 subnets: - cidrBlocks: - - 10.0.0.0/16 + - 50.0.0.0/16 - 2001:1234:5678:9abc::/64 name: control-plane-subnet role: control-plane - cidrBlocks: - - 10.1.0.0/16 + - 50.1.0.0/16 - 2001:1234:5678:9abd::/64 name: node-subnet role: node vnet: cidrBlocks: - - 10.0.0.0/8 + - 50.0.0.0/8 - 2001:1234:5678:9a00::/56 name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} @@ -448,6 +456,8 @@ spec: preKubeadmCommands: - bash -c /tmp/oot-cred-provider.sh - bash -c /tmp/kubeadm-bootstrap.sh + - echo '50.0.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' + >> /etc/hosts verbosity: 5 --- apiVersion: cluster.x-k8s.io/v1beta1 diff --git a/templates/test/ci/cluster-template-prow-dual-stack.yaml b/templates/test/ci/cluster-template-prow-dual-stack.yaml index b5ca8f3689f..f8007510e27 100644 --- a/templates/test/ci/cluster-template-prow-dual-stack.yaml +++ b/templates/test/ci/cluster-template-prow-dual-stack.yaml @@ -10,11 +10,11 @@ spec: clusterNetwork: pods: cidrBlocks: - - 10.244.0.0/16 + - 50.244.0.0/16 - 2001:1234:5678:9a40::/58 services: cidrBlocks: - - 10.0.0.0/16 + - 50.0.0.0/16 - fd00::/108 controlPlaneRef: apiVersion: controlplane.cluster.x-k8s.io/v1beta1 @@ -47,20 +47,22 @@ spec: publicIP: dnsName: ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com name: ${CLUSTER_NAME}-api-lb + - name: ${CLUSTER_NAME}-internal-lb-private-ip + privateIP: 50.0.0.100 subnets: - cidrBlocks: - - 10.0.0.0/16 + - 50.0.0.0/16 - 2001:1234:5678:9abc::/64 name: control-plane-subnet role: control-plane - cidrBlocks: - - 10.1.0.0/16 + - 50.1.0.0/16 - 2001:1234:5678:9abd::/64 name: node-subnet role: node vnet: cidrBlocks: - - 10.0.0.0/8 + - 50.0.0.0/8 - 2001:1234:5678:9a00::/56 name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} @@ -245,7 +247,7 @@ spec: cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: - - echo '10.0.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' + - echo '50.0.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' >> /etc/hosts --- apiVersion: cluster.x-k8s.io/v1beta1 diff --git a/templates/test/ci/prow-ci-version-dual-stack/kustomization.yaml b/templates/test/ci/prow-ci-version-dual-stack/kustomization.yaml index 4d01fd9ab39..f45fb4ba598 100644 --- a/templates/test/ci/prow-ci-version-dual-stack/kustomization.yaml +++ b/templates/test/ci/prow-ci-version-dual-stack/kustomization.yaml @@ -28,6 +28,13 @@ patches: - path: ../prow-dual-stack/patches/cluster-label-calico-dual-stack.yaml - path: patches/machine-deployment.yaml - path: ../patches/windows-addons-disabled.yaml +- target: + kind: KubeadmConfigTemplate + patch: |- + - op: add + path: /spec/template/spec/preKubeadmCommands/- + value: echo '50.0.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' >> /etc/hosts + sortOptions: order: fifo From 93ed804dc45aa84871a7d4971de27b85aaf21482 Mon Sep 17 00:00:00 2001 From: Nawaz Hussain Khazielakha Date: Thu, 14 Nov 2024 19:04:21 -0800 Subject: [PATCH 14/23] update ipv6 templates --- templates/cluster-template-ipv6.yaml | 10 ++++++---- templates/flavors/ipv6/kustomization.yaml | 11 +++++++++-- templates/flavors/ipv6/patches/ipv6.yaml | 16 +++++++++++++--- .../cluster-template-prow-ci-version-ipv6.yaml | 14 +++++++++++--- .../test/ci/cluster-template-prow-ipv6.yaml | 10 ++++++---- 5 files changed, 45 insertions(+), 16 deletions(-) diff --git a/templates/cluster-template-ipv6.yaml b/templates/cluster-template-ipv6.yaml index d0bb0e43a47..4da49126d1b 100644 --- a/templates/cluster-template-ipv6.yaml +++ b/templates/cluster-template-ipv6.yaml @@ -38,20 +38,22 @@ spec: publicIP: dnsName: ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com name: ${CLUSTER_NAME}-api-lb + - name: ${CLUSTER_NAME}-internal-lb-private-ip + privateIP: 60.0.0.100 subnets: - cidrBlocks: - - 10.0.0.0/16 + - 60.0.0.0/16 - 2001:1234:5678:9abc::/64 name: control-plane-subnet role: control-plane - cidrBlocks: - - 10.1.0.0/16 + - 60.1.0.0/16 - 2001:1234:5678:9abd::/64 name: node-subnet role: node vnet: cidrBlocks: - - 10.0.0.0/8 + - 60.0.0.0/8 - 2001:1234:5678:9a00::/56 name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} @@ -252,5 +254,5 @@ spec: cluster-dns: '[fd00::10]' name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: - - echo '10.0.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' + - echo '60.0.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' >> /etc/hosts diff --git a/templates/flavors/ipv6/kustomization.yaml b/templates/flavors/ipv6/kustomization.yaml index d663897e134..e665a2bd3f9 100644 --- a/templates/flavors/ipv6/kustomization.yaml +++ b/templates/flavors/ipv6/kustomization.yaml @@ -11,8 +11,15 @@ patches: - path: patches/ipv6.yaml - path: patches/kubeadm-controlplane.yaml - path: patches/controlplane-azuremachinetemplate.yaml -- path: ../../internal-load-balancer/azure-cluster-frontend-ip.yaml -- path: ../../internal-load-balancer/kubeadm-config-template-worker-node.yaml +- target: + kind: KubeadmConfigTemplate + patch: |- + - op: add + path: /spec/template/spec/preKubeadmCommands + value: [] + - op: add + path: /spec/template/spec/preKubeadmCommands/- + value: echo '60.0.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' >> /etc/hosts sortOptions: order: fifo diff --git a/templates/flavors/ipv6/patches/ipv6.yaml b/templates/flavors/ipv6/patches/ipv6.yaml index ea0eff7286d..17c5a2ad6d3 100644 --- a/templates/flavors/ipv6/patches/ipv6.yaml +++ b/templates/flavors/ipv6/patches/ipv6.yaml @@ -18,18 +18,28 @@ metadata: name: ${CLUSTER_NAME} spec: networkSpec: + apiServerLB: + # We pre-create this public IP and the DNS name to use it in the + # worker node's /etc/hosts. + frontendIPs: + - name: ${CLUSTER_NAME}-api-lb + publicIP: + name: ${CLUSTER_NAME}-api-lb + dnsName: ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com + - name: ${CLUSTER_NAME}-internal-lb-private-ip + privateIP: 60.0.0.100 vnet: cidrBlocks: - - "10.0.0.0/8" + - "60.0.0.0/8" - "2001:1234:5678:9a00::/56" subnets: - name: control-plane-subnet role: control-plane cidrBlocks: - - "10.0.0.0/16" + - "60.0.0.0/16" - "2001:1234:5678:9abc::/64" - name: node-subnet role: node cidrBlocks: - - "10.1.0.0/16" + - "60.1.0.0/16" - "2001:1234:5678:9abd::/64" diff --git a/templates/test/ci/cluster-template-prow-ci-version-ipv6.yaml b/templates/test/ci/cluster-template-prow-ci-version-ipv6.yaml index cbf94be6a1b..d5ff8507052 100644 --- a/templates/test/ci/cluster-template-prow-ci-version-ipv6.yaml +++ b/templates/test/ci/cluster-template-prow-ci-version-ipv6.yaml @@ -43,20 +43,28 @@ spec: name: ${CLUSTER_IDENTITY_NAME} location: ${AZURE_LOCATION} networkSpec: + apiServerLB: + frontendIPs: + - name: ${CLUSTER_NAME}-api-lb + publicIP: + dnsName: ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com + name: ${CLUSTER_NAME}-api-lb + - name: ${CLUSTER_NAME}-internal-lb-private-ip + privateIP: 60.0.0.100 subnets: - cidrBlocks: - - 10.0.0.0/16 + - 60.0.0.0/16 - 2001:1234:5678:9abc::/64 name: control-plane-subnet role: control-plane - cidrBlocks: - - 10.1.0.0/16 + - 60.1.0.0/16 - 2001:1234:5678:9abd::/64 name: node-subnet role: node vnet: cidrBlocks: - - 10.0.0.0/8 + - 60.0.0.0/8 - 2001:1234:5678:9a00::/56 name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} diff --git a/templates/test/ci/cluster-template-prow-ipv6.yaml b/templates/test/ci/cluster-template-prow-ipv6.yaml index 6cd89fc25f0..77d56946c45 100644 --- a/templates/test/ci/cluster-template-prow-ipv6.yaml +++ b/templates/test/ci/cluster-template-prow-ipv6.yaml @@ -45,20 +45,22 @@ spec: publicIP: dnsName: ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com name: ${CLUSTER_NAME}-api-lb + - name: ${CLUSTER_NAME}-internal-lb-private-ip + privateIP: 60.0.0.100 subnets: - cidrBlocks: - - 10.0.0.0/16 + - 60.0.0.0/16 - 2001:1234:5678:9abc::/64 name: control-plane-subnet role: control-plane - cidrBlocks: - - 10.1.0.0/16 + - 60.1.0.0/16 - 2001:1234:5678:9abd::/64 name: node-subnet role: node vnet: cidrBlocks: - - 10.0.0.0/8 + - 60.0.0.0/8 - 2001:1234:5678:9a00::/56 name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} @@ -260,7 +262,7 @@ spec: cluster-dns: '[fd00::10]' name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: - - echo '10.0.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' + - echo '60.0.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' >> /etc/hosts --- apiVersion: cluster.x-k8s.io/v1beta1 From d7bf52e7d3514ffc58bf9461feababb67d07d192 Mon Sep 17 00:00:00 2001 From: Nawaz Hussain Khazielakha Date: Thu, 14 Nov 2024 19:12:43 -0800 Subject: [PATCH 15/23] update flatcar templates --- templates/cluster-template-flatcar.yaml | 14 +++++++-- templates/flavors/flatcar/kustomization.yaml | 29 +++++++++++++++++-- .../ci/cluster-template-prow-flatcar.yaml | 14 +++++++-- 3 files changed, 49 insertions(+), 8 deletions(-) diff --git a/templates/cluster-template-flatcar.yaml b/templates/cluster-template-flatcar.yaml index 84750e8fe3a..4d1ea513b8c 100644 --- a/templates/cluster-template-flatcar.yaml +++ b/templates/cluster-template-flatcar.yaml @@ -35,12 +35,20 @@ spec: publicIP: dnsName: ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com name: ${CLUSTER_NAME}-api-lb + - name: ${CLUSTER_NAME}-internal-lb-private-ip + privateIP: 10.10.0.100 subnets: - - name: control-plane-subnet + - cidrBlocks: + - 10.10.0.0/24 + name: control-plane-subnet role: control-plane - - name: node-subnet + - cidrBlocks: + - 10.10.1.0/24 + name: node-subnet role: node vnet: + cidrBlocks: + - 10.10.0.0/16 name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} subscriptionID: ${AZURE_SUBSCRIPTION_ID} @@ -238,7 +246,7 @@ spec: preKubeadmCommands: - sed -i "s/@@HOSTNAME@@/$(curl -s -H Metadata:true --noproxy '*' 'http://169.254.169.254/metadata/instance?api-version=2020-09-01' | jq -r .compute.name)/g" /etc/kubeadm.yml - - echo '10.0.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' + - echo '10.10.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' >> /etc/hosts --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 diff --git a/templates/flavors/flatcar/kustomization.yaml b/templates/flavors/flatcar/kustomization.yaml index 826dc2b11c5..ed631fde30f 100644 --- a/templates/flavors/flatcar/kustomization.yaml +++ b/templates/flavors/flatcar/kustomization.yaml @@ -9,13 +9,38 @@ resources: patches: - path: patches/kubeadm-controlplane.yaml - path: ../../azure-cluster-identity/azurecluster-identity-ref.yaml -- path: ../../internal-load-balancer/azure-cluster-frontend-ip.yaml +- path: ../../internal-load-balancer/azure-cluster-cidrs-and-frontend-ips.yaml +- target: + kind: AzureCluster + patch: |- + - op: replace + path: /spec/networkSpec/apiServerLB/frontendIPs/1/privateIP + value: 10.10.0.100 +- target: + kind: AzureCluster + patch: |- + - op: replace + path: /spec/networkSpec/vnet/cidrBlocks/0 + value: 10.10.0.0/16 +- target: + kind: AzureCluster + patch: |- + - op: replace + path: /spec/networkSpec/subnets/0/cidrBlocks/0 + value: 10.10.0.0/24 +- target: + kind: AzureCluster + patch: |- + - op: replace + path: /spec/networkSpec/subnets/1/cidrBlocks/0 + value: 10.10.1.0/24 - target: kind: KubeadmConfigTemplate patch: |- - op: add path: /spec/template/spec/preKubeadmCommands/- - value: echo '10.0.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' >> /etc/hosts + value: echo '10.10.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' >> /etc/hosts + sortOptions: order: fifo diff --git a/templates/test/ci/cluster-template-prow-flatcar.yaml b/templates/test/ci/cluster-template-prow-flatcar.yaml index bffae1d36f2..2713d2a2158 100644 --- a/templates/test/ci/cluster-template-prow-flatcar.yaml +++ b/templates/test/ci/cluster-template-prow-flatcar.yaml @@ -42,12 +42,20 @@ spec: publicIP: dnsName: ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com name: ${CLUSTER_NAME}-api-lb + - name: ${CLUSTER_NAME}-internal-lb-private-ip + privateIP: 10.10.0.100 subnets: - - name: control-plane-subnet + - cidrBlocks: + - 10.10.0.0/24 + name: control-plane-subnet role: control-plane - - name: node-subnet + - cidrBlocks: + - 10.10.1.0/24 + name: node-subnet role: node vnet: + cidrBlocks: + - 10.10.0.0/16 name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} subscriptionID: ${AZURE_SUBSCRIPTION_ID} @@ -246,7 +254,7 @@ spec: preKubeadmCommands: - sed -i "s/@@HOSTNAME@@/$(curl -s -H Metadata:true --noproxy '*' 'http://169.254.169.254/metadata/instance?api-version=2020-09-01' | jq -r .compute.name)/g" /etc/kubeadm.yml - - echo '10.0.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' + - echo '10.10.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' >> /etc/hosts --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 From 29cedfe10eba17c49a554fb3f4bda5326d4fe5d4 Mon Sep 17 00:00:00 2001 From: Nawaz Hussain Khazielakha Date: Thu, 14 Nov 2024 19:13:47 -0800 Subject: [PATCH 16/23] update nvdia-gpu templates --- templates/cluster-template-nvidia-gpu.yaml | 14 +++++-- .../flavors/nvidia-gpu/kustomization.yaml | 37 ++++++++++++++++++- .../ci/cluster-template-prow-nvidia-gpu.yaml | 14 +++++-- 3 files changed, 58 insertions(+), 7 deletions(-) diff --git a/templates/cluster-template-nvidia-gpu.yaml b/templates/cluster-template-nvidia-gpu.yaml index 5cb94eeb512..3f31defa06c 100644 --- a/templates/cluster-template-nvidia-gpu.yaml +++ b/templates/cluster-template-nvidia-gpu.yaml @@ -35,12 +35,20 @@ spec: publicIP: dnsName: ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com name: ${CLUSTER_NAME}-api-lb + - name: ${CLUSTER_NAME}-internal-lb-private-ip + privateIP: 10.8.0.100 subnets: - - name: control-plane-subnet + - cidrBlocks: + - 10.8.0.0/24 + name: control-plane-subnet role: control-plane - - name: node-subnet + - cidrBlocks: + - 10.8.1.0/24 + name: node-subnet role: node vnet: + cidrBlocks: + - 10.8.0.0/16 name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} subscriptionID: ${AZURE_SUBSCRIPTION_ID} @@ -211,5 +219,5 @@ spec: cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: - - echo '10.0.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' + - echo '10.8.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' >> /etc/hosts diff --git a/templates/flavors/nvidia-gpu/kustomization.yaml b/templates/flavors/nvidia-gpu/kustomization.yaml index f8ebc7a9157..ed221ab5400 100644 --- a/templates/flavors/nvidia-gpu/kustomization.yaml +++ b/templates/flavors/nvidia-gpu/kustomization.yaml @@ -16,8 +16,43 @@ generatorOptions: patches: - path: ../../azure-cluster-identity/azurecluster-identity-ref.yaml -- path: ../../internal-load-balancer/azure-cluster-frontend-ip.yaml +- path: ../../internal-load-balancer/azure-cluster-cidrs-and-frontend-ips.yaml - path: ../../internal-load-balancer/kubeadm-config-template-worker-node.yaml +- target: + kind: KubeadmConfigTemplate + patch: |- + - op: add + path: /spec/template/spec/preKubeadmCommands/- + value: echo '10.8.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' >> /etc/hosts +- target: + kind: AzureCluster + patch: |- + - op: replace + path: /spec/networkSpec/apiServerLB/frontendIPs/1/privateIP + value: 10.8.0.100 +- target: + kind: AzureCluster + patch: |- + - op: replace + path: /spec/networkSpec/vnet/cidrBlocks/0 + value: 10.8.0.0/16 +- target: + kind: AzureCluster + patch: |- + - op: replace + path: /spec/networkSpec/subnets/0/cidrBlocks/0 + value: 10.8.0.0/24 +- target: + kind: AzureCluster + patch: |- + - op: replace + path: /spec/networkSpec/subnets/1/cidrBlocks/0 + value: 10.8.1.0/24 +- target: + kind: KubeadmConfigTemplate + patch: |- + - op: remove + path: /spec/template/spec/preKubeadmCommands/0 sortOptions: order: fifo diff --git a/templates/test/ci/cluster-template-prow-nvidia-gpu.yaml b/templates/test/ci/cluster-template-prow-nvidia-gpu.yaml index 0b7fde5f1b1..b32a80d3120 100644 --- a/templates/test/ci/cluster-template-prow-nvidia-gpu.yaml +++ b/templates/test/ci/cluster-template-prow-nvidia-gpu.yaml @@ -42,12 +42,20 @@ spec: publicIP: dnsName: ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com name: ${CLUSTER_NAME}-api-lb + - name: ${CLUSTER_NAME}-internal-lb-private-ip + privateIP: 10.8.0.100 subnets: - - name: control-plane-subnet + - cidrBlocks: + - 10.8.0.0/24 + name: control-plane-subnet role: control-plane - - name: node-subnet + - cidrBlocks: + - 10.8.1.0/24 + name: node-subnet role: node vnet: + cidrBlocks: + - 10.8.0.0/16 name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} subscriptionID: ${AZURE_SUBSCRIPTION_ID} @@ -219,7 +227,7 @@ spec: cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: - - echo '10.0.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' + - echo '10.8.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' >> /etc/hosts --- apiVersion: addons.cluster.x-k8s.io/v1alpha1 From 5e2927834c01bb6887316463e6dffaf85839b55f Mon Sep 17 00:00:00 2001 From: Nawaz Hussain Khazielakha Date: Thu, 14 Nov 2024 19:27:10 -0800 Subject: [PATCH 17/23] update windows templates --- templates/cluster-template-windows.yaml | 14 ++++++++--- templates/flavors/windows/kustomization.yaml | 24 +++++++++++++++++++ .../patches/kubeadm-config-template.yaml | 2 +- 3 files changed, 36 insertions(+), 4 deletions(-) diff --git a/templates/cluster-template-windows.yaml b/templates/cluster-template-windows.yaml index ecee4ae0eaf..c7ef3a1e373 100644 --- a/templates/cluster-template-windows.yaml +++ b/templates/cluster-template-windows.yaml @@ -39,12 +39,20 @@ spec: publicIP: dnsName: ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com name: ${CLUSTER_NAME}-api-lb + - name: ${CLUSTER_NAME}-internal-lb-private-ip + privateIP: 10.11.0.100 subnets: - - name: control-plane-subnet + - cidrBlocks: + - 10.11.0.0/24 + name: control-plane-subnet role: control-plane - - name: node-subnet + - cidrBlocks: + - 10.11.1.0/24 + name: node-subnet role: node vnet: + cidrBlocks: + - 10.11.0.0/16 name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} subscriptionID: ${AZURE_SUBSCRIPTION_ID} @@ -298,7 +306,7 @@ spec: - powershell C:/defender-exclude-calico.ps1 preKubeadmCommands: - powershell -Command "Add-Content -Path 'C:\\Windows\\System32\\drivers\\etc\\hosts' - -Value '10.0.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com'" + -Value '10.11.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com'" users: - groups: Administrators name: capi diff --git a/templates/flavors/windows/kustomization.yaml b/templates/flavors/windows/kustomization.yaml index 452f43e02c5..a58d4e6e6d2 100644 --- a/templates/flavors/windows/kustomization.yaml +++ b/templates/flavors/windows/kustomization.yaml @@ -8,6 +8,30 @@ resources: patches: - path: ../base-windows-containerd/cluster.yaml - path: patches/kubeadm-config-template.yaml +- target: + kind: AzureCluster + patch: |- + - op: replace + path: /spec/networkSpec/apiServerLB/frontendIPs/1/privateIP + value: 10.11.0.100 +- target: + kind: AzureCluster + patch: |- + - op: replace + path: /spec/networkSpec/vnet/cidrBlocks/0 + value: 10.11.0.0/16 +- target: + kind: AzureCluster + patch: |- + - op: replace + path: /spec/networkSpec/subnets/0/cidrBlocks/0 + value: 10.11.0.0/24 +- target: + kind: AzureCluster + patch: |- + - op: replace + path: /spec/networkSpec/subnets/1/cidrBlocks/0 + value: 10.11.1.0/24 sortOptions: order: fifo diff --git a/templates/flavors/windows/patches/kubeadm-config-template.yaml b/templates/flavors/windows/patches/kubeadm-config-template.yaml index 3509792752a..39cd050129a 100644 --- a/templates/flavors/windows/patches/kubeadm-config-template.yaml +++ b/templates/flavors/windows/patches/kubeadm-config-template.yaml @@ -9,4 +9,4 @@ spec: # so that worker nodes can access the API server using the internal IP. # 10.0.0.100 is the default IP that gets assigned to a internal load balancer. preKubeadmCommands: - - powershell -Command "Add-Content -Path 'C:\\Windows\\System32\\drivers\\etc\\hosts' -Value '10.0.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com'" + - powershell -Command "Add-Content -Path 'C:\\Windows\\System32\\drivers\\etc\\hosts' -Value '10.11.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com'" From ec8f3dc4b12092696079c55e055b6625b7396065 Mon Sep 17 00:00:00 2001 From: Nawaz Hussain Khazielakha Date: Thu, 14 Nov 2024 19:29:43 -0800 Subject: [PATCH 18/23] update ci/prow-spot templates --- .../test/ci/cluster-template-prow-spot.yaml | 14 ++++++-- .../test/ci/prow-spot/kustomization.yaml | 35 +++++++++++++++++++ 2 files changed, 46 insertions(+), 3 deletions(-) diff --git a/templates/test/ci/cluster-template-prow-spot.yaml b/templates/test/ci/cluster-template-prow-spot.yaml index 5dc7f601945..cd9c09ef977 100644 --- a/templates/test/ci/cluster-template-prow-spot.yaml +++ b/templates/test/ci/cluster-template-prow-spot.yaml @@ -42,12 +42,20 @@ spec: publicIP: dnsName: ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com name: ${CLUSTER_NAME}-api-lb + - name: ${CLUSTER_NAME}-internal-lb-private-ip + privateIP: 10.12.0.100 subnets: - - name: control-plane-subnet + - cidrBlocks: + - 10.12.0.0/24 + name: control-plane-subnet role: control-plane - - name: node-subnet + - cidrBlocks: + - 10.12.1.0/24 + name: node-subnet role: node vnet: + cidrBlocks: + - 10.12.0.0/16 name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} subscriptionID: ${AZURE_SUBSCRIPTION_ID} @@ -212,7 +220,7 @@ spec: cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: - - echo '10.0.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' + - echo '10.12.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' >> /etc/hosts --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 diff --git a/templates/test/ci/prow-spot/kustomization.yaml b/templates/test/ci/prow-spot/kustomization.yaml index 635bf56e4fe..2933e992c0c 100644 --- a/templates/test/ci/prow-spot/kustomization.yaml +++ b/templates/test/ci/prow-spot/kustomization.yaml @@ -17,6 +17,41 @@ patches: - path: ../patches/uami-control-plane.yaml - path: ../patches/cluster-label-calico.yaml - path: ../patches/cluster-label-cloud-provider-azure.yaml +- target: + kind: KubeadmConfigTemplate + patch: |- + - op: add + path: /spec/template/spec/preKubeadmCommands/- + value: echo '10.12.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' >> /etc/hosts +- target: + kind: AzureCluster + patch: |- + - op: replace + path: /spec/networkSpec/apiServerLB/frontendIPs/1/privateIP + value: 10.12.0.100 +- target: + kind: AzureCluster + patch: |- + - op: replace + path: /spec/networkSpec/vnet/cidrBlocks/0 + value: 10.12.0.0/16 +- target: + kind: AzureCluster + patch: |- + - op: replace + path: /spec/networkSpec/subnets/0/cidrBlocks/0 + value: 10.12.0.0/24 +- target: + kind: AzureCluster + patch: |- + - op: replace + path: /spec/networkSpec/subnets/1/cidrBlocks/0 + value: 10.12.1.0/24 +- target: + kind: KubeadmConfigTemplate + patch: |- + - op: remove + path: /spec/template/spec/preKubeadmCommands/0 sortOptions: order: fifo From 040f0f751248f18492910cc15c97b5973c949fb0 Mon Sep 17 00:00:00 2001 From: Nawaz Hussain Khazielakha Date: Thu, 14 Nov 2024 19:36:21 -0800 Subject: [PATCH 19/23] update ci/prow-custom-vnet templates --- .../ci/cluster-template-prow-custom-vnet.yaml | 14 +++++-- .../ci/prow-custom-vnet/kustomization.yaml | 41 +++++++++++++++++++ 2 files changed, 52 insertions(+), 3 deletions(-) diff --git a/templates/test/ci/cluster-template-prow-custom-vnet.yaml b/templates/test/ci/cluster-template-prow-custom-vnet.yaml index e9b090f4ca5..dea192908af 100644 --- a/templates/test/ci/cluster-template-prow-custom-vnet.yaml +++ b/templates/test/ci/cluster-template-prow-custom-vnet.yaml @@ -42,18 +42,26 @@ spec: publicIP: dnsName: ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com name: ${CLUSTER_NAME}-api-lb + - name: ${CLUSTER_NAME}-internal-lb-private-ip + privateIP: 10.13.0.100 subnets: - - name: ${AZURE_CUSTOM_VNET_NAME}-controlplane-subnet + - cidrBlocks: + - 10.13.0.0/24 + name: ${AZURE_CUSTOM_VNET_NAME}-controlplane-subnet role: control-plane securityGroup: name: control-plane-nsg - - name: ${AZURE_CUSTOM_VNET_NAME}-node-subnet + - cidrBlocks: + - 10.13.1.0/24 + name: ${AZURE_CUSTOM_VNET_NAME}-node-subnet role: node routeTable: name: node-routetable securityGroup: name: node-nsg vnet: + cidrBlocks: + - 10.13.0.0/16 name: ${AZURE_CUSTOM_VNET_NAME} resourceGroup: ${AZURE_CUSTOM_VNET_RESOURCE_GROUP} resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} @@ -216,7 +224,7 @@ spec: cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: - - echo '10.0.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' + - echo '10.13.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' >> /etc/hosts --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 diff --git a/templates/test/ci/prow-custom-vnet/kustomization.yaml b/templates/test/ci/prow-custom-vnet/kustomization.yaml index e078480baf6..34690eeaa2a 100644 --- a/templates/test/ci/prow-custom-vnet/kustomization.yaml +++ b/templates/test/ci/prow-custom-vnet/kustomization.yaml @@ -17,6 +17,47 @@ patches: - path: ../patches/uami-control-plane.yaml - path: ../patches/cluster-label-calico.yaml - path: ../patches/cluster-label-cloud-provider-azure.yaml +- target: + kind: KubeadmConfigTemplate + patch: |- + - op: add + path: /spec/template/spec/preKubeadmCommands/- + value: echo '10.13.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' >> /etc/hosts +- target: + kind: AzureCluster + patch: |- + - op: replace + path: /spec/networkSpec/apiServerLB/frontendIPs/1/privateIP + value: 10.13.0.100 +- target: + kind: AzureCluster + patch: |- + - op: replace + path: /spec/networkSpec/vnet/cidrBlocks/0 + value: 10.13.0.0/16 +- target: + kind: AzureCluster + patch: |- + - op: add + path: /spec/networkSpec/subnets/0/cidrBlocks + value: [] + - op: add + path: /spec/networkSpec/subnets/0/cidrBlocks/0 + value: 10.13.0.0/24 +- target: + kind: AzureCluster + patch: |- + - op: add + path: /spec/networkSpec/subnets/1/cidrBlocks + value: [] + - op: add + path: /spec/networkSpec/subnets/1/cidrBlocks/0 + value: 10.13.1.0/24 +- target: + kind: KubeadmConfigTemplate + patch: |- + - op: remove + path: /spec/template/spec/preKubeadmCommands/0 sortOptions: order: fifo From ef039fe2dec00c750dddf23b3e4ec1696e264309 Mon Sep 17 00:00:00 2001 From: Nawaz Hussain Khazielakha Date: Thu, 14 Nov 2024 20:06:26 -0800 Subject: [PATCH 20/23] fix verify errors --- Tiltfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Tiltfile b/Tiltfile index 64c9da828a9..ee4f2934177 100644 --- a/Tiltfile +++ b/Tiltfile @@ -324,7 +324,6 @@ def flavors(): if "aks" in settings.get("kustomize_substitutions", {}).get("MGMT_CLUSTER_NAME", ""): delete_all_workload_clusters += clear_aks_vnet_peerings() - local_resource( name = "delete-all-workload-clusters", cmd = ["sh", "-ec", delete_all_workload_clusters], From dfa6e38454aa61c031db4861a64a403fc119eab9 Mon Sep 17 00:00:00 2001 From: Nawaz Hussain Khazielakha Date: Mon, 18 Nov 2024 11:35:17 -0800 Subject: [PATCH 21/23] assign a /8 cidr to azure cni v1 templates --- templates/cluster-template-azure-cni-v1.yaml | 10 +++++----- templates/flavors/azure-cni-v1/kustomization.yaml | 10 +++++----- .../test/ci/cluster-template-prow-azure-cni-v1.yaml | 10 +++++----- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/templates/cluster-template-azure-cni-v1.yaml b/templates/cluster-template-azure-cni-v1.yaml index b2f2942f1e4..98b34270175 100644 --- a/templates/cluster-template-azure-cni-v1.yaml +++ b/templates/cluster-template-azure-cni-v1.yaml @@ -36,19 +36,19 @@ spec: dnsName: ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com name: ${CLUSTER_NAME}-api-lb - name: ${CLUSTER_NAME}-internal-lb-private-ip - privateIP: 10.3.0.100 + privateIP: 60.3.0.100 subnets: - cidrBlocks: - - 10.3.0.0/24 + - 60.3.0.0/16 name: control-plane-subnet role: control-plane - cidrBlocks: - - 10.3.1.0/24 + - 60.3.1.0/16 name: node-subnet role: node vnet: cidrBlocks: - - 10.3.0.0/16 + - 60.3.0.0/8 name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} subscriptionID: ${AZURE_SUBSCRIPTION_ID} @@ -213,7 +213,7 @@ spec: max-pods: "110" name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: - - echo '10.3.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' + - echo '60.3.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' >> /etc/hosts --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 diff --git a/templates/flavors/azure-cni-v1/kustomization.yaml b/templates/flavors/azure-cni-v1/kustomization.yaml index 277594f0f09..3726b5f34bc 100644 --- a/templates/flavors/azure-cni-v1/kustomization.yaml +++ b/templates/flavors/azure-cni-v1/kustomization.yaml @@ -18,31 +18,31 @@ patches: patch: |- - op: add path: /spec/template/spec/preKubeadmCommands/- - value: echo '10.3.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' >> /etc/hosts + value: echo '60.3.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' >> /etc/hosts - target: kind: AzureCluster patch: |- - op: replace path: /spec/networkSpec/apiServerLB/frontendIPs/1/privateIP - value: 10.3.0.100 + value: 60.3.0.100 - target: kind: AzureCluster patch: |- - op: replace path: /spec/networkSpec/vnet/cidrBlocks/0 - value: 10.3.0.0/16 + value: 60.3.0.0/8 - target: kind: AzureCluster patch: |- - op: replace path: /spec/networkSpec/subnets/0/cidrBlocks/0 - value: 10.3.0.0/24 + value: 60.3.0.0/16 - target: kind: AzureCluster patch: |- - op: replace path: /spec/networkSpec/subnets/1/cidrBlocks/0 - value: 10.3.1.0/24 + value: 60.3.1.0/16 - target: kind: KubeadmConfigTemplate patch: |- diff --git a/templates/test/ci/cluster-template-prow-azure-cni-v1.yaml b/templates/test/ci/cluster-template-prow-azure-cni-v1.yaml index afdf70a91de..775925e8887 100644 --- a/templates/test/ci/cluster-template-prow-azure-cni-v1.yaml +++ b/templates/test/ci/cluster-template-prow-azure-cni-v1.yaml @@ -42,19 +42,19 @@ spec: dnsName: ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com name: ${CLUSTER_NAME}-api-lb - name: ${CLUSTER_NAME}-internal-lb-private-ip - privateIP: 10.3.0.100 + privateIP: 60.3.0.100 subnets: - cidrBlocks: - - 10.3.0.0/24 + - 60.3.0.0/16 name: control-plane-subnet role: control-plane - cidrBlocks: - - 10.3.1.0/24 + - 60.3.1.0/16 name: node-subnet role: node vnet: cidrBlocks: - - 10.3.0.0/16 + - 60.3.0.0/8 name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} subscriptionID: ${AZURE_SUBSCRIPTION_ID} @@ -220,7 +220,7 @@ spec: max-pods: "110" name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: - - echo '10.3.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' + - echo '60.3.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' >> /etc/hosts --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 From a31fe3c50ba5a926e7a2fd77b1af91530508680e Mon Sep 17 00:00:00 2001 From: Nawaz Hussain Khazielakha Date: Mon, 18 Nov 2024 11:36:50 -0800 Subject: [PATCH 22/23] update windows templates with bigger cidr --- templates/cluster-template-windows.yaml | 10 +++++----- templates/flavors/windows/kustomization.yaml | 8 ++++---- .../windows/patches/kubeadm-config-template.yaml | 2 +- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/templates/cluster-template-windows.yaml b/templates/cluster-template-windows.yaml index c7ef3a1e373..433296810fd 100644 --- a/templates/cluster-template-windows.yaml +++ b/templates/cluster-template-windows.yaml @@ -40,19 +40,19 @@ spec: dnsName: ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com name: ${CLUSTER_NAME}-api-lb - name: ${CLUSTER_NAME}-internal-lb-private-ip - privateIP: 10.11.0.100 + privateIP: 70.11.0.100 subnets: - cidrBlocks: - - 10.11.0.0/24 + - 70.11.0.0/16 name: control-plane-subnet role: control-plane - cidrBlocks: - - 10.11.1.0/24 + - 70.11.1.0/16 name: node-subnet role: node vnet: cidrBlocks: - - 10.11.0.0/16 + - 70.11.0.0/8 name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} subscriptionID: ${AZURE_SUBSCRIPTION_ID} @@ -306,7 +306,7 @@ spec: - powershell C:/defender-exclude-calico.ps1 preKubeadmCommands: - powershell -Command "Add-Content -Path 'C:\\Windows\\System32\\drivers\\etc\\hosts' - -Value '10.11.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com'" + -Value '70.11.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com'" users: - groups: Administrators name: capi diff --git a/templates/flavors/windows/kustomization.yaml b/templates/flavors/windows/kustomization.yaml index a58d4e6e6d2..6ca23e93ad0 100644 --- a/templates/flavors/windows/kustomization.yaml +++ b/templates/flavors/windows/kustomization.yaml @@ -13,25 +13,25 @@ patches: patch: |- - op: replace path: /spec/networkSpec/apiServerLB/frontendIPs/1/privateIP - value: 10.11.0.100 + value: 70.11.0.100 - target: kind: AzureCluster patch: |- - op: replace path: /spec/networkSpec/vnet/cidrBlocks/0 - value: 10.11.0.0/16 + value: 70.11.0.0/8 - target: kind: AzureCluster patch: |- - op: replace path: /spec/networkSpec/subnets/0/cidrBlocks/0 - value: 10.11.0.0/24 + value: 70.11.0.0/16 - target: kind: AzureCluster patch: |- - op: replace path: /spec/networkSpec/subnets/1/cidrBlocks/0 - value: 10.11.1.0/24 + value: 70.11.1.0/16 sortOptions: order: fifo diff --git a/templates/flavors/windows/patches/kubeadm-config-template.yaml b/templates/flavors/windows/patches/kubeadm-config-template.yaml index 39cd050129a..503a286ff13 100644 --- a/templates/flavors/windows/patches/kubeadm-config-template.yaml +++ b/templates/flavors/windows/patches/kubeadm-config-template.yaml @@ -9,4 +9,4 @@ spec: # so that worker nodes can access the API server using the internal IP. # 10.0.0.100 is the default IP that gets assigned to a internal load balancer. preKubeadmCommands: - - powershell -Command "Add-Content -Path 'C:\\Windows\\System32\\drivers\\etc\\hosts' -Value '10.11.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com'" + - powershell -Command "Add-Content -Path 'C:\\Windows\\System32\\drivers\\etc\\hosts' -Value '70.11.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com'" From 6cb4ead69638ad799c2560ac33f8419172a97e7e Mon Sep 17 00:00:00 2001 From: Nawaz Hussain Khazielakha Date: Mon, 18 Nov 2024 12:01:10 -0800 Subject: [PATCH 23/23] allocate bigger dual-stack cidrs --- templates/cluster-template-dual-stack.yaml | 2 +- templates/flavors/dual-stack/patches/kubeadm-controlplane.yaml | 2 +- .../test/ci/cluster-template-prow-ci-version-dual-stack.yaml | 2 +- templates/test/ci/cluster-template-prow-dual-stack.yaml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/templates/cluster-template-dual-stack.yaml b/templates/cluster-template-dual-stack.yaml index ce8ad77db4b..360d25c2ab6 100644 --- a/templates/cluster-template-dual-stack.yaml +++ b/templates/cluster-template-dual-stack.yaml @@ -79,7 +79,7 @@ spec: extraArgs: allocate-node-cidrs: "true" cloud-provider: external - cluster-cidr: 10.244.0.0/16,2001:1234:5678:9a40::/58 + cluster-cidr: 50.244.0.0/16,2001:1234:5678:9a40::/58 cluster-name: ${CLUSTER_NAME} configure-cloud-routes: "true" etcd: diff --git a/templates/flavors/dual-stack/patches/kubeadm-controlplane.yaml b/templates/flavors/dual-stack/patches/kubeadm-controlplane.yaml index 05f45c680d2..fd9527f836a 100644 --- a/templates/flavors/dual-stack/patches/kubeadm-controlplane.yaml +++ b/templates/flavors/dual-stack/patches/kubeadm-controlplane.yaml @@ -26,6 +26,6 @@ spec: extraArgs: #required for ipv6 using calico allocate-node-cidrs: "true" - cluster-cidr: "10.244.0.0/16,2001:1234:5678:9a40::/58" + cluster-cidr: "50.244.0.0/16,2001:1234:5678:9a40::/58" configure-cloud-routes: "true" cloud-provider: external diff --git a/templates/test/ci/cluster-template-prow-ci-version-dual-stack.yaml b/templates/test/ci/cluster-template-prow-ci-version-dual-stack.yaml index 451ce8a8451..e1947e9dca4 100644 --- a/templates/test/ci/cluster-template-prow-ci-version-dual-stack.yaml +++ b/templates/test/ci/cluster-template-prow-ci-version-dual-stack.yaml @@ -89,7 +89,7 @@ spec: extraArgs: allocate-node-cidrs: "true" cloud-provider: external - cluster-cidr: 10.244.0.0/16,2001:1234:5678:9a40::/58 + cluster-cidr: 50.244.0.0/16,2001:1234:5678:9a40::/58 cluster-name: ${CLUSTER_NAME} configure-cloud-routes: "true" v: "4" diff --git a/templates/test/ci/cluster-template-prow-dual-stack.yaml b/templates/test/ci/cluster-template-prow-dual-stack.yaml index f8007510e27..b8d51e983c0 100644 --- a/templates/test/ci/cluster-template-prow-dual-stack.yaml +++ b/templates/test/ci/cluster-template-prow-dual-stack.yaml @@ -84,7 +84,7 @@ spec: extraArgs: allocate-node-cidrs: "true" cloud-provider: external - cluster-cidr: 10.244.0.0/16,2001:1234:5678:9a40::/58 + cluster-cidr: 50.244.0.0/16,2001:1234:5678:9a40::/58 cluster-name: ${CLUSTER_NAME} configure-cloud-routes: "true" v: "4"