Skip to content

Commit

Permalink
add foo to clear AKS Cluster's VNet peerings
Browse files Browse the repository at this point in the history
  • Loading branch information
@nawazkh
nawazkh committed Nov 9, 2024
1 parent fb3d377 commit 38e27e7
Showing 1 changed file with 51 additions and 37 deletions.
88 changes: 51 additions & 37 deletions Tiltfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ if "default_registry" in settings:
default_registry(settings.get("default_registry"))

os_arch = str(local("go env GOARCH")).rstrip("\n")

# TODO: no one is clearing MGMT_CLUSTER_NAME when using KIND, so this is always going to be true. Improve this logic.
if "aks" in settings.get("kustomize_substitutions", {}).get("MGMT_CLUSTER_NAME", ""):
print("Using AKS as management cluster, setting os_arch to amd64")
Expand Down Expand Up @@ -118,7 +119,7 @@ def fixup_yaml_empty_arrays(yaml_str):
return yaml_str.replace("storedVersions: null", "storedVersions: []")

def validate_auth():
substitutions = settings.get("kustomize_substitutions", {}) # all the env variables are exported here
substitutions = settings.get("kustomize_substitutions", {}) # all the env variables are exported here
os.environ.update(substitutions)
for sub in substitutions:
if sub[-4:] == "_B64":
Expand Down Expand Up @@ -317,15 +318,20 @@ def flavors():
for template in template_list:
deploy_worker_templates(template, substitutions)

delete_all_workload_clusters = kubectl_cmd + " delete clusters --all --wait=false"

if "aks" in settings.get("kustomize_substitutions", {}).get("MGMT_CLUSTER_NAME", ""):
delete_all_workload_clusters += clear_aks_vnet_peerings()

delete_all_workload_clusters += "; echo \"\n-----------------------------------\n\""
local_resource(
name = "delete-all-workload-clusters",
cmd = kubectl_cmd + " delete clusters --all --wait=false",
cmd = ["sh", "-ec", delete_all_workload_clusters],
auto_init = False,
trigger_mode = TRIGGER_MODE_MANUAL,
labels = ["flavors"],
)


def deploy_worker_templates(template, substitutions):
# validate template exists
if not os.path.exists(template):
Expand Down Expand Up @@ -383,10 +389,19 @@ def deploy_worker_templates(template, substitutions):

yaml = shlex.quote(yaml)
flavor_name = os.path.basename(flavor)
flavor_cmd = "RANDOM=$(bash -c 'echo $RANDOM'); export CLUSTER_NAME=" + flavor.replace("windows", "win") + "-$RANDOM; make generate-flavors; echo " + yaml + "> ./.tiltbuild/" + flavor + "; cat ./.tiltbuild/" + flavor + " | " + envsubst_cmd + " | " + kubectl_cmd + " apply -f -; echo \"Cluster \'$CLUSTER_NAME\' created, don't forget to delete\""
flavor_cmd = "RANDOM=$(bash -c 'echo $RANDOM')"
flavor_cmd += "; export CLUSTER_NAME=" + flavor.replace("windows", "win") + "-$RANDOM; echo " + yaml + "> ./.tiltbuild/" + flavor + "; cat ./.tiltbuild/" + flavor + " | " + envsubst_cmd + " | " + kubectl_cmd + " apply -f -"
flavor_cmd += "; echo \"Cluster \'$CLUSTER_NAME\' created, don't forget to delete\""

# wait for kubeconfig to be available
flavor_cmd += "; until " + kubectl_cmd + " get secret ${CLUSTER_NAME}-kubeconfig > /dev/null 2>&1; do sleep 5; done; " + kubectl_cmd + " get secret ${CLUSTER_NAME}-kubeconfig -o jsonpath={.data.value} | base64 --decode > ./${CLUSTER_NAME}.kubeconfig; chmod 600 ./${CLUSTER_NAME}.kubeconfig; until " + kubectl_cmd + " --kubeconfig=./${CLUSTER_NAME}.kubeconfig get nodes > /dev/null 2>&1; do sleep 5; done"
flavor_cmd += "; echo \"Waiting for kubeconfig to be available\""
flavor_cmd += "; until " + kubectl_cmd + " get secret ${CLUSTER_NAME}-kubeconfig > /dev/null 2>&1; do sleep 5; done"
flavor_cmd += "; " + kubectl_cmd + " get secret ${CLUSTER_NAME}-kubeconfig -o jsonpath={.data.value} | base64 --decode > ./${CLUSTER_NAME}.kubeconfig"
flavor_cmd += "; chmod 600 ./${CLUSTER_NAME}.kubeconfig"
flavor_cmd += "; echo \"Kubeconfig for $CLUSTER_NAME created and saved in the local\""
flavor_cmd += "; echo \"Waiting for $CLUSTER_NAME nodes to become available\""
flavor_cmd += "; until " + kubectl_cmd + " --kubeconfig=./${CLUSTER_NAME}.kubeconfig get nodes > /dev/null 2>&1; do sleep 5; done"
flavor_cmd += "; echo \"Nodes for $CLUSTER_NAME are available\""

# copy the kubeadm configmap to the calico-system namespace.
# This is a workaround needed for the calico-node-windows daemonset to be able to run in the calico-system namespace.
Expand All @@ -399,6 +414,7 @@ def deploy_worker_templates(template, substitutions):
flavor_cmd += peer_vnets()

flavor_cmd += get_addons(flavor_name)
flavor_cmd += "; echo \"\n-----------------------------------\n\""
local_resource(
name = flavor_name,
cmd = ["sh", "-ec", flavor_cmd],
Expand Down Expand Up @@ -460,60 +476,58 @@ def waitforsystem():

def peer_vnets():
# TODO: check for az cli to be installed in local

# get AKS management cluster vnet and resource group
#mgmt_vnet_name = "${AKS_MGMT_VNET_NAME}"
# mgmt_rg = "${AKS_RESOURCE_GROUP}"

# get workload cluster vnet and resource group
# workload_cluster_name = "${CLUSTER_NAME}"
# workload_vnet_name = workload_cluster_name + "-vnet"
# workload_rg = workload_cluster_name
# workload_cluster_fqdn_suffix = "${APISERVER_LB_DNS_SUFFIX}"
# private_dns_zone_name = "${AZURE_LOCATION}.cloudapp.azure.com"

# wait for AKS VNet to be in the state created
peering_cmd = "; echo \"\nPeering VNETs\" "
peering_cmd = "; echo \"--------Peering VNETs--------\""
peering_cmd += "; az network vnet wait --resource-group ${AKS_RESOURCE_GROUP} --name ${AKS_MGMT_VNET_NAME} --created --timeout 180"
peering_cmd += "; export MGMT_VNET_ID=$(az network vnet show --resource-group ${AKS_RESOURCE_GROUP} --name ${AKS_MGMT_VNET_NAME} --query id --output tsv)"
peering_cmd += "; echo \"${AKS_MGMT_VNET_NAME} found \""
peering_cmd += "; echo \" 1/8 ${AKS_MGMT_VNET_NAME} found \""

# wait for workload VNet to be created
peering_cmd += "; az network vnet wait --resource-group ${CLUSTER_NAME} --name ${CLUSTER_NAME}-vnet --created --timeout 180"
peering_cmd += "; export WORKLOAD_VNET_ID=$(az network vnet show --resource-group ${CLUSTER_NAME} --name ${CLUSTER_NAME}-vnet --query id --output tsv)"
peering_cmd += "; echo \"${CLUSTER_NAME}-vnet found \""
peering_cmd += "; echo \" 2/8 ${CLUSTER_NAME}-vnet found \""

# peer mgmt vnet
peering_cmd += "; az network vnet peering create --name mgmt-to-${CLUSTER_NAME} --resource-group ${AKS_RESOURCE_GROUP} --vnet-name ${AKS_MGMT_VNET_NAME} --remote-vnet \"${WORKLOAD_VNET_ID}\" --allow-vnet-access true --allow-forwarded-traffic true --only-show-errors"
peering_cmd += "; az network vnet peering wait --name mgmt-to-${CLUSTER_NAME} --resource-group ${AKS_RESOURCE_GROUP} --vnet-name ${AKS_MGMT_VNET_NAME} --created --timeout 300 --only-show-errors"
peering_cmd += "; echo \"mgmt-to-${CLUSTER_NAME} peering created\""
peering_cmd += "; az network vnet peering create --name mgmt-to-${CLUSTER_NAME} --resource-group ${AKS_RESOURCE_GROUP} --vnet-name ${AKS_MGMT_VNET_NAME} --remote-vnet \"${WORKLOAD_VNET_ID}\" --allow-vnet-access true --allow-forwarded-traffic true --only-show-errors --output none"
peering_cmd += "; az network vnet peering wait --name mgmt-to-${CLUSTER_NAME} --resource-group ${AKS_RESOURCE_GROUP} --vnet-name ${AKS_MGMT_VNET_NAME} --created --timeout 300 --only-show-errors --output none"
peering_cmd += "; echo \" 3/8 mgmt-to-${CLUSTER_NAME} peering created in ${AKS_MGMT_VNET_NAME}\""

# peer workload vnet
peering_cmd += "; az network vnet peering create --name ${CLUSTER_NAME}-to-mgmt --resource-group ${CLUSTER_NAME} --vnet-name ${CLUSTER_NAME}-vnet --remote-vnet \"${MGMT_VNET_ID}\" --allow-vnet-access true --allow-forwarded-traffic true --only-show-errors"
peering_cmd += "; az network vnet peering wait --name ${CLUSTER_NAME}-to-mgmt --resource-group ${CLUSTER_NAME} --vnet-name ${CLUSTER_NAME}-vnet --created --timeout 300 --only-show-errors"
peering_cmd += "; echo \"${CLUSTER_NAME}-to-mgmt peering created\""
peering_cmd += "; az network vnet peering create --name ${CLUSTER_NAME}-to-mgmt --resource-group ${CLUSTER_NAME} --vnet-name ${CLUSTER_NAME}-vnet --remote-vnet \"${MGMT_VNET_ID}\" --allow-vnet-access true --allow-forwarded-traffic true --only-show-errors --output none"
peering_cmd += "; az network vnet peering wait --name ${CLUSTER_NAME}-to-mgmt --resource-group ${CLUSTER_NAME} --vnet-name ${CLUSTER_NAME}-vnet --created --timeout 300 --only-show-errors --output none"
peering_cmd += "; echo \" 4/8 ${CLUSTER_NAME}-to-mgmt peering created in ${CLUSTER_NAME}-vnet\""

# create private DNS zone
peering_cmd += "; az network private-dns zone create --resource-group ${CLUSTER_NAME} --name ${AZURE_LOCATION}.cloudapp.azure.com --only-show-errors"
peering_cmd += "; az network private-dns zone wait --resource-group ${CLUSTER_NAME} --name ${AZURE_LOCATION}.cloudapp.azure.com --created --timeout 300 --only-show-errors"
peering_cmd += "; echo \"${AZURE_LOCATION}.cloudapp.azure.com private DNS zone created\""
peering_cmd += "; az network private-dns zone create --resource-group ${CLUSTER_NAME} --name ${AZURE_LOCATION}.cloudapp.azure.com --only-show-errors --output none"
peering_cmd += "; az network private-dns zone wait --resource-group ${CLUSTER_NAME} --name ${AZURE_LOCATION}.cloudapp.azure.com --created --timeout 300 --only-show-errors --output none"
peering_cmd += "; echo \" 5/8 ${AZURE_LOCATION}.cloudapp.azure.com private DNS zone created in ${CLUSTER_NAME}\""

# link private DNS Zone to workload vnet
peering_cmd += "; az network private-dns link vnet create --resource-group ${CLUSTER_NAME} --zone-name ${AZURE_LOCATION}.cloudapp.azure.com --name ${CLUSTER_NAME}-to-mgmt --virtual-network \"${WORKLOAD_VNET_ID}\" --registration-enabled false --only-show-errors"
peering_cmd += "; az network private-dns link vnet wait --resource-group ${CLUSTER_NAME} --zone-name ${AZURE_LOCATION}.cloudapp.azure.com --name ${CLUSTER_NAME}-to-mgmt --created --timeout 300 --only-show-errors"
peering_cmd += "; echo \"workload cluster linked with private DNS zone\""
peering_cmd += "; az network private-dns link vnet create --resource-group ${CLUSTER_NAME} --zone-name ${AZURE_LOCATION}.cloudapp.azure.com --name ${CLUSTER_NAME}-to-mgmt --virtual-network \"${WORKLOAD_VNET_ID}\" --registration-enabled false --only-show-errors --output none"
peering_cmd += "; az network private-dns link vnet wait --resource-group ${CLUSTER_NAME} --zone-name ${AZURE_LOCATION}.cloudapp.azure.com --name ${CLUSTER_NAME}-to-mgmt --created --timeout 300 --only-show-errors --output none"
peering_cmd += "; echo \" 6/8 workload cluster vnet ${CLUSTER_NAME}-vnet linked with private DNS zone\""

# link private DNS Zone to mgmt vnet
peering_cmd += "; az network private-dns link vnet create --resource-group ${CLUSTER_NAME} --zone-name ${AZURE_LOCATION}.cloudapp.azure.com --name mgmt-to-${CLUSTER_NAME} --virtual-network \"${MGMT_VNET_ID}\" --registration-enabled false --only-show-errors"
peering_cmd += "; az network private-dns link vnet wait --resource-group ${CLUSTER_NAME} --zone-name ${AZURE_LOCATION}.cloudapp.azure.com --name mgmt-to-${CLUSTER_NAME} --created --timeout 300 --only-show-errors"
peering_cmd += "; echo \"management cluster linked with private DNS zone\""
peering_cmd += "; az network private-dns link vnet create --resource-group ${CLUSTER_NAME} --zone-name ${AZURE_LOCATION}.cloudapp.azure.com --name mgmt-to-${CLUSTER_NAME} --virtual-network \"${MGMT_VNET_ID}\" --registration-enabled false --only-show-errors --output none"
peering_cmd += "; az network private-dns link vnet wait --resource-group ${CLUSTER_NAME} --zone-name ${AZURE_LOCATION}.cloudapp.azure.com --name mgmt-to-${CLUSTER_NAME} --created --timeout 300 --only-show-errors --output none"
peering_cmd += "; echo \" 7/8 management cluster vnet ${AKS_MGMT_VNET_NAME} linked with private DNS zone\""

# create private DNS zone record
peering_cmd += "; az network private-dns record-set a add-record --resource-group ${CLUSTER_NAME} --zone-name ${AZURE_LOCATION}.cloudapp.azure.com --record-set-name ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX} --ipv4-address 10.0.0.100 --only-show-errors"
peering_cmd += "; echo \"${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX} private DNS zone record created\""
# TODO: 10.0.0.100 should be customizable
peering_cmd += "; az network private-dns record-set a add-record --resource-group ${CLUSTER_NAME} --zone-name ${AZURE_LOCATION}.cloudapp.azure.com --record-set-name ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX} --ipv4-address 10.0.0.100 --only-show-errors --output none"
peering_cmd += "; echo \" 8/8 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX} private DNS zone record created\n\""

return peering_cmd

def clear_aks_vnet_peerings():
delete_peering_cmd = "; echo \"--------Clearing AKS VNETs Peerings--------\""
delete_peering_cmd += "; az network vnet wait --resource-group ${AKS_RESOURCE_GROUP} --name ${AKS_MGMT_VNET_NAME} --created --timeout 180"
delete_peering_cmd += "; echo \" 1/2 ${AKS_MGMT_VNET_NAME} found \""

delete_peering_cmd += "; az network vnet peering list --resource-group ${AKS_RESOURCE_GROUP} --vnet-name ${AKS_MGMT_VNET_NAME} --output tsv | xargs -I {} az network vnet peering delete --name {} --resource-group ${AKS_RESOURCE_GROUP} --vnet-name ${AKS_MGMT_VNET_NAME} --only-show-errors --output none"
delete_peering_cmd += "; echo \" 2/2 All peerings deleted in mgmt cluster's vnet ${AKS_MGMT_VNET_NAME}\""

return delete_peering_cmd

##############################
# Actual work happens here
Expand Down

0 comments on commit 38e27e7

Please sign in to comment.