Skip to content

Commit

Permalink
install vnets and peer them with private DNS zone
Browse files Browse the repository at this point in the history
  • Loading branch information
@nawazkh
nawazkh committed Nov 9, 2024
1 parent 27e6896 commit fb3d377
Show file tree
Hide file tree
Showing 2 changed files with 64 additions and 43 deletions.
101 changes: 61 additions & 40 deletions Tiltfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ if "default_registry" in settings:
default_registry(settings.get("default_registry"))

os_arch = str(local("go env GOARCH")).rstrip("\n")
# TODO: no one is clearing MGMT_CLUSTER_NAME when using KIND, so this is always going to be true. Improve this logic.
if "aks" in settings.get("kustomize_substitutions", {}).get("MGMT_CLUSTER_NAME", ""):
print("Using AKS as management cluster, setting os_arch to amd64")
os_arch = "amd64"
Expand Down Expand Up @@ -117,7 +118,7 @@ def fixup_yaml_empty_arrays(yaml_str):
return yaml_str.replace("storedVersions: null", "storedVersions: []")

def validate_auth():
substitutions = settings.get("kustomize_substitutions", {})
substitutions = settings.get("kustomize_substitutions", {}) # all the env variables are exported here
os.environ.update(substitutions)
for sub in substitutions:
if sub[-4:] == "_B64":
Expand Down Expand Up @@ -324,6 +325,7 @@ def flavors():
labels = ["flavors"],
)


def deploy_worker_templates(template, substitutions):
# validate template exists
if not os.path.exists(template):
Expand Down Expand Up @@ -392,8 +394,11 @@ def deploy_worker_templates(template, substitutions):
flavor_cmd += "; until " + kubectl_cmd + " --kubeconfig ./${CLUSTER_NAME}.kubeconfig get configmap kubeadm-config --namespace=kube-system > /dev/null 2>&1; do sleep 5; done"
flavor_cmd += "; " + kubectl_cmd + " --kubeconfig ./${CLUSTER_NAME}.kubeconfig create namespace calico-system --dry-run=client -o yaml | " + kubectl_cmd + " --kubeconfig ./${CLUSTER_NAME}.kubeconfig apply -f -; " + kubectl_cmd + " --kubeconfig ./${CLUSTER_NAME}.kubeconfig get configmap kubeadm-config --namespace=kube-system -o yaml | sed 's/namespace: kube-system/namespace: calico-system/' | " + kubectl_cmd + " --kubeconfig ./${CLUSTER_NAME}.kubeconfig apply -f -"

flavor_cmd += get_addons(flavor_name)
# TODO: no one is clearing MGMT_CLUSTER_NAME when using KIND, so this is always going to be true. Improve this logic.
if "aks" in settings.get("kustomize_substitutions", {}).get("MGMT_CLUSTER_NAME", ""):
flavor_cmd += peer_vnets()

flavor_cmd += get_addons(flavor_name)
local_resource(
name = flavor_name,
cmd = ["sh", "-ec", flavor_cmd],
Expand Down Expand Up @@ -454,42 +459,60 @@ def waitforsystem():
local(kubectl_cmd + " wait --for=condition=ready --timeout=300s pod --all -n capi-system")

def peer_vnets():
# only peer vnets and create a private DNS zone if AKS cluster is the management cluster
if "aks" in settings.get("kustomize_substitutions", {}).get("MGMT_CLUSTER_NAME", ""):
# TODO: check for az cli to be installed in local

# get AKS management cluster vnet and resource group
mgmt_vnet_name = settings.get("kustomize_substitutions", {}).get("AKS_MGMT_VNET_NAME", "")
mgmt_rg = settings.get("kustomize_substitutions", {}).get("AKS_RESOURCE_GROUP", "")
azure_location = settings.get("azure_location")

# get workload cluster vnet and resource group
workload_cluster_name = os.getenv("CLUSTER_NAME", "")
workload_vnet_name = os.getenv("AZURE_VNET_NAME", workload_cluster_name + "-vnet")
workload_rg = os.getenv("AZURE_RESOURCE_GROUP", workload_cluster_name)
workload_cluster_fqdn_suffix = settings.get("APISERVER_LB_DNS_SUFFIX", "")
private_dns_zone_name = azure_location + ".cloudapp.azure.com"

# peer vnets
local("az network vnet peering create --name mgmt-to-workload --resource-group " + mgmt_rg + " --vnet-name " + mgmt_vnet_name + " --remote-vnet " + workload_vnet_name " + --allow-vnet-access --allow-forwarded-traffic")
local("az network vnet peering create --name workload-to-mgmt --resource-group " + workload_rg + " --vnet-name " + workload_vnet_name + " --remote-vnet " + mgmt_vnet_name + " --allow-vnet-access --allow-forwarded-traffic")

# check peering
local("az vnet peering wait --name mgmt-to-workload --resource-group " + mgmt_rg + " --vnet-name " + mgmt_vnet_name + " --created")
local("az vnet peering wait --name workload-to-mgmt --resource-group " + workload_rg + " --vnet-name " + workload_vnet_name + " --created")

# create private DNS zone
local("az network private-dns zone create --resource-group " + workload_rg + " --name "+ azure_location +".cloudapp.azure.com")

# link private DNS Zone to mgmt vnet and workload vnet
local("az network private-dns link vnet create --resource-group " + mgmt_rg + " --zone-name " + private_dns_zone_name + " --name mgmt-to-workload --virtual-network " + mgmt_vnet_name + " --registration-enabled false")
local("az network private-dns link vnet create --resource-group " + workload_rg + " --zone-name " + private_dns_zone_name + " --name workload-to-mgmt --virtual-network " + workload_vnet_name + " --registration-enabled false")

# create private DNS zone record
local("az network private-dns record-set a add-record --resource-group " + workload_cluster_name + " --zone-name " + private_dns_zone_name + " --record-set-name " + workload_cluster_name +"." + workload_cluster_fqdn_suffix + " --ipv4-address 10.0.0.100")

else:
print("Skipping peering vnets and creating private DNS zone as AKS cluster is not the management cluster")
# TODO: check for az cli to be installed in local

# get AKS management cluster vnet and resource group
#mgmt_vnet_name = "${AKS_MGMT_VNET_NAME}"
# mgmt_rg = "${AKS_RESOURCE_GROUP}"

# get workload cluster vnet and resource group
# workload_cluster_name = "${CLUSTER_NAME}"
# workload_vnet_name = workload_cluster_name + "-vnet"
# workload_rg = workload_cluster_name
# workload_cluster_fqdn_suffix = "${APISERVER_LB_DNS_SUFFIX}"
# private_dns_zone_name = "${AZURE_LOCATION}.cloudapp.azure.com"

# wait for AKS VNet to be in the state created
peering_cmd = "; echo \"\nPeering VNETs\" "
peering_cmd += "; az network vnet wait --resource-group ${AKS_RESOURCE_GROUP} --name ${AKS_MGMT_VNET_NAME} --created --timeout 180"
peering_cmd += "; export MGMT_VNET_ID=$(az network vnet show --resource-group ${AKS_RESOURCE_GROUP} --name ${AKS_MGMT_VNET_NAME} --query id --output tsv)"
peering_cmd += "; echo \"${AKS_MGMT_VNET_NAME} found \""

# wait for workload VNet to be created
peering_cmd += "; az network vnet wait --resource-group ${CLUSTER_NAME} --name ${CLUSTER_NAME}-vnet --created --timeout 180"
peering_cmd += "; export WORKLOAD_VNET_ID=$(az network vnet show --resource-group ${CLUSTER_NAME} --name ${CLUSTER_NAME}-vnet --query id --output tsv)"
peering_cmd += "; echo \"${CLUSTER_NAME}-vnet found \""

# peer mgmt vnet
peering_cmd += "; az network vnet peering create --name mgmt-to-${CLUSTER_NAME} --resource-group ${AKS_RESOURCE_GROUP} --vnet-name ${AKS_MGMT_VNET_NAME} --remote-vnet \"${WORKLOAD_VNET_ID}\" --allow-vnet-access true --allow-forwarded-traffic true --only-show-errors"
peering_cmd += "; az network vnet peering wait --name mgmt-to-${CLUSTER_NAME} --resource-group ${AKS_RESOURCE_GROUP} --vnet-name ${AKS_MGMT_VNET_NAME} --created --timeout 300 --only-show-errors"
peering_cmd += "; echo \"mgmt-to-${CLUSTER_NAME} peering created\""

# peer workload vnet
peering_cmd += "; az network vnet peering create --name ${CLUSTER_NAME}-to-mgmt --resource-group ${CLUSTER_NAME} --vnet-name ${CLUSTER_NAME}-vnet --remote-vnet \"${MGMT_VNET_ID}\" --allow-vnet-access true --allow-forwarded-traffic true --only-show-errors"
peering_cmd += "; az network vnet peering wait --name ${CLUSTER_NAME}-to-mgmt --resource-group ${CLUSTER_NAME} --vnet-name ${CLUSTER_NAME}-vnet --created --timeout 300 --only-show-errors"
peering_cmd += "; echo \"${CLUSTER_NAME}-to-mgmt peering created\""

# create private DNS zone
peering_cmd += "; az network private-dns zone create --resource-group ${CLUSTER_NAME} --name ${AZURE_LOCATION}.cloudapp.azure.com --only-show-errors"
peering_cmd += "; az network private-dns zone wait --resource-group ${CLUSTER_NAME} --name ${AZURE_LOCATION}.cloudapp.azure.com --created --timeout 300 --only-show-errors"
peering_cmd += "; echo \"${AZURE_LOCATION}.cloudapp.azure.com private DNS zone created\""

# link private DNS Zone to workload vnet
peering_cmd += "; az network private-dns link vnet create --resource-group ${CLUSTER_NAME} --zone-name ${AZURE_LOCATION}.cloudapp.azure.com --name ${CLUSTER_NAME}-to-mgmt --virtual-network \"${WORKLOAD_VNET_ID}\" --registration-enabled false --only-show-errors"
peering_cmd += "; az network private-dns link vnet wait --resource-group ${CLUSTER_NAME} --zone-name ${AZURE_LOCATION}.cloudapp.azure.com --name ${CLUSTER_NAME}-to-mgmt --created --timeout 300 --only-show-errors"
peering_cmd += "; echo \"workload cluster linked with private DNS zone\""

# link private DNS Zone to mgmt vnet
peering_cmd += "; az network private-dns link vnet create --resource-group ${CLUSTER_NAME} --zone-name ${AZURE_LOCATION}.cloudapp.azure.com --name mgmt-to-${CLUSTER_NAME} --virtual-network \"${MGMT_VNET_ID}\" --registration-enabled false --only-show-errors"
peering_cmd += "; az network private-dns link vnet wait --resource-group ${CLUSTER_NAME} --zone-name ${AZURE_LOCATION}.cloudapp.azure.com --name mgmt-to-${CLUSTER_NAME} --created --timeout 300 --only-show-errors"
peering_cmd += "; echo \"management cluster linked with private DNS zone\""

# create private DNS zone record
peering_cmd += "; az network private-dns record-set a add-record --resource-group ${CLUSTER_NAME} --zone-name ${AZURE_LOCATION}.cloudapp.azure.com --record-set-name ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX} --ipv4-address 10.0.0.100 --only-show-errors"
peering_cmd += "; echo \"${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX} private DNS zone record created\""

return peering_cmd


##############################
Expand Down Expand Up @@ -520,5 +543,3 @@ waitforsystem()
create_crs()

flavors()

peer_vnets()
6 changes: 3 additions & 3 deletions scripts/aks-as-mgmt.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,9 +82,9 @@ main() {
echo "SERVICE_ACCOUNT_SIGNING_KEY_FILEPATH: $SERVICE_ACCOUNT_SIGNING_KEY_FILEPATH"
echo "REGISTRY: $REGISTRY"
echo "APISERVER_LB_DNS_SUFFIX: $APISERVER_LB_DNS_SUFFIX"
echo "AKS_MGMT_VNET_NAME: $AKS_MGMT_VNET_NAME"
echo "AKS_MGMT_VNET_CIDR: $AKS_MGMT_VNET_CIDR"
echo "AKS_MGMT_SERVICE_CIDR: $AKS_MGMT_SERVICE_CIDR"
echo "AKS_MGMT_VNET_NAME: $AKS_MGMT_VNET_NAME"
echo "AKS_MGMT_VNET_CIDR: $AKS_MGMT_VNET_CIDR"
echo "AKS_MGMT_SERVICE_CIDR: $AKS_MGMT_SERVICE_CIDR"
echo "AKS_MGMT_SUBNET_NAME: $AKS_MGMT_SUBNET_NAME"
echo "AKS_MGMT_SUBNET_CIDR: $AKS_MGMT_SUBNET_CIDR"
echo "AKS_MGMT_DNS_SERVICE_IP: $AKS_MGMT_DNS_SERVICE_IP"
Expand Down

0 comments on commit fb3d377

Please sign in to comment.