diff --git a/src/deploy-cromwell-on-azure/Configuration.cs b/src/deploy-cromwell-on-azure/Configuration.cs index a9909a89..fd70ee4a 100644 --- a/src/deploy-cromwell-on-azure/Configuration.cs +++ b/src/deploy-cromwell-on-azure/Configuration.cs @@ -31,6 +31,7 @@ public class Configuration : UserAccessibleConfiguration public abstract class UserAccessibleConfiguration { + public string AksNodeResourceGroupName { get; set; } public string IdentityResourceId { get; set; } public string AzureCloudName { get; set; } = AzureCloudConfig.DefaultAzureCloudName; public string BatchPrefix { get; set; } diff --git a/src/deploy-cromwell-on-azure/Deployer.cs b/src/deploy-cromwell-on-azure/Deployer.cs index c480c6e3..d0f34a07 100644 --- a/src/deploy-cromwell-on-azure/Deployer.cs +++ b/src/deploy-cromwell-on-azure/Deployer.cs @@ -436,9 +436,14 @@ await Execute("Connecting to Azure Services...", async () => managedIdentity = await CreateUserManagedIdentityAsync(resourceGroup); } - if (vnetAndSubnet is not null) + if (vnetAndSubnet is not null && vnetAndSubnet.Value.virtualNetwork is not null) { - ConsoleEx.WriteLine($"Creating VM in existing virtual network {vnetAndSubnet.Value.virtualNetwork.Name} and subnet {vnetAndSubnet.Value.vmSubnet.Name}"); + ConsoleEx.WriteLine($"Creating VM in existing virtual network {vnetAndSubnet.Value.virtualNetwork.Name}"); + } + + if (vnetAndSubnet is not null && vnetAndSubnet.Value.vmSubnet is not null) + { + ConsoleEx.WriteLine($"Using existing subnet {vnetAndSubnet.Value.vmSubnet.Name}"); } if (storageAccount is not null) @@ -455,19 +460,32 @@ await Task.WhenAll(new Task[] { Task.Run(async () => { - if (vnetAndSubnet is null) + if (vnetAndSubnet.Value.virtualNetwork is null) + { + configuration.VnetName = string.IsNullOrEmpty(configuration.VnetName) ? SdkContext.RandomResourceName($"{configuration.MainIdentifierPrefix}-", 15) : configuration.VnetName; + } + + if (vnetAndSubnet.Value.postgreSqlSubnet is null) { - configuration.VnetName = SdkContext.RandomResourceName($"{configuration.MainIdentifierPrefix}-", 15); configuration.PostgreSqlSubnetName = string.IsNullOrEmpty(configuration.PostgreSqlSubnetName) ? configuration.DefaultPostgreSqlSubnetName : configuration.PostgreSqlSubnetName; + } + + if (vnetAndSubnet.Value.batchSubnet is null) + { configuration.BatchSubnetName = string.IsNullOrEmpty(configuration.BatchSubnetName) ? configuration.DefaultBatchSubnetName : configuration.BatchSubnetName; - configuration.VmSubnetName = string.IsNullOrEmpty(configuration.VmSubnetName) ? configuration.DefaultVmSubnetName : configuration.VmSubnetName; - vnetAndSubnet = await CreateVnetAndSubnetsAsync(resourceGroup); + } - if (string.IsNullOrEmpty(this.configuration.BatchNodesSubnetId)) - { - this.configuration.BatchNodesSubnetId = vnetAndSubnet.Value.batchSubnet.Inner.Id; - } + if (vnetAndSubnet.Value.vmSubnet is null) + { + configuration.VmSubnetName = string.IsNullOrEmpty(configuration.VmSubnetName) ? configuration.DefaultVmSubnetName : configuration.VmSubnetName; } + + vnetAndSubnet = await CreateOrUseExistingVnetAndSubnetsAsync(resourceGroup); + + if (string.IsNullOrEmpty(this.configuration.BatchNodesSubnetId)) + { + this.configuration.BatchNodesSubnetId = vnetAndSubnet.Value.batchSubnet.Inner.Id; + } }), Task.Run(async () => { @@ -522,7 +540,7 @@ await Task.WhenAll(new Task[] { if (aksCluster is null && !configuration.ManualHelmDeployment) { - aksCluster = await ProvisionManagedCluster(resourceGroup, managedIdentity, logAnalyticsWorkspace, vnetAndSubnet?.virtualNetwork, vnetAndSubnet?.vmSubnet.Name, configuration.PrivateNetworking.GetValueOrDefault()); + aksCluster = await ProvisionManagedCluster(resourceGroup, managedIdentity, logAnalyticsWorkspace, vnetAndSubnet?.virtualNetwork, vnetAndSubnet?.vmSubnet.Name, configuration.PrivateNetworking.GetValueOrDefault(), configuration.AksNodeResourceGroupName); await EnableWorkloadIdentity(aksCluster, managedIdentity, resourceGroup); } }), @@ -806,7 +824,7 @@ private async Task GetExistingAKSClusterAsync(string aksClusterN cts.Token); } - private async Task ProvisionManagedCluster(IResource resourceGroupObject, IIdentity managedIdentity, IGenericResource logAnalyticsWorkspace, INetwork virtualNetwork, string subnetName, bool privateNetworking) + private async Task ProvisionManagedCluster(IResource resourceGroupObject, IIdentity managedIdentity, IGenericResource logAnalyticsWorkspace, INetwork virtualNetwork, string subnetName, bool privateNetworking, string nodeResourceGroupName) { var resourceGroup = resourceGroupObject.Name; var nodePoolName = "nodepool1"; @@ -830,7 +848,8 @@ private async Task ProvisionManagedCluster(IResource resourceGro Identity = new(managedIdentity.PrincipalId, managedIdentity.TenantId, Microsoft.Azure.Management.ContainerService.Models.ResourceIdentityType.UserAssigned) { UserAssignedIdentities = new Dictionary() - } + }, + NodeResourceGroup = nodeResourceGroupName }; cluster.Identity.UserAssignedIdentities.Add(managedIdentity.Id, new(managedIdentity.PrincipalId, managedIdentity.ClientId)); @@ -1487,49 +1506,72 @@ private Task AssignVmAsContributorToAppInsightsAsync(IIdentity managedIdentity, .CreateAsync(ct), cts.Token)); - private Task<(INetwork virtualNetwork, ISubnet vmSubnet, ISubnet postgreSqlSubnet, ISubnet batchSubnet)> CreateVnetAndSubnetsAsync(IResourceGroup resourceGroup) - => Execute( - $"Creating virtual network and subnets: {configuration.VnetName}...", - async () => - { - var defaultNsg = await CreateNetworkSecurityGroupAsync(resourceGroup, $"{configuration.VnetName}-default-nsg"); + private async Task<(INetwork virtualNetwork, ISubnet vmSubnet, ISubnet postgreSqlSubnet, ISubnet batchSubnet)> CreateOrUseExistingVnetAndSubnetsAsync(IResourceGroup resourceGroup) + { + var networkSecurityGroup = await CreateNetworkSecurityGroupAsync(resourceGroup, $"{configuration.VnetName}-default-nsg"); - var vnetDefinition = azureSubscriptionClient.Networks - .Define(configuration.VnetName) - .WithRegion(configuration.RegionName) - .WithExistingResourceGroup(resourceGroup) - .WithAddressSpace(configuration.VnetAddressSpace) - .DefineSubnet(configuration.VmSubnetName) - .WithAddressPrefix(configuration.VmSubnetAddressSpace) - .WithExistingNetworkSecurityGroup(defaultNsg) - .Attach(); + // Check for existing VNet + INetwork vnet = await azureSubscriptionClient.Networks.GetByResourceGroupAsync(resourceGroup.Name, configuration.VnetName) + ?? await DefineAndCreateVnetWithSubnets(resourceGroup, networkSecurityGroup); - vnetDefinition = vnetDefinition.DefineSubnet(configuration.PostgreSqlSubnetName) - .WithAddressPrefix(configuration.PostgreSqlSubnetAddressSpace) - .WithExistingNetworkSecurityGroup(defaultNsg) - .WithDelegation("Microsoft.DBforPostgreSQL/flexibleServers") - .Attach(); + // Ensure subnets are correctly set up + var vmSubnet = await EnsureSubnet(vnet, configuration.VmSubnetName, configuration.VmSubnetAddressSpace, networkSecurityGroup); + var postgreSqlSubnet = await EnsureSubnet(vnet, configuration.PostgreSqlSubnetName, configuration.PostgreSqlSubnetAddressSpace, networkSecurityGroup, true); + var batchSubnet = await EnsureSubnet(vnet, configuration.BatchSubnetName, configuration.BatchNodesSubnetAddressSpace, networkSecurityGroup); - vnetDefinition = vnetDefinition.DefineSubnet(configuration.BatchSubnetName) - .WithAddressPrefix(configuration.BatchNodesSubnetAddressSpace) - .WithExistingNetworkSecurityGroup(defaultNsg) - .Attach(); + return (vnet, vmSubnet, postgreSqlSubnet, batchSubnet); + } - var vnet = await vnetDefinition.CreateAsync(cts.Token); - var batchSubnet = vnet.Subnets.FirstOrDefault(s => s.Key.Equals(configuration.BatchSubnetName, StringComparison.OrdinalIgnoreCase)).Value; + async Task DefineAndCreateVnetWithSubnets(IResourceGroup resourceGroup, INetworkSecurityGroup defaultNsg) + { + var vnetDefinition = azureSubscriptionClient.Networks + .Define(configuration.VnetName) + .WithRegion(configuration.RegionName) + .WithExistingResourceGroup(resourceGroup) + .WithAddressSpace(configuration.VnetAddressSpace) + .DefineSubnet(configuration.VmSubnetName) + .WithAddressPrefix(configuration.VmSubnetAddressSpace) + .WithExistingNetworkSecurityGroup(defaultNsg) + .Attach() + .DefineSubnet(configuration.PostgreSqlSubnetName) + .WithAddressPrefix(configuration.PostgreSqlSubnetAddressSpace) + .WithExistingNetworkSecurityGroup(defaultNsg) + .WithDelegation("Microsoft.DBforPostgreSQL/flexibleServers") + .Attach() + .DefineSubnet(configuration.BatchSubnetName) + .WithAddressPrefix(configuration.BatchNodesSubnetAddressSpace) + .WithExistingNetworkSecurityGroup(defaultNsg) + .Attach(); + + return await vnetDefinition.CreateAsync(cts.Token); + } - // Use the new ResourceManager sdk to add the ACR service endpoint since it is absent from the fluent sdk. - var armBatchSubnet = (await armClient.GetSubnetResource(new ResourceIdentifier(batchSubnet.Inner.Id)).GetAsync(cancellationToken: cts.Token)).Value; + async Task EnsureSubnet(INetwork vnet, string subnetName, string subnetAddressSpace, INetworkSecurityGroup nsg, bool requiresDelegation = false) + { + // Check if subnet exists + if (!vnet.Subnets.TryGetValue(subnetName, out var subnet)) + { + // Create or update the VNet with a new subnet if it doesn't exist + var vnetUpdate = vnet.Update() + .DefineSubnet(subnetName) + .WithAddressPrefix(subnetAddressSpace) + .Attach(); - AddServiceEndpointsToSubnet(armBatchSubnet.Data); + // Add delegation if required + if (requiresDelegation) + { + vnetUpdate.UpdateSubnet(subnetName) + .WithDelegation("Microsoft.DBforPostgreSQL/flexibleServers") + .Parent(); + } - await armBatchSubnet.UpdateAsync(Azure.WaitUntil.Completed, armBatchSubnet.Data, cts.Token); + vnet = await vnetUpdate.ApplyAsync(); + subnet = vnet.Subnets[subnetName]; + } - return (vnet, - vnet.Subnets.FirstOrDefault(s => s.Key.Equals(configuration.VmSubnetName, StringComparison.OrdinalIgnoreCase)).Value, - vnet.Subnets.FirstOrDefault(s => s.Key.Equals(configuration.PostgreSqlSubnetName, StringComparison.OrdinalIgnoreCase)).Value, - batchSubnet); - }); + return subnet; + } + private Task CreateNetworkSecurityGroupAsync(IResourceGroup resourceGroup, string networkSecurityGroupName) { @@ -1915,8 +1957,7 @@ private async Task ValidateAndGetExistingStorageAccountAsync() return null; } - return (await GetExistingStorageAccountAsync(configuration.StorageAccountName)) - ?? throw new ValidationException($"If StorageAccountName is provided, the storage account must already exist in region {configuration.RegionName}, and be accessible to the current user.", displayExample: false); + return await GetExistingStorageAccountAsync(configuration.StorageAccountName); } private async Task ValidateAndGetExistingBatchAccountAsync() @@ -1930,82 +1971,79 @@ private async Task ValidateAndGetExistingBatchAccountAsync() ?? throw new ValidationException($"If BatchAccountName is provided, the batch account must already exist in region {configuration.RegionName}, and be accessible to the current user.", displayExample: false); } - private async Task<(INetwork virtualNetwork, ISubnet vmSubnet, ISubnet postgreSqlSubnet, ISubnet batchSubnet)?> ValidateAndGetExistingVirtualNetworkAsync() - { - static bool AllOrNoneSet(params string[] values) => values.All(v => !string.IsNullOrEmpty(v)) || values.All(v => string.IsNullOrEmpty(v)); - static bool NoneSet(params string[] values) => values.All(v => string.IsNullOrEmpty(v)); + private static bool AllOrNoneSet(params string[] values) => values.All(v => !string.IsNullOrEmpty(v)) || values.All(v => string.IsNullOrEmpty(v)); + + private static bool NoneSet(params string[] values) => values.All(v => string.IsNullOrEmpty(v)); - if (NoneSet(configuration.VnetResourceGroupName, configuration.VnetName, configuration.VmSubnetName)) + private async Task ValidateAndGetVirtualNetworkAsync() + { + if (NoneSet(configuration.VnetResourceGroupName, configuration.VnetName)) { if (configuration.PrivateNetworking.GetValueOrDefault()) { - throw new ValidationException($"{nameof(configuration.VnetResourceGroupName)}, {nameof(configuration.VnetName)} and {nameof(configuration.VmSubnetName)} are required when using private networking."); + throw new ValidationException($"{nameof(configuration.VnetResourceGroupName)}, {nameof(configuration.VnetName)} are required when using private networking."); } return null; } - if (!AllOrNoneSet(configuration.VnetResourceGroupName, configuration.VnetName, configuration.VmSubnetName, configuration.PostgreSqlSubnetName)) - { - throw new ValidationException($"{nameof(configuration.VnetResourceGroupName)}, {nameof(configuration.VnetName)}, {nameof(configuration.VmSubnetName)} and {nameof(configuration.PostgreSqlSubnetName)} are required when using an existing virtual network."); - } - - if (!AllOrNoneSet(configuration.VnetResourceGroupName, configuration.VnetName, configuration.VmSubnetName)) - { - throw new ValidationException($"{nameof(configuration.VnetResourceGroupName)}, {nameof(configuration.VnetName)} and {nameof(configuration.VmSubnetName)} are required when using an existing virtual network."); - } - - if (!await (await azureSubscriptionClient.ResourceGroups.ListAsync(true, cts.Token)).ToAsyncEnumerable().AnyAsync(rg => rg.Name.Equals(configuration.VnetResourceGroupName, StringComparison.OrdinalIgnoreCase), cts.Token)) + if (!await ResourceGroupExists(configuration.VnetResourceGroupName)) { throw new ValidationException($"Resource group '{configuration.VnetResourceGroupName}' does not exist."); } var vnet = await azureSubscriptionClient.Networks.GetByResourceGroupAsync(configuration.VnetResourceGroupName, configuration.VnetName, cts.Token); - - if (vnet is null) - { - throw new ValidationException($"Virtual network '{configuration.VnetName}' does not exist in resource group '{configuration.VnetResourceGroupName}'."); - } - - if (!vnet.RegionName.Equals(configuration.RegionName, StringComparison.OrdinalIgnoreCase)) + if (vnet != null && !vnet.RegionName.Equals(configuration.RegionName, StringComparison.OrdinalIgnoreCase)) { throw new ValidationException($"Virtual network '{configuration.VnetName}' must be in the same region that you are deploying to ({configuration.RegionName})."); } - var vmSubnet = vnet.Subnets.FirstOrDefault(s => s.Key.Equals(configuration.VmSubnetName, StringComparison.OrdinalIgnoreCase)).Value; + return vnet; + } - if (vmSubnet is null) + private async Task ResourceGroupExists(string resourceGroupName) + { + return await (await azureSubscriptionClient.ResourceGroups.ListAsync(true, cts.Token)).ToAsyncEnumerable().AnyAsync(rg => rg.Name.Equals(resourceGroupName, StringComparison.OrdinalIgnoreCase), cts.Token); + } + + private async Task ValidateAndGetSubnetAsync(INetwork virtualNetwork, string subnetName, string expectedDelegation = null) + { + var subnet = virtualNetwork?.Subnets.FirstOrDefault(s => s.Key.Equals(subnetName, StringComparison.OrdinalIgnoreCase)).Value; + + if (subnet == null) { - throw new ValidationException($"Virtual network '{configuration.VnetName}' does not contain subnet '{configuration.VmSubnetName}'"); + throw new ValidationException($"Virtual network '{configuration.VnetName}' does not contain subnet '{subnetName}'"); } - var resourceGraphClient = new ResourceGraphClient(tokenCredentials); - var postgreSqlSubnet = vnet.Subnets.FirstOrDefault(s => s.Key.Equals(configuration.PostgreSqlSubnetName, StringComparison.OrdinalIgnoreCase)).Value; - - if (postgreSqlSubnet is null) + if (!string.IsNullOrEmpty(expectedDelegation)) { - throw new ValidationException($"Virtual network '{configuration.VnetName}' does not contain subnet '{configuration.PostgreSqlSubnetName}'"); + var delegatedServices = subnet.Inner.Delegations.Select(d => d.ServiceName); + var hasIncorrectDelegations = expectedDelegation != null && !delegatedServices.Contains(expectedDelegation); + if (hasIncorrectDelegations) + { + throw new ValidationException($"Subnet '{subnetName}' must have '{expectedDelegation}' delegation."); + } } - var delegatedServices = postgreSqlSubnet.Inner.Delegations.Select(d => d.ServiceName); - var hasOtherDelegations = delegatedServices.Any(s => s != "Microsoft.DBforPostgreSQL/flexibleServers"); - var hasNoDelegations = !delegatedServices.Any(); + return subnet; + } - if (hasOtherDelegations) - { - throw new ValidationException($"Subnet '{configuration.PostgreSqlSubnetName}' can have 'Microsoft.DBforPostgreSQL/flexibleServers' delegation only."); - } + private async Task<(INetwork virtualNetwork, ISubnet vmSubnet, ISubnet postgreSqlSubnet, ISubnet batchSubnet)?> ValidateAndGetExistingVirtualNetworkAsync() + { + var vnet = await ValidateAndGetVirtualNetworkAsync(); + if (vnet == null) return null; - var resourcesInPostgreSqlSubnetQuery = $"where type =~ 'Microsoft.Network/networkInterfaces' | where properties.ipConfigurations[0].properties.subnet.id == '{postgreSqlSubnet.Inner.Id}'"; - var resourcesExist = (await resourceGraphClient.ResourcesAsync(new QueryRequest(new[] { configuration.SubscriptionId }, resourcesInPostgreSqlSubnetQuery), cts.Token)).TotalRecords > 0; + ISubnet vmSubnet = null; + ISubnet postgreSqlSubnet = null; + ISubnet batchSubnet = null; - if (hasNoDelegations && resourcesExist) + if (!configuration.PrivateNetworking.GetValueOrDefault()) { - throw new ValidationException($"Subnet '{configuration.PostgreSqlSubnetName}' must be either empty or have 'Microsoft.DBforPostgreSQL/flexibleServers' delegation."); + vmSubnet = await ValidateAndGetSubnetAsync(vnet, configuration.VmSubnetName); + postgreSqlSubnet = await ValidateAndGetSubnetAsync(vnet, configuration.PostgreSqlSubnetName, "Microsoft.DBforPostgreSQL/flexibleServers"); + batchSubnet = await ValidateAndGetSubnetAsync(vnet, configuration.BatchSubnetName); } - var batchSubnet = vnet.Subnets.FirstOrDefault(s => s.Key.Equals(configuration.BatchSubnetName, StringComparison.OrdinalIgnoreCase)).Value; - return (vnet, vmSubnet, postgreSqlSubnet, batchSubnet); } diff --git a/src/deploy-cromwell-on-azure/deploy-private-coa.sh b/src/deploy-cromwell-on-azure/deploy-private-coa.sh new file mode 100644 index 00000000..c3e0c370 --- /dev/null +++ b/src/deploy-cromwell-on-azure/deploy-private-coa.sh @@ -0,0 +1,152 @@ +#!/bin/bash + +# This script deploys a private instance of Cromwell on Azure (CoA) within a specified Azure subscription +# and location. It sets up necessary Azure resources including a resource group, virtual network, subnets, +# managed identity, storage account, and a container registry. It supports custom prefixes and allows +# deployment to different Azure clouds (e.g., Azure US Government, Azure China) with configurable options +# for private networking. The script handles resource creation, assigns necessary permissions to the +# managed identity, prepares a VM jumpbox for deployment from within the virtual network, and executes +# the CoA deployment binary. This script is designed to ensure a secure and isolated environment for running +# Cromwell on Azure, suitable for sensitive or regulated workloads. + +# Usage: deploy-private-coa.sh + +subscription=$1 +location=$2 #eastus, usgovarizona +prefix=${3:-"coa"} +azure_cloud_name=${4:-"azurecloud"} # azureusgovernment, azurechinacloud + +if [ -z "$subscription" ] || [ -z "$location" ]; then + echo "Usage: $0 [azure_cloud_name] [prefix]" + echo "Note: azure_cloud_name defaults to 'azurecloud' and prefix defaults to 'coa' if not provided." + exit 1 +fi + +coa_identifier="${prefix}coa" +resource_group_name="${prefix}-coa-main" +aks_name="${prefix}coaaks" +aks_resource_group_name="${prefix}-coa-aks-nodes" +vnet_name="${prefix}-coa-vnet" +subnet_name="${prefix}-coa-subnet" +deployer_subnet_name="${prefix}-coa-deployer-subnet" +vmsubnet_name="${prefix}-coa-aks-subnet" +batchsubnet_name="${prefix}-coa-batch-subnet" +sqlsubnet_name="${prefix}-sql-subnet" +mycontainerregistry="${prefix}coacr" +storage_account_name="${prefix}coastorage" +managed_identity_name="${prefix}-coa-mi" +private_endpoint_name="${prefix}-coa-pe" +private_endpoint_name_storage="${prefix}-coa-pe-storage" +private_endpoint_name_cr="${prefix}-coa-pe-storage-cr" +private_cr_dns_zone_name="${prefix}-coa-cr-dns-zone" +private_cr_zone_name="privatelink.azurecr.io" +tes_image_name="mcr.microsoft.com/CromwellOnAzure/tes:5.3.0.10760" +trigger_service_image_name="mcr.microsoft.com/CromwellOnAzure/triggerservice:5.3.0.10760" +temp_deployer_vm_name="${prefix}-coa-deploy" +coa_binary="deploy-cromwell-on-azure" +coa_binary_path="/tmp/coa" + +create_resource_group_if_not_exists() { + local rg_name=$1 + local rg_location=$2 + + if [ $(az group exists --name "$rg_name") = false ]; then + echo "Creating resource group '$rg_name' in location '$rg_location'." + az group create --name "$rg_name" --location "$rg_location" + else + echo "Resource group '$rg_name' already exists." + fi +} + +rm -f ../ga4gh-tes/nuget.config + +if [ -f "./deploy-cromwell-on-azure-linux" ]; then + coa_binary="deploy-cromwell-on-azure-linux" +elif [ -f "./deploy-cromwell-on-azure" ]; then + coa_binary="deploy-cromwell-on-azure" +else + # publish a new deployer binary + dotnet publish -r linux-x64 -c Release -o ./ /p:PublishSingleFile=true /p:DebugType=none /p:IncludeNativeLibrariesForSelfExtract=true +fi + +create_resource_group_if_not_exists $resource_group_name $location + +echo "Creating identity..." +managed_identity_id=$(az identity create -g $resource_group_name -n $managed_identity_name -l $location --query id --output tsv) + +echo "Waiting for identity to propagate..." +sleep 10 # Waits for 10 seconds + +echo "Creating virtual network..." +# Create VNet and Subnet +az network vnet create \ + --resource-group $resource_group_name \ + --name $vnet_name \ + --address-prefixes 10.1.0.0/16 \ + --subnet-name $deployer_subnet_name \ + --subnet-prefixes 10.1.100.0/24 + +echo "Creating deployer subnet..." +# Get the subnet ID for the VM creation +deployer_subnet_id=$(az network vnet subnet show \ + --resource-group $resource_group_name \ + --vnet-name $vnet_name \ + --name $deployer_subnet_name \ + --query id -o tsv) + +echo "Creating VM jumpbox within the virtual network to deploy from..." + +# Create the VM and specify the VNet's subnet by using the subnet ID +vm_public_ip=$(az vm create \ + --resource-group $resource_group_name \ + --name $temp_deployer_vm_name \ + --image Ubuntu2204 \ + --admin-username azureuser \ + --generate-ssh-keys \ + --subnet $deployer_subnet_id \ + --query publicIpAddress -o tsv) + +echo "Opening port 22 for SSH access..." +az vm open-port --port 22 --resource-group $resource_group_name --name $temp_deployer_vm_name + +echo "Waiting for port to open..." +sleep 10 + +echo "Installing AZ CLI and logging in..." +ssh -o StrictHostKeyChecking=no azureuser@$vm_public_ip "curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash; az cloud set -n $azure_cloud_name; az login --use-device-code" + +echo "Creating directory..." +ssh -o StrictHostKeyChecking=no azureuser@$vm_public_ip "mkdir -p /tmp/coa" + +echo "Copying CoA deployment binary..." +scp -o StrictHostKeyChecking=no $coa_binary azureuser@$vm_public_ip:/tmp/coa/$coa_binary + +echo "Installing Helm..." +ssh -o StrictHostKeyChecking=no azureuser@${vm_public_ip} "curl -fsSL https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash" + +echo "Setting CoA deployer binary to executable..." +ssh -o StrictHostKeyChecking=no azureuser@$vm_public_ip "chmod +x $coa_binary_path/$coa_binary" + +echo "Executing CoA deployer binary..." +ssh -o StrictHostKeyChecking=no azureuser@$vm_public_ip "$coa_binary_path/$coa_binary \ + --IdentityResourceId $managed_identity_id \ + --SubscriptionId $subscription \ + --ResourceGroupName $resource_group_name \ + --RegionName $location \ + --AzureCloudName $azure_cloud_name \ + --MainIdentifierPrefix $coa_identifier \ + --PrivateNetworking true \ + --DisableBatchNodesPublicIpAddress true \ + --AksClusterName $aks_name \ + --AksNodeResourceGroupName $aks_resource_group_name \ + --StorageAccountName $storage_account_name \ + --VnetName $vnet_name \ + --VnetResourceGroupName $resource_group_name \ + --SubnetName $subnet_name \ + --VmSubnetName $vmsubnet_name \ + --BatchSubnetName $batchsubnet_name \ + --PostgreSqlSubnetName $sqlsubnet_name \ + --HelmBinaryPath /usr/local/bin/helm \ + --TesImageName $tes_image_name \ + --TriggerServiceImageName $trigger_service_image_name \ + --DebugLogging true"