TOAZ-372 Support for deployment in Azure Government Cloud (Cromwell) (#…

…7670)

Co-authored-by: Blair L Murri <[email protected]>
Co-authored-by: Jonathon Saunders <[email protected]>

cloudSupport/src/main/scala/cromwell/cloudsupport/azure/AzureConfiguration.scala

@@ -0,0 +1,28 @@
+package cromwell.cloudsupport.azure
+
+import com.azure.core.management.AzureEnvironment
+import com.typesafe.config.ConfigFactory
+import net.ceedubs.ficus.Ficus._
+
+object AzureConfiguration {
+  private val conf = ConfigFactory.load().getConfig("azure")
+  val azureEnvironment =
+    AzureEnvironmentConverter.fromString(
+      conf.as[Option[String]]("azure-environment").getOrElse(AzureEnvironmentConverter.Azure)
+    )
+  val azureTokenScopeManagement = conf.as[String]("token-scope-management")
+}
+
+object AzureEnvironmentConverter {
+  val Azure: String = "AzureCloud"
+  val AzureGov: String = "AzureUSGovernmentCloud"
+  val AzureChina: String = "AzureChinaCloud"
+
+  def fromString(s: String): AzureEnvironment = s match {
+    case AzureGov => AzureEnvironment.AZURE_US_GOVERNMENT
+    case AzureChina => AzureEnvironment.AZURE_CHINA
+    // a bit redundant, but I want to have a explicit case for Azure for clarity, even though it's the default
+    case Azure => AzureEnvironment.AZURE
+    case _ => AzureEnvironment.AZURE
+  }
+}

cloudSupport/src/main/scala/cromwell/cloudsupport/azure/AzureCredentials.scala

@@ -2,7 +2,6 @@ package cromwell.cloudsupport.azure
 
 import cats.implicits.catsSyntaxValidatedId
 import com.azure.core.credential.TokenRequestContext
-import com.azure.core.management.AzureEnvironment
 import com.azure.core.management.profile.AzureProfile
 import com.azure.identity.DefaultAzureCredentialBuilder
 import common.validation.ErrorOr.ErrorOr
@@ -20,8 +19,8 @@ case object AzureCredentials {
 
   final val tokenAcquisitionTimeout = 5.seconds
 
-  val azureProfile = new AzureProfile(AzureEnvironment.AZURE)
-  val tokenScope = "https://management.azure.com/.default"
+  val azureProfile = new AzureProfile(AzureConfiguration.azureEnvironment)
+  val tokenScope = AzureConfiguration.azureTokenScopeManagement
 
   private def tokenRequestContext: TokenRequestContext = {
     val trc = new TokenRequestContext()

cloudSupport/src/main/scala/cromwell/cloudsupport/azure/AzureUtils.scala

@@ -1,6 +1,5 @@
 package cromwell.cloudsupport.azure
 
-import com.azure.core.management.AzureEnvironment
 import com.azure.core.management.profile.AzureProfile
 import com.azure.identity.DefaultAzureCredentialBuilder
 import com.azure.resourcemanager.AzureResourceManager
@@ -33,7 +32,7 @@ object AzureUtils {
       .map(Success(_))
       .getOrElse(Failure(new Exception("Could not parse storage account")))
 
-    val azureProfile = new AzureProfile(AzureEnvironment.AZURE)
+    val azureProfile = new AzureProfile(AzureConfiguration.azureEnvironment)
 
     def azureCredentialBuilder = new DefaultAzureCredentialBuilder()
       .authorityHost(azureProfile.getEnvironment.getActiveDirectoryEndpoint)

core/src/main/resources/reference.conf

@@ -330,6 +330,11 @@ call-caching {
   max-failed-copy-attempts = 1000000
 }
 
+azure {
+    azure-environment = "AzureCloud"
+    token-scope-management = "https://management.azure.com/.default"
+}
+
 google {
 
   application-name = "cromwell"

filesystems/blob/src/main/scala/cromwell/filesystems/blob/BlobPathBuilder.scala

@@ -3,6 +3,7 @@ package cromwell.filesystems.blob
 import akka.http.scaladsl.model.Uri
 import com.azure.storage.blob.nio.AzureBlobFileAttributes
 import com.google.common.net.UrlEscapers
+import cromwell.cloudsupport.azure.AzureConfiguration
 import cromwell.core.path.{NioPath, Path, PathBuilder}
 import cromwell.filesystems.blob.BlobPathBuilder._
 
@@ -13,13 +14,13 @@ import scala.language.postfixOps
 import scala.util.{Failure, Success, Try}
 
 object BlobPathBuilder {
-  private val blobHostnameSuffix = ".blob.core.windows.net"
+  private val blobHostnameSuffix = s".blob${AzureConfiguration.azureEnvironment.getStorageEndpointSuffix}"
   sealed trait BlobPathValidation
   case class ValidBlobPath(path: String, container: BlobContainerName, endpoint: EndpointURL) extends BlobPathValidation
   case class UnparsableBlobPath(errorMessage: Throwable) extends BlobPathValidation
 
   def invalidBlobHostMessage(endpoint: EndpointURL) =
-    s"Malformed Blob URL for this builder: The endpoint $endpoint doesn't contain the expected host string '{SA}.blob.core.windows.net/'"
+    s"Malformed Blob URL for this builder: The endpoint $endpoint doesn't contain the expected host string '{SA}.${blobHostnameSuffix}/'"
   def invalidBlobContainerMessage(endpoint: EndpointURL) =
     s"Malformed Blob URL for this builder: Could not parse container"
   val externalToken =
@@ -103,7 +104,8 @@ object BlobPath {
   // 1) If the path starts with http:/ (single slash!) transform it to the containerName:<path inside container>
   //    format the library expects
   // 2) If the path looks like <container>:<path>, strip off the <container>: to leave the absolute path inside the container.
-  private val brokenPathRegex = "https:/([a-z0-9]+).blob.core.windows.net/([-a-zA-Z0-9]+)/(.*)".r
+  private val brokenPathRegex =
+    s"https:/([a-z0-9]+).blob${AzureConfiguration.azureEnvironment.getStorageEndpointSuffix}/([-a-zA-Z0-9]+)/(.*)".r
 
   // Blob files larger than 5 GB upload in parallel parts [0][1] and do not get a native `CONTENT-MD5` property.
   // Instead, some uploaders such as TES [2] calculate the md5 themselves and store it under this key in metadata.