Skip to content

Commit

Permalink
Merge branch 'main' into bmurri/simplify-task-management
Browse files Browse the repository at this point in the history
  • Loading branch information
@BMurri
BMurri committed Dec 11, 2024
2 parents a8a37c2 + 18fddd2 commit ce7e38d
Show file tree
Hide file tree
Showing 10 changed files with 345 additions and 63 deletions.
169 changes: 167 additions & 2 deletions src/deploy-cromwell-on-azure.Tests/HoconUtilTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -174,7 +174,7 @@ class = ""cromwell.filesystems.blob.BlobFileSystemManager""
blob : {
enabled : true
}
}
}
workflow-options {
workflow-log-dir: ""/cromwell-workflow-logs""
Expand Down Expand Up @@ -213,7 +213,7 @@ class = ""cromwell.filesystems.blob.BlobFileSystemManager""
}
}
}
}
}
database {
db.url = ""jdbc:postgresql://db.postgres.database.azure.com/cromwell_db?sslmode=require""
Expand All @@ -234,6 +234,171 @@ class : ""cromwell.filesystems.blob.BlobFileSystemManager"",
}
}
}
}".ReplaceLineEndings(), hocon.ToString(conf).ReplaceLineEndings());
}

[TestMethod]
public void TestRemove()
{
using HoconUtil hocon = new(@"include required(classpath(""application""))
akka.http.host-connection-pool.max-open-requests = 16384
akka.http.host-connection-pool.max-connections = 2000
call-caching {
enabled = false
}
system {
input-read-limits {
lines = 1000000
}
}
filesystems : {
blob : {
class : ""cromwell.filesystems.blob.BlobPathBuilderFactory"",
global : {
class : ""cromwell.filesystems.blob.BlobFileSystemManager"",
config : {
subscription : ""subscription""
}
}
}
}
engine.filesystems : {
local : {
enabled : true
},
http : {
enabled : true
},
blob : {
enabled : true
}
}
workflow-options {
workflow-log-dir: ""/cromwell-workflow-logs""
workflow-log-temporary: false
}
backend : {
default : ""TES"",
providers : {
TES : {
actor-factory : ""cromwell.backend.impl.tes.TesBackendLifecycleActorFactory"",
config : {
filesystems : {
http : {
enabled : true
},
local : {
enabled : true
},
blob : {
enabled : true
}
},
root : ""https://storageAccount.blob.storageSuffix/cromwell-executions/"",
dockerRoot : ""/cromwell-executions"",
endpoint : ""http://tes/v1/tasks"",
use_tes_11_preview_backend_parameters : true,
default-runtime-attributes : {
cpu : 1,
failOnStderr : false,
continueOnReturnCode : 0,
memory : ""2 GB"",
disk : ""10 GB"",
preemptible : true
}
}
}
}
}
database {
db.url = ""jdbc:postgresql://db.postgres.database.azure.com/cromwell_db?sslmode=require""
db.user = ""cromwell""
db.password = ""password""
db.driver = ""org.postgresql.Driver""
profile = ""slick.jdbc.PostgresProfile$""
db.connectionTimeout = 15000
}");
var conf = hocon.Parse();

_ = hocon.Remove(conf, "filesystems.blob");
_ = hocon.Remove(conf, "engine.filesystems.blob");
_ = hocon.Remove(conf, "backend.providers.TES.config.filesystems.blob");

Assert.AreEqual(@"include required(classpath(""application""))
akka.http.host-connection-pool.max-open-requests = 16384
akka.http.host-connection-pool.max-connections = 2000
call-caching {
enabled = false
}
system {
input-read-limits {
lines = 1000000
}
}
engine.filesystems : {
local : {
enabled : true
},
http : {
enabled : true
}
}
workflow-options {
workflow-log-dir: ""/cromwell-workflow-logs""
workflow-log-temporary: false
}
backend : {
default : ""TES"",
providers : {
TES : {
actor-factory : ""cromwell.backend.impl.tes.TesBackendLifecycleActorFactory"",
config : {
filesystems : {
http : {
enabled : true
},
local : {
enabled : true
}
},
root : ""https://storageAccount.blob.storageSuffix/cromwell-executions/"",
dockerRoot : ""/cromwell-executions"",
endpoint : ""http://tes/v1/tasks"",
use_tes_11_preview_backend_parameters : true,
default-runtime-attributes : {
cpu : 1,
failOnStderr : false,
continueOnReturnCode : 0,
memory : ""2 GB"",
disk : ""10 GB"",
preemptible : true
}
}
}
}
}
database {
db.url = ""jdbc:postgresql://db.postgres.database.azure.com/cromwell_db?sslmode=require""
db.user = ""cromwell""
db.password = ""password""
db.driver = ""org.postgresql.Driver""
profile = ""slick.jdbc.PostgresProfile$""
db.connectionTimeout = 15000
}".ReplaceLineEndings(), hocon.ToString(conf).ReplaceLineEndings());
}
}
Expand Down
43 changes: 35 additions & 8 deletions src/deploy-cromwell-on-azure/Deployer.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,7 @@ public class Deployer(Configuration configuration)

private Configuration configuration { get; } = configuration;
[System.Diagnostics.CodeAnalysis.SuppressMessage("Performance", "CA1859:Use concrete types when possible for improved performance", Justification = "We are using the base type everywhere.")]
[System.Diagnostics.CodeAnalysis.SuppressMessage("CodeQuality", "IDE0079:Remove unnecessary suppression", Justification = "CA1859 suppression seems appropriate in this use case.")]
private TokenCredential tokenCredential { get; set; }
private SubscriptionResource armSubscription { get; set; }
private ArmClient armClient { get; set; }
Expand All @@ -126,6 +127,7 @@ public class Deployer(Configuration configuration)
private KubernetesManager kubernetesManager { get; set; }
internal static AzureCloudConfig azureCloudConfig { get; private set; }
private static readonly System.Collections.Concurrent.ConcurrentDictionary<ResourceIdentifier, Azure.Storage.StorageSharedKeyCredential> storageKeys = [];
internal static bool IsStorageInPublicCloud { get; private set; }

private static async Task<T> EnsureResourceDataAsync<T>(T resource, Predicate<T> HasData, Func<T, Func<CancellationToken, Task<Response<T>>>> GetAsync, CancellationToken cancellationToken, Action<T> OnAcquisition = null) where T : ArmResource
{
Expand Down Expand Up @@ -184,6 +186,8 @@ public async Task<int> DeployAsync()
cloudEnvironment = new(azureCloudConfig.ArmEnvironment.Value, azureCloudConfig.AuthorityHost);
});

IsStorageInPublicCloud = "core.windows.net".Equals(azureCloudConfig.Suffixes.StorageSuffix, StringComparison.OrdinalIgnoreCase);

await Execute("Validating command line arguments...", () =>
{
ValidateInitialCommandLineArgs();
Expand Down Expand Up @@ -411,7 +415,7 @@ await Execute("Connecting to Azure Services...", async () =>
}
}

if (installedVersion is null || installedVersion < new Version(5, 5, 1))
if (IsStorageInPublicCloud && (installedVersion is null || installedVersion < new Version(5, 5, 1)))
{
var cromwellConfig = GetBlobClient(storageAccountData, ConfigurationContainerName, CromwellConfigurationFileName);
var configContent = await DownloadTextFromStorageAccountAsync(cromwellConfig, cts.Token);
Expand Down Expand Up @@ -446,6 +450,26 @@ class = ""cromwell.filesystems.blob.BlobFileSystemManager""
}
}

if (!IsStorageInPublicCloud && (installedVersion.Major == 5 && installedVersion.Minor == 5 && installedVersion.Build == 1)) // special case: revert 5.5.1 changes
{
var cromwellConfig = GetBlobClient(storageAccountData, ConfigurationContainerName, CromwellConfigurationFileName);
var configContent = await DownloadTextFromStorageAccountAsync(cromwellConfig, cts.Token);

if (configContent.Contains(".blob.", StringComparison.Ordinal))
{
using HoconUtil hocon = new(configContent);
var conf = hocon.Parse();

var changes = Hocon.HoconParser.Parse($@"backend.providers.TES.config.root = ""/{ExecutionsContainerName}""").Value.GetObject();
conf.Value.GetObject().Merge(changes);
_ = hocon.Remove(conf, "filesystems.blob");
_ = hocon.Remove(conf, "engine.filesystems.blob");
_ = hocon.Remove(conf, "backend.providers.TES.config.filesystems.blob");

await UploadTextToStorageAccountAsync(cromwellConfig, hocon.ToString(conf).ReplaceLineEndings("\r\n"), cts.Token);
}
}

//if (installedVersion is null || installedVersion < new Version(x, y, z))
//{
//}
Expand Down Expand Up @@ -1512,6 +1536,7 @@ private Task WritePersonalizedFilesToStorageAccountAsync(StorageAccountData stor
// Configure Cromwell config file for PostgreSQL on Azure.
await UploadTextToStorageAccountAsync(GetBlobClient(storageAccount, ConfigurationContainerName, CromwellConfigurationFileName), Utility.PersonalizeContent(
[
new Utility.ConfigNamedConditional("AzurePublic", IsStorageInPublicCloud),
new Utility.ConfigReplaceTextItem("{DatabaseUrl}", $"\"jdbc:postgresql://{configuration.PostgreSqlServerName}.{azureCloudConfig.Suffixes.PostgresqlServerEndpointSuffix}/{configuration.PostgreSqlCromwellDatabaseName}?sslmode=require\""),
new Utility.ConfigReplaceTextItem("{DatabaseUser}", $"\"{configuration.PostgreSqlCromwellUserLogin}\""),
new Utility.ConfigReplaceTextItem("{DatabasePassword}", $"\"{configuration.PostgreSqlCromwellUserPassword}\""),
Expand Down Expand Up @@ -1729,6 +1754,8 @@ private Task<PrivateDnsZoneResource> CreatePrivateDnsZoneAsync(VirtualNetworkRes
return dnsZone;
});

[System.Diagnostics.CodeAnalysis.SuppressMessage("Performance", "CA1861:Avoid constant arrays as arguments", Justification = "Only called once per process invocation.")]
[System.Diagnostics.CodeAnalysis.SuppressMessage("CodeQuality", "IDE0079:Remove unnecessary suppression", Justification = "CA1861 suppression is appropriate in this use case.")]
private Task ExecuteQueriesOnAzurePostgreSQLDbFromK8(IKubernetes kubernetesClient, string podName, string aksNamespace)
=> Execute(
$"Executing script to create users in tes_db and cromwell_db...",
Expand All @@ -1740,13 +1767,13 @@ private Task ExecuteQueriesOnAzurePostgreSQLDbFromK8(IKubernetes kubernetesClien
var adminUser = configuration.PostgreSqlAdministratorLogin;

var commands = new List<string[]> {
new string[] { "apt", "-qq", "update" },
new string[] { "apt", "-qq", "install", "-y", "postgresql-client" },
new string[] { "bash", "-lic", $"echo {configuration.PostgreSqlServerName}.{azureCloudConfig.Suffixes.PostgresqlServerEndpointSuffix}:{configuration.PostgreSqlServerPort}:{configuration.PostgreSqlCromwellDatabaseName}:{adminUser}:{configuration.PostgreSqlAdministratorPassword} > ~/.pgpass" },
new string[] { "bash", "-lic", $"echo {configuration.PostgreSqlServerName}.{azureCloudConfig.Suffixes.PostgresqlServerEndpointSuffix}:{configuration.PostgreSqlServerPort}:{configuration.PostgreSqlTesDatabaseName}:{adminUser}:{configuration.PostgreSqlAdministratorPassword} >> ~/.pgpass" },
new string[] { "bash", "-lic", "chmod 0600 ~/.pgpass" },
new string[] { "/usr/bin/psql", "-h", serverPath, "-U", adminUser, "-d", configuration.PostgreSqlCromwellDatabaseName, "-c", cromwellScript },
new string[] { "/usr/bin/psql", "-h", serverPath, "-U", adminUser, "-d", configuration.PostgreSqlTesDatabaseName, "-c", tesScript }
new [] { "apt", "-qq", "update" },
new [] { "apt", "-qq", "install", "-y", "postgresql-client" },
new [] { "bash", "-lic", $"echo {configuration.PostgreSqlServerName}.{azureCloudConfig.Suffixes.PostgresqlServerEndpointSuffix}:{configuration.PostgreSqlServerPort}:{configuration.PostgreSqlCromwellDatabaseName}:{adminUser}:{configuration.PostgreSqlAdministratorPassword} > ~/.pgpass" },
new [] { "bash", "-lic", $"echo {configuration.PostgreSqlServerName}.{azureCloudConfig.Suffixes.PostgresqlServerEndpointSuffix}:{configuration.PostgreSqlServerPort}:{configuration.PostgreSqlTesDatabaseName}:{adminUser}:{configuration.PostgreSqlAdministratorPassword} >> ~/.pgpass" },
new [] { "bash", "-lic", "chmod 0600 ~/.pgpass" },
new [] { "/usr/bin/psql", "-h", serverPath, "-U", adminUser, "-d", configuration.PostgreSqlCromwellDatabaseName, "-c", cromwellScript },
new [] { "/usr/bin/psql", "-h", serverPath, "-U", adminUser, "-d", configuration.PostgreSqlTesDatabaseName, "-c", tesScript }
};

await kubernetesManager.ExecuteCommandsOnPodAsync(kubernetesClient, podName, commands, aksNamespace);
Expand Down
27 changes: 24 additions & 3 deletions src/deploy-cromwell-on-azure/HoconUtil.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,27 @@ public HoconRoot Parse()
return result;
}


[System.Diagnostics.CodeAnalysis.SuppressMessage("Performance", "CA1822:Mark members as static", Justification = "Maintain API")]
[System.Diagnostics.CodeAnalysis.SuppressMessage("CodeQuality", "IDE0079:Remove unnecessary suppression", Justification = "CA1822 suppression is appropriate in this use case.")]
public bool Remove(HoconRoot config, string path)
{
var value = config.Value.GetObject();
var pathParts = path.Split('.');

foreach (var part in pathParts.Take(pathParts.Length - 1))
{
if (!value.TryGetValue(part, out var nextValue))
{
return false;
}

value = nextValue.GetObject();
}

return value.Remove(pathParts.Last());
}

/// <summary>
/// Converts <paramref name="config"/> into a more stable configuration
/// </summary>
Expand All @@ -134,10 +155,10 @@ public string ToString(HoconRoot config)
// If the element was not removed
if (config.TryGetValue(section.Field.Value.Field.Path, out var result))
{
// Determine if element was changed. Note that formatting is normalized in that case.
// Determine if element was changed. Note that formatting is normalized if the element was modified.
sb.Append(section.Field.Value.Field.Value.Equals(result)
? section.Text.ReplaceLineEndings()
: $"{section.Field.Value.Field.Path} : {result.ToString(withBraces ? 3 : 1, 2)} {Environment.NewLine}{Environment.NewLine}");
: $"{section.Field.Value.Field.Path} : {result.ToString(withBraces ? 3 : 1, 2)}{Environment.NewLine}{Environment.NewLine}");
}
}
}
Expand Down Expand Up @@ -193,7 +214,7 @@ static bool IsChildPathOf(HoconPath path, HoconPath parentPath)
return false;
}

for (int i = 0; i < parentPath.Count; i++)
for (var i = 0; i < parentPath.Count; ++i)
{
if (path[i] != parentPath[i])
{
Expand Down
Loading

0 comments on commit ce7e38d

Please sign in to comment.