Merge branch 'develop' into sts_AN-144_cost_capping_gpu

.github/workflows/integration_tests.yml

@@ -32,12 +32,30 @@ jobs:
       matrix:
         # Batch test fixes to land later
         include:
-          - build_type: centaurGcpBatch
+          - build_type: centaurPapiV2beta
+            build_mysql: 5.6
+            friendly_name: Life Sciences, MySQL 5.6 (current Terra production)
+          - build_type: centaurPapiV2beta
             build_mysql: 5.7
-            friendly_name: Centaur GCP Batch with MySQL 5.7
+            friendly_name: Life Sciences, MySQL 5.7
+          - build_type: centaurPapiV2beta
+            build_mysql: 8.0
+            friendly_name: Life Sciences, MySQL 8.0 (required Terra production)
           - build_type: centaurPapiV2beta
+            build_mysql: 8.4
+            friendly_name: Life Sciences, MySQL 8.4 (aspirational Terra production)
+          - build_type: centaurGcpBatch
+            build_mysql: 5.6
+            friendly_name: GCP Batch, MySQL 5.6
+          - build_type: centaurGcpBatch
             build_mysql: 5.7
-            friendly_name: Centaur Papi V2 Beta with MySQL 5.7
+            friendly_name: GCP Batch, MySQL 5.7
+          - build_type: centaurGcpBatch
+            build_mysql: 8.0
+            friendly_name: GCP Batch, MySQL 8.0
+          - build_type: centaurGcpBatch
+            build_mysql: 8.4
+            friendly_name: GCP Batch, MySQL 8.4
           - build_type: centaurPapiV2betaRestart
             build_mysql: 5.7
             friendly_name: Centaur Papi V2 Beta (restart)

.gitignore

@@ -40,7 +40,6 @@ scripts/docker-compose-mysql/compose/mysql/data
 # Vault rendered resources
 artifactory_credentials.properties
 aws_credentials
-centaur_secure.inc.conf
 cromwell-centaur-requester-pays-service-account.json
 cromwell-centaur-service-account.json
 cromwell-service-account.json

CHANGELOG.md

@@ -55,6 +55,10 @@ The `IX_WORKFLOW_STORE_ENTRY_WS` index is removed from `WORKFLOW_STORE_ENTRY`.
 
 The index had low cardinality and workflow pickup is faster without it. Migration time depends on workflow store size, but should be very fast for most installations. Terminal workflows are removed from the workflow store, so only running workflows contribute to the cost.
 
+#### Index additions
+
+The `IX_METADATA_ENTRY_WEU_MK` index is added to `METADATA_ENTRY`. In pre-release testing, the migration proceeded at about 3 million rows per minute. Please plan downtime accordingly.
+
 ### Bug fixes and small changes
 
  * Changed default boot disk size from 10GB to 20GB in PipelinesAPI and Google Batch backends

database/migration/src/main/resources/metadata_changesets/metadata_index_add_workflow_key.xml

@@ -0,0 +1,23 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<databaseChangeLog objectQuotingStrategy="QUOTE_ALL_OBJECTS"
+                   xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
+                   xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+                   xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.3.xsd">
+
+    <changeSet id="metadata_index_add_workflow_key" author="anichols" dbms="hsqldb,mariadb,mysql,postgresql">
+        <!--
+            This index creates at about 3M rows per minute on MySQL.
+            That would be an impossible multi-day downtime in Terra, so we manually pre-create the index asynchronously.
+            This changeset detects environments where this has been done and immediately marks itself as applied.
+        -->
+        <preConditions onFail="MARK_RAN">
+            <not>
+                <indexExists indexName="IX_METADATA_ENTRY_WEU_MK"/>
+            </not>
+        </preConditions>
+        <createIndex indexName="IX_METADATA_ENTRY_WEU_MK" tableName="METADATA_ENTRY">
+            <column name="WORKFLOW_EXECUTION_UUID"/>
+            <column name="METADATA_KEY"/>
+        </createIndex>
+    </changeSet>
+</databaseChangeLog>

database/migration/src/main/resources/sql_metadata_changelog.xml

@@ -19,6 +19,7 @@
     <include file="metadata_changesets/remove_non_summarizable_metadata_from_queue.xml" relativeToChangelogFile="true" />
     <include file="metadata_changesets/update_metadata_archive_index.xml" relativeToChangelogFile="true" />
     <include file="metadata_changesets/reset_archive_statuses_to_null.xml" relativeToChangelogFile="true" />
+    <include file="metadata_changesets/metadata_index_add_workflow_key.xml" relativeToChangelogFile="true" />
     <!-- WARNING!
       This changeset should always be last.
       It it always run (and should always run last) to set table ownership correctly.

database/sql/src/main/scala/cromwell/database/slick/tables/MetadataEntryComponent.scala

@@ -57,6 +57,23 @@ trait MetadataEntryComponent {
 
     // TODO: rename index via liquibase
     def ixMetadataEntryWeu = index("METADATA_WORKFLOW_IDX", workflowExecutionUuid, unique = false)
+
+    /**
+      * Index designed to accelerate common key-specific queries across an entire workflow, such as:
+      * - Get workflow-level `outputs%` keys (no tasks, requireEmptyJobKey = true)
+      * - Get all `vmStartTime%`, `vmEndTime%`, `vmCostPerHour%` keys in the workflow (include tasks, requireEmptyJobKey = false)
+      *
+      * It is NOT good, as in may make actively slower, queries that reference a specific job. If we do more
+      * with getting metadata for individual jobs, recommend creating this index with all 5 columns:
+      * - WORKFLOW_EXECUTION_UUID, CALL_FQN, JOB_SCATTER_INDEX, JOB_RETRY_ATTEMPT, METADATA_KEY
+      *
+      * Do NOT recommend this alternate order, as wildcards in the middle are inefficient and this can be
+      * slower than no indexes. Tested with 20M row `69e8259c` workflow in October 2024.
+      * - WORKFLOW_EXECUTION_UUID, METADATA_KEY, CALL_FQN, JOB_SCATTER_INDEX, JOB_RETRY_ATTEMPT
+      *
+      * @return A reference to the index
+      */
+    def ixMetadataEntryWeuMk = index("IX_METADATA_ENTRY_WEU_MK", (workflowExecutionUuid, metadataKey), unique = false)
   }
 
   val metadataEntries = TableQuery[MetadataEntries]

project/Dependencies.scala

@@ -46,7 +46,6 @@ object Dependencies {
   private val ficusV = "1.5.2"
   private val fs2V = "2.5.9" // scala-steward:off (CROM-6564)
   private val googleApiClientV = "2.1.4"
-  private val googleCloudBigQueryV = "2.25.0"
   // latest date via: https://github.com/googleapis/google-api-java-client-services/blob/main/clients/google-api-services-cloudkms/v1.metadata.json
   private val googleCloudKmsV = "v1-rev20230421-2.0.0"
   private val googleCloudMonitoringV = "3.2.5"
@@ -558,7 +557,6 @@ object Dependencies {
   val centaurDependencies: List[ModuleID] = List(
     "org.apache.commons" % "commons-math3" % commonsMathV,
     "com.github.kxbmap" %% "configs" % configsV,
-    "com.google.cloud" % "google-cloud-bigquery" % googleCloudBigQueryV % IntegrationTest,
     "org.gnieh" %% "diffson-spray-json" % diffsonSprayJsonV
   ) ++ circeDependencies ++ slf4jBindingDependencies ++ cloudSupportDependencies ++ http4sDependencies
 

src/ci/resources/centaur_application.conf

@@ -40,5 +40,4 @@ centaur {
 
     include "cromwell_database.inc.conf"
   }
-  include "centaur_secure.inc.conf"
 }