Merge branch 'main' into version-property-names

NationalSecurityAgency · Sep 13, 2024 · eba7bd6 · eba7bd6
2 parents 6d6264c + fd0977c
commit eba7bd6
Show file tree

Hide file tree

Showing 10 changed files with 43 additions and 12 deletions.
diff --git a/pom.xml b/pom.xml
@@ -8,7 +8,7 @@
         <relativePath>../../microservice-service-parent/pom.xml</relativePath>
     </parent>
     <artifactId>spring-boot-starter-datawave-query</artifactId>
-    <version>1.0.2-SNAPSHOT</version>
+    <version>1.0.6-SNAPSHOT</version>
     <url>https://github.com/NationalSecurityAgency/datawave-spring-boot-starter-query</url>
     <licenses>
         <license>
@@ -24,11 +24,11 @@
     </scm>
     <properties>
         <datawave.webservice.namespace>http://webservice.datawave/v1</datawave.webservice.namespace>
-        <version.datawave>7.1.2</version.datawave>
-        <version.datawave.hazelcast-client>4.0.1</version.datawave.hazelcast-client>
-        <version.datawave.starter>4.0.1</version.datawave.starter>
-        <version.datawave.starter-metadata>3.0.1</version.datawave.starter-metadata>
-        <version.datawave.starter-metrics>3.0.2</version.datawave.starter-metrics>
+        <version.datawave>7.3.0</version.datawave>
+        <version.datawave.hazelcast-client>4.0.2</version.datawave.hazelcast-client>
+        <version.datawave.starter>4.0.2</version.datawave.starter>
+        <version.datawave.starter-metadata>3.0.2</version.datawave.starter-metadata>
+        <version.datawave.starter-metrics>3.0.3</version.datawave.starter-metrics>
         <version.guava>31.0.1-jre</version.guava>
     </properties>
     <dependencyManagement>

diff --git a/src/main/java/datawave/microservice/query/config/DefaultParameterProperties.java b/src/main/java/datawave/microservice/query/config/DefaultParameterProperties.java
@@ -10,7 +10,7 @@ public class DefaultParameterProperties {
     @NotEmpty
     private String pool = "unassigned";
     @Nonnegative
-    private int maxConcurrentTasks = 10;
+    private int maxConcurrentTasks = 8;
 
     public String getPool() {
         return pool;

diff --git a/src/main/java/datawave/microservice/query/config/QueryProperties.java b/src/main/java/datawave/microservice/query/config/QueryProperties.java
@@ -44,6 +44,8 @@ public class QueryProperties {
     private QueryExpirationProperties expiration = new QueryExpirationProperties();
     private NextCallProperties nextCall = new NextCallProperties();
     private DefaultParameterProperties defaultParams = new DefaultParameterProperties();
+    private String poolOverride = null;
+    private String poolHeader = "Pool";
 
     public List<String> getAdminRoles() {
         return adminRoles;
@@ -156,4 +158,20 @@ public DefaultParameterProperties getDefaultParams() {
     public void setDefaultParams(DefaultParameterProperties defaultParams) {
         this.defaultParams = defaultParams;
     }
+
+    public String getPoolOverride() {
+        return poolOverride;
+    }
+
+    public void setPoolOverride(String poolOverride) {
+        this.poolOverride = poolOverride;
+    }
+
+    public String getPoolHeader() {
+        return poolHeader;
+    }
+
+    public void setPoolHeader(String poolHeader) {
+        this.poolHeader = poolHeader;
+    }
 }
diff --git a/src/main/java/datawave/microservice/query/mapreduce/status/MapReduceQueryStatus.java b/src/main/java/datawave/microservice/query/mapreduce/status/MapReduceQueryStatus.java
@@ -23,7 +23,7 @@
 
 public class MapReduceQueryStatus implements Serializable {
     private static final long serialVersionUID = -8731070243293603952L;
-
+    
     public enum MapReduceQueryState {
         DEFINED, SUBMITTED, RUNNING, SUCCEEDED, CLOSED, CANCELED, FAILED
     }

diff --git a/src/main/java/datawave/microservice/query/storage/QueryStatus.java b/src/main/java/datawave/microservice/query/storage/QueryStatus.java
@@ -28,7 +28,7 @@
 
 public class QueryStatus implements Serializable {
     private static final long serialVersionUID = -1633953472983032183L;
-
+    
     /**
      * These are the possible query states correlating with the activity that the user is requesting.<br>
      * DEFINE: define a query<br>

diff --git a/src/main/java/datawave/microservice/query/storage/TaskStates.java b/src/main/java/datawave/microservice/query/storage/TaskStates.java
@@ -24,7 +24,7 @@
 
 public class TaskStates implements Serializable {
     private static final long serialVersionUID = 1361359960334155427L;
-
+    
     /**
      * The possible task states: <br>
      * READY: ready to run <br>

diff --git a/src/main/resources/EdgeQueryLogicFactory.xml b/src/main/resources/EdgeQueryLogicFactory.xml
@@ -14,6 +14,7 @@
         <property name="maxResults" value="${datawave.query.logic.logics.EdgeQuery.maxResults}" />
         <property name="maxWork" value="${datawave.query.logic.logics.EdgeQuery.maxWork}" />
         <property name="queryThreads" value="${datawave.query.logic.logics.EdgeQuery.queryThreads}" />
+        <property name="maxConcurrentTasks" value="${datawave.query.logic.logics.EdgeQuery.maxConcurrentTasks}" />
         <property name="auditType" value="${datawave.query.logic.logics.EdgeQuery.auditType}" />
         <property name="dataTypes">
             <list value-type="datawave.data.type.Type">

diff --git a/src/main/resources/QueryLogicFactory.xml b/src/main/resources/QueryLogicFactory.xml
@@ -381,6 +381,7 @@
         <property name="maxResults" value="${datawave.query.logic.logics.FieldIndexCountQuery.maxResults}" />
         <property name="maxWork" value="${datawave.query.logic.logics.FieldIndexCountQuery.maxWork}" />
         <property name="queryThreads" value="${datawave.query.logic.logics.FieldIndexCountQuery.queryThreads}" />
+        <property name="maxConcurrentTasks" value="${datawave.query.logic.logics.FieldIndexCountQuery.maxConcurrentTasks}" />
         <property name="modelTableName" value="${datawave.query.logic.logics.FieldIndexCountQuery.modelTableName}" />
         <property name="modelName" value="${datawave.query.logic.logics.FieldIndexCountQuery.modelName}" />
         <property name="maxUniqueValues" value="${datawave.query.logic.logics.FieldIndexCountQuery.maxUniqueValues}" />
@@ -402,6 +403,7 @@
         <property name="maxResults" value="${datawave.query.logic.logics.ErrorFieldIndexCountQuery.maxResults}" />
         <property name="maxWork" value="${datawave.query.logic.logics.ErrorFieldIndexCountQuery.maxWork}" />
         <property name="queryThreads" value="${datawave.query.logic.logics.ErrorFieldIndexCountQuery.queryThreads}" />
+        <property name="maxConcurrentTasks" value="${datawave.query.logic.logics.ErrorFieldIndexCountQuery.maxConcurrentTasks}" />
         <property name="modelTableName" value="${datawave.query.logic.logics.ErrorFieldIndexCountQuery.modelTableName}" />
         <property name="modelName" value="${datawave.query.logic.logics.ErrorFieldIndexCountQuery.modelName}" />
         <property name="maxUniqueValues" value="${datawave.query.logic.logics.ErrorFieldIndexCountQuery.maxUniqueValues}" />
@@ -482,6 +484,7 @@
         <property name="indexTableName" value="${datawave.query.logic.logics.HitHighlights.indexTableName}" />
         <property name="reverseIndexTableName" value="${datawave.query.logic.logics.HitHighlights.reverseIndexTableName}" />
         <property name="queryThreads" value="${datawave.query.logic.logics.HitHighlights.queryThreads}" />
+        <property name="maxConcurrentTasks" value="${datawave.query.logic.logics.HitHighlights.maxConcurrentTasks}" />
         <property name="fullTableScanEnabled" value="${datawave.query.logic.logics.HitHighlights.fullTableScanEnabled}" />
         <property name="minimumSelectivity" value="${datawave.query.logic.logics.HitHighlights.minimumSelectivity}" />
         <property name="includeDataTypeAsField" value="${datawave.query.logic.logics.HitHighlights.includeDataTypeAsField}" />

diff --git a/src/test/java/datawave/microservice/query/storage/QueryTaskCheckpointTest.java b/src/test/java/datawave/microservice/query/storage/QueryTaskCheckpointTest.java
@@ -1,7 +1,9 @@
 package datawave.microservice.query.storage;
 
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Map;
 import java.util.Random;
 import java.util.SortedSet;
@@ -219,8 +221,10 @@ public void testTaskDescription() throws JsonProcessingException {
         ShardQueryConfiguration config = new ShardQueryConfiguration();
         config.setQuery(query);
         QueryData queryData = new QueryData("table", "logic",
-                        Collections.singletonList(new Range(new Key("row1", "cf1", "cq1", "(FOO)"), true, new Key("row2", "cf2", "cq2", "(BAR)"), false)),
-                        Collections.emptyList(), Collections.singletonList(new IteratorSetting(10, "test", "test", Collections.singletonMap("key", "value"))));
+                        new HashSet<>(Collections
+                                        .singleton(new Range(new Key("row1", "cf1", "cq1", "(FOO)"), true, new Key("row2", "cf2", "cq2", "(BAR)"), false))),
+                        new HashSet<>(),
+                        new ArrayList<>(Collections.singletonList(new IteratorSetting(10, "test", "test", Collections.singletonMap("key", "value")))));
         config.setQueries(Collections.singletonList(queryData));
         TaskDescription desc = new TaskDescription(key, config.getQueries());
 
@@ -229,6 +233,7 @@ public void testTaskDescription() throws JsonProcessingException {
 
         String json = new ObjectMapper().registerModule(new GuavaModule()).writeValueAsString(desc);
         TaskDescription desc2 = new ObjectMapper().registerModule(new GuavaModule()).readerFor(TaskDescription.class).readValue(json);
+        // NOTE: This comparison is very brittle. It breaks if the collection types don't match (i.e. arraylist vs set vs singletonlist, etc.)
         Assertions.assertEquals(desc, desc2);
         Assertions.assertEquals(desc.hashCode(), desc2.hashCode());
 

diff --git a/src/test/resources/config/application-QueryStarterDefaults.yml b/src/test/resources/config/application-QueryStarterDefaults.yml
@@ -326,6 +326,7 @@ datawave:
           queryThreads: 16
           auditType: "NONE"
           logicDescription: "Retrieve graph edges matching the search term(s)"
+          maxConcurrentTasks: 10
           querySyntaxParsers:
             "JEXL": ""
             "LIST": ""
@@ -358,6 +359,7 @@ datawave:
           maxUniqueValues: 20000
           auditType: "NONE"
           logicDescription: "Indexed Fields Only:  Given FIELDNAME returns counts for each unique value. Given FIELDNAME:FIELDVALUE returns count for only that value."
+          maxConcurrentTasks: 10
 
         ErrorFieldIndexCountQuery:
           checkpointable: false
@@ -373,6 +375,7 @@ datawave:
           maxUniqueValues: 20000
           auditType: "NONE"
           logicDescription: "FieldIndex count query (experimental)"
+          maxConcurrentTasks: 10
 
         TermFrequencyQuery:
           tableName: ${warehouse.tables.shard.name}
@@ -460,6 +463,7 @@ datawave:
           ivaratorMaxOpenFiles: ${warehouse.defaults.ivaratorMaxOpenFiles}
           ivaratorCacheScanPersistThreshold: 100000
           ivaratorCacheScanTimeoutMinutes: ${warehouse.defaults.ivaratorCacheScanTimeoutMinutes}
+          maxConcurrentTasks: 10
           querySyntaxParsers:
             JEXL: ""
             LUCENE: "LuceneToJexlQueryParser"