Merge branch 'integration' into feature/returnFieldExpressions

NationalSecurityAgency · Jun 7, 2023 · 8cdc7b8 · 8cdc7b8
2 parents 4f33448 + bcf2b1a
commit 8cdc7b8
Show file tree

Hide file tree

Showing 84 changed files with 1,713 additions and 394 deletions.
diff --git a/common-test/pom.xml b/common-test/pom.xml
@@ -4,7 +4,7 @@
     <parent>
         <groupId>gov.nsa.datawave</groupId>
         <artifactId>datawave-parent</artifactId>
-        <version>5.6.0-SNAPSHOT</version>
+        <version>5.7.0-SNAPSHOT</version>
     </parent>
     <artifactId>datawave-common-test</artifactId>
     <name>${project.artifactId}</name>

diff --git a/contrib/datawave-quickstart/bin/query.sh b/contrib/datawave-quickstart/bin/query.sh
@@ -55,7 +55,7 @@ function datawaveQuery() {
 
    local curlcmd="/usr/bin/curl \
     --silent --write-out 'HTTP_STATUS_CODE:%{http_code};TOTAL_TIME:%{time_total};CONTENT_TYPE:%{content_type}' \
-    --insecure --cert "${DW_CURL_CERT}" --key "${DW_CURL_KEY_RSA}" --cacert "${DW_CURL_CA}" \
+    --insecure --cert "${DW_CURL_CERT}" --key "${DW_CURL_KEY_RSA}" --cacert "${DW_CURL_CA}" --keepalive-time 180 \
     --header 'Content-Type: application/x-www-form-urlencoded;charset=UTF-8' --header 'Accept: application/json' \
     ${DW_REQUEST_HEADERS} ${DW_CURL_DATA} -X POST ${DW_QUERY_URI}/${DW_QUERY_LOGIC}/${DW_QUERY_CREATE_MODE}"
 
@@ -333,7 +333,7 @@ function closeQuery() {
 
    local curlcmd="/usr/bin/curl \
    --silent --write-out 'HTTP_STATUS_CODE:%{http_code};TOTAL_TIME:%{time_total};CONTENT_TYPE:%{content_type}' \
-   --insecure --cert "${DW_CURL_CERT}" --key "${DW_CURL_KEY_RSA}" --cacert "${DW_CURL_CA}" \
+   --insecure --cert "${DW_CURL_CERT}" --key "${DW_CURL_KEY_RSA}" --cacert "${DW_CURL_CA} --keepalive-time 180 " \
    -X PUT ${DW_QUERY_URI}/${DW_QUERY_ID}/close"
 
    local response="$( eval "${curlcmd}" )"
@@ -368,7 +368,7 @@ function getNextPage() {
 
    local curlcmd="/usr/bin/curl \
    --silent --write-out 'HTTP_STATUS_CODE:%{http_code};TOTAL_TIME:%{time_total};CONTENT_TYPE:%{content_type}' \
-   --insecure --header 'Accept: application/json' ${DW_REQUEST_HEADERS} --cert "${DW_CURL_CERT}" --key "${DW_CURL_KEY_RSA}" --cacert "${DW_CURL_CA}" \
+   --insecure --header 'Accept: application/json' ${DW_REQUEST_HEADERS} --cert "${DW_CURL_CERT}" --key "${DW_CURL_KEY_RSA}" --cacert "${DW_CURL_CA} --keepalive-time 180 " \
    -X GET ${DW_QUERY_URI}/${DW_QUERY_ID}/next"
 
    local response="$( eval "${curlcmd}" )"

diff --git a/contrib/datawave-quickstart/bin/services/datawave/test-web/run.sh b/contrib/datawave-quickstart/bin/services/datawave/test-web/run.sh
@@ -280,7 +280,7 @@ function runTest() {
 
     TEST_COMMAND="${CURL} ${CURL_ADDITIONAL_OPTS} --silent \
 --write-out 'HTTP_STATUS_CODE:%{http_code};TOTAL_TIME:%{time_total};CONTENT_TYPE:%{content_type}' \
---insecure --cert '${DW_CURL_CERT}' --key '${DW_CURL_KEY_RSA}' --cacert '${DW_CURL_CA}' ${TEST_URL_OPTS}"
+--insecure --cert '${DW_CURL_CERT}' --keepalive-time 180 --key '${DW_CURL_KEY_RSA}' --cacert '${DW_CURL_CA}' ${TEST_URL_OPTS}"
 
     if [ "${LIST_TESTS}" == true ] ; then
         printCurrentTestInfo
@@ -533,4 +533,4 @@ if [ "${LIST_TESTS}" != true ] ; then
    printTestSummary
    cleanup
    exitWithTestStatus
-fi
+fi
diff --git a/docs/pom.xml b/docs/pom.xml
@@ -4,7 +4,7 @@
     <parent>
         <groupId>gov.nsa.datawave</groupId>
         <artifactId>datawave-parent</artifactId>
-        <version>5.6.0-SNAPSHOT</version>
+        <version>5.7.0-SNAPSHOT</version>
     </parent>
     <artifactId>datawave-docs</artifactId>
     <!-- Declare all datawave modules as dependencies in order to have the javadoc include all the datawave source. -->

diff --git a/pom.xml b/pom.xml
@@ -3,7 +3,7 @@
     <modelVersion>4.0.0</modelVersion>
     <groupId>gov.nsa.datawave</groupId>
     <artifactId>datawave-parent</artifactId>
-    <version>5.6.0-SNAPSHOT</version>
+    <version>5.7.0-SNAPSHOT</version>
     <packaging>pom</packaging>
     <name>DataWave</name>
     <description>DataWave is a Java-based ingest and query framework that leverages Apache Accumulo to provide fast, secure access to your data.</description>

diff --git a/properties/default.properties b/properties/default.properties
@@ -137,8 +137,11 @@ jboss.runas.user=jboss
 
 # Defines the size parameters of the worker's task thread pool
 # Suggest setting values here based on accumulo connection pool sizes, available cores, and expected access patterns
+# From WildFly manual: Workers for I/O channel notification.  The maximum number of threads for the worker task thread pool.
+#   default cpuCount * 16.  Once this is filled, tasks that cannot be queued will be rejected.
 wildfly.io.worker.default.task-max-threads=16
 # How many I/O (selector) threads should be maintained. Generally this number should be a small constant multiple of the number of available cores.
+# From WildFly manual: Specify the number of I/O threads to create for the worker.  Default cpuCount * 2
 wildfly.io.worker.default.io-threads=2
 
 ############################

diff --git a/warehouse/accumulo-extensions/pom.xml b/warehouse/accumulo-extensions/pom.xml
@@ -4,7 +4,7 @@
     <parent>
         <groupId>gov.nsa.datawave</groupId>
         <artifactId>datawave-warehouse-parent</artifactId>
-        <version>5.6.0-SNAPSHOT</version>
+        <version>5.7.0-SNAPSHOT</version>
     </parent>
     <artifactId>datawave-accumulo-extensions</artifactId>
     <name>${project.artifactId}</name>

diff --git a/warehouse/assemble/datawave/pom.xml b/warehouse/assemble/datawave/pom.xml
@@ -4,7 +4,7 @@
     <parent>
         <groupId>gov.nsa.datawave</groupId>
         <artifactId>assemble-parent</artifactId>
-        <version>5.6.0-SNAPSHOT</version>
+        <version>5.7.0-SNAPSHOT</version>
     </parent>
     <artifactId>assemble-datawave</artifactId>
     <packaging>pom</packaging>

diff --git a/warehouse/assemble/pom.xml b/warehouse/assemble/pom.xml
@@ -4,7 +4,7 @@
     <parent>
         <groupId>gov.nsa.datawave</groupId>
         <artifactId>datawave-warehouse-parent</artifactId>
-        <version>5.6.0-SNAPSHOT</version>
+        <version>5.7.0-SNAPSHOT</version>
     </parent>
     <artifactId>assemble-parent</artifactId>
     <packaging>pom</packaging>

diff --git a/warehouse/assemble/webservice/pom.xml b/warehouse/assemble/webservice/pom.xml
@@ -4,7 +4,7 @@
     <parent>
         <groupId>gov.nsa.datawave</groupId>
         <artifactId>assemble-parent</artifactId>
-        <version>5.6.0-SNAPSHOT</version>
+        <version>5.7.0-SNAPSHOT</version>
     </parent>
     <artifactId>assemble-webservice</artifactId>
     <name>${project.artifactId}</name>

diff --git a/warehouse/common/pom.xml b/warehouse/common/pom.xml
@@ -4,7 +4,7 @@
     <parent>
         <groupId>gov.nsa.datawave</groupId>
         <artifactId>datawave-warehouse-parent</artifactId>
-        <version>5.6.0-SNAPSHOT</version>
+        <version>5.7.0-SNAPSHOT</version>
     </parent>
     <artifactId>datawave-common</artifactId>
     <name>${project.artifactId}</name>

diff --git a/warehouse/core/pom.xml b/warehouse/core/pom.xml
@@ -4,7 +4,7 @@
     <parent>
         <groupId>gov.nsa.datawave</groupId>
         <artifactId>datawave-warehouse-parent</artifactId>
-        <version>5.6.0-SNAPSHOT</version>
+        <version>5.7.0-SNAPSHOT</version>
     </parent>
     <artifactId>datawave-core</artifactId>
     <packaging>jar</packaging>

diff --git a/warehouse/core/src/test/java/datawave/ingest/protobuf/TermWeightPositionTest.java b/warehouse/core/src/test/java/datawave/ingest/protobuf/TermWeightPositionTest.java
@@ -101,6 +101,24 @@ public void testComparator() {
         Collections.sort(result);
         Assert.assertEquals(listExpected, result);
     }
+
+    @Test
+    public void testBuilderReset() {
+        TermWeightPosition.Builder builder = new TermWeightPosition.Builder();
+        TermWeightPosition expected = builder.setOffset(1).setPrevSkips(0).setScore(0).setZeroOffsetMatch(true).build();
+        TermWeightPosition position = builder.setOffset(1).setPrevSkips(0).setScore(0).setZeroOffsetMatch(true).build();
+        Assert.assertEquals(expected, position);
+
+        expected = builder.setOffset(1).setPrevSkips(0).setScore(0).setZeroOffsetMatch(true).build();
+        builder.reset();
+        position = builder.setOffset(1).setPrevSkips(0).setScore(0).setZeroOffsetMatch(true).build();
+        Assert.assertEquals(expected, position);
+
+        expected = builder.setOffset(1).setPrevSkips(0).setScore(0).setZeroOffsetMatch(false).build();
+        builder.reset();
+        position = builder.setOffset(1).setPrevSkips(0).setScore(0).setZeroOffsetMatch(false).build();
+        Assert.assertEquals(expected, position);
+    }
 
     @Test
     public void testPositionScoreToTermWeightScore() {

diff --git a/warehouse/data-dictionary-core/pom.xml b/warehouse/data-dictionary-core/pom.xml
@@ -4,7 +4,7 @@
     <parent>
         <groupId>gov.nsa.datawave</groupId>
         <artifactId>datawave-warehouse-parent</artifactId>
-        <version>5.6.0-SNAPSHOT</version>
+        <version>5.7.0-SNAPSHOT</version>
     </parent>
     <artifactId>datawave-data-dictionary-core</artifactId>
     <packaging>jar</packaging>

diff --git a/warehouse/edge-dictionary-core/pom.xml b/warehouse/edge-dictionary-core/pom.xml
@@ -4,7 +4,7 @@
     <parent>
         <groupId>gov.nsa.datawave</groupId>
         <artifactId>datawave-warehouse-parent</artifactId>
-        <version>5.6.0-SNAPSHOT</version>
+        <version>5.7.0-SNAPSHOT</version>
     </parent>
     <artifactId>datawave-edge-dictionary-core</artifactId>
     <packaging>jar</packaging>

diff --git a/warehouse/edge-model-configuration-core/pom.xml b/warehouse/edge-model-configuration-core/pom.xml
@@ -4,7 +4,7 @@
     <parent>
         <groupId>gov.nsa.datawave</groupId>
         <artifactId>datawave-warehouse-parent</artifactId>
-        <version>5.6.0-SNAPSHOT</version>
+        <version>5.7.0-SNAPSHOT</version>
     </parent>
     <artifactId>datawave-edge-model-configuration-core</artifactId>
     <packaging>jar</packaging>

diff --git a/warehouse/index-stats/pom.xml b/warehouse/index-stats/pom.xml
@@ -4,7 +4,7 @@
     <parent>
         <groupId>gov.nsa.datawave</groupId>
         <artifactId>datawave-warehouse-parent</artifactId>
-        <version>5.6.0-SNAPSHOT</version>
+        <version>5.7.0-SNAPSHOT</version>
     </parent>
     <artifactId>datawave-index-stats</artifactId>
     <packaging>jar</packaging>

diff --git a/warehouse/ingest-configuration/pom.xml b/warehouse/ingest-configuration/pom.xml
@@ -4,7 +4,7 @@
     <parent>
         <groupId>gov.nsa.datawave</groupId>
         <artifactId>datawave-warehouse-parent</artifactId>
-        <version>5.6.0-SNAPSHOT</version>
+        <version>5.7.0-SNAPSHOT</version>
     </parent>
     <artifactId>datawave-ingest-configuration</artifactId>
     <dependencies>

diff --git a/warehouse/ingest-core/pom.xml b/warehouse/ingest-core/pom.xml
@@ -4,7 +4,7 @@
     <parent>
         <groupId>gov.nsa.datawave</groupId>
         <artifactId>datawave-warehouse-parent</artifactId>
-        <version>5.6.0-SNAPSHOT</version>
+        <version>5.7.0-SNAPSHOT</version>
     </parent>
     <artifactId>datawave-ingest-core</artifactId>
     <packaging>jar</packaging>

diff --git a/...house/ingest-core/src/test/java/datawave/ingest/data/config/XMLFieldConfigHelperTest.java b/...house/ingest-core/src/test/java/datawave/ingest/data/config/XMLFieldConfigHelperTest.java
@@ -82,7 +82,7 @@ private HttpServer createFileServer(String path, int port) throws Exception {
     }
 
     private String readFile(String path) {
-        StringBuffer sb = new StringBuffer();
+        StringBuilder sb = new StringBuilder();
 
         InputStream istream = getClass().getClassLoader().getResourceAsStream(path);
         try (Scanner scanner = new Scanner(istream)) {

diff --git a/.../ingest-core/src/test/java/datawave/ingest/mapreduce/job/SafeFileOutputCommitterTest.java b/.../ingest-core/src/test/java/datawave/ingest/mapreduce/job/SafeFileOutputCommitterTest.java
@@ -437,7 +437,7 @@ private void validateContent(Path dir) throws IOException {
     private void validateContent(File dir, String fileName) throws IOException {
         File expectedFile = new File(dir, fileName);
         assertTrue("Could not find " + expectedFile, expectedFile.exists());
-        StringBuffer expectedOutput = new StringBuffer();
+        StringBuilder expectedOutput = new StringBuilder();
         expectedOutput.append(key1).append('\t').append(val1).append("\n");
         expectedOutput.append(val1).append("\n");
         expectedOutput.append(val2).append("\n");

diff --git a/warehouse/ingest-csv/pom.xml b/warehouse/ingest-csv/pom.xml
@@ -4,7 +4,7 @@
     <parent>
         <groupId>gov.nsa.datawave</groupId>
         <artifactId>datawave-warehouse-parent</artifactId>
-        <version>5.6.0-SNAPSHOT</version>
+        <version>5.7.0-SNAPSHOT</version>
     </parent>
     <artifactId>datawave-ingest-csv</artifactId>
     <packaging>jar</packaging>

diff --git a/warehouse/ingest-json/pom.xml b/warehouse/ingest-json/pom.xml
@@ -4,7 +4,7 @@
     <parent>
         <groupId>gov.nsa.datawave</groupId>
         <artifactId>datawave-warehouse-parent</artifactId>
-        <version>5.6.0-SNAPSHOT</version>
+        <version>5.7.0-SNAPSHOT</version>
     </parent>
     <artifactId>datawave-ingest-json</artifactId>
     <packaging>jar</packaging>

diff --git a/warehouse/ingest-nyctlc/pom.xml b/warehouse/ingest-nyctlc/pom.xml
@@ -4,7 +4,7 @@
     <parent>
         <groupId>gov.nsa.datawave</groupId>
         <artifactId>datawave-warehouse-parent</artifactId>
-        <version>5.6.0-SNAPSHOT</version>
+        <version>5.7.0-SNAPSHOT</version>
     </parent>
     <artifactId>datawave-ingest-nyctlc</artifactId>
     <packaging>jar</packaging>

diff --git a/warehouse/ingest-nyctlc/src/main/java/datawave/ingest/nyctlc/NYCTLCReader.java b/warehouse/ingest-nyctlc/src/main/java/datawave/ingest/nyctlc/NYCTLCReader.java
@@ -53,27 +53,30 @@ public boolean nextKeyValue() throws IOException {
         // followed by a blank line, followed by our entries
         // This is here to account for that
         boolean hasNext, completeRecord;
+        StringBuilder sb = new StringBuilder();
         do {
             hasNext = super.nextKeyValue();
 
             if (this.value != null && !this.value.toString().isEmpty() && !this.value.toString().equals(rawHeader)) {
                 // update value to be list of field/value pairings
-                StringBuffer fvBuf = new StringBuffer();
                 String[] values = this.value.toString().split(((NYCTLCHelper) helper).getSeparator());
-                if (values.length > ((NYCTLCHelper) helper).getParsedHeader().length)
+                if (values.length > ((NYCTLCHelper) helper).getParsedHeader().length) {
                     log.debug("More values present than expected.");
-
+                }
                 int numFields = Math.min(values.length, ((NYCTLCHelper) helper).getParsedHeader().length);
 
                 completeRecord = true;
                 for (int fieldIdx = 0; fieldIdx < numFields; fieldIdx++) {
-                    fvBuf.append(((NYCTLCHelper) helper).getParsedHeader()[fieldIdx] + "=" + values[fieldIdx]);
-                    if ((fieldIdx + 1) < numFields)
-                        fvBuf.append(((NYCTLCHelper) helper).getSeparator());
+                    sb.append(((NYCTLCHelper) helper).getParsedHeader()[fieldIdx] + "=" + values[fieldIdx]);
+                    if ((fieldIdx + 1) < numFields) {
+                        sb.append(((NYCTLCHelper) helper).getSeparator());
+                    }
                 }
-                this.value = new Text(fvBuf.toString());
-            } else
+                this.value = new Text(sb.toString());
+                sb.setLength(0);
+            } else {
                 completeRecord = false;
+            }
         } while (hasNext && !completeRecord);
 
         return hasNext;

diff --git a/warehouse/ingest-scripts/pom.xml b/warehouse/ingest-scripts/pom.xml
@@ -4,7 +4,7 @@
     <parent>
         <groupId>gov.nsa.datawave</groupId>
         <artifactId>datawave-warehouse-parent</artifactId>
-        <version>5.6.0-SNAPSHOT</version>
+        <version>5.7.0-SNAPSHOT</version>
     </parent>
     <artifactId>datawave-ingest-scripts</artifactId>
     <name>${project.artifactId}</name>

diff --git a/warehouse/ingest-wikipedia/pom.xml b/warehouse/ingest-wikipedia/pom.xml
@@ -4,7 +4,7 @@
     <parent>
         <groupId>gov.nsa.datawave</groupId>
         <artifactId>datawave-warehouse-parent</artifactId>
-        <version>5.6.0-SNAPSHOT</version>
+        <version>5.7.0-SNAPSHOT</version>
     </parent>
     <artifactId>datawave-ingest-wikipedia</artifactId>
     <packaging>jar</packaging>

diff --git a/warehouse/metrics-core/pom.xml b/warehouse/metrics-core/pom.xml
@@ -4,7 +4,7 @@
     <parent>
         <groupId>gov.nsa.datawave</groupId>
         <artifactId>datawave-warehouse-parent</artifactId>
-        <version>5.6.0-SNAPSHOT</version>
+        <version>5.7.0-SNAPSHOT</version>
     </parent>
     <artifactId>datawave-metrics-core</artifactId>
     <packaging>jar</packaging>

diff --git a/warehouse/ops-tools/config-compare/pom.xml b/warehouse/ops-tools/config-compare/pom.xml
@@ -4,7 +4,7 @@
     <parent>
         <groupId>gov.nsa.datawave</groupId>
         <artifactId>datawave-ops-tools-parent</artifactId>
-        <version>5.6.0-SNAPSHOT</version>
+        <version>5.7.0-SNAPSHOT</version>
     </parent>
     <artifactId>datawave-ops-tools-config-compare</artifactId>
     <dependencies>

diff --git a/warehouse/ops-tools/index-validation/pom.xml b/warehouse/ops-tools/index-validation/pom.xml
@@ -4,7 +4,7 @@
     <parent>
         <groupId>gov.nsa.datawave</groupId>
         <artifactId>datawave-ops-tools-parent</artifactId>
-        <version>5.6.0-SNAPSHOT</version>
+        <version>5.7.0-SNAPSHOT</version>
     </parent>
     <artifactId>datawave-ops-tools-index-validation</artifactId>
     <packaging>jar</packaging>

diff --git a/warehouse/ops-tools/pom.xml b/warehouse/ops-tools/pom.xml
@@ -4,7 +4,7 @@
     <parent>
         <groupId>gov.nsa.datawave</groupId>
         <artifactId>datawave-warehouse-parent</artifactId>
-        <version>5.6.0-SNAPSHOT</version>
+        <version>5.7.0-SNAPSHOT</version>
     </parent>
     <artifactId>datawave-ops-tools-parent</artifactId>
     <packaging>pom</packaging>

diff --git a/warehouse/pom.xml b/warehouse/pom.xml
@@ -4,7 +4,7 @@
     <parent>
         <groupId>gov.nsa.datawave</groupId>
         <artifactId>datawave-parent</artifactId>
-        <version>5.6.0-SNAPSHOT</version>
+        <version>5.7.0-SNAPSHOT</version>
     </parent>
     <artifactId>datawave-warehouse-parent</artifactId>
     <packaging>pom</packaging>

diff --git a/warehouse/query-core/pom.xml b/warehouse/query-core/pom.xml
@@ -4,7 +4,7 @@
     <parent>
         <groupId>gov.nsa.datawave</groupId>
         <artifactId>datawave-warehouse-parent</artifactId>
-        <version>5.6.0-SNAPSHOT</version>
+        <version>5.7.0-SNAPSHOT</version>
     </parent>
     <artifactId>datawave-query-core</artifactId>
     <packaging>jar</packaging>

diff --git a/warehouse/query-core/src/main/java/datawave/query/QueryParameters.java b/warehouse/query-core/src/main/java/datawave/query/QueryParameters.java
@@ -137,6 +137,10 @@ public class QueryParameters {
      * Used to limit the number of values returned for specific fields
      */
     public static final String LIMIT_FIELDS = "limit.fields";
+    /**
+     * Used to tie field groups together such that if a field in one group is not being limited the fields in matching groups will not be limited.
+     */
+    public static final String MATCHING_FIELD_SETS = "matching.field.sets";
 
     public static final String GROUP_FIELDS = "group.fields";
     public static final String GROUP_FIELDS_BATCH_SIZE = "group.fields.batch.size";