diff --git a/.github/workflows/settings.xml b/.github/workflows/settings.xml new file mode 100644 index 00000000..d8be2eb4 --- /dev/null +++ b/.github/workflows/settings.xml @@ -0,0 +1,23 @@ + + + + + + + + + + + + + + + + github-datawave + ${env.USER_NAME} + ${env.ACCESS_TOKEN} + + + diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 234556ee..543fd726 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -31,11 +31,13 @@ jobs: ${{ runner.os }}-maven- - name: Format code run: | - mvn -V -B -e clean formatter:format sortpom:sort -Pautoformat + mvn -s $GITHUB_WORKSPACE/.github/workflows/settings.xml -V -B -e clean formatter:format sortpom:sort -Pautoformat git status git diff-index --quiet HEAD || (echo "Error! There are modified files after formatting." && false) env: MAVEN_OPTS: "-Dhttps.protocols=TLSv1.2 -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=WARN -Djava.awt.headless=true" + USER_NAME: ${{ secrets.USER_NAME }} + ACCESS_TOKEN: ${{ secrets.ACCESS_TOKEN }} # Build the code and run the unit/integration tests. build-and-test: @@ -56,9 +58,11 @@ jobs: ${{ runner.os }}-maven-format- ${{ runner.os }}-maven- - name: Build and Run Unit Tests - run: mvn -V -B -e -Ddist clean verify + run: mvn -s $GITHUB_WORKSPACE/.github/workflows/settings.xml -V -B -e -Ddist clean verify env: MAVEN_OPTS: "-Dhttps.protocols=TLSv1.2 -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=WARN -Djava.awt.headless=true" + USER_NAME: ${{ secrets.USER_NAME }} + ACCESS_TOKEN: ${{ secrets.ACCESS_TOKEN }} # Here's an example of how you'd deploy the image to the github package registry. # We don't want to do this by default since packages on github cannot be deleted @@ -68,11 +72,13 @@ jobs: # env: # IMAGE_REGISTRY: "docker.pkg.github.com" # IMAGE_USERNAME: "NationalSecurityAgency" + # USER_NAME: ${{ secrets.USER_NAME }} + # ACCESS_TOKEN: ${{ secrets.ACCESS_TOKEN }} # run: | # # Set up env vars - # IMAGE_NAME=$(mvn -q -N -Pdocker -f service/pom.xml -Dexec.executable='echo' -Dexec.args='${project.version}' exec:exec) - # IMAGE_PREFIX=$(mvn -q -N -Pdocker -f service/pom.xml -Dexec.executable='echo' -Dexec.args='${docker.image.prefix}' exec:exec) - # IMAGE_TAG=$(mvn -q -N -Pdocker -f service/pom.xml -Dexec.executable='echo' -Dexec.args='${project.artifactId}' exec:exec) + # IMAGE_NAME=$(mvn -s $GITHUB_WORKSPACE/.github/workflows/settings.xml -q -N -Pdocker -f service/pom.xml -Dexec.executable='echo' -Dexec.args='${project.version}' exec:exec) + # IMAGE_PREFIX=$(mvn -s $GITHUB_WORKSPACE/.github/workflows/settings.xml -q -N -Pdocker -f service/pom.xml -Dexec.executable='echo' -Dexec.args='${docker.image.prefix}' exec:exec) + # IMAGE_TAG=$(mvn -s $GITHUB_WORKSPACE/.github/workflows/settings.xml -q -N -Pdocker -f service/pom.xml -Dexec.executable='echo' -Dexec.args='${project.artifactId}' exec:exec) # REMOTE_IMAGE_NAME="${IMAGE_REGISTRY}/${IMAGE_USERNAME}/${IMAGE_PREFIX}${IMAGE_NAME}" # # Log in to the package registry # echo ${{ secrets.GITHUB_TOKEN }} | docker login docker.pkg.github.com --username ${GITHUB_ACTOR} --password-stdin diff --git a/README.md b/README.md new file mode 100644 index 00000000..c6b87b28 --- /dev/null +++ b/README.md @@ -0,0 +1,66 @@ +# Query Service + +[![Apache License][li]][ll] ![Build Status](https://github.com/NationalSecurityAgency/datawave/workflows/Tests/badge.svg) + +The query service is a user-facing DATAWAVE microservice that serves as the main REST interface for DataWave query functionality. + +### Query Context + +*https://host:port/query/v1/* + +### User API + +| Done? | New? |Admin? | Method | Operation | Description | Path Param | Request Body | Response Body | +|:--------|:--------|:--------|:--------------|:-----------------------------------------|:-----------------------------------------------------------------------------------------------------------|:------------------------|:-------------------------------|:-------------------------------------------| +| ✓ | | | `GET` | /listQueryLogic | List QueryLogic types that are currently available | N/A | N/A | [QueryLogicResponse] | +| ✓ | | | `POST` | /{queryLogic}/define | Define a query using the specified query logic and params | [QueryLogicName] | [QueryParameters] | [GenericResponse] | +| ✓ | | | `POST` | /{queryLogic}/create | Create a query using the specified query logic and params | [QueryLogicName] | [QueryParameters] | [GenericResponse] | +| ✓ | | | `POST` | /{queryLogic}/plan | Generate a query plan using the specified query logic and params | [QueryLogicName] | [QueryParameters] | [GenericResponse] | +| ✓ | | | `POST` | /{queryLogic}/predict | Generate a query prediction using the specified query logic and params | [QueryLogicName] | [QueryParameters] | [GenericResponse] | +| ✓ | | | `POST` | /{queryLogic}/async/create | Create a query using the specified query logic and params | [QueryLogicName] | [QueryParameters] | [GenericResponse] | +| ✓ | | | `PUT` `POST` | /{id}/reset | Resets the specified query | [QueryId] | N/A | [VoidResponse]
[GenericResponse] | +| ✓ | | | `POST` | /{queryLogic}/createAndNext | Create a query using the specified query logic and params, and get the first page | [QueryLogicName] | [QueryParameters] | [BaseQueryResponse] | +| ✓ | | | `POST` | /{queryLogic}/async/createAndNext | Create a query using the specified query logic and params, and get the first page | [QueryLogicName] | [QueryParameters] | [BaseQueryResponse] | +| | | | `GET` | /lookupContentUUID/{uuidType}/{uuid} | Returns content associated with the given UUID | [UUIDType], [UUID] | N/A | [BaseQueryResponse] or [StreamingOutput] | +| | | | `POST` | /lookupContentUUID | Returns content associated with the given batch of UUIDs | N/A | [QueryParameters] | [BaseQueryResponse] or [StreamingOutput] | +| | | | `GET` | /lookupUUID/{uuidType}/{uuid} | Returns event associated with the given batch of UUID | [UUIDType], [UUID] | N/A | [BaseQueryResponse] or [StreamingOutput] | +| | | | `POST` | /lookupUUID | Returns event(s) associated with the given batch of UUIDs | N/A | [QueryParameters] | [BaseQueryResponse] or [StreamingOutput] | +| ✓ | | | `GET` | /{id}/plan | Returns the plan for the specified query | [QueryId] | N/A | [GenericResponse] | +| ✓ | | | `GET` | /{id}/predictions | Returns the predictions for the specified query | [QueryId] | N/A | [GenericResponse] | +| ✓ | | | `GET` | /{id}/async/next | Returns the next page of results for the specified query | [QueryId] | N/A | [BaseQueryResponse] | +| ✓ | | | `GET` | /{id}/next | Returns the next page of results for the specified query | [QueryId] | N/A | [BaseQueryResponse] | +| ✓ | | | `PUT` `POST` | /{id}/close | Closes the specified query | [QueryId] | N/A | [VoidResponse] | +| ✓ | | ✓ | `PUT` `POST` | /{id}/adminClose | Closes the specified query | [QueryId] | N/A | [VoidResponse] | +| ✓ | ✓ | ✓ | `PUT` `POST` | /adminCloseAll | Closes all running queries | N/A | N/A | [VoidResponse] | +| ✓ | | | `PUT` `POST` | /{id}/cancel | Cancels the specified query | [QueryId] | N/A | [VoidResponse] | +| ✓ | | ✓ | `PUT` `POST` | /{id}/adminCancel | Cancels the specified query | [QueryId] | N/A | [VoidResponse] | +| ✓ | ✓ | ✓ | `PUT` `POST` | /adminCancelAll | Cancels all running queries | N/A | N/A | [VoidResponse] | +| ✓ | | | `GET` | /listAll | Returns a list of queries associated with the current user | N/A | N/A | [QueryImplListResponse] | +| ✓ | | | `GET` | /{id} | Returns query info for the specified query | [QueryId] | N/A | [QueryImplListResponse] | +| ✓ | | | `GET` | /list | Returns a list of queries for this caller, filtering by the (optional) query id, and (optional) query name | N/A | [QueryId], [QueryName] | [QueryImplListResponse] | +| ✓ | ✓ | ✓ | `GET` | /adminList | Returns a list of queries, filtered by the (optional) user, (optional) query id, and (optional) query name | N/A | [User], [QueryId], [QueryName] | [QueryImplListResponse] | +| ✓ | | | `DELETE` | /{id}/remove | Remove (delete) the specified query | [QueryId] | N/A | [VoidResponse] | +| ✓ | ✓ | ✓ | `DELETE` | /{id}/adminRemove | Remove (delete) the specified query | [QueryId] | N/A | [VoidResponse] | +| ✓ | ✓ | ✓ | `DELETE` | /{id}/adminRemoveAll | Removes all queries which aren't running | N/A | N/A | [VoidResponse] | +| ✓ | | | `POST` | /{id}/duplicate | Duplicates the specified query | [QueryId] | [QueryParameters] | [GenericResponse] | +| ✓ | | | `PUT` `POST` | /{id}/update | Updates the specified query | [QueryId] | [QueryParameters] | [GenericResponse] | +| ✓ | | | `GET` | /{id}/listAll | Returns a list of queries associated with the specified user | [UserId] | N/A | [QueryImplListResponse] | +| ✓ | | | `POST` | /purgeQueryCache | Purges the cache of query objects | N/A | N/A | [VoidResponse] | +| ✓ | | | `GET` | /enableTracing | Enables tracing for queries which match the given criteria | N/A | [QueryRegex], [User] | [VoidResponse] | +| ✓ | | | `GET` | /disableTracing | Disables tracing for queries which match the given criteria | N/A | [QueryRegex], [User] | [VoidResponse] | +| ✓ | | | `GET` | /disableAllTracing | Disables tracing for all queries | N/A | N/A | [VoidResponse] | +| | | | `POST` | /{logicName}/execute | Create a query using the specified query logic and params, and stream the results | [QueryLogicName] | [QueryParameters] | [StreamingOutput] | +| | | | `POST` | /{logicName}/async/execute | Create a query using the specified query logic and params, and stream the results | [QueryLogicName] | [QueryParameters] | [StreamingOutput] | + +--- + +### Getting Started + +TBD + +For now, refer to the [Datawave Docker Compose Readme][getting-started] + +[getting-started]:https://github.com/NationalSecurityAgency/datawave/blob/feature/queryMicroservices/docker/README.md#datawave-docker-compose + +[li]: http://img.shields.io/badge/license-ASL-blue.svg +[ll]: https://www.apache.org/licenses/LICENSE-2.0 diff --git a/api/pom.xml b/api/pom.xml new file mode 100644 index 00000000..0d1c496c --- /dev/null +++ b/api/pom.xml @@ -0,0 +1,171 @@ + + + 4.0.0 + + gov.nsa.datawave.microservice + datawave-microservice-parent + 4.0.0-SNAPSHOT + ../../../microservice-parent/pom.xml + + query-api + 1.0.0-SNAPSHOT + https://github.com/NationalSecurityAgency/datawave-query-service + + scm:git:https://github.com/NationalSecurityAgency/datawave-query-service.git + scm:git:git@github.com:NationalSecurityAgency/datawave-query-service.git + HEAD + https://github.com/NationalSecurityAgency/datawave-query-service + + + http://webservice.datawave.nsa/v1 + 4.0.0-SNAPSHOT + 3.0.0-SNAPSHOT + 2.0.2 + + + + + gov.nsa.datawave.microservice + base-rest-responses + ${version.microservice.base-rest-responses} + + + log4j + log4j + + + slf4j-reload4j + org.slf4j + + + + + gov.nsa.datawave.microservice + type-utils + ${version.microservice.type-utils} + + + slf4j-reload4j + org.slf4j + + + log4j + log4j + + + + + jakarta.validation + jakarta.validation-api + ${version.validation-api} + + + + + + gov.nsa.datawave.microservice + base-rest-responses + + + gov.nsa.datawave.microservice + type-utils + + + jakarta.validation + jakarta.validation-api + + + org.junit.jupiter + junit-jupiter-engine + test + + + + + + true + + + false + + github-datawave + https://maven.pkg.github.com/NationalSecurityAgency/datawave + + + + + + true + src/main/resources + + source-templates/** + + + + + + org.apache.maven.plugins + maven-jar-plugin + 2.4 + + + jboss + + jar + + + jboss + + + + + + + META-INF/beans.xml + META-INF/jboss-ejb3.xml + + + + + maven-resources-plugin + + + copy-templated-sources + validate + + copy-resources + + + ${project.build.directory}/generated-sources/templated-sources + + + src/main/resources/source-templates + true + + + + + + + + org.codehaus.mojo + build-helper-maven-plugin + 3.3.0 + + + add-source + generate-sources + + add-source + + + + target/generated-sources/templated-sources + + + + + + + + diff --git a/api/src/main/java/datawave/microservice/query/DefaultQueryParameters.java b/api/src/main/java/datawave/microservice/query/DefaultQueryParameters.java new file mode 100644 index 00000000..6001b714 --- /dev/null +++ b/api/src/main/java/datawave/microservice/query/DefaultQueryParameters.java @@ -0,0 +1,631 @@ +package datawave.microservice.query; + +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Arrays; +import java.util.Calendar; +import java.util.Date; +import java.util.List; +import java.util.Map; + +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.time.DateUtils; +import org.springframework.util.LinkedMultiValueMap; +import org.springframework.util.MultiValueMap; + +import com.google.common.base.Preconditions; +import com.google.common.base.Splitter; + +public class DefaultQueryParameters implements QueryParameters { + + private static final List KNOWN_PARAMS = Arrays.asList(QUERY_STRING, QUERY_NAME, QUERY_PERSISTENCE, QUERY_PAGESIZE, QUERY_PAGETIMEOUT, + QUERY_AUTHORIZATIONS, QUERY_EXPIRATION, QUERY_TRACE, QUERY_BEGIN, QUERY_END, QUERY_VISIBILITY, QUERY_LOGIC_NAME, QUERY_POOL, + QUERY_MAX_RESULTS_OVERRIDE, QUERY_MAX_CONCURRENT_TASKS, QUERY_SYSTEM_FROM); + + protected String query; + protected String queryName; + protected QueryPersistence persistenceMode; + protected int pagesize; + protected int pageTimeout; + protected boolean isMaxResultsOverridden; + protected long maxResultsOverride; + protected String auths; + protected Date expirationDate; + protected boolean trace; + protected Date beginDate; + protected Date endDate; + protected String visibility; + protected String logicName; + protected String systemFrom; + protected String pool; + protected boolean isMaxConcurrentTasksOverridden; + protected int maxConcurrentTasks; + protected boolean expandFields; + protected boolean expandValues; + protected Map> requestHeaders; + + public DefaultQueryParameters() { + clear(); + } + + /** + * Configure internal variables via the incoming parameter map, performing validation of values. + * + * QueryParameters are considered valid if the following required parameters are present. + *
    + *
  1. 'query'
    2. + *
  2. 'queryName'
    3. + *
  3. 'persistence'
    4. + *
  4. 'auths'
    5. + *
  5. 'expiration'
    6. + *
  6. 'queryLogicName'
    7. + *
+ * + * QueryParameters may also include the following optional parameters. + *
    + *
  1. 'pagesize'
    2. + *
  2. 'pageTimeout'
    3. + *
  3. 'begin'
    4. + *
  4. 'end'
    5. + *
+ * + * @param parameters + * - a Map of QueryParameters + * @throws IllegalArgumentException + * when a bad argument is encountered + */ + public void validate(Map> parameters) throws IllegalArgumentException { + for (String param : KNOWN_PARAMS) { + List values = parameters.get(param); + if (null == values) { + continue; + } + if (values.isEmpty() || values.size() > 1) { + throw new IllegalArgumentException("Known parameter [" + param + "] only accepts one value"); + } + if (QUERY_STRING.equals(param)) { + this.query = values.get(0); + } else if (QUERY_NAME.equals(param)) { + this.queryName = values.get(0); + } else if (QUERY_PERSISTENCE.equals(param)) { + this.persistenceMode = QueryPersistence.valueOf(values.get(0)); + } else if (QUERY_PAGESIZE.equals(param)) { + this.pagesize = Integer.parseInt(values.get(0)); + } else if (QUERY_PAGETIMEOUT.equals(param)) { + this.pageTimeout = Integer.parseInt(values.get(0)); + } else if (QUERY_MAX_RESULTS_OVERRIDE.equals(param)) { + this.maxResultsOverride = Long.parseLong(values.get(0)); + this.isMaxResultsOverridden = true; + } else if (QUERY_MAX_CONCURRENT_TASKS.equals(param)) { + this.maxConcurrentTasks = Integer.parseInt(values.get(0)); + this.isMaxConcurrentTasksOverridden = true; + } else if (QUERY_AUTHORIZATIONS.equals(param)) { + // ensure that auths are comma separated with no empty values or spaces + Splitter splitter = Splitter.on(',').omitEmptyStrings().trimResults(); + this.auths = StringUtils.join(splitter.splitToList(values.get(0)), ","); + } else if (QUERY_EXPIRATION.equals(param)) { + try { + this.expirationDate = parseEndDate(values.get(0)); + } catch (ParseException e) { + throw new IllegalArgumentException("Error parsing expiration date", e); + } + } else if (QUERY_TRACE.equals(param)) { + this.trace = Boolean.parseBoolean(values.get(0)); + } else if (QUERY_BEGIN.equals(param)) { + try { + this.beginDate = values.get(0) == null ? null : parseStartDate(values.get(0)); + } catch (ParseException e) { + throw new IllegalArgumentException("Error parsing begin date", e); + } + } else if (QUERY_END.equals(param)) { + try { + this.endDate = values.get(0) == null ? null : parseEndDate(values.get(0)); + } catch (ParseException e) { + throw new IllegalArgumentException("Error parsing end date", e); + } + } else if (QUERY_VISIBILITY.equals(param)) { + this.visibility = values.get(0); + } else if (QUERY_LOGIC_NAME.equals(param)) { + this.logicName = values.get(0); + } else if (QUERY_POOL.equals(param)) { + this.pool = values.get(0); + } else if (QUERY_PLAN_EXPAND_FIELDS.equals(param)) { + this.expandFields = Boolean.parseBoolean(values.get(0)); + } else if (QUERY_PLAN_EXPAND_VALUES.equals(param)) { + this.expandValues = Boolean.parseBoolean(values.get(0)); + } else if (QUERY_SYSTEM_FROM.equals(param)) { + this.systemFrom = values.get(0); + } else { + throw new IllegalArgumentException("Unknown condition."); + } + } + + try { + Preconditions.checkNotNull(this.query, "QueryParameter 'query' cannot be null"); + Preconditions.checkNotNull(this.queryName, "QueryParameter 'queryName' cannot be null"); + Preconditions.checkNotNull(this.persistenceMode, "QueryParameter 'persistence' mode cannot be null"); + Preconditions.checkNotNull(this.auths, "QueryParameter 'auths' cannot be null"); + Preconditions.checkNotNull(this.expirationDate, "QueryParameter 'expirationDate' cannot be null"); + Preconditions.checkNotNull(this.logicName, "QueryParameter 'logicName' cannot be null"); + } catch (NullPointerException e) { + throw new IllegalArgumentException("Missing one or more required QueryParameters", e); + } + } + + @Override + public boolean equals(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + + DefaultQueryParameters that = (DefaultQueryParameters) o; + + if (pagesize != that.pagesize) + return false; + if (pageTimeout != that.pageTimeout) + return false; + if (isMaxResultsOverridden != that.isMaxResultsOverridden) + return false; + if (isMaxResultsOverridden) { + if (maxResultsOverride != that.maxResultsOverride) + return false; + } + if (isMaxConcurrentTasksOverridden != that.isMaxConcurrentTasksOverridden) + return false; + if (maxConcurrentTasks != that.maxConcurrentTasks) + return false; + if (expandFields != that.expandFields) + return false; + if (expandValues != that.expandValues) + return false; + if (trace != that.trace) + return false; + if (!auths.equals(that.auths)) + return false; + if (beginDate != null ? !beginDate.equals(that.beginDate) : that.beginDate != null) + return false; + if (visibility != null ? !visibility.equals(that.visibility) : that.visibility != null) + return false; + if (endDate != null ? !endDate.equals(that.endDate) : that.endDate != null) + return false; + if (!expirationDate.equals(that.expirationDate)) + return false; + if (logicName != null ? !logicName.equals(that.logicName) : that.logicName != null) + return false; + if (pool != null ? !pool.equals(that.pool) : that.pool != null) + return false; + if (persistenceMode != that.persistenceMode) + return false; + if (!query.equals(that.query)) + return false; + if (!queryName.equals(that.queryName)) + return false; + if (requestHeaders != null ? !requestHeaders.equals(that.requestHeaders) : that.requestHeaders != null) + return false; + if (systemFrom != null ? !systemFrom.equals(that.systemFrom) : that.systemFrom != null) + return false; + return true; + } + + @Override + public int hashCode() { + int result = query.hashCode(); + result = 31 * result + queryName.hashCode(); + result = 31 * result + persistenceMode.hashCode(); + result = 31 * result + pagesize; + result = 31 * result + pageTimeout; + if (isMaxResultsOverridden) { + result = 31 * result + (int) (maxResultsOverride); + } + if (isMaxConcurrentTasksOverridden) { + result = 31 * result + maxConcurrentTasks; + } + result = 31 * result + Boolean.hashCode(expandFields); + result = 31 * result + Boolean.hashCode(expandValues); + result = 31 * result + auths.hashCode(); + result = 31 * result + expirationDate.hashCode(); + result = 31 * result + (trace ? 1 : 0); + result = 31 * result + (beginDate != null ? beginDate.hashCode() : 0); + result = 31 * result + (endDate != null ? endDate.hashCode() : 0); + result = 31 * result + (visibility != null ? visibility.hashCode() : 0); + result = 31 * result + (logicName != null ? logicName.hashCode() : 0); + result = 31 * result + (pool != null ? pool.hashCode() : 0); + result = 31 * result + (requestHeaders != null ? requestHeaders.hashCode() : 0); + result = 31 * result + (systemFrom != null ? systemFrom.hashCode() : 0); + return result; + } + + public static synchronized String formatDate(Date d) throws ParseException { + String formatPattern = "yyyyMMdd HHmmss.SSS"; + SimpleDateFormat formatter = new SimpleDateFormat(formatPattern); + formatter.setLenient(false); + return formatter.format(d); + } + + protected static final String defaultStartTime = "000000"; + protected static final String defaultStartMillisec = "000"; + protected static final String defaultEndTime = "235959"; + protected static final String defaultEndMillisec = "999"; + protected static final String formatPattern = "yyyyMMdd HHmmss.SSS"; + private static final SimpleDateFormat dateFormat; + + static { + dateFormat = new SimpleDateFormat(formatPattern); + dateFormat.setLenient(false); + } + + public static Date parseStartDate(String s) throws ParseException { + return parseDate(s, defaultStartTime, defaultStartMillisec); + } + + public static Date parseEndDate(String s) throws ParseException { + return parseDate(s, defaultEndTime, defaultEndMillisec); + } + + public static synchronized Date parseDate(String s, String defaultTime, String defaultMillisec) throws ParseException { + Date d; + ParseException e = null; + synchronized (DefaultQueryParameters.dateFormat) { + String str = s; + if (str.equals("+24Hours")) { + d = DateUtils.addDays(new Date(), 1); + } else { + if (StringUtils.isNotBlank(defaultTime) && !str.contains(" ")) { + str = str + " " + defaultTime; + } + + if (StringUtils.isNotBlank(defaultMillisec) && !str.contains(".")) { + str = str + "." + defaultMillisec; + } + + try { + d = DefaultQueryParameters.dateFormat.parse(str); + // if any time value in HHmmss was set either by default or by the user + // then we want to include ALL of that second by setting the milliseconds to 999 + if (DateUtils.getFragmentInMilliseconds(d, Calendar.HOUR_OF_DAY) > 0) { + DateUtils.setMilliseconds(d, 999); + } + } catch (ParseException pe) { + throw new RuntimeException("Unable to parse date " + str + " with format " + formatPattern, e); + } + } + } + return d; + } + + /** + * Convenience method to generate a {@code Map>} from the specified arguments. If an argument is null, it's associated parameter name + * (key) will not be added to the map, which is why Integer and Boolean wrappers are used for greater flexibility. + * + * The 'parameters' argument will not be parsed, so its internal elements will not be placed into the map. If non-null, the 'parameters' value will be + * mapped directly to the QUERY_PARAMS key. + * + * No attempt is made to determine whether or not the given arguments constitute a valid query. If validation is desired, see the {@link #validate(Map)} + * method + * + * @param queryLogicName + * - name of QueryLogic to use + * @param query + * - the raw query string + * @param queryName + * - client-supplied name of query + * @param queryVisibility + * - query + * @param beginDate + * - start date + * @param endDate + * - end date + * @param queryAuthorizations + * - what auths the query should run with + * @param expirationDate + * - expiration date + * @param pagesize + * - page size + * @param pageTimeout + * - page timeout + * @param maxResultsOverride + * - max results override + * @param persistenceMode + * - persistence mode + * @param systemFrom + * - system from + * @param parameters + * - additional parameters passed in as map + * @param trace + * - trace flag + * @return parameter map + * @throws ParseException + * on date parse/format error + */ + public static Map> paramsToMap(String queryLogicName, String query, String queryName, String queryVisibility, Date beginDate, + Date endDate, String queryAuthorizations, Date expirationDate, Integer pagesize, Integer pageTimeout, Long maxResultsOverride, + QueryPersistence persistenceMode, String systemFrom, String parameters, Boolean trace) throws ParseException { + + MultiValueMap p = new LinkedMultiValueMap<>(); + if (queryLogicName != null) { + p.set(QueryParameters.QUERY_LOGIC_NAME, queryLogicName); + } + if (query != null) { + p.set(QueryParameters.QUERY_STRING, query); + } + if (queryName != null) { + p.set(QueryParameters.QUERY_NAME, queryName); + } + if (queryVisibility != null) { + p.set(QueryParameters.QUERY_VISIBILITY, queryVisibility); + } + if (beginDate != null) { + p.set(QueryParameters.QUERY_BEGIN, formatDate(beginDate)); + } + if (endDate != null) { + p.set(QueryParameters.QUERY_END, formatDate(endDate)); + } + if (queryAuthorizations != null) { + // ensure that auths are comma separated with no empty values or spaces + Splitter splitter = Splitter.on(',').omitEmptyStrings().trimResults(); + p.set(QueryParameters.QUERY_AUTHORIZATIONS, StringUtils.join(splitter.splitToList(queryAuthorizations), ",")); + } + if (expirationDate != null) { + p.set(QueryParameters.QUERY_EXPIRATION, formatDate(expirationDate)); + } + if (pagesize != null) { + p.set(QueryParameters.QUERY_PAGESIZE, pagesize.toString()); + } + if (pageTimeout != null) { + p.set(QueryParameters.QUERY_PAGETIMEOUT, pageTimeout.toString()); + } + if (maxResultsOverride != null) { + p.set(QueryParameters.QUERY_MAX_RESULTS_OVERRIDE, maxResultsOverride.toString()); + } + if (persistenceMode != null) { + p.set(QueryParameters.QUERY_PERSISTENCE, persistenceMode.name()); + } + if (trace != null) { + p.set(QueryParameters.QUERY_TRACE, trace.toString()); + } + if (systemFrom != null) { + p.set(QueryParameters.QUERY_SYSTEM_FROM, systemFrom); + } + if (parameters != null) { + p.set(QueryParameters.QUERY_PARAMS, parameters); + } + + return p; + } + + @Override + public String getQuery() { + return query; + } + + @Override + public void setQuery(String query) { + this.query = query; + } + + @Override + public String getQueryName() { + return queryName; + } + + @Override + public void setQueryName(String queryName) { + this.queryName = queryName; + } + + @Override + public QueryPersistence getPersistenceMode() { + return persistenceMode; + } + + @Override + public void setPersistenceMode(QueryPersistence persistenceMode) { + this.persistenceMode = persistenceMode; + } + + @Override + public int getPagesize() { + return pagesize; + } + + @Override + public void setPagesize(int pagesize) { + this.pagesize = pagesize; + } + + @Override + public int getPageTimeout() { + return pageTimeout; + } + + @Override + public void setPageTimeout(int pageTimeout) { + this.pageTimeout = pageTimeout; + } + + @Override + public long getMaxResultsOverride() { + return maxResultsOverride; + } + + @Override + public void setMaxResultsOverride(long maxResultsOverride) { + this.maxResultsOverride = maxResultsOverride; + } + + @Override + public boolean isMaxResultsOverridden() { + return this.isMaxResultsOverridden; + } + + @Override + public String getAuths() { + return auths; + } + + @Override + public void setAuths(String auths) { + this.auths = auths; + } + + @Override + public Date getExpirationDate() { + return expirationDate; + } + + @Override + public void setExpirationDate(Date expirationDate) { + this.expirationDate = expirationDate; + } + + @Override + public boolean isTrace() { + return trace; + } + + @Override + public void setTrace(boolean trace) { + this.trace = trace; + } + + @Override + public Date getBeginDate() { + return beginDate; + } + + @Override + public Date getEndDate() { + return endDate; + } + + @Override + public void setBeginDate(Date beginDate) { + this.beginDate = beginDate; + } + + @Override + public void setEndDate(Date endDate) { + this.endDate = endDate; + } + + @Override + public String getVisibility() { + return visibility; + } + + @Override + public void setVisibility(String visibility) { + this.visibility = visibility; + } + + @Override + public String getLogicName() { + return logicName; + } + + @Override + public void setLogicName(String logicName) { + this.logicName = logicName; + } + + @Override + public String getSystemFrom() { + return systemFrom; + } + + @Override + public void setSystemFrom(String systemFrom) { + this.systemFrom = systemFrom; + } + + @Override + public String getPool() { + return pool; + } + + @Override + public void setPool(String pool) { + this.pool = pool; + } + + @Override + public int getMaxConcurrentTasks() { + return maxConcurrentTasks; + } + + @Override + public void setMaxConcurrentTasks(int maxConcurrentTasks) { + this.maxConcurrentTasks = maxConcurrentTasks; + } + + @Override + public boolean isMaxConcurrentTasksOverridden() { + return isMaxConcurrentTasksOverridden; + } + + @Override + public boolean isExpandFields() { + return expandFields; + } + + @Override + public void setExpandFields(boolean expandFields) { + this.expandFields = expandFields; + } + + @Override + public boolean isExpandValues() { + return expandValues; + } + + @Override + public void setExpandValues(boolean expandValues) { + this.expandValues = expandValues; + } + + @Override + public Map> getRequestHeaders() { + return requestHeaders; + } + + @Override + public void setRequestHeaders(Map> requestHeaders) { + this.requestHeaders = requestHeaders; + } + + @Override + public MultiValueMap getUnknownParameters(Map> allQueryParameters) { + MultiValueMap p = new LinkedMultiValueMap<>(); + for (String key : allQueryParameters.keySet()) { + if (!KNOWN_PARAMS.contains(key)) { + p.put(key, allQueryParameters.get(key)); + } + } + return p; + } + + @Override + public void clear() { + this.query = null; + this.queryName = null; + this.persistenceMode = QueryPersistence.TRANSIENT; + this.pagesize = 10; + this.pageTimeout = -1; + this.isMaxResultsOverridden = false; + this.auths = null; + this.expirationDate = DateUtils.addDays(new Date(), 1); + this.trace = false; + this.beginDate = null; + this.endDate = null; + this.visibility = null; + this.logicName = null; + this.pool = null; + this.isMaxConcurrentTasksOverridden = false; + this.maxConcurrentTasks = 0; + this.expandFields = true; + this.expandValues = false; + this.requestHeaders = null; + this.systemFrom = null; + } +} diff --git a/api/src/main/java/datawave/microservice/query/Query.java b/api/src/main/java/datawave/microservice/query/Query.java new file mode 100644 index 00000000..e5eaca9a --- /dev/null +++ b/api/src/main/java/datawave/microservice/query/Query.java @@ -0,0 +1,167 @@ +package datawave.microservice.query; + +import java.io.Externalizable; +import java.io.IOException; +import java.io.ObjectInput; +import java.io.ObjectOutput; +import java.text.ParseException; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlSeeAlso; + +import datawave.microservice.query.QueryImpl.Parameter; +import datawave.webservice.query.util.QueryUncaughtExceptionHandler; + +@XmlAccessorType(XmlAccessType.NONE) +@XmlSeeAlso(QueryImpl.class) +public abstract class Query implements Externalizable { + + private static final long serialVersionUID = -5980134700364340930L; + + public abstract void initialize(String userDN, List dnList, String queryLogicName, QueryParameters qp, + Map> optionalQueryParameters); + + public abstract String getQueryLogicName(); + + public abstract UUID getId(); + + public abstract void setId(UUID id); + + public abstract String getQueryName(); + + public abstract void setQueryName(String queryName); + + public abstract String getUserDN(); + + public abstract void setUserDN(String userDN); + + public abstract String getQuery(); + + public abstract void setQuery(String query); + + public abstract String getQueryAuthorizations(); + + public abstract void setQueryAuthorizations(String authorizations); + + public abstract Date getExpirationDate(); + + public abstract void setExpirationDate(Date expirationDate); + + public abstract int getPagesize(); + + public abstract void setPagesize(int pagesize); + + public abstract int getPageTimeout(); + + public abstract void setPageTimeout(int pageTimeout); + + public abstract long getMaxResultsOverride(); + + public abstract void setMaxResultsOverride(long maxResults); + + public abstract boolean isMaxResultsOverridden(); + + public abstract void setMaxResultsOverridden(boolean maxResultsOverridden); + + public abstract Set getParameters(); + + public abstract void setParameters(Set params); + + public abstract void setQueryLogicName(String name); + + public abstract Date getBeginDate(); + + public abstract void setBeginDate(Date beginDate); + + public abstract Date getEndDate(); + + public abstract void setEndDate(Date endDate); + + public abstract String getSystemFrom(); + + public abstract void setSystemFrom(String systemFrom); + + public abstract Query duplicate(String newQueryName); + + public abstract Parameter findParameter(String parameter); + + public abstract void setParameters(Map parameters); + + public abstract void addParameter(String key, String val); + + public abstract void addParameters(Map parameters); + + public abstract void setDnList(List dnList); + + public abstract List getDnList(); + + public abstract QueryUncaughtExceptionHandler getUncaughtExceptionHandler(); + + public abstract void setUncaughtExceptionHandler(QueryUncaughtExceptionHandler uncaughtExceptionHandler); + + public abstract void setOwner(String owner); + + public abstract String getOwner(); + + public abstract void setColumnVisibility(String colviz); + + public abstract String getColumnVisibility(); + + public abstract Map> toMap(); + + public abstract void readMap(Map> map) throws ParseException; + + public abstract Map getCardinalityFields(); + + public abstract void setOptionalQueryParameters(Map> optionalQueryParameters); + + public abstract Map> getOptionalQueryParameters(); + + public abstract void removeParameter(String key); + + @Override + public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException { + Map> map = new HashMap<>(); + int numKeys = in.readInt(); + for (int i = 0; i < numKeys; i++) { + String key = in.readUTF(); + int numValues = in.readInt(); + List values = new ArrayList<>(numValues); + for (int j = 0; j < numValues; j++) { + String value = in.readUTF(); + values.add(value); + } + map.put(key, values); + } + try { + readMap(map); + } catch (ParseException pe) { + throw new IOException("Could not parse value", pe); + } + } + + @Override + public void writeExternal(ObjectOutput out) throws IOException { + Map> map = toMap(); + Set keys = map.keySet(); + out.writeInt(keys.size()); + for (String key : keys) { + out.writeUTF(key); + List values = map.get(key); + out.writeInt(values.size()); + for (String value : values) { + out.writeUTF(value); + } + } + } + + public abstract void populateTrackingMap(Map trackingMap); +} diff --git a/api/src/main/java/datawave/microservice/query/QueryImpl.java b/api/src/main/java/datawave/microservice/query/QueryImpl.java new file mode 100644 index 00000000..3640ce1f --- /dev/null +++ b/api/src/main/java/datawave/microservice/query/QueryImpl.java @@ -0,0 +1,1011 @@ +package datawave.microservice.query; + +import static datawave.microservice.query.QueryParameters.QUERY_SYSTEM_FROM; + +import java.io.IOException; +import java.io.Serializable; +import java.text.ParseException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Objects; +import java.util.Set; +import java.util.UUID; + +import javax.naming.InvalidNameException; +import javax.naming.ldap.LdapName; +import javax.naming.ldap.Rdn; +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; +import javax.xml.bind.annotation.XmlTransient; +import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter; + +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.collections4.MapUtils; +import org.apache.commons.lang3.builder.ToStringBuilder; +import org.springframework.util.LinkedMultiValueMap; +import org.springframework.util.MultiValueMap; + +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import com.fasterxml.jackson.databind.ser.std.ToStringSerializer; + +import datawave.webservice.query.util.OptionallyEncodedStringAdapter; +import datawave.webservice.query.util.QueryUncaughtExceptionHandler; +import io.protostuff.Input; +import io.protostuff.Message; +import io.protostuff.Output; +import io.protostuff.Schema; +import io.protostuff.UninitializedMessageException; + +@XmlRootElement(name = "QueryImpl") +@XmlAccessorType(XmlAccessType.NONE) +public class QueryImpl extends Query implements Serializable, Message { + + public static final String PARAMETER_SEPARATOR = ";"; + public static final String PARAMETER_NAME_VALUE_SEPARATOR = ":"; + + public static final String USER_DN = "userDN"; + public static final String DN_LIST = "dnList"; + public static final String COLUMN_VISIBILITY = "columnVisibility"; + public static final String QUERY_LOGIC_NAME = QueryParameters.QUERY_LOGIC_NAME; + public static final String QUERY_NAME = QueryParameters.QUERY_NAME; + public static final String EXPIRATION_DATE = QueryParameters.QUERY_EXPIRATION; + public static final String QUERY_ID = "uuid"; + public static final String PAGESIZE = QueryParameters.QUERY_PAGESIZE; + public static final String PAGE_TIMEOUT = QueryParameters.QUERY_PAGETIMEOUT; + public static final String MAX_RESULTS_OVERRIDE = QueryParameters.QUERY_MAX_RESULTS_OVERRIDE; + public static final String QUERY = QueryParameters.QUERY_STRING; + public static final String QUERY_AUTHORIZATIONS = QueryParameters.QUERY_AUTHORIZATIONS; + public static final String OWNER = "owner"; + public static final String PARAMETERS = "parameters"; + public static final String BEGIN_DATE = QueryParameters.QUERY_BEGIN; + public static final String END_DATE = QueryParameters.QUERY_END; + public static final String QUERY_USER_FIELD = "QUERY_USER"; + public static final String QUERY_LOGIC_NAME_FIELD = "QUERY_LOGIC_NAME"; + public static final String POOL = QueryParameters.QUERY_POOL; + + @XmlAccessorType(XmlAccessType.FIELD) + public static final class Parameter implements Serializable, Message { + + private static final long serialVersionUID = 2L; + + @XmlElement(name = "name") + private String parameterName; + @XmlElement(name = "value") + private String parameterValue; + + public Parameter() {} + + public Parameter(String name, String value) { + this.parameterName = name; + this.parameterValue = value; + } + + public String getParameterName() { + return parameterName; + } + + public void setParameterName(String parameterName) { + this.parameterName = parameterName; + } + + public String getParameterValue() { + return parameterValue; + } + + public void setParameterValue(String parameterValue) { + this.parameterValue = parameterValue; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(256); + sb.append("[name=").append(this.parameterName); + sb.append(",value=").append(this.parameterValue).append("]"); + return sb.toString(); + } + + @Override + public boolean equals(Object o) { + if (null == o) + return false; + if (!(o instanceof Parameter)) + return false; + if (this == o) + return true; + Parameter other = (Parameter) o; + if (this.getParameterName().equals(other.getParameterName()) && this.getParameterValue().equals(other.getParameterValue())) + return true; + else + return false; + } + + @Override + public int hashCode() { + return getParameterName() == null ? 0 : getParameterName().hashCode(); + } + + @XmlTransient + public static final Schema SCHEMA = new Schema() { + public Parameter newMessage() { + return new Parameter(); + } + + public Class typeClass() { + return Parameter.class; + } + + public String messageName() { + return Parameter.class.getSimpleName(); + } + + public String messageFullName() { + return Parameter.class.getName(); + } + + public boolean isInitialized(Parameter message) { + return message.parameterName != null && message.parameterValue != null; + } + + public void writeTo(Output output, Parameter message) throws IOException { + if (message.parameterName == null) + throw new UninitializedMessageException(message); + output.writeString(1, message.parameterName, false); + + if (message.parameterValue == null) + throw new UninitializedMessageException(message); + output.writeString(2, message.parameterValue, false); + } + + public void mergeFrom(Input input, Parameter message) throws IOException { + int number; + while ((number = input.readFieldNumber(this)) != 0) { + switch (number) { + case 1: + message.parameterName = input.readString(); + break; + case 2: + message.parameterValue = input.readString(); + break; + default: + input.handleUnknownField(number, this); + break; + } + } + } + + public String getFieldName(int number) { + switch (number) { + case 1: + return "parameterName"; + case 2: + return "parameterValue"; + default: + return null; + } + } + + public int getFieldNumber(String name) { + final Integer number = fieldMap.get(name); + return number == null ? 0 : number.intValue(); + } + + final java.util.HashMap fieldMap = new java.util.HashMap(); + + { + fieldMap.put("parameterName", 1); + fieldMap.put("parameterValue", 2); + } + }; + + public static Schema getSchema() { + return SCHEMA; + } + + @Override + public Schema cachedSchema() { + return SCHEMA; + } + + } + + private static final long serialVersionUID = 2L; + + @XmlElement + protected String queryLogicName; + @XmlElement + protected String id; + @XmlElement + protected String queryName; + @XmlElement + protected String userDN; + @XmlElement + @XmlJavaTypeAdapter(OptionallyEncodedStringAdapter.class) + @JsonSerialize(using = ToStringSerializer.class) + protected String query; + @XmlElement + protected Date beginDate; + @XmlElement + protected Date endDate; + @XmlElement + protected String queryAuthorizations; + @XmlElement + protected Date expirationDate; + @XmlElement + protected int pagesize; + @XmlElement + protected int pageTimeout; + @XmlElement + protected boolean maxResultsOverridden; + @XmlElement + protected long maxResultsOverride; + @XmlElement + protected HashSet parameters = new HashSet(); + @XmlElement + protected List dnList; + @XmlElement + protected String owner; + @XmlElement + protected String columnVisibility; + @XmlElement + protected String systemFrom; + @XmlElement + protected String pool; + @XmlTransient + protected Map> optionalQueryParameters; + + protected transient QueryUncaughtExceptionHandler uncaughtExceptionHandler; + + protected transient HashMap paramLookup = new HashMap(); + + public String getQueryLogicName() { + return queryLogicName; + } + + public UUID getId() { + if (null == id) + return null; + return java.util.UUID.fromString(id); + } + + public String getQueryName() { + return queryName; + } + + public String getUserDN() { + return userDN; + } + + public String getQuery() { + return query; + } + + public String getQueryAuthorizations() { + return queryAuthorizations; + } + + public Date getExpirationDate() { + return expirationDate; + } + + public int getPagesize() { + return pagesize; + } + + public int getPageTimeout() { + return pageTimeout; + } + + public String getPool() { + return pool; + } + + public long getMaxResultsOverride() { + return maxResultsOverride; + } + + public boolean isMaxResultsOverridden() { + return maxResultsOverridden; + } + + public void setMaxResultsOverridden(boolean maxResultsOverridden) { + this.maxResultsOverridden = maxResultsOverridden; + } + + public Set getParameters() { + return parameters == null ? null : Collections.unmodifiableSet(parameters); + } + + public void setQueryLogicName(String name) { + this.queryLogicName = name; + } + + public void setId(UUID id) { + this.id = id.toString(); + } + + public void setQueryName(String queryName) { + this.queryName = queryName; + } + + public void setUserDN(String userDN) { + this.userDN = userDN; + } + + public void setQuery(String query) { + this.query = query; + } + + public void setQueryAuthorizations(String queryAuthorizations) { + this.queryAuthorizations = queryAuthorizations; + } + + public void setExpirationDate(Date expirationDate) { + this.expirationDate = expirationDate; + } + + public void setMaxResultsOverride(long maxResults) { + this.maxResultsOverride = maxResults; + } + + public void setPagesize(int pagesize) { + this.pagesize = pagesize; + } + + public void setPageTimeout(int pageTimeout) { + this.pageTimeout = pageTimeout; + } + + public void setPool(String pool) { + this.pool = pool; + } + + public void setParameters(Set parameters) { + this.parameters.clear(); + this.parameters.addAll(parameters); + this.paramLookup.clear(); + for (Parameter p : this.parameters) { + this.paramLookup.put(p.getParameterName(), p); + } + } + + public void addParameter(String key, String val) { + Parameter p = new Parameter(key, val); + this.parameters.add(p); + this.paramLookup.put(p.getParameterName(), p); + } + + public void addParameters(Map parameters) { + for (Entry p : parameters.entrySet()) { + addParameter(p.getKey(), p.getValue()); + } + } + + public void setParameters(Map parameters) { + HashSet paramObjs = new HashSet(parameters.size()); + for (Entry param : parameters.entrySet()) { + Parameter p = new Parameter(param.getKey(), param.getValue()); + paramObjs.add(p); + } + this.setParameters(paramObjs); + } + + public List getDnList() { + return dnList; + } + + public void setDnList(List dnList) { + this.dnList = dnList; + } + + public String getColumnVisibility() { + return columnVisibility; + } + + public void setColumnVisibility(String columnVisibility) { + this.columnVisibility = columnVisibility; + } + + public Date getBeginDate() { + return beginDate; + } + + public void setBeginDate(Date beginDate) { + this.beginDate = beginDate; + } + + public Date getEndDate() { + return endDate; + } + + public void setEndDate(Date endDate) { + this.endDate = endDate; + } + + @Override + public String getSystemFrom() { + return systemFrom; + } + + @Override + public void setSystemFrom(String systemFrom) { + this.systemFrom = systemFrom; + } + + public Map> getOptionalQueryParameters() { + return optionalQueryParameters; + } + + public void setOptionalQueryParameters(Map> optionalQueryParameters) { + this.optionalQueryParameters = optionalQueryParameters; + } + + @Override + public QueryImpl duplicate(String newQueryName) { + QueryImpl query = new QueryImpl(); + query.setQueryLogicName(this.getQueryLogicName()); + query.setQueryName(newQueryName); + query.setExpirationDate(this.getExpirationDate()); + query.setId(java.util.UUID.randomUUID()); + query.setPagesize(this.getPagesize()); + query.setPageTimeout(this.getPageTimeout()); + query.setMaxResultsOverridden(this.isMaxResultsOverridden()); + query.setMaxResultsOverride(this.getMaxResultsOverride()); + query.setQuery(this.getQuery()); + query.setQueryAuthorizations(this.getQueryAuthorizations()); + query.setUserDN(this.getUserDN()); + query.setOwner(this.getOwner()); + query.setColumnVisibility(this.getColumnVisibility()); + query.setBeginDate(this.getBeginDate()); + query.setEndDate(this.getEndDate()); + query.setSystemFrom(this.getSystemFrom()); + query.setPool(this.getPool()); + if (CollectionUtils.isNotEmpty(this.parameters)) + query.setParameters(new HashSet(this.parameters)); + query.setDnList(this.dnList); + if (MapUtils.isNotEmpty(this.optionalQueryParameters)) { + Map> optionalDuplicate = new HashMap<>(); + this.optionalQueryParameters.entrySet().stream().forEach(e -> optionalDuplicate.put(e.getKey(), new ArrayList(e.getValue()))); + query.setOptionalQueryParameters(optionalDuplicate); + } + query.setUncaughtExceptionHandler(this.getUncaughtExceptionHandler()); + return query; + } + + @Override + public String toString() { + ToStringBuilder tsb = new ToStringBuilder(this); + tsb.append(QUERY_LOGIC_NAME, this.getQueryLogicName()); + tsb.append(QUERY_NAME, this.getQueryName()); + tsb.append(EXPIRATION_DATE, this.getExpirationDate()); + tsb.append(QUERY_ID, this.getId()); + tsb.append(PAGESIZE, this.getPagesize()); + tsb.append(PAGE_TIMEOUT, this.getPageTimeout()); + tsb.append(MAX_RESULTS_OVERRIDE, (this.isMaxResultsOverridden() ? this.getMaxResultsOverride() : "NA")); + tsb.append(QUERY, this.getQuery()); + tsb.append(QUERY_AUTHORIZATIONS, this.getQueryAuthorizations()); + tsb.append(USER_DN, this.getUserDN()); + tsb.append(OWNER, this.getOwner()); + tsb.append(PARAMETERS, this.getParameters()); + tsb.append(DN_LIST, this.getDnList()); + tsb.append(COLUMN_VISIBILITY, this.getColumnVisibility()); + tsb.append(BEGIN_DATE, this.getBeginDate()); + tsb.append(END_DATE, this.getEndDate()); + tsb.append(QUERY_SYSTEM_FROM, this.getSystemFrom()); + tsb.append(POOL, this.getPool()); + return tsb.toString(); + } + + @Override + public boolean equals(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + QueryImpl that = (QueryImpl) o; + return pagesize == that.pagesize && pageTimeout == that.pageTimeout && maxResultsOverridden == that.maxResultsOverridden + && maxResultsOverride == that.maxResultsOverride && Objects.equals(queryLogicName, that.queryLogicName) && Objects.equals(id, that.id) + && Objects.equals(queryName, that.queryName) && Objects.equals(userDN, that.userDN) && Objects.equals(query, that.query) + && Objects.equals(beginDate, that.beginDate) && Objects.equals(endDate, that.endDate) + && Objects.equals(queryAuthorizations, that.queryAuthorizations) && Objects.equals(expirationDate, that.expirationDate) + && Objects.equals(parameters, that.parameters) && Objects.equals(dnList, that.dnList) && Objects.equals(owner, that.owner) + && Objects.equals(columnVisibility, that.columnVisibility) && Objects.equals(optionalQueryParameters, that.optionalQueryParameters) + && Objects.equals(systemFrom, that.systemFrom) && Objects.equals(pool, that.pool); + } + + @Override + public int hashCode() { + return Objects.hash(queryLogicName, id, queryName, userDN, query, beginDate, endDate, queryAuthorizations, expirationDate, pagesize, pageTimeout, + maxResultsOverridden, maxResultsOverride, parameters, dnList, owner, columnVisibility, optionalQueryParameters, systemFrom, pool); + } + + public Parameter findParameter(String parameter) { + if (!paramLookup.containsKey(parameter)) { + return new Parameter(parameter, ""); + } else { + return paramLookup.get(parameter); + } + } + + @XmlTransient + private static final Schema SCHEMA = new Schema() { + public QueryImpl newMessage() { + return new QueryImpl(); + } + + public Class typeClass() { + return QueryImpl.class; + } + + public String messageName() { + return QueryImpl.class.getSimpleName(); + } + + public String messageFullName() { + return QueryImpl.class.getName(); + } + + public boolean isInitialized(QueryImpl message) { + return message.queryLogicName != null && message.id != null && message.userDN != null && message.query != null + && message.queryAuthorizations != null && message.expirationDate != null && message.pagesize > 0 && message.pageTimeout != 0; + } + + public void writeTo(Output output, QueryImpl message) throws IOException { + if (message.queryLogicName == null) + throw new UninitializedMessageException(message, SCHEMA); + output.writeString(1, message.queryLogicName, false); + + if (message.id == null) + throw new UninitializedMessageException(message, SCHEMA); + output.writeString(2, message.id, false); + + if (message.queryName != null) + output.writeString(3, message.queryName, false); + + if (message.userDN == null) + throw new UninitializedMessageException(message, SCHEMA); + output.writeString(4, message.userDN, false); + + if (message.query == null) + throw new UninitializedMessageException(message, SCHEMA); + output.writeString(5, message.query, false); + + if (message.beginDate != null) + output.writeInt64(6, message.beginDate.getTime(), false); + + if (message.endDate != null) + output.writeInt64(7, message.endDate.getTime(), false); + + if (message.queryAuthorizations == null) + throw new UninitializedMessageException(message, SCHEMA); + output.writeString(8, message.queryAuthorizations, false); + + if (message.expirationDate == null) + throw new UninitializedMessageException(message, SCHEMA); + output.writeInt64(9, message.expirationDate.getTime(), false); + + if (message.pagesize <= 0) + throw new UninitializedMessageException(message, SCHEMA); + output.writeUInt32(10, message.pagesize, false); + + if (message.parameters != null) { + for (Parameter p : message.parameters) { + output.writeObject(11, p, Parameter.SCHEMA, true); + } + } + + if (message.owner == null) + throw new UninitializedMessageException(message, SCHEMA); + output.writeString(12, message.owner, false); + + if (null != message.dnList) { + for (String dn : message.dnList) + output.writeString(13, dn, true); + } + + if (message.columnVisibility != null) { + output.writeString(14, message.columnVisibility, false); + } + + if (message.pageTimeout == 0) + throw new UninitializedMessageException(message, SCHEMA); + output.writeUInt32(15, message.pageTimeout, false); + + if (message.systemFrom != null) + output.writeString(16, message.systemFrom, false); + + if (message.pool != null) { + output.writeString(17, message.pool, false); + } + } + + public void mergeFrom(Input input, QueryImpl message) throws IOException { + int number; + while ((number = input.readFieldNumber(this)) != 0) { + switch (number) { + case 1: + message.queryLogicName = input.readString(); + break; + case 2: + message.id = input.readString(); + break; + case 3: + message.queryName = input.readString(); + break; + case 4: + message.userDN = input.readString(); + break; + case 5: + message.query = input.readString(); + break; + + case 6: + message.beginDate = new Date(input.readInt64()); + break; + case 7: + message.endDate = new Date(input.readInt64()); + break; + case 8: + message.queryAuthorizations = input.readString(); + break; + case 9: + message.expirationDate = new Date(input.readInt64()); + break; + case 10: + message.pagesize = input.readUInt32(); + break; + case 11: + if (message.parameters == null) + message.parameters = new HashSet(); + Parameter p = input.mergeObject(null, Parameter.SCHEMA); + message.addParameter(p.getParameterName(), p.getParameterValue()); + break; + case 12: + message.owner = input.readString(); + break; + case 13: + if (null == message.dnList) + message.dnList = new ArrayList(); + message.dnList.add(input.readString()); + break; + case 14: + message.columnVisibility = input.readString(); + break; + case 15: + message.pageTimeout = input.readUInt32(); + break; + case 16: + message.systemFrom = input.readString(); + case 17: + message.pool = input.readString(); + default: + input.handleUnknownField(number, this); + break; + } + } + } + + public String getFieldName(int number) { + switch (number) { + case 1: + return QueryParameters.QUERY_LOGIC_NAME; + case 2: + return QUERY_ID; + case 3: + return QueryParameters.QUERY_NAME; + case 4: + return USER_DN; + case 5: + return QUERY; + case 6: + return BEGIN_DATE; + case 7: + return END_DATE; + case 8: + return QUERY_AUTHORIZATIONS; + case 9: + return EXPIRATION_DATE; + case 10: + return PAGESIZE; + case 11: + return PARAMETERS; + case 12: + return OWNER; + case 13: + return DN_LIST; + case 14: + return COLUMN_VISIBILITY; + case 15: + return PAGE_TIMEOUT; + case 16: + return QUERY_SYSTEM_FROM; + case 17: + return POOL; + default: + return null; + } + } + + public int getFieldNumber(String name) { + final Integer number = fieldMap.get(name); + return number == null ? 0 : number.intValue(); + } + + final java.util.HashMap fieldMap = new java.util.HashMap(); + + { + fieldMap.put(QUERY_LOGIC_NAME, 1); + fieldMap.put(QUERY_ID, 2); + fieldMap.put(QUERY_NAME, 3); + fieldMap.put(USER_DN, 4); + fieldMap.put(QUERY, 5); + fieldMap.put(BEGIN_DATE, 6); + fieldMap.put(END_DATE, 7); + fieldMap.put(QUERY_AUTHORIZATIONS, 8); + fieldMap.put(EXPIRATION_DATE, 9); + fieldMap.put(PAGESIZE, 10); + fieldMap.put(PARAMETERS, 11); + fieldMap.put(OWNER, 12); + fieldMap.put(DN_LIST, 13); + fieldMap.put(COLUMN_VISIBILITY, 14); + fieldMap.put(PAGE_TIMEOUT, 15); + fieldMap.put(QUERY_SYSTEM_FROM, 16); + fieldMap.put(POOL, 17); + } + }; + + public QueryUncaughtExceptionHandler getUncaughtExceptionHandler() { + return this.uncaughtExceptionHandler; + } + + public void setUncaughtExceptionHandler(QueryUncaughtExceptionHandler uncaughtExceptionHandler) { + this.uncaughtExceptionHandler = uncaughtExceptionHandler; + } + + public void initialize(String userDN, List dnList, String queryLogicName, QueryParameters qp, Map> optionalQueryParameters) { + this.dnList = dnList; + this.expirationDate = qp.getExpirationDate(); + this.id = java.util.UUID.randomUUID().toString(); + this.pagesize = qp.getPagesize(); + this.pageTimeout = qp.getPageTimeout(); + this.query = qp.getQuery(); + this.queryAuthorizations = qp.getAuths(); + this.queryLogicName = queryLogicName; + this.queryName = qp.getQueryName(); + this.userDN = userDN; + this.owner = getOwner(this.userDN); + this.beginDate = qp.getBeginDate(); + this.endDate = qp.getEndDate(); + this.systemFrom = qp.getSystemFrom(); + this.pool = qp.getPool(); + if (optionalQueryParameters != null) { + for (Entry> entry : optionalQueryParameters.entrySet()) { + if (entry.getValue().get(0) != null) { + this.addParameter(entry.getKey(), entry.getValue().get(0)); + } + } + } + } + + private static String getCommonName(String dn) { + String[] comps = getComponents(dn, "CN"); + return comps.length >= 1 ? comps[0] : null; + } + + private static String[] getComponents(String dn, String componentName) { + componentName = componentName.toUpperCase(); + ArrayList components = new ArrayList(); + try { + LdapName name = new LdapName(dn); + for (Rdn rdn : name.getRdns()) { + if (componentName.equals(rdn.getType().toUpperCase())) { + components.add(String.valueOf(rdn.getValue())); + } + } + } catch (InvalidNameException e) { + // ignore -- invalid name, so can't find components + } + return components.toArray(new String[0]); + } + + public static String getOwner(String dn) { + String sid = null; + if (dn != null) { + String cn = getCommonName(dn); + if (cn == null) + cn = dn; + sid = cn; + int idx = cn.lastIndexOf(' '); + if (idx >= 0) + sid = cn.substring(idx + 1); + } + return sid; + } + + public void setOwner(String owner) { + this.owner = owner; + } + + public String getOwner() { + return this.owner; + } + + public Map> toMap() { + MultiValueMap p = new LinkedMultiValueMap<>(); + if (this.id != null) { + p.set(QUERY_ID, this.id); + } + if (this.queryAuthorizations != null) { + p.set(QueryParameters.QUERY_AUTHORIZATIONS, this.queryAuthorizations); + } + if (this.expirationDate != null) { + try { + p.set(QueryParameters.QUERY_EXPIRATION, DefaultQueryParameters.formatDate(this.expirationDate)); + } catch (ParseException e) { + throw new RuntimeException("Error formatting date", e); + } + } + if (this.queryName != null) { + p.set(QueryParameters.QUERY_NAME, this.queryName); + } + if (this.queryLogicName != null) { + p.set(QueryParameters.QUERY_LOGIC_NAME, this.queryLogicName); + } + // no null check on primitives + p.set(QueryParameters.QUERY_PAGESIZE, Integer.toString(this.pagesize)); + if (this.query != null) { + p.set(QueryParameters.QUERY_STRING, this.query); + } + if (this.userDN != null) { + p.set(USER_DN, this.userDN); + } + if (this.dnList != null) { + p.put(DN_LIST, this.dnList); + } + if (this.owner != null) { + p.set(OWNER, this.owner); + } + if (this.columnVisibility != null) { + p.set(COLUMN_VISIBILITY, this.columnVisibility); + } + if (this.beginDate != null) { + try { + p.set(QueryParameters.QUERY_BEGIN, DefaultQueryParameters.formatDate(this.beginDate)); + } catch (ParseException e) { + throw new RuntimeException("Error formatting date", e); + } + } + if (this.endDate != null) { + try { + p.set(QueryParameters.QUERY_END, DefaultQueryParameters.formatDate(this.endDate)); + } catch (ParseException e) { + throw new RuntimeException("Error formatting date", e); + } + } + p.set(PAGE_TIMEOUT, Integer.toString(this.pageTimeout)); + if (this.isMaxResultsOverridden()) { + p.set(MAX_RESULTS_OVERRIDE, Long.toString(this.maxResultsOverride)); + } + if (this.pool != null) { + p.set(QueryParameters.QUERY_POOL, this.pool); + } + + if (this.systemFrom != null) { + p.set("systemFrom", this.systemFrom); + } + + if (this.parameters != null) { + for (Parameter parameter : parameters) { + p.set(parameter.getParameterName(), parameter.getParameterValue()); + } + } + if (this.optionalQueryParameters != null) { + p.putAll(this.optionalQueryParameters); + } + return p; + } + + @Override + public void readMap(Map> map) throws ParseException { + for (String key : map.keySet()) { + switch (key) { + case QUERY_ID: + setId(UUID.fromString(map.get(key).get(0))); + break; + case QueryParameters.QUERY_AUTHORIZATIONS: + setQueryAuthorizations(map.get(key).get(0)); + break; + case QueryParameters.QUERY_EXPIRATION: + setExpirationDate(DefaultQueryParameters.parseDate(map.get(key).get(0), null, null)); + break; + case QueryParameters.QUERY_NAME: + setQueryName(map.get(key).get(0)); + break; + case QueryParameters.QUERY_LOGIC_NAME: + setQueryLogicName(map.get(key).get(0)); + break; + case QueryParameters.QUERY_PAGESIZE: + setPagesize(Integer.parseInt(map.get(key).get(0))); + break; + case QueryParameters.QUERY_STRING: + setQuery(map.get(key).get(0)); + break; + case USER_DN: + setUserDN(map.get(key).get(0)); + setOwner(getOwner(getUserDN())); + break; + case DN_LIST: + setDnList(map.get(key)); + break; + case OWNER: + setOwner(map.get(key).get(0)); + break; + case COLUMN_VISIBILITY: + setColumnVisibility(map.get(key).get(0)); + break; + case QueryParameters.QUERY_BEGIN: + setBeginDate(DefaultQueryParameters.parseStartDate(map.get(key).get(0))); + break; + case QueryParameters.QUERY_END: + setEndDate(DefaultQueryParameters.parseEndDate(map.get(key).get(0))); + break; + case PAGE_TIMEOUT: + setPageTimeout(Integer.parseInt(map.get(key).get(0))); + break; + case MAX_RESULTS_OVERRIDE: + String maxResultsOverride = map.get(key).get(0); + if (maxResultsOverride != null) { + setMaxResultsOverridden(true); + setMaxResultsOverride(Long.parseLong(maxResultsOverride)); + } + break; + case POOL: + setPool(map.get(key).get(0)); + default: + addParameter(key, map.get(key).get(0)); + break; + } + } + } + + @Override + public Map getCardinalityFields() { + Map cardinalityFields = new HashMap(); + cardinalityFields.put(QUERY_USER_FIELD, getOwner()); + cardinalityFields.put(QUERY_LOGIC_NAME_FIELD, getQueryLogicName()); + return cardinalityFields; + } + + @Override + public Schema cachedSchema() { + return SCHEMA; + } + + @Override + public void removeParameter(String key) { + this.parameters.remove(paramLookup.get(key)); + this.paramLookup.remove(key); + } + + @Override + public void populateTrackingMap(Map trackingMap) { + if (trackingMap != null) { + if (this.owner != null) { + trackingMap.put("query.user", this.owner); + } + if (this.id != null) { + trackingMap.put("query.id", this.id); + } + if (this.query != null) { + trackingMap.put("query.query", this.query); + } + } + } +} diff --git a/api/src/main/java/datawave/microservice/query/QueryParameters.java b/api/src/main/java/datawave/microservice/query/QueryParameters.java new file mode 100644 index 00000000..367d7563 --- /dev/null +++ b/api/src/main/java/datawave/microservice/query/QueryParameters.java @@ -0,0 +1,121 @@ +package datawave.microservice.query; + +import java.util.Date; +import java.util.List; +import java.util.Map; + +import org.springframework.util.MultiValueMap; + +import datawave.validation.ParameterValidator; + +/** + * QueryParameters passed in from a client, they are validated and passed through to the iterator stack as QueryOptions. + * + */ +public interface QueryParameters extends ParameterValidator { + + String QUERY_STRING = "query"; + String QUERY_NAME = "queryName"; + String QUERY_PERSISTENCE = "persistence"; + String QUERY_PAGESIZE = "pagesize"; + String QUERY_PAGETIMEOUT = "pageTimeout"; + String QUERY_MAX_RESULTS_OVERRIDE = "max.results.override"; + String QUERY_AUTHORIZATIONS = "auths"; + String QUERY_EXPIRATION = "expiration"; + String QUERY_TRACE = "trace"; + String QUERY_BEGIN = "begin"; + String QUERY_END = "end"; + String QUERY_PARAMS = "params"; + String QUERY_VISIBILITY = "columnVisibility"; + String QUERY_LOGIC_NAME = "logicName"; + String QUERY_POOL = "pool"; + String QUERY_MAX_CONCURRENT_TASKS = "maxConcurrentTasks"; + String QUERY_PLAN_EXPAND_FIELDS = "expand.fields"; + String QUERY_PLAN_EXPAND_VALUES = "expand.values"; + String QUERY_SYSTEM_FROM = "systemFrom"; + + String getQuery(); + + void setQuery(String query); + + String getQueryName(); + + void setQueryName(String queryName); + + QueryPersistence getPersistenceMode(); + + void setPersistenceMode(QueryPersistence persistenceMode); + + int getPagesize(); + + void setPagesize(int pagesize); + + int getPageTimeout(); + + void setPageTimeout(int pageTimeout); + + long getMaxResultsOverride(); + + void setMaxResultsOverride(long maxResults); + + boolean isMaxResultsOverridden(); + + String getAuths(); + + void setAuths(String auths); + + Date getExpirationDate(); + + void setExpirationDate(Date expirationDate); + + boolean isTrace(); + + void setTrace(boolean trace); + + Date getBeginDate(); + + Date getEndDate(); + + void setBeginDate(Date beginDate); + + void setEndDate(Date endDate); + + String getVisibility(); + + void setVisibility(String visibility); + + String getLogicName(); + + void setLogicName(String logicName); + + String getSystemFrom(); + + void setSystemFrom(String systemFrom); + + String getPool(); + + void setPool(String pool); + + int getMaxConcurrentTasks(); + + void setMaxConcurrentTasks(int maxConcurrentTasks); + + boolean isMaxConcurrentTasksOverridden(); + + void setExpandFields(boolean expandFields); + + boolean isExpandFields(); + + void setExpandValues(boolean expandVues); + + boolean isExpandValues(); + + Map> getRequestHeaders(); + + void setRequestHeaders(Map> requestHeaders); + + MultiValueMap getUnknownParameters(Map> allQueryParameters); + + void clear(); + +} diff --git a/api/src/main/java/datawave/microservice/query/QueryPersistence.java b/api/src/main/java/datawave/microservice/query/QueryPersistence.java new file mode 100644 index 00000000..3c9a0533 --- /dev/null +++ b/api/src/main/java/datawave/microservice/query/QueryPersistence.java @@ -0,0 +1,7 @@ +package datawave.microservice.query; + +public enum QueryPersistence { + + PERSISTENT, TRANSIENT; + +} diff --git a/api/src/main/java/datawave/microservice/query/config/QueryExpirationProperties.java b/api/src/main/java/datawave/microservice/query/config/QueryExpirationProperties.java new file mode 100644 index 00000000..8e0f4c35 --- /dev/null +++ b/api/src/main/java/datawave/microservice/query/config/QueryExpirationProperties.java @@ -0,0 +1,242 @@ +package datawave.microservice.query.config; + +import java.util.concurrent.TimeUnit; + +import javax.validation.constraints.NotNull; +import javax.validation.constraints.Positive; + +import org.springframework.validation.annotation.Validated; + +@Validated +public class QueryExpirationProperties { + @Positive + private long idleTimeout = 15; + @NotNull + private TimeUnit idleTimeoutUnit = TimeUnit.MINUTES; + @Positive + private long progressTimeout = 5; + @NotNull + private TimeUnit progressTimeoutUnit = TimeUnit.MINUTES; + @Positive + private long callTimeout = 60; + @NotNull + private TimeUnit callTimeoutUnit = TimeUnit.MINUTES; + @Positive + private long callTimeoutInterval = 1; + @NotNull + private TimeUnit callTimeoutIntervalUnit = TimeUnit.MINUTES; + @Positive + private long pageMinTimeout = 1; + @NotNull + private TimeUnit pageMinTimeoutUnit = TimeUnit.MINUTES; + @Positive + private long pageMaxTimeout = 60; + @NotNull + private TimeUnit pageMaxTimeoutUnit = TimeUnit.MINUTES; + @Positive + private long shortCircuitCheckTime = callTimeout / 2; + @NotNull + private TimeUnit shortCircuitCheckTimeUnit = TimeUnit.MINUTES; + @Positive + private long shortCircuitTimeout = Math.round(0.97 * callTimeout); + @NotNull + private TimeUnit shortCircuitTimeoutUnit = TimeUnit.MINUTES; + + @Positive + @Deprecated // to be replaced by the long running query timeout + private int maxLongRunningTimeoutRetries = 3; + + @Positive + private long longRunningQueryTimeout = (maxLongRunningTimeoutRetries + 1) * callTimeoutUnit.toMinutes(callTimeout); + + @NotNull + private TimeUnit longRunningQueryTimeoutUnit = TimeUnit.MINUTES; + + public long getIdleTimeout() { + return idleTimeout; + } + + public long getIdleTimeoutMillis() { + return idleTimeoutUnit.toMillis(idleTimeout); + } + + public void setIdleTimeout(long idleTimeout) { + this.idleTimeout = idleTimeout; + } + + public TimeUnit getIdleTimeoutUnit() { + return idleTimeoutUnit; + } + + public void setIdleTimeoutUnit(TimeUnit idleTimeoutUnit) { + this.idleTimeoutUnit = idleTimeoutUnit; + } + + public long getProgressTimeout() { + return progressTimeout; + } + + public long getProgressTimeoutMillis() { + return progressTimeoutUnit.toMillis(progressTimeout); + } + + public void setProgressTimeout(long progressTimeout) { + this.progressTimeout = progressTimeout; + } + + public TimeUnit getProgressTimeoutUnit() { + return progressTimeoutUnit; + } + + public void setProgressTimeoutUnit(TimeUnit progressTimeoutUnit) { + this.progressTimeoutUnit = progressTimeoutUnit; + } + + public long getCallTimeout() { + return callTimeout; + } + + public long getCallTimeoutMillis() { + return callTimeoutUnit.toMillis(callTimeout); + } + + public void setCallTimeout(long callTimeout) { + this.callTimeout = callTimeout; + } + + public TimeUnit getCallTimeoutUnit() { + return callTimeoutUnit; + } + + public void setCallTimeoutUnit(TimeUnit callTimeoutUnit) { + this.callTimeoutUnit = callTimeoutUnit; + } + + public long getCallTimeoutInterval() { + return callTimeoutInterval; + } + + public long getCallTimeoutIntervalMillis() { + return callTimeoutIntervalUnit.toMillis(callTimeoutInterval); + } + + public void setCallTimeoutInterval(long callTimeoutInterval) { + this.callTimeoutInterval = callTimeoutInterval; + } + + public TimeUnit getCallTimeoutIntervalUnit() { + return callTimeoutIntervalUnit; + } + + public void setCallTimeoutIntervalUnit(TimeUnit callTimeoutIntervalUnit) { + this.callTimeoutIntervalUnit = callTimeoutIntervalUnit; + } + + public long getPageMinTimeout() { + return pageMinTimeout; + } + + public long getPageMinTimeoutMillis() { + return pageMinTimeoutUnit.toMillis(pageMinTimeout); + } + + public void setPageMinTimeout(long pageMinTimeout) { + this.pageMinTimeout = pageMinTimeout; + } + + public TimeUnit getPageMinTimeoutUnit() { + return pageMinTimeoutUnit; + } + + public void setPageMinTimeoutUnit(TimeUnit pageMinTimeoutUnit) { + this.pageMinTimeoutUnit = pageMinTimeoutUnit; + } + + public long getPageMaxTimeout() { + return pageMaxTimeout; + } + + public long getPageMaxTimeoutMillis() { + return pageMaxTimeoutUnit.toMillis(pageMaxTimeout); + } + + public void setPageMaxTimeout(long pageMaxTimeout) { + this.pageMaxTimeout = pageMaxTimeout; + } + + public TimeUnit getPageMaxTimeoutUnit() { + return pageMaxTimeoutUnit; + } + + public void setPageMaxTimeoutUnit(TimeUnit pageMaxTimeoutUnit) { + this.pageMaxTimeoutUnit = pageMaxTimeoutUnit; + } + + public long getShortCircuitCheckTime() { + return shortCircuitCheckTime; + } + + public long getShortCircuitCheckTimeMillis() { + return shortCircuitCheckTimeUnit.toMillis(shortCircuitCheckTime); + } + + public void setShortCircuitCheckTime(long shortCircuitCheckTime) { + this.shortCircuitCheckTime = shortCircuitCheckTime; + } + + public TimeUnit getShortCircuitCheckTimeUnit() { + return shortCircuitCheckTimeUnit; + } + + public void setShortCircuitCheckTimeUnit(TimeUnit shortCircuitCheckTimeUnit) { + this.shortCircuitCheckTimeUnit = shortCircuitCheckTimeUnit; + } + + public long getShortCircuitTimeout() { + return shortCircuitTimeout; + } + + public long getShortCircuitTimeoutMillis() { + return shortCircuitTimeoutUnit.toMillis(shortCircuitTimeout); + } + + public void setShortCircuitTimeout(long shortCircuitTimeout) { + this.shortCircuitTimeout = shortCircuitTimeout; + } + + public TimeUnit getShortCircuitTimeoutUnit() { + return shortCircuitTimeoutUnit; + } + + public void setShortCircuitTimeoutUnit(TimeUnit shortCircuitTimeoutUnit) { + this.shortCircuitTimeoutUnit = shortCircuitTimeoutUnit; + } + + public int getMaxLongRunningTimeoutRetries() { + return maxLongRunningTimeoutRetries; + } + + public void setMaxLongRunningTimeoutRetries(int maxLongRunningTimeoutRetries) { + this.maxLongRunningTimeoutRetries = maxLongRunningTimeoutRetries; + } + + public long getLongRunningQueryTimeout() { + return longRunningQueryTimeout; + } + + public void setLongRunningQueryTimeout(long longRunningQueryTimeout) { + this.longRunningQueryTimeout = longRunningQueryTimeout; + } + + public TimeUnit getLongRunningQueryTimeoutUnit() { + return longRunningQueryTimeoutUnit; + } + + public void setLongRunningQueryTimeoutUnit(TimeUnit longRunningQueryTimeoutUnit) { + this.longRunningQueryTimeoutUnit = longRunningQueryTimeoutUnit; + } + + public long getLongRunningQueryTimeoutMillis() { + return longRunningQueryTimeoutUnit.toMillis(longRunningQueryTimeout); + } +} diff --git a/api/src/main/resources/META-INF/beans.xml b/api/src/main/resources/META-INF/beans.xml new file mode 100644 index 00000000..4ca201f8 --- /dev/null +++ b/api/src/main/resources/META-INF/beans.xml @@ -0,0 +1,9 @@ + + + + \ No newline at end of file diff --git a/api/src/main/resources/META-INF/jboss-ejb3.xml b/api/src/main/resources/META-INF/jboss-ejb3.xml new file mode 100644 index 00000000..8cf49db8 --- /dev/null +++ b/api/src/main/resources/META-INF/jboss-ejb3.xml @@ -0,0 +1,16 @@ + + + + + + + * + datawave + + + + \ No newline at end of file diff --git a/api/src/main/resources/source-templates/datawave/microservice/query/package-info.java b/api/src/main/resources/source-templates/datawave/microservice/query/package-info.java new file mode 100644 index 00000000..ad28e37a --- /dev/null +++ b/api/src/main/resources/source-templates/datawave/microservice/query/package-info.java @@ -0,0 +1,6 @@ +@XmlSchema(namespace="${datawave.webservice.namespace}", elementFormDefault=XmlNsForm.QUALIFIED, xmlns={@XmlNs(prefix = "", namespaceURI = "${datawave.webservice.namespace}")}) +package datawave.microservice.query; + +import javax.xml.bind.annotation.XmlNs; +import javax.xml.bind.annotation.XmlNsForm; +import javax.xml.bind.annotation.XmlSchema; diff --git a/pom.xml b/pom.xml new file mode 100644 index 00000000..e1b60905 --- /dev/null +++ b/pom.xml @@ -0,0 +1,47 @@ + + + 4.0.0 + + gov.nsa.datawave.microservice + datawave-microservice-parent + 4.0.0-SNAPSHOT + ../../microservice-parent/pom.xml + + query-service-parent + 1.0.0-SNAPSHOT + pom + https://github.com/NationalSecurityAgency/datawave-query-service + + api + + + scm:git:https://github.com/NationalSecurityAgency/datawave-query-service.git + scm:git:git@github.com:NationalSecurityAgency/datawave-query-service.git + https://github.com/NationalSecurityAgency/datawave-query-service + + + + + true + + + false + + github-datawave + https://maven.pkg.github.com/NationalSecurityAgency/datawave + + + + + services + + + !skipServices + + + + service + + + + diff --git a/service/pom.xml b/service/pom.xml new file mode 100644 index 00000000..98b7c465 --- /dev/null +++ b/service/pom.xml @@ -0,0 +1,334 @@ + + + 4.0.0 + + gov.nsa.datawave.microservice + datawave-microservice-service-parent + 5.0.0-SNAPSHOT + ../../../microservice-service-parent/pom.xml + + query-service + 1.0.0-SNAPSHOT + DATAWAVE Query Microservice + https://github.com/NationalSecurityAgency/datawave-query-service + + scm:git:https://github.com/NationalSecurityAgency/datawave-query-service.git + scm:git:git@github.com:NationalSecurityAgency/datawave-query-service.git + HEAD + https://github.com/NationalSecurityAgency/datawave-query-service + + + datawave.microservice.query.QueryService + 3.3.4 + 1.0.0-SNAPSHOT + 4.0.0-SNAPSHOT + 1.0.0-SNAPSHOT + 1.0.0-SNAPSHOT + 6.4.3-1 + 3.3.1-1 + 0.30 + + + + + gov.nsa.datawave.microservice + query-api + ${version.microservice.query-api} + + + gov.nsa.datawave.microservice + spring-boot-starter-datawave-audit + ${version.microservice.starter-audit} + + + gov.nsa.datawave.microservice + spring-boot-starter-datawave-cached-results + ${version.microservice.starter-cached-results} + + + gov.nsa.datawave.microservice + spring-boot-starter-datawave-query + ${version.microservice.starter-query} + + + org.apache.hadoop + hadoop-client + ${version.hadoop} + + + * + org.eclipse.jetty + + + + + org.apache.hadoop + hadoop-common + ${version.hadoop} + + + * + org.mortbay.jetty + + + * + org.eclipse.jetty + + + * + log4j + + + * + org.slf4j + + + * + org.codehaus.jackson + + + * + commons-logging + + + avro + org.apache.avro + + + * + org.apache.curator + + + * + org.apache.zookeeper + + + + + org.apache.hadoop + hadoop-hdfs + ${version.hadoop} + + + * + org.mortbay.jetty + + + * + org.eclipse.jetty + + + * + io.netty + + + * + log4j + + + * + commons-logging + + + + + gov.nsa.datawave.microservice + spring-boot-starter-datawave-cached-results + ${version.microservice.starter-cached-results} + pom + import + + + gov.nsa.datawave.microservice + spring-boot-starter-datawave-query + ${version.microservice.starter-query} + pom + import + + + gov.nsa.datawave.microservice + spring-boot-starter-datawave-query + ${version.microservice.starter-query} + test-jar + test + + + + + + gov.nsa.datawave.core + datawave-core-query + + + log4j + log4j + + + slf4j-reload4j + org.slf4j + + + + + gov.nsa.datawave.microservice + query-api + + + gov.nsa.datawave.microservice + spring-boot-starter-datawave-audit + + + gov.nsa.datawave.microservice + spring-boot-starter-datawave-cached-results + + + gov.nsa.datawave.microservice + spring-boot-starter-datawave-query + + + org.apache.hadoop + hadoop-common + + + org.apache.hadoop + hadoop-hdfs + + + org.springframework.boot + spring-boot-starter-thymeleaf + + + org.webjars + foundation + ${version.webjars.foundation} + + + org.webjars + jquery + ${version.webjars.jquery} + + + org.webjars + webjars-locator-core + ${version.webjars.locator-core} + + + gov.nsa.datawave.microservice + spring-boot-starter-datawave-query + test-jar + test + + + org.junit.jupiter + junit-jupiter-engine + test + + + org.springframework.cloud + spring-cloud-stream-test-support + test + + + + + + true + + + false + + github-datawave + https://maven.pkg.github.com/NationalSecurityAgency/datawave + + + + + + org.apache.maven.plugins + maven-dependency-plugin + 2.8 + + + unpack-test-resources + process-test-resources + + unpack + + + + + gov.nsa.datawave.microservice + spring-boot-starter-datawave-query + test-jar + **/application-*.yml + ${project.build.testOutputDirectory} + + + + + + unpack + package + + unpack + + + + + gov.nsa.datawave.microservice + spring-boot-starter-datawave-query + **/QueryLogicFactory.xml + ${project.build.outputDirectory} + + + + + + + + ch.qos.reload4j + reload4j + 1.2.22 + + + org.apache.maven.shared + maven-common-artifact-filters + 1.4 + + + log4j + log4j + + + + + + + org.springframework.boot + spring-boot-maven-plugin + + + pl.project13.maven + git-commit-id-plugin + + + + + + docker + + + microservice-docker + + + + + + com.spotify + docker-maven-plugin + + + + + + diff --git a/service/src/main/docker/Dockerfile b/service/src/main/docker/Dockerfile new file mode 100644 index 00000000..7c37f330 --- /dev/null +++ b/service/src/main/docker/Dockerfile @@ -0,0 +1,11 @@ +FROM azul/zulu-openjdk-alpine:11 + +LABEL version=${project.version} \ + run="docker run ${docker.image.prefix}${project.artifactId}:latest" \ + description="${project.description}" + +ADD ${project.build.finalName}-exec.jar /app.jar +RUN apk add libc6-compat curl + +EXPOSE 8443 8080 +ENTRYPOINT ["java","-jar","app.jar"] \ No newline at end of file diff --git a/service/src/main/java/datawave/microservice/query/QueryController.java b/service/src/main/java/datawave/microservice/query/QueryController.java new file mode 100644 index 00000000..822f1d23 --- /dev/null +++ b/service/src/main/java/datawave/microservice/query/QueryController.java @@ -0,0 +1,2738 @@ +package datawave.microservice.query; + +import static datawave.core.query.logic.lookup.LookupQueryLogic.LOOKUP_KEY_VALUE_DELIMITER; +import static datawave.microservice.query.QueryParameters.QUERY_AUTHORIZATIONS; +import static datawave.microservice.query.QueryParameters.QUERY_BEGIN; +import static datawave.microservice.query.QueryParameters.QUERY_END; +import static datawave.microservice.query.QueryParameters.QUERY_MAX_CONCURRENT_TASKS; +import static datawave.microservice.query.QueryParameters.QUERY_MAX_RESULTS_OVERRIDE; +import static datawave.microservice.query.QueryParameters.QUERY_NAME; +import static datawave.microservice.query.QueryParameters.QUERY_PAGESIZE; +import static datawave.microservice.query.QueryParameters.QUERY_PAGETIMEOUT; +import static datawave.microservice.query.QueryParameters.QUERY_PARAMS; +import static datawave.microservice.query.QueryParameters.QUERY_PLAN_EXPAND_FIELDS; +import static datawave.microservice.query.QueryParameters.QUERY_PLAN_EXPAND_VALUES; +import static datawave.microservice.query.QueryParameters.QUERY_POOL; +import static datawave.microservice.query.QueryParameters.QUERY_STRING; +import static datawave.microservice.query.QueryParameters.QUERY_VISIBILITY; +import static datawave.microservice.query.lookup.LookupService.LOOKUP_CONTEXT; +import static datawave.microservice.query.lookup.LookupService.LOOKUP_STREAMING; +import static datawave.microservice.query.lookup.LookupService.LOOKUP_UUID_PAIRS; +import static datawave.microservice.query.translateid.TranslateIdService.TRANSLATE_ID; +import static datawave.query.QueryParameters.QUERY_SYNTAX; + +import java.util.List; +import java.util.function.Supplier; + +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.security.access.annotation.Secured; +import org.springframework.security.core.annotation.AuthenticationPrincipal; +import org.springframework.util.MultiValueMap; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestHeader; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.servlet.mvc.method.annotation.ResponseBodyEmitter; + +import com.codahale.metrics.annotation.Timed; + +import datawave.microservice.authorization.user.DatawaveUserDetails; +import datawave.microservice.query.lookup.LookupService; +import datawave.microservice.query.stream.StreamingProperties; +import datawave.microservice.query.stream.StreamingService; +import datawave.microservice.query.stream.listener.CountingResponseBodyEmitterListener; +import datawave.microservice.query.stream.listener.StreamingResponseListener; +import datawave.microservice.query.translateid.TranslateIdService; +import datawave.microservice.query.web.annotation.EnrichQueryMetrics; +import datawave.microservice.query.web.filter.BaseMethodStatsFilter; +import datawave.microservice.query.web.filter.CountingResponseBodyEmitter; +import datawave.microservice.query.web.filter.QueryMetricsEnrichmentFilterAdvice; +import datawave.webservice.query.exception.QueryException; +import datawave.webservice.result.BaseQueryResponse; +import datawave.webservice.result.GenericResponse; +import datawave.webservice.result.QueryImplListResponse; +import datawave.webservice.result.QueryLogicResponse; +import datawave.webservice.result.VoidResponse; +import io.swagger.v3.oas.annotations.ExternalDocumentation; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.Parameters; +import io.swagger.v3.oas.annotations.enums.ParameterIn; +import io.swagger.v3.oas.annotations.headers.Header; +import io.swagger.v3.oas.annotations.media.ArraySchema; +import io.swagger.v3.oas.annotations.media.Content; +import io.swagger.v3.oas.annotations.media.Schema; +import io.swagger.v3.oas.annotations.responses.ApiResponse; +import io.swagger.v3.oas.annotations.responses.ApiResponses; +import io.swagger.v3.oas.annotations.tags.Tag; + +@Tag(name = "Query Controller /v1", description = "DataWave Query Management", + externalDocs = @ExternalDocumentation(description = "Query Service Documentation", + url = "https://github.com/NationalSecurityAgency/datawave-query-service")) +@RestController +@RequestMapping(path = "/v1/query", produces = MediaType.APPLICATION_JSON_VALUE) +public class QueryController { + private final QueryManagementService queryManagementService; + private final LookupService lookupService; + private final StreamingService streamingService; + private final TranslateIdService translateIdService; + + private final StreamingProperties streamingProperties; + + private final Supplier serverUserDetailsSupplier; + + // Note: baseMethodStatsContext needs to be request scoped + private final BaseMethodStatsFilter.BaseMethodStatsContext baseMethodStatsContext; + // Note: queryMetricsEnrichmentContest needs to be request scoped + private final QueryMetricsEnrichmentFilterAdvice.QueryMetricsEnrichmentContext queryMetricsEnrichmentContext; + + public QueryController(QueryManagementService queryManagementService, LookupService lookupService, StreamingService streamingService, + TranslateIdService translateIdService, StreamingProperties streamingProperties, + @Qualifier("serverUserDetailsSupplier") Supplier serverUserDetailsSupplier, + BaseMethodStatsFilter.BaseMethodStatsContext baseMethodStatsContext, + QueryMetricsEnrichmentFilterAdvice.QueryMetricsEnrichmentContext queryMetricsEnrichmentContext) { + this.queryManagementService = queryManagementService; + this.lookupService = lookupService; + this.streamingService = streamingService; + this.translateIdService = translateIdService; + this.streamingProperties = streamingProperties; + this.serverUserDetailsSupplier = serverUserDetailsSupplier; + this.baseMethodStatsContext = baseMethodStatsContext; + this.queryMetricsEnrichmentContext = queryMetricsEnrichmentContext; + } + + /** + * @see QueryManagementService#define(String, MultiValueMap, String, DatawaveUserDetails) + */ + // @formatter:off + @Operation( + summary = "Defines a query using the given query logic and parameters.", + description = "Defined queries cannot be started and run.
" + + "Auditing is not performed when defining a query.
" + + "Updates can be made to any parameter using update.
" + + "Create a runnable query from a defined query using duplicate, reset, or mapreduce/submit.
" + + "Delete a defined query using remove.
" + + "Aside from a limited set of admin actions, only the query owner can act on a defined query.") + @ApiResponses({ + @ApiResponse( + description = "if successful, returns a generic response containing the query id", + responseCode = "200", + content = @Content(schema = @Schema(implementation = GenericResponse.class)), + headers = { + @Header( + name = "Pool", + description = "the executor pool to target", + schema = @Schema(defaultValue = "default"))}), + @ApiResponse( + description = "if parameter validation fails
" + + "if query logic parameter validation fails
" + + "if security marking validation fails", + responseCode = "400", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if the user doesn't have access to the requested query logic", + responseCode = "401", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if query storage fails
" + + "if there is an unknown error", + responseCode = "500", + content = @Content(schema = @Schema(implementation = VoidResponse.class)))}) + @Parameters({ + @Parameter( + name = QUERY_BEGIN, + in = ParameterIn.QUERY, + description = "The query begin date", + required = true, + schema = @Schema(implementation = String.class), + example = "\"19660908 000000.000\""), + @Parameter( + name = QUERY_END, + in = ParameterIn.QUERY, + description = "The query end date", + required = true, + schema = @Schema(implementation = String.class), + example = "\"20161002 235959.999\""), + @Parameter( + name = QUERY_NAME, + in = ParameterIn.QUERY, + description = "The query name", + required = true, + schema = @Schema(implementation = String.class), + example = "Developer Test Query"), + @Parameter( + name = QUERY_STRING, + in = ParameterIn.QUERY, + description = "The query string", + required = true, + schema = @Schema(implementation = String.class), + example = "GENRES:[Action to Western]"), + @Parameter( + name = QUERY_AUTHORIZATIONS, + in = ParameterIn.QUERY, + description = "The query auths", + required = true, + schema = @Schema(implementation = String.class), + example = "PUBLIC,PRIVATE,BAR,FOO"), + @Parameter( + name = QUERY_VISIBILITY, + in = ParameterIn.QUERY, + description = "The visibility to use when storing metrics for this query", + required = true, + schema = @Schema(implementation = String.class), + example = "PUBLIC"), + @Parameter( + name = QUERY_SYNTAX, + in = ParameterIn.QUERY, + description = "The syntax used in the query", + schema = @Schema(implementation = String.class), + example = "LUCENE"), + @Parameter( + name = QUERY_MAX_CONCURRENT_TASKS, + in = ParameterIn.QUERY, + description = "The max number of concurrent tasks to run for this query", + schema = @Schema(implementation = Integer.class), + example = "10"), + @Parameter( + name = QUERY_POOL, + in = ParameterIn.QUERY, + description = "The executor pool to run against", + schema = @Schema(implementation = String.class), + example = "pool1"), + @Parameter( + name = QUERY_PAGESIZE, + in = ParameterIn.QUERY, + description = "The requested page size", + schema = @Schema(implementation = Integer.class), + example = "10"), + @Parameter( + name = QUERY_PAGETIMEOUT, + in = ParameterIn.QUERY, + description = "The call timeout when requesting a page, in minutes", + schema = @Schema(implementation = Integer.class), + example = "60"), + @Parameter( + name = QUERY_MAX_RESULTS_OVERRIDE, + in = ParameterIn.QUERY, + description = "The max results override value", + schema = @Schema(implementation = Integer.class), + example = "5000"), + @Parameter( + name = QUERY_PARAMS, + in = ParameterIn.QUERY, + description = "Additional query parameters", + schema = @Schema(implementation = String.class), + example = "KEY_1:VALUE_1;KEY_2:VALUE_2") + }) + // @formatter:on + @Timed(name = "dw.query.defineQuery", absolute = true) + @EnrichQueryMetrics(methodType = EnrichQueryMetrics.MethodType.CREATE) + @RequestMapping(path = "{queryLogic}/define", method = {RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", + "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) + public GenericResponse define(@Parameter(description = "The query logic", example = "EventQuery") @PathVariable String queryLogic, + @Parameter(hidden = true) @RequestParam MultiValueMap parameters, @RequestHeader HttpHeaders headers, + @AuthenticationPrincipal DatawaveUserDetails currentUser) throws QueryException { + return queryManagementService.define(queryLogic, parameters, getPool(headers), currentUser); + } + + /** + * @see QueryManagementService#listQueryLogic(DatawaveUserDetails) + */ + @Operation(summary = "Gets a list of descriptions for the configured query logics, sorted by query logic name.", + description = "The descriptions include things like the audit type, optional and required parameters, required roles, and response class.") + @Timed(name = "dw.query.listQueryLogic", absolute = true) + @RequestMapping(path = "listQueryLogic", method = {RequestMethod.GET}, + produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "text/html"}) + public QueryLogicResponse listQueryLogic(@AuthenticationPrincipal DatawaveUserDetails currentUser) { + return queryManagementService.listQueryLogic(currentUser); + } + + /** + * @see QueryManagementService#create(String, MultiValueMap, String, DatawaveUserDetails) + */ + // @formatter:off + @Operation( + summary = "Creates a query using the given query logic and parameters.", + description = "Created queries will start running immediately.
" + + "Auditing is performed before the query is started.
" + + "Query results can be retrieved using next.
" + + "Updates can be made to any parameter which doesn't affect the scope of the query using update.
" + + "Stop a running query gracefully using close or forcefully using cancel.
" + + "Stop, and restart a running query using reset.
" + + "Create a copy of a running query using duplicate.
" + + "Aside from a limited set of admin actions, only the query owner can act on a running query.") + @ApiResponses({ + @ApiResponse( + description = "if successful, returns a generic response containing the query id", + responseCode = "200", + content = @Content(schema = @Schema(implementation = GenericResponse.class)), + headers = { + @Header( + name = "Pool", + description = "the executor pool to target", + schema = @Schema(defaultValue = "default"))}), + @ApiResponse( + description = "if parameter validation fails
" + + "if query logic parameter validation fails
" + + "if security marking validation fails
" + + "if auditing fails", + responseCode = "400", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if the user doesn't have access to the requested query logic", + responseCode = "401", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if query storage fails
" + + "if there is an unknown error", + responseCode = "500", + content = @Content(schema = @Schema(implementation = VoidResponse.class)))}) + @Parameters({ + @Parameter( + name = QUERY_BEGIN, + in = ParameterIn.QUERY, + description = "The query begin date", + required = true, + schema = @Schema(implementation = String.class), + example = "\"19660908 000000.000\""), + @Parameter( + name = QUERY_END, + in = ParameterIn.QUERY, + description = "The query end date", + required = true, + schema = @Schema(implementation = String.class), + example = "\"20161002 235959.999\""), + @Parameter( + name = QUERY_NAME, + in = ParameterIn.QUERY, + description = "The query name", + required = true, + schema = @Schema(implementation = String.class), + example = "Developer Test Query"), + @Parameter( + name = QUERY_STRING, + in = ParameterIn.QUERY, + description = "The query string", + required = true, + schema = @Schema(implementation = String.class), + example = "GENRES:[Action to Western]"), + @Parameter( + name = QUERY_AUTHORIZATIONS, + in = ParameterIn.QUERY, + description = "The query auths", + required = true, + schema = @Schema(implementation = String.class), + example = "PUBLIC,PRIVATE,BAR,FOO"), + @Parameter( + name = QUERY_VISIBILITY, + in = ParameterIn.QUERY, + description = "The visibility to use when storing metrics for this query", + required = true, + schema = @Schema(implementation = String.class), + example = "PUBLIC"), + @Parameter( + name = QUERY_SYNTAX, + in = ParameterIn.QUERY, + description = "The syntax used in the query", + schema = @Schema(implementation = String.class), + example = "LUCENE"), + @Parameter( + name = QUERY_MAX_CONCURRENT_TASKS, + in = ParameterIn.QUERY, + description = "The max number of concurrent tasks to run for this query", + schema = @Schema(implementation = Integer.class), + example = "10"), + @Parameter( + name = QUERY_POOL, + in = ParameterIn.QUERY, + description = "The executor pool to run against", + schema = @Schema(implementation = String.class), + example = "pool1"), + @Parameter( + name = QUERY_PAGESIZE, + in = ParameterIn.QUERY, + description = "The requested page size", + schema = @Schema(implementation = Integer.class), + example = "10"), + @Parameter( + name = QUERY_PAGETIMEOUT, + in = ParameterIn.QUERY, + description = "The call timeout when requesting a page, in minutes", + schema = @Schema(implementation = Integer.class), + example = "60"), + @Parameter( + name = QUERY_MAX_RESULTS_OVERRIDE, + in = ParameterIn.QUERY, + description = "The max results override value", + schema = @Schema(implementation = Integer.class), + example = "5000"), + @Parameter( + name = QUERY_PARAMS, + in = ParameterIn.QUERY, + description = "Additional query parameters", + schema = @Schema(implementation = String.class), + example = "KEY_1:VALUE_1;KEY_2:VALUE_2") + }) + // @formatter:on + @Timed(name = "dw.query.createQuery", absolute = true) + @EnrichQueryMetrics(methodType = EnrichQueryMetrics.MethodType.CREATE) + @RequestMapping(path = "{queryLogic}/create", method = {RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", + "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) + public GenericResponse create(@Parameter(description = "The query logic", example = "EventQuery") @PathVariable String queryLogic, + @Parameter(hidden = true) @RequestParam MultiValueMap parameters, @RequestHeader HttpHeaders headers, + @AuthenticationPrincipal DatawaveUserDetails currentUser) throws QueryException { + return queryManagementService.create(queryLogic, parameters, getPool(headers), currentUser); + } + + /** + * @see QueryManagementService#plan(String, MultiValueMap, String, DatawaveUserDetails) + */ + // @formatter:off + @Operation( + summary = "Generates a query plan using the given query logic and parameters.", + description = "Created queries will begin planning immediately.
" + + "Auditing is performed if we are expanding indices.
" + + "Query plan will be returned in the response.
" + + "Updates can be made to any parameter which doesn't affect the scope of the query using update.
" + + "Stop a running query gracefully using close or forcefully using cancel.
" + + "Stop, and restart a running query using reset.
" + + "Create a copy of a running query using duplicate.
" + + "Aside from a limited set of admin actions, only the query owner can act on a running query.") + @ApiResponses({ + @ApiResponse( + description = "if successful, returns a generic response containing the query plan", + responseCode = "200", + content = @Content(schema = @Schema(implementation = GenericResponse.class)), + headers = { + @Header( + name = "Pool", + description = "the executor pool to target", + schema = @Schema(defaultValue = "default"))}), + @ApiResponse( + description = "if parameter validation fails
" + + "if query logic parameter validation fails
" + + "if security marking validation fails
" + + "if auditing fails", + responseCode = "400", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if the user doesn't have access to the requested query logic", + responseCode = "401", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if query storage fails
" + + "if there is an unknown error", + responseCode = "500", + content = @Content(schema = @Schema(implementation = VoidResponse.class)))}) + @Parameters({ + @Parameter( + name = QUERY_BEGIN, + in = ParameterIn.QUERY, + description = "The query begin date", + required = true, + schema = @Schema(implementation = String.class), + example = "\"19660908 000000.000\""), + @Parameter( + name = QUERY_END, + in = ParameterIn.QUERY, + description = "The query end date", + required = true, + schema = @Schema(implementation = String.class), + example = "\"20161002 235959.999\""), + @Parameter( + name = QUERY_NAME, + in = ParameterIn.QUERY, + description = "The query name", + required = true, + schema = @Schema(implementation = String.class), + example = "Developer Test Query"), + @Parameter( + name = QUERY_STRING, + in = ParameterIn.QUERY, + description = "The query string", + required = true, + schema = @Schema(implementation = String.class), + example = "GENRES:[Action to Western]"), + @Parameter( + name = QUERY_AUTHORIZATIONS, + in = ParameterIn.QUERY, + description = "The query auths", + required = true, + schema = @Schema(implementation = String.class), + example = "PUBLIC,PRIVATE,BAR,FOO"), + @Parameter( + name = QUERY_VISIBILITY, + in = ParameterIn.QUERY, + description = "The visibility to use when storing metrics for this query", + required = true, + schema = @Schema(implementation = String.class), + example = "PUBLIC"), + @Parameter( + name = QUERY_PLAN_EXPAND_FIELDS, + in = ParameterIn.QUERY, + description = "Whether to expand unfielded terms", + schema = @Schema(implementation = Boolean.class), + example = "true" + ), + @Parameter( + name = QUERY_PLAN_EXPAND_VALUES, + in = ParameterIn.QUERY, + description = "Whether to expand regex and/or ranges into discrete values
" + + "If 'true', auditing will be performed", + schema = @Schema(implementation = Boolean.class), + example = "true" + ), + @Parameter( + name = QUERY_SYNTAX, + in = ParameterIn.QUERY, + description = "The syntax used in the query", + schema = @Schema(implementation = String.class), + example = "LUCENE"), + @Parameter( + name = QUERY_MAX_CONCURRENT_TASKS, + in = ParameterIn.QUERY, + description = "The max number of concurrent tasks to run for this query", + schema = @Schema(implementation = Integer.class), + example = "10"), + @Parameter( + name = QUERY_POOL, + in = ParameterIn.QUERY, + description = "The executor pool to run against", + schema = @Schema(implementation = String.class), + example = "pool1"), + @Parameter( + name = QUERY_PAGESIZE, + in = ParameterIn.QUERY, + description = "The requested page size", + schema = @Schema(implementation = Integer.class), + example = "10"), + @Parameter( + name = QUERY_PAGETIMEOUT, + in = ParameterIn.QUERY, + description = "The call timeout when requesting a page, in minutes", + schema = @Schema(implementation = Integer.class), + example = "60"), + @Parameter( + name = QUERY_MAX_RESULTS_OVERRIDE, + in = ParameterIn.QUERY, + description = "The max results override value", + schema = @Schema(implementation = Integer.class), + example = "5000"), + @Parameter( + name = QUERY_PARAMS, + in = ParameterIn.QUERY, + description = "Additional query parameters", + schema = @Schema(implementation = String.class), + example = "KEY_1:VALUE_1;KEY_2:VALUE_2") + }) + // @formatter:on + @Timed(name = "dw.query.planQuery", absolute = true) + @RequestMapping(path = "{queryLogic}/plan", method = {RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", + "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) + public GenericResponse plan(@Parameter(description = "The query logic", example = "EventQuery") @PathVariable String queryLogic, + @Parameter(hidden = true) @RequestParam MultiValueMap parameters, @RequestHeader HttpHeaders headers, + @AuthenticationPrincipal DatawaveUserDetails currentUser) throws QueryException { + return queryManagementService.plan(queryLogic, parameters, getPool(headers), currentUser); + } + + /** + * @see QueryManagementService#predict(String, MultiValueMap, String, DatawaveUserDetails) + */ + // @formatter:off + @Operation( + summary = "Generates a query prediction using the given query logic and parameters.", + description = "Created queries will begin predicting immediately.
" + + "Auditing is not performed.
" + + "Query prediction will be returned in the response.
" + + "Updates can be made to any parameter which doesn't affect the scope of the query using update.
") + @ApiResponses({ + @ApiResponse( + description = "if successful, returns a generic response containing the query plan", + responseCode = "200", + content = @Content(schema = @Schema(implementation = GenericResponse.class)), + headers = { + @Header( + name = "Pool", + description = "the executor pool to target", + schema = @Schema(defaultValue = "default"))}), + @ApiResponse( + description = "if parameter validation fails
" + + "if query logic parameter validation fails
" + + "if security marking validation fails
" + + "if auditing fails", + responseCode = "400", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if the user doesn't have access to the requested query logic", + responseCode = "401", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if query storage fails
" + + "if there is an unknown error", + responseCode = "500", + content = @Content(schema = @Schema(implementation = VoidResponse.class)))}) + @Parameters({ + @Parameter( + name = QUERY_BEGIN, + in = ParameterIn.QUERY, + description = "The query begin date", + required = true, + schema = @Schema(implementation = String.class), + example = "\"19660908 000000.000\""), + @Parameter( + name = QUERY_END, + in = ParameterIn.QUERY, + description = "The query end date", + required = true, + schema = @Schema(implementation = String.class), + example = "\"20161002 235959.999\""), + @Parameter( + name = QUERY_NAME, + in = ParameterIn.QUERY, + description = "The query name", + required = true, + schema = @Schema(implementation = String.class), + example = "Developer Test Query"), + @Parameter( + name = QUERY_STRING, + in = ParameterIn.QUERY, + description = "The query string", + required = true, + schema = @Schema(implementation = String.class), + example = "GENRES:[Action to Western]"), + @Parameter( + name = QUERY_AUTHORIZATIONS, + in = ParameterIn.QUERY, + description = "The query auths", + required = true, + schema = @Schema(implementation = String.class), + example = "PUBLIC,PRIVATE,BAR,FOO"), + @Parameter( + name = QUERY_VISIBILITY, + in = ParameterIn.QUERY, + description = "The visibility to use when storing metrics for this query", + required = true, + schema = @Schema(implementation = String.class), + example = "PUBLIC"), + @Parameter( + name = QUERY_SYNTAX, + in = ParameterIn.QUERY, + description = "The syntax used in the query", + schema = @Schema(implementation = String.class), + example = "LUCENE"), + @Parameter( + name = QUERY_MAX_CONCURRENT_TASKS, + in = ParameterIn.QUERY, + description = "The max number of concurrent tasks to run for this query", + schema = @Schema(implementation = Integer.class), + example = "10"), + @Parameter( + name = QUERY_POOL, + in = ParameterIn.QUERY, + description = "The executor pool to run against", + schema = @Schema(implementation = String.class), + example = "pool1"), + @Parameter( + name = QUERY_PAGESIZE, + in = ParameterIn.QUERY, + description = "The requested page size", + schema = @Schema(implementation = Integer.class), + example = "10"), + @Parameter( + name = QUERY_PAGETIMEOUT, + in = ParameterIn.QUERY, + description = "The call timeout when requesting a page, in minutes", + schema = @Schema(implementation = Integer.class), + example = "60"), + @Parameter( + name = QUERY_MAX_RESULTS_OVERRIDE, + in = ParameterIn.QUERY, + description = "The max results override value", + schema = @Schema(implementation = Integer.class), + example = "5000"), + @Parameter( + name = QUERY_PARAMS, + in = ParameterIn.QUERY, + description = "Additional query parameters", + schema = @Schema(implementation = String.class), + example = "KEY_1:VALUE_1;KEY_2:VALUE_2") + }) + // @formatter:on + @Timed(name = "dw.query.predictQuery", absolute = true) + @RequestMapping(path = "{queryLogic}/predict", method = {RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", + "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) + public GenericResponse predict(@Parameter(description = "The query logic", example = "EventQuery") @PathVariable String queryLogic, + @Parameter(hidden = true) @RequestParam MultiValueMap parameters, @RequestHeader HttpHeaders headers, + @AuthenticationPrincipal DatawaveUserDetails currentUser) throws QueryException { + return queryManagementService.predict(queryLogic, parameters, getPool(headers), currentUser); + } + + /** + * @see LookupService#lookupUUID(MultiValueMap, String, DatawaveUserDetails) + * @see LookupService#lookupUUID(MultiValueMap, String, DatawaveUserDetails, StreamingResponseListener) + */ + // @formatter:off + @Operation( + summary = "Creates an event lookup query using the query logic associated with the given uuid type(s) and parameters, and returns the first page of results.", + description = "Lookup queries will start running immediately.
" + + "Auditing is performed before the query is started.
" + + "Each of the uuid pairs must map to the same query logic.
" + + "After the first page is returned, the query will be closed.") + @ApiResponses({ + @ApiResponse( + description = "if successful, returns a base query response containing the first page of results", + responseCode = "200", + content = @Content(schema = @Schema(implementation = BaseQueryResponse.class)), + headers = { + @Header( + name = "Pool", + description = "the executor pool to target", + schema = @Schema(defaultValue = "default"))}), + @ApiResponse( + description = "if no query results are found", + responseCode = "204", + content = @Content(schema = @Schema(hidden = true))), + @ApiResponse( + description = "if parameter validation fails
" + + "if query logic parameter validation fails
" + + "if security marking validation fails
" + + "if auditing fails", + responseCode = "400", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if the user doesn't have access to the requested query logic", + responseCode = "401", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if query storage fails
" + + "if the next call times out
" + + "if the next task is rejected by the executor
" + + "if there is an unknown error", + responseCode = "500", + content = @Content(schema = @Schema(implementation = VoidResponse.class)))}) + @Parameters({ + @Parameter( + name = LOOKUP_STREAMING, + in = ParameterIn.QUERY, + description = "if true, streams all results back", + schema = @Schema(implementation = Boolean.class), + example = "true"), + @Parameter( + name = LOOKUP_CONTEXT, + in = ParameterIn.QUERY, + description = "The lookup UUID type context", + example = "default", + array = @ArraySchema(schema = @Schema(implementation = String.class))), + @Parameter( + name = QUERY_BEGIN, + in = ParameterIn.QUERY, + description = "The query begin date", + required = true, + schema = @Schema(implementation = String.class), + example = "\"19660908 000000.000\""), + @Parameter( + name = QUERY_END, + in = ParameterIn.QUERY, + description = "The query end date", + required = true, + schema = @Schema(implementation = String.class), + example = "\"20161002 235959.999\""), + @Parameter( + name = QUERY_NAME, + in = ParameterIn.QUERY, + description = "The query name", + required = true, + schema = @Schema(implementation = String.class), + example = "Developer Test Query"), + @Parameter( + name = QUERY_AUTHORIZATIONS, + in = ParameterIn.QUERY, + description = "The query auths", + required = true, + schema = @Schema(implementation = String.class), + example = "PUBLIC,PRIVATE,BAR,FOO"), + @Parameter( + name = QUERY_VISIBILITY, + in = ParameterIn.QUERY, + description = "The visibility to use when storing metrics for this query", + required = true, + schema = @Schema(implementation = String.class), + example = "PUBLIC"), + @Parameter( + name = QUERY_MAX_CONCURRENT_TASKS, + in = ParameterIn.QUERY, + description = "The max number of concurrent tasks to run for this query", + schema = @Schema(implementation = Integer.class), + example = "10"), + @Parameter( + name = QUERY_POOL, + in = ParameterIn.QUERY, + description = "The executor pool to run against", + schema = @Schema(implementation = String.class), + example = "pool1"), + @Parameter( + name = QUERY_PAGESIZE, + in = ParameterIn.QUERY, + description = "The requested page size", + schema = @Schema(implementation = Integer.class), + example = "10"), + @Parameter( + name = QUERY_PAGETIMEOUT, + in = ParameterIn.QUERY, + description = "The call timeout when requesting a page, in minutes", + schema = @Schema(implementation = Integer.class), + example = "60"), + @Parameter( + name = QUERY_MAX_RESULTS_OVERRIDE, + in = ParameterIn.QUERY, + description = "The max results override value", + schema = @Schema(implementation = Integer.class), + example = "5000"), + @Parameter( + name = QUERY_PARAMS, + in = ParameterIn.QUERY, + description = "Additional query parameters", + schema = @Schema(implementation = String.class), + example = "KEY_1:VALUE_1;KEY_2:VALUE_2") + }) + // @formatter:on + @Timed(name = "dw.query.lookupUUID", absolute = true) + @RequestMapping(path = "lookupUUID/{uuidType}/{uuid}", method = {RequestMethod.GET}, produces = {"application/xml", "text/xml", "application/json", + "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) + public Object lookupUUID(@Parameter(description = "The UUID type", example = "PAGE_TITLE") @PathVariable(required = false) String uuidType, + @Parameter(description = "The UUID", example = "anarchism") @PathVariable(required = false) String uuid, + @Parameter(hidden = true) @RequestParam MultiValueMap parameters, @RequestHeader HttpHeaders headers, + @AuthenticationPrincipal DatawaveUserDetails currentUser) throws QueryException { + parameters.add(LOOKUP_UUID_PAIRS, String.join(LOOKUP_KEY_VALUE_DELIMITER, uuidType, uuid)); + + if (Boolean.parseBoolean(parameters.getFirst(LOOKUP_STREAMING))) { + MediaType contentType = determineContentType(headers.getAccept(), MediaType.parseMediaType(streamingProperties.getDefaultContentType())); + CountingResponseBodyEmitter emitter = baseMethodStatsContext.createCountingResponseBodyEmitter(streamingProperties.getCallTimeoutMillis()); + lookupService.lookupUUID(parameters, getPool(headers), currentUser, new CountingResponseBodyEmitterListener(emitter, contentType)); + return emitter; + } else { + return lookupService.lookupUUID(parameters, getPool(headers), currentUser); + } + } + + /** + * @see LookupService#lookupUUID(MultiValueMap, String, DatawaveUserDetails) + * @see LookupService#lookupUUID(MultiValueMap, String, DatawaveUserDetails, StreamingResponseListener) + */ + // @formatter:off + @Operation( + summary = "Creates an event lookup query using the query logic associated with the given uuid type(s) and parameters, and returns the first page of results.", + description = "Lookup queries will start running immediately.
" + + "Auditing is performed before the query is started.
" + + "Each of the uuid pairs must map to the same query logic.
" + + "After the first page is returned, the query will be closed.") + @ApiResponses({ + @ApiResponse( + description = "if successful, returns a base query response containing the first page of results", + responseCode = "200", + content = @Content(schema = @Schema(implementation = BaseQueryResponse.class)), + headers = { + @Header( + name = "Pool", + description = "the executor pool to target", + schema = @Schema(defaultValue = "default"))}), + @ApiResponse( + description = "if no query results are found", + responseCode = "204", + content = @Content(schema = @Schema(hidden = true))), + @ApiResponse( + description = "if parameter validation fails
" + + "if query logic parameter validation fails
" + + "if security marking validation fails
" + + "if auditing fails", + responseCode = "400", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if the user doesn't have access to the requested query logic", + responseCode = "401", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if query storage fails
" + + "if the next call times out
" + + "if the next task is rejected by the executor
" + + "if there is an unknown error", + responseCode = "500", + content = @Content(schema = @Schema(implementation = VoidResponse.class)))}) + @Parameters({ + @Parameter( + name = LOOKUP_STREAMING, + in = ParameterIn.QUERY, + description = "if true, streams all results back", + schema = @Schema(implementation = Boolean.class), + example = "true"), + @Parameter( + name = LOOKUP_CONTEXT, + in = ParameterIn.QUERY, + description = "The lookup UUID type context", + example = "default", + array = @ArraySchema(schema = @Schema(implementation = String.class))), + @Parameter( + name = LOOKUP_UUID_PAIRS, + in = ParameterIn.QUERY, + description = "The lookup UUID pairs
" + + "To lookup multiple UUID pairs, submit multiples of this parameter", + required = true, + example = "PAGE_TITLE:anarchism", + array = @ArraySchema(schema = @Schema(implementation = String.class))), + @Parameter( + name = QUERY_BEGIN, + in = ParameterIn.QUERY, + description = "The query begin date", + required = true, + schema = @Schema(implementation = String.class), + example = "\"19660908 000000.000\""), + @Parameter( + name = QUERY_END, + in = ParameterIn.QUERY, + description = "The query end date", + required = true, + schema = @Schema(implementation = String.class), + example = "\"20161002 235959.999\""), + @Parameter( + name = QUERY_NAME, + in = ParameterIn.QUERY, + description = "The query name", + required = true, + schema = @Schema(implementation = String.class), + example = "Developer Test Query"), + @Parameter( + name = QUERY_AUTHORIZATIONS, + in = ParameterIn.QUERY, + description = "The query auths", + required = true, + schema = @Schema(implementation = String.class), + example = "PUBLIC,PRIVATE,BAR,FOO"), + @Parameter( + name = QUERY_VISIBILITY, + in = ParameterIn.QUERY, + description = "The visibility to use when storing metrics for this query", + required = true, + schema = @Schema(implementation = String.class), + example = "PUBLIC"), + @Parameter( + name = QUERY_MAX_CONCURRENT_TASKS, + in = ParameterIn.QUERY, + description = "The max number of concurrent tasks to run for this query", + schema = @Schema(implementation = Integer.class), + example = "10"), + @Parameter( + name = QUERY_POOL, + in = ParameterIn.QUERY, + description = "The executor pool to run against", + schema = @Schema(implementation = String.class), + example = "pool1"), + @Parameter( + name = QUERY_PAGESIZE, + in = ParameterIn.QUERY, + description = "The requested page size", + schema = @Schema(implementation = Integer.class), + example = "10"), + @Parameter( + name = QUERY_PAGETIMEOUT, + in = ParameterIn.QUERY, + description = "The call timeout when requesting a page, in minutes", + schema = @Schema(implementation = Integer.class), + example = "60"), + @Parameter( + name = QUERY_MAX_RESULTS_OVERRIDE, + in = ParameterIn.QUERY, + description = "The max results override value", + schema = @Schema(implementation = Integer.class), + example = "5000"), + @Parameter( + name = QUERY_PARAMS, + in = ParameterIn.QUERY, + description = "Additional query parameters", + schema = @Schema(implementation = String.class), + example = "KEY_1:VALUE_1;KEY_2:VALUE_2") + }) + // @formatter:on + @Timed(name = "dw.query.lookupUUIDBatch", absolute = true) + @RequestMapping(path = "lookupUUID", method = {RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", + "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) + public Object lookupUUIDBatch(@Parameter(hidden = true) @RequestParam MultiValueMap parameters, @RequestHeader HttpHeaders headers, + @AuthenticationPrincipal DatawaveUserDetails currentUser) throws QueryException { + if (Boolean.parseBoolean(parameters.getFirst(LOOKUP_STREAMING))) { + MediaType contentType = determineContentType(headers.getAccept(), MediaType.parseMediaType(streamingProperties.getDefaultContentType())); + CountingResponseBodyEmitter emitter = baseMethodStatsContext.createCountingResponseBodyEmitter(streamingProperties.getCallTimeoutMillis()); + lookupService.lookupUUID(parameters, getPool(headers), currentUser, new CountingResponseBodyEmitterListener(emitter, contentType)); + return emitter; + } else { + return lookupService.lookupUUID(parameters, getPool(headers), currentUser); + } + } + + /** + * @see LookupService#lookupContentUUID(MultiValueMap, String, DatawaveUserDetails) + * @see LookupService#lookupContentUUID(MultiValueMap, String, DatawaveUserDetails, StreamingResponseListener) + */ + // @formatter:off + @Operation( + summary = "Creates a content lookup query using the query logic associated with the given uuid type(s) and parameters, and returns the first page of results.", + description = "Lookup queries will start running immediately.
" + + "Auditing is performed before the query is started.
" + + "Each of the uuid pairs must map to the same query logic.
" + + "After the first page is returned, the query will be closed.") + @ApiResponses({ + @ApiResponse( + description = "if successful, returns a base query response containing the first page of results", + responseCode = "200", + content = @Content(schema = @Schema(implementation = BaseQueryResponse.class)), + headers = { + @Header( + name = "Pool", + description = "the executor pool to target", + schema = @Schema(defaultValue = "default"))}), + @ApiResponse( + description = "if no query results are found", + responseCode = "204", + content = @Content(schema = @Schema(hidden = true))), + @ApiResponse( + description = "if parameter validation fails
" + + "if query logic parameter validation fails
" + + "if security marking validation fails
" + + "if auditing fails", + responseCode = "400", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if the user doesn't have access to the requested query logic", + responseCode = "401", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if query storage fails
" + + "if the next call times out
" + + "if the next task is rejected by the executor
" + + "if there is an unknown error", + responseCode = "500", + content = @Content(schema = @Schema(implementation = VoidResponse.class)))}) + @Parameters({ + @Parameter( + name = LOOKUP_STREAMING, + in = ParameterIn.QUERY, + description = "if true, streams all results back", + schema = @Schema(implementation = Boolean.class), + example = "true"), + @Parameter( + name = LOOKUP_CONTEXT, + in = ParameterIn.QUERY, + description = "The lookup UUID type context", + example = "default", + array = @ArraySchema(schema = @Schema(implementation = String.class))), + @Parameter( + name = QUERY_BEGIN, + in = ParameterIn.QUERY, + description = "The query begin date", + required = true, + schema = @Schema(implementation = String.class), + example = "\"19660908 000000.000\""), + @Parameter( + name = QUERY_END, + in = ParameterIn.QUERY, + description = "The query end date", + required = true, + schema = @Schema(implementation = String.class), + example = "\"20161002 235959.999\""), + @Parameter( + name = QUERY_NAME, + in = ParameterIn.QUERY, + description = "The query name", + required = true, + schema = @Schema(implementation = String.class), + example = "Developer Test Query"), + @Parameter( + name = QUERY_AUTHORIZATIONS, + in = ParameterIn.QUERY, + description = "The query auths", + required = true, + schema = @Schema(implementation = String.class), + example = "PUBLIC,PRIVATE,BAR,FOO"), + @Parameter( + name = QUERY_VISIBILITY, + in = ParameterIn.QUERY, + description = "The visibility to use when storing metrics for this query", + required = true, + schema = @Schema(implementation = String.class), + example = "PUBLIC"), + @Parameter( + name = QUERY_MAX_CONCURRENT_TASKS, + in = ParameterIn.QUERY, + description = "The max number of concurrent tasks to run for this query", + schema = @Schema(implementation = Integer.class), + example = "10"), + @Parameter( + name = QUERY_POOL, + in = ParameterIn.QUERY, + description = "The executor pool to run against", + schema = @Schema(implementation = String.class), + example = "pool1"), + @Parameter( + name = QUERY_PAGESIZE, + in = ParameterIn.QUERY, + description = "The requested page size", + schema = @Schema(implementation = Integer.class), + example = "10"), + @Parameter( + name = QUERY_PAGETIMEOUT, + in = ParameterIn.QUERY, + description = "The call timeout when requesting a page, in minutes", + schema = @Schema(implementation = Integer.class), + example = "60"), + @Parameter( + name = QUERY_MAX_RESULTS_OVERRIDE, + in = ParameterIn.QUERY, + description = "The max results override value", + schema = @Schema(implementation = Integer.class), + example = "5000"), + @Parameter( + name = QUERY_PARAMS, + in = ParameterIn.QUERY, + description = "Additional query parameters", + schema = @Schema(implementation = String.class), + example = "KEY_1:VALUE_1;KEY_2:VALUE_2") + }) + // @formatter:on + @Timed(name = "dw.query.lookupContentUUID", absolute = true) + @RequestMapping(path = "lookupContentUUID/{uuidType}/{uuid}", method = {RequestMethod.GET}, produces = {"application/xml", "text/xml", "application/json", + "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) + public Object lookupContentUUID(@Parameter(description = "The UUID type", example = "PAGE_TITLE") @PathVariable(required = false) String uuidType, + @Parameter(description = "The UUID", example = "anarchism") @PathVariable(required = false) String uuid, + @Parameter(hidden = true) @RequestParam MultiValueMap parameters, @RequestHeader HttpHeaders headers, + @AuthenticationPrincipal DatawaveUserDetails currentUser) throws QueryException { + parameters.add(LOOKUP_UUID_PAIRS, String.join(LOOKUP_KEY_VALUE_DELIMITER, uuidType, uuid)); + + if (Boolean.parseBoolean(parameters.getFirst(LOOKUP_STREAMING))) { + MediaType contentType = determineContentType(headers.getAccept(), MediaType.parseMediaType(streamingProperties.getDefaultContentType())); + CountingResponseBodyEmitter emitter = baseMethodStatsContext.createCountingResponseBodyEmitter(streamingProperties.getCallTimeoutMillis()); + lookupService.lookupContentUUID(parameters, getPool(headers), currentUser, new CountingResponseBodyEmitterListener(emitter, contentType)); + return emitter; + } else { + return lookupService.lookupContentUUID(parameters, getPool(headers), currentUser); + } + } + + /** + * @see LookupService#lookupContentUUID(MultiValueMap, String, DatawaveUserDetails) + * @see LookupService#lookupContentUUID(MultiValueMap, String, DatawaveUserDetails) + */ + // @formatter:off + @Operation( + summary = "Creates a batch content lookup query using the query logic associated with the given uuid type(s) and parameters, and returns the first page of results.", + description = "Lookup queries will start running immediately.
" + + "Auditing is performed before the query is started.
" + + "Each of the uuid pairs must map to the same query logic.
" + + "After the first page is returned, the query will be closed.") + @ApiResponses({ + @ApiResponse( + description = "if successful, returns a base query response containing the first page of results", + responseCode = "200", + content = @Content(schema = @Schema(implementation = BaseQueryResponse.class)), + headers = { + @Header( + name = "Pool", + description = "the executor pool to target", + schema = @Schema(defaultValue = "default"))}), + @ApiResponse( + description = "if no query results are found", + responseCode = "204", + content = @Content(schema = @Schema(hidden = true))), + @ApiResponse( + description = "if parameter validation fails
" + + "if query logic parameter validation fails
" + + "if security marking validation fails
" + + "if auditing fails", + responseCode = "400", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if the user doesn't have access to the requested query logic", + responseCode = "401", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if query storage fails
" + + "if the next call times out
" + + "if the next task is rejected by the executor
" + + "if there is an unknown error", + responseCode = "500", + content = @Content(schema = @Schema(implementation = VoidResponse.class)))}) + @Parameters({ + @Parameter( + name = LOOKUP_STREAMING, + in = ParameterIn.QUERY, + description = "if true, streams all results back", + schema = @Schema(implementation = Boolean.class), + example = "true"), + @Parameter( + name = LOOKUP_CONTEXT, + in = ParameterIn.QUERY, + description = "The lookup UUID type context", + example = "default", + array = @ArraySchema(schema = @Schema(implementation = String.class))), + @Parameter( + name = LOOKUP_UUID_PAIRS, + in = ParameterIn.QUERY, + description = "The lookup UUID pairs
" + + "To lookup multiple UUID pairs, submit multiples of this parameter", + required = true, + example = "PAGE_TITLE:anarchism", + array = @ArraySchema(schema = @Schema(implementation = String.class))), + @Parameter( + name = QUERY_BEGIN, + in = ParameterIn.QUERY, + description = "The query begin date", + required = true, + schema = @Schema(implementation = String.class), + example = "\"19660908 000000.000\""), + @Parameter( + name = QUERY_END, + in = ParameterIn.QUERY, + description = "The query end date", + required = true, + schema = @Schema(implementation = String.class), + example = "\"20161002 235959.999\""), + @Parameter( + name = QUERY_NAME, + in = ParameterIn.QUERY, + description = "The query name", + required = true, + schema = @Schema(implementation = String.class), + example = "Developer Test Query"), + @Parameter( + name = QUERY_AUTHORIZATIONS, + in = ParameterIn.QUERY, + description = "The query auths", + required = true, + schema = @Schema(implementation = String.class), + example = "PUBLIC,PRIVATE,BAR,FOO"), + @Parameter( + name = QUERY_VISIBILITY, + in = ParameterIn.QUERY, + description = "The visibility to use when storing metrics for this query", + required = true, + schema = @Schema(implementation = String.class), + example = "PUBLIC"), + @Parameter( + name = QUERY_MAX_CONCURRENT_TASKS, + in = ParameterIn.QUERY, + description = "The max number of concurrent tasks to run for this query", + schema = @Schema(implementation = Integer.class), + example = "10"), + @Parameter( + name = QUERY_POOL, + in = ParameterIn.QUERY, + description = "The executor pool to run against", + schema = @Schema(implementation = String.class), + example = "pool1"), + @Parameter( + name = QUERY_PAGESIZE, + in = ParameterIn.QUERY, + description = "The requested page size", + schema = @Schema(implementation = Integer.class), + example = "10"), + @Parameter( + name = QUERY_PAGETIMEOUT, + in = ParameterIn.QUERY, + description = "The call timeout when requesting a page, in minutes", + schema = @Schema(implementation = Integer.class), + example = "60"), + @Parameter( + name = QUERY_MAX_RESULTS_OVERRIDE, + in = ParameterIn.QUERY, + description = "The max results override value", + schema = @Schema(implementation = Integer.class), + example = "5000"), + @Parameter( + name = QUERY_PARAMS, + in = ParameterIn.QUERY, + description = "Additional query parameters", + schema = @Schema(implementation = String.class), + example = "KEY_1:VALUE_1;KEY_2:VALUE_2") + }) + // @formatter:on + @Timed(name = "dw.query.lookupContentUUIDBatch", absolute = true) + @RequestMapping(path = "lookupContentUUID", method = {RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", + "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) + public Object lookupContentUUIDBatch(@Parameter(hidden = true) @RequestParam MultiValueMap parameters, @RequestHeader HttpHeaders headers, + @AuthenticationPrincipal DatawaveUserDetails currentUser) throws QueryException { + if (Boolean.parseBoolean(parameters.getFirst(LOOKUP_STREAMING))) { + MediaType contentType = determineContentType(headers.getAccept(), MediaType.parseMediaType(streamingProperties.getDefaultContentType())); + CountingResponseBodyEmitter emitter = baseMethodStatsContext.createCountingResponseBodyEmitter(streamingProperties.getCallTimeoutMillis()); + lookupService.lookupContentUUID(parameters, getPool(headers), currentUser, new CountingResponseBodyEmitterListener(emitter, contentType)); + return emitter; + } else { + return lookupService.lookupContentUUID(parameters, getPool(headers), currentUser); + } + } + + /** + * @see TranslateIdService#translateId(String, MultiValueMap, String, DatawaveUserDetails) + */ + // @formatter:off + @Operation( + summary = "Get one or more ID(s), if any, that correspond to the given ID.", + description = "This method only returns the first page, so set pagesize appropriately.
" + + "Since the underlying query is automatically closed, callers are NOT expected to request additional pages or close the query.") + @ApiResponses({ + @ApiResponse( + description = "if successful, returns a base query response containing the first page of results", + responseCode = "200", + content = @Content(schema = @Schema(implementation = BaseQueryResponse.class)), + headers = { + @Header( + name = "Pool", + description = "the executor pool to target", + schema = @Schema(defaultValue = "default"))}), + @ApiResponse( + description = "if no query results are found", + responseCode = "204", + content = @Content(schema = @Schema(hidden = true))), + @ApiResponse( + description = "if parameter validation fails
" + + "if query logic parameter validation fails
" + + "if security marking validation fails
" + + "if auditing fails", + responseCode = "400", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if the user doesn't have access to the requested query logic", + responseCode = "401", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if query storage fails
" + + "if the next call times out
" + + "if the next task is rejected by the executor
" + + "if there is an unknown error", + responseCode = "500", + content = @Content(schema = @Schema(implementation = VoidResponse.class)))}) + @Parameters({ + @Parameter( + name = QUERY_BEGIN, + in = ParameterIn.QUERY, + description = "The query begin date", + required = true, + schema = @Schema(implementation = String.class), + example = "\"19660908 000000.000\""), + @Parameter( + name = QUERY_END, + in = ParameterIn.QUERY, + description = "The query end date", + required = true, + schema = @Schema(implementation = String.class), + example = "\"20161002 235959.999\""), + @Parameter( + name = QUERY_NAME, + in = ParameterIn.QUERY, + description = "The query name", + required = true, + schema = @Schema(implementation = String.class), + example = "Developer Test Query"), + @Parameter( + name = QUERY_AUTHORIZATIONS, + in = ParameterIn.QUERY, + description = "The query auths", + required = true, + schema = @Schema(implementation = String.class), + example = "PUBLIC,PRIVATE,BAR,FOO"), + @Parameter( + name = QUERY_VISIBILITY, + in = ParameterIn.QUERY, + description = "The visibility to use when storing metrics for this query", + required = true, + schema = @Schema(implementation = String.class), + example = "PUBLIC"), + @Parameter( + name = QUERY_MAX_CONCURRENT_TASKS, + in = ParameterIn.QUERY, + description = "The max number of concurrent tasks to run for this query", + schema = @Schema(implementation = Integer.class), + example = "10"), + @Parameter( + name = QUERY_POOL, + in = ParameterIn.QUERY, + description = "The executor pool to run against", + schema = @Schema(implementation = String.class), + example = "pool1"), + @Parameter( + name = QUERY_PAGESIZE, + in = ParameterIn.QUERY, + description = "The requested page size", + schema = @Schema(implementation = Integer.class), + example = "10"), + @Parameter( + name = QUERY_PAGETIMEOUT, + in = ParameterIn.QUERY, + description = "The call timeout when requesting a page, in minutes", + schema = @Schema(implementation = Integer.class), + example = "60"), + @Parameter( + name = QUERY_MAX_RESULTS_OVERRIDE, + in = ParameterIn.QUERY, + description = "The max results override value", + schema = @Schema(implementation = Integer.class), + example = "5000"), + @Parameter( + name = QUERY_PARAMS, + in = ParameterIn.QUERY, + description = "Additional query parameters", + schema = @Schema(implementation = String.class), + example = "KEY_1:VALUE_1;KEY_2:VALUE_2") + }) + // @formatter:on + @RequestMapping(path = "translateId/{id}", method = {RequestMethod.GET}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", + "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) + public BaseQueryResponse translateId(@Parameter(description = "The ID to translate") @PathVariable String id, + @Parameter(hidden = true) @RequestParam MultiValueMap parameters, @RequestHeader HttpHeaders headers, + @AuthenticationPrincipal DatawaveUserDetails currentUser) throws QueryException { + return translateIdService.translateId(id, parameters, getPool(headers), currentUser); + } + + /** + * @see TranslateIdService#translateIds(MultiValueMap, String, DatawaveUserDetails) + */ + // @formatter:off + @Operation( + summary = "Get the ID(s), if any, associated with the specified IDs.", + description = "Because the query created by this call may return multiple pages, callers are expected to request additional pages and eventually close the query.") + @ApiResponses({ + @ApiResponse( + description = "if successful, returns a base query response containing the first page of results", + responseCode = "200", + content = @Content(schema = @Schema(implementation = BaseQueryResponse.class)), + headers = { + @Header( + name = "Pool", + description = "the executor pool to target", + schema = @Schema(defaultValue = "default"))}), + @ApiResponse( + description = "if no query results are found", + responseCode = "204", + content = @Content(schema = @Schema(hidden = true))), + @ApiResponse( + description = "if parameter validation fails
" + + "if query logic parameter validation fails
" + + "if security marking validation fails
" + + "if auditing fails", + responseCode = "400", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if the user doesn't have access to the requested query logic", + responseCode = "401", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if query storage fails
" + + "if the next call times out
" + + "if the next task is rejected by the executor
" + + "if there is an unknown error", + responseCode = "500", + content = @Content(schema = @Schema(implementation = VoidResponse.class)))}) + @Parameters({ + @Parameter( + name = TRANSLATE_ID, + in = ParameterIn.QUERY, + description = "The IDs to translate", + required = true, + schema = @Schema(implementation = String.class), + example = "\"19660908 000000.000\""), + @Parameter( + name = QUERY_BEGIN, + in = ParameterIn.QUERY, + description = "The query begin date", + required = true, + schema = @Schema(implementation = String.class), + example = "\"19660908 000000.000\""), + @Parameter( + name = QUERY_END, + in = ParameterIn.QUERY, + description = "The query end date", + required = true, + schema = @Schema(implementation = String.class), + example = "\"20161002 235959.999\""), + @Parameter( + name = QUERY_NAME, + in = ParameterIn.QUERY, + description = "The query name", + required = true, + schema = @Schema(implementation = String.class), + example = "Developer Test Query"), + @Parameter( + name = QUERY_AUTHORIZATIONS, + in = ParameterIn.QUERY, + description = "The query auths", + required = true, + schema = @Schema(implementation = String.class), + example = "PUBLIC,PRIVATE,BAR,FOO"), + @Parameter( + name = QUERY_VISIBILITY, + in = ParameterIn.QUERY, + description = "The visibility to use when storing metrics for this query", + required = true, + schema = @Schema(implementation = String.class), + example = "PUBLIC"), + @Parameter( + name = QUERY_MAX_CONCURRENT_TASKS, + in = ParameterIn.QUERY, + description = "The max number of concurrent tasks to run for this query", + schema = @Schema(implementation = Integer.class), + example = "10"), + @Parameter( + name = QUERY_POOL, + in = ParameterIn.QUERY, + description = "The executor pool to run against", + schema = @Schema(implementation = String.class), + example = "pool1"), + @Parameter( + name = QUERY_PAGESIZE, + in = ParameterIn.QUERY, + description = "The requested page size", + schema = @Schema(implementation = Integer.class), + example = "10"), + @Parameter( + name = QUERY_PAGETIMEOUT, + in = ParameterIn.QUERY, + description = "The call timeout when requesting a page, in minutes", + schema = @Schema(implementation = Integer.class), + example = "60"), + @Parameter( + name = QUERY_MAX_RESULTS_OVERRIDE, + in = ParameterIn.QUERY, + description = "The max results override value", + schema = @Schema(implementation = Integer.class), + example = "5000"), + @Parameter( + name = QUERY_PARAMS, + in = ParameterIn.QUERY, + description = "Additional query parameters", + schema = @Schema(implementation = String.class), + example = "KEY_1:VALUE_1;KEY_2:VALUE_2") + }) + // @formatter:on + @RequestMapping(path = "translateIDs", method = {RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", + "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) + public BaseQueryResponse translateIDs(@Parameter(hidden = true) @RequestParam MultiValueMap parameters, @RequestHeader HttpHeaders headers, + @AuthenticationPrincipal DatawaveUserDetails currentUser) throws QueryException { + return translateIdService.translateIds(parameters, getPool(headers), currentUser); + } + + /** + * @see QueryManagementService#createAndNext(String, MultiValueMap, String, DatawaveUserDetails) + */ + // @formatter:off + @Operation( + summary = "Creates a query using the given query logic and parameters, and returns the first page of results.", + description = "Created queries will start running immediately.
" + + "Auditing is performed before the query is started.
" + + "Subsequent query results can be retrieved using next.
" + + "Updates can be made to any parameter which doesn't affect the scope of the query using update.
" + + "Stop a running query gracefully using close or forcefully using cancel.
" + + "Stop, and restart a running query using reset.
" + + "Create a copy of a running query using duplicate.
" + + "Aside from a limited set of admin actions, only the query owner can act on a running query.") + @ApiResponses({ + @ApiResponse( + description = "if successful, returns a base query response containing the first page of results", + responseCode = "200", + content = @Content(schema = @Schema(implementation = BaseQueryResponse.class)), + headers = { + @Header( + name = "Pool", + description = "the executor pool to target", + schema = @Schema(defaultValue = "default"))}), + @ApiResponse( + description = "if no query results are found", + responseCode = "204", + content = @Content(schema = @Schema(hidden = true))), + @ApiResponse( + description = "if parameter validation fails
" + + "if query logic parameter validation fails
" + + "if security marking validation fails
" + + "if auditing fails", + responseCode = "400", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if the user doesn't have access to the requested query logic", + responseCode = "401", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if query storage fails
" + + "if the next call times out
" + + "if the next task is rejected by the executor
" + + "if there is an unknown error", + responseCode = "500", + content = @Content(schema = @Schema(implementation = VoidResponse.class)))}) + @Parameters({ + @Parameter( + name = QUERY_BEGIN, + in = ParameterIn.QUERY, + description = "The query begin date", + required = true, + schema = @Schema(implementation = String.class), + example = "\"19660908 000000.000\""), + @Parameter( + name = QUERY_END, + in = ParameterIn.QUERY, + description = "The query end date", + required = true, + schema = @Schema(implementation = String.class), + example = "\"20161002 235959.999\""), + @Parameter( + name = QUERY_NAME, + in = ParameterIn.QUERY, + description = "The query name", + required = true, + schema = @Schema(implementation = String.class), + example = "Developer Test Query"), + @Parameter( + name = QUERY_STRING, + in = ParameterIn.QUERY, + description = "The query string", + required = true, + schema = @Schema(implementation = String.class), + example = "GENRES:[Action to Western]"), + @Parameter( + name = QUERY_AUTHORIZATIONS, + in = ParameterIn.QUERY, + description = "The query auths", + required = true, + schema = @Schema(implementation = String.class), + example = "PUBLIC,PRIVATE,BAR,FOO"), + @Parameter( + name = QUERY_VISIBILITY, + in = ParameterIn.QUERY, + description = "The visibility to use when storing metrics for this query", + required = true, + schema = @Schema(implementation = String.class), + example = "PUBLIC"), + @Parameter( + name = QUERY_SYNTAX, + in = ParameterIn.QUERY, + description = "The syntax used in the query", + schema = @Schema(implementation = String.class), + example = "LUCENE"), + @Parameter( + name = QUERY_MAX_CONCURRENT_TASKS, + in = ParameterIn.QUERY, + description = "The max number of concurrent tasks to run for this query", + schema = @Schema(implementation = Integer.class), + example = "10"), + @Parameter( + name = QUERY_POOL, + in = ParameterIn.QUERY, + description = "The executor pool to run against", + schema = @Schema(implementation = String.class), + example = "pool1"), + @Parameter( + name = QUERY_PAGESIZE, + in = ParameterIn.QUERY, + description = "The requested page size", + schema = @Schema(implementation = Integer.class), + example = "10"), + @Parameter( + name = QUERY_PAGETIMEOUT, + in = ParameterIn.QUERY, + description = "The call timeout when requesting a page, in minutes", + schema = @Schema(implementation = Integer.class), + example = "60"), + @Parameter( + name = QUERY_MAX_RESULTS_OVERRIDE, + in = ParameterIn.QUERY, + description = "The max results override value", + schema = @Schema(implementation = Integer.class), + example = "5000"), + @Parameter( + name = QUERY_PARAMS, + in = ParameterIn.QUERY, + description = "Additional query parameters", + schema = @Schema(implementation = String.class), + example = "KEY_1:VALUE_1;KEY_2:VALUE_2") + }) + // @formatter:on + @Timed(name = "dw.query.createAndNext", absolute = true) + @EnrichQueryMetrics(methodType = EnrichQueryMetrics.MethodType.CREATE_AND_NEXT) + @RequestMapping(path = "{queryLogic}/createAndNext", method = {RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", + "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) + public BaseQueryResponse createAndNext(@Parameter(description = "The query logic", example = "EventQuery") @PathVariable String queryLogic, + @Parameter(hidden = true) @RequestParam MultiValueMap parameters, @RequestHeader HttpHeaders headers, + @AuthenticationPrincipal DatawaveUserDetails currentUser) throws QueryException { + return queryManagementService.createAndNext(queryLogic, parameters, getPool(headers), currentUser); + } + + /** + * @see QueryManagementService#next(String, DatawaveUserDetails) + */ + // @formatter:off + @Operation( + summary = "Gets the next page of results for the specified query.", + description = "Next can only be called on a running query.
" + + "If configuration allows, multiple next calls may be run concurrently for a query.
" + + "Only the query owner can call next on the specified query.") + @ApiResponses({ + @ApiResponse( + description = "if successful, returns a base query response containing the next page of results", + responseCode = "200", + content = @Content(schema = @Schema(implementation = BaseQueryResponse.class))), + @ApiResponse( + description = "if no query results are found", + responseCode = "204", + content = @Content(schema = @Schema(hidden = true))), + @ApiResponse( + description = "if the query is not running", + responseCode = "400", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if the user doesn't own the query", + responseCode = "401", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if the query cannot be found", + responseCode = "404", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if query lock acquisition fails
" + + "if the next call is interrupted
" + + "if the query times out
" + + "if the next task is rejected by the executor
" + + "if next call execution fails
" + + "if query logic creation fails
" + + "if there is an unknown error", + responseCode = "500", + content = @Content(schema = @Schema(implementation = VoidResponse.class)))}) + // @formatter:on + @Timed(name = "dw.query.next", absolute = true) + @EnrichQueryMetrics(methodType = EnrichQueryMetrics.MethodType.NEXT) + @RequestMapping(path = "{queryId}/next", method = {RequestMethod.GET}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", + "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) + public BaseQueryResponse next(@Parameter(description = "The query ID") @PathVariable String queryId, + @AuthenticationPrincipal DatawaveUserDetails currentUser) throws QueryException { + return queryManagementService.next(queryId, currentUser); + } + + /** + * @see QueryManagementService#cancel(String, DatawaveUserDetails) + */ + // @formatter:off + @Operation( + summary = "Cancels the specified query.", + description = "Cancel can only be called on a running query, or a query that is in the process of closing.
" + + "Outstanding next calls will be stopped immediately, but will return partial results if applicable.
" + + "Aside from admins, only the query owner can cancel the specified query.") + @ApiResponses({ + @ApiResponse( + description = "if successful, returns a void response indicating that the query was canceled", + responseCode = "200", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if the query is not running", + responseCode = "400", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if the user doesn't own the query", + responseCode = "401", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if the query cannot be found", + responseCode = "404", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if query lock acquisition fails
" + + "if the cancel call is interrupted
" + + "if there is an unknown error", + responseCode = "500", + content = @Content(schema = @Schema(implementation = VoidResponse.class)))}) + // @formatter:on + @Timed(name = "dw.query.cancel", absolute = true) + @RequestMapping(path = "{queryId}/cancel", method = {RequestMethod.PUT, RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", + "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) + public VoidResponse cancel(@Parameter(description = "The query ID") @PathVariable String queryId, @AuthenticationPrincipal DatawaveUserDetails currentUser) + throws QueryException { + return queryManagementService.cancel(queryId, currentUser); + } + + /** + * @see QueryManagementService#adminCancel(String, DatawaveUserDetails) + */ + // @formatter:off + @Operation( + summary = "Cancels the specified query using admin privileges.", + description = "Cancel can only be called on a running query, or a query that is in the process of closing.
" + + "Outstanding next calls will be stopped immediately, but will return partial results if applicable.
" + + "Only admin users should be allowed to call this method.") + @ApiResponses({ + @ApiResponse( + description = "if successful, returns a void response indicating that the query was canceled", + responseCode = "200", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if the query is not running", + responseCode = "400", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if the user doesn't own the query", + responseCode = "401", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if the query cannot be found", + responseCode = "404", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if query lock acquisition fails
" + + "if the cancel call is interrupted
" + + "if there is an unknown error", + responseCode = "500", + content = @Content(schema = @Schema(implementation = VoidResponse.class)))}) + // @formatter:on + @Timed(name = "dw.query.adminCancel", absolute = true) + @Secured({"Administrator", "JBossAdministrator"}) + @RequestMapping(path = "{queryId}/adminCancel", method = {RequestMethod.PUT, RequestMethod.POST}, produces = {"application/xml", "text/xml", + "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) + public VoidResponse adminCancel(@Parameter(description = "The query ID") @PathVariable String queryId, + @AuthenticationPrincipal DatawaveUserDetails currentUser) throws QueryException { + return queryManagementService.adminCancel(queryId, currentUser); + } + + /** + * @see QueryManagementService#close(String, DatawaveUserDetails) + */ + // @formatter:off + @Operation( + summary = "Closes the specified query.", + description = "Close can only be called on a running query.
" + + "Outstanding next calls will be allowed to run until they can return a full page, or they timeout.
" + + "Aside from admins, only the query owner can close the specified query.") + @ApiResponses({ + @ApiResponse( + description = "if successful, returns a void response indicating that the query was closed", + responseCode = "200", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if the query is not running", + responseCode = "400", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if the user doesn't own the query", + responseCode = "401", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if the query cannot be found", + responseCode = "404", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if query lock acquisition fails
" + + "if the close call is interrupted
" + + "if there is an unknown error", + responseCode = "500", + content = @Content(schema = @Schema(implementation = VoidResponse.class)))}) + // @formatter:on + @Timed(name = "dw.query.close", absolute = true) + @RequestMapping(path = "{queryId}/close", method = {RequestMethod.PUT, RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", + "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) + public VoidResponse close(@Parameter(description = "The query ID") @PathVariable String queryId, @AuthenticationPrincipal DatawaveUserDetails currentUser) + throws QueryException { + return queryManagementService.close(queryId, currentUser); + } + + /** + * @see QueryManagementService#adminClose(String, DatawaveUserDetails) + */ + // @formatter:off + @Operation( + summary = "Closes the specified query using admin privileges.", + description = "Close can only be called on a running query.
" + + "Outstanding next calls will be allowed to run until they can return a full page, or they timeout.
" + + "Only admin users should be allowed to call this method.") + @ApiResponses({ + @ApiResponse( + description = "if successful, returns a void response indicating that the query was closed", + responseCode = "200", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if the query is not running", + responseCode = "400", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if the user doesn't own the query", + responseCode = "401", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if the query cannot be found", + responseCode = "404", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if query lock acquisition fails
" + + "if the close call is interrupted
" + + "if there is an unknown error", + responseCode = "500", + content = @Content(schema = @Schema(implementation = VoidResponse.class)))}) + // @formatter:on + @Timed(name = "dw.query.adminClose", absolute = true) + @Secured({"Administrator", "JBossAdministrator"}) + @RequestMapping(path = "{queryId}/adminClose", method = {RequestMethod.PUT, RequestMethod.POST}, produces = {"application/xml", "text/xml", + "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) + public VoidResponse adminClose(@Parameter(description = "The query ID") @PathVariable String queryId, + @AuthenticationPrincipal DatawaveUserDetails currentUser) throws QueryException { + return queryManagementService.adminClose(queryId, currentUser); + } + + /** + * @see QueryManagementService#reset(String, DatawaveUserDetails) + */ + // @formatter:off + @Operation( + summary = "Stops, and restarts the specified query.", + description = "Reset can be called on any query, whether it's running or not.
" + + "If the specified query is still running, it will be canceled. See cancel.
" + + "Reset creates a new, identical query, with a new query id.
" + + "Reset queries will start running immediately.
" + + "Auditing is performed before the new query is started.") + @ApiResponses({ + @ApiResponse( + description = "if successful, returns a generic response containing the new query id", + responseCode = "200", + content = @Content(schema = @Schema(implementation = BaseQueryResponse.class))), + @ApiResponse( + description = "if parameter validation fails
" + + "if query logic parameter validation fails
" + + "if security marking validation fails
" + + "if auditing fails", + responseCode = "400", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if the user doesn't own the query
" + + "if the user doesn't have access to the requested query logic", + responseCode = "401", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if the query cannot be found", + responseCode = "404", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if query lock acquisition fails
" + + "if the cancel call is interrupted
" + + "if query storage fails
" + + "if there is an unknown error", + responseCode = "500", + content = @Content(schema = @Schema(implementation = VoidResponse.class)))}) + // @formatter:on + @Timed(name = "dw.query.reset", absolute = true) + @RequestMapping(path = "{queryId}/reset", method = {RequestMethod.PUT, RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", + "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) + public GenericResponse reset(@Parameter(description = "The query ID") @PathVariable String queryId, + @AuthenticationPrincipal DatawaveUserDetails currentUser) throws QueryException { + return queryManagementService.reset(queryId, currentUser); + } + + /** + * @see QueryManagementService#remove(String, DatawaveUserDetails) + */ + // @formatter:off + @Operation( + summary = "Removes the specified query from query storage.", + description = "Remove can only be called on a query that is not running.
" + + "Aside from admins, only the query owner can remove the specified query.") + @ApiResponses({ + @ApiResponse( + description = "if successful, returns a void response indicating that the query was removed", + responseCode = "200", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if the query is running", + responseCode = "400", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if the user doesn't own the query", + responseCode = "401", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if the query cannot be found", + responseCode = "404", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if there is an unknown error", + responseCode = "500", + content = @Content(schema = @Schema(implementation = VoidResponse.class)))}) + // @formatter:on + @Timed(name = "dw.query.remove", absolute = true) + @RequestMapping(path = "{queryId}/remove", method = {RequestMethod.DELETE}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", + "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) + public VoidResponse remove(@Parameter(description = "The query ID") @PathVariable String queryId, @AuthenticationPrincipal DatawaveUserDetails currentUser) + throws QueryException { + return queryManagementService.remove(queryId, currentUser); + } + + /** + * @see QueryManagementService#adminRemove(String, DatawaveUserDetails) + */ + // @formatter:off + @Operation( + summary = "Removes the specified query from query storage using admin privileges.", + description = "Remove can only be called on a query that is not running.
" + + "Only admin users should be allowed to call this method.") + @ApiResponses({ + @ApiResponse( + description = "if successful, returns a void response indicating that the query was removed", + responseCode = "200", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if the query is running", + responseCode = "400", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if the query cannot be found", + responseCode = "404", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if there is an unknown error", + responseCode = "500", + content = @Content(schema = @Schema(implementation = VoidResponse.class)))}) + // @formatter:on + @Timed(name = "dw.query.adminRemove", absolute = true) + @Secured({"Administrator", "JBossAdministrator"}) + @RequestMapping(path = "{queryId}/adminRemove", method = {RequestMethod.DELETE}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", + "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) + public VoidResponse adminRemove(@Parameter(description = "The query ID") @PathVariable String queryId, + @AuthenticationPrincipal DatawaveUserDetails currentUser) throws QueryException { + return queryManagementService.adminRemove(queryId, currentUser); + } + + /** + * @see QueryManagementService#update(String, MultiValueMap, DatawaveUserDetails) + */ + // @formatter:off + @Operation( + summary = "Updates the specified query.", + description = "Update can only be called on a defined, or running query.
" + + "Auditing is not performed when updating a defined query.
" + + "No auditable parameters should be updated when updating a running query.
" + + "Any query parameter can be updated for a defined query.
" + + "Query parameters which don't affect the scope of the query can be updated for a running query.
" + + "The list of parameters that can be updated for a running query is configurable.
" + + "Auditable parameters should never be added to the updatable parameters configuration.
" + + "Query string, date range, query logic, and auths should never be updated for a running query.
" + + "Only the query owner can call update on the specified query.") + @ApiResponses({ + @ApiResponse( + description = "if successful, returns a generic response containing the query id", + responseCode = "200", + content = @Content(schema = @Schema(implementation = GenericResponse.class))), + @ApiResponse( + description = "if parameter validation fails
" + + "if query logic parameter validation fails
" + + "if security marking validation fails
" + + "if the query is not defined, or running
" + + " if no parameters are specified", + responseCode = "400", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if the user doesn't own the query
" + + "if the user doesn't have access to the requested query logic", + responseCode = "401", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if the query cannot be found", + responseCode = "404", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if query lock acquisition fails
" + + "if the update call is interrupted
" + + "if query storage fails
" + + "if there is an unknown error", + responseCode = "500", + content = @Content(schema = @Schema(implementation = VoidResponse.class)))}) + @Parameters({ + @Parameter( + name = QUERY_BEGIN, + in = ParameterIn.QUERY, + description = "The query begin date", + schema = @Schema(implementation = String.class), + example = "\"19660908 000000.000\""), + @Parameter( + name = QUERY_END, + in = ParameterIn.QUERY, + description = "The query end date", + schema = @Schema(implementation = String.class), + example = "\"20161002 235959.999\""), + @Parameter( + name = QUERY_NAME, + in = ParameterIn.QUERY, + description = "The query name", + schema = @Schema(implementation = String.class), + example = "Developer Test Query"), + @Parameter( + name = QUERY_STRING, + in = ParameterIn.QUERY, + description = "The query string", + schema = @Schema(implementation = String.class), + example = "GENRES:[Action to Western]"), + @Parameter( + name = QUERY_AUTHORIZATIONS, + in = ParameterIn.QUERY, + description = "The query auths", + schema = @Schema(implementation = String.class), + example = "PUBLIC,PRIVATE,BAR,FOO"), + @Parameter( + name = QUERY_VISIBILITY, + in = ParameterIn.QUERY, + description = "The visibility to use when storing metrics for this query", + schema = @Schema(implementation = String.class), + example = "PUBLIC"), + @Parameter( + name = QUERY_SYNTAX, + in = ParameterIn.QUERY, + description = "The syntax used in the query", + schema = @Schema(implementation = String.class), + example = "LUCENE"), + @Parameter( + name = QUERY_MAX_CONCURRENT_TASKS, + in = ParameterIn.QUERY, + description = "The max number of concurrent tasks to run for this query", + schema = @Schema(implementation = Integer.class), + example = "10"), + @Parameter( + name = QUERY_POOL, + in = ParameterIn.QUERY, + description = "The executor pool to run against", + schema = @Schema(implementation = String.class), + example = "pool1"), + @Parameter( + name = QUERY_PAGESIZE, + in = ParameterIn.QUERY, + description = "The requested page size", + schema = @Schema(implementation = Integer.class), + example = "10"), + @Parameter( + name = QUERY_PAGETIMEOUT, + in = ParameterIn.QUERY, + description = "The call timeout when requesting a page, in minutes", + schema = @Schema(implementation = Integer.class), + example = "60"), + @Parameter( + name = QUERY_MAX_RESULTS_OVERRIDE, + in = ParameterIn.QUERY, + description = "The max results override value", + schema = @Schema(implementation = Integer.class), + example = "5000"), + @Parameter( + name = QUERY_PARAMS, + in = ParameterIn.QUERY, + description = "Additional query parameters", + schema = @Schema(implementation = String.class), + example = "KEY_1:VALUE_1;KEY_2:VALUE_2") + }) + // @formatter:on + @Timed(name = "dw.query.update", absolute = true) + @RequestMapping(path = "{queryId}/update", method = {RequestMethod.PUT, RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", + "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) + public GenericResponse update(@Parameter(description = "The query ID") @PathVariable String queryId, + @Parameter(hidden = true) @RequestParam MultiValueMap parameters, @AuthenticationPrincipal DatawaveUserDetails currentUser) + throws QueryException { + return queryManagementService.update(queryId, parameters, currentUser); + } + + /** + * @see QueryManagementService#duplicate(String, MultiValueMap, DatawaveUserDetails) + */ + // @formatter:off + @Operation( + summary = "Creates a copy of the specified query.", + description = "Duplicate can be called on any query, whether it's running or not.
" + + "Duplicate creates a new, identical query, with a new query id.
" + + "Provided parameter updates will be applied to the new query.
" + + "Duplicated queries will start running immediately.
" + + "Auditing is performed before the new query is started.") + @ApiResponses({ + @ApiResponse( + description = "if successful, returns a generic response containing the query id", + responseCode = "200", + content = @Content(schema = @Schema(implementation = GenericResponse.class))), + @ApiResponse( + description = "if parameter validation fails
" + + "if query logic parameter validation fails
" + + "if security marking validation fails
" + + "if auditing fails", + responseCode = "400", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if the user doesn't own the query
" + + "if the user doesn't have access to the requested query logic", + responseCode = "401", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if the query cannot be found", + responseCode = "404", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if query lock acquisition fails
" + + "if query storage fails
" + + "if there is an unknown error", + responseCode = "500", + content = @Content(schema = @Schema(implementation = VoidResponse.class)))}) + @Parameters({ + @Parameter( + name = QUERY_BEGIN, + in = ParameterIn.QUERY, + description = "The query begin date", + schema = @Schema(implementation = String.class), + example = "\"19660908 000000.000\""), + @Parameter( + name = QUERY_END, + in = ParameterIn.QUERY, + description = "The query end date", + schema = @Schema(implementation = String.class), + example = "\"20161002 235959.999\""), + @Parameter( + name = QUERY_NAME, + in = ParameterIn.QUERY, + description = "The query name", + schema = @Schema(implementation = String.class), + example = "Developer Test Query"), + @Parameter( + name = QUERY_STRING, + in = ParameterIn.QUERY, + description = "The query string", + schema = @Schema(implementation = String.class), + example = "GENRES:[Action to Western]"), + @Parameter( + name = QUERY_AUTHORIZATIONS, + in = ParameterIn.QUERY, + description = "The query auths", + schema = @Schema(implementation = String.class), + example = "PUBLIC,PRIVATE,BAR,FOO"), + @Parameter( + name = QUERY_VISIBILITY, + in = ParameterIn.QUERY, + description = "The visibility to use when storing metrics for this query", + schema = @Schema(implementation = String.class), + example = "PUBLIC"), + @Parameter( + name = QUERY_SYNTAX, + in = ParameterIn.QUERY, + description = "The syntax used in the query", + schema = @Schema(implementation = String.class), + example = "LUCENE"), + @Parameter( + name = QUERY_MAX_CONCURRENT_TASKS, + in = ParameterIn.QUERY, + description = "The max number of concurrent tasks to run for this query", + schema = @Schema(implementation = Integer.class), + example = "10"), + @Parameter( + name = QUERY_POOL, + in = ParameterIn.QUERY, + description = "The executor pool to run against", + schema = @Schema(implementation = String.class), + example = "pool1"), + @Parameter( + name = QUERY_PAGESIZE, + in = ParameterIn.QUERY, + description = "The requested page size", + schema = @Schema(implementation = Integer.class), + example = "10"), + @Parameter( + name = QUERY_PAGETIMEOUT, + in = ParameterIn.QUERY, + description = "The call timeout when requesting a page, in minutes", + schema = @Schema(implementation = Integer.class), + example = "60"), + @Parameter( + name = QUERY_MAX_RESULTS_OVERRIDE, + in = ParameterIn.QUERY, + description = "The max results override value", + schema = @Schema(implementation = Integer.class), + example = "5000"), + @Parameter( + name = QUERY_PARAMS, + in = ParameterIn.QUERY, + description = "Additional query parameters", + schema = @Schema(implementation = String.class), + example = "KEY_1:VALUE_1;KEY_2:VALUE_2") + }) + // @formatter:on + @Timed(name = "dw.query.duplicate", absolute = true) + @RequestMapping(path = "{queryId}/duplicate", method = {RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", + "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) + public GenericResponse duplicate(@Parameter(description = "The query ID") @PathVariable String queryId, + @Parameter(hidden = true) @RequestParam MultiValueMap parameters, @AuthenticationPrincipal DatawaveUserDetails currentUser) + throws QueryException { + return queryManagementService.duplicate(queryId, parameters, currentUser); + } + + /** + * @see QueryManagementService#list(String, String, DatawaveUserDetails) + */ + // @formatter:off + @Operation( + summary = "Gets a list of queries for the calling user.", + description = "Returns all matching queries owned by the calling user, filtering by query id and query name.") + @ApiResponses({ + @ApiResponse( + description = "if successful, returns a list response containing the matching queries", + responseCode = "200", + content = @Content(schema = @Schema(implementation = QueryImplListResponse.class))), + @ApiResponse( + description = "if there is an unknown error", + responseCode = "500", + content = @Content(schema = @Schema(implementation = VoidResponse.class)))}) + // @formatter:on + @Timed(name = "dw.query.list", absolute = true) + @RequestMapping(path = "list", method = {RequestMethod.GET}, produces = {"text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", + "application/x-protobuf", "application/x-protostuff"}) + public QueryImplListResponse list(@Parameter(description = "The query ID") @RequestParam(required = false) String queryId, + @Parameter(description = "The query name") @RequestParam(required = false) String queryName, + @AuthenticationPrincipal DatawaveUserDetails currentUser) throws QueryException { + return queryManagementService.list(queryId, queryName, currentUser); + } + + /** + * @see QueryManagementService#adminList(String, String, String, DatawaveUserDetails) + */ + // @formatter:off + @Operation( + summary = "Gets a list of queries for the specified user using admin privileges.", + description = "Returns all matching queries owned by any user, filtered by user ID, query ID, and query name.
" + + "Only admin users should be allowed to call this method.") + @ApiResponses({ + @ApiResponse( + description = "if successful, returns a list response containing the matching queries", + responseCode = "200", + content = @Content(schema = @Schema(implementation = QueryImplListResponse.class))), + @ApiResponse( + description = "if there is an unknown error", + responseCode = "500", + content = @Content(schema = @Schema(implementation = VoidResponse.class)))}) + // @formatter:on + @Timed(name = "dw.query.adminList", absolute = true) + @Secured({"Administrator", "JBossAdministrator"}) + @RequestMapping(path = "adminList", method = {RequestMethod.GET}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", + "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) + public QueryImplListResponse adminList(@Parameter(description = "The query ID") @RequestParam(required = false) String queryId, + @Parameter(description = "The user id") @RequestParam(required = false) String user, + @Parameter(description = "The query name") @RequestParam(required = false) String queryName, + @AuthenticationPrincipal DatawaveUserDetails currentUser) throws QueryException { + return queryManagementService.adminList(queryId, queryName, user, currentUser); + } + + /** + * @see QueryManagementService#list(String, String, DatawaveUserDetails) + */ + // @formatter:off + @Operation( + summary = "Gets the matching query for the calling user.", + description = "Returns all matching queries owned by the calling user, filtering by query id.") + @ApiResponses({ + @ApiResponse( + description = "if successful, returns a list response containing the matching query", + responseCode = "200", + content = @Content(schema = @Schema(implementation = QueryImplListResponse.class))), + @ApiResponse( + description = "if there is an unknown error", + responseCode = "500", + content = @Content(schema = @Schema(implementation = VoidResponse.class)))}) + // @formatter:on + @Timed(name = "dw.query.get", absolute = true) + @RequestMapping(path = "{queryId}", method = {RequestMethod.GET}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", + "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) + public QueryImplListResponse get(@Parameter(description = "The query ID") @PathVariable String queryId, + @AuthenticationPrincipal DatawaveUserDetails currentUser) throws QueryException { + return queryManagementService.list(queryId, null, currentUser); + } + + /** + * @see QueryManagementService#plan(String, DatawaveUserDetails) + */ + // @formatter:off + @Operation( + summary = "Gets the plan for the given query for the calling user.") + @ApiResponses({ + @ApiResponse( + description = "if successful, returns the query plan for the matching query", + responseCode = "200", + content = @Content(schema = @Schema(implementation = GenericResponse.class))), + @ApiResponse( + description = "if the user doesn't own the query", + responseCode = "401", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if the query cannot be found", + responseCode = "404", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if query lock acquisition fails
" + + "if query storage fails
" + + "if there is an unknown error", + responseCode = "500", + content = @Content(schema = @Schema(implementation = VoidResponse.class)))}) + // @formatter:on + @Timed(name = "dw.query.plan", absolute = true) + @RequestMapping(path = "{queryId}/plan", method = {RequestMethod.GET}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", + "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) + public GenericResponse plan(@Parameter(description = "The query ID") @PathVariable String queryId, + @AuthenticationPrincipal DatawaveUserDetails currentUser) throws QueryException { + return queryManagementService.plan(queryId, currentUser); + } + + /** + * @see QueryManagementService#predictions(String, DatawaveUserDetails) + */ + // @formatter:off + @Operation( + summary = "Gets the predictions for the given query for the calling user.") + @ApiResponses({ + @ApiResponse( + description = "if successful, returns the query predictions for the matching query", + responseCode = "200", + content = @Content(schema = @Schema(implementation = GenericResponse.class))), + @ApiResponse( + description = "if the user doesn't own the query", + responseCode = "401", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if the query cannot be found", + responseCode = "404", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if query lock acquisition fails
" + + "if query storage fails
" + + "if there is an unknown error", + responseCode = "500", + content = @Content(schema = @Schema(implementation = VoidResponse.class)))}) + // @formatter:on + @Timed(name = "dw.query.predictions", absolute = true) + @RequestMapping(path = "{queryId}/predictions", method = {RequestMethod.GET}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", + "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) + public GenericResponse predictions(@Parameter(description = "The query ID") @PathVariable String queryId, + @AuthenticationPrincipal DatawaveUserDetails currentUser) throws QueryException { + return queryManagementService.predictions(queryId, currentUser); + } + + /** + * @see QueryManagementService#adminCancelAll(DatawaveUserDetails) + */ + // @formatter:off + @Operation( + summary = "Cancels all queries using admin privileges.", + description = "Cancel can only be called on a running query, or a query that is in the process of closing.
" + + "Queries that are not running will be ignored by this method.
" + + "Outstanding next calls will be stopped immediately, but will return partial results if applicable.
" + + "Only admin users should be allowed to call this method.") + @ApiResponses({ + @ApiResponse( + description = "if successful, returns a void response specifying which queries were canceled", + responseCode = "200", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if a query cannot be found", + responseCode = "404", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if query lock acquisition fails
" + + "if the cancel call is interrupted
" + + "if there is an unknown error", + responseCode = "500", + content = @Content(schema = @Schema(implementation = VoidResponse.class)))}) + // @formatter:on + @Timed(name = "dw.query.adminCancelAll", absolute = true) + @Secured({"Administrator", "JBossAdministrator"}) + @RequestMapping(path = "adminCancelAll", method = {RequestMethod.PUT, RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", + "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) + public VoidResponse adminCancelAll(@AuthenticationPrincipal DatawaveUserDetails currentUser) throws QueryException { + return queryManagementService.adminCancelAll(currentUser); + } + + /** + * @see QueryManagementService#adminCloseAll(DatawaveUserDetails) + */ + // @formatter:off + @Operation( + summary = "Closes all queries using admin privileges.", + description = "Close can only be called on a running query.
" + + "Queries that are not running will be ignored by this method.
" + + "Outstanding next calls will be allowed to run until they can return a full page, or they timeout.
" + + "Only admin users should be allowed to call this method.") + @ApiResponses({ + @ApiResponse( + description = "if successful, returns a void response specifying which queries were closed", + responseCode = "200", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if a query cannot be found", + responseCode = "404", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if query lock acquisition fails
" + + "if the close call is interrupted
" + + "if there is an unknown error", + responseCode = "500", + content = @Content(schema = @Schema(implementation = VoidResponse.class)))}) + // @formatter:on + @Timed(name = "dw.query.adminCloseAll", absolute = true) + @Secured({"Administrator", "JBossAdministrator"}) + @RequestMapping(path = "adminCloseAll", method = {RequestMethod.PUT, RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", + "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) + public VoidResponse adminCloseAll(@AuthenticationPrincipal DatawaveUserDetails currentUser) throws QueryException { + return queryManagementService.adminCloseAll(currentUser); + } + + /** + * @see QueryManagementService#adminRemoveAll(DatawaveUserDetails) + */ + // @formatter:off + @Operation( + summary = "Removes all queries from query storage using admin privileges.", + description = "Remove can only be called on a query that is not running.
" + + "Queries that are running will be ignored by this method.
" + + "Only admin users should be allowed to call this method.") + @ApiResponses({ + @ApiResponse( + description = "if successful, returns a void response specifying which queries were removed", + responseCode = "200", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if a query cannot be found", + responseCode = "404", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if there is an unknown error", + responseCode = "500", + content = @Content(schema = @Schema(implementation = VoidResponse.class)))}) + // @formatter:on + @Timed(name = "dw.query.adminRemoveAll", absolute = true) + @Secured({"Administrator", "JBossAdministrator"}) + @RequestMapping(path = "adminRemoveAll", method = {RequestMethod.DELETE}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", + "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) + public VoidResponse adminRemoveAll(@AuthenticationPrincipal DatawaveUserDetails currentUser) throws QueryException { + return queryManagementService.adminRemoveAll(currentUser); + } + + /** + * @see StreamingService#createAndExecute(String, MultiValueMap, String, DatawaveUserDetails, DatawaveUserDetails, StreamingResponseListener) + */ + // @formatter:off + @Operation( + summary = "Creates a query using the given query logic and parameters, and streams back all pages of results.", + description = "Created queries will start running immediately.
" + + "Auditing is performed before the query is started.
" + + "Stop a running query gracefully using close or forcefully using cancel.
" + + "Stop, and restart a running query using reset.
" + + "Create a copy of a running query using duplicate.
" + + "Aside from a limited set of admin actions, only the query owner can act on a running query.") + @ApiResponses({ + @ApiResponse( + description = "if successful, returns multiple base query responses containing pages of results", + responseCode = "200", + content = @Content(array = @ArraySchema(schema = @Schema(implementation = BaseQueryResponse.class))), + headers = { + @Header( + name = "Pool", + description = "the executor pool to target", + schema = @Schema(defaultValue = "default"))}), + @ApiResponse( + description = "if no query results are found", + responseCode = "204", + content = @Content(schema = @Schema(hidden = true))), + @ApiResponse( + description = "if parameter validation fails
" + + "if query logic parameter validation fails
" + + "if security marking validation fails
" + + "if auditing fails", + responseCode = "400", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if the user doesn't have access to the requested query logic", + responseCode = "401", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if query storage fails
" + + "if there is an unknown error", + responseCode = "500", + content = @Content(schema = @Schema(implementation = VoidResponse.class)))}) + @Parameters({ + @Parameter( + name = QUERY_BEGIN, + in = ParameterIn.QUERY, + description = "The query begin date", + required = true, + schema = @Schema(implementation = String.class), + example = "\"19660908 000000.000\""), + @Parameter( + name = QUERY_END, + in = ParameterIn.QUERY, + description = "The query end date", + required = true, + schema = @Schema(implementation = String.class), + example = "\"20161002 235959.999\""), + @Parameter( + name = QUERY_NAME, + in = ParameterIn.QUERY, + description = "The query name", + required = true, + schema = @Schema(implementation = String.class), + example = "Developer Test Query"), + @Parameter( + name = QUERY_STRING, + in = ParameterIn.QUERY, + description = "The query string", + required = true, + schema = @Schema(implementation = String.class), + example = "GENRES:[Action to Western]"), + @Parameter( + name = QUERY_AUTHORIZATIONS, + in = ParameterIn.QUERY, + description = "The query auths", + required = true, + schema = @Schema(implementation = String.class), + example = "PUBLIC,PRIVATE,BAR,FOO"), + @Parameter( + name = QUERY_VISIBILITY, + in = ParameterIn.QUERY, + description = "The visibility to use when storing metrics for this query", + required = true, + schema = @Schema(implementation = String.class), + example = "PUBLIC"), + @Parameter( + name = QUERY_SYNTAX, + in = ParameterIn.QUERY, + description = "The syntax used in the query", + schema = @Schema(implementation = String.class), + example = "LUCENE"), + @Parameter( + name = QUERY_MAX_CONCURRENT_TASKS, + in = ParameterIn.QUERY, + description = "The max number of concurrent tasks to run for this query", + schema = @Schema(implementation = Integer.class), + example = "10"), + @Parameter( + name = QUERY_POOL, + in = ParameterIn.QUERY, + description = "The executor pool to run against", + schema = @Schema(implementation = String.class), + example = "pool1"), + @Parameter( + name = QUERY_PAGESIZE, + in = ParameterIn.QUERY, + description = "The requested page size", + schema = @Schema(implementation = Integer.class), + example = "10"), + @Parameter( + name = QUERY_PAGETIMEOUT, + in = ParameterIn.QUERY, + description = "The call timeout when requesting a page, in minutes", + schema = @Schema(implementation = Integer.class), + example = "60"), + @Parameter( + name = QUERY_MAX_RESULTS_OVERRIDE, + in = ParameterIn.QUERY, + description = "The max results override value", + schema = @Schema(implementation = Integer.class), + example = "5000"), + @Parameter( + name = QUERY_PARAMS, + in = ParameterIn.QUERY, + description = "Additional query parameters", + schema = @Schema(implementation = String.class), + example = "KEY_1:VALUE_1;KEY_2:VALUE_2") + }) + // @formatter:on + @Timed(name = "dw.query.createAndExecuteQuery", absolute = true) + @RequestMapping(path = "{queryLogic}/createAndExecute", method = {RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", + "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) + public ResponseEntity createAndExecute( + @Parameter(description = "The query logic", example = "EventQuery") @PathVariable String queryLogic, + @Parameter(hidden = true) @RequestParam MultiValueMap parameters, @RequestHeader HttpHeaders headers, + @AuthenticationPrincipal DatawaveUserDetails currentUser) throws QueryException { + MediaType contentType = determineContentType(headers.getAccept(), MediaType.parseMediaType(streamingProperties.getDefaultContentType())); + CountingResponseBodyEmitter emitter = baseMethodStatsContext.createCountingResponseBodyEmitter(streamingProperties.getCallTimeoutMillis()); + String queryId = streamingService.createAndExecute(queryLogic, parameters, getPool(headers), currentUser, serverUserDetailsSupplier.get(), + new CountingResponseBodyEmitterListener(emitter, contentType)); + + // unfortunately this needs to be set manually. ResponseBodyAdvice does not run for streaming endpoints + queryMetricsEnrichmentContext.setMethodType(EnrichQueryMetrics.MethodType.CREATE); + queryMetricsEnrichmentContext.setQueryId(queryId); + + return createStreamingResponse(emitter, contentType); + } + + /** + * @see StreamingService#execute(String, DatawaveUserDetails, DatawaveUserDetails, StreamingResponseListener) + */ + // @formatter:off + @Operation( + summary = "Gets all pages of results for the given query and streams them back.", + description = "Execute can only be called on a running query.
" + + "Only the query owner can call execute on the specified query.") + @ApiResponses({ + @ApiResponse( + description = "if successful, returns multiple base query responses containing pages of results", + responseCode = "200", + content = @Content(array = @ArraySchema(schema = @Schema(implementation = BaseQueryResponse.class)))), + @ApiResponse( + description = "if no query results are found", + responseCode = "204", + content = @Content(schema = @Schema(hidden = true))), + @ApiResponse( + description = "if the query is not running", + responseCode = "400", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if the user doesn't own the query", + responseCode = "401", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if the query cannot be found", + responseCode = "404", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if query lock acquisition fails
" + + "if the execute call is interrupted
" + + "if the query times out
" + + "if the next task is rejected by the executor
" + + "if next call execution fails
" + + "if query logic creation fails
" + + "if there is an unknown error", + responseCode = "500", + content = @Content(schema = @Schema(implementation = VoidResponse.class)))}) + // @formatter:on + @Timed(name = "dw.query.executeQuery", absolute = true) + @RequestMapping(path = "{queryId}/execute", method = {RequestMethod.GET}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", + "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) + public ResponseEntity execute(@Parameter(description = "The query ID") @PathVariable String queryId, + @AuthenticationPrincipal DatawaveUserDetails currentUser, @RequestHeader HttpHeaders headers) { + MediaType contentType = determineContentType(headers.getAccept(), MediaType.parseMediaType(streamingProperties.getDefaultContentType())); + CountingResponseBodyEmitter emitter = baseMethodStatsContext.createCountingResponseBodyEmitter(streamingProperties.getCallTimeoutMillis()); + streamingService.execute(queryId, currentUser, serverUserDetailsSupplier.get(), new CountingResponseBodyEmitterListener(emitter, contentType)); + + return createStreamingResponse(emitter, contentType); + } + + private MediaType determineContentType(List acceptedMediaTypes, MediaType defaultMediaType) { + MediaType mediaType = null; + + if (acceptedMediaTypes != null && !acceptedMediaTypes.isEmpty()) { + MediaType.sortBySpecificityAndQuality(acceptedMediaTypes); + mediaType = acceptedMediaTypes.get(0); + } + + if (mediaType == null || MediaType.ALL.equals(mediaType)) { + mediaType = defaultMediaType; + } + + return mediaType; + } + + private String getPool(HttpHeaders headers) { + return headers.getFirst("Pool"); + } + + private ResponseEntity createStreamingResponse(ResponseBodyEmitter emitter, MediaType contentType) { + HttpHeaders responseHeaders = new HttpHeaders(); + responseHeaders.setContentType(contentType); + return new ResponseEntity<>(emitter, responseHeaders, HttpStatus.OK); + } +} diff --git a/service/src/main/java/datawave/microservice/query/QueryManagementService.java b/service/src/main/java/datawave/microservice/query/QueryManagementService.java new file mode 100644 index 00000000..07b9167f --- /dev/null +++ b/service/src/main/java/datawave/microservice/query/QueryManagementService.java @@ -0,0 +1,2614 @@ +package datawave.microservice.query; + +import static datawave.microservice.query.QueryImpl.DN_LIST; +import static datawave.microservice.query.QueryImpl.QUERY_ID; +import static datawave.microservice.query.QueryImpl.USER_DN; +import static datawave.microservice.query.QueryParameters.QUERY_LOGIC_NAME; +import static datawave.microservice.query.storage.QueryStatus.QUERY_STATE.CANCEL; +import static datawave.microservice.query.storage.QueryStatus.QUERY_STATE.CLOSE; +import static datawave.microservice.query.storage.QueryStatus.QUERY_STATE.CREATE; +import static datawave.microservice.query.storage.QueryStatus.QUERY_STATE.DEFINE; +import static datawave.microservice.query.storage.QueryStatus.QUERY_STATE.FAIL; +import static datawave.microservice.query.storage.QueryStatus.QUERY_STATE.PLAN; +import static datawave.microservice.query.storage.QueryStatus.QUERY_STATE.PREDICT; + +import java.io.IOException; +import java.lang.reflect.Method; +import java.text.MessageFormat; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +import org.apache.accumulo.core.security.Authorizations; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.http.HttpStatus; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.cloud.bus.BusProperties; +import org.springframework.cloud.bus.event.RemoteQueryRequestEvent; +import org.springframework.context.ApplicationEventPublisher; +import org.springframework.core.task.TaskRejectedException; +import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; +import org.springframework.stereotype.Service; +import org.springframework.util.LinkedMultiValueMap; +import org.springframework.util.MultiValueMap; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; + +import datawave.core.common.audit.PrivateAuditConstants; +import datawave.core.query.cache.ResultsPage; +import datawave.core.query.logic.QueryLogic; +import datawave.core.query.logic.QueryLogicFactory; +import datawave.core.query.util.QueryUtil; +import datawave.marking.SecurityMarking; +import datawave.microservice.audit.AuditClient; +import datawave.microservice.authorization.federation.FederatedAuthorizationService; +import datawave.microservice.authorization.user.DatawaveUserDetails; +import datawave.microservice.authorization.util.AuthorizationsUtil; +import datawave.microservice.query.config.QueryProperties; +import datawave.microservice.query.messaging.QueryResultsManager; +import datawave.microservice.query.remote.QueryRequest; +import datawave.microservice.query.remote.QueryRequestHandler; +import datawave.microservice.query.runner.NextCall; +import datawave.microservice.query.storage.QueryStatus; +import datawave.microservice.query.storage.QueryStorageCache; +import datawave.microservice.query.storage.TaskKey; +import datawave.microservice.query.util.QueryStatusUpdateUtil; +import datawave.microservice.querymetric.BaseQueryMetric; +import datawave.microservice.querymetric.QueryMetricClient; +import datawave.microservice.querymetric.QueryMetricType; +import datawave.security.util.ProxiedEntityUtils; +import datawave.webservice.common.audit.AuditParameters; +import datawave.webservice.common.audit.Auditor; +import datawave.webservice.query.exception.BadRequestQueryException; +import datawave.webservice.query.exception.DatawaveErrorCode; +import datawave.webservice.query.exception.NoResultsQueryException; +import datawave.webservice.query.exception.NotFoundQueryException; +import datawave.webservice.query.exception.QueryCanceledQueryException; +import datawave.webservice.query.exception.QueryException; +import datawave.webservice.query.exception.TimeoutQueryException; +import datawave.webservice.query.exception.UnauthorizedQueryException; +import datawave.webservice.query.result.event.ResponseObjectFactory; +import datawave.webservice.query.result.logic.QueryLogicDescription; +import datawave.webservice.query.util.QueryUncaughtExceptionHandler; +import datawave.webservice.result.BaseQueryResponse; +import datawave.webservice.result.GenericResponse; +import datawave.webservice.result.QueryImplListResponse; +import datawave.webservice.result.QueryLogicResponse; +import datawave.webservice.result.VoidResponse; + +@Service +public class QueryManagementService implements QueryRequestHandler { + private final Logger log = LoggerFactory.getLogger(this.getClass()); + + private static final ObjectMapper mapper = new ObjectMapper(); + + private final QueryProperties queryProperties; + + private final ApplicationEventPublisher eventPublisher; + private final BusProperties busProperties; + + // Note: QueryParameters needs to be request scoped + private final QueryParameters queryParameters; + // Note: SecurityMarking needs to be request scoped + private final SecurityMarking securityMarking; + // Note: BaseQueryMetric needs to be request scoped + private final BaseQueryMetric baseQueryMetric; + + private final QueryLogicFactory queryLogicFactory; + private final QueryMetricClient queryMetricClient; + private final ResponseObjectFactory responseObjectFactory; + private final QueryStorageCache queryStorageCache; + private final QueryResultsManager queryResultsManager; + private final AuditClient auditClient; + private final ThreadPoolTaskExecutor nextCallExecutor; + + private final QueryStatusUpdateUtil queryStatusUpdateUtil; + private final MultiValueMap nextCallMap = new LinkedMultiValueMap<>(); + + private final String selfDestination; + + // Note: for requests which don't originate with a rest call, provide ThreadLocal queryParameters + private final ThreadLocal queryParametersOverride; + // Note: for requests which don't originate with a rest call, provide ThreadLocal securityMarkings + private final ThreadLocal securityMarkingOverride; + // Note: for requests which don't originate with a rest call, provide ThreadLocal baseQueryMetric + private final ThreadLocal baseQueryMetricOverride; + + private final Map queryLatchMap = new ConcurrentHashMap<>(); + + public QueryManagementService(QueryProperties queryProperties, ApplicationEventPublisher eventPublisher, BusProperties busProperties, + QueryParameters queryParameters, SecurityMarking securityMarking, BaseQueryMetric baseQueryMetric, QueryLogicFactory queryLogicFactory, + QueryMetricClient queryMetricClient, ResponseObjectFactory responseObjectFactory, QueryStorageCache queryStorageCache, + QueryResultsManager queryResultsManager, AuditClient auditClient, ThreadPoolTaskExecutor nextCallExecutor) { + this.queryProperties = queryProperties; + this.eventPublisher = eventPublisher; + this.busProperties = busProperties; + this.queryParameters = queryParameters; + this.securityMarking = securityMarking; + this.baseQueryMetric = baseQueryMetric; + this.queryLogicFactory = queryLogicFactory; + this.queryMetricClient = queryMetricClient; + this.responseObjectFactory = responseObjectFactory; + this.queryStorageCache = queryStorageCache; + this.queryResultsManager = queryResultsManager; + this.auditClient = auditClient; + this.nextCallExecutor = nextCallExecutor; + this.queryStatusUpdateUtil = new QueryStatusUpdateUtil(this.queryProperties, this.queryStorageCache); + this.selfDestination = getSelfDestination(); + this.queryParametersOverride = new ThreadLocal<>(); + this.securityMarkingOverride = new ThreadLocal<>(); + this.baseQueryMetricOverride = new ThreadLocal<>(); + } + + /** + * Gets a list of descriptions for the configured query logics, sorted by query logic name. + *

+ * The descriptions include things like the audit type, optional and required parameters, required roles, and response class. + * + * @param currentUser + * the user who called this method, not null + * @return the query logic descriptions + */ + public QueryLogicResponse listQueryLogic(DatawaveUserDetails currentUser) { + log.info("Request: listQueryLogic from {}", ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName())); + + QueryLogicResponse response = new QueryLogicResponse(); + List> queryLogicList = queryLogicFactory.getQueryLogicList(); + List logicConfigurationList = new ArrayList<>(); + + // reference query necessary to avoid NPEs in getting the Transformer and BaseResponse + Query q = new QueryImpl(); + Date now = new Date(); + q.setExpirationDate(now); + q.setQuery("test"); + q.setQueryAuthorizations("ALL"); + + for (QueryLogic queryLogic : queryLogicList) { + try { + QueryLogicDescription logicDesc = new QueryLogicDescription(queryLogic.getLogicName()); + logicDesc.setAuditType(queryLogic.getAuditType(null).toString()); + logicDesc.setLogicDescription(queryLogic.getLogicDescription()); + + Set optionalQueryParameters = queryLogic.getOptionalQueryParameters(); + if (optionalQueryParameters != null) { + logicDesc.setSupportedParams(new ArrayList<>(optionalQueryParameters)); + } + Set requiredQueryParameters = queryLogic.getRequiredQueryParameters(); + if (requiredQueryParameters != null) { + logicDesc.setRequiredParams(new ArrayList<>(requiredQueryParameters)); + } + Set exampleQueries = queryLogic.getExampleQueries(); + if (exampleQueries != null) { + logicDesc.setExampleQueries(new ArrayList<>(exampleQueries)); + } + Set requiredRoles = queryLogic.getRequiredRoles(); + if (requiredRoles != null) { + List requiredRolesList = new ArrayList<>(queryLogic.getRequiredRoles()); + logicDesc.setRequiredRoles(requiredRolesList); + } + + try { + logicDesc.setResponseClass(queryLogic.getResponseClass(q)); + } catch (QueryException e) { + log.error("Unable to get response class for query logic: {}", queryLogic.getLogicName(), e); + response.addException(e); + logicDesc.setResponseClass("unknown"); + } + + List querySyntax = new ArrayList<>(); + try { + Method m = queryLogic.getClass().getMethod("getQuerySyntaxParsers"); + Object result = m.invoke(queryLogic); + if (result instanceof Map) { + Map map = (Map) result; + for (Object o : map.keySet()) + querySyntax.add(o.toString()); + } + } catch (Exception e) { + log.warn("Unable to get query syntax for query logic: {}", queryLogic.getClass().getCanonicalName()); + } + if (querySyntax.isEmpty()) { + querySyntax.add("CUSTOM"); + } + logicDesc.setQuerySyntax(querySyntax); + + logicConfigurationList.add(logicDesc); + } catch (Exception e) { + log.error("Error setting query logic description", e); + } + } + logicConfigurationList.sort(Comparator.comparing(QueryLogicDescription::getName)); + response.setQueryLogicList(logicConfigurationList); + + return response; + } + + /** + * Defines a query using the given query logic and parameters. + *

+ * Defined queries cannot be started and run.
+ * Auditing is not performed when defining a query.
+ * Updates can be made to any parameter using {@link #update}.
+ * Create a runnable query from a defined query using {@link #duplicate} or {@link #reset}.
+ * Delete a defined query using {@link #remove}.
+ * Aside from a limited set of admin actions, only the query owner can act on a defined query. + * + * @param queryLogicName + * the requested query logic, not null + * @param parameters + * the query parameters, not null + * @param currentUser + * the user who called this method, not null + * @param pool + * the pool to target, may be null + * @return a generic response containing the query id + * @throws BadRequestQueryException + * if parameter validation fails + * @throws BadRequestQueryException + * if query logic parameter validation fails + * @throws UnauthorizedQueryException + * if the user doesn't have access to the requested query logic + * @throws BadRequestQueryException + * if security marking validation fails + * @throws QueryException + * if query storage fails + * @throws QueryException + * if there is an unknown error + */ + public GenericResponse define(String queryLogicName, MultiValueMap parameters, String pool, DatawaveUserDetails currentUser) + throws QueryException { + String user = ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()); + if (log.isDebugEnabled()) { + log.info("Request: {}/define from {} with params: {}", queryLogicName, user, parameters); + } else { + log.info("Request: {}/define from {}", queryLogicName, user); + } + + try { + TaskKey taskKey = storeQuery(queryLogicName, parameters, pool, currentUser, DEFINE); + GenericResponse response = new GenericResponse<>(); + response.setResult(taskKey.getQueryId()); + return response; + } catch (QueryException e) { + throw e; + } catch (Exception e) { + log.error("Unknown error defining query", e); + throw new QueryException(DatawaveErrorCode.QUERY_SETUP_ERROR, e, "Unknown error defining query."); + } + } + + /** + * Creates a query using the given query logic and parameters. + *

+ * Created queries will start running immediately.
+ * Auditing is performed before the query is started.
+ * Query results can be retrieved using {@link #executeNext}.
+ * Updates can be made to any parameter which doesn't affect the scope of the query using {@link #update}.
+ * Stop a running query gracefully using {@link #close} or forcefully using {@link #cancel}.
+ * Stop, and restart a running query using {@link #reset}.
+ * Create a copy of a running query using {@link #duplicate}.
+ * Aside from a limited set of admin actions, only the query owner can act on a running query. + * + * @param queryLogicName + * the requested query logic, not null + * @param parameters + * the query parameters, not null + * @param pool + * the pool to target, may be null + * @param currentUser + * the user who called this method, not null + * @return a generic response containing the query id + * @throws BadRequestQueryException + * if parameter validation fails + * @throws BadRequestQueryException + * if query logic parameter validation fails + * @throws UnauthorizedQueryException + * if the user doesn't have access to the requested query logic + * @throws BadRequestQueryException + * if security marking validation fails + * @throws BadRequestQueryException + * if auditing fails + * @throws QueryException + * if query storage fails + * @throws QueryException + * if there is an unknown error + */ + public GenericResponse create(String queryLogicName, MultiValueMap parameters, String pool, DatawaveUserDetails currentUser) + throws QueryException { + String user = ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()); + if (log.isDebugEnabled()) { + log.info("Request: {}/create from {} with params: {}", queryLogicName, user, parameters); + } else { + log.info("Request: {}/create from {}", queryLogicName, user); + } + + try { + TaskKey taskKey = storeQuery(queryLogicName, parameters, pool, currentUser, CREATE); + GenericResponse response = new GenericResponse<>(); + response.setResult(taskKey.getQueryId()); + response.setHasResults(true); + return response; + } catch (QueryException e) { + throw e; + } catch (Exception e) { + log.error("Unknown error creating query", e); + throw new QueryException(DatawaveErrorCode.QUERY_SETUP_ERROR, e, "Unknown error creating query."); + } + } + + /** + * Generates a query plan using the given query logic and parameters. + *

+ * Created queries will begin planning immediately.
+ * Auditing is performed if we are expanding indices.
+ * Query plan will be returned in the response.
+ * Updates can be made to any parameter which doesn't affect the scope of the query using {@link #update}.
+ * + * @param queryLogicName + * the requested query logic, not null + * @param parameters + * the query parameters, not null + * @param pool + * the pool to target, may be null + * @param currentUser + * the user who called this method, not null + * @return a generic response containing the query plan + * @throws BadRequestQueryException + * if parameter validation fails + * @throws BadRequestQueryException + * if query logic parameter validation fails + * @throws UnauthorizedQueryException + * if the user doesn't have access to the requested query logic + * @throws BadRequestQueryException + * if security marking validation fails + * @throws BadRequestQueryException + * if auditing fails + * @throws QueryException + * if query storage fails + * @throws QueryException + * if there is an unknown error + */ + public GenericResponse plan(String queryLogicName, MultiValueMap parameters, String pool, DatawaveUserDetails currentUser) + throws QueryException { + String user = ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()); + if (log.isDebugEnabled()) { + log.info("Request: {}/plan from {} with params: {}", queryLogicName, user, parameters); + } else { + log.info("Request: {}/plan from {}", queryLogicName, user); + } + + try { + TaskKey taskKey = storeQuery(queryLogicName, parameters, pool, currentUser, PLAN); + String queryPlan = queryStorageCache.getQueryStatus(taskKey.getQueryId()).getPlan(); + queryStorageCache.deleteQuery(taskKey.getQueryId()); + GenericResponse response = new GenericResponse<>(); + response.setResult(queryPlan); + response.setHasResults(true); + return response; + } catch (QueryException e) { + throw e; + } catch (Exception e) { + log.error("Unknown error planning query", e); + throw new QueryException(DatawaveErrorCode.QUERY_SETUP_ERROR, e, "Unknown error planning query."); + } + } + + /** + * Generates a query prediction using the given query logic and parameters. + *

+ * Created queries will begin predicting immediately.
+ * Auditing is not performed.
+ * Query prediction will be returned in the response.
+ * Updates can be made to any parameter which doesn't affect the scope of the query using {@link #update}.
+ * + * @param queryLogicName + * the requested query logic, not null + * @param parameters + * the query parameters, not null + * @param pool + * the pool to target, may be null + * @param currentUser + * the user who called this method, not null + * @return a generic response containing the query prediction + * @throws BadRequestQueryException + * if parameter validation fails + * @throws BadRequestQueryException + * if query logic parameter validation fails + * @throws UnauthorizedQueryException + * if the user doesn't have access to the requested query logic + * @throws BadRequestQueryException + * if security marking validation fails + * @throws BadRequestQueryException + * if auditing fails + * @throws QueryException + * if query storage fails + * @throws QueryException + * if there is an unknown error + */ + public GenericResponse predict(String queryLogicName, MultiValueMap parameters, String pool, DatawaveUserDetails currentUser) + throws QueryException { + String user = ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()); + if (log.isDebugEnabled()) { + log.info("Request: {}/predict from {} with params: {}", queryLogicName, user, parameters); + } else { + log.info("Request: {}/predict from {}", queryLogicName, user); + } + + try { + TaskKey taskKey = storeQuery(queryLogicName, parameters, pool, currentUser, PREDICT); + String queryPrediction = "no predictions"; + QueryStatus status = queryStorageCache.getQueryStatus(taskKey.getQueryId()); + if (status != null) { + Set predictions = status.getPredictions(); + if (CollectionUtils.isNotEmpty(predictions)) { + queryPrediction = predictions.toString(); + } + } + queryStorageCache.deleteQuery(taskKey.getQueryId()); + GenericResponse response = new GenericResponse<>(); + response.setResult(queryPrediction); + response.setHasResults(true); + return response; + } catch (QueryException e) { + throw e; + } catch (Exception e) { + log.error("Unknown error predicting query", e); + throw new QueryException(DatawaveErrorCode.QUERY_SETUP_ERROR, e, "Unknown error predicting query."); + } + } + + private TaskKey storeQuery(String queryLogicName, MultiValueMap parameters, String pool, DatawaveUserDetails currentUser, + QueryStatus.QUERY_STATE queryType) throws QueryException { + return storeQuery(queryLogicName, parameters, pool, currentUser, queryType, null); + } + + /** + * Validates the query request, creates an entry in the query storage cache, and publishes a create event to the executor service. + *

+ * Validation is run against the requested logic, the parameters, and the security markings in {@link #validateQuery}.
+ * Auditing is performed when {@code isCreateRequest} is true using {@link #audit}.
+ * If {@code queryId} is null, a query id will be generated automatically. + * + * @param queryLogicName + * the requested query logic, not null + * @param parameters + * the query parameters, not null + * @param pool + * the pool to target, may be null + * @param currentUser + * the user who called this method, not null + * @param queryType + * whether this is a define, create, or plan call + * @param queryId + * the desired query id, may be null + * @return the task key returned from query storage + * @throws BadRequestQueryException + * if parameter validation fails + * @throws BadRequestQueryException + * if query logic parameter validation fails + * @throws UnauthorizedQueryException + * if the user doesn't have access to the requested query logic + * @throws BadRequestQueryException + * if security marking validation fails + * @throws BadRequestQueryException + * if auditing fails + * @throws QueryException + * if query storage fails + * @throws QueryException + * if there is an unknown error + */ + private TaskKey storeQuery(String queryLogicName, MultiValueMap parameters, String pool, DatawaveUserDetails currentUser, + QueryStatus.QUERY_STATE queryType, String queryId) throws BadRequestQueryException, QueryException { + long callStartTimeMillis = System.currentTimeMillis(); + + // validate query and get a query logic + QueryLogic queryLogic = validateQuery(queryLogicName, parameters, currentUser); + + String userId = ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()); + log.trace("{} has authorizations {}", userId, currentUser.getPrimaryUser().getAuths()); + + Query query = createQuery(queryLogicName, parameters, currentUser, queryId); + + // if this is a create request, or a plan request where we are expanding values, send an audit record to the auditor + if (queryType == CREATE || (queryType == PLAN && queryParameters.isExpandValues())) { + audit(query, queryLogic, parameters, currentUser); + } + + // downgrade the auths + QueryParameters queryParameters = getQueryParameters(); + Set downgradedAuthorizations; + try { + if (queryParameters.getAuths() == null) { + // if no requested auths, then use the overall auths for any filtering of the query operations + queryLogic.preInitialize(query, AuthorizationsUtil.buildAuthorizations(currentUser.getAuthorizations())); + } else { + queryLogic.preInitialize(query, + AuthorizationsUtil.buildAuthorizations(Collections.singleton(AuthorizationsUtil.splitAuths(query.getQueryAuthorizations())))); + } + // the query principal is our local principal unless the query logic has a different user operations + DatawaveUserDetails queryUserDetails = (DatawaveUserDetails) ((queryLogic.getUserOperations() == null) ? currentUser + : queryLogic.getUserOperations().getRemoteUser(currentUser)); + // the overall principal (the one with combined auths across remote user operations) is our own user operations (probably the UserOperationsBean) + // don't call remote user operations if it's asked not to + DatawaveUserDetails overallUserDetails = (queryLogic.getUserOperations() == null + || "false".equalsIgnoreCase(parameters.getFirst(FederatedAuthorizationService.INCLUDE_REMOTE_SERVICES))) ? queryUserDetails + : queryLogic.getUserOperations().getRemoteUser(queryUserDetails); + downgradedAuthorizations = AuthorizationsUtil.getDowngradedAuthorizations(queryParameters.getAuths(), overallUserDetails, queryUserDetails); + } catch (Exception e) { + throw new BadRequestQueryException("Unable to downgrade authorizations", e, HttpStatus.SC_BAD_REQUEST + "-1"); + } + + try { + String computedPool = getPoolName(pool, isAdminUser(currentUser)); + + // persist the query w/ query id in the query storage cache + TaskKey taskKey = null; + if (queryType == DEFINE) { + // @formatter:off + taskKey = queryStorageCache.defineQuery( + computedPool, + query, + currentUser, + downgradedAuthorizations, + getMaxConcurrentTasks(queryLogic)); + // @formatter:on + } else if (queryType == CREATE) { + // @formatter:off + taskKey = queryStorageCache.createQuery( + computedPool, + query, + currentUser, + downgradedAuthorizations, + getMaxConcurrentTasks(queryLogic)); + + sendRequestAwaitResponse( + QueryRequest.create(taskKey.getQueryId()), + computedPool, + queryProperties.isAwaitExecutorCreateResponse(), + callStartTimeMillis); + // @formatter:on + } else if (queryType == PLAN) { + // @formatter:off + taskKey = queryStorageCache.planQuery( + computedPool, + query, + currentUser, + downgradedAuthorizations); + + sendRequestAwaitResponse( + QueryRequest.plan(taskKey.getQueryId()), + computedPool, + true, + callStartTimeMillis); + // @formatter:on + } else if (queryType == PREDICT) { + // @formatter:off + taskKey = queryStorageCache.predictQuery( + computedPool, + query, + currentUser, + downgradedAuthorizations); + + sendRequestAwaitResponse( + QueryRequest.predict(taskKey.getQueryId()), + computedPool, + true, + callStartTimeMillis); + // @formatter:on + } + + if (taskKey == null) { + log.error("Task Key not created for query"); + throw new QueryException(DatawaveErrorCode.RUNNING_QUERY_CACHE_ERROR); + } + + // update the query metric + BaseQueryMetric baseQueryMetric = getBaseQueryMetric(); + if (queryType == DEFINE || queryType == CREATE) { + baseQueryMetric.setQueryId(taskKey.getQueryId()); + baseQueryMetric.setLifecycle(BaseQueryMetric.Lifecycle.DEFINED); + baseQueryMetric.populate(query); + baseQueryMetric.setProxyServers(getDNs(currentUser)); + } + + return taskKey; + } catch (Exception e) { + log.error("Unknown error storing query", e); + throw new QueryException(DatawaveErrorCode.RUNNING_QUERY_CACHE_ERROR, e); + } + } + + private void sendRequestAwaitResponse(QueryRequest request, String computedPool, boolean isAwaitResponse, long startTimeMillis) throws QueryException { + if (isAwaitResponse) { + // before publishing the message, create a latch based on the query ID + queryLatchMap.put(request.getQueryId(), new CountDownLatch(1)); + } + + // publish an event to the executor pool + publishExecutorEvent(request, computedPool); + + if (isAwaitResponse) { + log.info("Waiting on query {} response from the executor.", request.getMethod().name()); + + try { + boolean isFinished = false; + while (!isFinished && System.currentTimeMillis() < (startTimeMillis + queryProperties.getExpiration().getCallTimeoutMillis())) { + try { + // wait for the executor response + if (queryLatchMap.get(request.getQueryId()).await(queryProperties.getExpiration().getCallTimeoutInterval(), + queryProperties.getExpiration().getCallTimeoutIntervalUnit())) { + log.info("Received query {} response from the executor.", request.getMethod().name()); + isFinished = true; + } + + // did the request fail? + QueryStatus queryStatus = queryStorageCache.getQueryStatus(request.getQueryId()); + if (queryStatus.getQueryState() == FAIL) { + log.error("Query {} failed for queryId {}: {}", request.getMethod().name(), request.getQueryId(), queryStatus.getFailureMessage()); + throw new QueryException(queryStatus.getErrorCode(), "Query " + request.getMethod().name() + " failed for queryId " + + request.getQueryId() + ": " + queryStatus.getFailureMessage()); + } + } catch (InterruptedException e) { + log.warn("Interrupted while waiting for query {} latch for queryId {}", request.getMethod().name(), request.getQueryId()); + } + } + } finally { + queryLatchMap.remove(request.getQueryId()); + } + } + } + + /** + * Creates a query using the given query logic and parameters, and returns the first page of results. + *

+ * Created queries will start running immediately.
+ * Auditing is performed before the query is started.
+ * Subsequent query results can be retrieved using {@link #executeNext}.
+ * Updates can be made to any parameter which doesn't affect the scope of the query using {@link #update}.
+ * Stop a running query gracefully using {@link #close} or forcefully using {@link #cancel}.
+ * Stop, and restart a running query using {@link #reset}.
+ * Create a copy of a running query using {@link #duplicate}.
+ * Aside from a limited set of admin actions, only the query owner can act on a running query. + * + * @param queryLogicName + * the requested query logic, not null + * @param parameters + * the query parameters, not null + * @param pool + * the pool to target, may be null + * @param currentUser + * the user who called this method, not null + * @return a base query response containing the first page of results + * @throws BadRequestQueryException + * if parameter validation fails + * @throws BadRequestQueryException + * if query logic parameter validation fails + * @throws UnauthorizedQueryException + * if the user doesn't have access to the requested query logic + * @throws BadRequestQueryException + * if security marking validation fails + * @throws BadRequestQueryException + * if auditing fails + * @throws QueryException + * if query storage fails + * @throws TimeoutQueryException + * if the next call times out + * @throws NoResultsQueryException + * if no query results are found + * @throws QueryException + * if this next task is rejected by the executor + * @throws QueryException + * if there is an unknown error + */ + public BaseQueryResponse createAndNext(String queryLogicName, MultiValueMap parameters, String pool, DatawaveUserDetails currentUser) + throws QueryException { + String user = ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()); + if (log.isDebugEnabled()) { + log.info("Request: {}/createAndNext from {} with params: {}", queryLogicName, user, parameters); + } else { + log.info("Request: {}/createAndNext from {}", queryLogicName, user); + } + + String queryId = null; + try { + queryId = create(queryLogicName, parameters, pool, currentUser).getResult(); + return executeNext(queryId, currentUser); + } catch (Exception e) { + QueryException qe; + if (!(e instanceof QueryException)) { + log.error("Unknown error calling create and next. {}", queryId, e); + qe = new QueryException(DatawaveErrorCode.QUERY_NEXT_ERROR, e, "Unknown error calling create and next. " + queryId); + } else { + qe = (QueryException) e; + } + + if (queryId != null && !(qe instanceof NoResultsQueryException)) { + getBaseQueryMetric().setError(qe); + } + + throw qe; + } + } + + /** + * Gets the next page of results for the specified query. + *

+ * Next can only be called on a running query.
+ * If configuration allows, multiple next calls may be run concurrently for a query.
+ * Only the query owner can call next on the specified query. + * + * @param queryId + * the query id, not null + * @param currentUser + * the user who called this method, not null + * @return a base query response containing the next page of results + * @throws NotFoundQueryException + * if the query cannot be found + * @throws UnauthorizedQueryException + * if the user doesn't own the query + * @throws BadRequestQueryException + * if the query is not running + * @throws QueryException + * if query lock acquisition fails + * @throws QueryException + * if the next call is interrupted + * @throws TimeoutQueryException + * if the query times out + * @throws NoResultsQueryException + * if no query results are found + * @throws QueryException + * if this next task is rejected by the executor + * @throws QueryException + * if next call execution fails + * @throws QueryException + * if query logic creation fails + * @throws QueryException + * if there is an unknown error + */ + public BaseQueryResponse next(String queryId, DatawaveUserDetails currentUser) throws QueryException { + log.info("Request: next from {} for {}", ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()), queryId); + + try { + // make sure the query is valid, and the user can act on it + QueryStatus queryStatus = validateRequest(queryId, currentUser); + + // make sure the state is created + if (queryStatus.getQueryState() == CREATE) { + return executeNext(queryId, currentUser); + } else { + throw new BadRequestQueryException("Cannot call next on a query that is not running", HttpStatus.SC_BAD_REQUEST + "-1"); + } + } catch (Exception e) { + QueryException qe; + if (!(e instanceof QueryException)) { + log.error("Unknown error getting next page for query {}", queryId, e); + qe = new QueryException(DatawaveErrorCode.QUERY_NEXT_ERROR, e, "Unknown error getting next page for query " + queryId); + } else { + qe = (QueryException) e; + } + + if (!(qe instanceof NoResultsQueryException)) { + getBaseQueryMetric().setError(qe); + } + + throw qe; + } + } + + /** + * Gets the next page of results for the given query, and publishes a next event to the executor service. + *

+ * If configuration allows, multiple next calls may be run concurrently for a query. + * + * @param queryId + * the query id, not null + * @param currentUser + * the user who called this method, not null + * @return a base query response containing the next page of results + * @throws NotFoundQueryException + * if the query cannot be found + * @throws QueryException + * if query lock acquisition fails + * @throws InterruptedException + * if the next call is interrupted + * @throws TimeoutQueryException + * if the query times out + * @throws NoResultsQueryException + * if no query results are found + * @throws QueryException + * if this next task is rejected by the executor + * @throws QueryException + * if next call execution fails + * @throws QueryException + * if query logic creation fails + */ + private BaseQueryResponse executeNext(String queryId, DatawaveUserDetails currentUser) throws InterruptedException, QueryException { + // before we spin up a separate thread, make sure we are allowed to call next + boolean success = false; + QueryStatus queryStatus = queryStatusUpdateUtil.lockedUpdate(queryId, queryStatusUpdateUtil::claimNextCall); + try { + // publish a next event to the executor pool + publishNextEvent(queryId, queryStatus.getQueryKey().getQueryPool()); + + // get the query logic + String queryLogicName = queryStatus.getQuery().getQueryLogicName(); + QueryLogic queryLogic = queryLogicFactory.getQueryLogic(queryStatus.getQuery().getQueryLogicName(), currentUser); + + // update query metrics + BaseQueryMetric baseQueryMetric = getBaseQueryMetric(); + baseQueryMetric.setQueryId(queryId); + baseQueryMetric.setQueryLogic(queryLogicName); + + // @formatter:off + final NextCall nextCall = new NextCall.Builder() + .setQueryProperties(queryProperties) + .setResultsQueueManager(queryResultsManager) + .setQueryStorageCache(queryStorageCache) + .setQueryStatusUpdateUtil(queryStatusUpdateUtil) + .setQueryId(queryId) + .setQueryLogic(queryLogic) + .build(); + // @formatter:on + + nextCallMap.add(queryId, nextCall); + try { + // submit the next call to the executor + nextCall.setFuture(nextCallExecutor.submit(nextCall)); + + // wait for the results to be ready + ResultsPage resultsPage = nextCall.getFuture().get(); + + // update the query metric + nextCall.updateQueryMetric(baseQueryMetric); + + // format the response + if (!resultsPage.getResults().isEmpty()) { + BaseQueryResponse response = queryLogic.getTransformer(queryStatus.getQuery()).createResponse(resultsPage); + + // after all of our work is done, perform our final query status update for this next call + queryStatus = queryStatusUpdateUtil.lockedUpdate(queryId, status -> { + queryStatusUpdateUtil.releaseNextCall(status, queryResultsManager); + status.setLastPageNumber(status.getLastPageNumber() + 1); + status.setNumResultsReturned(status.getNumResultsReturned() + resultsPage.getResults().size()); + }); + success = true; + + response.setHasResults(true); + response.setPageNumber(queryStatus.getLastPageNumber()); + response.setLogicName(queryLogicName); + response.setQueryId(queryId); + return response; + } else { + if (nextCall.isCanceled()) { + log.debug("Query [{}]: Canceled while handling next call", queryId); + throw new QueryCanceledQueryException(DatawaveErrorCode.QUERY_CANCELED, MessageFormat.format("{0} canceled;", queryId)); + } else if (baseQueryMetric.getLifecycle() == BaseQueryMetric.Lifecycle.NEXTTIMEOUT) { + log.debug("Query [{}]: Timed out during next call", queryId); + throw new TimeoutQueryException(DatawaveErrorCode.QUERY_TIMEOUT, MessageFormat.format("{0} timed out.", queryId)); + } else { + log.debug("Query [{}]: No results found for next call - closing query", queryId); + // if there are no results, and we didn't timeout, close the query + close(queryId); + throw new NoResultsQueryException(DatawaveErrorCode.NO_QUERY_RESULTS_FOUND, MessageFormat.format("{0}", queryId)); + } + } + } catch (TaskRejectedException e) { + throw new QueryException(DatawaveErrorCode.QUERY_NEXT_ERROR, e, "Next task rejected by the executor for query " + queryId); + } catch (ExecutionException e) { + // try to unwrap the execution exception and throw a query exception + throw new QueryException(DatawaveErrorCode.QUERY_NEXT_ERROR, e.getCause(), "Next call execution failed"); + } finally { + // remove this next call from the map, and decrement the next count for this query + nextCallMap.get(queryId).remove(nextCall); + } + } catch (CloneNotSupportedException e) { + throw new QueryException(DatawaveErrorCode.QUERY_NEXT_ERROR, e, + "Unable to create instance of the requested query logic " + queryStatus.getQuery().getQueryLogicName()); + } finally { + // update query status if we failed + if (!success) { + queryStatusUpdateUtil.lockedUpdate(queryId, status -> queryStatusUpdateUtil.releaseNextCall(status, queryResultsManager)); + } + } + } + + /** + * Cancels the specified query. + *

+ * Cancel can only be called on a running query, or a query that is in the process of closing.
+ * Outstanding next calls will be stopped immediately, but will return partial results if applicable.
+ * Aside from admins, only the query owner can cancel the specified query. + * + * @param queryId + * the query id, not null + * @param currentUser + * the user who called this method, not null + * @return a void response indicating that the query was canceled + * @throws NotFoundQueryException + * if the query cannot be found + * @throws UnauthorizedQueryException + * if the user doesn't own the query + * @throws BadRequestQueryException + * if the query is not running + * @throws QueryException + * if query lock acquisition fails + * @throws QueryException + * if the cancel call is interrupted + * @throws QueryException + * if there is an unknown error + */ + public VoidResponse cancel(String queryId, DatawaveUserDetails currentUser) throws QueryException { + log.info("Request: cancel from {} for {}", ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()), queryId); + + return cancel(queryId, currentUser, false); + } + + /** + * Cancels the specified query using admin privileges. + *

+ * Cancel can only be called on a running query, or a query that is in the process of closing.
+ * Outstanding next calls will be stopped immediately, but will return partial results if applicable.
+ * Only admin users should be allowed to call this method. + * + * @param queryId + * the query id, not null + * @param currentUser + * the user who called this method, not null + * @return a void response indicating that the query was canceled + * @throws NotFoundQueryException + * if the query cannot be found + * @throws BadRequestQueryException + * if the query is not running + * @throws QueryException + * if query lock acquisition fails + * @throws QueryException + * if the cancel call is interrupted + * @throws QueryException + * if there is an unknown error + */ + public VoidResponse adminCancel(String queryId, DatawaveUserDetails currentUser) throws QueryException { + log.info("Request: adminCancel from {} for {}", ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()), queryId); + + return cancel(queryId, currentUser, true); + } + + /** + * Cancels all queries using admin privileges. + *

+ * Cancel can only be called on a running query, or a query that is in the process of closing.
+ * Queries that are not running will be ignored by this method.
+ * Outstanding next calls will be stopped immediately, but will return partial results if applicable.
+ * Only admin users should be allowed to call this method. + * + * @param currentUser + * the user who called this method, not null + * @return a void response specifying which queries were canceled + * @throws NotFoundQueryException + * if a query cannot be found + * @throws QueryException + * if query lock acquisition fails + * @throws QueryException + * if the cancel call is interrupted + * @throws QueryException + * if there is an unknown error + */ + public VoidResponse adminCancelAll(DatawaveUserDetails currentUser) throws QueryException { + log.info("Request: adminCancelAll from {}", ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName())); + + try { + List queryStatuses = queryStorageCache.getQueryStatus(); + queryStatuses.removeIf(s -> s.getQueryState() != CREATE); + + VoidResponse response = new VoidResponse(); + for (QueryStatus queryStatus : queryStatuses) { + cancel(queryStatus.getQueryKey().getQueryId(), true); + response.addMessage(queryStatus.getQueryKey().getQueryId() + " canceled."); + } + return response; + } catch (QueryException e) { + throw e; + } catch (Exception e) { + QueryException queryException = new QueryException(DatawaveErrorCode.CANCELLATION_ERROR, e, "Error encountered while canceling all queries."); + log.error("Error encountered while canceling all queries", queryException); + throw queryException; + } + } + + /** + * Cancels the specified query. + *

+ * Cancel can only be called on a running query, or a query that is in the process of closing.
+ * Outstanding next calls will be stopped immediately, but will return partial results if applicable.
+ * Publishes a cancel event to the query and executor services.
+ * Query ownership will only be validated when {@code adminOverride} is set to true. + * + * @param queryId + * the query id, not null + * @param currentUser + * the user who called this method, not null + * @param adminOverride + * whether or not this is an admin action + * @return a void response indicating that the query was canceled + * @throws NotFoundQueryException + * if the query cannot be found + * @throws UnauthorizedQueryException + * if the user doesn't own the query + * @throws BadRequestQueryException + * if the query is not running + * @throws QueryException + * if query lock acquisition fails + * @throws QueryException + * if the cancel call is interrupted + * @throws QueryException + * if there is an unknown error + */ + private VoidResponse cancel(String queryId, DatawaveUserDetails currentUser, boolean adminOverride) throws QueryException { + try { + // make sure the query is valid, and the user can act on it + QueryStatus queryStatus = validateRequest(queryId, currentUser, adminOverride); + + // if the query is running, or if the query is closing and finishing up a next call + if (queryStatus.isRunning()) { + cancel(queryId, true); + } else { + throw new BadRequestQueryException("Cannot call cancel on a query that is not running", HttpStatus.SC_BAD_REQUEST + "-1"); + } + + VoidResponse response = new VoidResponse(); + response.addMessage(queryId + " canceled."); + return response; + } catch (QueryException e) { + throw e; + } catch (Exception e) { + QueryException queryException = new QueryException(DatawaveErrorCode.CANCELLATION_ERROR, e, "Unknown error canceling query " + queryId); + log.error("Unknown error canceling query {}", queryId, queryException); + throw queryException; + } + } + + /** + * Cancels the specified query, and optionally publishes a cancel event to the query and executor services. + *

+ * Cancels any locally-running next calls.
+ * Called with {@code publishEvent} set to true when the user calls cancel.
+ * When {@code publishEvent} is true, changes the query state to {@link QueryStatus.QUERY_STATE#CANCEL}.
+ * Called with {@code publishEvent} set to false when handling a remote cancel event. + * + * @param queryId + * the query id, not null + * @param publishEvent + * whether or not to publish an event + * @throws NotFoundQueryException + * if the query cannot be found + * @throws QueryException + * if query lock acquisition fails + * @throws InterruptedException + * if the cancel call is interrupted + */ + public void cancel(String queryId, boolean publishEvent) throws InterruptedException, QueryException { + // if we have an active next call for this query locally, cancel it + List nextCalls = nextCallMap.get(queryId); + if (nextCalls != null) { + nextCalls.forEach(NextCall::cancel); + } + + if (publishEvent) { + // only the initial event publisher should update the status + QueryStatus queryStatus = queryStatusUpdateUtil.lockedUpdate(queryId, status -> { + // update query state to CANCELED + status.setQueryState(CANCEL); + }); + + // delete the results queue + queryResultsManager.deleteQuery(queryId); + + QueryRequest cancelRequest = QueryRequest.cancel(queryId); + + // publish a cancel event to all of the query services + publishSelfEvent(cancelRequest); + + // publish a cancel event to the executor pool + publishExecutorEvent(cancelRequest, queryStatus.getQueryKey().getQueryPool()); + + try { + QueryLogic logic = queryLogicFactory.getQueryLogic(queryStatus.getQuery().getQueryLogicName()); + if (logic.getCollectQueryMetrics()) { + // update query metrics + BaseQueryMetric baseQueryMetric = getBaseQueryMetric(); + baseQueryMetric.setQueryId(queryId); + baseQueryMetric.setLifecycle(BaseQueryMetric.Lifecycle.CANCELLED); + baseQueryMetric.setLastUpdated(new Date()); + try { + // @formatter:off + queryMetricClient.submit( + new QueryMetricClient.Request.Builder() + .withUser((DatawaveUserDetails) logic.getServerUser()) + .withMetric(baseQueryMetric.duplicate()) + .withMetricType(QueryMetricType.DISTRIBUTED) + .build()); + // @formatter:on + } catch (Exception e) { + log.error("Error updating query metric", e); + } + } + } catch (CloneNotSupportedException e) { + log.warn("Could not determine whether the query logic supports metrics"); + } + } + } + + /** + * Closes the specified query. + *

+ * Close can only be called on a running query.
+ * Outstanding next calls will be allowed to run until they can return a full page, or they timeout.
+ * Aside from admins, only the query owner can close the specified query. + * + * @param queryId + * the query id, not null + * @param currentUser + * the user who called this method, not null + * @return a void response indicating that the query was closed + * @throws NotFoundQueryException + * if the query cannot be found + * @throws UnauthorizedQueryException + * if the user doesn't own the query + * @throws BadRequestQueryException + * if the query is not running + * @throws QueryException + * if query lock acquisition fails + * @throws QueryException + * if the cancel call is interrupted + * @throws QueryException + * if there is an unknown error + */ + public VoidResponse close(String queryId, DatawaveUserDetails currentUser) throws QueryException { + log.info("Request: close from {} for {}", ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()), queryId); + + return close(queryId, currentUser, false); + } + + /** + * Closes the specified query using admin privileges. + *

+ * Close can only be called on a running query.
+ * Outstanding next calls will be allowed to run until they can return a full page, or they timeout.
+ * Only admin users should be allowed to call this method. + * + * @param queryId + * the query id, not null + * @param currentUser + * the user who called this method, not null + * @return a void response indicating that the query was closed + * @throws NotFoundQueryException + * if the query cannot be found + * @throws BadRequestQueryException + * if the query is not running + * @throws QueryException + * if query lock acquisition fails + * @throws QueryException + * if the cancel call is interrupted + * @throws QueryException + * if there is an unknown error + */ + public VoidResponse adminClose(String queryId, DatawaveUserDetails currentUser) throws QueryException { + log.info("Request: adminClose from {} for {}", ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()), queryId); + + return close(queryId, currentUser, true); + } + + /** + * Closes all queries using admin privileges. + *

+ * Close can only be called on a running query.
+ * Queries that are not running will be ignored by this method.
+ * Outstanding next calls will be allowed to run until they can return a full page, or they timeout.
+ * Only admin users should be allowed to call this method. + * + * @param currentUser + * the user who called this method, not null + * @return a void response specifying which queries were closed + * @throws NotFoundQueryException + * if a query cannot be found + * @throws QueryException + * if query lock acquisition fails + * @throws QueryException + * if the close call is interrupted + * @throws QueryException + * if there is an unknown error + */ + public VoidResponse adminCloseAll(DatawaveUserDetails currentUser) throws QueryException { + log.info("Request: adminCloseAll from {}", ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName())); + + try { + List queryStatuses = queryStorageCache.getQueryStatus(); + queryStatuses.removeIf(s -> s.getQueryState() != CREATE); + + VoidResponse response = new VoidResponse(); + for (QueryStatus queryStatus : queryStatuses) { + close(queryStatus.getQueryKey().getQueryId()); + response.addMessage(queryStatus.getQueryKey().getQueryId() + " closed."); + } + return response; + } catch (QueryException e) { + throw e; + } catch (Exception e) { + QueryException queryException = new QueryException(DatawaveErrorCode.QUERY_CLOSE_ERROR, e, "Error encountered while closing all queries."); + log.error("Error encountered while closing all queries", queryException); + throw queryException; + } + } + + /** + * Closes the specified query. + *

+ * Close can only be called on a running query.
+ * Outstanding next calls will be allowed to run until they can return a full page, or they timeout.
+ * Query ownership will only be validated when {@code adminOverride} is set to true. + * + * @param queryId + * the query id, not null + * @param currentUser + * the user who called this method, not null + * @param adminOverride + * whether or not this is an admin action + * @return a void response indicating that the query was closed + * @throws NotFoundQueryException + * if the query cannot be found + * @throws UnauthorizedQueryException + * if the user doesn't own the query + * @throws BadRequestQueryException + * if the query is not running + * @throws QueryException + * if query lock acquisition fails + * @throws QueryException + * if the cancel call is interrupted + * @throws QueryException + * if there is an unknown error + */ + private VoidResponse close(String queryId, DatawaveUserDetails currentUser, boolean adminOverride) throws QueryException { + try { + // make sure the query is valid, and the user can act on it + QueryStatus queryStatus = validateRequest(queryId, currentUser, adminOverride); + + if (queryStatus.getQueryState() == CREATE) { + close(queryId); + } else { + throw new BadRequestQueryException("Cannot call close on a query that is not running", HttpStatus.SC_BAD_REQUEST + "-1"); + } + + VoidResponse response = new VoidResponse(); + response.addMessage(queryId + " closed."); + return response; + } catch (QueryException e) { + throw e; + } catch (Exception e) { + QueryException queryException = new QueryException(DatawaveErrorCode.QUERY_CLOSE_ERROR, e, "Unknown error closing query " + queryId); + log.error("Unknown error closing query {}", queryId, queryException); + throw queryException; + } + } + + /** + * Closes the specified query, and publishes a close event to the executor services. + *

+ * Changes the query state to {@link QueryStatus.QUERY_STATE#CLOSE}. + * + * @param queryId + * the query id, not null + * @throws NotFoundQueryException + * if the query cannot be found + * @throws QueryException + * if query lock acquisition fails + * @throws InterruptedException + * if the cancel call is interrupted + */ + public void close(String queryId) throws InterruptedException, QueryException { + QueryStatus queryStatus = queryStatusUpdateUtil.lockedUpdate(queryId, status -> { + // update query state to CLOSED + status.setQueryState(CLOSE); + }); + + // if the query has no active next calls, delete the results queue + if (queryStatus.getActiveNextCalls() == 0) { + queryResultsManager.deleteQuery(queryId); + } + + // publish a close event to the executor pool + publishExecutorEvent(QueryRequest.close(queryId), queryStatus.getQueryKey().getQueryPool()); + + try { + QueryLogic logic = queryLogicFactory.getQueryLogic(queryStatus.getQuery().getQueryLogicName()); + if (logic.getCollectQueryMetrics()) { + // update query metrics + BaseQueryMetric baseQueryMetric = getBaseQueryMetric(); + baseQueryMetric.setQueryId(queryId); + baseQueryMetric.setLifecycle(BaseQueryMetric.Lifecycle.CLOSED); + baseQueryMetric.setLastUpdated(new Date()); + try { + // @formatter:off + queryMetricClient.submit( + new QueryMetricClient.Request.Builder() + .withUser((DatawaveUserDetails) logic.getServerUser()) + .withMetric(baseQueryMetric.duplicate()) + .withMetricType(QueryMetricType.DISTRIBUTED) + .build()); + // @formatter:on + } catch (Exception e) { + log.error("Error updating query metric", e); + } + } + } catch (CloneNotSupportedException e) { + log.warn("Could not determine whether the query logic supports metrics"); + } + } + + /** + * Stops, and restarts the specified query. + *

+ * Reset can be called on any query, whether it's running or not.
+ * If the specified query is still running, it will be canceled. See {@link #cancel}.
+ * Reset creates a new, identical query, with a new query id.
+ * Reset queries will start running immediately.
+ * Auditing is performed before the new query is started. + * + * @param queryId + * the query id, not null + * @param currentUser + * the user who called this method, not null + * @return a generic response containing the new query id + * @throws NotFoundQueryException + * if the query cannot be found + * @throws UnauthorizedQueryException + * if the user doesn't own the query + * @throws QueryException + * if query lock acquisition fails + * @throws QueryException + * if the cancel call is interrupted + * @throws BadRequestQueryException + * if parameter validation fails + * @throws BadRequestQueryException + * if query logic parameter validation fails + * @throws UnauthorizedQueryException + * if the user doesn't have access to the requested query logic + * @throws BadRequestQueryException + * if security marking validation fails + * @throws BadRequestQueryException + * if auditing fails + * @throws QueryException + * if query storage fails + * @throws QueryException + * if there is an unknown error + */ + public GenericResponse reset(String queryId, DatawaveUserDetails currentUser) throws QueryException { + log.info("Request: reset from {} for {}", ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()), queryId); + + try { + // make sure the query is valid, and the user can act on it + QueryStatus queryStatus = validateRequest(queryId, currentUser); + + // cancel the query if it is running + if (queryStatus.isRunning()) { + cancel(queryStatus.getQueryKey().getQueryId(), true); + } + + // create a new query which is an exact copy of the specified query + TaskKey taskKey = duplicate(queryStatus, new LinkedMultiValueMap<>(), currentUser); + + GenericResponse response = new GenericResponse<>(); + response.addMessage(queryId + " reset."); + response.setResult(taskKey.getQueryId()); + response.setHasResults(true); + return response; + } catch (QueryException e) { + throw e; + } catch (Exception e) { + log.error("Unknown error resetting query {}", queryId, e); + throw new QueryException(DatawaveErrorCode.QUERY_RESET_ERROR, e, "Unknown error resetting query " + queryId); + } + } + + /** + * Removes the specified query from query storage. + *

+ * Remove can only be called on a query that is not running.
+ * Aside from admins, only the query owner can remove the specified query. + * + * @param queryId + * the query id, not null + * @param currentUser + * the user who called this method, not null + * @return a void response indicating that the query was removed + * @throws NotFoundQueryException + * if the query cannot be found + * @throws UnauthorizedQueryException + * if the user doesn't own the query + * @throws BadRequestQueryException + * if the query is running + * @throws QueryException + * if there is an unknown error + */ + public VoidResponse remove(String queryId, DatawaveUserDetails currentUser) throws QueryException { + log.info("Request: remove from {} for {}", ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()), queryId); + + return remove(queryId, currentUser, false); + } + + /** + * Removes the specified query from query storage using admin privileges. + *

+ * Remove can only be called on a query that is not running.
+ * Only admin users should be allowed to call this method. + * + * @param queryId + * the query id, not null + * @param currentUser + * the user who called this method, not null + * @return a void response indicating that the query was removed + * @throws NotFoundQueryException + * if the query cannot be found + * @throws BadRequestQueryException + * if the query is running + * @throws QueryException + * if there is an unknown error + */ + public VoidResponse adminRemove(String queryId, DatawaveUserDetails currentUser) throws QueryException { + log.info("Request: adminRemove from {} for {}", ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()), queryId); + + return remove(queryId, currentUser, true); + } + + /** + * Removes all queries from query storage using admin privileges. + *

+ * Remove can only be called on a query that is not running.
+ * Queries that are running will be ignored by this method.
+ * Only admin users should be allowed to call this method. + * + * @param currentUser + * the user who called this method, not null + * @return a void response specifying which queries were removed + * @throws NotFoundQueryException + * if a query cannot be found + * @throws QueryException + * if there is an unknown error + */ + public VoidResponse adminRemoveAll(DatawaveUserDetails currentUser) throws QueryException { + log.info("Request: adminRemoveAll from {}", ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName())); + + try { + List queryStatuses = queryStorageCache.getQueryStatus(); + queryStatuses.removeIf(QueryStatus::isRunning); + + VoidResponse response = new VoidResponse(); + for (QueryStatus queryStatus : queryStatuses) { + if (remove(queryStatus)) { + response.addMessage(queryStatus.getQueryKey().getQueryId() + " removed."); + } + } + return response; + } catch (Exception e) { + QueryException queryException = new QueryException(DatawaveErrorCode.QUERY_REMOVAL_ERROR, e, "Error encountered while removing all queries."); + log.error("Error encountered while removing all queries", queryException); + throw queryException; + } + } + + /** + * Removes the specified query from query storage. + *

+ * Remove can only be called on a query that is not running.
+ * Query ownership will only be validated when {@code adminOverride} is set to true. + * + * @param queryId + * the query id, not null + * @param currentUser + * the user who called this method, not null + * @param adminOverride + * whether or not this is an admin action + * @return a void response indicating that the query was removed + * @throws NotFoundQueryException + * if the query cannot be found + * @throws UnauthorizedQueryException + * if the user doesn't own the query + * @throws BadRequestQueryException + * if the query is running + * @throws QueryException + * if there is an unknown error + */ + private VoidResponse remove(String queryId, DatawaveUserDetails currentUser, boolean adminOverride) throws QueryException { + try { + // make sure the query is valid, and the user can act on it + QueryStatus queryStatus = validateRequest(queryId, currentUser, adminOverride); + + // remove the query if it is not running + if (!queryStatus.isRunning()) { + if (!remove(queryStatus)) { + throw new QueryException("Failed to remove " + queryId); + } + } else { + throw new BadRequestQueryException("Cannot remove a running query.", HttpStatus.SC_BAD_REQUEST + "-1"); + } + + VoidResponse response = new VoidResponse(); + response.addMessage(queryId + " removed."); + return response; + } catch (QueryException e) { + throw e; + } catch (Exception e) { + log.error("Unknown error removing query {}", queryId, e); + throw new QueryException(DatawaveErrorCode.QUERY_REMOVAL_ERROR, e, "Unknown error removing query " + queryId); + } + } + + private boolean remove(QueryStatus queryStatus) throws IOException { + return queryStorageCache.deleteQuery(queryStatus.getQueryKey().getQueryId()); + } + + /** + * Updates the specified query. + *

+ * Update can only be called on a defined, or running query.
+ * Auditing is not performed when updating a defined query.
+ * No auditable parameters should be updated when updating a running query.
+ * Any query parameter can be updated for a defined query.
+ * Query parameters which don't affect the scope of the query can be updated for a running query.
+ * The list of parameters that can be updated for a running query is configurable.
+ * Auditable parameters should never be added to the updatable parameters configuration.
+ * Query string, date range, query logic, and auths should never be updated for a running query.
+ * Only the query owner can call update on the specified query. + * + * @param queryId + * the query id, not null + * @param parameters + * the query parameter updates, not null + * @param currentUser + * the user who called this method, not null + * @return a generic response containing the query id + * @throws NotFoundQueryException + * if the query cannot be found + * @throws UnauthorizedQueryException + * if the user doesn't own the query + * @throws BadRequestQueryException + * if parameter validation fails + * @throws BadRequestQueryException + * if query logic parameter validation fails + * @throws UnauthorizedQueryException + * if the user doesn't have access to the requested query logic + * @throws BadRequestQueryException + * if security marking validation fails + * @throws BadRequestQueryException + * if the query is not defined, or running + * @throws QueryException + * if query lock acquisition fails + * @throws QueryException + * if the update call is interrupted + * @throws BadRequestQueryException + * if no parameters are specified + * @throws QueryException + * if query storage fails + * @throws QueryException + * if there is an unknown error + */ + public GenericResponse update(String queryId, MultiValueMap parameters, DatawaveUserDetails currentUser) throws QueryException { + String user = ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()); + if (log.isDebugEnabled()) { + log.info("Request: {}/update from {} with params: {}", queryId, user, parameters); + } else { + log.info("Request: {}/update from {}", queryId, user); + } + + try { + // make sure the query is valid, and the user can act on it + QueryStatus queryStatus = validateRequest(queryId, currentUser); + + GenericResponse response = new GenericResponse<>(); + if (!parameters.isEmpty()) { + // recreate the query parameters + MultiValueMap currentParams = new LinkedMultiValueMap<>(queryStatus.getQuery().toMap()); + + // remove some of the copied params + currentParams.remove(QUERY_ID); + currentParams.remove(USER_DN); + currentParams.remove(DN_LIST); + + boolean updated; + if (queryStatus.getQueryState() == DEFINE) { + // update all parameters if the state is defined + updated = updateParameters(parameters, currentParams); + + // redefine the query + if (updated) { + storeQuery(currentParams.getFirst(QUERY_LOGIC_NAME), currentParams, queryStatus.getQueryKey().getQueryPool(), currentUser, DEFINE, + queryId); + } + } else if (queryStatus.isRunning()) { + // if the query is created/running, update safe parameters only + List unsafeParams = new ArrayList<>(parameters.keySet()); + List safeParams = new ArrayList<>(queryProperties.getUpdatableParams()); + safeParams.retainAll(parameters.keySet()); + unsafeParams.removeAll(safeParams); + + // only update a running query if the params are all safe + if (unsafeParams.isEmpty()) { + updated = updateParameters(safeParams, parameters, currentParams); + + if (updated) { + // validate the update + String queryLogicName = currentParams.getFirst(QUERY_LOGIC_NAME); + validateQuery(queryLogicName, currentParams, currentUser); + + // create a new query object + Query query = createQuery(queryLogicName, currentParams, currentUser, queryId); + + // save the new query object in the cache + queryStatusUpdateUtil.lockedUpdate(queryId, status -> status.setQuery(query)); + } + } else { + throw new BadRequestQueryException("Cannot update the following parameters for a running query: " + String.join(", ", unsafeParams), + HttpStatus.SC_BAD_REQUEST + "-1"); + } + } else { + throw new BadRequestQueryException("Cannot update a query unless it is defined or running.", HttpStatus.SC_BAD_REQUEST + "-1"); + } + + response.setResult(queryId); + if (updated) { + response.addMessage(queryId + " updated."); + } else { + response.addMessage(queryId + " unchanged."); + } + } else { + throw new BadRequestQueryException("No parameters specified for update.", HttpStatus.SC_BAD_REQUEST + "-1"); + } + + return response; + } catch (QueryException e) { + throw e; + } catch (Exception e) { + log.error("Unknown error updating query {}", queryId, e); + throw new QueryException(DatawaveErrorCode.QUERY_UPDATE_ERROR, e, "Unknown error updating query " + queryId); + } + } + + /** + * Creates a copy of the specified query. + *

+ * Duplicate can be called on any query, whether it's running or not.
+ * Duplicate creates a new, identical query, with a new query id.
+ * Provided parameter updates will be applied to the new query.
+ * Duplicated queries will start running immediately.
+ * Auditing is performed before the new query is started. + * + * @param queryId + * the query id, not null + * @param parameters + * the query parameter updates, not null + * @param currentUser + * the user who called this method, not null + * @return a generic response containing the new query id + * @throws NotFoundQueryException + * if the query cannot be found + * @throws UnauthorizedQueryException + * if the user doesn't own the query + * @throws QueryException + * if query lock acquisition fails + * @throws BadRequestQueryException + * if parameter validation fails + * @throws BadRequestQueryException + * if query logic parameter validation fails + * @throws UnauthorizedQueryException + * if the user doesn't have access to the requested query logic + * @throws BadRequestQueryException + * if security marking validation fails + * @throws BadRequestQueryException + * if auditing fails + * @throws QueryException + * if query storage fails + * @throws QueryException + * if there is an unknown error + */ + public GenericResponse duplicate(String queryId, MultiValueMap parameters, DatawaveUserDetails currentUser) throws QueryException { + String user = ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()); + if (log.isDebugEnabled()) { + log.info("Request: {}/duplicate from {} with params: {}", queryId, user, parameters); + } else { + log.info("Request: {}/duplicate from {}", queryId, user); + } + + try { + // make sure the query is valid, and the user can act on it + QueryStatus queryStatus = validateRequest(queryId, currentUser); + + // define a duplicate query from the existing query + TaskKey taskKey = duplicate(queryStatus, parameters, currentUser); + + GenericResponse response = new GenericResponse<>(); + response.addMessage(queryId + " duplicated."); + response.setResult(taskKey.getQueryId()); + response.setHasResults(true); + return response; + } catch (QueryException e) { + throw e; + } catch (Exception e) { + log.error("Unknown error duplicating query {}", queryId, e); + throw new QueryException(DatawaveErrorCode.QUERY_DUPLICATION_ERROR, e, "Unknown error duplicating query " + queryId); + } + } + + /** + * Creates a copy of the specified query. + *

+ * Duplicate can be called on any query, whether it's running or not.
+ * Duplicate creates a new, identical query, with a new query id.
+ * Provided parameter updates will be applied to the new query.
+ * Duplicated queries will start running immediately.
+ * Auditing is performed before the new query is started. + * + * @param queryStatus + * the query status, not null + * @param parameters + * the query parameter updates, not null + * @param currentUser + * the user who called this method, not null + * @return a generic response containing the new query id + * @throws NotFoundQueryException + * if the query cannot be found + * @throws UnauthorizedQueryException + * if the user doesn't own the query + * @throws QueryException + * if query lock acquisition fails + * @throws BadRequestQueryException + * if parameter validation fails + * @throws BadRequestQueryException + * if query logic parameter validation fails + * @throws UnauthorizedQueryException + * if the user doesn't have access to the requested query logic + * @throws BadRequestQueryException + * if security marking validation fails + * @throws BadRequestQueryException + * if auditing fails + * @throws QueryException + * if query storage fails + */ + private TaskKey duplicate(QueryStatus queryStatus, MultiValueMap parameters, DatawaveUserDetails currentUser) throws QueryException { + // recreate the query parameters + MultiValueMap currentParams = new LinkedMultiValueMap<>(queryStatus.getQuery().toMap()); + + // remove some of the copied params + currentParams.remove(QUERY_ID); + currentParams.remove(USER_DN); + currentParams.remove(DN_LIST); + + // updated all of the passed in parameters + updateParameters(parameters, currentParams); + + // define a duplicate query + return storeQuery(currentParams.getFirst(QUERY_LOGIC_NAME), currentParams, queryStatus.getQueryKey().getQueryPool(), currentUser, CREATE); + } + + private boolean updateParameters(MultiValueMap newParameters, MultiValueMap currentParams) throws BadRequestQueryException { + return updateParameters(new ArrayList<>(newParameters.keySet()), newParameters, currentParams); + } + + /** + * Updates the current parameters with the new parameters, limited to the specified parameter names. + * + * @param parameterNames + * the parameters names to override, not null + * @param newParameters + * the new parameters, may be null + * @param currentParams + * the current parameters, not null + * @return true is the current parameters were changed + * @throws BadRequestQueryException + * if a parameter is passed without a value + */ + private boolean updateParameters(Collection parameterNames, MultiValueMap newParameters, MultiValueMap currentParams) + throws BadRequestQueryException { + boolean paramsUpdated = false; + for (String paramName : parameterNames) { + if (CollectionUtils.isNotEmpty(newParameters.get(paramName))) { + if (!newParameters.get(paramName).get(0).equals(currentParams.getFirst(paramName))) { + // if the new value differs from the old value, update the old value + currentParams.put(paramName, newParameters.remove(paramName)); + paramsUpdated = true; + } + } else { + throw new BadRequestQueryException("Cannot update a query parameter without a value: " + paramName, HttpStatus.SC_BAD_REQUEST + "-1"); + } + } + return paramsUpdated; + } + + /** + * Gets a list of queries for the calling user. + *

+ * Returns all matching queries owned by the calling user, filtering by query id and query name. + * + * @param queryId + * the query id, may be null + * @param queryName + * the query name, may be null + * @param currentUser + * the user who called this method, not null + * @return a list response containing the matching queries + * @throws QueryException + * if there is an unknown error + */ + public QueryImplListResponse list(String queryId, String queryName, DatawaveUserDetails currentUser) throws QueryException { + log.info("Request: list from {} for queryId: {}, queryName: {}", ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()), queryId, + queryName); + + return list(queryId, queryName, ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getDn().subjectDN())); + } + + /** + * Gets a list of queries for the specified user using admin privileges. + *

+ * Returns all matching queries owned by any user, filtered by user ID, query ID, and query name.
+ * Only admin users should be allowed to call this method. + * + * @param queryId + * the query id, may be null + * @param queryName + * the query name, may be null + * @param userId + * the user whose queries we want to list, may be null + * @param currentUser + * the user who called this method, not null + * @return a list response containing the matching queries + * @throws QueryException + * if there is an unknown error + */ + public QueryImplListResponse adminList(String queryId, String queryName, String userId, DatawaveUserDetails currentUser) throws QueryException { + log.info("Request: adminList from {} for queryId: {}, queryName: {}, userId: {}", + ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()), queryId, queryName, userId); + + return list(queryId, queryName, userId); + } + + /** + * Gets a list of all matching queries, filtered by user ID, query ID, and query name. + * + * @param queryId + * the query id, may be null + * @param queryName + * the query name, may be null + * @param userId + * the user whose queries we want to list, may be null + * @return a list response containing the matching queries + * @throws QueryException + * if there is an unknown error + */ + private QueryImplListResponse list(String queryId, String queryName, String userId) throws QueryException { + try { + List queries; + if (StringUtils.isNotBlank(queryId)) { + // get the query for the given id + queries = new ArrayList<>(); + + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + if (queryStatus != null) { + queries.add(queryStatus.getQuery()); + } + } else { + // get all of the queries + queries = queryStorageCache.getQueryStatus().stream().map(QueryStatus::getQuery).collect(Collectors.toList()); + } + + // only keep queries with the given userId and query name + queries.removeIf(q -> (userId != null && !q.getOwner().equals(userId)) || (queryName != null && !q.getQueryName().equals(queryName))); + + QueryImplListResponse response = new QueryImplListResponse(); + response.setQuery(queries); + return response; + } catch (Exception e) { + log.error("Unknown error listing queries for {}", userId, e); + throw new QueryException(DatawaveErrorCode.QUERY_LISTING_ERROR, e, "Unknown error listing queries for " + userId); + } + } + + /** + * Gets the plan for the given query for the calling user. + * + * @param queryId + * the query id, not null + * @param currentUser + * the user who called this method, not null + * @return the query plan for the matching query + * @throws NotFoundQueryException + * if the query cannot be found + * @throws UnauthorizedQueryException + * if the user doesn't own the query + * @throws QueryException + * if query lock acquisition fails + * @throws QueryException + * if query storage fails + * @throws QueryException + * if there is an unknown error + */ + public GenericResponse plan(String queryId, DatawaveUserDetails currentUser) throws QueryException { + log.info("Request: plan from {} for queryId: {}", ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()), queryId); + + try { + // make sure the query is valid, and the user can act on it + QueryStatus queryStatus = validateRequest(queryId, currentUser); + + GenericResponse response = new GenericResponse<>(); + if (queryStatus.getPlan() != null) { + response.setResult(queryStatus.getPlan()); + response.setHasResults(true); + } + return response; + } catch (QueryException e) { + throw e; + } catch (Exception e) { + log.error("Unknown error getting plan for query {}", queryId, e); + throw new QueryException(DatawaveErrorCode.QUERY_PLAN_ERROR, e, "Unknown error getting plan for query " + queryId); + } + } + + /** + * Gets the predictions for the given query for the calling user. + * + * @param queryId + * the query id, not null + * @param currentUser + * the user who called this method, not null + * @return the query predictions for the matching query + * @throws NotFoundQueryException + * if the query cannot be found + * @throws UnauthorizedQueryException + * if the user doesn't own the query + * @throws QueryException + * if query lock acquisition fails + * @throws QueryException + * if query storage fails + * @throws QueryException + * if there is an unknown error + */ + public GenericResponse predictions(String queryId, DatawaveUserDetails currentUser) throws QueryException { + log.info("Request: predictions from {} for queryId: {}", ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()), queryId); + + try { + // make sure the query is valid, and the user can act on it + QueryStatus queryStatus = validateRequest(queryId, currentUser); + + GenericResponse response = new GenericResponse<>(); + if (CollectionUtils.isNotEmpty(queryStatus.getPredictions())) { + response.setResult(queryStatus.getPredictions().toString()); + response.setHasResults(true); + } + return response; + } catch (QueryException e) { + throw e; + } catch (Exception e) { + log.error("Unknown error getting predictions for query {}", queryId, e); + throw new QueryException(DatawaveErrorCode.QUERY_PLAN_ERROR, e, "Unknown error getting predictions for query " + queryId); + } + } + + public QueryStatus validateRequest(String queryId, DatawaveUserDetails currentUser) throws QueryException { + return validateRequest(queryId, currentUser, false); + } + + /** + * Validates the user request by ensuring that the query exists, and the user owns the query or is an admin. + * + * @param queryId + * the query id, not null + * @param currentUser + * the user who called this method, not null + * @param adminOverride + * whether or not this is an admin request + * @return a query status object for the specified query + * @throws NotFoundQueryException + * if the query cannot be found + * @throws UnauthorizedQueryException + * if the user doesn't own the query + */ + public QueryStatus validateRequest(String queryId, DatawaveUserDetails currentUser, boolean adminOverride) + throws NotFoundQueryException, UnauthorizedQueryException { + // does the query exist? + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + if (queryStatus == null) { + throw new NotFoundQueryException(DatawaveErrorCode.NO_QUERY_OBJECT_MATCH, MessageFormat.format("{0}", queryId)); + } + + // admin requests can operate on any query, regardless of ownership + if (!adminOverride) { + // does the current user own this query? + String userId = ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getDn().subjectDN()); + Query query = queryStatus.getQuery(); + if (!query.getOwner().equals(userId)) { + throw new UnauthorizedQueryException(DatawaveErrorCode.QUERY_OWNER_MISMATCH, MessageFormat.format("{0} != {1}", userId, query.getOwner())); + } + } + + return queryStatus; + } + + /** + * Receives remote query requests and handles them. + *

+ * Cancels any running next calls when the request method is {@link QueryRequest.Method#CANCEL}.
+ * Takes no action for other remote query requests. + * + * @param queryRequest + * the remote query request, not null + * @param originService + * the address of the service which sent the request + * @param destinationService + * the address of the service which this event was sent to + */ + @Override + public void handleRemoteRequest(QueryRequest queryRequest, String originService, String destinationService) { + try { + if (queryRequest.getMethod() == QueryRequest.Method.CANCEL) { + log.trace("Received remote cancel request from {} for {}.", originService, destinationService); + cancel(queryRequest.getQueryId(), false); + } else if (queryRequest.getMethod() == QueryRequest.Method.CREATE || queryRequest.getMethod() == QueryRequest.Method.PLAN + || queryRequest.getMethod() == QueryRequest.Method.PREDICT) { + log.trace("Received remote {} request from {} for {}.", queryRequest.getMethod().name(), originService, destinationService); + if (queryLatchMap.containsKey(queryRequest.getQueryId())) { + queryLatchMap.get(queryRequest.getQueryId()).countDown(); + } else { + log.warn("Unable to decrement {} latch for query {}", queryRequest.getMethod().name(), queryRequest.getQueryId()); + } + } else { + log.debug("No handling specified for remote query request method: {} from {} for {}", queryRequest.getMethod(), originService, + destinationService); + } + } catch (Exception e) { + log.error("Unknown error handling remote request: {} from {} for {}", queryRequest, originService, destinationService); + } + } + + /** + * Creates and submits an audit record to the audit service. + *

+ * The audit request submission will fail if the audit service is unable to validate the audit message. + * + * @param query + * the query to be audited, not null + * @param queryLogic + * the query logic, not null + * @param parameters + * the query parameters, not null + * @param currentUser + * the user who called this method, not null + * @throws BadRequestQueryException + * if the audit parameters fail validation + * @throws BadRequestQueryException + * if there is an error auditing the query + */ + public void audit(Query query, QueryLogic queryLogic, MultiValueMap parameters, DatawaveUserDetails currentUser) + throws BadRequestQueryException { + List selectors = null; + try { + selectors = queryLogic.getSelectors(query); + } catch (Exception e) { + log.error("Error accessing query selector", e); + } + + // @formatter:off + audit(query.getId().toString(), + queryLogic.getAuditType(query), + queryLogic.getLogicName(), + query.getQuery(), + selectors, + parameters, + currentUser); + // @formatter:on + } + + /** + * Creates and submits an audit record to the audit service. + *

+ * The audit request submission will fail if the audit service is unable to validate the audit message. + * + * @param auditId + * the id to use when auditing, not null + * @param auditType + * the audit type, not null + * @param logicName + * the logic name, not null + * @param query + * the query, not null + * @param selectors + * the list of selectors, may be null + * @param parameters + * the query parameters, not null + * @param currentUser + * the user who called this method, not null + * @throws BadRequestQueryException + * if the audit parameters fail validation + * @throws BadRequestQueryException + * if there is an error auditing the query + */ + public void audit(String auditId, Auditor.AuditType auditType, String logicName, String query, List selectors, + MultiValueMap parameters, DatawaveUserDetails currentUser) throws BadRequestQueryException { + + // if we haven't already, validate the markings + if (getSecurityMarking().toColumnVisibilityString() == null) { + validateSecurityMarkings(parameters); + } + + // set some audit parameters which are used internally + setInternalAuditParameters(logicName, currentUser.getPrimaryUser().getDn().subjectDN(), parameters); + + parameters.add(PrivateAuditConstants.AUDIT_TYPE, auditType.name()); + if (auditType != Auditor.AuditType.NONE) { + // audit the query before execution + try { + if (CollectionUtils.isNotEmpty(selectors)) { + parameters.put(PrivateAuditConstants.SELECTORS, selectors); + } + + // is the user didn't set an audit id, use the query id + if (!parameters.containsKey(AuditParameters.AUDIT_ID)) { + parameters.set(AuditParameters.AUDIT_ID, auditId); + } + + // @formatter:off + AuditClient.Request auditRequest = new AuditClient.Request.Builder() + .withParams(parameters) + .withQueryExpression(query) + .withDatawaveUserDetails(currentUser) + .withMarking(getSecurityMarking()) + .withAuditType(auditType) + .withQueryLogic(logicName) + .build(); + // @formatter:on + + log.info("[{}] Sending audit request with parameters {}", auditId, auditRequest); + + auditClient.submit(auditRequest); + } catch (IllegalArgumentException e) { + log.error("Error validating audit parameters", e); + throw new BadRequestQueryException(DatawaveErrorCode.MISSING_REQUIRED_PARAMETER, e); + } catch (Exception e) { + log.error("Error auditing query", e); + throw new BadRequestQueryException(DatawaveErrorCode.QUERY_AUDITING_ERROR, e); + } + } + } + + protected boolean isAdminUser(DatawaveUserDetails currentUser) { + boolean isAdminUser = false; + for (String role : currentUser.getPrimaryUser().getRoles()) { + if (queryProperties.getAdminRoles().contains(role)) { + isAdminUser = true; + break; + } + } + return isAdminUser; + } + + /** + * Gets the pool name for this request. + *

+ * If no pool is specified in the query parameters, the default pool will be used. + * + * @return the pool name for this query + */ + protected String getPoolName(String pool, boolean isAdminUser) { + QueryParameters queryParameters = getQueryParameters(); + return (isAdminUser && (queryParameters.getPool() != null)) ? queryParameters.getPool() + : ((pool != null) ? pool : queryProperties.getDefaultParams().getPool()); + } + + /** + * Gets the pool-specific executor service name. + * + * @param poolName + * the pool name, null returns *-null + * @return the pool-specific executor service name + */ + protected String getPooledExecutorName(String poolName) { + return String.join("-", Arrays.asList(queryProperties.getExecutorServiceName(), poolName)); + } + + /** + * Publishes a next event for the given query id and pool. + * + * @param queryId + * the query id, not null + * @param queryPool + * the pool to use, not null + */ + public void publishNextEvent(String queryId, String queryPool) { + publishExecutorEvent(QueryRequest.next(queryId), queryPool); + } + + private void publishExecutorEvent(QueryRequest queryRequest, String queryPool) { + // @formatter:off + eventPublisher.publishEvent( + new RemoteQueryRequestEvent( + this, + busProperties.getId(), + getPooledExecutorName(queryPool), + queryRequest)); + // @formatter:on + } + + private void publishSelfEvent(QueryRequest queryRequest) { + // @formatter:off + eventPublisher.publishEvent( + new RemoteQueryRequestEvent( + this, + busProperties.getId(), + selfDestination, + queryRequest)); + // @formatter:on + } + + private String getSelfDestination() { + String id = busProperties.getId(); + if (id.contains(":")) { + return id.substring(0, id.indexOf(":")); + } + return id; + } + + /** + * Gets the maximum number of concurrent query tasks allowed for this logic. + *

+ * If the max concurrent tasks is overridden, that value will be used.
+ * Otherwise, the value is determined via the query logic, if defined, or via the configuration default. + * + * @param queryLogic + * the requested query logic, not null + * @return the maximum concurrent tasks limit + */ + protected int getMaxConcurrentTasks(QueryLogic queryLogic) { + // if there's an override, use it + QueryParameters queryParameters = getQueryParameters(); + if (queryParameters.isMaxConcurrentTasksOverridden()) { + return queryParameters.getMaxConcurrentTasks(); + } + // if the query logic has a limit, use it + else if (queryLogic.getMaxConcurrentTasks() > 0) { + return queryLogic.getMaxConcurrentTasks(); + } + // otherwise, use the configuration default + else { + return queryProperties.getDefaultParams().getMaxConcurrentTasks(); + } + } + + /** + * Creates and initializes a query object using the provided query parameters. + *

+ * If a query id is not specified, then a random one will be generated. + * + * @param queryLogicName + * the requested query logic, not null + * @param parameters + * the query parameters, not null + * @param currentUser + * the current user, not null + * @param queryId + * the desired query id, may be null + * @return an instantiated query object + */ + protected Query createQuery(String queryLogicName, MultiValueMap parameters, DatawaveUserDetails currentUser, String queryId) { + String userDn = currentUser.getPrimaryUser().getDn().subjectDN(); + List dnList = getDNs(currentUser); + + QueryParameters queryParameters = getQueryParameters(); + SecurityMarking securityMarking = getSecurityMarking(); + + Query query = responseObjectFactory.getQueryImpl(); + query.initialize(userDn, dnList, queryLogicName, queryParameters, queryParameters.getUnknownParameters(parameters)); + query.setColumnVisibility(securityMarking.toColumnVisibilityString()); + query.setUncaughtExceptionHandler(new QueryUncaughtExceptionHandler()); + Thread.currentThread().setUncaughtExceptionHandler(query.getUncaughtExceptionHandler()); + if (queryId != null) { + query.setId(UUID.fromString(queryId)); + } + return query; + } + + /** + * Validates the query parameters, security markings, and instantiates the requested query logic. + *

+ * If the query is not valid, an exception will be thrown. + * + * @param queryLogicName + * the requested query logic, not null + * @param parameters + * the query parameters, not null + * @param currentUser + * the user who called this method, not null + * @return the query logic + * @throws BadRequestQueryException + * if parameter validation fails + * @throws BadRequestQueryException + * if query logic parameter validation fails + * @throws UnauthorizedQueryException + * if the user doesn't have access to the requested query logic + * @throws BadRequestQueryException + * if security marking validation fails + */ + public QueryLogic validateQuery(String queryLogicName, MultiValueMap parameters, DatawaveUserDetails currentUser) + throws BadRequestQueryException, UnauthorizedQueryException { + // validate the query parameters + validateParameters(queryLogicName, parameters); + + // create the query logic, and perform query logic parameter validation + QueryLogic queryLogic = createQueryLogic(queryLogicName, currentUser); + validateQueryLogic(queryLogic, parameters, currentUser); + + // validate the security markings + validateSecurityMarkings(parameters); + + return queryLogic; + } + + /** + * Performs query parameter validation. + *

+ * If the parameters are not valid, an exception will be thrown. + * + * @param queryLogicName + * the requested query logic, not null + * @param parameters + * the query parameters, not null + * @throws BadRequestQueryException + * if parameter validation fails + * @throws BadRequestQueryException + * if an invalid page size was requested + * @throws BadRequestQueryException + * if an invalid page timeout was requested + * @throws BadRequestQueryException + * if an invalid begin and end date was requested + */ + protected void validateParameters(String queryLogicName, MultiValueMap parameters) throws BadRequestQueryException { + // add query logic name to parameters + parameters.set(QUERY_LOGIC_NAME, queryLogicName); + + log.debug(writeValueAsString(parameters)); + + // Pull "params" values into individual query parameters for validation on the query logic. + // This supports the deprecated "params" value (both on the old and new API). Once we remove the deprecated + // parameter, this code block can go away. + if (parameters.get(QueryParameters.QUERY_PARAMS) != null) { + // @formatter:off + parameters.get(QueryParameters.QUERY_PARAMS).stream() + .flatMap(params -> QueryUtil.parseParameters(params).stream()) + .forEach(x -> parameters.add(x.getParameterName(), x.getParameterValue())); + // @formatter:on + } + + parameters.remove(AuditParameters.QUERY_SECURITY_MARKING_COLVIZ); + parameters.remove(AuditParameters.USER_DN); + parameters.remove(AuditParameters.QUERY_AUDIT_TYPE); + + // Ensure that all required parameters exist prior to validating the values. + QueryParameters queryParameters = getQueryParameters(); + try { + queryParameters.validate(parameters); + } catch (IllegalArgumentException e) { + log.error("Unable to validate query parameters", e); + throw new BadRequestQueryException("Unable to validate query parameters.", e, HttpStatus.SC_BAD_REQUEST + "-1"); + } + + // The pageSize and expirationDate checks will always be false when called from the RemoteQueryExecutor. + // Leaving for now until we can test to ensure that is always the case. + if (queryParameters.getPagesize() <= 0) { + log.error("Invalid page size: {}", queryParameters.getPagesize()); + throw new BadRequestQueryException(DatawaveErrorCode.INVALID_PAGE_SIZE); + } + + long pageMinTimeoutMillis = queryProperties.getExpiration().getPageMinTimeoutMillis(); + long pageMaxTimeoutMillis = queryProperties.getExpiration().getPageMaxTimeoutMillis(); + long pageTimeoutMillis = TimeUnit.MINUTES.toMillis(queryParameters.getPageTimeout()); + if (queryParameters.getPageTimeout() != -1 && (pageTimeoutMillis < pageMinTimeoutMillis || pageTimeoutMillis > pageMaxTimeoutMillis)) { + log.error("Invalid page timeout: {}", queryParameters.getPageTimeout()); + throw new BadRequestQueryException(DatawaveErrorCode.INVALID_PAGE_TIMEOUT); + } + + // Ensure begin date does not occur after the end date (if dates are not null) + if ((queryParameters.getBeginDate() != null && queryParameters.getEndDate() != null) + && queryParameters.getBeginDate().after(queryParameters.getEndDate())) { + log.error("Invalid begin and/or end date: {}", queryParameters.getBeginDate() + " - " + queryParameters.getEndDate()); + throw new BadRequestQueryException(DatawaveErrorCode.BEGIN_DATE_AFTER_END_DATE); + } + } + + /** + * Creates a query logic instance for the given user. + *

+ * The user's roles will be checked when instantiating the query logic. + * + * @param queryLogicName + * the requested query logic, not null + * @param currentUser + * the user who called this method, not null + * @return the requested query logic + * @throws BadRequestQueryException + * if the query logic does not exist + * @throws BadRequestQueryException + * if the user does not have the required roles for the query logic + */ + protected QueryLogic createQueryLogic(String queryLogicName, DatawaveUserDetails currentUser) throws BadRequestQueryException { + // will throw IllegalArgumentException if not defined + try { + return queryLogicFactory.getQueryLogic(queryLogicName, currentUser); + } catch (Exception e) { + log.error("Failed to get query logic for {}", queryLogicName, e); + throw new BadRequestQueryException(DatawaveErrorCode.QUERY_LOGIC_ERROR, e); + } + } + + /** + * Performs query parameter validation using the query logic. + * + * @param queryLogic + * the requested query logic, not null + * @param parameters + * the query parameters, not null + * @param currentUser + * the user who called this method, not null + * @throws BadRequestQueryException + * if query logic parameter validation fails + * @throws BadRequestQueryException + * if an invalid page size was requested + * @throws BadRequestQueryException + * if an invalid max results override was requested + * @throws BadRequestQueryException + * if an invalid max concurrent tasks override was requested + * @throws UnauthorizedQueryException + * if the user doesn't have access to the requested query logic + */ + protected void validateQueryLogic(QueryLogic queryLogic, MultiValueMap parameters, DatawaveUserDetails currentUser) + throws BadRequestQueryException, UnauthorizedQueryException { + try { + queryLogic.validate(parameters); + } catch (IllegalArgumentException e) { + log.error("Unable to validate query parameters with query logic", e); + throw new BadRequestQueryException("Unable to validate query parameters with query logic.", e, HttpStatus.SC_BAD_REQUEST + "-1"); + } + + // always check against the max + QueryParameters queryParameters = getQueryParameters(); + if (queryLogic.getMaxPageSize() > 0 && queryParameters.getPagesize() > queryLogic.getMaxPageSize()) { + log.error("Invalid page size: {} vs {}", queryParameters.getPagesize(), queryLogic.getMaxPageSize()); + throw new BadRequestQueryException(DatawaveErrorCode.PAGE_SIZE_TOO_LARGE, MessageFormat.format("Max = {0}.", queryLogic.getMaxPageSize())); + } + + // If the user is not privileged, make sure they didn't exceed the limits for the following parameters + if (!currentUser.getPrimaryUser().getRoles().contains(queryProperties.getPrivilegedRole())) { + // validate the max results override relative to the max results on a query logic + // privileged users however can set whatever they want + if (queryParameters.isMaxResultsOverridden() && queryLogic.getMaxResults() >= 0) { + if (queryParameters.getMaxResultsOverride() < 0 || (queryLogic.getMaxResults() < queryParameters.getMaxResultsOverride())) { + log.error("Invalid max results override: {} vs {}", queryParameters.getMaxResultsOverride(), queryLogic.getMaxResults()); + throw new BadRequestQueryException(DatawaveErrorCode.INVALID_MAX_RESULTS_OVERRIDE, + MessageFormat.format("Max = {0}.", queryLogic.getMaxResults())); + } + } + + // validate the max concurrent tasks override relative to the max concurrent tasks on a query logic + // privileged users however can set whatever they want + if (queryParameters.isMaxConcurrentTasksOverridden() && queryLogic.getMaxConcurrentTasks() >= 0) { + if (queryParameters.getMaxConcurrentTasks() < 0 || (queryLogic.getMaxConcurrentTasks() < queryParameters.getMaxConcurrentTasks())) { + log.error("Invalid max concurrent tasks override: {} vs {}", queryParameters.getMaxConcurrentTasks(), queryLogic.getMaxConcurrentTasks()); + throw new BadRequestQueryException(DatawaveErrorCode.INVALID_MAX_CONCURRENT_TASKS_OVERRIDE, + MessageFormat.format("Max = {0}.", queryLogic.getMaxConcurrentTasks())); + } + } + } + + // Verify that the calling principal has access to the query logic. + List dnList = getDNs(currentUser); + if (!queryLogic.containsDNWithAccess(dnList)) { + throw new UnauthorizedQueryException("None of the DNs used have access to this query logic: " + dnList, 401); + } + } + + /** + * Performs security marking validation using the configured security markings. See {@link SecurityMarking#validate}. + * + * @param parameters + * the query parameters, not null + * @throws BadRequestQueryException + * if security marking validation fails + */ + public void validateSecurityMarkings(MultiValueMap parameters) throws BadRequestQueryException { + try { + getSecurityMarking().validate(parameters); + } catch (IllegalArgumentException e) { + log.error("Failed security markings validation", e); + throw new BadRequestQueryException(DatawaveErrorCode.SECURITY_MARKING_CHECK_ERROR, e); + } + } + + /** + * Sets some audit parameters which are used internally to assist with auditing. + *

+ * If any of these parameters exist in the current parameter map, they will first be removed. + * + * @param queryLogicName + * the requested query logic, not null + * @param userDn + * the user dn, not null + * @param parameters + * the query parameters, not null + */ + public void setInternalAuditParameters(String queryLogicName, String userDn, MultiValueMap parameters) { + // Set private audit-related parameters, stripping off any that the user might have passed in first. + // These are parameters that aren't passed in by the user, but rather are computed from other sources. + PrivateAuditConstants.stripPrivateParameters(parameters); + parameters.add(PrivateAuditConstants.LOGIC_CLASS, queryLogicName); + parameters.set(PrivateAuditConstants.COLUMN_VISIBILITY, getSecurityMarking().toColumnVisibilityString()); + parameters.add(PrivateAuditConstants.USER_DN, userDn); + } + + private String writeValueAsString(Object object) { + String stringValue; + try { + stringValue = mapper.writeValueAsString(object); + } catch (JsonProcessingException e) { + stringValue = String.valueOf(object); + } + return stringValue; + } + + public QueryParameters getQueryParameters() { + if (queryParametersOverride.get() != null) { + return queryParametersOverride.get(); + } else { + return queryParameters; + } + } + + public SecurityMarking getSecurityMarking() { + if (securityMarkingOverride.get() != null) { + return securityMarkingOverride.get(); + } else { + return securityMarking; + } + } + + public BaseQueryMetric getBaseQueryMetric() { + if (baseQueryMetricOverride.get() != null) { + return baseQueryMetricOverride.get(); + } else { + return baseQueryMetric; + } + } + + public ThreadLocal getQueryParametersOverride() { + return queryParametersOverride; + } + + public ThreadLocal getSecurityMarkingOverride() { + return securityMarkingOverride; + } + + public ThreadLocal getBaseQueryMetricOverride() { + return baseQueryMetricOverride; + } + + public List getDNs(DatawaveUserDetails user) { + return user.getProxiedUsers().stream().map(u -> u.getDn().subjectDN()).collect(Collectors.toList()); + } +} diff --git a/service/src/main/java/datawave/microservice/query/QueryService.java b/service/src/main/java/datawave/microservice/query/QueryService.java new file mode 100644 index 00000000..aaf85e33 --- /dev/null +++ b/service/src/main/java/datawave/microservice/query/QueryService.java @@ -0,0 +1,17 @@ +package datawave.microservice.query; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.autoconfigure.web.servlet.error.ErrorMvcAutoConfiguration; +import org.springframework.cloud.client.discovery.EnableDiscoveryClient; + +/** + * Launcher for the query service + */ +@EnableDiscoveryClient +@SpringBootApplication(scanBasePackages = "datawave.microservice", exclude = {ErrorMvcAutoConfiguration.class}) +public class QueryService { + public static void main(String[] args) { + SpringApplication.run(QueryService.class, args); + } +} diff --git a/service/src/main/java/datawave/microservice/query/WebController.java b/service/src/main/java/datawave/microservice/query/WebController.java new file mode 100644 index 00000000..004aa248 --- /dev/null +++ b/service/src/main/java/datawave/microservice/query/WebController.java @@ -0,0 +1,30 @@ +package datawave.microservice.query; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.info.GitProperties; +import org.springframework.stereotype.Controller; +import org.springframework.ui.Model; +import org.springframework.web.bind.annotation.GetMapping; + +@Controller +public class WebController { + + @Autowired + private GitProperties gitProperties; + + @GetMapping("/") + public String index(Model model) { + model.addAttribute("gitBuildUserEmail", gitProperties.get("build.user.email")); + model.addAttribute("gitBranch", gitProperties.getBranch()); + model.addAttribute("gitCommitId", gitProperties.getCommitId()); + model.addAttribute("gitBuildTime", gitProperties.getInstant("build.time")); + + return "index"; + } + + @GetMapping("/query_help.html") + public String query_help(Model model) { + return "query_help"; + } + +} diff --git a/service/src/main/java/datawave/microservice/query/cachedresults/CachedResultsQueryController.java b/service/src/main/java/datawave/microservice/query/cachedresults/CachedResultsQueryController.java new file mode 100644 index 00000000..4a40f4cb --- /dev/null +++ b/service/src/main/java/datawave/microservice/query/cachedresults/CachedResultsQueryController.java @@ -0,0 +1,317 @@ +package datawave.microservice.query.cachedresults; + +import static datawave.core.query.cachedresults.CachedResultsQueryParameters.CONDITIONS; +import static datawave.core.query.cachedresults.CachedResultsQueryParameters.FIELDS; +import static datawave.core.query.cachedresults.CachedResultsQueryParameters.GROUPING; +import static datawave.core.query.cachedresults.CachedResultsQueryParameters.ORDER; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.http.MediaType; +import org.springframework.security.access.annotation.Secured; +import org.springframework.security.core.annotation.AuthenticationPrincipal; +import org.springframework.util.MultiValueMap; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +import com.codahale.metrics.annotation.Timed; + +import datawave.microservice.authorization.user.DatawaveUserDetails; +import datawave.webservice.query.exception.QueryException; +import datawave.webservice.result.BaseQueryResponse; +import datawave.webservice.result.CachedResultsDescribeResponse; +import datawave.webservice.result.CachedResultsResponse; +import datawave.webservice.result.GenericResponse; +import datawave.webservice.result.VoidResponse; +import io.swagger.v3.oas.annotations.ExternalDocumentation; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.Parameters; +import io.swagger.v3.oas.annotations.enums.ParameterIn; +import io.swagger.v3.oas.annotations.media.Schema; +import io.swagger.v3.oas.annotations.responses.ApiResponse; +import io.swagger.v3.oas.annotations.responses.ApiResponses; +import io.swagger.v3.oas.annotations.tags.Tag; + +@Tag(name = "Cached Results Query Controller /v1", description = "DataWave Cached Results Query Management", + externalDocs = @ExternalDocumentation(description = "Cached Results Query Documentation", + url = "https://github.com/NationalSecurityAgency/datawave-spring-boot-starter-cached-results")) +@RestController +@RequestMapping(path = "/v1/cachedresults", produces = MediaType.APPLICATION_JSON_VALUE) +@ConditionalOnProperty(name = "datawave.query.cached-results.enabled", havingValue = "true", matchIfMissing = true) +public class CachedResultsQueryController { + + private final CachedResultsQueryService cachedResultsQueryService; + + public CachedResultsQueryController(CachedResultsQueryService cachedResultsQueryService) { + this.cachedResultsQueryService = cachedResultsQueryService; + } + + /** + * @see CachedResultsQueryService#load(String, String, DatawaveUserDetails) + */ + // @formatter:off + @Operation(summary = "Loads a query into MySQL using the given defined query ID and parameters.") + @ApiResponses({ + @ApiResponse( + description = "if successful, returns a generic response containing the view name.", + responseCode = "200")}) + // @formatter:on + @RequestMapping(path = "{definedQueryId}/load", method = {RequestMethod.GET}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", + "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) + public GenericResponse load(@Parameter(description = "The defined query ID") @PathVariable String definedQueryId, + @Parameter(description = "The user-defined alias") @RequestParam(required = false) String alias, + @AuthenticationPrincipal DatawaveUserDetails currentUser) throws QueryException { + return cachedResultsQueryService.load(definedQueryId, alias, currentUser); + } + + /** + * @see CachedResultsQueryService#create(String, MultiValueMap, DatawaveUserDetails) + */ + // @formatter:off + @Operation( + summary = "Creates a MySQL query which will be run against the loaded results.", + description = "Auditing is performed before the SQL query is started.") + @ApiResponses({ + @ApiResponse( + description = "if successful, returns a cached results response containing information about the SQL query.", + responseCode = "200")}) + @Parameters({ + @Parameter( + name = FIELDS, + in = ParameterIn.QUERY, + description = "The fields to return", + schema = @Schema(implementation = String.class)), + @Parameter( + name = CONDITIONS, + in = ParameterIn.QUERY, + description = "The conditions to apply", + schema = @Schema(implementation = String.class)), + @Parameter( + name = GROUPING, + in = ParameterIn.QUERY, + description = "The fields to group by", + schema = @Schema(implementation = String.class)), + @Parameter( + name = ORDER, + in = ParameterIn.QUERY, + description = "The fields to order by", + schema = @Schema(implementation = String.class)) + }) + // @formatter:on + @RequestMapping(path = "{key}/create", method = {RequestMethod.POST}, + produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", + "application/x-protostuff"}) + @Timed(name = "dw.cachedr.create", absolute = true) + public CachedResultsResponse create(@Parameter(description = "The defined query id, view name, or alias") @PathVariable String key, + @Parameter(hidden = true) @RequestParam MultiValueMap parameters, @AuthenticationPrincipal DatawaveUserDetails currentUser) + throws QueryException { + return cachedResultsQueryService.create(key, parameters, currentUser); + } + + /** + * @see CachedResultsQueryService#loadAndCreate(String, MultiValueMap, DatawaveUserDetails) + */ + // @formatter:off + @Operation( + summary = "Loads a query into MySQL using the given defined query ID and parameters, and creates a MySQL query which will be run against the loaded results", + description = "Auditing is performed before the SQL query is started.") + @ApiResponses({ + @ApiResponse( + description = "if successful, returns a cached results response containing information about the SQL query.", + responseCode = "200")}) + @Parameters({ + @Parameter( + name = FIELDS, + in = ParameterIn.QUERY, + description = "The fields to return", + schema = @Schema(implementation = String.class)), + @Parameter( + name = CONDITIONS, + in = ParameterIn.QUERY, + description = "The conditions to apply", + schema = @Schema(implementation = String.class)), + @Parameter( + name = GROUPING, + in = ParameterIn.QUERY, + description = "The fields to group by", + schema = @Schema(implementation = String.class)), + @Parameter( + name = ORDER, + in = ParameterIn.QUERY, + description = "The fields to order by", + schema = @Schema(implementation = String.class)) + }) + // @formatter:on + @RequestMapping(path = "{key}/loadAndCreate", method = {RequestMethod.POST}, + produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", + "application/x-protostuff"}) + @Timed(name = "dw.cachedr.loadAndCreate", absolute = true) + public CachedResultsResponse loadAndCreate(@Parameter(description = "The defined query ID") @PathVariable String definedQueryId, + @Parameter(hidden = true) @RequestParam MultiValueMap parameters, @AuthenticationPrincipal DatawaveUserDetails currentUser) + throws QueryException { + return cachedResultsQueryService.loadAndCreate(definedQueryId, parameters, currentUser); + } + + /** + * @see CachedResultsQueryService#getRows(String, Integer, Integer, DatawaveUserDetails) + */ + // @formatter:off + @Operation( + summary = "Gets the requested rows from rowBegin to rowEnd for the specified query.") + @ApiResponses({ + @ApiResponse( + description = "if successful, returns a base query response containing the requested rows.", + responseCode = "200")}) + // @formatter:on + @RequestMapping(path = "{key}/getRows", method = {RequestMethod.GET}, + produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", + "application/x-protostuff"}) + @Timed(name = "dw.cachedr.getRows", absolute = true) + public BaseQueryResponse getRows(@Parameter(description = "The defined query id, view name, or alias") @PathVariable("key") String key, + @RequestParam(defaultValue = "1") Integer rowBegin, @RequestParam Integer rowEnd, @AuthenticationPrincipal DatawaveUserDetails currentUser) + throws QueryException { + return cachedResultsQueryService.getRows(key, rowBegin, rowEnd, currentUser); + } + + /** + * @see CachedResultsQueryService#status(String, DatawaveUserDetails) + */ + // @formatter:off + @Operation( + summary = "Gets the status for the specified query.") + @ApiResponses({ + @ApiResponse( + description = "if successful, returns a generic response containing the query status.", + responseCode = "200")}) + // @formatter:on + @RequestMapping(path = "{key}/status", method = {RequestMethod.GET}, + produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", + "application/x-protostuff"}) + @Timed(name = "dw.cachedr.status", absolute = true) + public GenericResponse status(@Parameter(description = "The defined query id, view name, or alias") @PathVariable("key") String key, + @AuthenticationPrincipal DatawaveUserDetails currentUser) throws QueryException { + return cachedResultsQueryService.status(key, currentUser); + } + + /** + * @see CachedResultsQueryService#describe(String, DatawaveUserDetails) + */ + // @formatter:off + @Operation( + summary = "Gets the description for the specified query.") + @ApiResponses({ + @ApiResponse( + description = "if successful, returns a cached results describe response containing the query description.", + responseCode = "200")}) + // @formatter:on + @RequestMapping(path = "{key}/describe", method = {RequestMethod.GET}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", + "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) + public CachedResultsDescribeResponse describe(@Parameter(description = "The defined query id, view name, or alias") @PathVariable("key") String key, + @AuthenticationPrincipal DatawaveUserDetails currentUser) throws QueryException { + return cachedResultsQueryService.describe(key, currentUser); + } + + /** + * @see CachedResultsQueryService#cancel(String, DatawaveUserDetails) + */ + // @formatter:off + @Operation(summary = "Cancels the specified query.") + // @formatter:on + @RequestMapping(path = "{key}/cancel", method = {RequestMethod.PUT}, + produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", + "application/x-protostuff"}) + @Timed(name = "dw.cachedr.cancel", absolute = true) + public VoidResponse cancel(@Parameter(description = "The defined query id, view name, or alias") @PathVariable("key") String key, + @AuthenticationPrincipal DatawaveUserDetails currentUser) throws QueryException { + return cachedResultsQueryService.cancel(key, currentUser); + } + + /** + * @see CachedResultsQueryService#adminCancel(String, DatawaveUserDetails) + */ + // @formatter:off + @Operation(summary = "Cancels the specified query using admin privileges.") + // @formatter:on + @Secured({"Administrator", "JBossAdministrator"}) + @RequestMapping(path = "{key}/adminCancel", method = {RequestMethod.PUT}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", + "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) + public VoidResponse adminCancel(@Parameter(description = "The defined query id, view name, or alias") @PathVariable("key") String key, + @AuthenticationPrincipal DatawaveUserDetails currentUser) throws QueryException { + return cachedResultsQueryService.adminCancel(key, currentUser); + } + + /** + * @see CachedResultsQueryService#close(String, DatawaveUserDetails) + */ + // @formatter:off + @Operation(summary = "Closes the specified query.") + // @formatter:on + @RequestMapping(path = "{key}/close", method = {RequestMethod.PUT}, + produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", + "application/x-protostuff"}) + @Timed(name = "dw.cachedr.close", absolute = true) + public VoidResponse close(@Parameter(description = "The defined query id, view name, or alias") @PathVariable("key") String key, + @AuthenticationPrincipal DatawaveUserDetails currentUser) throws QueryException { + return cachedResultsQueryService.close(key, currentUser); + } + + /** + * @see CachedResultsQueryService#adminClose(String, DatawaveUserDetails) + */ + // @formatter:off + @Operation(summary = "Closes the specified query using admin privileges.") + // @formatter:on + @Secured({"Administrator", "JBossAdministrator"}) + @RequestMapping(path = "{key}/adminClose", method = {RequestMethod.PUT}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", + "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) + public VoidResponse adminClose(@Parameter(description = "The defined query id, view name, or alias") @PathVariable("key") String key, + @AuthenticationPrincipal DatawaveUserDetails currentUser) throws QueryException { + return cachedResultsQueryService.adminClose(key, currentUser); + } + + /** + * @see CachedResultsQueryService#setAlias(String, String, DatawaveUserDetails) + */ + // @formatter:off + @Operation( + summary = "Sets the alias for the specified query.") + @ApiResponses({ + @ApiResponse( + description = "if successful, returns a cached results response containing the updated alias.", + responseCode = "200")}) + // @formatter:on + @RequestMapping(path = "{key}/setAlias", method = {RequestMethod.POST}, + produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", + "application/x-protostuff"}) + @Timed(name = "dw.cachedr.setAlias", absolute = true) + public CachedResultsResponse setAlias(@Parameter(description = "The defined query id, view name, or alias") @PathVariable String key, + @RequestParam String alias, @AuthenticationPrincipal DatawaveUserDetails currentUser) throws QueryException { + return cachedResultsQueryService.setAlias(key, alias, currentUser); + } + + /** + * @see CachedResultsQueryService#update(String, String, String, String, String, Integer, DatawaveUserDetails) + */ + // @formatter:off + @Operation( + summary = "Updates the cached results query.", + description = "Auditing is performed if the SQL query changes.") + @ApiResponses({ + @ApiResponse( + description = "if successful, returns a cached results response containing the updated alias.", + responseCode = "200")}) + // @formatter:on + @RequestMapping(path = "{key}/update", method = {RequestMethod.POST}, + produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", + "application/x-protostuff"}) + @Timed(name = "dw.cachedr.update", absolute = true) + public CachedResultsResponse update(@Parameter(description = "The defined query id, view name, or alias") @PathVariable String key, + @RequestParam(required = false) String fields, @RequestParam(required = false) String conditions, + @RequestParam(required = false) String grouping, @RequestParam(required = false) String order, + @RequestParam(required = false) Integer pagesize, @AuthenticationPrincipal DatawaveUserDetails currentUser) throws QueryException { + return cachedResultsQueryService.update(key, fields, conditions, grouping, order, pagesize, currentUser); + } +} diff --git a/service/src/main/java/datawave/microservice/query/cachedresults/LocalQueryService.java b/service/src/main/java/datawave/microservice/query/cachedresults/LocalQueryService.java new file mode 100644 index 00000000..1352b685 --- /dev/null +++ b/service/src/main/java/datawave/microservice/query/cachedresults/LocalQueryService.java @@ -0,0 +1,65 @@ +package datawave.microservice.query.cachedresults; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.context.annotation.Primary; +import org.springframework.stereotype.Service; +import org.springframework.util.LinkedMultiValueMap; + +import datawave.microservice.authorization.user.DatawaveUserDetails; +import datawave.microservice.query.QueryManagementService; +import datawave.security.authorization.ProxiedUserDetails; +import datawave.webservice.query.exception.QueryException; +import datawave.webservice.result.BaseQueryResponse; +import datawave.webservice.result.GenericResponse; +import datawave.webservice.result.VoidResponse; + +@Primary +@Service +@ConditionalOnProperty(name = "datawave.query.cached-results.enabled", havingValue = "true", matchIfMissing = true) +public class LocalQueryService implements QueryService { + + private static final Logger log = LoggerFactory.getLogger(LocalQueryService.class); + + final private QueryManagementService queryManagementService; + + public LocalQueryService(QueryManagementService queryManagementService) { + this.queryManagementService = queryManagementService; + } + + @Override + public GenericResponse duplicate(String queryId, ProxiedUserDetails currentUser) throws QueryException { + log.info("LocalQueryService duplicate {} for {}", queryId, currentUser.getPrimaryUser()); + + return queryManagementService.duplicate(queryId, new LinkedMultiValueMap<>(), (DatawaveUserDetails) currentUser); + } + + @Override + public BaseQueryResponse next(String queryId, ProxiedUserDetails currentUser) throws QueryException { + log.info("LocalQueryService next {} for {}", queryId, currentUser.getPrimaryUser()); + + return queryManagementService.next(queryId, (DatawaveUserDetails) currentUser); + } + + @Override + public VoidResponse close(String queryId, ProxiedUserDetails currentUser) throws QueryException { + log.info("LocalQueryService close {} for {}", queryId, currentUser.getPrimaryUser()); + + return queryManagementService.close(queryId, (DatawaveUserDetails) currentUser); + } + + @Override + public VoidResponse cancel(String queryId, ProxiedUserDetails currentUser) throws QueryException { + log.info("LocalQueryService cancel {} for {}", queryId, currentUser.getPrimaryUser()); + + return queryManagementService.cancel(queryId, (DatawaveUserDetails) currentUser); + } + + @Override + public VoidResponse remove(String queryId, ProxiedUserDetails currentUser) throws QueryException { + log.info("LocalQueryService remove {} for {}", queryId, currentUser.getPrimaryUser()); + + return queryManagementService.remove(queryId, (DatawaveUserDetails) currentUser); + } +} diff --git a/service/src/main/java/datawave/microservice/query/config/QueryServiceConfiguration.java b/service/src/main/java/datawave/microservice/query/config/QueryServiceConfiguration.java new file mode 100644 index 00000000..8ae6a20c --- /dev/null +++ b/service/src/main/java/datawave/microservice/query/config/QueryServiceConfiguration.java @@ -0,0 +1,78 @@ +package datawave.microservice.query.config; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.cloud.context.config.annotation.RefreshScope; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; +import org.springframework.web.context.annotation.RequestScope; + +import datawave.marking.ColumnVisibilitySecurityMarking; +import datawave.marking.SecurityMarking; +import datawave.microservice.query.DefaultQueryParameters; +import datawave.microservice.query.QueryParameters; +import datawave.microservice.query.stream.StreamingProperties; +import datawave.microservice.querymetric.BaseQueryMetric; +import datawave.microservice.querymetric.QueryMetricFactory; +import datawave.microservice.querymetric.QueryMetricFactoryImpl; + +@Configuration +public class QueryServiceConfiguration { + + @Bean + @ConditionalOnMissingBean + @RequestScope + public QueryParameters queryParameters() { + DefaultQueryParameters queryParameters = new DefaultQueryParameters(); + queryParameters.clear(); + return queryParameters; + } + + @Bean + @ConditionalOnMissingBean + @RequestScope + public SecurityMarking securityMarking() { + SecurityMarking securityMarking = new ColumnVisibilitySecurityMarking(); + securityMarking.clear(); + return securityMarking; + } + + @Bean + @ConditionalOnMissingBean + @RequestScope + public BaseQueryMetric baseQueryMetric() { + return queryMetricFactory().createMetric(); + } + + @Bean + @ConditionalOnMissingBean(type = "QueryMetricFactory") + public QueryMetricFactory queryMetricFactory() { + return new QueryMetricFactoryImpl(); + } + + @RefreshScope + @Bean + public ThreadPoolTaskExecutor nextCallExecutor(QueryProperties queryProperties) { + ThreadPoolTaskExecutorProperties executorProperties = queryProperties.getNextCall().getExecutor(); + ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor(); + executor.setCorePoolSize(executorProperties.getCorePoolSize()); + executor.setMaxPoolSize(executorProperties.getMaxPoolSize()); + executor.setQueueCapacity(executorProperties.getQueueCapacity()); + executor.setThreadNamePrefix(executorProperties.getThreadNamePrefix()); + executor.initialize(); + return executor; + } + + @RefreshScope + @Bean + public ThreadPoolTaskExecutor streamingCallExecutor(StreamingProperties streamingProperties) { + ThreadPoolTaskExecutorProperties executorProperties = streamingProperties.getExecutor(); + ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor(); + executor.setCorePoolSize(executorProperties.getCorePoolSize()); + executor.setMaxPoolSize(executorProperties.getMaxPoolSize()); + executor.setQueueCapacity(executorProperties.getQueueCapacity()); + executor.setThreadNamePrefix(executorProperties.getThreadNamePrefix()); + executor.initialize(); + return executor; + } +} diff --git a/service/src/main/java/datawave/microservice/query/lookup/LookupService.java b/service/src/main/java/datawave/microservice/query/lookup/LookupService.java new file mode 100644 index 00000000..4c29a2e2 --- /dev/null +++ b/service/src/main/java/datawave/microservice/query/lookup/LookupService.java @@ -0,0 +1,731 @@ +package datawave.microservice.query.lookup; + +import static datawave.core.query.logic.lookup.LookupQueryLogic.LOOKUP_KEY_VALUE_DELIMITER; +import static datawave.microservice.query.QueryParameters.QUERY_AUTHORIZATIONS; +import static datawave.microservice.query.QueryParameters.QUERY_BEGIN; +import static datawave.microservice.query.QueryParameters.QUERY_END; +import static datawave.microservice.query.QueryParameters.QUERY_LOGIC_NAME; +import static datawave.microservice.query.QueryParameters.QUERY_NAME; +import static datawave.microservice.query.QueryParameters.QUERY_PARAMS; +import static datawave.microservice.query.QueryParameters.QUERY_STRING; +import static datawave.query.QueryParameters.QUERY_SYNTAX; + +import java.text.ParseException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Date; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.stream.Collectors; + +import org.apache.commons.lang.time.DateUtils; +import org.apache.http.HttpStatus; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; +import org.springframework.util.LinkedMultiValueMap; +import org.springframework.util.MultiValueMap; + +import com.google.common.collect.Iterables; + +import datawave.core.query.logic.QueryLogic; +import datawave.core.query.logic.QueryLogicFactory; +import datawave.core.query.logic.lookup.LookupQueryLogic; +import datawave.core.query.util.QueryUtil; +import datawave.microservice.authorization.user.DatawaveUserDetails; +import datawave.microservice.authorization.util.AuthorizationsUtil; +import datawave.microservice.query.DefaultQueryParameters; +import datawave.microservice.query.Query; +import datawave.microservice.query.QueryImpl; +import datawave.microservice.query.QueryManagementService; +import datawave.microservice.query.QueryParameters; +import datawave.microservice.query.stream.StreamingService; +import datawave.microservice.query.stream.listener.StreamingResponseListener; +import datawave.query.data.UUIDType; +import datawave.security.authorization.AuthorizationException; +import datawave.security.authorization.ProxiedUserDetails; +import datawave.security.util.ProxiedEntityUtils; +import datawave.webservice.query.exception.BadRequestQueryException; +import datawave.webservice.query.exception.DatawaveErrorCode; +import datawave.webservice.query.exception.NoResultsQueryException; +import datawave.webservice.query.exception.QueryException; +import datawave.webservice.query.exception.TimeoutQueryException; +import datawave.webservice.query.exception.UnauthorizedQueryException; +import datawave.webservice.query.result.event.Metadata; +import datawave.webservice.result.BaseQueryResponse; +import datawave.webservice.result.EventQueryResponseBase; + +@Service +public class LookupService { + private final Logger log = LoggerFactory.getLogger(this.getClass()); + + public static final String LOOKUP_UUID_PAIRS = "uuidPairs"; + public static final String LUCENE_UUID_SYNTAX = "LUCENE-UUID"; + public static final String LOOKUP_STREAMING = "streaming"; + public static final String LOOKUP_CONTEXT = "context"; + + public static final String PARAM_HIT_LIST = "hit.list"; + protected static final String EMPTY_STRING = ""; + private static final String SPACE = " "; + private static final String REGEX_GROUPING_CHARS = "[()]"; + private static final String REGEX_NONWORD_CHARS = "[\\W&&[^:_\\.\\s-]]"; + private static final String REGEX_OR_OPERATOR = "[\\s][oO][rR][\\s]"; + private static final String REGEX_WHITESPACE_CHARS = "\\s"; + + private static final String CONTENT_QUERY_TERM_DELIMITER = ":"; + private static final String CONTENT_QUERY_VALUE_DELIMITER = "/"; + private static final String CONTENT_QUERY_TERM_SEPARATOR = " "; + private static final String DOCUMENT_FIELD_PREFIX = "DOCUMENT" + CONTENT_QUERY_TERM_DELIMITER; + + private final LookupProperties lookupProperties; + + private final QueryLogicFactory queryLogicFactory; + private final QueryManagementService queryManagementService; + private final StreamingService streamingService; + + public LookupService(LookupProperties lookupProperties, QueryLogicFactory queryLogicFactory, QueryManagementService queryManagementService, + StreamingService streamingService) { + this.lookupProperties = lookupProperties; + this.queryLogicFactory = queryLogicFactory; + this.queryManagementService = queryManagementService; + this.streamingService = streamingService; + } + + /** + * Creates a batch event lookup query using the query logic associated with the given uuid type(s) and parameters, and returns the first page of results. + *

+ * Lookup queries will start running immediately.
+ * Auditing is performed before the query is started.
+ * Each of the uuid pairs must map to the same query logic.
+ * After the first page is returned, the query will be closed. + * + * @param parameters + * the query parameters, not null + * @param pool + * the pool to target, may be null + * @param currentUser + * the user who called this method, not null + * @param listener + * the listener which will handle the result pages, not null + * @throws BadRequestQueryException + * if parameter validation fails + * @throws BadRequestQueryException + * if query logic parameter validation fails + * @throws UnauthorizedQueryException + * if the user doesn't have access to the requested query logic + * @throws BadRequestQueryException + * if security marking validation fails + * @throws BadRequestQueryException + * if auditing fails + * @throws QueryException + * if query storage fails + * @throws TimeoutQueryException + * if the next call times out + * @throws NoResultsQueryException + * if no query results are found + * @throws QueryException + * if this next task is rejected by the executor + * @throws QueryException + * if there is an unknown error + */ + public void lookupUUID(MultiValueMap parameters, String pool, DatawaveUserDetails currentUser, StreamingResponseListener listener) + throws QueryException { + String user = ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()); + if (log.isDebugEnabled()) { + log.info("Request: lookupUUID from {} with params: {}", user, parameters); + } else { + log.info("Request: lookupUUID from {}", user); + } + + try { + lookup(parameters, pool, currentUser, listener); + } catch (QueryException e) { + throw e; + } catch (Exception e) { + log.error("Unknown error looking up UUID", e); + throw new QueryException(DatawaveErrorCode.QUERY_SETUP_ERROR, e, "Unknown error looking up UUID."); + } + } + + /** + * Creates a batch event lookup query using the query logic associated with the given uuid type(s) and parameters, and returns the first page of results. + *

+ * Lookup queries will start running immediately.
+ * Auditing is performed before the query is started.
+ * Each of the uuid pairs must map to the same query logic.
+ * After the first page is returned, the query will be closed. + * + * @param parameters + * the query parameters, not null + * @param pool + * the pool to target, may be null + * @param currentUser + * the user who called this method, not null + * @return a base query response containing the first page of results + * @throws BadRequestQueryException + * if parameter validation fails + * @throws BadRequestQueryException + * if query logic parameter validation fails + * @throws UnauthorizedQueryException + * if the user doesn't have access to the requested query logic + * @throws BadRequestQueryException + * if security marking validation fails + * @throws BadRequestQueryException + * if auditing fails + * @throws QueryException + * if query storage fails + * @throws TimeoutQueryException + * if the next call times out + * @throws NoResultsQueryException + * if no query results are found + * @throws QueryException + * if this next task is rejected by the executor + * @throws QueryException + * if there is an unknown error + */ + public BaseQueryResponse lookupUUID(MultiValueMap parameters, String pool, DatawaveUserDetails currentUser) throws QueryException { + String user = ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()); + if (log.isDebugEnabled()) { + log.info("Request: lookupUUID from {} with params: {}", user, parameters); + } else { + log.info("Request: lookupUUID from {}", user); + } + + try { + return lookup(parameters, pool, currentUser, null); + } catch (QueryException e) { + throw e; + } catch (Exception e) { + log.error("Unknown error looking up UUID", e); + throw new QueryException(DatawaveErrorCode.QUERY_SETUP_ERROR, e, "Unknown error looking up UUID."); + } + } + + /** + * Creates a batch content lookup query using the query logic associated with the given uuid type(s) and parameters, and returns the first page of results. + *

+ * Lookup queries will start running immediately.
+ * Auditing is performed before the query is started.
+ * Each of the uuid pairs must map to the same query logic.
+ * After the first page is returned, the query will be closed. + * + * @param parameters + * the query parameters, not null + * @param pool + * the pool to target, may be null + * @param currentUser + * the user who called this method, not null + * @param listener + * the listener which will handle the result pages, not null + * @return a base query response containing the first page of results + * @throws BadRequestQueryException + * if parameter validation fails + * @throws BadRequestQueryException + * if query logic parameter validation fails + * @throws UnauthorizedQueryException + * if the user doesn't have access to the requested query logic + * @throws BadRequestQueryException + * if security marking validation fails + * @throws BadRequestQueryException + * if auditing fails + * @throws QueryException + * if query storage fails + * @throws TimeoutQueryException + * if the next call times out + * @throws NoResultsQueryException + * if no query results are found + * @throws QueryException + * if this next task is rejected by the executor + * @throws QueryException + * if there is an unknown error + */ + public T lookupContentUUID(MultiValueMap parameters, String pool, DatawaveUserDetails currentUser, StreamingResponseListener listener) + throws QueryException { + String user = ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()); + if (log.isDebugEnabled()) { + log.info("Request: lookupContentUUID from {} with params: {}", user, parameters); + } else { + log.info("Request: lookupContentUUID from {}", user); + } + + try { + // first lookup the UUIDs, then get the content for each UUID + return lookupContent(parameters, pool, currentUser, listener); + } catch (QueryException e) { + throw e; + } catch (Exception e) { + log.error("Unknown error looking up UUID content", e); + throw new QueryException(DatawaveErrorCode.QUERY_SETUP_ERROR, e, "Unknown error looking up UUID content."); + } + } + + /** + * Creates a batch content lookup query using the query logic associated with the given uuid type(s) and parameters, and returns the first page of results. + *

+ * Lookup queries will start running immediately.
+ * Auditing is performed before the query is started.
+ * Each of the uuid pairs must map to the same query logic.
+ * After the first page is returned, the query will be closed. + * + * @param parameters + * the query parameters, not null + * @param pool + * the pool to target, may be null + * @param currentUser + * the user who called this method, not null + * @return a base query response containing the first page of results + * @throws BadRequestQueryException + * if parameter validation fails + * @throws BadRequestQueryException + * if query logic parameter validation fails + * @throws UnauthorizedQueryException + * if the user doesn't have access to the requested query logic + * @throws BadRequestQueryException + * if security marking validation fails + * @throws BadRequestQueryException + * if auditing fails + * @throws QueryException + * if query storage fails + * @throws TimeoutQueryException + * if the next call times out + * @throws NoResultsQueryException + * if no query results are found + * @throws QueryException + * if this next task is rejected by the executor + * @throws QueryException + * if there is an unknown error + */ + public BaseQueryResponse lookupContentUUID(MultiValueMap parameters, String pool, DatawaveUserDetails currentUser) throws QueryException { + String user = ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()); + if (log.isDebugEnabled()) { + log.info("Request: lookupContentUUID from {} with params: {}", user, parameters); + } else { + log.info("Request: lookupContentUUID from {}", user); + } + + try { + // first lookup the UUIDs, then get the content for each UUID + return lookupContent(parameters, pool, currentUser, null); + } catch (QueryException e) { + throw e; + } catch (Exception e) { + log.error("Unknown error looking up UUID content", e); + throw new QueryException(DatawaveErrorCode.QUERY_SETUP_ERROR, e, "Unknown error looking up UUID content."); + } + } + + private T lookup(MultiValueMap parameters, String pool, DatawaveUserDetails currentUser, StreamingResponseListener listener) + throws QueryException, AuthorizationException { + List lookupTerms = parameters.get(LOOKUP_UUID_PAIRS); + if (lookupTerms == null || lookupTerms.isEmpty()) { + log.error("Unable to validate lookupUUID parameters: No UUID Pairs"); + throw new BadRequestQueryException(DatawaveErrorCode.MISSING_REQUIRED_PARAMETER); + } + + String uuidTypeContext = parameters.getFirst(LOOKUP_CONTEXT); + + // flatten out the terms + lookupTerms = lookupTerms.stream().flatMap(x -> Arrays.stream(reformatQuery(x).split(REGEX_WHITESPACE_CHARS))).collect(Collectors.toList()); + + // validate the lookup terms + LookupQueryLogic lookupQueryLogic = validateLookupTerms(uuidTypeContext, lookupTerms); + + // perform the event lookup + return lookupEvents(lookupQueryLogic, new LinkedMultiValueMap<>(parameters), pool, currentUser, listener); + } + + private String reformatQuery(String query) { + String reformattedQuery = EMPTY_STRING; + if (query != null) { + reformattedQuery = query; + reformattedQuery = reformattedQuery.replaceAll(REGEX_GROUPING_CHARS, SPACE); // Replace grouping characters with whitespace + reformattedQuery = reformattedQuery.replaceAll(REGEX_NONWORD_CHARS, EMPTY_STRING); // Remove most, but not all, non-word characters + reformattedQuery = reformattedQuery.replaceAll(REGEX_OR_OPERATOR, SPACE); // Remove OR operators + } + return reformattedQuery; + } + + private BaseQueryResponse lookupEvents(LookupQueryLogic lookupQueryLogic, MultiValueMap parameters, String pool, + DatawaveUserDetails currentUser) throws QueryException, AuthorizationException { + return lookupEvents(lookupQueryLogic, parameters, pool, currentUser, null); + } + + private T lookupEvents(LookupQueryLogic lookupQueryLogic, MultiValueMap parameters, String pool, DatawaveUserDetails currentUser, + StreamingResponseListener listener) throws QueryException, AuthorizationException { + String queryId = null; + try { + // add the query logic name and query string to our parameters + parameters.put(QUERY_LOGIC_NAME, Collections.singletonList(lookupQueryLogic.getLogicName())); + parameters.put(QUERY_STRING, Collections.singletonList(parameters.getFirst(LOOKUP_UUID_PAIRS))); + + // update the parameters for query + setupEventQueryParameters(parameters, lookupQueryLogic, currentUser); + + // create the query + queryId = queryManagementService.create(parameters.getFirst(QUERY_LOGIC_NAME), parameters, pool, currentUser).getResult(); + + if (listener != null) { + // stream results to the listener + streamingService.execute(queryId, currentUser, (DatawaveUserDetails) lookupQueryLogic.getServerUser(), listener); + return null; + } else { + // get the first page of results + // noinspection unchecked + return (T) queryManagementService.next(queryId, currentUser); + } + } finally { + // close the query if applicable + if (listener == null && queryId != null) { + queryManagementService.close(queryId, currentUser); + } + } + } + + protected LookupQueryLogic validateLookupTerms(String uuidTypeContext, List lookupUUIDPairs) throws QueryException { + return validateLookupTerms(uuidTypeContext, lookupUUIDPairs, null); + } + + protected LookupQueryLogic validateLookupTerms(String uuidTypeContext, List lookupUUIDPairs, MultiValueMap lookupUUIDMap) + throws QueryException { + String queryLogicName = null; + + // make sure there aren't too many terms to lookup + if (lookupProperties.getBatchLookupLimit() > 0 && lookupUUIDPairs.size() <= lookupProperties.getBatchLookupLimit()) { + + // validate each of the uuid pairs + for (String uuidPair : lookupUUIDPairs) { + String[] fieldValue = uuidPair.split(LOOKUP_KEY_VALUE_DELIMITER); + + // there should be a field and value present - no more, no less + if (fieldValue.length == 2) { + String field = fieldValue[0]; + String value = fieldValue[1]; + + // neither the field or value should be empty + if (!field.isEmpty() && !value.isEmpty()) { + + // is this a supported uuid type/field? + UUIDType uuidType = lookupProperties.getTypes().get(field.toUpperCase()); + if (uuidType != null) { + if (queryLogicName == null) { + queryLogicName = uuidType.getQueryLogic(uuidTypeContext); + } + // if we are mixing and matching query logics + else if (!queryLogicName.equals(uuidType.getQueryLogic(uuidTypeContext))) { + String message = "Multiple UUID types '" + queryLogicName + "' and '" + uuidType.getQueryLogic(uuidTypeContext) + + "' not supported within the same lookup request"; + log.error(message); + throw new BadRequestQueryException(new IllegalArgumentException(message), HttpStatus.SC_BAD_REQUEST + "-1"); + } + } + // if uuid type is null + else { + String message = "Invalid type '" + field.toUpperCase() + "' for UUID " + value + + " not supported with the LuceneToJexlUUIDQueryParser"; + log.error(message); + throw new BadRequestQueryException(new IllegalArgumentException(message), HttpStatus.SC_BAD_REQUEST + "-1"); + } + + if (lookupUUIDMap != null) { + lookupUUIDMap.add(field, value); + } + } + // if the field or value is empty + else { + String message = "Empty UUID type or value extracted from uuidPair " + uuidPair; + log.error(message); + throw new BadRequestQueryException(new IllegalArgumentException(message), HttpStatus.SC_BAD_REQUEST + "-1"); + } + } + // if there isn't a field AND a value + else { + String message = "Unable to determine UUID type and value from uuidPair " + uuidPair; + log.error(message); + throw new BadRequestQueryException(new IllegalArgumentException(message), HttpStatus.SC_BAD_REQUEST + "-1"); + } + } + } + // too many terms to lookup + else { + String message = "The " + lookupUUIDPairs.size() + " specified UUIDs exceed the maximum number of " + lookupProperties.getBatchLookupLimit() + + " allowed for a given lookup request"; + log.error(message); + throw new BadRequestQueryException(new IllegalArgumentException(message), HttpStatus.SC_BAD_REQUEST + "-1"); + } + + try { + QueryLogic queryLogic = queryLogicFactory.getQueryLogic(queryLogicName); + + if (queryLogic instanceof LookupQueryLogic) { + return (LookupQueryLogic) queryLogic; + } else { + log.error("Lookup UUID can only be run with a LookupQueryLogic"); + throw new BadRequestQueryException(DatawaveErrorCode.QUERY_SETUP_ERROR, "Lookup UUID can only be run with a LookupQueryLogic"); + } + } catch (CloneNotSupportedException e) { + throw new QueryException(DatawaveErrorCode.QUERY_SETUP_ERROR, e, "Unable to create instance of the requested query logic " + queryLogicName); + } + } + + @SuppressWarnings("ConstantConditions") + public Query createSettings(Map> queryParameters) { + log.debug("Initial query parameters: " + queryParameters); + Query query = new QueryImpl(); + if (queryParameters != null) { + MultiValueMap expandedQueryParameters = new LinkedMultiValueMap<>(); + List params = queryParameters.get(QueryParameters.QUERY_PARAMS); + String delimitedParams = null; + if (params != null && !params.isEmpty()) { + delimitedParams = params.get(0); + } + if (delimitedParams != null) { + for (QueryImpl.Parameter pm : QueryUtil.parseParameters(delimitedParams)) { + expandedQueryParameters.add(pm.getParameterName(), pm.getParameterValue()); + } + } + expandedQueryParameters.putAll(queryParameters); + log.debug("Final query parameters: " + expandedQueryParameters); + query.setOptionalQueryParameters(expandedQueryParameters); + for (String key : expandedQueryParameters.keySet()) { + if (expandedQueryParameters.get(key).size() == 1) { + query.addParameter(key, expandedQueryParameters.getFirst(key)); + } + } + } + return query; + } + + public String getAuths(MultiValueMap queryParameters, QueryLogic queryLogic, DatawaveUserDetails currentUser) + throws AuthorizationException { + Query query = createSettings(queryParameters); + + String userAuths; + try { + String queryAuths = null; + if (queryParameters.containsKey(QUERY_AUTHORIZATIONS)) { + queryAuths = queryParameters.getFirst(QUERY_AUTHORIZATIONS); + queryLogic.preInitialize(query, AuthorizationsUtil.buildAuthorizations(Collections.singleton(AuthorizationsUtil.splitAuths(queryAuths)))); + } else { + // if no requested auths, then use the overall auths for any filtering of the query operations + queryLogic.preInitialize(query, AuthorizationsUtil.buildAuthorizations(currentUser.getAuthorizations())); + } + + // the query principal is our local principal unless the query logic has a different user operations + ProxiedUserDetails queryPrincipal = ((queryLogic.getUserOperations() == null) ? currentUser + : queryLogic.getUserOperations().getRemoteUser(currentUser)); + + if (queryAuths != null) { + userAuths = AuthorizationsUtil.downgradeUserAuths(queryAuths, currentUser, queryPrincipal); + } else { + userAuths = AuthorizationsUtil.buildUserAuthorizationString(queryPrincipal); + } + } catch (Exception e) { + log.error("Failed to get user query authorizations", e); + throw new AuthorizationException("Failed to get user query authorizations", e); + } + + return userAuths; + } + + protected void setupEventQueryParameters(MultiValueMap parameters, LookupQueryLogic queryLogic, DatawaveUserDetails currentUser) + throws AuthorizationException { + String user = ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()); + final String queryName = user + "-" + UUID.randomUUID().toString(); + + final String endDate; + try { + endDate = DefaultQueryParameters.formatDate(DateUtils.addDays(new Date(), 2)); + } catch (ParseException e) { + throw new RuntimeException("Unable to format new query end date"); + } + + setOptionalQueryParameters(parameters); + + // Override the extraneous query details + parameters.set(QUERY_SYNTAX, LUCENE_UUID_SYNTAX); + parameters.set(QUERY_NAME, queryName); + parameters.set(QUERY_BEGIN, lookupProperties.getBeginDate()); + parameters.set(QUERY_END, endDate); + + parameters.set(QUERY_AUTHORIZATIONS, getAuths(parameters, queryLogic, currentUser)); + } + + protected void setOptionalQueryParameters(MultiValueMap parameters) { + if (lookupProperties.getColumnVisibility() != null) { + parameters.set(QueryParameters.QUERY_VISIBILITY, lookupProperties.getColumnVisibility()); + } + } + + private T lookupContent(MultiValueMap parameters, String pool, DatawaveUserDetails currentUser, StreamingResponseListener listener) + throws QueryException, AuthorizationException { + List lookupTerms = parameters.get(LOOKUP_UUID_PAIRS); + if (lookupTerms == null || lookupTerms.isEmpty()) { + log.error("Unable to validate lookupContentUUID parameters: No UUID Pairs"); + throw new BadRequestQueryException(DatawaveErrorCode.MISSING_REQUIRED_PARAMETER); + } + + String uuidTypeContext = parameters.getFirst(LOOKUP_CONTEXT); + + // flatten out the terms + lookupTerms = lookupTerms.stream().flatMap(x -> Arrays.stream(reformatQuery(x).split(REGEX_WHITESPACE_CHARS))).collect(Collectors.toList()); + + MultiValueMap lookupTermMap = new LinkedMultiValueMap<>(); + + // validate the lookup terms + LookupQueryLogic lookupQueryLogic = validateLookupTerms(uuidTypeContext, lookupTerms, lookupTermMap); + + BaseQueryResponse response = null; + + boolean isEventLookupRequired = lookupQueryLogic.isEventLookupRequired(lookupTermMap); + + // do the event lookup if necessary + if (isEventLookupRequired) { + response = lookupEvents(lookupQueryLogic, new LinkedMultiValueMap<>(parameters), pool, currentUser); + } + + // perform the content lookup + Set contentLookupTerms; + if (!isEventLookupRequired) { + contentLookupTerms = lookupQueryLogic.getContentLookupTerms(lookupTermMap); + } else { + contentLookupTerms = getContentLookupTerms(response); + } + + return lookupContent(contentLookupTerms, parameters, pool, currentUser, (DatawaveUserDetails) lookupQueryLogic.getServerUser(), listener); + } + + private Set getContentLookupTerms(BaseQueryResponse response) { + Set contentQueries = new HashSet<>(); + + if (response instanceof EventQueryResponseBase) { + ((EventQueryResponseBase) response).getEvents().forEach(e -> contentQueries.add(createContentLookupTerm(e.getMetadata()))); + } + + return contentQueries; + } + + private String createContentLookupTerm(Metadata eventMetadata) { + return DOCUMENT_FIELD_PREFIX + + String.join(CONTENT_QUERY_VALUE_DELIMITER, eventMetadata.getRow(), eventMetadata.getDataType(), eventMetadata.getInternalId()); + } + + private T lookupContent(Set contentLookupTerms, MultiValueMap parameters, String pool, DatawaveUserDetails currentUser, + DatawaveUserDetails serverUser, StreamingResponseListener listener) throws QueryException { + // create queries from the content lookup terms + List contentQueries = createContentQueries(contentLookupTerms); + + // Required so that we can return identifiers alongside the content returned in the content lookup. + String params = parameters.getFirst(QUERY_PARAMS) != null ? parameters.getFirst(QUERY_PARAMS) : ""; + params += ";" + PARAM_HIT_LIST + ":true"; + + EventQueryResponseBase mergedResponse = null; + for (String contentQuery : contentQueries) { + MultiValueMap queryParameters = new LinkedMultiValueMap<>(parameters); + + // set the content query string + queryParameters.put(QUERY_STRING, Collections.singletonList(contentQuery)); + queryParameters.put(QUERY_PARAMS, Collections.singletonList(params)); + + // update parameters for the query + setContentQueryParameters(queryParameters, currentUser); + + if (listener != null) { + streamingService.createAndExecute(queryParameters.getFirst(QUERY_LOGIC_NAME), queryParameters, pool, currentUser, serverUser, listener); + } else { + // run the query + EventQueryResponseBase contentQueryResponse = runContentQuery(queryParameters, pool, currentUser); + + // merge the response + if (contentQueryResponse != null) { + if (mergedResponse == null) { + mergedResponse = contentQueryResponse; + } else { + mergedResponse.merge(contentQueryResponse); + } + } + } + } + + // noinspection unchecked + return (T) mergedResponse; + } + + private List createContentQueries(Set contentLookupTerms) { + List contentQueries = new ArrayList<>(); + + Iterables.partition(contentLookupTerms, lookupProperties.getBatchLookupLimit()) + .forEach(termBatch -> contentQueries.add(String.join(CONTENT_QUERY_TERM_SEPARATOR, termBatch))); + + return contentQueries; + } + + protected void setContentQueryParameters(MultiValueMap parameters, DatawaveUserDetails currentUser) { + String user = ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()); + + setOptionalQueryParameters(parameters); + + // all content queries use the same query logic + parameters.put(QUERY_LOGIC_NAME, Collections.singletonList(lookupProperties.getContentQueryLogicName())); + + parameters.set(QUERY_NAME, user + '-' + UUID.randomUUID()); + + parameters.set(QUERY_BEGIN, lookupProperties.getBeginDate()); + + final Date endDate = new Date(); + try { + parameters.set(QUERY_END, DefaultQueryParameters.formatDate(endDate)); + } catch (ParseException e1) { + throw new RuntimeException("Error formatting end date: " + endDate); + } + + final String userAuths = AuthorizationsUtil.buildUserAuthorizationString(currentUser); + parameters.set(QUERY_AUTHORIZATIONS, userAuths); + } + + protected EventQueryResponseBase runContentQuery(MultiValueMap parameters, String pool, DatawaveUserDetails currentUser) { + EventQueryResponseBase mergedResponse = null; + String queryId = null; + boolean isQueryFinished = false; + + do { + BaseQueryResponse nextResponse = null; + try { + if (queryId == null) { + nextResponse = queryManagementService.createAndNext(parameters.getFirst(QUERY_LOGIC_NAME), parameters, pool, currentUser); + queryId = nextResponse.getQueryId(); + } else { + nextResponse = queryManagementService.next(queryId, currentUser); + } + } catch (NoResultsQueryException e) { + log.debug("No results found for content query '{}'", parameters.getFirst(QUERY_STRING)); + } catch (QueryException e) { + log.info("Encountered error while getting results for content query '{}'", parameters.getFirst(QUERY_STRING)); + } + + if (nextResponse instanceof EventQueryResponseBase) { + EventQueryResponseBase nextEventQueryResponse = (EventQueryResponseBase) nextResponse; + + // Prevent NPE due to attempted merge when total events is null + if (nextEventQueryResponse.getTotalEvents() == null) { + final Long totalEvents = nextEventQueryResponse.getReturnedEvents(); + nextEventQueryResponse.setTotalEvents((totalEvents != null) ? totalEvents : 0L); + } + + // save or update the merged response + if (mergedResponse == null) { + mergedResponse = nextEventQueryResponse; + } else { + mergedResponse.merge(nextEventQueryResponse); + } + } else { + isQueryFinished = true; + } + } while (!isQueryFinished); + + return mergedResponse; + } +} diff --git a/service/src/main/java/datawave/microservice/query/mapreduce/MapReduceQueryController.java b/service/src/main/java/datawave/microservice/query/mapreduce/MapReduceQueryController.java new file mode 100644 index 00000000..569f4a0c --- /dev/null +++ b/service/src/main/java/datawave/microservice/query/mapreduce/MapReduceQueryController.java @@ -0,0 +1,575 @@ +package datawave.microservice.query.mapreduce; + +import static datawave.microservice.query.QueryParameters.QUERY_AUTHORIZATIONS; +import static datawave.microservice.query.mapreduce.config.MapReduceQueryProperties.FORMAT; +import static datawave.microservice.query.mapreduce.config.MapReduceQueryProperties.JOB_NAME; +import static datawave.microservice.query.mapreduce.config.MapReduceQueryProperties.OUTPUT_FORMAT; +import static datawave.microservice.query.mapreduce.config.MapReduceQueryProperties.OUTPUT_TABLE_NAME; +import static datawave.microservice.query.mapreduce.config.MapReduceQueryProperties.PARAMETERS; +import static datawave.microservice.query.mapreduce.config.MapReduceQueryProperties.QUERY_ID; +import static datawave.microservice.query.mapreduce.jobs.OozieJob.WORKFLOW; + +import java.io.IOException; +import java.util.Map; + +import org.apache.commons.compress.archivers.tar.TarArchiveEntry; +import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; +import org.apache.commons.io.IOUtils; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FileStatus; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.http.HttpHeaders; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.security.access.annotation.Secured; +import org.springframework.security.core.annotation.AuthenticationPrincipal; +import org.springframework.util.MultiValueMap; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody; + +import datawave.microservice.authorization.user.DatawaveUserDetails; +import datawave.microservice.query.mapreduce.config.MapReduceQueryProperties; +import datawave.webservice.query.exception.QueryException; +import datawave.webservice.result.GenericResponse; +import datawave.webservice.result.VoidResponse; +import datawave.webservice.results.mr.MapReduceInfoResponseList; +import datawave.webservice.results.mr.MapReduceJobDescriptionList; +import io.swagger.v3.oas.annotations.ExternalDocumentation; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.Parameters; +import io.swagger.v3.oas.annotations.enums.ParameterIn; +import io.swagger.v3.oas.annotations.media.Content; +import io.swagger.v3.oas.annotations.media.Schema; +import io.swagger.v3.oas.annotations.responses.ApiResponse; +import io.swagger.v3.oas.annotations.responses.ApiResponses; +import io.swagger.v3.oas.annotations.tags.Tag; + +@Tag(name = "MapReduce Query Controller /v1", description = "DataWave MapReduce Query Management", + externalDocs = @ExternalDocumentation(description = "MapReduce Query Service Documentation", + url = "https://github.com/NationalSecurityAgency/datawave-mapreduce-query-service")) +@RestController +@RequestMapping(path = "/v1/mapreduce", produces = MediaType.APPLICATION_JSON_VALUE) +@ConditionalOnProperty(name = MapReduceQueryProperties.PREFIX + ".enabled", havingValue = "true", matchIfMissing = true) +public class MapReduceQueryController { + private final MapReduceQueryManagementService mapReduceQueryManagementService; + + public MapReduceQueryController(MapReduceQueryManagementService mapReduceQueryManagementService) { + this.mapReduceQueryManagementService = mapReduceQueryManagementService; + } + + /** + * @see MapReduceQueryManagementService#listConfigurations(String, DatawaveUserDetails) + */ + // @formatter:off + @Operation( + summary = "Gets a list of the available map reduce jobs and their configurations.", + description = "Returns all matching map reduce jobs available to the user, filtering by job type.") + @ApiResponses({ + @ApiResponse( + description = "if successful, returns a map reduce list response containing the matching job configurations", + responseCode = "200", + content = @Content(schema = @Schema(implementation = MapReduceJobDescriptionList.class))), + @ApiResponse( + description = "if there is an unknown error", + responseCode = "500", + content = @Content(schema = @Schema(implementation = VoidResponse.class)))}) + // @formatter:on + @RequestMapping(path = "listConfigurations", method = RequestMethod.GET, produces = {"application/xml", "text/xml", "application/json", "text/yaml", + "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) + public MapReduceJobDescriptionList listConfigurations( + @Parameter(description = "The type of jobs to list") @RequestParam(required = false, defaultValue = "none") String jobType, + @AuthenticationPrincipal DatawaveUserDetails currentUser) { + return mapReduceQueryManagementService.listConfigurations(jobType, currentUser); + } + + /** + * @see MapReduceQueryManagementService#oozieSubmit(MultiValueMap, DatawaveUserDetails) + */ + // @formatter:off + @Operation( + summary = "Execute a configured oozie workflow.", + description = "Runs the selected oozie workflow.") + @ApiResponses({ + @ApiResponse( + description = "if successful, returns a generic response containing the oozie workflow id", + responseCode = "200", + content = @Content(schema = @Schema(implementation = GenericResponse.class))), + @ApiResponse( + description = "if the job configuration can't be found
" + + "if parameter validation fails
" + + "if auditing fails", + responseCode = "400", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if the user doesn't have access to the requested job configuration", + responseCode = "401", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if query storage fails
" + + "if there is an unknown error", + responseCode = "500", + content = @Content(schema = @Schema(implementation = VoidResponse.class)))}) + @Parameters({ + @Parameter( + name = WORKFLOW, + in = ParameterIn.QUERY, + description = "The oozie workflow to execute", + required = true, + schema = @Schema(implementation = String.class), + example = "OozieJob"), + @Parameter( + name = QUERY_AUTHORIZATIONS, + in = ParameterIn.QUERY, + description = "The query auths", + required = true, + schema = @Schema(implementation = String.class), + example = "PUBLIC,PRIVATE,BAR,FOO"), + @Parameter( + name = PARAMETERS, + in = ParameterIn.QUERY, + description = "Additional query parameters", + schema = @Schema(implementation = String.class), + example = "KEY_1:VALUE_1;KEY_2:VALUE_2") + }) + // @formatter:on + @RequestMapping(path = "oozieSubmit", method = RequestMethod.POST, produces = {"application/xml", "text/xml", "application/json", "text/yaml", + "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) + public GenericResponse oozieSubmit(@Parameter(hidden = true) @RequestParam MultiValueMap parameters, + @AuthenticationPrincipal DatawaveUserDetails currentUser) throws QueryException { + return mapReduceQueryManagementService.oozieSubmit(parameters, currentUser); + } + + /** + * @see MapReduceQueryManagementService#submit(MultiValueMap, DatawaveUserDetails) + */ + // @formatter:off + @Operation( + summary = "Execute a configured map reduce job.", + description = "Runs the selected map reduce job.
" + + "Loads the specified defined query, and runs it as a map reduce job.
" + + "By default, results will be written to an HDFS output directory.
" + + "If 'outputTableName' is specified, results will be written to a table in Accumulo instead.") + @ApiResponses({ + @ApiResponse( + description = "if successful, returns a generic response containing the map reduce query id", + responseCode = "200", + content = @Content(schema = @Schema(implementation = GenericResponse.class))), + @ApiResponse( + description = "if the job configuration can't be found
" + + "if parameter validation fails
" + + "if auditing fails", + responseCode = "400", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if the user doesn't have access to the requested job configuration
" + + "if the user doesn't own the defined query", + responseCode = "401", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if the query cannot be found", + responseCode = "404", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if query storage fails
" + + "if there is an unknown error", + responseCode = "500", + content = @Content(schema = @Schema(implementation = VoidResponse.class)))}) + @Parameters({ + @Parameter( + name = JOB_NAME, + in = ParameterIn.QUERY, + description = "The name of the map reduce job configuration to execute", + required = true, + schema = @Schema(implementation = String.class), + example = "OozieJob"), + @Parameter( + name = QUERY_AUTHORIZATIONS, + in = ParameterIn.QUERY, + description = "The query auths", + required = true, + schema = @Schema(implementation = String.class), + example = "PUBLIC,PRIVATE,BAR,FOO"), + @Parameter( + name = QUERY_ID, + in = ParameterIn.QUERY, + description = "The id of the query to run as a map reduce job", + required = true, + schema = @Schema(implementation = String.class)), + @Parameter( + name = FORMAT, + in = ParameterIn.QUERY, + description = "The serialization format to use when writing results", + required = true, + schema = @Schema(implementation = String.class), + example = "XML"), + @Parameter( + name = OUTPUT_TABLE_NAME, + in = ParameterIn.QUERY, + description = "The name of the table where the results should be written", + schema = @Schema(implementation = String.class)), + @Parameter( + name = OUTPUT_FORMAT, + in = ParameterIn.QUERY, + description = "The hadoop file output format to use when writing results", + schema = @Schema(implementation = String.class), + example = "TEXT"), + @Parameter( + name = PARAMETERS, + in = ParameterIn.QUERY, + description = "Additional query parameters", + schema = @Schema(implementation = String.class), + example = "KEY_1:VALUE_1;KEY_2:VALUE_2") + }) + // @formatter:on + @RequestMapping(path = "submit", method = RequestMethod.POST, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", + "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) + public GenericResponse submit(@Parameter(hidden = true) @RequestParam MultiValueMap parameters, + @AuthenticationPrincipal DatawaveUserDetails currentUser) throws QueryException { + return mapReduceQueryManagementService.submit(parameters, currentUser); + } + + /** + * @see MapReduceQueryManagementService#cancel(String, DatawaveUserDetails) + */ + // @formatter:off + @Operation( + summary = "Cancels the specified query.", + description = "Cancel can only be called on a running query.
" + + "Aside from admins, only the query owner can cancel the specified query.") + @ApiResponses({ + @ApiResponse( + description = "if successful, returns a void response indicating that the query was canceled", + responseCode = "200", + content = @Content(schema = @Schema(implementation = GenericResponse.class))), + @ApiResponse( + description = "if the query is not running", + responseCode = "400", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if the user doesn't own the query", + responseCode = "401", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if the query cannot be found", + responseCode = "404", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if query lock acquisition fails
" + + "if the cancel call is interrupted
" + + "if there is an unknown error", + responseCode = "500", + content = @Content(schema = @Schema(implementation = VoidResponse.class)))}) + // @formatter:on + @RequestMapping(path = "{id}/cancel", method = {RequestMethod.POST, RequestMethod.PUT}, produces = {"application/xml", "text/xml", "application/json", + "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) + public GenericResponse cancel(@Parameter(description = "The map reduce query id") @PathVariable String id, + @AuthenticationPrincipal DatawaveUserDetails currentUser) throws QueryException { + return mapReduceQueryManagementService.cancel(id, currentUser); + } + + /** + * @see MapReduceQueryManagementService#adminCancel(String, DatawaveUserDetails) + */ + // @formatter:off + @Operation( + summary = "Cancels the specified query using admin privileges.", + description = "Cancel can only be called on a running query.
" + + "Only admin users should be allowed to call this method.") + @ApiResponses({ + @ApiResponse( + description = "if successful, returns a void response indicating that the query was canceled", + responseCode = "200", + content = @Content(schema = @Schema(implementation = GenericResponse.class))), + @ApiResponse( + description = "if the query is not running", + responseCode = "400", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if the query cannot be found", + responseCode = "404", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if query lock acquisition fails
" + + "if the cancel call is interrupted
" + + "if there is an unknown error", + responseCode = "500", + content = @Content(schema = @Schema(implementation = VoidResponse.class)))}) + // @formatter:on + @Secured({"Administrator", "JBossAdministrator"}) + @RequestMapping(path = "{id}/adminCancel", method = {RequestMethod.POST, RequestMethod.PUT}, produces = {"application/xml", "text/xml", "application/json", + "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) + public GenericResponse adminCancel(@Parameter(description = "The map reduce query id") @PathVariable String id, + @AuthenticationPrincipal DatawaveUserDetails currentUser) throws QueryException { + return mapReduceQueryManagementService.adminCancel(id, currentUser); + } + + /** + * @see MapReduceQueryManagementService#restart(String, DatawaveUserDetails) + */ + // @formatter:off + @Operation( + summary = "Stops, and restarts the specified query.", + description = "Restart can be called on any query, whether it's running or not.
" + + "If the specified query is still running, it will be canceled. See cancel.
" + + "Restart creates a new, identical query, with a new query id.
" + + "Restart queries will start running immediately.
" + + "Auditing is performed before the new query is started.") + @ApiResponses({ + @ApiResponse( + description = "if successful, returns a generic response containing the new query id", + responseCode = "200", + content = @Content(schema = @Schema(implementation = GenericResponse.class))), + @ApiResponse( + description = "if parameter validation fails
" + + "if auditing fails", + responseCode = "400", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if the user doesn't have access to the requested job configuration
" + + "if the user doesn't own the defined query", + responseCode = "401", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if the query cannot be found", + responseCode = "404", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if query storage fails
" + + "if there is an unknown error", + responseCode = "500", + content = @Content(schema = @Schema(implementation = VoidResponse.class)))}) + // @formatter:on + @RequestMapping(path = "{id}/restart", method = {RequestMethod.PUT, RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", + "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) + public GenericResponse restart(@Parameter(description = "The map reduce query id") @PathVariable String id, + @AuthenticationPrincipal DatawaveUserDetails currentUser) throws QueryException { + return mapReduceQueryManagementService.restart(id, currentUser); + } + + /** + * @see MapReduceQueryManagementService#list(String, DatawaveUserDetails) + */ + // @formatter:off + @Operation( + summary = "Gets a list of result info for the specified query for the calling user.", + description = "Returns a list of result info for the specified query owned by the calling user.") + @ApiResponses({ + @ApiResponse( + description = "if successful, returns a map reduce info list response", + responseCode = "200", + content = @Content(schema = @Schema(implementation = MapReduceInfoResponseList.class))), + @ApiResponse( + description = "if there is an unknown error", + responseCode = "500", + content = @Content(schema = @Schema(implementation = VoidResponse.class)))}) + // @formatter:on + @RequestMapping(path = "{id}/list", method = RequestMethod.GET, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", + "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) + public MapReduceInfoResponseList list(@Parameter(description = "The map reduce query id") @PathVariable String id, + @AuthenticationPrincipal DatawaveUserDetails currentUser) throws QueryException { + return mapReduceQueryManagementService.list(id, currentUser); + } + + /** + * @see MapReduceQueryManagementService#getFile(String,String, DatawaveUserDetails) + */ + // @formatter:off + @Operation( + summary = "Gets a result file for the specified query for the calling user.", + description = "Returns a result file for the specified query owned by the calling user.") + @ApiResponses({ + @ApiResponse( + description = "if successful, a map reduce query result file", + responseCode = "200", + content = @Content(schema = @Schema(implementation = StreamingResponseBody.class))), + @ApiResponse( + description = "if there is an unknown error", + responseCode = "500", + content = @Content(schema = @Schema(implementation = VoidResponse.class)))}) + // @formatter:on + @RequestMapping(path = "{id}/getFile/{fileName}", method = RequestMethod.GET, produces = MediaType.APPLICATION_OCTET_STREAM_VALUE) + public ResponseEntity getFile(@Parameter(description = "The map reduce query id") @PathVariable String id, + @Parameter(description = "The file name") @PathVariable String fileName, @AuthenticationPrincipal DatawaveUserDetails currentUser) + throws QueryException { + final Map.Entry resultFile = mapReduceQueryManagementService.getFile(id, fileName, currentUser); + + // @formatter:off + return ResponseEntity.ok() + .contentType(MediaType.APPLICATION_OCTET_STREAM) + .header(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=\"" + fileName + "\"").body(outputStream -> { + try (FSDataInputStream inputStream = resultFile.getValue()) { + IOUtils.copy(inputStream, outputStream); + } + }); + // @formatter:on + } + + /** + * @see MapReduceQueryManagementService#getAllFiles(String, DatawaveUserDetails) + */ + // @formatter:off + @Operation( + summary = "Gets all result files for the specified query for the calling user.", + description = "Returns all result files for the specified query owned by the calling user.") + @ApiResponses({ + @ApiResponse( + description = "if successful, a tarball containing map reduce query result files", + responseCode = "200", + content = @Content(schema = @Schema(implementation = StreamingResponseBody.class))), + @ApiResponse( + description = "if there is an unknown error", + responseCode = "500", + content = @Content(schema = @Schema(implementation = VoidResponse.class)))}) + // @formatter:on + @RequestMapping(path = "{id}/getAllFiles", method = RequestMethod.GET, produces = MediaType.APPLICATION_OCTET_STREAM_VALUE) + public ResponseEntity getAllFiles(@Parameter(description = "The map reduce query id") @PathVariable String id, + @AuthenticationPrincipal DatawaveUserDetails currentUser) throws QueryException { + final Map resultFiles = mapReduceQueryManagementService.getAllFiles(id, currentUser); + + // @formatter:off + return ResponseEntity.ok() + .contentType(MediaType.APPLICATION_OCTET_STREAM) + .header(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=\"" + id + ".tar\"") + .body(outputStream -> { + TarArchiveOutputStream tarArchiveOutputStream = new TarArchiveOutputStream(outputStream); + tarArchiveOutputStream.setLongFileMode(TarArchiveOutputStream.LONGFILE_POSIX); + try { + for (Map.Entry resultFile : resultFiles.entrySet()) { + String fileName = resultFile.getKey().getPath().toString(); + TarArchiveEntry entry = new TarArchiveEntry(id + "/" + fileName, false); + entry.setSize(resultFile.getKey().getLen()); + tarArchiveOutputStream.putArchiveEntry(entry); + try { + IOUtils.copy(resultFile.getValue(), tarArchiveOutputStream); + } finally { + resultFile.getValue().close(); + } + tarArchiveOutputStream.closeArchiveEntry(); + } + tarArchiveOutputStream.finish(); + } finally { + try { + tarArchiveOutputStream.close(); + } catch (IOException ioe) { + // do nothing + } + } + }); + // @formatter:on + } + + /** + * @see MapReduceQueryManagementService#list(DatawaveUserDetails) + */ + // @formatter:off + @Operation( + summary = "Gets a list of result info for all map reduce queries owned by the calling user.", + description = "Returns a list of result info for all map reduce queries owned by the calling user.") + @ApiResponses({ + @ApiResponse( + description = "if successful, returns a map reduce info list response", + responseCode = "200", + content = @Content(schema = @Schema(implementation = MapReduceInfoResponseList.class))), + @ApiResponse( + description = "if there is an unknown error", + responseCode = "500", + content = @Content(schema = @Schema(implementation = VoidResponse.class)))}) + // @formatter:on + @RequestMapping(path = "list", method = RequestMethod.GET, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", + "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) + public MapReduceInfoResponseList list(@AuthenticationPrincipal DatawaveUserDetails currentUser) throws QueryException { + return mapReduceQueryManagementService.list(currentUser); + } + + /** + * @see MapReduceQueryManagementService#remove(String, DatawaveUserDetails) + */ + // @formatter:off + @Operation( + summary = "Removes the specified map reduce query from query storage.", + description = "If the map reduce query is running, it wil be canceled.
" + + "Aside from admins, only the query owner can remove the specified query.") + @ApiResponses({ + @ApiResponse( + description = "if successful, returns a void response indicating that the map reduce query was removed", + responseCode = "200", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if the user doesn't own the query", + responseCode = "401", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if the query cannot be found", + responseCode = "404", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if there is an unknown error", + responseCode = "500", + content = @Content(schema = @Schema(implementation = VoidResponse.class)))}) + // @formatter:on + @RequestMapping(path = "{id}/remove", method = RequestMethod.DELETE, produces = {"application/xml", "text/xml", "application/json", "text/yaml", + "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) + public VoidResponse remove(@Parameter(description = "The map reduce query id") @PathVariable String id, + @AuthenticationPrincipal DatawaveUserDetails currentUser) throws QueryException { + return mapReduceQueryManagementService.remove(id, currentUser); + } + + /** + * @see MapReduceQueryManagementService#adminRemove(String, DatawaveUserDetails) + */ + // @formatter:off + @Operation( + summary = "Removes the specified map reduce query from query storage using admin privileges.", + description = "If the map reduce query is running, it wil be canceled.
" + + "Only admin users should be allowed to call this method.") + @ApiResponses({ + @ApiResponse( + description = "if successful, returns a void response indicating that the map reduce query was removed", + responseCode = "200", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if the user doesn't own the query", + responseCode = "401", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if the query cannot be found", + responseCode = "404", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if there is an unknown error", + responseCode = "500", + content = @Content(schema = @Schema(implementation = VoidResponse.class)))}) + // @formatter:on + @Secured({"Administrator", "JBossAdministrator"}) + @RequestMapping(path = "{id}/adminRemove", method = {RequestMethod.DELETE}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", + "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) + public VoidResponse adminRemove(@Parameter(description = "The map reduce query id") @PathVariable String id, + @AuthenticationPrincipal DatawaveUserDetails currentUser) throws QueryException { + return mapReduceQueryManagementService.adminRemove(id, currentUser); + } + + // @formatter:off + @Operation( + summary = "Updates the state of the map reduce job.", + description = "This method is intended to be called by the map reduce job to update the state of the query.") + @ApiResponses({ + @ApiResponse( + description = "if successful, returns a void response indicating that the state was updated", + responseCode = "200", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if there is an unknown error", + responseCode = "500", + content = @Content(schema = @Schema(implementation = VoidResponse.class)))}) + // @formatter:on + @RequestMapping(path = "updateState", method = RequestMethod.GET, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", + "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) + public VoidResponse updateState(@RequestParam String jobId, @RequestParam String jobStatus) throws QueryException { + return mapReduceQueryManagementService.updateState(jobId, jobStatus); + } +} diff --git a/service/src/main/java/datawave/microservice/query/mapreduce/MapReduceQueryManagementService.java b/service/src/main/java/datawave/microservice/query/mapreduce/MapReduceQueryManagementService.java new file mode 100644 index 00000000..d76ab038 --- /dev/null +++ b/service/src/main/java/datawave/microservice/query/mapreduce/MapReduceQueryManagementService.java @@ -0,0 +1,766 @@ +package datawave.microservice.query.mapreduce; + +import static datawave.microservice.query.mapreduce.config.MapReduceQueryProperties.JOB_NAME; +import static datawave.microservice.query.mapreduce.config.MapReduceQueryProperties.PARAMETERS; +import static datawave.microservice.query.mapreduce.config.MapReduceQueryProperties.QUERY_ID; +import static datawave.microservice.query.mapreduce.jobs.OozieJob.WORKFLOW; +import static datawave.microservice.query.mapreduce.remote.MapReduceQueryRequest.Method.OOZIE_SUBMIT; +import static datawave.microservice.query.mapreduce.remote.MapReduceQueryRequest.Method.SUBMIT; +import static datawave.microservice.query.mapreduce.status.MapReduceQueryStatus.MapReduceQueryState.CANCELED; +import static datawave.microservice.query.mapreduce.status.MapReduceQueryStatus.MapReduceQueryState.FAILED; +import static datawave.microservice.query.storage.QueryStatus.QUERY_STATE.DEFINE; + +import java.io.IOException; +import java.net.URI; +import java.text.MessageFormat; +import java.util.AbstractMap; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; +import java.util.function.Supplier; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.LocatedFileStatus; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.RemoteIterator; +import org.apache.hadoop.mapred.JobClient; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.RunningJob; +import org.apache.hadoop.mapreduce.JobID; +import org.apache.hadoop.mapreduce.JobStatus; +import org.apache.http.HttpStatus; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.cloud.bus.BusProperties; +import org.springframework.cloud.bus.event.RemoteMapReduceQueryRequestEvent; +import org.springframework.context.ApplicationEventPublisher; +import org.springframework.stereotype.Service; +import org.springframework.util.LinkedMultiValueMap; +import org.springframework.util.MultiValueMap; + +import datawave.core.query.logic.QueryLogic; +import datawave.microservice.authorization.user.DatawaveUserDetails; +import datawave.microservice.authorization.util.AuthorizationsUtil; +import datawave.microservice.query.Query; +import datawave.microservice.query.QueryManagementService; +import datawave.microservice.query.QueryParameters; +import datawave.microservice.query.config.QueryProperties; +import datawave.microservice.query.mapreduce.config.MapReduceJobProperties; +import datawave.microservice.query.mapreduce.config.MapReduceQueryProperties; +import datawave.microservice.query.mapreduce.jobs.MapReduceJob; +import datawave.microservice.query.mapreduce.jobs.OozieJob; +import datawave.microservice.query.mapreduce.remote.MapReduceQueryRequest; +import datawave.microservice.query.mapreduce.remote.MapReduceQueryRequestHandler; +import datawave.microservice.query.mapreduce.status.MapReduceQueryCache; +import datawave.microservice.query.mapreduce.status.MapReduceQueryStatus; +import datawave.microservice.query.storage.QueryStatus; +import datawave.security.util.ProxiedEntityUtils; +import datawave.webservice.common.audit.AuditParameters; +import datawave.webservice.query.exception.BadRequestQueryException; +import datawave.webservice.query.exception.DatawaveErrorCode; +import datawave.webservice.query.exception.NotFoundQueryException; +import datawave.webservice.query.exception.QueryException; +import datawave.webservice.query.exception.UnauthorizedQueryException; +import datawave.webservice.result.GenericResponse; +import datawave.webservice.result.VoidResponse; +import datawave.webservice.results.mr.MapReduceInfoResponse; +import datawave.webservice.results.mr.MapReduceInfoResponseList; +import datawave.webservice.results.mr.MapReduceJobDescription; +import datawave.webservice.results.mr.MapReduceJobDescriptionList; +import datawave.webservice.results.mr.ResultFile; + +@Service +@ConditionalOnProperty(name = MapReduceQueryProperties.PREFIX + ".enabled", havingValue = "true", matchIfMissing = true) +public class MapReduceQueryManagementService implements MapReduceQueryRequestHandler { + + private final Logger log = LoggerFactory.getLogger(this.getClass()); + + private static final String PARAMETER_SEPARATOR = ";"; + private static final String PARAMETER_NAME_VALUE_SEPARATOR = ":"; + private final QueryProperties queryProperties; + + private final MapReduceQueryProperties mapReduceQueryProperties; + + private final ApplicationEventPublisher eventPublisher; + + private final BusProperties busProperties; + + private final QueryManagementService queryManagementService; + + private final MapReduceQueryCache mapReduceQueryCache; + + private final Map> mapReduceJobs; + + private final Configuration configuration; + + private final Map queryLatchMap = new ConcurrentHashMap<>(); + + public MapReduceQueryManagementService(QueryProperties queryProperties, MapReduceQueryProperties mapReduceQueryProperties, + ApplicationEventPublisher eventPublisher, BusProperties busProperties, QueryManagementService queryManagementService, + MapReduceQueryCache mapReduceQueryCache, Map> mapReduceJobs) { + this.queryProperties = queryProperties; + this.mapReduceQueryProperties = mapReduceQueryProperties; + this.eventPublisher = eventPublisher; + this.busProperties = busProperties; + this.queryManagementService = queryManagementService; + this.mapReduceQueryCache = mapReduceQueryCache; + this.mapReduceJobs = mapReduceJobs; + this.configuration = new Configuration(); + if (mapReduceQueryProperties.getFsConfigResources() != null) { + for (String resource : mapReduceQueryProperties.getFsConfigResources()) { + this.configuration.addResource(new Path(resource)); + } + } + } + + public MapReduceJobDescriptionList listConfigurations(String jobType, DatawaveUserDetails currentUser) { + log.info("Request: listConfigurations from {} for {}", ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()), jobType); + + MapReduceJobDescriptionList response = new MapReduceJobDescriptionList(); + List jobs = new ArrayList<>(); + if (jobType.equals("none")) { + jobType = null; + } + for (Map.Entry entry : mapReduceQueryProperties.getJobs().entrySet()) { + if (jobType != null && !entry.getValue().getJobType().equals(jobType)) { + continue; + } + jobs.add(createMapReduceJobDescription(entry.getKey(), entry.getValue())); + } + response.setResults(jobs); + return response; + } + + protected MapReduceJobDescription createMapReduceJobDescription(String name, MapReduceJobProperties jobProperties) { + MapReduceJobDescription desc = new MapReduceJobDescription(); + desc.setName(name); + desc.setJobType(jobProperties.getJobType()); + desc.setDescription(jobProperties.getDescription()); + List required = new ArrayList<>(jobProperties.getRequiredRuntimeParameters().keySet()); + desc.setRequiredRuntimeParameters(required); + List optional = new ArrayList<>(jobProperties.getOptionalRuntimeParameters().keySet()); + desc.setOptionalRuntimeParameters(optional); + return desc; + } + + public GenericResponse oozieSubmit(MultiValueMap parameters, DatawaveUserDetails currentUser) throws QueryException { + + String user = ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()); + if (log.isDebugEnabled()) { + log.info("Request: submit from {} with params: {}", user, parameters); + } else { + log.info("Request: submit from {}", user); + } + + try { + MultiValueMap parsedParameters = parseParameters(parameters); + + String workflow = parsedParameters.getFirst(WORKFLOW); + + String id = submitOozieWorkflow(workflow, parsedParameters, currentUser); + GenericResponse response = new GenericResponse<>(); + response.setResult(id); + return response; + } catch (QueryException e) { + throw e; + } catch (Exception e) { + log.error("Unknown error submitting oozie workflow", e); + throw new QueryException(DatawaveErrorCode.QUERY_SETUP_ERROR, e, "Unknown error submitting oozie workflow."); + } + } + + public GenericResponse submit(MultiValueMap parameters, DatawaveUserDetails currentUser) throws QueryException { + String user = ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()); + if (log.isDebugEnabled()) { + log.info("Request: submit from {} with params: {}", user, parameters); + } else { + log.info("Request: submit from {}", user); + } + + try { + MultiValueMap parsedParameters = parseParameters(parameters); + + String jobName = parsedParameters.getFirst(JOB_NAME); + + // make sure the query is valid, and the user can act on it + QueryStatus queryStatus = queryManagementService.validateRequest(parsedParameters.getFirst(QUERY_ID), currentUser); + + // make sure the state is define + if (queryStatus.getQueryState() == DEFINE) { + String id = submitJob(jobName, parsedParameters, queryStatus, currentUser); + GenericResponse response = new GenericResponse<>(); + response.setResult(id); + return response; + } else { + throw new BadRequestQueryException("Submit can only be called on a defined query", HttpStatus.SC_BAD_REQUEST + "-1"); + } + } catch (QueryException e) { + throw e; + } catch (Exception e) { + log.error("Unknown error submitting job", e); + throw new QueryException(DatawaveErrorCode.QUERY_SETUP_ERROR, e, "Unknown error submitting job."); + } + } + + protected String submitJob(String jobName, MultiValueMap parameters, QueryStatus queryStatus, DatawaveUserDetails currentUser) + throws Exception { + + // validate the job + MapReduceJob mapReduceJob = validateJob(jobName, parameters, currentUser); + + // create the audit parameters from the query definition + MultiValueMap auditParameters = new LinkedMultiValueMap<>(parameters); + auditParameters.addAll(new LinkedMultiValueMap<>(queryStatus.getQuery().toMap())); + + // validate the query and get the query logic + QueryLogic queryLogic = queryManagementService.validateQuery(queryStatus.getQuery().getQueryLogicName(), auditParameters, currentUser); + + String id = mapReduceJob.createId(currentUser); + + // audit the job + auditParameters.add(AuditParameters.AUDIT_ID, id); + queryManagementService.audit(queryStatus.getQuery(), queryLogic, auditParameters, currentUser); + + // store the job in the cache + mapReduceQueryCache.createQuery(id, jobName, parameters, queryStatus.getQuery(), currentUser); + + // send the job off to the mapreduce service + sendRequestAwaitResponse(MapReduceQueryRequest.submit(id), true); + + // return the id + return id; + } + + protected String submitOozieWorkflow(String workflow, MultiValueMap parameters, DatawaveUserDetails currentUser) throws Exception { + // validate the job + OozieJob oozieJob = (OozieJob) validateJob(workflow, parameters, currentUser); + + String id = oozieJob.createId(currentUser); + + // create the audit parameters from the query definition + MultiValueMap auditParameters = new LinkedMultiValueMap<>(parameters); + + // audit the job + // @formatter:off + queryManagementService.audit(id, + oozieJob.getAuditType(), + workflow, + oozieJob.getQuery(auditParameters), + oozieJob.getSelectors(auditParameters), + auditParameters, + currentUser); + // @formatter:on + + // store the job in the cache + mapReduceQueryCache.createQuery(id, workflow, parameters, currentUser); + + // send the job off to the mapreduce service + sendRequestAwaitResponse(MapReduceQueryRequest.oozieSubmit(id), true); + + // return the id + return id; + } + + protected MultiValueMap parseParameters(MultiValueMap parameters) { + List encodedParams = parameters.remove(PARAMETERS); + if (encodedParams != null) { + for (String encodedParam : encodedParams) { + parameters.addAll(parseParameters(encodedParam)); + } + } + return parameters; + } + + protected MultiValueMap parseParameters(String params) { + MultiValueMap parameters = new LinkedMultiValueMap<>(); + if (null != params) { + String[] entries = params.split(PARAMETER_SEPARATOR); + for (String entry : entries) { + String[] keyValue = entry.split(PARAMETER_NAME_VALUE_SEPARATOR); + if (keyValue.length == 2) { + parameters.add(keyValue[0], keyValue[1]); + } + } + } + return parameters; + } + + protected MapReduceJob validateJob(String jobName, MultiValueMap parameters, DatawaveUserDetails currentUser) + throws BadRequestQueryException, UnauthorizedQueryException { + + // get the map reduce job + MapReduceJob mapReduceJob = mapReduceJobs.get(jobName).get(); + if (mapReduceJob == null) { + throw new BadRequestQueryException(DatawaveErrorCode.JOB_CONFIGURATION_ERROR, "No job configuration with name " + jobName); + } + + // validate the user's roles + validateRoles(currentUser.getPrimaryUser().getRoles(), mapReduceJob.getMapReduceJobProperties().getRequiredRoles()); + + // validate the user's auths + validateAuths(parameters.getFirst(QueryParameters.QUERY_AUTHORIZATIONS), mapReduceJob.getMapReduceJobProperties().getRequiredAuths()); + + // validate the parameters + mapReduceJob.validateParameters(parameters); + + return mapReduceJob; + } + + protected void validateRoles(Collection userRoles, Collection requiredRoles) throws UnauthorizedQueryException { + if (requiredRoles != null && !requiredRoles.isEmpty()) { + if (!userRoles.containsAll(requiredRoles)) { + throw new UnauthorizedQueryException(DatawaveErrorCode.JOB_EXECUTION_UNAUTHORIZED, + MessageFormat.format("Requires the following roles: {0}", requiredRoles)); + } + } + } + + protected void validateAuths(String requestedAuths, Collection requiredAuths) throws UnauthorizedQueryException { + if (requiredAuths != null && !requiredAuths.isEmpty()) { + Set userAuths = new HashSet<>(AuthorizationsUtil.splitAuths(requestedAuths)); + if (!userAuths.containsAll(requiredAuths)) { + throw new UnauthorizedQueryException(DatawaveErrorCode.JOB_EXECUTION_UNAUTHORIZED, + MessageFormat.format("Requires the following auths: {0}", requiredAuths)); + } + } + } + + private void sendRequestAwaitResponse(MapReduceQueryRequest request, boolean isAwaitResponse) throws QueryException { + if (isAwaitResponse) { + // before publishing the message, create a latch based on the query ID + queryLatchMap.put(request.getId(), new CountDownLatch(1)); + } + + // publish an event to the map reduce query service + publishMapReduceQueryEvent(request); + + if (isAwaitResponse) { + long startTimeMillis = System.currentTimeMillis(); + + log.info("Waiting on map reduce query {} response from the map reduce query service.", request.getMethod().name()); + + try { + boolean isFinished = false; + while (!isFinished && System.currentTimeMillis() < (startTimeMillis + queryProperties.getExpiration().getCallTimeoutMillis())) { + try { + // wait for the executor response + if (queryLatchMap.get(request.getId()).await(queryProperties.getExpiration().getCallTimeoutInterval(), + queryProperties.getExpiration().getCallTimeoutIntervalUnit())) { + log.info("Received map reduce query {} response from the map reduce query service.", request.getMethod().name()); + isFinished = true; + } + + // did the request fail? + MapReduceQueryStatus mapReduceQueryStatus = mapReduceQueryCache.getQueryStatus(request.getId()); + if (mapReduceQueryStatus.getState() == FAILED) { + log.error("Map reduce query {} failed for id {}: {}", request.getMethod().name(), request.getId(), + mapReduceQueryStatus.getFailureMessage()); + throw new QueryException(mapReduceQueryStatus.getErrorCode(), "Map reduce query " + request.getMethod().name() + " failed for id " + + request.getId() + ": " + mapReduceQueryStatus.getFailureMessage()); + } + } catch (InterruptedException e) { + log.warn("Interrupted while waiting for map reduce query {} latch for id {}", request.getMethod().name(), request.getId()); + } + } + } finally { + queryLatchMap.remove(request.getId()); + } + } + } + + private void publishMapReduceQueryEvent(MapReduceQueryRequest mapReduceQueryRequest) { + // @formatter:off + eventPublisher.publishEvent( + new RemoteMapReduceQueryRequestEvent( + this, + busProperties.getId(), + mapReduceQueryProperties.getMapReduceQueryServiceName(), + mapReduceQueryRequest)); + // @formatter:on + } + + @Override + public void handleRemoteRequest(MapReduceQueryRequest queryRequest, String originService, String destinationService) { + try { + if (queryRequest.getMethod() == SUBMIT || queryRequest.getMethod() == OOZIE_SUBMIT) { + log.trace("Received remote {} request from {} for {}.", queryRequest.getMethod().name(), originService, destinationService); + if (queryLatchMap.containsKey(queryRequest.getId())) { + queryLatchMap.get(queryRequest.getId()).countDown(); + } else { + log.warn("Unable to decrement {} latch for query {}", queryRequest.getMethod().name(), queryRequest.getId()); + } + } else { + log.debug("No handling specified for remote map reduce query request method: {} from {} for {}", queryRequest.getMethod(), originService, + destinationService); + } + } catch (Exception e) { + log.error("Unknown error handling remote request: {} from {} for {}", queryRequest, originService, destinationService); + } + } + + public GenericResponse cancel(String id, DatawaveUserDetails currentUser) throws QueryException { + log.info("Request: cancel from {} for {}", ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()), id); + + return cancel(id, currentUser, false); + } + + public GenericResponse adminCancel(String id, DatawaveUserDetails currentUser) throws QueryException { + log.info("Request: adminCancel from {} for {}", ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()), id); + + return cancel(id, currentUser, true); + } + + private GenericResponse cancel(String id, DatawaveUserDetails currentUser, boolean adminOverride) throws QueryException { + try { + // make sure the query is valid, and the user can act on it + MapReduceQueryStatus mapReduceQueryStatus = validateRequest(id, currentUser, adminOverride); + + // if the map reduce job is submitted or running + if (mapReduceQueryStatus.isRunning()) { + cancel(id, mapReduceQueryStatus.getJobId(), mapReduceQueryStatus.getResultsDirectory()); + + GenericResponse response = new GenericResponse<>(); + response.setResult(true); + + return response; + } else { + throw new BadRequestQueryException("Cannot call cancel on a query that is not running", HttpStatus.SC_BAD_REQUEST + "-1"); + } + } catch (QueryException e) { + throw e; + } catch (Exception e) { + log.error("Unknown error canceling map reduce query {}", id, e); + throw new QueryException(DatawaveErrorCode.CANCELLATION_ERROR, e, "Unknown error canceling map reduce query " + id); + } + } + + private void cancel(String id, String jobId, String resultsDirectory) throws QueryException, IOException, InterruptedException { + cancel(id, jobId); + removeDirectory(resultsDirectory); + } + + private void cancel(String id, String jobId) throws IOException, QueryException, InterruptedException { + // cancel the map reduce job + try (JobClient job = new JobClient(new JobConf(configuration))) { + JobID mrJobId = JobID.forName(jobId); + if (mrJobId instanceof org.apache.hadoop.mapred.JobID) { + RunningJob runningJob = job.getJob((org.apache.hadoop.mapred.JobID) mrJobId); + if (null != runningJob) { + // killing the job will trigger hadoop to update the status via the callback URL + runningJob.killJob(); + } else { + mapReduceQueryCache.updateQueryStatus(id, (mrQueryStatus) -> mrQueryStatus.setState(CANCELED), + mapReduceQueryProperties.getLockWaitTimeMillis(), mapReduceQueryProperties.getLockLeaseTimeMillis()); + } + } + } + } + + private void removeDirectory(String directory) throws IOException, QueryException { + FileSystem filesystem = FileSystem.get(configuration); + Path dir = new Path(directory); + if (filesystem.exists(dir) && !filesystem.delete(dir, true)) { + log.error("Unknown error deleting directory: {}", dir); + throw new QueryException(DatawaveErrorCode.MAPRED_RESULTS_DELETE_ERROR, "Unknown error deleting directory: " + dir); + } + } + + public GenericResponse restart(String id, DatawaveUserDetails currentUser) throws QueryException { + log.info("Request: restart from {} for {}", ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()), id); + + try { + // make sure the map reduce query is valid, and the user can act on it + MapReduceQueryStatus mapReduceQueryStatus = validateRequest(id, currentUser); + + // if the map reduce job is submitted or running, cancel it + if (mapReduceQueryStatus.isRunning()) { + cancel(id, mapReduceQueryStatus.getJobId(), mapReduceQueryStatus.getResultsDirectory()); + } + + MultiValueMap parsedParameters = mapReduceQueryStatus.getParameters(); + + // make sure the original query is valid, and the user can act on it + QueryStatus queryStatus = queryManagementService.validateRequest(parsedParameters.getFirst(QUERY_ID), currentUser); + + String newId = submitJob(mapReduceQueryStatus.getJobName(), parsedParameters, queryStatus, currentUser); + GenericResponse response = new GenericResponse<>(); + response.setResult(newId); + return response; + } catch (QueryException e) { + throw e; + } catch (Exception e) { + log.error("Unknown error listing map reduce info", e); + throw new QueryException(DatawaveErrorCode.QUERY_LISTING_ERROR, e, "Unknown error listing map reduce info."); + } + } + + public MapReduceInfoResponseList list(String id, DatawaveUserDetails currentUser) throws QueryException { + log.info("Request: list from {} for {}", ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()), id); + + try { + // make sure the query is valid, and the user can act on it + MapReduceQueryStatus mapReduceQueryStatus = validateRequest(id, currentUser); + + MapReduceInfoResponseList respList = new MapReduceInfoResponseList(); + respList.setResults(Collections.singletonList(createMapReduceInfoResponse(mapReduceQueryStatus))); + + return respList; + } catch (QueryException e) { + throw e; + } catch (Exception e) { + log.error("Unknown error listing map reduce info", e); + throw new QueryException(DatawaveErrorCode.QUERY_LISTING_ERROR, e, "Unknown error listing map reduce info."); + } + } + + protected MapReduceInfoResponse createMapReduceInfoResponse(MapReduceQueryStatus mapReduceQueryStatus) throws QueryException { + MapReduceInfoResponse mapReduceInfoResponse = mapReduceQueryStatus.toMapReduceInfoResponse(); + + List files = listFiles(mapReduceQueryStatus.getResultsDirectory()); + List resultFiles = new ArrayList<>(); + for (LocatedFileStatus file : files) { + ResultFile resultFile = new ResultFile(); + resultFile.setFileName(file.getPath().toString()); + resultFile.setLength(file.getLen()); + resultFiles.add(resultFile); + } + mapReduceInfoResponse.setResultFiles(resultFiles); + return mapReduceInfoResponse; + } + + public MapReduceInfoResponseList list(DatawaveUserDetails currentUser) throws QueryException { + log.info("Request: list for {}", ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName())); + + try { + Set ids = mapReduceQueryCache.lookupQueryIdsByUsername(currentUser.getUsername()); + + List mapReduceInfoResponses = new ArrayList<>(); + for (String id : ids) { + MapReduceQueryStatus status = mapReduceQueryCache.getQueryStatus(id); + mapReduceInfoResponses.add(createMapReduceInfoResponse(status)); + } + + MapReduceInfoResponseList respList = new MapReduceInfoResponseList(); + respList.setResults(mapReduceInfoResponses); + + return respList; + } catch (QueryException e) { + throw e; + } catch (Exception e) { + log.error("Unknown error listing map reduce info", e); + throw new QueryException(DatawaveErrorCode.QUERY_LISTING_ERROR, e, "Unknown error listing map reduce info."); + } + } + + public MapReduceQueryStatus validateRequest(String id, DatawaveUserDetails currentUser) throws NotFoundQueryException, UnauthorizedQueryException { + return validateRequest(id, currentUser, false); + } + + public MapReduceQueryStatus validateRequest(String id, DatawaveUserDetails currentUser, boolean adminOverride) + throws NotFoundQueryException, UnauthorizedQueryException { + // does the map reduce job exist? + MapReduceQueryStatus mapReduceQueryStatus = mapReduceQueryCache.getQueryStatus(id); + if (mapReduceQueryStatus == null) { + throw new NotFoundQueryException(DatawaveErrorCode.NO_QUERY_OBJECT_MATCH, MessageFormat.format("{0}", id)); + } + + // admin requests can operate on any job, regardless of ownership + if (!adminOverride) { + // does the current user own this job? + String userId = ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getDn().subjectDN()); + Query query = mapReduceQueryStatus.getQuery(); + if (!query.getOwner().equals(userId)) { + throw new UnauthorizedQueryException(DatawaveErrorCode.QUERY_OWNER_MISMATCH, MessageFormat.format("{0} != {1}", userId, query.getOwner())); + } + } + + return mapReduceQueryStatus; + } + + public Map.Entry getFile(String id, String fileName, DatawaveUserDetails currentUser) throws QueryException { + log.info("Request: getFile from {} for {}, {}", ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()), id, fileName); + + try { + // make sure the query is valid, and the user can act on it + MapReduceQueryStatus mapReduceQueryStatus = validateRequest(id, currentUser); + + String resultsDirectory = mapReduceQueryStatus.getResultsDirectory(); + + Path resultsPath = new Path(new URI(resultsDirectory)); + Path resultFile = new Path(resultsPath, fileName); + + FileSystem filesystem = FileSystem.get(configuration); + FileStatus fileStatus = filesystem.getFileStatus(resultFile); + + if (!fileStatus.isFile()) { + throw new BadRequestQueryException("Requested path is not a file: " + fileName, HttpStatus.SC_BAD_REQUEST + "-1"); + } + + // update the file status to reflect a relative file path + fileStatus.setPath(getRelativeFilePath(filesystem.getFileStatus(resultsPath), fileStatus)); + + return new AbstractMap.SimpleEntry<>(fileStatus, getFileInputStream(resultFile)); + } catch (QueryException e) { + throw e; + } catch (Exception e) { + log.error("Unknown error retrieving result file", e); + throw new QueryException(DatawaveErrorCode.QUERY_LISTING_ERROR, e, "Unknown error retrieving result file."); + } + } + + private FSDataInputStream getFileInputStream(FileSystem filesystem, Path filePath) throws QueryException { + try { + return filesystem.open(filePath); + } catch (IOException e) { + throw new NotFoundQueryException("Unable to open result file", e, HttpStatus.SC_INTERNAL_SERVER_ERROR + "-1"); + } + } + + private FSDataInputStream getFileInputStream(Path filePath) throws QueryException, IOException { + return getFileInputStream(FileSystem.get(configuration), filePath); + } + + public Map getAllFiles(String id, DatawaveUserDetails currentUser) throws QueryException { + log.info("Request: getAllFiles from {} for {}", ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()), id); + + Map resultFiles = new HashMap<>(); + try { + // make sure the query is valid, and the user can act on it + MapReduceQueryStatus mapReduceQueryStatus = validateRequest(id, currentUser); + + FileSystem fs = FileSystem.get(configuration); + + FileStatus basePathStatus = fs.getFileStatus(new Path(mapReduceQueryStatus.getResultsDirectory())); + + if (mapReduceQueryStatus.getResultsDirectory() != null && !mapReduceQueryStatus.getResultsDirectory().isEmpty()) { + for (LocatedFileStatus fileStatus : listFiles(mapReduceQueryStatus.getResultsDirectory())) { + if (fileStatus.isFile()) { + FSDataInputStream inputStream = getFileInputStream(fs, fileStatus.getPath()); + + // update the file status to reflect a relative file path + fileStatus.setPath(getRelativeFilePath(basePathStatus, fileStatus)); + + resultFiles.put(fileStatus, inputStream); + } + } + } + return resultFiles; + } catch (QueryException e) { + for (Map.Entry resultFile : resultFiles.entrySet()) { + try { + resultFile.getValue().close(); + } catch (Exception ex) { + log.error("Unknown error closing input stream", e); + } + } + + throw e; + } catch (Exception e) { + for (Map.Entry resultFile : resultFiles.entrySet()) { + try { + resultFile.getValue().close(); + } catch (Exception ex) { + log.error("Unknown error closing input stream", e); + } + } + + log.error("Unknown error retrieving result file", e); + throw new QueryException(DatawaveErrorCode.QUERY_LISTING_ERROR, e, "Unknown error retrieving result file."); + } + } + + private Path getRelativeFilePath(FileStatus basePath, FileStatus filePath) { + int basePathLength = basePath.getPath().toUri().getPath().length(); + return new Path(filePath.getPath().toUri().getPath().substring(basePathLength + 1)); + } + + public VoidResponse remove(String id, DatawaveUserDetails currentUser) throws QueryException { + log.info("Request: remove from {} for {}", ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()), id); + + return remove(id, currentUser, false); + } + + public VoidResponse adminRemove(String id, DatawaveUserDetails currentUser) throws QueryException { + log.info("Request: adminRemove from {} for {}", ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()), id); + + return remove(id, currentUser, true); + } + + private VoidResponse remove(String id, DatawaveUserDetails currentUser, boolean adminOverride) throws QueryException { + try { + // make sure the query is valid, and the user can act on it + MapReduceQueryStatus mapReduceQueryStatus = validateRequest(id, currentUser, adminOverride); + + // if the map reduce job is submitted or running, cancel it + if (mapReduceQueryStatus.isRunning()) { + cancel(id, mapReduceQueryStatus.getJobId()); + } + + // remove the working directory + removeDirectory(mapReduceQueryStatus.getWorkingDirectory()); + + // remove the cache entry + mapReduceQueryCache.removeQuery(mapReduceQueryStatus.getId()); + + return new VoidResponse(); + } catch (QueryException e) { + throw e; + } catch (Exception e) { + log.error("Unknown error removing map reduce query {}", id, e); + throw new QueryException(DatawaveErrorCode.QUERY_REMOVAL_ERROR, e, "Unknown error removing map reduce query " + id); + } + } + + private List listFiles(String resultsDirectory) throws QueryException { + try { + List files = new ArrayList<>(); + + if (resultsDirectory != null && !resultsDirectory.isEmpty()) { + FileSystem fs = FileSystem.get(configuration); + Path resultsPath = new Path(new URI(resultsDirectory)); + if (fs.exists(resultsPath)) { + RemoteIterator fileStatusIter = fs.listFiles(resultsPath, true); + while (fileStatusIter.hasNext()) { + LocatedFileStatus fileStatus = fileStatusIter.next(); + if (fileStatus.isFile()) { + files.add(fileStatus); + } + } + } + } + + return files; + } catch (Exception e) { + log.error("Unknown error listing files", e); + throw new QueryException(DatawaveErrorCode.FILE_LIST_ERROR, e, "Unknown error listing files."); + } + } + + public VoidResponse updateState(String jobId, String jobStatus) throws QueryException { + log.info("Request: updateState for {} to {}", jobId, jobStatus); + + VoidResponse response = new VoidResponse(); + try { + String id = mapReduceQueryCache.lookupQueryIdByJobId(jobId); + if (id != null) { + mapReduceQueryCache.updateQueryStatus(id, (mapReduceQueryStatus) -> { + mapReduceQueryStatus.setState(JobStatus.State.valueOf(jobStatus)); + }, mapReduceQueryProperties.getLockWaitTimeMillis(), mapReduceQueryProperties.getLockLeaseTimeMillis()); + } + return response; + } catch (Exception e) { + throw new QueryException(DatawaveErrorCode.MAPRED_UPDATE_STATUS_ERROR, e); + } + } +} diff --git a/service/src/main/java/datawave/microservice/query/mapreduce/config/MapReduceQueryControllerConfig.java b/service/src/main/java/datawave/microservice/query/mapreduce/config/MapReduceQueryControllerConfig.java new file mode 100644 index 00000000..ec814f18 --- /dev/null +++ b/service/src/main/java/datawave/microservice/query/mapreduce/config/MapReduceQueryControllerConfig.java @@ -0,0 +1,42 @@ +package datawave.microservice.query.mapreduce.config; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; +import org.springframework.security.config.annotation.web.configuration.WebSecurityCustomizer; +import org.springframework.web.servlet.config.annotation.AsyncSupportConfigurer; +import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; + +@Configuration +@ConditionalOnProperty(name = MapReduceQueryProperties.PREFIX + ".enabled", havingValue = "true", matchIfMissing = true) +public class MapReduceQueryControllerConfig { + @Autowired + MapReduceQueryProperties mapReduceQueryProperties; + + @Bean + public WebSecurityCustomizer ignoreMapReduceUpdateState() { + return (web) -> web.ignoring().antMatchers("/v1/mapreduce/updateState"); + } + + @Bean + public ThreadPoolTaskExecutor mvcTaskExecutor() { + ThreadPoolTaskExecutor taskExecutor = new ThreadPoolTaskExecutor(); + taskExecutor.setCorePoolSize(mapReduceQueryProperties.getExecutor().getCorePoolSize()); + taskExecutor.setMaxPoolSize(mapReduceQueryProperties.getExecutor().getMaxPoolSize()); + taskExecutor.setQueueCapacity(mapReduceQueryProperties.getExecutor().getQueueCapacity()); + taskExecutor.setThreadNamePrefix(mapReduceQueryProperties.getExecutor().getThreadNamePrefix()); + return taskExecutor; + } + + @Bean + public WebMvcConfigurer taskExecutorConfiguration() { + return new WebMvcConfigurer() { + @Override + public void configureAsyncSupport(AsyncSupportConfigurer configurer) { + configurer.setTaskExecutor(mvcTaskExecutor()); + } + }; + } +} diff --git a/service/src/main/java/datawave/microservice/query/monitor/MonitorTask.java b/service/src/main/java/datawave/microservice/query/monitor/MonitorTask.java new file mode 100644 index 00000000..8ce898b8 --- /dev/null +++ b/service/src/main/java/datawave/microservice/query/monitor/MonitorTask.java @@ -0,0 +1,129 @@ +package datawave.microservice.query.monitor; + +import java.io.IOException; +import java.util.concurrent.Callable; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import datawave.microservice.query.QueryManagementService; +import datawave.microservice.query.config.QueryExpirationProperties; +import datawave.microservice.query.messaging.QueryResultsManager; +import datawave.microservice.query.monitor.cache.MonitorStatus; +import datawave.microservice.query.monitor.cache.MonitorStatusCache; +import datawave.microservice.query.monitor.config.MonitorProperties; +import datawave.microservice.query.storage.QueryStatus; +import datawave.microservice.query.storage.QueryStorageCache; +import datawave.webservice.query.exception.QueryException; + +public class MonitorTask implements Callable { + private final Logger log = LoggerFactory.getLogger(this.getClass()); + + private final MonitorProperties monitorProperties; + private final QueryExpirationProperties expirationProperties; + private final MonitorStatusCache monitorStatusCache; + private final QueryStorageCache queryStorageCache; + private final QueryResultsManager queryQueueManager; + private final QueryManagementService queryManagementService; + + public MonitorTask(MonitorProperties monitorProperties, QueryExpirationProperties expirationProperties, MonitorStatusCache monitorStatusCache, + QueryStorageCache queryStorageCache, QueryResultsManager queryQueueManager, QueryManagementService queryManagementService) { + this.monitorProperties = monitorProperties; + this.expirationProperties = expirationProperties; + this.monitorStatusCache = monitorStatusCache; + this.queryStorageCache = queryStorageCache; + this.queryQueueManager = queryQueueManager; + this.queryManagementService = queryManagementService; + } + + @Override + public Void call() throws Exception { + if (tryLock()) { + boolean success = false; + MonitorStatus monitorStatus = null; + try { + long currentTimeMillis = System.currentTimeMillis(); + monitorStatus = monitorStatusCache.getStatus(); + if (monitorStatus.isExpired(currentTimeMillis, monitorProperties.getMonitorIntervalMillis())) { + monitor(currentTimeMillis); + success = true; + } + } finally { + if (success) { + monitorStatus.setLastChecked(System.currentTimeMillis()); + monitorStatusCache.setStatus(monitorStatus); + } + unlock(); + } + } + return null; + } + + // Check for the following conditions + // 1) Is query progress idle? If so, poke the query + // 2) Is the user idle? If so, close the query + // 3) Are there any other conditions that we should check for? + private void monitor(long currentTimeMillis) { + for (QueryStatus status : queryStorageCache.getQueryStatus()) { + String queryId = status.getQueryKey().getQueryId(); + + // if the query is not running + if (!status.isRunning()) { + + // if the query has been inactive too long (i.e. no interaction from the user or software) + if (status.isInactive(currentTimeMillis, monitorProperties.getInactiveQueryTimeToLiveMillis())) { + deleteQuery(queryId); + } + // delete the results queue if it exists + else { + queryQueueManager.deleteQuery(queryId); + } + } + // if the query is running + else { + // if the query isn't making progress + if (status.isProgressIdle(currentTimeMillis, expirationProperties.getProgressTimeoutMillis())) { + defibrillateQuery(queryId, status.getQueryKey().getQueryPool()); + } + // if the user hasn't interacted with the query + else if (status.isUserIdle(currentTimeMillis, expirationProperties.getIdleTimeoutMillis())) { + cancelQuery(queryId); + } + } + } + } + + private void cancelQuery(String queryId) { + try { + queryManagementService.cancel(queryId, true); + } catch (InterruptedException e) { + log.error("Interrupted while trying to cancel idle query: " + queryId, e); + } catch (QueryException e) { + log.error("Encountered error while trying to cancel idle query: " + queryId, e); + } + } + + private void defibrillateQuery(String queryId, String queryPool) { + // publish a next event to the executor pool + queryManagementService.publishNextEvent(queryId, queryPool); + } + + private void deleteQuery(String queryId) { + try { + // deletes everything for a query + // the result queue, the query status, the tasks, the task states + queryStorageCache.deleteQuery(queryId); + } catch (IOException e) { + log.error("Encountered error while trying to evict inactive query: " + queryId, e); + } + } + + private boolean tryLock() throws InterruptedException { + return monitorStatusCache.tryLock(monitorProperties.getLockWaitTime(), monitorProperties.getLockWaitTimeUnit(), monitorProperties.getLockLeaseTime(), + monitorProperties.getLockLeaseTimeUnit()); + } + + private void unlock() { + monitorStatusCache.unlock(); + } +} diff --git a/service/src/main/java/datawave/microservice/query/monitor/QueryMonitor.java b/service/src/main/java/datawave/microservice/query/monitor/QueryMonitor.java new file mode 100644 index 00000000..75d1ca60 --- /dev/null +++ b/service/src/main/java/datawave/microservice/query/monitor/QueryMonitor.java @@ -0,0 +1,91 @@ +package datawave.microservice.query.monitor; + +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +import datawave.microservice.query.QueryManagementService; +import datawave.microservice.query.config.QueryExpirationProperties; +import datawave.microservice.query.config.QueryProperties; +import datawave.microservice.query.messaging.QueryResultsManager; +import datawave.microservice.query.monitor.cache.MonitorStatusCache; +import datawave.microservice.query.monitor.config.MonitorProperties; +import datawave.microservice.query.storage.QueryStorageCache; + +@Component +@ConditionalOnProperty(name = "datawave.query.monitor.enabled", havingValue = "true", matchIfMissing = true) +public class QueryMonitor { + private final Logger log = LoggerFactory.getLogger(this.getClass()); + + private final MonitorProperties monitorProperties; + private final QueryExpirationProperties expirationProperties; + private final MonitorStatusCache monitorStatusCache; + private final QueryStorageCache queryStorageCache; + private final QueryResultsManager queryResultsManager; + private final QueryManagementService queryManagementService; + private final ExecutorService executor = Executors.newSingleThreadExecutor(); + + private long taskStartTime; + private Future taskFuture; + + public QueryMonitor(MonitorProperties monitorProperties, QueryProperties queryProperties, MonitorStatusCache monitorStatusCache, + QueryStorageCache queryStorageCache, QueryResultsManager queryResultsManager, QueryManagementService queryManagementService) { + this.monitorProperties = monitorProperties; + this.expirationProperties = queryProperties.getExpiration(); + this.monitorStatusCache = monitorStatusCache; + this.queryStorageCache = queryStorageCache; + this.queryResultsManager = queryResultsManager; + this.queryManagementService = queryManagementService; + } + + // this runs in a separate thread every 30 seconds (by default) + @Scheduled(cron = "${datawave.query.monitor.scheduler-crontab:*/30 * * * * ?}") + public void monitorTaskScheduler() { + // perform some upkeep + if (taskFuture != null) { + if (taskFuture.isDone()) { + try { + taskFuture.get(); + } catch (InterruptedException e) { + log.warn("Query Monitor task was interrupted"); + } catch (ExecutionException e) { + log.error("Query Monitor task failed", e.getCause()); + } + taskFuture = null; + } else if (isTaskLeaseExpired()) { + // if the lease has expired for the future, cancel it and wait for next scheduled task + taskFuture.cancel(true); + } + } + + // schedule a new monitor task if the previous one has finished/expired + if (taskFuture == null && isMonitorIntervalExpired()) { + taskStartTime = System.currentTimeMillis(); + // @formatter:off + taskFuture = executor.submit( + new MonitorTask( + monitorProperties, + expirationProperties, + monitorStatusCache, + queryStorageCache, + queryResultsManager, + queryManagementService)); + // @formatter:on + } + } + + private boolean isTaskLeaseExpired() { + return (System.currentTimeMillis() - taskStartTime) > monitorProperties.getMonitorIntervalMillis(); + } + + private boolean isMonitorIntervalExpired() { + return (System.currentTimeMillis() - monitorStatusCache.getStatus().getLastCheckedMillis()) > monitorProperties.getMonitorIntervalMillis(); + } +} diff --git a/service/src/main/java/datawave/microservice/query/monitor/cache/MonitorStatus.java b/service/src/main/java/datawave/microservice/query/monitor/cache/MonitorStatus.java new file mode 100644 index 00000000..e459b5dd --- /dev/null +++ b/service/src/main/java/datawave/microservice/query/monitor/cache/MonitorStatus.java @@ -0,0 +1,19 @@ +package datawave.microservice.query.monitor.cache; + +import java.io.Serializable; + +public class MonitorStatus implements Serializable { + private long lastCheckedMillis; + + public long getLastCheckedMillis() { + return lastCheckedMillis; + } + + public void setLastChecked(long lastCheckedMillis) { + this.lastCheckedMillis = lastCheckedMillis; + } + + public boolean isExpired(long currentTimeMillis, long expirationTimeoutMillis) { + return (currentTimeMillis - lastCheckedMillis) >= expirationTimeoutMillis; + } +} diff --git a/service/src/main/java/datawave/microservice/query/monitor/cache/MonitorStatusCache.java b/service/src/main/java/datawave/microservice/query/monitor/cache/MonitorStatusCache.java new file mode 100644 index 00000000..ef1b31d2 --- /dev/null +++ b/service/src/main/java/datawave/microservice/query/monitor/cache/MonitorStatusCache.java @@ -0,0 +1,94 @@ +package datawave.microservice.query.monitor.cache; + +import java.util.concurrent.TimeUnit; + +import org.springframework.cache.annotation.CacheConfig; +import org.springframework.cache.annotation.CacheEvict; +import org.springframework.cache.annotation.CachePut; + +import datawave.microservice.cached.LockableCacheInspector; + +@CacheConfig(cacheNames = MonitorStatusCache.CACHE_NAME) +public class MonitorStatusCache { + private final LockableCacheInspector cacheInspector; + + public static final String CACHE_NAME = "QueryMonitorCache"; + public static final String CACHE_KEY = "MonitorStatus"; + + public MonitorStatusCache(LockableCacheInspector cacheInspector) { + this.cacheInspector = cacheInspector; + } + + /** + * Get the query monitor status + * + * @return the stored monitor status + */ + public MonitorStatus getStatus() { + MonitorStatus status = cacheInspector.list(CACHE_NAME, MonitorStatus.class, CACHE_KEY); + if (status == null) { + lock(); + try { + status = cacheInspector.list(CACHE_NAME, MonitorStatus.class, CACHE_KEY); + if (status == null) { + status = setStatus(new MonitorStatus()); + } + } finally { + unlock(); + } + } + return status; + } + + /** + * Store the query monitor status + * + * @param monitorStatus + * The monitor status to store + * @return the stored monitor status + */ + @CachePut(key = "'" + CACHE_KEY + "'") + public MonitorStatus setStatus(MonitorStatus monitorStatus) { + return monitorStatus; + } + + /** + * Deletes the query monitor status + */ + @CacheEvict(key = "'" + CACHE_KEY + "'") + public void deleteStatus() { + + } + + public void lock() { + cacheInspector.lock(CACHE_NAME, CACHE_KEY); + } + + public void lock(long leaseTime, TimeUnit leaseTimeUnit) { + cacheInspector.lock(CACHE_NAME, CACHE_KEY, leaseTime, leaseTimeUnit); + } + + public boolean tryLock() { + return cacheInspector.tryLock(CACHE_NAME, CACHE_KEY); + } + + public boolean tryLock(long waitTime, TimeUnit waitTimeUnit) throws InterruptedException { + return cacheInspector.tryLock(CACHE_NAME, CACHE_KEY, waitTime, waitTimeUnit); + } + + public boolean tryLock(long waitTime, TimeUnit waitTimeUnit, long leaseTime, TimeUnit leaseTimeUnit) throws InterruptedException { + return cacheInspector.tryLock(CACHE_NAME, CACHE_KEY, waitTime, waitTimeUnit, leaseTime, leaseTimeUnit); + } + + public void unlock() { + cacheInspector.unlock(CACHE_NAME, CACHE_KEY); + } + + public void forceUnlock() { + cacheInspector.forceUnlock(CACHE_NAME, CACHE_KEY); + } + + public boolean isLocked() { + return cacheInspector.isLocked(CACHE_NAME, CACHE_KEY); + } +} diff --git a/service/src/main/java/datawave/microservice/query/monitor/config/MonitorConfig.java b/service/src/main/java/datawave/microservice/query/monitor/config/MonitorConfig.java new file mode 100644 index 00000000..afce3d57 --- /dev/null +++ b/service/src/main/java/datawave/microservice/query/monitor/config/MonitorConfig.java @@ -0,0 +1,43 @@ +package datawave.microservice.query.monitor.config; + +import java.util.function.Function; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.cache.CacheManager; +import org.springframework.cache.annotation.EnableCaching; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.scheduling.annotation.EnableScheduling; + +import com.hazelcast.spring.cache.HazelcastCacheManager; + +import datawave.microservice.cached.CacheInspector; +import datawave.microservice.cached.LockableCacheInspector; +import datawave.microservice.cached.LockableHazelcastCacheInspector; +import datawave.microservice.cached.UniversalLockableCacheInspector; +import datawave.microservice.query.monitor.cache.MonitorStatusCache; + +@EnableCaching +@EnableScheduling +@Configuration("QueryMonitorConfig") +@ConditionalOnProperty(name = "datawave.query.monitor.enabled", havingValue = "true", matchIfMissing = true) +@EnableConfigurationProperties(MonitorProperties.class) +public class MonitorConfig { + private final Logger log = LoggerFactory.getLogger(this.getClass()); + + @Bean + public MonitorStatusCache monitorStatusCache(@Qualifier("cacheInspectorFactory") Function cacheInspectorFactory, + CacheManager cacheManager) { + log.debug("Using " + cacheManager.getClass() + " for caching"); + LockableCacheInspector lockableCacheInspector; + if (cacheManager instanceof HazelcastCacheManager) + lockableCacheInspector = new LockableHazelcastCacheInspector(cacheManager); + else + lockableCacheInspector = new UniversalLockableCacheInspector(cacheInspectorFactory.apply(cacheManager)); + return new MonitorStatusCache(lockableCacheInspector); + } +} diff --git a/service/src/main/java/datawave/microservice/query/monitor/config/MonitorProperties.java b/service/src/main/java/datawave/microservice/query/monitor/config/MonitorProperties.java new file mode 100644 index 00000000..b8518a17 --- /dev/null +++ b/service/src/main/java/datawave/microservice/query/monitor/config/MonitorProperties.java @@ -0,0 +1,123 @@ +package datawave.microservice.query.monitor.config; + +import java.util.concurrent.TimeUnit; + +import javax.validation.constraints.NotNull; +import javax.validation.constraints.Positive; +import javax.validation.constraints.PositiveOrZero; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.validation.annotation.Validated; + +@Validated +@ConfigurationProperties(prefix = "datawave.query.monitor") +public class MonitorProperties { + private String schedulerCrontab; + @PositiveOrZero + private long monitorInterval = TimeUnit.MILLISECONDS.toMillis(30); + @NotNull + private TimeUnit monitorIntervalUnit = TimeUnit.MILLISECONDS; + // The amount of time to wait for the monitor lock to be acquired + @PositiveOrZero + private long lockWaitTime = 0; + @NotNull + private TimeUnit lockWaitTimeUnit = TimeUnit.MILLISECONDS; + // The amount of time that the monitor lock will be held before being automatically released + @Positive + private long lockLeaseTime = TimeUnit.MINUTES.toMillis(1); + @NotNull + private TimeUnit lockLeaseTimeUnit = TimeUnit.MILLISECONDS; + // The amount of time that an inactive query should remain in the query cache + @PositiveOrZero + private long inactiveQueryTimeToLive = 1; + @NotNull + private TimeUnit inactiveQueryTimeToLiveUnit = TimeUnit.DAYS; + + public String getSchedulerCrontab() { + return schedulerCrontab; + } + + public void setSchedulerCrontab(String schedulerCrontab) { + this.schedulerCrontab = schedulerCrontab; + } + + public long getMonitorInterval() { + return monitorInterval; + } + + public long getMonitorIntervalMillis() { + return monitorIntervalUnit.toMillis(monitorInterval); + } + + public void setMonitorInterval(long monitorInterval) { + this.monitorInterval = monitorInterval; + } + + public TimeUnit getMonitorIntervalUnit() { + return monitorIntervalUnit; + } + + public void setMonitorIntervalUnit(TimeUnit monitorIntervalUnit) { + this.monitorIntervalUnit = monitorIntervalUnit; + } + + public long getLockWaitTime() { + return lockWaitTime; + } + + public long getLockWaitTimeMillis() { + return lockWaitTimeUnit.toMillis(lockWaitTime); + } + + public void setLockWaitTime(long lockWaitTime) { + this.lockWaitTime = lockWaitTime; + } + + public TimeUnit getLockWaitTimeUnit() { + return lockWaitTimeUnit; + } + + public void setLockWaitTimeUnit(TimeUnit lockWaitTimeUnit) { + this.lockWaitTimeUnit = lockWaitTimeUnit; + } + + public long getLockLeaseTime() { + return lockLeaseTime; + } + + public long getLockLeaseTimeMillis() { + return lockLeaseTimeUnit.toMillis(lockLeaseTime); + } + + public void setLockLeaseTime(long lockLeaseTime) { + this.lockLeaseTime = lockLeaseTime; + } + + public TimeUnit getLockLeaseTimeUnit() { + return lockLeaseTimeUnit; + } + + public void setLockLeaseTimeUnit(TimeUnit lockLeaseTimeUnit) { + this.lockLeaseTimeUnit = lockLeaseTimeUnit; + } + + public long getInactiveQueryTimeToLive() { + return inactiveQueryTimeToLive; + } + + public long getInactiveQueryTimeToLiveMillis() { + return inactiveQueryTimeToLiveUnit.toMillis(inactiveQueryTimeToLive); + } + + public void setInactiveQueryTimeToLive(long inactiveQueryTimeToLive) { + this.inactiveQueryTimeToLive = inactiveQueryTimeToLive; + } + + public TimeUnit getInactiveQueryTimeToLiveUnit() { + return inactiveQueryTimeToLiveUnit; + } + + public void setInactiveQueryTimeToLiveUnit(TimeUnit inactiveQueryTimeToLiveUnit) { + this.inactiveQueryTimeToLiveUnit = inactiveQueryTimeToLiveUnit; + } +} diff --git a/service/src/main/java/datawave/microservice/query/runner/NextCall.java b/service/src/main/java/datawave/microservice/query/runner/NextCall.java new file mode 100644 index 00000000..1e3b2899 --- /dev/null +++ b/service/src/main/java/datawave/microservice/query/runner/NextCall.java @@ -0,0 +1,496 @@ +package datawave.microservice.query.runner; + +import static datawave.microservice.query.messaging.AcknowledgementCallback.Status.ACK; + +import java.util.LinkedList; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.Callable; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import datawave.core.query.cache.ResultsPage; +import datawave.core.query.logic.QueryLogic; +import datawave.core.query.logic.ResultPostprocessor; +import datawave.microservice.query.config.NextCallProperties; +import datawave.microservice.query.config.QueryExpirationProperties; +import datawave.microservice.query.config.QueryProperties; +import datawave.microservice.query.messaging.QueryResultsListener; +import datawave.microservice.query.messaging.QueryResultsManager; +import datawave.microservice.query.messaging.QueryResultsPublisher; +import datawave.microservice.query.messaging.Result; +import datawave.microservice.query.storage.QueryStatus; +import datawave.microservice.query.storage.QueryStorageCache; +import datawave.microservice.query.storage.TaskStates; +import datawave.microservice.query.util.QueryStatusUpdateUtil; +import datawave.microservice.querymetric.BaseQueryMetric; +import datawave.microservice.querymetric.QueryMetric; +import datawave.webservice.query.data.ObjectSizeOf; +import datawave.webservice.query.exception.QueryException; + +public class NextCall implements Callable> { + private final Logger log = LoggerFactory.getLogger(this.getClass()); + + private final NextCallProperties nextCallProperties; + private final QueryResultsManager queryResultsManager; + private final QueryStorageCache queryStorageCache; + private final String queryId; + private final QueryStatusUpdateUtil queryStatusUpdateUtil; + + private volatile boolean canceled = false; + private volatile Future> future = null; + + private final long callTimeoutMillis; + private final long shortCircuitCheckTimeMillis; + private final long shortCircuitTimeoutMillis; + private final long queryStartTimeMillis; + private final long longRunningQueryTimeoutMillis; + private final boolean allowLongRunningQueryEmptyPages; + + private final int userResultsPerPage; + private final boolean maxResultsOverridden; + private final long maxResultsOverride; + private final long maxResults; + private final int logicResultsPerPage; + private final long logicBytesPerPage; + private final long logicMaxWork; + private final long maxResultsPerPage; + private final ResultPostprocessor resultPostprocessor; + + private final List results = new LinkedList<>(); + private long pageSizeBytes; + private long startTimeMillis; + private long stopTimeMillis; + private ResultsPage.Status status = ResultsPage.Status.COMPLETE; + + private long lastQueryStatusUpdateTime = 0L; + private QueryStatus queryStatus; + private long lastTaskStatesUpdateTime = 0L; + private TaskStates taskStates; + private long numResultsConsumed = 0L; + private boolean returnIntermediateResult = false; + + private long hitMaxResultsTimeMillis = 0L; + + private BaseQueryMetric.Lifecycle lifecycle; + + private NextCall(Builder builder) { + this.nextCallProperties = builder.nextCallProperties; + this.queryResultsManager = builder.queryResultsManager; + this.queryStorageCache = builder.queryStorageCache; + this.queryId = builder.queryId; + this.queryStatusUpdateUtil = builder.queryStatusUpdateUtil; + + QueryStatus status = getQueryStatus(); + long pageTimeoutMillis = TimeUnit.MINUTES.toMillis(status.getQuery().getPageTimeout()); + if (pageTimeoutMillis >= builder.expirationProperties.getPageMinTimeoutMillis() + && pageTimeoutMillis <= builder.expirationProperties.getPageMaxTimeoutMillis()) { + callTimeoutMillis = pageTimeoutMillis; + shortCircuitCheckTimeMillis = callTimeoutMillis / 2; + shortCircuitTimeoutMillis = Math.round(0.97 * callTimeoutMillis); + } else { + callTimeoutMillis = builder.expirationProperties.getCallTimeoutMillis(); + shortCircuitCheckTimeMillis = builder.expirationProperties.getShortCircuitCheckTimeMillis(); + shortCircuitTimeoutMillis = builder.expirationProperties.getShortCircuitTimeoutMillis(); + } + + this.userResultsPerPage = status.getQuery().getPagesize(); + this.maxResultsOverridden = status.getQuery().isMaxResultsOverridden(); + this.maxResultsOverride = status.getQuery().getMaxResultsOverride(); + this.allowLongRunningQueryEmptyPages = status.isAllowLongRunningQueryEmptyPages(); + this.queryStartTimeMillis = status.getQueryStartMillis(); + this.longRunningQueryTimeoutMillis = builder.expirationProperties.getLongRunningQueryTimeoutMillis(); + + this.logicResultsPerPage = builder.queryLogic.getMaxPageSize(); + this.logicBytesPerPage = builder.queryLogic.getPageByteTrigger(); + this.logicMaxWork = builder.queryLogic.getMaxWork(); + + this.maxResultsPerPage = Math.min(userResultsPerPage, logicResultsPerPage); + + this.maxResults = builder.queryLogic.getResultLimit(status.getQuery()); + if (this.maxResults != builder.queryLogic.getMaxResults()) { + log.info("Maximum results set to " + this.maxResults + " instead of default " + builder.queryLogic.getMaxResults() + ", user " + + status.getQuery().getUserDN() + " has a DN configured with a different limit"); + } + + this.resultPostprocessor = builder.queryLogic.getResultPostprocessor(getQueryStatus().getConfig()); + } + + @Override + public ResultsPage call() throws Exception { + startTimeMillis = System.currentTimeMillis(); + + try (QueryResultsListener resultListener = queryResultsManager.createListener(UUID.randomUUID().toString(), queryId)) { + // keep waiting for results until we're finished + // Note: isFinished should be checked once per result + while (!isFinished(queryId)) { + Result result = resultListener.receive(nextCallProperties.getResultPollInterval(), nextCallProperties.getResultPollIntervalUnit()); + if (result != null) { + result.acknowledge(ACK); + + Object payload = result.getPayload(); + if (payload != null) { + results.add(payload); + + resultPostprocessor.apply(results); + + numResultsConsumed++; + + if (logicBytesPerPage > 0) { + pageSizeBytes += ObjectSizeOf.Sizer.getObjectSize(payload); + } + } else { + log.debug("Null result encountered, no more results"); + break; + } + } + } + } catch (Exception e) { + log.error("Encountered an error while fetching results from the listener", e); + throw e; + } + + // if we are aggregating results and we short-circuit, + // return the intermediate result(s) to the queue + if (returnIntermediateResult) { + try (QueryResultsPublisher publisher = queryResultsManager.createPublisher(queryId)) { + for (Object result : results) { + publisher.publish(new Result(UUID.randomUUID().toString(), result)); + } + } + results.clear(); + } + + // update some values for metrics + stopTimeMillis = System.currentTimeMillis(); + if (lifecycle == null && !results.isEmpty()) { + lifecycle = BaseQueryMetric.Lifecycle.RESULTS; + } + + // update num results consumed for query status + updateNumResultsConsumed(); + + return new ResultsPage<>(results, status); + } + + public void updateQueryMetric(BaseQueryMetric baseQueryMetric) { + baseQueryMetric.addPageTime(results.size(), stopTimeMillis - startTimeMillis, startTimeMillis, stopTimeMillis); + baseQueryMetric.setLifecycle(lifecycle); + } + + private boolean isFinished(String queryId) throws QueryException { + boolean finished = false; + long callTimeMillis = System.currentTimeMillis() - startTimeMillis; + QueryStatus queryStatus = getQueryStatus(); + + // if the query state is FAILED, throw an exception up to the query management service with the failure message + if (queryStatus.getQueryState() == QueryStatus.QUERY_STATE.FAIL) { + log.error("Query [{}]: query failed, aborting next call. Cause: {}", queryId, queryStatus.getFailureMessage()); + + throw new QueryException(queryStatus.getErrorCode(), queryStatus.getFailureMessage()); + } + + // 1) have we hit the user's results-per-page limit? + if (results.size() >= userResultsPerPage) { + log.info("Query [{}]: user requested max page size [{}] has been reached, aborting next call", queryId, userResultsPerPage); + + finished = true; + } + + // 2) have we hit the query logic's results-per-page limit? + if (!finished && logicResultsPerPage > 0 && results.size() >= logicResultsPerPage) { + log.info("Query [{}]: query logic max page size [{}] has been reached, aborting next call", queryId, logicResultsPerPage); + + finished = true; + } + + // 3) was this query canceled? + if (!finished && (canceled || queryStatus.getQueryState() == QueryStatus.QUERY_STATE.CANCEL)) { + log.info("Query [{}]: query cancelled, aborting next call", queryId); + + // no query metric lifecycle update - assumption is that the cancel call handled that + // set to partial for now - if there are no results, it will switch to NONE later + status = ResultsPage.Status.PARTIAL; + + finished = true; + } + + // 4) have we hit the query logic's bytes-per-page limit? + if (!finished && logicBytesPerPage > 0 && pageSizeBytes >= logicBytesPerPage) { + log.info("Query [{}]: query logic max page byte size has been reached, aborting next call", queryId); + + status = ResultsPage.Status.PARTIAL; + + finished = true; + } + + // 5) have we retrieved all of the results? + if (!finished && (queryStatus.getCreateStage() == QueryStatus.CREATE_STAGE.RESULTS) && !getTaskStates().hasUnfinishedTasks()) { + + // update the number of results consumed + queryStatus = updateNumResultsConsumed(); + + // how many results do the query services think are left + long queryResultsRemaining = queryStatus.getNumResultsGenerated() - queryStatus.getNumResultsConsumed(); + + // check to see if the number of results consumed is >= to the number of results generated + if (queryResultsRemaining < 0) { + log.warn("Query [{}]: The number of results consumed [{}] exceeds the number of results generated [{}]", queryId, + queryStatus.getNumResultsConsumed(), queryStatus.getNumResultsGenerated()); + } + + // how many results does the broker think are left + long brokerResultsRemaining = queryResultsManager.getNumResultsRemaining(queryId); + + log.info("All tasks appear to be completed for " + queryId + " with " + queryResultsRemaining + " yet to be retrieved and " + brokerResultsRemaining + + " left in broker"); + + // if the broker thinks there are not results left, we may be done + if (brokerResultsRemaining == 0) { + + // if the query service thinks there are no results left, we are done + // we can have negative results remaining if we consumed duplicate records + if (queryResultsRemaining <= 0) { + log.info("Query [{}]: all query tasks complete, and all results retrieved, aborting next call", queryId); + + status = ResultsPage.Status.PARTIAL; + + finished = true; + } + // if the query services think there are results left, we may need to wait + // this can happen if messages are in flux with the message broker due to nacking + else { + // if we aren't in a max results waiting period, start waiting + if (hitMaxResultsTimeMillis == 0) { + hitMaxResultsTimeMillis = System.currentTimeMillis(); + } + // if we are finished waiting, we are done + else if (System.currentTimeMillis() >= (hitMaxResultsTimeMillis + nextCallProperties.getMaxResultsTimeoutMillis())) { + log.info("Query [{}]: all query tasks complete, but timed out waiting for all results to be retrieved, aborting next call", queryId); + + status = ResultsPage.Status.PARTIAL; + + finished = true; + } + } + } + } + + // 6) have we hit the max results (or the max results override)? + if (!finished) { + long numResultsReturned = queryStatus.getNumResultsReturned(); + long numResults = numResultsReturned + results.size(); + if (this.maxResultsOverridden) { + if (maxResultsOverride >= 0 && numResults >= maxResultsOverride) { + log.info("Query [{}]: max results override has been reached, aborting next call", queryId); + + lifecycle = QueryMetric.Lifecycle.MAXRESULTS; + + status = ResultsPage.Status.PARTIAL; + + finished = true; + } + } else if (maxResults >= 0 && numResults >= maxResults) { + log.info("Query [{}]: logic max results has been reached, aborting next call", queryId); + + lifecycle = QueryMetric.Lifecycle.MAXRESULTS; + + status = ResultsPage.Status.PARTIAL; + + finished = true; + } + } + + // 7) have we reached the "max work" limit? (i.e. next count + seek count) + if (!finished && logicMaxWork > 0 && (queryStatus.getNextCount() + queryStatus.getSeekCount()) >= logicMaxWork) { + log.info("Query [{}]: logic max work has been reached, aborting next call", queryId); + + lifecycle = BaseQueryMetric.Lifecycle.MAXWORK; + + status = ResultsPage.Status.PARTIAL; + + finished = true; + } + + // 8) are we going to timeout before getting a full page? if so, return partial results + if (!finished && shortCircuitTimeout(callTimeMillis)) { + log.info("Query [{}]: logic max expire before page is full, returning existing results: {} of {} results in {}ms", queryId, results.size(), + maxResultsPerPage, callTimeMillis); + + status = ResultsPage.Status.PARTIAL; + + finished = true; + } + + // 9) have we been in this next call too long? + if (!finished && callExpiredTimeout(callTimeMillis)) { + log.info("Query [{}]: max call time reached, returning existing results: {} of {} results in {}ms", queryId, results.size(), maxResultsPerPage, + callTimeMillis); + + lifecycle = BaseQueryMetric.Lifecycle.NEXTTIMEOUT; + + status = ResultsPage.Status.PARTIAL; + + finished = true; + } + + return finished; + } + + private QueryStatus updateNumResultsConsumed() { + if (numResultsConsumed > 0) { + try { + queryStatus = queryStatusUpdateUtil.lockedUpdate(queryId, queryStatus1 -> { + queryStatus1.incrementNumResultsConsumed(numResultsConsumed); + numResultsConsumed = 0; + }); + lastQueryStatusUpdateTime = System.currentTimeMillis(); + } catch (Exception e) { + log.warn("Unable to update number of results consumed for query {}", queryId); + } + } + return queryStatus; + } + + private boolean shortCircuitTimeout(long callTimeMillis) { + boolean timeout = false; + + // return prematurely if we have at least 1 result, and we aren't aggregating results + if (!results.isEmpty() && !queryStatus.getConfig().isReduceResults()) { + // if after the page size short circuit check time + if (callTimeMillis >= shortCircuitCheckTimeMillis) { + float percentTimeComplete = (float) callTimeMillis / (float) (callTimeoutMillis); + float percentResultsComplete = (float) results.size() / (float) maxResultsPerPage; + // if the percent results complete is less than the percent time complete, then break out + if (percentResultsComplete < percentTimeComplete) { + timeout = true; + } + } + + // if after the page short circuit timeout, then break out + if (callTimeMillis >= shortCircuitTimeoutMillis) { + timeout = true; + } + } else if (allowLongRunningQueryEmptyPages) { + // in the case of allowing long-running query timeouts, we can return an empty page + // before the query times out. However we can only allow this query to run so long... + if (callTimeMillis >= shortCircuitTimeoutMillis) { + if ((System.currentTimeMillis() - queryStartTimeMillis) < longRunningQueryTimeoutMillis) { + timeout = true; + + // if we are aggregating results, return the intermediate + // result to the queue before returning a blank page + returnIntermediateResult = queryStatus.getConfig().isReduceResults(); + } + } + } + + return timeout; + } + + private boolean callExpiredTimeout(long callTimeMillis) { + return callTimeMillis >= callTimeoutMillis; + } + + private QueryStatus getQueryStatus() { + if (queryStatus == null || isQueryStatusExpired()) { + lastQueryStatusUpdateTime = System.currentTimeMillis(); + queryStatus = queryStorageCache.getQueryStatus(queryId); + } + return queryStatus; + } + + private TaskStates getTaskStates() { + if (taskStates == null || isTaskStatesExpired()) { + lastTaskStatesUpdateTime = System.currentTimeMillis(); + taskStates = queryStorageCache.getTaskStates(queryId); + } + return taskStates; + } + + private boolean isQueryStatusExpired() { + return (System.currentTimeMillis() - lastQueryStatusUpdateTime) > nextCallProperties.getStatusUpdateIntervalMillis(); + } + + private boolean isTaskStatesExpired() { + return (System.currentTimeMillis() - lastTaskStatesUpdateTime) > nextCallProperties.getStatusUpdateIntervalMillis(); + } + + public boolean isCanceled() { + return canceled; + } + + public void cancel() { + this.canceled = true; + } + + public Future> getFuture() { + return future; + } + + public void setFuture(Future> future) { + this.future = future; + } + + public BaseQueryMetric.Lifecycle getLifecycle() { + return lifecycle; + } + + public static class Builder { + private NextCallProperties nextCallProperties; + private QueryExpirationProperties expirationProperties; + private QueryResultsManager queryResultsManager; + private QueryStorageCache queryStorageCache; + private String queryId; + private QueryStatusUpdateUtil queryStatusUpdateUtil; + private QueryLogic queryLogic; + + public Builder setQueryProperties(QueryProperties queryProperties) { + this.nextCallProperties = queryProperties.getNextCall(); + this.expirationProperties = queryProperties.getExpiration(); + return this; + } + + public Builder setNextCallProperties(NextCallProperties nextCallProperties) { + this.nextCallProperties = nextCallProperties; + return this; + } + + public Builder setExpirationProperties(QueryExpirationProperties expirationProperties) { + this.expirationProperties = expirationProperties; + return this; + } + + public Builder setResultsQueueManager(QueryResultsManager queryResultsManager) { + this.queryResultsManager = queryResultsManager; + return this; + } + + public Builder setQueryStorageCache(QueryStorageCache queryStorageCache) { + this.queryStorageCache = queryStorageCache; + return this; + } + + public Builder setQueryId(String queryId) { + this.queryId = queryId; + return this; + } + + public Builder setQueryStatusUpdateUtil(QueryStatusUpdateUtil queryStatusUpdateUtil) { + this.queryStatusUpdateUtil = queryStatusUpdateUtil; + return this; + } + + public Builder setQueryLogic(QueryLogic queryLogic) { + this.queryLogic = queryLogic; + return this; + } + + public NextCall build() { + return new NextCall(this); + } + } +} diff --git a/service/src/main/java/datawave/microservice/query/stream/StreamingService.java b/service/src/main/java/datawave/microservice/query/stream/StreamingService.java new file mode 100644 index 00000000..3a8630b3 --- /dev/null +++ b/service/src/main/java/datawave/microservice/query/stream/StreamingService.java @@ -0,0 +1,124 @@ +package datawave.microservice.query.stream; + +import java.util.function.Supplier; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; +import org.springframework.stereotype.Service; +import org.springframework.util.MultiValueMap; + +import datawave.microservice.authorization.user.DatawaveUserDetails; +import datawave.microservice.query.QueryManagementService; +import datawave.microservice.query.stream.listener.StreamingResponseListener; +import datawave.microservice.query.stream.runner.StreamingCall; +import datawave.microservice.querymetric.QueryMetricClient; +import datawave.security.util.ProxiedEntityUtils; +import datawave.webservice.query.exception.BadRequestQueryException; +import datawave.webservice.query.exception.NoResultsQueryException; +import datawave.webservice.query.exception.QueryException; +import datawave.webservice.query.exception.UnauthorizedQueryException; + +@Service +public class StreamingService { + private final Logger log = LoggerFactory.getLogger(this.getClass()); + + private final QueryManagementService queryManagementService; + private final QueryMetricClient queryMetricClient; + + private final ThreadPoolTaskExecutor streamingCallExecutor; + + public StreamingService(QueryManagementService queryManagementService, QueryMetricClient queryMetricClient, ThreadPoolTaskExecutor streamingCallExecutor) { + this.queryManagementService = queryManagementService; + this.queryMetricClient = queryMetricClient; + this.streamingCallExecutor = streamingCallExecutor; + } + + /** + * Creates a query using the given query logic and parameters, and streams all pages of results to the configured listener. + *

+ * Created queries will start running immediately.
+ * Auditing is performed before the query is started.
+ * Stop a running query gracefully using {@link QueryManagementService#close} or forcefully using {@link QueryManagementService#cancel}.
+ * Stop, and restart a running query using {@link QueryManagementService#reset}.
+ * Create a copy of a running query using {@link QueryManagementService#duplicate}.
+ * Aside from a limited set of admin actions, only the query owner can act on a running query. + * + * @param queryLogicName + * the requested query logic, not null + * @param parameters + * the query parameters, not null + * @param pool + * the pool to target, may be null + * @param currentUser + * the user who called this method, not null + * @param listener + * the listener which will handle the result pages, not null + * @return the query id + * @throws BadRequestQueryException + * if parameter validation fails + * @throws BadRequestQueryException + * if query logic parameter validation fails + * @throws UnauthorizedQueryException + * if the user doesn't have access to the requested query logic + * @throws BadRequestQueryException + * if security marking validation fails + * @throws BadRequestQueryException + * if auditing fails + * @throws QueryException + * if query storage fails + * @throws NoResultsQueryException + * if no query results are found + * @throws QueryException + * if there is an unknown error + */ + public String createAndExecute(String queryLogicName, MultiValueMap parameters, String pool, DatawaveUserDetails currentUser, + DatawaveUserDetails serverUser, StreamingResponseListener listener) throws QueryException { + String user = ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()); + if (log.isDebugEnabled()) { + log.info("Request: {}/createAndExecute from {} with params: {}", queryLogicName, user, parameters); + } else { + log.info("Request: {}/createAndExecute from {}", queryLogicName, user); + } + + String queryId = queryManagementService.create(queryLogicName, parameters, pool, currentUser).getResult(); + submitStreamingCall(queryId, currentUser, serverUser, listener); + return queryId; + } + + /** + * Gets all pages of results for the given query and streams them to the configured listener. + *

+ * Execute can only be called on a running query.
+ * Execute is a non-blocking call, and will return immediately.
+ * Only the query owner can call execute on the specified query. + * + * @param queryId + * the query id, not null + * @param currentUser + * the user who called this method, not null + * @param serverUser + * the server user, not null + * @param listener + * the listener which will handle the result pages, not null + */ + public void execute(String queryId, DatawaveUserDetails currentUser, DatawaveUserDetails serverUser, StreamingResponseListener listener) { + log.info("Request: {}/execute from {}", queryId, ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName())); + + submitStreamingCall(queryId, currentUser, serverUser, listener); + } + + private void submitStreamingCall(String queryId, DatawaveUserDetails currentUser, DatawaveUserDetails serverUser, StreamingResponseListener listener) { + // @formatter:off + streamingCallExecutor.submit( + new StreamingCall.Builder() + .setQueryManagementService(queryManagementService) + .setQueryMetricClient(queryMetricClient) + .setQueryId(queryId) + .setCurrentUser(currentUser) + .setServerUser(serverUser) + .setListener(listener) + .build()); + // @formatter:on + } +} diff --git a/service/src/main/java/datawave/microservice/query/stream/listener/CountingResponseBodyEmitterListener.java b/service/src/main/java/datawave/microservice/query/stream/listener/CountingResponseBodyEmitterListener.java new file mode 100644 index 00000000..5ac002a5 --- /dev/null +++ b/service/src/main/java/datawave/microservice/query/stream/listener/CountingResponseBodyEmitterListener.java @@ -0,0 +1,45 @@ +package datawave.microservice.query.stream.listener; + +import java.io.IOException; + +import org.springframework.http.MediaType; + +import datawave.microservice.query.web.filter.CountingResponseBodyEmitter; +import datawave.webservice.result.BaseQueryResponse; + +public class CountingResponseBodyEmitterListener implements StreamingResponseListener { + private final CountingResponseBodyEmitter countingEmitter; + private final MediaType mediaType; + + public CountingResponseBodyEmitterListener(CountingResponseBodyEmitter countingEmitter, MediaType mediaType) { + this.countingEmitter = countingEmitter; + this.mediaType = mediaType; + } + + @Override + public void onResponse(BaseQueryResponse response) throws IOException { + countingEmitter.send(response, mediaType); + } + + @Override + public void close() { + countingEmitter.complete(); + } + + @Override + public void closeWithError(Throwable t) { + countingEmitter.completeWithError(t); + } + + public CountingResponseBodyEmitter getCountingEmitter() { + return countingEmitter; + } + + public MediaType getMediaType() { + return mediaType; + } + + public long getBytesWritten() { + return (countingEmitter != null) ? countingEmitter.getBytesWritten() : 0L; + } +} diff --git a/service/src/main/java/datawave/microservice/query/stream/listener/StreamingResponseListener.java b/service/src/main/java/datawave/microservice/query/stream/listener/StreamingResponseListener.java new file mode 100644 index 00000000..9c989710 --- /dev/null +++ b/service/src/main/java/datawave/microservice/query/stream/listener/StreamingResponseListener.java @@ -0,0 +1,18 @@ +package datawave.microservice.query.stream.listener; + +import java.io.IOException; + +import datawave.webservice.result.BaseQueryResponse; + +public interface StreamingResponseListener { + + void onResponse(BaseQueryResponse response) throws IOException; + + default void close() { + // do nothing + } + + default void closeWithError(Throwable t) { + // do nothing + } +} diff --git a/service/src/main/java/datawave/microservice/query/stream/runner/StreamingCall.java b/service/src/main/java/datawave/microservice/query/stream/runner/StreamingCall.java new file mode 100644 index 00000000..18c4c0d1 --- /dev/null +++ b/service/src/main/java/datawave/microservice/query/stream/runner/StreamingCall.java @@ -0,0 +1,189 @@ +package datawave.microservice.query.stream.runner; + +import java.io.IOException; +import java.util.List; +import java.util.concurrent.Callable; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import datawave.microservice.authorization.user.DatawaveUserDetails; +import datawave.microservice.query.QueryManagementService; +import datawave.microservice.query.stream.listener.CountingResponseBodyEmitterListener; +import datawave.microservice.query.stream.listener.StreamingResponseListener; +import datawave.microservice.querymetric.BaseQueryMetric; +import datawave.microservice.querymetric.QueryMetricClient; +import datawave.microservice.querymetric.QueryMetricType; +import datawave.webservice.query.exception.DatawaveErrorCode; +import datawave.webservice.query.exception.NoResultsQueryException; +import datawave.webservice.query.exception.QueryException; +import datawave.webservice.result.BaseQueryResponse; + +public class StreamingCall implements Callable { + private final Logger log = LoggerFactory.getLogger(this.getClass()); + + final private QueryManagementService queryManagementService; + final private QueryMetricClient queryMetricClient; + final private BaseQueryMetric baseQueryMetric; + + final private DatawaveUserDetails currentUser; + final private DatawaveUserDetails serverUser; + final private String queryId; + + final private StreamingResponseListener listener; + + private StreamingCall(Builder builder) { + this.queryManagementService = builder.queryManagementService; + this.queryMetricClient = builder.queryMetricClient; + this.baseQueryMetric = builder.queryManagementService.getBaseQueryMetric().duplicate(); + + this.currentUser = builder.currentUser; + this.serverUser = builder.serverUser; + this.queryId = builder.queryId; + + this.listener = builder.listener; + } + + @Override + public Void call() throws Exception { + // since this is running in a separate thread, we need to set and use the thread-local baseQueryMetric + ThreadLocal baseQueryMetricOverride = queryManagementService.getBaseQueryMetricOverride(); + baseQueryMetricOverride.set(baseQueryMetric); + + try { + boolean isFinished = false; + do { + final BaseQueryResponse nextResponse = next(queryId, currentUser); + if (nextResponse != null) { + onResponse(nextResponse); + updateMetrics(); + } else { + isFinished = true; + } + } while (!isFinished); + + listener.close(); + return null; + } catch (Exception e) { + log.error("Error encountered while processing streaming results for query {}", queryId, e); + listener.closeWithError(e); + throw e; + } finally { + baseQueryMetricOverride.remove(); + } + } + + private BaseQueryResponse next(String queryId, DatawaveUserDetails currentUser) { + BaseQueryResponse nextResponse = null; + try { + long startTimeMillis = System.currentTimeMillis(); + nextResponse = queryManagementService.next(queryId, currentUser); + long nextCallTimeMillis = System.currentTimeMillis() - startTimeMillis; + + BaseQueryMetric.PageMetric lastPageMetric = getLastPageMetric(); + if (lastPageMetric != null) { + lastPageMetric.setCallTime(nextCallTimeMillis); + } + } catch (NoResultsQueryException e) { + log.debug("No results found for query '{}'", queryId); + } catch (QueryException e) { + log.info("Encountered error while getting results for query '{}'", queryId); + } + return nextResponse; + } + + private void onResponse(BaseQueryResponse nextResponse) throws QueryException { + try { + long startBytesWritten = getBytesWritten(); + long startTimeMillis = System.currentTimeMillis(); + listener.onResponse(nextResponse); + long serializationTimeMillis = System.currentTimeMillis() - startTimeMillis; + + BaseQueryMetric.PageMetric lastPageMetric = getLastPageMetric(); + if (lastPageMetric != null) { + lastPageMetric.setSerializationTime(serializationTimeMillis); + lastPageMetric.setBytesWritten(getBytesWritten() - startBytesWritten); + } + } catch (IOException e) { + throw new QueryException(DatawaveErrorCode.UNKNOWN_SERVER_ERROR, e, "Unknown error sending next page for query " + queryId); + } + } + + private long getBytesWritten() { + long bytesWritten = 0L; + if (listener instanceof CountingResponseBodyEmitterListener) { + bytesWritten = ((CountingResponseBodyEmitterListener) listener).getBytesWritten(); + } + return bytesWritten; + } + + private void updateMetrics() { + // send out the metrics + try { + // @formatter:off + queryMetricClient.submit( + new QueryMetricClient.Request.Builder() + .withUser(serverUser) + .withMetric(baseQueryMetric.duplicate()) + .withMetricType(QueryMetricType.DISTRIBUTED) + .build()); + // @formatter:on + } catch (Exception e) { + log.error("Error updating query metric", e); + } + } + + private BaseQueryMetric.PageMetric getLastPageMetric() { + BaseQueryMetric.PageMetric pageMetric = null; + List pageTimes = baseQueryMetric.getPageTimes(); + if (!pageTimes.isEmpty()) { + pageMetric = pageTimes.get(pageTimes.size() - 1); + } + return pageMetric; + } + + public static class Builder { + private QueryManagementService queryManagementService; + private QueryMetricClient queryMetricClient; + + private DatawaveUserDetails currentUser; + private DatawaveUserDetails serverUser; + private String queryId; + + private StreamingResponseListener listener; + + public Builder setQueryManagementService(QueryManagementService queryManagementService) { + this.queryManagementService = queryManagementService; + return this; + } + + public Builder setQueryMetricClient(QueryMetricClient queryMetricClient) { + this.queryMetricClient = queryMetricClient; + return this; + } + + public Builder setCurrentUser(DatawaveUserDetails currentUser) { + this.currentUser = currentUser; + return this; + } + + public Builder setServerUser(DatawaveUserDetails serverUser) { + this.serverUser = serverUser; + return this; + } + + public Builder setQueryId(String queryId) { + this.queryId = queryId; + return this; + } + + public Builder setListener(StreamingResponseListener listener) { + this.listener = listener; + return this; + } + + public StreamingCall build() { + return new StreamingCall(this); + } + } +} diff --git a/service/src/main/java/datawave/microservice/query/translateid/TranslateIdService.java b/service/src/main/java/datawave/microservice/query/translateid/TranslateIdService.java new file mode 100644 index 00000000..1fb51746 --- /dev/null +++ b/service/src/main/java/datawave/microservice/query/translateid/TranslateIdService.java @@ -0,0 +1,218 @@ +package datawave.microservice.query.translateid; + +import static datawave.microservice.query.QueryParameters.QUERY_AUTHORIZATIONS; +import static datawave.microservice.query.QueryParameters.QUERY_BEGIN; +import static datawave.microservice.query.QueryParameters.QUERY_END; +import static datawave.microservice.query.QueryParameters.QUERY_LOGIC_NAME; +import static datawave.microservice.query.QueryParameters.QUERY_NAME; +import static datawave.microservice.query.QueryParameters.QUERY_STRING; +import static datawave.query.QueryParameters.QUERY_SYNTAX; +import static datawave.webservice.query.exception.DatawaveErrorCode.MISSING_REQUIRED_PARAMETER; + +import java.text.ParseException; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.UUID; +import java.util.stream.Collectors; + +import org.apache.commons.lang.time.DateUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; +import org.springframework.util.LinkedMultiValueMap; +import org.springframework.util.MultiValueMap; + +import datawave.microservice.authorization.user.DatawaveUserDetails; +import datawave.microservice.authorization.util.AuthorizationsUtil; +import datawave.microservice.query.DefaultQueryParameters; +import datawave.microservice.query.QueryManagementService; +import datawave.microservice.query.QueryParameters; +import datawave.security.util.ProxiedEntityUtils; +import datawave.webservice.query.exception.BadRequestQueryException; +import datawave.webservice.query.exception.DatawaveErrorCode; +import datawave.webservice.query.exception.NoResultsQueryException; +import datawave.webservice.query.exception.QueryException; +import datawave.webservice.query.exception.TimeoutQueryException; +import datawave.webservice.query.exception.UnauthorizedQueryException; +import datawave.webservice.result.BaseQueryResponse; + +@Service +public class TranslateIdService { + private final Logger log = LoggerFactory.getLogger(this.getClass()); + + public static final String TRANSLATE_ID = "id"; + public static final String TRANSLATE_TLD_ONLY = "TLDonly"; + public static final String LUCENE_SYNTAX = "LUCENE"; + + private final TranslateIdProperties translateIdProperties; + + private final QueryManagementService queryManagementService; + + public TranslateIdService(TranslateIdProperties translateIdProperties, QueryManagementService queryManagementService) { + this.translateIdProperties = translateIdProperties; + this.queryManagementService = queryManagementService; + } + + /** + * Get one or more ID(s), if any, that correspond to the given ID. This method only returns the first page, so set pagesize appropriately. Since the + * underlying query is automatically closed, callers are NOT expected to request additional pages or close the query. + * + * @param id + * the id to translate + * @param parameters + * the query parameters, not null + * @param pool + * the pool to target, may be null + * @param currentUser + * the user who called this method, not null + * @return a base query response containing the first page of results + * @throws BadRequestQueryException + * if parameter validation fails + * @throws BadRequestQueryException + * if query logic parameter validation fails + * @throws UnauthorizedQueryException + * if the user doesn't have access to the requested query logic + * @throws BadRequestQueryException + * if security marking validation fails + * @throws BadRequestQueryException + * if auditing fails + * @throws QueryException + * if query storage fails + * @throws TimeoutQueryException + * if the next call times out + * @throws NoResultsQueryException + * if no query results are found + * @throws QueryException + * if this next task is rejected by the executor + * @throws QueryException + * if there is an unknown error + */ + public BaseQueryResponse translateId(String id, MultiValueMap parameters, String pool, DatawaveUserDetails currentUser) + throws QueryException { + String queryId = null; + try { + parameters.set(TRANSLATE_ID, id); + + BaseQueryResponse response = translateIds(parameters, pool, currentUser); + queryId = response.getQueryId(); + return response; + } catch (QueryException e) { + throw e; + } catch (Exception e) { + log.error("Unknown error with translateId", e); + throw new QueryException(DatawaveErrorCode.QUERY_SETUP_ERROR, e, "Unknown error with translateId."); + } finally { + // close the query if applicable + if (queryId != null) { + queryManagementService.close(queryId, currentUser); + } + } + } + + /** + * Get the ID(s), if any, associated with the specified IDs. Because the query created by this call may return multiple pages, callers are expected to + * request additional pages and eventually close the query. + * + * @param parameters + * the query parameters, not null + * @param pool + * the pool to target, may be null + * @param currentUser + * the user who called this method, not null + * @return a base query response containing the first page of results + * @throws BadRequestQueryException + * if parameter validation fails + * @throws BadRequestQueryException + * if query logic parameter validation fails + * @throws UnauthorizedQueryException + * if the user doesn't have access to the requested query logic + * @throws BadRequestQueryException + * if security marking validation fails + * @throws BadRequestQueryException + * if auditing fails + * @throws QueryException + * if query storage fails + * @throws TimeoutQueryException + * if the next call times out + * @throws NoResultsQueryException + * if no query results are found + * @throws QueryException + * if this next task is rejected by the executor + * @throws QueryException + * if there is an unknown error + */ + public BaseQueryResponse translateIds(MultiValueMap parameters, String pool, DatawaveUserDetails currentUser) throws QueryException { + if (!parameters.containsKey(TRANSLATE_ID)) { + throw new BadRequestQueryException(MISSING_REQUIRED_PARAMETER, "Missing required parameter: " + TRANSLATE_ID); + } + + try { + MultiValueMap queryParams = setupQueryParameters(parameters, currentUser); + return queryManagementService.createAndNext(parameters.getFirst(QUERY_LOGIC_NAME), queryParams, pool, currentUser); + } catch (QueryException e) { + throw e; + } catch (Exception e) { + log.error("Unknown error with translateIds", e); + throw new QueryException(DatawaveErrorCode.QUERY_SETUP_ERROR, e, "Unknown error with translateIds."); + } + } + + protected MultiValueMap setupQueryParameters(MultiValueMap parameters, DatawaveUserDetails currentUser) { + MultiValueMap queryParams = new LinkedMultiValueMap<>(); + + // copy over any query parameters which are explicitly allowed to be set, ignoring ones that aren't + for (String queryParam : translateIdProperties.getAllowedQueryParameters()) { + if (parameters.containsKey(queryParam)) { + queryParams.put(queryParam, parameters.get(queryParam)); + } + } + + String user = ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()); + String queryName = user + "-" + UUID.randomUUID().toString(); + + String queryLogic; + if (Boolean.parseBoolean(parameters.getFirst(TRANSLATE_TLD_ONLY))) { + queryLogic = translateIdProperties.getTldQueryLogicName(); + } else { + queryLogic = translateIdProperties.getQueryLogicName(); + } + + String endDate; + try { + endDate = DefaultQueryParameters.formatDate(DateUtils.addDays(new Date(), 2)); + } catch (ParseException e) { + throw new RuntimeException("Unable to format new query end date"); + } + + setOptionalQueryParameters(queryParams); + + queryParams.set(QUERY_SYNTAX, LUCENE_SYNTAX); + queryParams.add(QUERY_NAME, queryName); + queryParams.add(QUERY_LOGIC_NAME, queryLogic); + queryParams.add(QUERY_STRING, buildQuery(parameters.get(TRANSLATE_ID))); + queryParams.set(QUERY_AUTHORIZATIONS, AuthorizationsUtil.buildUserAuthorizationString(currentUser)); + queryParams.add(QUERY_BEGIN, translateIdProperties.getBeginDate()); + queryParams.set(QUERY_END, endDate); + + return queryParams; + } + + protected void setOptionalQueryParameters(MultiValueMap parameters) { + if (translateIdProperties.getColumnVisibility() != null) { + parameters.set(QueryParameters.QUERY_VISIBILITY, translateIdProperties.getColumnVisibility()); + } + } + + private String buildQuery(List ids) { + List uuidTypes = new ArrayList<>(); + translateIdProperties.getTypes().keySet().forEach(uuidType -> uuidTypes.add(uuidType.toUpperCase())); + + // @formatter:off + return ids.stream() + .map(id -> "\"" + id + "\"") + .flatMap(id -> uuidTypes.stream().map(uuidType -> uuidType + ":" + id)) + .collect(Collectors.joining(" OR ")); + // @formatter:on + } +} diff --git a/service/src/main/java/datawave/microservice/query/web/BaseQueryResponseAdvice.java b/service/src/main/java/datawave/microservice/query/web/BaseQueryResponseAdvice.java new file mode 100644 index 00000000..5a25f0f2 --- /dev/null +++ b/service/src/main/java/datawave/microservice/query/web/BaseQueryResponseAdvice.java @@ -0,0 +1,36 @@ +package datawave.microservice.query.web; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnClass; +import org.springframework.core.MethodParameter; +import org.springframework.http.MediaType; +import org.springframework.http.server.ServerHttpRequest; +import org.springframework.http.server.ServerHttpResponse; +import org.springframework.lang.NonNull; +import org.springframework.web.bind.annotation.ControllerAdvice; +import org.springframework.web.servlet.mvc.method.annotation.ResponseBodyAdvice; + +import datawave.Constants; +import datawave.webservice.result.BaseQueryResponse; + +/** + * A {@link ControllerAdvice} that implements {@link ResponseBodyAdvice} in order to allow access to {@link BaseQueryResponse} objects before they are written + * out to the response body. This is primarily used to write the page number, is last page, and partial results headers for the response. + */ +@ControllerAdvice +@ConditionalOnClass(BaseQueryResponse.class) +public class BaseQueryResponseAdvice implements ResponseBodyAdvice { + + @Override + public boolean supports(@NonNull MethodParameter returnType, @NonNull Class converterType) { + return BaseQueryResponse.class.isAssignableFrom(returnType.getParameterType()); + } + + @Override + public BaseQueryResponse beforeBodyWrite(BaseQueryResponse baseQueryResponse, @NonNull MethodParameter returnType, @NonNull MediaType selectedContentType, + @NonNull Class selectedConverterType, @NonNull ServerHttpRequest request, ServerHttpResponse response) { + response.getHeaders().add(Constants.PAGE_NUMBER, String.valueOf(baseQueryResponse.getPageNumber())); + response.getHeaders().add(Constants.IS_LAST_PAGE, String.valueOf(!baseQueryResponse.getHasResults())); + response.getHeaders().add(Constants.PARTIAL_RESULTS, String.valueOf(baseQueryResponse.isPartialResults())); + return baseQueryResponse; + } +} diff --git a/service/src/main/java/datawave/microservice/query/web/QuerySessionIdAdvice.java b/service/src/main/java/datawave/microservice/query/web/QuerySessionIdAdvice.java new file mode 100644 index 00000000..3bec74f6 --- /dev/null +++ b/service/src/main/java/datawave/microservice/query/web/QuerySessionIdAdvice.java @@ -0,0 +1,97 @@ +package datawave.microservice.query.web; + +import java.util.Arrays; +import java.util.UUID; + +import javax.servlet.http.Cookie; + +import org.apache.commons.lang.StringUtils; +import org.apache.log4j.Logger; +import org.springframework.core.MethodParameter; +import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; +import org.springframework.http.converter.HttpMessageConverter; +import org.springframework.http.server.ServerHttpRequest; +import org.springframework.http.server.ServerHttpResponse; +import org.springframework.http.server.ServletServerHttpResponse; +import org.springframework.lang.NonNull; +import org.springframework.web.bind.annotation.ControllerAdvice; +import org.springframework.web.servlet.mvc.method.annotation.ResponseBodyAdvice; + +import datawave.Constants; +import datawave.microservice.query.web.annotation.GenerateQuerySessionId; + +@ControllerAdvice +public class QuerySessionIdAdvice implements ResponseBodyAdvice { + private final Logger log = Logger.getLogger(QuerySessionIdAdvice.class); + + @Override + public boolean supports(@NonNull MethodParameter returnType, @NonNull Class> converterType) { + return Arrays.stream(returnType.getMethodAnnotations()).anyMatch(GenerateQuerySessionId.class::isInstance); + } + + @Override + public Object beforeBodyWrite(Object body, @NonNull MethodParameter returnType, @NonNull MediaType selectedContentType, + @NonNull Class> selectedConverterType, @NonNull ServerHttpRequest request, + @NonNull ServerHttpResponse response) { + try { + GenerateQuerySessionId annotation = (GenerateQuerySessionId) Arrays.stream(returnType.getMethodAnnotations()) + .filter(GenerateQuerySessionId.class::isInstance).findAny().orElse(null); + + if (annotation != null) { + ServletServerHttpResponse httpServletResponse = (ServletServerHttpResponse) response; + String path = annotation.cookieBasePath(); + + String id = ""; + boolean setCookie = true; + switch (HttpStatus.Series.valueOf(httpServletResponse.getServletResponse().getStatus())) { + case SERVER_ERROR: + case CLIENT_ERROR: + // If we're sending an error response, then there's no need to set a cookie since + // there's no query "session" to stick to this server. + setCookie = false; + break; + + default: + if (StringUtils.isEmpty(QuerySessionIdContext.getQueryId())) { + log.error("queryId was not set."); + } else { + id = QuerySessionIdContext.getQueryId(); + } + break; + } + + if (setCookie) { + Cookie cookie = new Cookie(Constants.QUERY_COOKIE_NAME, generateCookieValue()); + cookie.setPath(path + id); + httpServletResponse.getServletResponse().addCookie(cookie); + } + } + + return body; + } finally { + QuerySessionIdContext.remove(); + } + } + + private static String generateCookieValue() { + return Integer.toString(UUID.randomUUID().hashCode() & Integer.MAX_VALUE); + } + + public static class QuerySessionIdContext { + + private static final ThreadLocal queryId = new ThreadLocal<>(); + + public static String getQueryId() { + return (String) queryId.get(); + } + + public static void setQueryId(String queryId) { + QuerySessionIdContext.queryId.set(queryId); + } + + private static void remove() { + queryId.remove(); + } + } +} diff --git a/service/src/main/java/datawave/microservice/query/web/annotation/EnrichQueryMetrics.java b/service/src/main/java/datawave/microservice/query/web/annotation/EnrichQueryMetrics.java new file mode 100644 index 00000000..e188854d --- /dev/null +++ b/service/src/main/java/datawave/microservice/query/web/annotation/EnrichQueryMetrics.java @@ -0,0 +1,18 @@ +package datawave.microservice.query.web.annotation; + +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Target({ElementType.TYPE, ElementType.METHOD}) +@Retention(RetentionPolicy.RUNTIME) +@Documented +public @interface EnrichQueryMetrics { + enum MethodType { + NONE, CREATE, NEXT, CREATE_AND_NEXT + }; + + MethodType methodType(); +} diff --git a/service/src/main/java/datawave/microservice/query/web/annotation/GenerateQuerySessionId.java b/service/src/main/java/datawave/microservice/query/web/annotation/GenerateQuerySessionId.java new file mode 100644 index 00000000..4da1849b --- /dev/null +++ b/service/src/main/java/datawave/microservice/query/web/annotation/GenerateQuerySessionId.java @@ -0,0 +1,15 @@ +package datawave.microservice.query.web.annotation; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Target({ElementType.TYPE, ElementType.METHOD, ElementType.PARAMETER}) +@Retention(RetentionPolicy.RUNTIME) +public @interface GenerateQuerySessionId { + /** + * @return The base path for the generated cookie. The base path will be combined with the query id in order to form the cookie domain. + */ + String cookieBasePath(); +} diff --git a/service/src/main/java/datawave/microservice/query/web/filter/BaseMethodStatsFilter.java b/service/src/main/java/datawave/microservice/query/web/filter/BaseMethodStatsFilter.java new file mode 100644 index 00000000..428604e0 --- /dev/null +++ b/service/src/main/java/datawave/microservice/query/web/filter/BaseMethodStatsFilter.java @@ -0,0 +1,323 @@ +package datawave.microservice.query.web.filter; + +import static datawave.microservice.config.web.Constants.REQUEST_LOGIN_TIME_ATTRIBUTE; +import static datawave.microservice.config.web.Constants.REQUEST_START_TIME_NS_ATTRIBUTE; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Enumeration; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import javax.servlet.FilterChain; +import javax.servlet.ServletException; +import javax.servlet.ServletOutputStream; +import javax.servlet.ServletResponse; +import javax.servlet.WriteListener; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import javax.servlet.http.HttpServletResponseWrapper; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.http.MediaType; +import org.springframework.lang.NonNull; +import org.springframework.util.LinkedMultiValueMap; +import org.springframework.util.MultiValueMap; +import org.springframework.web.context.annotation.RequestScope; +import org.springframework.web.filter.OncePerRequestFilter; + +public abstract class BaseMethodStatsFilter extends OncePerRequestFilter { + + private static final String START_NS_ATTRIBUTE = "STATS_START_NS"; + private static final String STOP_NS_ATTRIBUTE = "STATS_STOP_NS"; + + @Autowired + private BaseMethodStatsContext baseMethodStatsContext; + + protected static class RequestMethodStats { + private String uri; + private String method; + private long loginTime = -1; + private long callStartTime; + private MultiValueMap requestHeaders = new LinkedMultiValueMap<>(); + private MultiValueMap formParameters = new LinkedMultiValueMap<>(); + + public String getUri() { + return uri; + } + + public String getMethod() { + return method; + } + + public long getLoginTime() { + return loginTime; + } + + public long getCallStartTime() { + return callStartTime; + } + + public MultiValueMap getRequestHeaders() { + return requestHeaders; + } + + public MultiValueMap getFormParameters() { + return formParameters; + } + } + + protected static class ResponseMethodStats { + private int statusCode = -1; + private long loginTime = -1; + private long callTime = -1; + private long serializationTime = -1; + private long bytesWritten = -1; + private MultiValueMap responseHeaders = new LinkedMultiValueMap<>(); + + public int getStatusCode() { + return statusCode; + } + + public long getLoginTime() { + return loginTime; + } + + public long getCallTime() { + return callTime; + } + + public long getSerializationTime() { + return serializationTime; + } + + public long getBytesWritten() { + return bytesWritten; + } + + public MultiValueMap getResponseHeaders() { + return responseHeaders; + } + } + + @Override + public void doFilterInternal(@NonNull HttpServletRequest request, @NonNull HttpServletResponse response, @NonNull FilterChain chain) + throws IOException, ServletException { + preProcess(request, response); + + if (!(response instanceof CountingHttpServletResponseWrapper)) { + response = new CountingHttpServletResponseWrapper(response); + } + if (baseMethodStatsContext.getCountingHttpServletResponseWrapper() == null) { + baseMethodStatsContext.setCountingHttpServletResponseWrapper((CountingHttpServletResponseWrapper) response); + } + + chain.doFilter(request, response); + postProcess(request, response); + } + + public void preProcess(HttpServletRequest request, HttpServletResponse response) { + if (baseMethodStatsContext.getRequestStats() == null) { + baseMethodStatsContext.setRequestStats(createRequestMethodStats(request, response)); + + long start = System.nanoTime(); + request.setAttribute(START_NS_ATTRIBUTE, start); + } + preProcess(baseMethodStatsContext.getRequestStats()); + } + + public void preProcess(RequestMethodStats requestStats) { + // do nothing + } + + public void postProcess(HttpServletRequest request, HttpServletResponse response) { + if (baseMethodStatsContext.getResponseStats() == null) { + long stop = System.nanoTime(); + request.setAttribute(STOP_NS_ATTRIBUTE, stop); + + baseMethodStatsContext.setResponseStats(createResponseMethodStats(request, response)); + } + postProcess(baseMethodStatsContext.getResponseStats()); + } + + abstract public void postProcess(ResponseMethodStats responseStats); + + protected RequestMethodStats createRequestMethodStats(HttpServletRequest request, HttpServletResponse response) { + RequestMethodStats requestStats = new RequestMethodStats(); + + requestStats.uri = request.getRequestURI(); + requestStats.method = request.getMethod(); + + requestStats.callStartTime = System.nanoTime(); + try { + requestStats.callStartTime = (long) request.getAttribute(REQUEST_START_TIME_NS_ATTRIBUTE); + } catch (Exception e) { + // do nothing + } + try { + requestStats.loginTime = (long) request.getAttribute(REQUEST_LOGIN_TIME_ATTRIBUTE); + } catch (Exception e) { + // do nothing + } + + for (Enumeration headerNames = request.getHeaderNames(); headerNames.hasMoreElements();) { + String header = headerNames.nextElement(); + for (Enumeration headerValues = request.getHeaders(header); headerValues.hasMoreElements();) { + requestStats.requestHeaders.add(header, headerValues.nextElement()); + } + } + + if (request.getContentType() != null && MediaType.parseMediaType(request.getContentType()).isCompatibleWith(MediaType.APPLICATION_FORM_URLENCODED)) { + Map formParameters = request.getParameterMap(); + if (formParameters != null) { + formParameters.forEach((k, v) -> requestStats.formParameters.addAll(k, Arrays.asList(v))); + } + } + + return requestStats; + } + + private ResponseMethodStats createResponseMethodStats(HttpServletRequest request, HttpServletResponse response) { + ResponseMethodStats responseStats = new ResponseMethodStats(); + + long start = System.nanoTime(); + try { + start = (long) request.getAttribute(START_NS_ATTRIBUTE); + } catch (Exception e) { + // do nothing + } + + long stop = System.nanoTime(); + try { + stop = (long) request.getAttribute(STOP_NS_ATTRIBUTE); + } catch (Exception e) { + // do nothing + } + + responseStats.serializationTime = TimeUnit.NANOSECONDS.toMillis(stop - start); + responseStats.loginTime = baseMethodStatsContext.getRequestStats().getLoginTime(); + responseStats.callTime = TimeUnit.NANOSECONDS.toMillis(stop - baseMethodStatsContext.getRequestStats().getCallStartTime()); + + if (response instanceof CountingHttpServletResponseWrapper) { + responseStats.bytesWritten = ((CountingHttpServletResponseWrapper) response).getBytesWritten(); + } + + for (String header : response.getHeaderNames()) { + responseStats.responseHeaders.add(header, response.getHeaders(header)); + } + + return responseStats; + } + + static class CountingHttpServletResponseWrapper extends HttpServletResponseWrapper { + private final ServletResponse response; + private CountingServletOutputStream cos; + + /** + * Creates a ServletResponse adaptor wrapping the given response object. + * + * @param response + * the {@link ServletResponse} to be wrapped + * @throws IllegalArgumentException + * if the response is null. + */ + public CountingHttpServletResponseWrapper(HttpServletResponse response) { + super(response); + this.response = response; + } + + @Override + public ServletOutputStream getOutputStream() throws IOException { + if (cos == null) { + cos = new CountingServletOutputStream(response.getOutputStream()); + } + return cos; + } + + public long getBytesWritten() { + return cos != null ? cos.getBytesWritten() : 0L; + } + } + + private static class CountingServletOutputStream extends ServletOutputStream { + private final ServletOutputStream outputStream; + private long count = 0; + + public CountingServletOutputStream(ServletOutputStream outputStream) { + this.outputStream = outputStream; + } + + @Override + public boolean isReady() { + return outputStream.isReady(); + } + + @Override + public void setWriteListener(WriteListener writeListener) { + outputStream.setWriteListener(writeListener); + } + + @Override + public void write(int b) throws IOException { + outputStream.write(b); + count++; + } + + @Override + public void write(byte[] b, int off, int len) throws IOException { + outputStream.write(b, off, len); + count += len; + } + + public long getBytesWritten() { + return count; + } + } + + public static class BaseMethodStatsContext { + private RequestMethodStats requestStats; + private ResponseMethodStats responseStats; + private CountingHttpServletResponseWrapper countingHttpServletResponseWrapper; + + public RequestMethodStats getRequestStats() { + return requestStats; + } + + public void setRequestStats(RequestMethodStats requestStats) { + this.requestStats = requestStats; + } + + public ResponseMethodStats getResponseStats() { + return responseStats; + } + + public void setResponseStats(ResponseMethodStats responseStats) { + this.responseStats = responseStats; + } + + CountingHttpServletResponseWrapper getCountingHttpServletResponseWrapper() { + return countingHttpServletResponseWrapper; + } + + public void setCountingHttpServletResponseWrapper(CountingHttpServletResponseWrapper countingHttpServletResponseWrapper) { + this.countingHttpServletResponseWrapper = countingHttpServletResponseWrapper; + } + + public CountingResponseBodyEmitter createCountingResponseBodyEmitter(Long timeout) { + return new CountingResponseBodyEmitter(timeout, countingHttpServletResponseWrapper); + } + } + + @Configuration + public static class BaseMethodStatsFilterConfig { + @Bean + @ConditionalOnMissingBean + @RequestScope + public BaseMethodStatsContext baseMethodStatsContext() { + return new BaseMethodStatsContext(); + } + } +} diff --git a/service/src/main/java/datawave/microservice/query/web/filter/CountingResponseBodyEmitter.java b/service/src/main/java/datawave/microservice/query/web/filter/CountingResponseBodyEmitter.java new file mode 100644 index 00000000..5bd267c7 --- /dev/null +++ b/service/src/main/java/datawave/microservice/query/web/filter/CountingResponseBodyEmitter.java @@ -0,0 +1,16 @@ +package datawave.microservice.query.web.filter; + +import org.springframework.web.servlet.mvc.method.annotation.ResponseBodyEmitter; + +public class CountingResponseBodyEmitter extends ResponseBodyEmitter { + private final BaseMethodStatsFilter.CountingHttpServletResponseWrapper countingResponse; + + public CountingResponseBodyEmitter(Long timeout, BaseMethodStatsFilter.CountingHttpServletResponseWrapper countingResponse) { + super(timeout); + this.countingResponse = countingResponse; + } + + public long getBytesWritten() { + return (countingResponse != null) ? countingResponse.getBytesWritten() : 0L; + } +} diff --git a/service/src/main/java/datawave/microservice/query/web/filter/LoggingStatsFilter.java b/service/src/main/java/datawave/microservice/query/web/filter/LoggingStatsFilter.java new file mode 100644 index 00000000..a9628294 --- /dev/null +++ b/service/src/main/java/datawave/microservice/query/web/filter/LoggingStatsFilter.java @@ -0,0 +1,80 @@ +package datawave.microservice.query.web.filter; + +import java.util.List; +import java.util.Map; + +import org.apache.log4j.Logger; +import org.springframework.stereotype.Component; +import org.springframework.util.MultiValueMap; + +@Component +public class LoggingStatsFilter extends BaseMethodStatsFilter { + private final Logger log = Logger.getLogger(this.getClass()); + + @Override + public void preProcess(RequestMethodStats requestStats) { + if (!log.isTraceEnabled() && requestStats != null) { + StringBuilder message = new StringBuilder(); + message.append(" URI: ").append(requestStats.getUri()); + message.append(" Method: ").append(requestStats.getMethod()); + message.append(" Request Headers {"); + for (Map.Entry> header : requestStats.getRequestHeaders().entrySet()) { + message.append(" ").append(header.getKey()).append(" -> "); + String sep = ""; + for (Object o : header.getValue()) { + message.append(sep).append(o); + sep = ","; + } + } + message.append("}"); + message.append(" Form Parameters {"); + try { + MultiValueMap formParams = requestStats.getFormParameters(); + if (formParams == null || formParams.isEmpty()) { + message.append(" None "); + } else { + for (Map.Entry> header : formParams.entrySet()) { + message.append(" ").append(header.getKey()).append(" -> "); + String sep = ""; + for (Object o : header.getValue()) { + message.append(sep).append(o); + sep = ","; + } + } + } + } catch (NullPointerException npe) { + log.warn("Unable to log request due to NPE"); + } catch (Exception e) { + if (null != e.getMessage()) + log.warn("Unable to log request due to error: " + e.getMessage()); + else + log.warn("Unable to log request due to error", e); + } + message.append("}"); + log.trace(message.toString()); + } + } + + @Override + public void postProcess(ResponseMethodStats responseStats) { + if (!log.isTraceEnabled() && responseStats != null) { + StringBuilder message = new StringBuilder(); + message.append(" Post Process: StatusCode: ").append(responseStats.getStatusCode()); + message.append(" Response Headers {"); + for (Map.Entry> header : responseStats.getResponseHeaders().entrySet()) { + message.append(" ").append(header.getKey()).append(" -> "); + String sep = ""; + for (Object o : header.getValue()) { + message.append(sep).append(o); + sep = ","; + } + } + message.append("} Serialization time: ").append(responseStats.getSerializationTime()).append("ms"); + message.append(" Bytes written: ").append(responseStats.getBytesWritten()); + message.append(" Login Time: ").append(responseStats.getLoginTime()).append("ms"); + message.append(" Call Time: ").append(responseStats.getCallTime()).append("ms"); + + log.trace(message); + } + } +} diff --git a/service/src/main/java/datawave/microservice/query/web/filter/QueryMetricsEnrichmentFilterAdvice.java b/service/src/main/java/datawave/microservice/query/web/filter/QueryMetricsEnrichmentFilterAdvice.java new file mode 100644 index 00000000..fba4f1b5 --- /dev/null +++ b/service/src/main/java/datawave/microservice/query/web/filter/QueryMetricsEnrichmentFilterAdvice.java @@ -0,0 +1,208 @@ +package datawave.microservice.query.web.filter; + +import java.util.Arrays; +import java.util.Date; +import java.util.List; +import java.util.Objects; + +import org.apache.log4j.Logger; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.core.MethodParameter; +import org.springframework.http.MediaType; +import org.springframework.http.converter.HttpMessageConverter; +import org.springframework.http.server.ServerHttpRequest; +import org.springframework.http.server.ServerHttpResponse; +import org.springframework.lang.NonNull; +import org.springframework.web.bind.annotation.ControllerAdvice; +import org.springframework.web.context.annotation.RequestScope; +import org.springframework.web.servlet.mvc.method.annotation.ResponseBodyAdvice; + +import datawave.core.query.logic.QueryLogic; +import datawave.core.query.logic.QueryLogicFactory; +import datawave.microservice.authorization.user.DatawaveUserDetails; +import datawave.microservice.query.storage.QueryStatus; +import datawave.microservice.query.storage.QueryStorageCache; +import datawave.microservice.query.web.annotation.EnrichQueryMetrics; +import datawave.microservice.querymetric.BaseQueryMetric; +import datawave.microservice.querymetric.QueryMetricClient; +import datawave.microservice.querymetric.QueryMetricType; +import datawave.webservice.result.BaseQueryResponse; +import datawave.webservice.result.GenericResponse; + +@ControllerAdvice +public class QueryMetricsEnrichmentFilterAdvice extends BaseMethodStatsFilter implements ResponseBodyAdvice { + + private final Logger log = Logger.getLogger(this.getClass()); + + private final QueryLogicFactory queryLogicFactory; + + private final QueryStorageCache queryStorageCache; + + private final QueryMetricClient queryMetricClient; + + // Note: BaseQueryMetric needs to be request scoped + private final BaseQueryMetric baseQueryMetric; + + // Note: QueryMetricsEnrichmentContext needs to be request scoped + private final QueryMetricsEnrichmentContext queryMetricsEnrichmentContext; + + public QueryMetricsEnrichmentFilterAdvice(QueryLogicFactory queryLogicFactory, QueryStorageCache queryStorageCache, QueryMetricClient queryMetricClient, + BaseQueryMetric baseQueryMetric, QueryMetricsEnrichmentContext queryMetricsEnrichmentContext) { + this.queryLogicFactory = queryLogicFactory; + this.queryStorageCache = queryStorageCache; + this.queryMetricClient = queryMetricClient; + this.baseQueryMetric = baseQueryMetric; + this.queryMetricsEnrichmentContext = queryMetricsEnrichmentContext; + } + + @Override + public boolean supports(MethodParameter returnType, @NonNull Class> converterType) { + boolean supports = false; + EnrichQueryMetrics annotation = (EnrichQueryMetrics) Arrays.stream(returnType.getMethodAnnotations()).filter(EnrichQueryMetrics.class::isInstance) + .findAny().orElse(null); + if (annotation != null) { + try { + Class returnClass = Objects.requireNonNull(returnType.getMethod()).getReturnType(); + if (GenericResponse.class.isAssignableFrom(returnClass)) { + supports = true; + queryMetricsEnrichmentContext.setMethodType(annotation.methodType()); + } else if (BaseQueryResponse.class.isAssignableFrom(returnClass)) { + supports = true; + queryMetricsEnrichmentContext.setMethodType(annotation.methodType()); + } else { + log.error("Unexpected response class for metrics annotated query method " + returnType.getMethod().getName() + ". Response class was " + + returnClass.toString()); + } + } catch (NullPointerException e) { + // do nothing + } + } + + return supports; + } + + @Override + public Object beforeBodyWrite(Object body, @NonNull MethodParameter returnType, @NonNull MediaType selectedContentType, + @NonNull Class> selectedConverterType, @NonNull ServerHttpRequest request, + @NonNull ServerHttpResponse response) { + if (body instanceof GenericResponse) { + @SuppressWarnings({"unchecked", "rawtypes"}) + GenericResponse genericResponse = (GenericResponse) body; + queryMetricsEnrichmentContext.setQueryId(genericResponse.getResult()); + } else if (body instanceof BaseQueryResponse) { + BaseQueryResponse baseResponse = (BaseQueryResponse) body; + queryMetricsEnrichmentContext.setQueryId(baseResponse.getQueryId()); + } + + return body; + } + + @Override + public void postProcess(ResponseMethodStats responseStats) { + String queryId = queryMetricsEnrichmentContext.getQueryId(); + EnrichQueryMetrics.MethodType methodType = queryMetricsEnrichmentContext.getMethodType(); + + if (queryId != null && methodType != null) { + // determine which query logic is being used + String queryLogic = null; + if (baseQueryMetric.getQueryLogic() != null) { + queryLogic = baseQueryMetric.getQueryLogic(); + } else { + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + if (queryStatus != null) { + queryLogic = queryStatus.getQuery().getQueryLogicName(); + } + } + + // retrieve the server user and determine whether metrics are enabled + boolean isMetricsEnabled = false; + DatawaveUserDetails serverUser = null; + try { + QueryLogic logic = queryLogicFactory.getQueryLogic(queryLogic); + isMetricsEnabled = logic.getCollectQueryMetrics(); + serverUser = (DatawaveUserDetails) logic.getServerUser(); + } catch (Exception e) { + log.warn("Unable to retrieve the server user and determine if query logic '" + queryLogic + "' supports metrics"); + } + + if (isMetricsEnabled) { + try { + switch (queryMetricsEnrichmentContext.getMethodType()) { + case CREATE: + baseQueryMetric.setCreateCallTime(responseStats.getCallTime()); + baseQueryMetric.setLoginTime(responseStats.getLoginTime()); + break; + case CREATE_AND_NEXT: + baseQueryMetric.setCreateCallTime(responseStats.getCallTime()); + baseQueryMetric.setLoginTime(responseStats.getLoginTime()); + List pageTimes = baseQueryMetric.getPageTimes(); + if (pageTimes != null && !pageTimes.isEmpty()) { + BaseQueryMetric.PageMetric pm = pageTimes.get(pageTimes.size() - 1); + pm.setCallTime(responseStats.getCallTime()); + pm.setLoginTime(responseStats.getLoginTime()); + pm.setSerializationTime(responseStats.getSerializationTime()); + pm.setBytesWritten(responseStats.getBytesWritten()); + } + break; + case NEXT: + pageTimes = baseQueryMetric.getPageTimes(); + if (pageTimes != null && !pageTimes.isEmpty()) { + BaseQueryMetric.PageMetric pm = pageTimes.get(pageTimes.size() - 1); + pm.setCallTime(responseStats.getCallTime()); + pm.setLoginTime(responseStats.getLoginTime()); + pm.setSerializationTime(responseStats.getSerializationTime()); + pm.setBytesWritten(responseStats.getBytesWritten()); + } + break; + } + + baseQueryMetric.setLastUpdated(new Date()); + // @formatter:off + queryMetricClient.submit( + new QueryMetricClient.Request.Builder() + .withUser(serverUser) + .withMetric(baseQueryMetric.duplicate()) + .withMetricType(QueryMetricType.DISTRIBUTED) + .build()); + // @formatter:on + } catch (Exception e) { + log.error("Unable to record metrics for query '" + queryMetricsEnrichmentContext.getQueryId() + "' and method '" + + queryMetricsEnrichmentContext.getMethodType() + "': " + e.getLocalizedMessage(), e); + } + } + } + } + + public static class QueryMetricsEnrichmentContext { + private String queryId; + private EnrichQueryMetrics.MethodType methodType; + + public String getQueryId() { + return queryId; + } + + public void setQueryId(String queryId) { + this.queryId = queryId; + } + + public EnrichQueryMetrics.MethodType getMethodType() { + return methodType; + } + + public void setMethodType(EnrichQueryMetrics.MethodType methodType) { + this.methodType = methodType; + } + } + + @Configuration + public static class QueryMetricsEnrichmentFilterAdviceConfig { + @Bean + @ConditionalOnMissingBean + @RequestScope + public QueryMetricsEnrichmentContext queryMetricsEnrichmentContext() { + return new QueryMetricsEnrichmentContext(); + } + } +} diff --git a/service/src/main/resources/config/application.yml b/service/src/main/resources/config/application.yml new file mode 100644 index 00000000..813c6cb5 --- /dev/null +++ b/service/src/main/resources/config/application.yml @@ -0,0 +1,10 @@ +server: + compression: + # enable response compression + enabled: true + + # mime-types that should be compressed (wildcards NOT supported) + mime-types: "application/javascript, application/json, application/xml, application/x-yaml, application/x-protobuf, application/x-protostuff, text/css, text/html, text/javascript, text/plain, text/xml, text/x-yaml, text/yaml" + + # response size at which compression kicks in + min-response-size: 4KB \ No newline at end of file diff --git a/service/src/main/resources/config/bootstrap.yml b/service/src/main/resources/config/bootstrap.yml new file mode 100644 index 00000000..f937e421 --- /dev/null +++ b/service/src/main/resources/config/bootstrap.yml @@ -0,0 +1,70 @@ +spring: + application: + name: query + config: + # Locations to check for configuration overrides + import: "optional:file:/etc/config/override[.yml],optional:file:/etc/config/override[.properties]" + cloud: + config: + # Disable consul-first config by default. We'll turn it back on in the consul profile if that profile is enabled. + discovery: + enabled: false + # Always fail fast so we can retry if the config server is not up yet + failFast: true + # Give the config server time to start up if it hasn't already + retry: + max-attempts: 60 + uri: '${CONFIG_SERVER_URL:http://configuration:8888/configserver}' + stream: + rabbit: + bindings: + queryMetricSource-out-0: + producer: + # Note: This must match CONFIRM_ACK_CHANNEL in QueryMetricOperations.java or producer confirms will not work. + confirmAckChannel: 'confirmAckChannel' + # Starting with spring-boot 2.6, circular references are disabled by default + # This is still needed for the evaluation-only function + main: + allow-circular-references: true + +datawave: + table: + cache: + enabled: false + +--- + +# For the dev profile, check localhost for the config server by default +spring: + config.activate.on-profile: 'dev' + cloud: + config: + uri: '${CONFIG_SERVER_URL:http://localhost:8888/configserver}' + +--- + +spring: + config.activate.on-profile: 'consul' + cloud: + config: + # Use Consul to locate the configuration server and bootstrap app config. + discovery: + enabled: true + # Give the config server a long time to come up and register itself in Consul + retry: + max-attempts: 120 + # Allow the default Consul host to be overridden via an environment variable + consul: + host: ${CONSUL_HOST:localhost} + +--- + +# For the "No Messaging" profile, we need to disable the AMQP bus, our custom RabbitMQ discovery, and the RabbitMQ health indicator. +spring: + config.activate.on-profile: 'nomessaging' + cloud: + bus: + enabled: false + rabbitmq: + discovery: + enabled: false diff --git a/service/src/main/resources/log4j2.yml b/service/src/main/resources/log4j2.yml new file mode 100644 index 00000000..9fdbc6bf --- /dev/null +++ b/service/src/main/resources/log4j2.yml @@ -0,0 +1,46 @@ +Configuration: + status: warn + monitorInterval: 60 + + Properties: + Property: + - name: logDir + value: "logs/" + - name: PID + value: "????" + - name: LOG_PATTERN + value: "%clr{%d{yyyy-MM-dd HH:mm:ss.SSS}}{faint} %clr{%5p} %clr{${sys:PID}}{magenta} %clr{---}{faint} %clr{[%15.15t]}{faint} %clr{%-40.40c{1.}}{cyan} %clr{:}{faint} %m%n%wEx" + + Appenders: + Console: + name: Console + target: SYSTEM_OUT + follow: true + PatternLayout: + pattern: "${LOG_PATTERN}" + + RollingFile: + - name: File + fileName: "${sys:logDir}/query-service.log" + filePattern: "${sys:logDir}/query-service.log.%d{yyyy-MM-dd}-%i.gz" + append: true + bufferedIO: true + bufferSize: 8192 + Policies: + TimeBasedTriggeringPolicy: + interval: 1 + SizeBasedTriggeringPolicy: + size: 250MB + DefaultRolloverStrategy: + max: 10 + PatternLayout: + pattern: "${LOG_PATTERN}" + + Loggers: + Root: + level: info + AppenderRef: + - ref: Console + level: info + - ref: File + level: trace \ No newline at end of file diff --git a/service/src/main/resources/static/css/dwdocs.css b/service/src/main/resources/static/css/dwdocs.css new file mode 100644 index 00000000..0027edae --- /dev/null +++ b/service/src/main/resources/static/css/dwdocs.css @@ -0,0 +1,17 @@ + +/* Text-level semantics + ========================================================================== */ + +.pageHeader { + background-color: #3c3c3c; +} +.images{ + padding: 5px; +} +.images .icons{ + +} +.images .menu{ + +} + diff --git a/service/src/main/resources/static/images/api_icon_finished.png b/service/src/main/resources/static/images/api_icon_finished.png new file mode 100644 index 00000000..d9a231a7 Binary files /dev/null and b/service/src/main/resources/static/images/api_icon_finished.png differ diff --git a/service/src/main/resources/static/images/dwdocs_logo.png b/service/src/main/resources/static/images/dwdocs_logo.png new file mode 100644 index 00000000..cbb67048 Binary files /dev/null and b/service/src/main/resources/static/images/dwdocs_logo.png differ diff --git a/service/src/main/resources/static/images/dwquery_logo.png b/service/src/main/resources/static/images/dwquery_logo.png new file mode 100644 index 00000000..41c39787 Binary files /dev/null and b/service/src/main/resources/static/images/dwquery_logo.png differ diff --git a/service/src/main/resources/static/images/releaseNotes_icon.png b/service/src/main/resources/static/images/releaseNotes_icon.png new file mode 100644 index 00000000..7844cc29 Binary files /dev/null and b/service/src/main/resources/static/images/releaseNotes_icon.png differ diff --git a/service/src/main/resources/static/images/userGuide_icon.png b/service/src/main/resources/static/images/userGuide_icon.png new file mode 100644 index 00000000..c7a635e7 Binary files /dev/null and b/service/src/main/resources/static/images/userGuide_icon.png differ diff --git a/service/src/main/resources/static/js/index.js b/service/src/main/resources/static/js/index.js new file mode 100644 index 00000000..d9e72b01 --- /dev/null +++ b/service/src/main/resources/static/js/index.js @@ -0,0 +1,17 @@ + $(document).foundation() + +// This code expects the following templated values to be set in index.html +// var buildUserEmail = "[[${gitBuildUserEmail}]]"; +// var branch = "[[${gitBranch}]]"; +// var commitId = "[[${gitCommitId}]]"; +// var buildTime = "[[${gitBuildTime}]]"; + +// Set additional content when ready +$(function() { + var gitInfoHtml = 'Built By: ' + buildUserEmail + '
' + + 'Branch: ' + branch + '
' + + 'Commit: ' + commitId + '
' + + 'Date/Time: ' + buildTime; + + $("#gitInfo").html(gitInfoHtml); +}); diff --git a/service/src/main/resources/templates/index.html b/service/src/main/resources/templates/index.html new file mode 100644 index 00000000..5d605c87 --- /dev/null +++ b/service/src/main/resources/templates/index.html @@ -0,0 +1,106 @@ + + + + + + + DataWave Docs + + + + +
+
+
    +
    • +
+
+
+ +
+
+
+

Welcome to DataWave

+
+ +
+
+ +
API Documents
+
+
+ +
Query Help
+
+
+ +
Release Notes
+
+
+ + + + + + +
+
+
+
+
+ + + diff --git a/service/src/main/resources/templates/query_help.html b/service/src/main/resources/templates/query_help.html new file mode 100644 index 00000000..566b1c00 --- /dev/null +++ b/service/src/main/resources/templates/query_help.html @@ -0,0 +1,241 @@ + + + + + DataWave Web Service Query Documentation - Version ###version### + + + +

Query Overview

+

The web service allows users to query the warehouse in several different ways. The Query + api presents a single set of methods for starting, using, and closing a query. The web service hosts different types of + query logic each with their own response type. Each query has the same lifecycle: +

    +
  1. User defines the query be calling either the Query/create or Query/define method. This returns a query id.
    2. +
  2. User iterates over the results by repetitive calls to Query/{queryId}/next. Each call to next will return a page of results. The page size was + set in step 1, or can be changed with a call to the update operation. The type of response is dependent on the query logic, and is listed on the + query logic page.
    3. +
  3. When the user receives a HTTP 204 response code (No Content), then there are no more results. The user should release the resources associated + with their query by calling Query/{queryId}/close
    4. +
+

JEXL Query Syntax

+

Query logic classes will use different types of syntax for the query string. Most of the queries that retrieve raw events will accept JEXL (default) + or our modified LUCENE syntax. We support a subset of the language elements in the Apache Commons JEXL syntax and we have implemented some of our own + custom functions. We support the following JEXL operators: +

    +
  • ==
    • +
  • !=
    • +
  • <
    • +
  • ≤
    • +
  • >
    • +
  • ≥
    • +
  • =~ (regex)
    • +
  • !~ (negative regex)
    • +
+

We have created the following functions, see the examples section below to see their usage: +

    +
  • Content functions: phrase(), adjacent(), and within()
    • +
  • Geo Functions: within_bounding_box() and within_circle()
    • +
  • GeoWave Functions: intersects_bounding_box(), intersects_radius_km(), contains(), covers(), coveredBy(), crosses(), intersects(), overlaps(), and within() +
  • Utility Functions: between() and length()
    • +
+

Since JEXL is an expression language and not a text query language, we also added support for unfielded queries. For most of the query + logic types that query raw events, we have provided the capability to search on any indexed field. To achieve this, the user should + submit their query with the field name: _ANYFIELD_. See the examples below for its usage. +

LUCENE Query Syntax

+

We have modified the LUCENE syntax such that the NOT operator is not unary, AND's are not fuzzy, and the implicit operator is the AND not the OR. + Our LUCENE syntax has the following form: + 

+		ModClause ::= DisjQuery [NOT DisjQuery]*
+		DisjQuery ::= ConjQuery [ OR ConjQuery ]*
+		ConjQuery ::= Query [ AND Query ]*
+		Query ::= Clause [ Clause ]*
+		Clause ::= Term | [ ModClause ]
+		Term ::=
+			field:selector |
+			field:selec* |
+			field:selec*or |
+			field:*lector |
+			field:selec?or |
+			selector | (can use wildcards)
+			field:[begin TO end] |
+			field:{begin TO end} |
+			"quick brown dog" |
+			"quick brown dog"~20 |
+			#FUNCTION(ARG1, ARG2)
+
+

Note that to search for punctuation characters within a term, you need to escape it with a backslash. +

We have also modified the LUCENE syntax to provide support for the JEXL syntax that LUCENE does not support. The table below maps the + JEXL operators to the supported LUCENE syntax: +
+ + + + + + + + + + + + + + + + +
JEXL OperatorLUCENE Operator
filter:includeRegex(regex)#INCLUDE(regex)
filter:includeRegex(field, regex)#INCLUDE(field, regex)
filter:includeRegex(field1, regex1) <op> filter:includeRegex(field2, regex2) ...#INCLUDE(op, field1, regex1, field2, regex2 ...) where op is 'or' or 'and'
filter:excludeRegex(regex)#EXCLUDE(regex)
filter:excludeRegex(field, regex)#EXCLUDE(field, regex)
filter:excludeRegex(field1, regex1) <op> filter:excludeRegex(field2, regex2) ...#EXCLUDE(op, field1, regex1, field2, regex2 ...) where op is 'or' or 'and'
filter:isNull(field)#ISNULL(field)
not(filter:isNull(field))#ISNOTNULL(field)
filter:occurrence(field,operator,count))#OCCURRENCE(field,operator,count)
filter:timeFunction(field1,field2,operator,equivalence,goal)#TIME_FUNCTION(field1,field2,operator,equivalence,goal)
filter:text(field)#TEXT(field)
filter:text(field, value)#TEXT(field, value)
filter:text(field1, value1) <op> filter:text(field2, value2) ...#TEXT(op, field1, value1, field2, value2 ...) where op is 'or' or 'and'
filter:compare(field1, op, mode, field2)#COMPARE(field1, op, mode, field2) where op is '>', '>=', '<', '<=', '==', '=', or '!=' and mode is 'ALL' or 'ANY'
+

Notes: +

    +
  1. None of these filter functions can be applied against index-only fields except for filter:text (#TEXT).
    2. +
  2. No indicies will be used for the INCLUDE and EXCLUDE functions, however they will be used for the TEXT function. All of these will be matched against the original unnormalized value. +
  3. The occurrence function is used to count the number of instances of a field in the event. Valid operators are '==' (or '='),'>','>=','<','<=', and '!='.
    4. +
+

Some geo functions are supplied as well that may prove useful although the within_bounding_box function may be done with a simple range comparison (i.e. LAT_LON_USER <= <lat1>_<lon1> and LAT_LON_USER >= <lat2>_<lon2>.

+ + + + + +
JEXL OperatorLUCENE Operator
geo:within_bounding_box(latLonField, lowerLeft, upperRight)#GEO(bounding_box, latLonField, 'lowerLeft', 'upperRight')
geo:within_bounding_box(lonField, latField, minLon, minLat, maxLon, maxLat)#GEO(bounding_box, lonField, latField, minLon, minLat, maxLon, maxLat)
geo:within_circle(latLonField, center, radius)#GEO(circle, latLonField, center, radius)
+

Notes: +

    +
  1. All lat and lon values are in decimal.
    2. +
  2. The lowerLeft, upperRight, and center are of the form <lat>_<lon> and must be surrounded by single quotes.
    3. +
  3. The radius is in decimal degrees as well.
    4. +
+

New GeoWave geo functions are available as follows:

+ + + + + + + + + + + + + + + + + + + + + + + + + +
JEXL OperatorLUCENE Operator
geowave:intersects_bounding_box(geometryField, westLon, eastLon, southLat, northLat)#INTERSECTS_BOUNDING_BOX(geometryField, westLon, eastLon, southLat, northLat)
geowave:intersects_radius_km(geometryField, centerLon, centerLat, radiusKm)#INTERSECTS_RADIUS_KM(geometryField, centerLon, centerLat, radiusKm)
geowave:contains(geometryField, Well-Known Text)#CONTAINS(geometryField, centerLon, centerLat, radiusDegrees)
geowave:covers(geometryField, Well-Known Text)#COVERS(geometryField, Well-Known Text)
geowave:coveredBy(geometryField, Well-Known Text)#COVERED_BY(geometryField, Well-Known Text)
geowave:crosses(geometryField, Well-Known Text)#CROSSES(geometryField, Well-Known Text)
geowave:intersects(geometryField, Well-Known Text)#INTERSECTS(geometryField, Well-Known Text)
geowave:overlaps(geometryField, Well-Known Text)#OVERLAPS(geometryField, Well-Known Text)
geowave:within(geometryField, Well-Known Text)#WITHIN(geometryField, Well-Known Text)
+

Notes: +

    +
  1. All lat and lon values are in decimal degrees.
    2. +
  2. The lowerLeft, upperRight, and center are of the form <lat>_<lon> and must be surrounded by single quotes.
    3. +
  3. Geometry is represented according to the Open Geospatial Consortium standard for Well-Known Text. It is in decimal degrees longitude for x, amd latitude for y. For example, a point at New York can be represented as 'POINT (-74.01 40.71) ' and a box at New York can be repesented as 'POLYGON(( -74.1 40.75, -74.1 40.69, -73.9 40.69, -73.9 40.75, -74.1 40.75))'
    4. +
+ +

An #EVALUATION_ONLY function has recently been introduced which is intended to be used to defer execution of part of a query to the evaluation phase. This could be used to defer a range to the evaluation phase, or possibly to defer GeoWave function evaluation to the evaluation phase, avoiding the need to generate geo indices when additional indexed fields are present (i.e. indexed fields other than the geo field). For example:

+ + + + + + + + + + +
JEXL Example
INDEXED_FIELD == 'some value' && ((ASTEvaluationOnly = true) && geowave:intersects(geometryField, Well-Known Text))
LUCENE Example
INDEXED_FIELD: 'some value' AND #EVALUATION_ONLY('#INTERSECTS(geometryField, Well-Known Text)')
+ + +

There are some additional functions that are supplied to handle dates more smoothly. It is intended that the need for these functions + may go away in future versions (bolded parameters are literal, other parameters are substituted with appropriate values): + + + + + + + + + + + + + + + + + + +
JEXL OperatorLUCENE Operator
filter:betweenDates(field, start date, end date)#DATE(field, start date, end date) or #DATE(field, between, start date, end date)
filter:betweenDates(field, start date, end date, start/end date format)#DATE(field, start date, end date, start/end date format) or #DATE(field, between, start date, end date, start/end date format)
filter:betweenDates(field, field date format, start date, end date, start/end date format)#DATE(field, field date format, start date, end date, start/end date format) or #DATE(field, between, field date format, start date, end date, start/end date format)
filter:afterDate(field, date)#DATE(field, after, date)
filter:afterDate(field, date, date format)#DATE(field, after, date, date format)
filter:afterDate(field, field date format, date, date format)#DATE(field, after, field date format, date, date format)
filter:beforeDate(field, date)#DATE(field, before, date)
filter:beforeDate(field, date, date format)#DATE(field, before, date, date format)
filter:beforeDate(field, field date format, date, date format)#DATE(field, before, field date format, date, date format)
filter:betweenLoadDates(LOAD_DATE, start date, end date)#LOADED(start date, end date) or #LOADED(between, start date, end date)
filter:betweenLoadDates(LOAD_DATE, start date, end date, start/end date format)#LOADED(start date, end date, start/end date format) or #LOADED(between, start date, end date, start/end date format)
filter:afterLoadDate(LOAD_DATE, date)#LOADED(after, date)
filter:afterLoadDate(LOAD_DATE, date, date format)#LOADED(after, date, date format)
filter:beforeLoadDate(LOAD_DATE, date)#LOADED(before, date)
filter:beforeLoadDate(LOAD_DATE, date, date format)#LOADED(before, date, date format)
filter:timeFunction(DOWNTIME, UPTIME, '-', '>', 2522880000000L)#TIME_FUNCTION(DOWNTIME, UPTIME, '-', '>', '2522880000000L')
+

Notes: +

    +
  1. None of these filter functions can be applied against index-only fields.
    2. +
  2. Between functions are inclusive, and the other functions are exclusive of the entered dates.
    3. +
  3. Date formats must be entered in the Java SimpleDateFormat object format.
    4. +
  4. If the entered date format is not specified, then the following list of date formats will be tried:
    5. +
      +
    • yyyyMMdd:HH:mm:ss:SSSZ
      • +
    • yyyyMMdd:HH:mm:ss:SSS
      • +
    • EEE MMM dd HH:mm:ss zzz yyyy
      • +
    • d MMM yyyy HH:mm:ss 'GMT'
      • +
    • yyyy-MM-dd HH:mm:ss.SSS Z
      • +
    • yyyy-MM-dd HH:mm:ss.SSS
      • +
    • yyyy-MM-dd HH:mm:ss.S Z
      • +
    • yyyy-MM-dd HH:mm:ss.S
      • +
    • yyyy-MM-dd HH:mm:ss Z
      • +
    • yyyy-MM-dd HH:mm:ssz
      • +
    • yyyy-MM-dd HH:mm:ss
      • +
    • yyyyMMdd HHmmss
      • +
    • yyyy-MM-dd'T'HH'|'mm
      • +
    • yyyy-MM-dd'T'HH':'mm':'ss'.'SSS'Z'
      • +
    • yyyy-MM-dd'T'HH':'mm':'ss'Z'
      • +
    • MM'/'dd'/'yyyy HH':'mm':'ss
      • +
    • E MMM d HH:mm:ss z yyyy
      • +
    • E MMM d HH:mm:ss Z yyyy
      • +
    • yyyyMMdd_HHmmss
      • +
    • yyyy-MM-dd
      • +
    • MM/dd/yyyy
      • +
    • yyyy-MMMM
      • +
    • yyyy-MMM
      • +
    • yyyyMMddHHmmss
      • +
    • yyyyMMddHHmm
      • +
    • yyyyMMddHH
      • +
    • yyyyMMdd
      • +
    +
  5. A special date format of 'e' can be supplied to mean milliseconds since epoch.
    6. +
+

Unique Functions

+

In queries, unique functions are available to find unique results for fields that are + specified, to the granularity of DAY, HOUR, or MINUTE. The field passed to the function needs to be a + dateTime field if using a time granularity, otherwise the original field will be returned as a unique result. Multiple granularity levels can + be provided to the same field in multiple ways, and granularities will be merged across multiple functions if necessary. + Potential function options that the user may use are listed below.

+
    +
  • #unique(field): Find unique results for every value of field. No restriction on the type of field.
    • +
  • #unique(datetimeField[DAY]): Find every unique result of datetimeField to the day.
    • +
  • #unique_by_day(datetimeField): Same as above, just a different means of formatting.
    • +
  • #unique(datetimeField[HOUR]): Find every unique result of datetimeField to the hour.
    • +
  • #unique_by_hour(datetimeField): Same result as above.
    • +
  • #unique(datetimeField[MINUTE]): Find every unique result of datetimeField to the minute.
    • +
  • #unique_by_minute(datetimeField): Same result as above.
    • +
  • #unique(field,datetimeField1[DAY],datetimeField2[HOUR],datetimeField3[MINUTE]): Find every unique result for field, field1 to the day, field2 to the hour, field3 to the minute.
    • +
  • #unique(field,datetimeField[DAY,HOUR]): Consecutive granularities for the same field are supported. Find each unique result for field, and datetimeField to the day, as well as to the hour.
    • +
  • #unique(field1,field2[DAY]) AND #unique_by_minute(field1,field2,field3) AND #unique_by_minute(field4): Find unique results for field1 with no restrictions as well as with minute restrictions, field2 to the day and the minute, field3 by the minute, and field4 by the minute.
    • +
+

As referenced above, multiple granularity levels can be specified for the same field in different ways. The following queries are equivalent, and will find + each unique result for field, to the hour and to the minute specified:

+
    +
  • #unique(field[HOUR,MINUTE])
    • +
  • #unique(field[HOUR],field[MINUTE])
    • +
  • #unique(field[HOUR]) AND #unique(field[MINUTE])
    • +
  • #unique(field[HOUR]) AND #unique_by_minute(field)
    • +
  • #unique_by_hour(field) AND #unique_by_minute(field)
    • +
+
+ diff --git a/service/src/test/java/datawave/microservice/query/AbstractQueryServiceTest.java b/service/src/test/java/datawave/microservice/query/AbstractQueryServiceTest.java new file mode 100644 index 00000000..032522bb --- /dev/null +++ b/service/src/test/java/datawave/microservice/query/AbstractQueryServiceTest.java @@ -0,0 +1,585 @@ +package datawave.microservice.query; + +import static datawave.microservice.query.QueryParameters.QUERY_MAX_CONCURRENT_TASKS; +import static datawave.microservice.query.QueryParameters.QUERY_MAX_RESULTS_OVERRIDE; +import static datawave.microservice.query.QueryParameters.QUERY_PAGESIZE; +import static datawave.security.authorization.DatawaveUser.UserType.USER; +import static datawave.webservice.common.audit.AuditParameters.AUDIT_ID; +import static org.springframework.test.web.client.ExpectedCount.never; +import static org.springframework.test.web.client.match.MockRestRequestMatchers.anything; +import static org.springframework.test.web.client.match.MockRestRequestMatchers.requestTo; +import static org.springframework.test.web.client.response.MockRestResponseCreators.withSuccess; + +import java.io.IOException; +import java.nio.charset.Charset; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +import org.apache.http.NameValuePair; +import org.apache.http.client.utils.URLEncodedUtils; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.springframework.beans.DirectFieldAccessor; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.web.server.LocalServerPort; +import org.springframework.boot.web.client.RestTemplateBuilder; +import org.springframework.cloud.bus.ServiceMatcher; +import org.springframework.cloud.bus.event.RemoteQueryRequestEvent; +import org.springframework.context.ApplicationEvent; +import org.springframework.context.ApplicationEventPublisher; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Lazy; +import org.springframework.context.annotation.Primary; +import org.springframework.context.annotation.Profile; +import org.springframework.http.HttpMethod; +import org.springframework.http.HttpStatus; +import org.springframework.http.RequestEntity; +import org.springframework.http.ResponseEntity; +import org.springframework.http.client.ClientHttpResponse; +import org.springframework.test.web.client.MockRestServiceServer; +import org.springframework.test.web.client.RequestMatcher; +import org.springframework.util.LinkedMultiValueMap; +import org.springframework.util.MultiValueMap; +import org.springframework.web.client.DefaultResponseErrorHandler; +import org.springframework.web.client.RestTemplate; +import org.springframework.web.util.UriComponents; +import org.springframework.web.util.UriComponentsBuilder; + +import com.fasterxml.jackson.core.JacksonException; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonDeserializer; +import com.fasterxml.jackson.databind.Module; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.module.SimpleModule; +import com.hazelcast.config.Config; +import com.hazelcast.core.Hazelcast; +import com.hazelcast.core.HazelcastInstance; + +import datawave.marking.ColumnVisibilitySecurityMarking; +import datawave.microservice.audit.AuditClient; +import datawave.microservice.authorization.jwt.JWTRestTemplate; +import datawave.microservice.authorization.user.DatawaveUserDetails; +import datawave.microservice.query.config.QueryProperties; +import datawave.microservice.query.messaging.QueryResultsManager; +import datawave.microservice.query.messaging.QueryResultsPublisher; +import datawave.microservice.query.messaging.Result; +import datawave.microservice.query.remote.QueryRequest; +import datawave.microservice.query.storage.QueryStatus; +import datawave.microservice.query.storage.QueryStorageCache; +import datawave.microservice.query.storage.TaskKey; +import datawave.microservice.query.storage.TaskStates; +import datawave.security.authorization.DatawaveUser; +import datawave.security.authorization.SubjectIssuerDNPair; +import datawave.webservice.query.exception.QueryExceptionType; +import datawave.webservice.query.result.event.DefaultEvent; +import datawave.webservice.query.result.event.DefaultField; +import datawave.webservice.result.BaseResponse; +import datawave.webservice.result.DefaultEventQueryResponse; +import datawave.webservice.result.GenericResponse; +import datawave.webservice.result.QueryImplListResponse; +import datawave.webservice.result.QueryLogicResponse; +import datawave.webservice.result.VoidResponse; + +public abstract class AbstractQueryServiceTest { + protected static final String EXPECTED_AUDIT_URI = "http://localhost:11111/audit/v1/audit"; + protected static final String TEST_QUERY_STRING = "FIELD:SOME_VALUE"; + protected static final String TEST_QUERY_NAME = "The Greatest Query in the World - Tribute"; + protected static final String TEST_QUERY_AUTHORIZATIONS = "ALL"; + protected static final String TEST_QUERY_BEGIN = "20000101 000000.000"; + protected static final String TEST_QUERY_END = "20500101 000000.000"; + protected static final String TEST_VISIBILITY_MARKING = "ALL"; + protected static final long TEST_MAX_RESULTS_OVERRIDE = 369L; + protected static final long TEST_PAGESIZE = 123L; + protected static final long TEST_WAIT_TIME_MILLIS = TimeUnit.SECONDS.toMillis(30); + + @LocalServerPort + protected int webServicePort; + + @Autowired + protected RestTemplateBuilder restTemplateBuilder; + + protected JWTRestTemplate jwtRestTemplate; + + protected SubjectIssuerDNPair DN; + protected String userDN = "userDn"; + + protected SubjectIssuerDNPair altDN; + protected String altUserDN = "altUserDN"; + + @Autowired + protected QueryStorageCache queryStorageCache; + + @Autowired + protected QueryResultsManager queryQueueManager; + + @Autowired + protected AuditClient auditClient; + + @Autowired + protected QueryProperties queryProperties; + + @Autowired + protected LinkedList queryRequestEvents; + + protected List auditIds; + protected MockRestServiceServer mockServer; + + @BeforeEach + public void setup() { + auditIds = new ArrayList<>(); + + jwtRestTemplate = restTemplateBuilder.build(JWTRestTemplate.class); + jwtRestTemplate.setErrorHandler(new NoOpResponseErrorHandler()); + DN = SubjectIssuerDNPair.of(userDN, "issuerDn"); + altDN = SubjectIssuerDNPair.of(altUserDN, "issuerDN"); + + RestTemplate auditorRestTemplate = (RestTemplate) new DirectFieldAccessor(auditClient).getPropertyValue("jwtRestTemplate"); + mockServer = MockRestServiceServer.createServer(auditorRestTemplate); + + queryRequestEvents.clear(); + } + + @AfterEach + public void teardown() throws Exception { + queryStorageCache.clear(); + queryRequestEvents.clear(); + } + + protected void publishEventsToQueue(String queryId, int numEvents, MultiValueMap fieldValues, String visibility) throws Exception { + QueryResultsPublisher publisher = queryQueueManager.createPublisher(queryId); + for (int resultId = 0; resultId < numEvents; resultId++) { + DefaultEvent event = new DefaultEvent(); + long currentTime = System.currentTimeMillis(); + List fields = new ArrayList<>(); + for (Map.Entry> entry : fieldValues.entrySet()) { + for (String value : entry.getValue()) { + fields.add(new DefaultField(entry.getKey(), visibility, new HashMap<>(), currentTime, value)); + } + } + event.setFields(fields); + publisher.publish(new Result(Integer.toString(resultId), event)); + } + } + + protected String createQuery(DatawaveUserDetails authUser, MultiValueMap map) { + return newQuery(authUser, map, "create"); + } + + protected String defineQuery(DatawaveUserDetails authUser, MultiValueMap map) { + return newQuery(authUser, map, "define"); + } + + protected String newQuery(DatawaveUserDetails authUser, MultiValueMap map, String createOrDefine) { + UriComponents uri = createUri("EventQuery/" + createOrDefine); + + // not testing audit with this method + auditIgnoreSetup(); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); + ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, GenericResponse.class); + + return (String) resp.getBody().getResult(); + } + + protected Future> nextQuery(DatawaveUserDetails authUser, String queryId) { + UriComponents uri = createUri(queryId + "/next"); + RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, null, null, HttpMethod.GET, uri); + + // make the next call asynchronously + return Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, DefaultEventQueryResponse.class)); + } + + protected Future> adminCloseQuery(DatawaveUserDetails authUser, String queryId) { + return stopQuery(authUser, queryId, "adminClose"); + } + + protected Future> closeQuery(DatawaveUserDetails authUser, String queryId) { + return stopQuery(authUser, queryId, "close"); + } + + protected Future> adminCancelQuery(DatawaveUserDetails authUser, String queryId) { + return stopQuery(authUser, queryId, "adminCancel"); + } + + protected Future> cancelQuery(DatawaveUserDetails authUser, String queryId) { + return stopQuery(authUser, queryId, "cancel"); + } + + protected Future> stopQuery(DatawaveUserDetails authUser, String queryId, String closeOrCancel) { + UriComponents uri = createUri(queryId + "/" + closeOrCancel); + RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, null, null, HttpMethod.PUT, uri); + + // make the next call asynchronously + return Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, VoidResponse.class)); + } + + protected Future> adminCloseAllQueries(DatawaveUserDetails authUser) { + UriComponents uri = createUri("/adminCloseAll"); + RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, null, null, HttpMethod.PUT, uri); + + // make the next call asynchronously + return Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, VoidResponse.class)); + } + + protected Future> adminCancelAllQueries(DatawaveUserDetails authUser) { + UriComponents uri = createUri("/adminCancelAll"); + RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, null, null, HttpMethod.PUT, uri); + + // make the next call asynchronously + return Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, VoidResponse.class)); + } + + protected Future> resetQuery(DatawaveUserDetails authUser, String queryId) { + UriComponents uri = createUri(queryId + "/reset"); + RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, null, null, HttpMethod.PUT, uri); + + // make the next call asynchronously + return Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, GenericResponse.class)); + } + + protected Future> removeQuery(DatawaveUserDetails authUser, String queryId) { + UriComponents uri = createUri(queryId + "/remove"); + RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, null, null, HttpMethod.DELETE, uri); + + // make the next call asynchronously + return Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, VoidResponse.class)); + } + + protected Future> adminRemoveQuery(DatawaveUserDetails authUser, String queryId) { + UriComponents uri = createUri(queryId + "/adminRemove"); + RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, null, null, HttpMethod.DELETE, uri); + + // make the next call asynchronously + return Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, VoidResponse.class)); + } + + protected Future> adminRemoveAllQueries(DatawaveUserDetails authUser) { + UriComponents uri = createUri("/adminRemoveAll"); + RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, null, null, HttpMethod.DELETE, uri); + + // make the next call asynchronously + return Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, VoidResponse.class)); + } + + protected Future> updateQuery(DatawaveUserDetails authUser, String queryId, MultiValueMap map) { + UriComponents uri = createUri(queryId + "/update"); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.PUT, uri); + + // make the update call asynchronously + return Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, GenericResponse.class)); + } + + protected Future> duplicateQuery(DatawaveUserDetails authUser, String queryId, MultiValueMap map) { + UriComponents uri = createUri(queryId + "/duplicate"); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); + + // make the update call asynchronously + return Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, GenericResponse.class)); + } + + protected Future> listQueries(DatawaveUserDetails authUser, String queryId, String queryName) { + UriComponentsBuilder uriBuilder = uriBuilder("/list"); + if (queryId != null) { + uriBuilder.queryParam("queryId", queryId); + } + if (queryName != null) { + uriBuilder.queryParam("queryName", queryName); + } + UriComponents uri = uriBuilder.build(); + + RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, null, null, HttpMethod.GET, uri); + + // make the next call asynchronously + return Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, QueryImplListResponse.class)); + } + + protected Future> getQuery(DatawaveUserDetails authUser, String queryId) { + UriComponents uri = createUri(queryId); + + RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, null, null, HttpMethod.GET, uri); + + // make the next call asynchronously + return Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, QueryImplListResponse.class)); + } + + protected Future> listQueryLogic(DatawaveUserDetails authUser) { + UriComponents uri = createUri("/listQueryLogic"); + + RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, null, null, HttpMethod.GET, uri); + + // make the next call asynchronously + return Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, QueryLogicResponse.class)); + } + + protected Future> adminListQueries(DatawaveUserDetails authUser, String queryId, String user, String queryName) { + UriComponentsBuilder uriBuilder = uriBuilder("/adminList"); + if (queryId != null) { + uriBuilder.queryParam("queryId", queryId); + } + if (queryName != null) { + uriBuilder.queryParam("queryName", queryName); + } + if (user != null) { + uriBuilder.queryParam("user", user); + } + UriComponents uri = uriBuilder.build(); + + RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, null, null, HttpMethod.GET, uri); + + // make the next call asynchronously + return Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, QueryImplListResponse.class)); + } + + protected DatawaveUserDetails createUserDetails() { + return createUserDetails(null, null); + } + + protected DatawaveUserDetails createUserDetails(Collection roles, Collection auths) { + Collection userRoles = roles != null ? roles : Collections.singleton("AuthorizedUser"); + Collection userAuths = auths != null ? auths : Collections.singleton("ALL"); + DatawaveUser datawaveUser = new DatawaveUser(DN, USER, userAuths, userRoles, null, System.currentTimeMillis()); + return new DatawaveUserDetails(Collections.singleton(datawaveUser), datawaveUser.getCreationTime()); + } + + protected DatawaveUserDetails createAltUserDetails() { + return createAltUserDetails(null, null); + } + + protected DatawaveUserDetails createAltUserDetails(Collection roles, Collection auths) { + Collection userRoles = roles != null ? roles : Collections.singleton("AuthorizedUser"); + Collection userAuths = auths != null ? auths : Collections.singleton("ALL"); + DatawaveUser datawaveUser = new DatawaveUser(altDN, USER, userAuths, userRoles, null, System.currentTimeMillis()); + return new DatawaveUserDetails(Collections.singleton(datawaveUser), datawaveUser.getCreationTime()); + } + + protected UriComponentsBuilder uriBuilder(String path) { + return UriComponentsBuilder.newInstance().scheme("https").host("localhost").port(webServicePort).path("/query/v1/query/" + path); + } + + protected UriComponents createUri(String path) { + return uriBuilder(path).build(); + } + + protected MultiValueMap createParams() { + MultiValueMap map = new LinkedMultiValueMap<>(); + map.set(DefaultQueryParameters.QUERY_STRING, TEST_QUERY_STRING); + map.set(DefaultQueryParameters.QUERY_NAME, TEST_QUERY_NAME); + map.set(DefaultQueryParameters.QUERY_AUTHORIZATIONS, TEST_QUERY_AUTHORIZATIONS); + map.set(DefaultQueryParameters.QUERY_BEGIN, TEST_QUERY_BEGIN); + map.set(DefaultQueryParameters.QUERY_END, TEST_QUERY_END); + map.set(ColumnVisibilitySecurityMarking.VISIBILITY_MARKING, TEST_VISIBILITY_MARKING); + map.set(QUERY_MAX_CONCURRENT_TASKS, Integer.toString(1)); + map.set(QUERY_MAX_RESULTS_OVERRIDE, Long.toString(TEST_MAX_RESULTS_OVERRIDE)); + map.set(QUERY_PAGESIZE, Long.toString(TEST_PAGESIZE)); + return map; + } + + protected void assertDefaultEvent(List fields, List values, DefaultEvent event) { + Assertions.assertEquals(fields, event.getFields().stream().map(DefaultField::getName).collect(Collectors.toList())); + Assertions.assertEquals(values, event.getFields().stream().map(DefaultField::getValueString).collect(Collectors.toList())); + } + + protected void assertQueryResponse(String queryId, String logicName, long pageNumber, boolean partialResults, long operationTimeInMS, int numFields, + List fieldNames, int numEvents, DefaultEventQueryResponse queryResponse) { + Assertions.assertEquals(queryId, queryResponse.getQueryId()); + Assertions.assertEquals(logicName, queryResponse.getLogicName()); + Assertions.assertEquals(pageNumber, queryResponse.getPageNumber()); + Assertions.assertEquals(partialResults, queryResponse.isPartialResults()); + Assertions.assertEquals(operationTimeInMS, queryResponse.getOperationTimeMS()); + Assertions.assertEquals(numFields, queryResponse.getFields().size()); + Assertions.assertEquals(fieldNames, queryResponse.getFields()); + Assertions.assertEquals(numEvents, queryResponse.getEvents().size()); + } + + protected void assertQueryRequestEvent(String destination, QueryRequest.Method method, String queryId, RemoteQueryRequestEvent queryRequestEvent) { + Assertions.assertEquals(destination, queryRequestEvent.getDestinationService()); + Assertions.assertEquals(queryId, queryRequestEvent.getRequest().getQueryId()); + Assertions.assertEquals(method, queryRequestEvent.getRequest().getMethod()); + } + + protected void assertQueryStatus(QueryStatus.QUERY_STATE queryState, long numResultsReturned, long numResultsGenerated, long activeNextCalls, + long lastPageNumber, long lastCallTimeMillis, QueryStatus queryStatus) { + Assertions.assertEquals(queryState, queryStatus.getQueryState()); + Assertions.assertEquals(numResultsReturned, queryStatus.getNumResultsReturned()); + Assertions.assertEquals(numResultsGenerated, queryStatus.getNumResultsGenerated()); + Assertions.assertEquals(activeNextCalls, queryStatus.getActiveNextCalls()); + Assertions.assertEquals(lastPageNumber, queryStatus.getLastPageNumber()); + Assertions.assertTrue(queryStatus.getLastUsedMillis() > lastCallTimeMillis); + Assertions.assertTrue(queryStatus.getLastUpdatedMillis() > lastCallTimeMillis); + } + + protected void assertQuery(String queryString, String queryName, String authorizations, String begin, String end, String visibility, Query query) + throws ParseException { + SimpleDateFormat sdf = new SimpleDateFormat(DefaultQueryParameters.formatPattern); + Assertions.assertEquals(queryString, query.getQuery()); + Assertions.assertEquals(queryName, query.getQueryName()); + Assertions.assertEquals(authorizations, query.getQueryAuthorizations()); + Assertions.assertEquals(sdf.parse(begin), query.getBeginDate()); + Assertions.assertEquals(sdf.parse(end), query.getEndDate()); + Assertions.assertEquals(visibility, query.getColumnVisibility()); + } + + protected void assertTasksCreated(String queryId) throws IOException { + // verify that the query task states were created + TaskStates taskStates = queryStorageCache.getTaskStates(queryId); + Assertions.assertNotNull(taskStates); + + // verify that a query task was created + List taskKeys = queryStorageCache.getTasks(queryId); + Assertions.assertFalse(taskKeys.isEmpty()); + } + + protected void assertTasksNotCreated(String queryId) throws IOException { + // verify that the query task states were not created + TaskStates taskStates = queryStorageCache.getTaskStates(queryId); + Assertions.assertNull(taskStates); + + // verify that a query task was not created + List taskKeys = queryStorageCache.getTasks(queryId); + Assertions.assertTrue(taskKeys.isEmpty()); + } + + public RequestMatcher auditIdGrabber() { + return request -> { + List params = URLEncodedUtils.parse(request.getBody().toString(), Charset.defaultCharset()); + params.stream().filter(p -> p.getName().equals(AUDIT_ID)).forEach(p -> auditIds.add(p.getValue())); + }; + } + + protected void auditIgnoreSetup() { + mockServer.expect(anything()).andRespond(withSuccess()); + } + + protected void auditSentSetup() { + mockServer.expect(requestTo(EXPECTED_AUDIT_URI)).andExpect(auditIdGrabber()).andRespond(withSuccess()); + } + + protected void auditNotSentSetup() { + mockServer.expect(never(), requestTo(EXPECTED_AUDIT_URI)).andExpect(auditIdGrabber()).andRespond(withSuccess()); + } + + protected void assertAuditSent(String queryId) { + mockServer.verify(); + Assertions.assertEquals(1, auditIds.size()); + if (queryId != null) { + Assertions.assertEquals(queryId, auditIds.get(0)); + } + } + + protected void assertAuditNotSent() { + mockServer.verify(); + Assertions.assertEquals(0, auditIds.size()); + } + + protected void assertQueryException(String message, String cause, String code, QueryExceptionType queryException) { + Assertions.assertEquals(message, queryException.getMessage()); + Assertions.assertEquals(cause, queryException.getCause()); + Assertions.assertEquals(code, queryException.getCode()); + } + + protected BaseResponse assertBaseResponse(boolean hasResults, HttpStatus.Series series, ResponseEntity response) { + Assertions.assertEquals(series, response.getStatusCode().series()); + Assertions.assertNotNull(response); + BaseResponse baseResponse = (BaseResponse) response.getBody(); + Assertions.assertNotNull(baseResponse); + Assertions.assertEquals(hasResults, baseResponse.getHasResults()); + return baseResponse; + } + + @SuppressWarnings("unchecked") + protected GenericResponse assertGenericResponse(boolean hasResults, HttpStatus.Series series, ResponseEntity response) { + Assertions.assertEquals(series, response.getStatusCode().series()); + Assertions.assertNotNull(response); + GenericResponse genericResponse = (GenericResponse) response.getBody(); + Assertions.assertNotNull(genericResponse); + Assertions.assertEquals(hasResults, genericResponse.getHasResults()); + return genericResponse; + } + + protected static class NoOpResponseErrorHandler extends DefaultResponseErrorHandler { + @Override + public void handleError(ClientHttpResponse response) throws IOException { + // do nothing + } + } + + @Configuration + @Profile("QueryServiceTest") + public static class QueryServiceTestConfiguration { + @Bean + public Module queryImplDeserializer(@Lazy ObjectMapper objectMapper) { + SimpleModule module = new SimpleModule(); + module.addDeserializer(Query.class, new JsonDeserializer<>() { + @Override + public Query deserialize(JsonParser jsonParser, DeserializationContext deserializationContext) throws IOException, JacksonException { + return objectMapper.readValue(jsonParser, QueryImpl.class); + } + }); + return module; + } + + @Bean + public HazelcastInstance hazelcastInstance() { + Config config = new Config(); + config.setClusterName(UUID.randomUUID().toString()); + return Hazelcast.newHazelcastInstance(config); + } + + @Bean + public LinkedList queryRequestEvents() { + return new LinkedList<>(); + } + + @Bean + @Primary + public ApplicationEventPublisher eventPublisher(@Lazy QueryManagementService queryManagementService, ServiceMatcher serviceMatcher) { + return new ApplicationEventPublisher() { + @Override + public void publishEvent(ApplicationEvent event) { + saveEvent(event); + processEvent(event); + } + + @Override + public void publishEvent(Object event) { + saveEvent(event); + processEvent(event); + } + + private void saveEvent(Object event) { + if (event instanceof RemoteQueryRequestEvent) { + queryRequestEvents().push(((RemoteQueryRequestEvent) event)); + } + } + + private void processEvent(Object event) { + if (event instanceof RemoteQueryRequestEvent) { + RemoteQueryRequestEvent queryEvent = (RemoteQueryRequestEvent) event; + boolean isSelfRequest = serviceMatcher.isFromSelf(queryEvent); + if (!isSelfRequest) { + queryManagementService.handleRemoteRequest(queryEvent.getRequest(), queryEvent.getOriginService(), + queryEvent.getDestinationService()); + } + } + } + }; + } + } +} diff --git a/service/src/test/java/datawave/microservice/query/QueryServiceCancelTest.java b/service/src/test/java/datawave/microservice/query/QueryServiceCancelTest.java new file mode 100644 index 00000000..77da88b9 --- /dev/null +++ b/service/src/test/java/datawave/microservice/query/QueryServiceCancelTest.java @@ -0,0 +1,494 @@ +package datawave.microservice.query; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.http.HttpMethod; +import org.springframework.http.RequestEntity; +import org.springframework.http.ResponseEntity; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit.jupiter.SpringExtension; +import org.springframework.web.util.UriComponents; + +import com.google.common.collect.Iterables; + +import datawave.microservice.authorization.service.RemoteAuthorizationServiceUserDetailsService; +import datawave.microservice.authorization.user.DatawaveUserDetails; +import datawave.microservice.query.remote.QueryRequest; +import datawave.microservice.query.storage.QueryStatus; +import datawave.webservice.result.DefaultEventQueryResponse; +import datawave.webservice.result.VoidResponse; + +@ExtendWith(SpringExtension.class) +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +@ActiveProfiles({"QueryStarterDefaults", "QueryStarterOverrides", "QueryServiceTest", RemoteAuthorizationServiceUserDetailsService.ACTIVATION_PROFILE}) +@ContextConfiguration(classes = {QueryService.class}) +public class QueryServiceCancelTest extends AbstractQueryServiceTest { + + @Test + public void testCancelSuccess() throws Exception { + DatawaveUserDetails authUser = createUserDetails(); + + // create a valid query + long currentTimeMillis = System.currentTimeMillis(); + String queryId = createQuery(authUser, createParams()); + + // cancel the query + Future> cancelFuture = cancelQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity cancelResponse = cancelFuture.get(); + + Assertions.assertEquals(200, cancelResponse.getStatusCodeValue()); + + // verify that query status was created correctly + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + + // @formatter:off + assertQueryStatus( + QueryStatus.QUERY_STATE.CANCEL, + 0, + 0, + 0, + 0, + currentTimeMillis, + queryStatus); + // @formatter:on + + // verify that the query tasks are still present + assertTasksCreated(queryId); + + // verify that the cancel event was published + Assertions.assertEquals(3, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "query:**", + QueryRequest.Method.CANCEL, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CANCEL, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @Test + public void testCancelSuccess_activeNextCall() throws Exception { + DatawaveUserDetails authUser = createUserDetails(); + + // create a valid query + long currentTimeMillis = System.currentTimeMillis(); + String queryId = createQuery(authUser, createParams()); + + // call next on the query + Future> nextFuture = nextQuery(authUser, queryId); + + boolean nextCallActive = queryStorageCache.getQueryStatus(queryId).getActiveNextCalls() > 0; + while (!nextCallActive) { + try { + nextFuture.get(500, TimeUnit.MILLISECONDS); + } catch (TimeoutException e) { + nextCallActive = queryStorageCache.getQueryStatus(queryId).getActiveNextCalls() > 0; + if ((System.currentTimeMillis() - currentTimeMillis) > TEST_WAIT_TIME_MILLIS) { + throw e; + } + } + } + + // cancel the query + Future> cancelFuture = cancelQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity cancelResponse = cancelFuture.get(); + + Assertions.assertEquals(200, cancelResponse.getStatusCodeValue()); + + // wait for the next call to drop out before checking status + // since we canceled, this should quit immediately, but just in case, we add a timeout + nextFuture.get(10, TimeUnit.SECONDS); + + // verify that query status was created correctly + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + + // @formatter:off + assertQueryStatus( + QueryStatus.QUERY_STATE.CANCEL, + 0, + 0, + 0, + 0, + currentTimeMillis, + queryStatus); + // @formatter:on + + // wait for the next call to return + nextFuture.get(); + + // verify that the query tasks are still present + assertTasksCreated(queryId); + + // verify that the close event was published + Assertions.assertEquals(4, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.NEXT, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "query:**", + QueryRequest.Method.CANCEL, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CANCEL, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @Test + public void testCancelFailure_queryNotFound() throws Exception { + DatawaveUserDetails authUser = createUserDetails(); + + String queryId = UUID.randomUUID().toString(); + + // cancel the query + Future> cancelFuture = cancelQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity cancelResponse = cancelFuture.get(); + + Assertions.assertEquals(404, cancelResponse.getStatusCodeValue()); + + // @formatter:off + assertQueryException( + "No query object matches this id. " + queryId, + "Exception with no cause caught", + "404-1", + Iterables.getOnlyElement(cancelResponse.getBody().getExceptions())); + // @formatter:on + + } + + @Test + public void testCancelFailure_ownershipFailure() throws Exception { + DatawaveUserDetails authUser = createUserDetails(); + DatawaveUserDetails altAuthUser = createAltUserDetails(); + + // create a valid query + String queryId = createQuery(authUser, createParams()); + + // make the cancel call as an alternate user asynchronously + Future> future = cancelQuery(altAuthUser, queryId); + + // the response should come back right away + ResponseEntity response = future.get(); + + Assertions.assertEquals(401, response.getStatusCodeValue()); + + // @formatter:off + assertQueryException( + "Current user does not match user that defined query. altuserdn != userdn", + "Exception with no cause caught", + "401-1", + Iterables.getOnlyElement(response.getBody().getExceptions())); + // @formatter:on + + // verify that the next events were published + Assertions.assertEquals(1, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @Test + public void testCancelFailure_queryNotRunning() throws Exception { + DatawaveUserDetails authUser = createUserDetails(); + + // create a valid query + String queryId = createQuery(authUser, createParams()); + + // cancel the query + Future> cancelFuture = cancelQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity cancelResponse = cancelFuture.get(); + + Assertions.assertEquals(200, cancelResponse.getStatusCodeValue()); + + // try to cancel the query again + cancelFuture = cancelQuery(authUser, queryId); + + // the response should come back right away + cancelResponse = cancelFuture.get(); + + Assertions.assertEquals(400, cancelResponse.getStatusCodeValue()); + + // @formatter:off + assertQueryException( + "Cannot call cancel on a query that is not running", + "Exception with no cause caught", + "400-1", + Iterables.getOnlyElement(cancelResponse.getBody().getExceptions())); + // @formatter:on + + // verify that the next events were published + Assertions.assertEquals(3, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "query:**", + QueryRequest.Method.CANCEL, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CANCEL, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @Test + public void testAdminCancelSuccess() throws Exception { + DatawaveUserDetails authUser = createUserDetails(); + DatawaveUserDetails adminUser = createAltUserDetails(Arrays.asList("AuthorizedUser", "Administrator"), null); + + // create a valid query + long currentTimeMillis = System.currentTimeMillis(); + String queryId = createQuery(authUser, createParams()); + + // cancel the query as the admin user + Future> cancelFuture = adminCancelQuery(adminUser, queryId); + + // the response should come back right away + ResponseEntity cancelResponse = cancelFuture.get(); + + Assertions.assertEquals(200, cancelResponse.getStatusCodeValue()); + + // verify that query status was created correctly + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + + // @formatter:off + assertQueryStatus( + QueryStatus.QUERY_STATE.CANCEL, + 0, + 0, + 0, + 0, + currentTimeMillis, + queryStatus); + // @formatter:on + + // verify that the query tasks are still present + assertTasksCreated(queryId); + + // verify that the cancel event was published + Assertions.assertEquals(3, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "query:**", + QueryRequest.Method.CANCEL, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CANCEL, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @Test + public void testAdminCancelFailure_notAdminUser() throws Exception { + DatawaveUserDetails authUser = createUserDetails(); + + // create a valid query + String queryId = createQuery(authUser, createParams()); + + UriComponents uri = createUri(queryId + "/adminCancel"); + RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, null, null, HttpMethod.PUT, uri); + + // cancel the query + Future> closeFuture = Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, String.class)); + + // the response should come back right away + ResponseEntity closeResponse = closeFuture.get(); + + Assertions.assertEquals(403, closeResponse.getStatusCodeValue()); + + // verify that the create event was published + Assertions.assertEquals(1, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @Test + public void testAdminCancelAllSuccess() throws Exception { + DatawaveUserDetails adminUser = createUserDetails(Arrays.asList("AuthorizedUser", "Administrator"), null); + + // create a bunch of queries + List queryIds = new ArrayList<>(); + long currentTimeMillis = System.currentTimeMillis(); + for (int i = 0; i < 10; i++) { + String queryId = createQuery(adminUser, createParams()); + mockServer.reset(); + + queryIds.add(queryId); + + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + // close all queries as the admin user + Future> cancelFuture = adminCancelAllQueries(adminUser); + + // the response should come back right away + ResponseEntity cancelResponse = cancelFuture.get(); + + Assertions.assertEquals(200, cancelResponse.getStatusCodeValue()); + + // verify that query status was created correctly + List queryStatusList = queryStorageCache.getQueryStatus(); + + for (QueryStatus queryStatus : queryStatusList) { + // @formatter:off + assertQueryStatus( + QueryStatus.QUERY_STATE.CANCEL, + 0, + 0, + 0, + 0, + currentTimeMillis, + queryStatus); + // @formatter:on + + String queryId = queryStatus.getQueryKey().getQueryId(); + + // verify that the query tasks are still present + assertTasksCreated(queryStatus.getQueryKey().getQueryId()); + + // @formatter:off + assertQueryRequestEvent( + "query:**", + QueryRequest.Method.CANCEL, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CANCEL, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + // verify that there are no more events + Assertions.assertEquals(0, queryRequestEvents.size()); + } + + @Test + public void testAdminCancelAllFailure_notAdminUser() throws Exception { + DatawaveUserDetails authUser = createUserDetails(); + + // create a bunch of queries + List queryIds = new ArrayList<>(); + long currentTimeMillis = System.currentTimeMillis(); + for (int i = 0; i < 10; i++) { + String queryId = createQuery(authUser, createParams()); + mockServer.reset(); + + queryIds.add(queryId); + + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + // cancel all queries as the admin user + UriComponents uri = createUri("/adminCancelAll"); + RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, null, null, HttpMethod.PUT, uri); + + // make the next call asynchronously + Future> cancelFuture = Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, String.class)); + + // the response should come back right away + ResponseEntity cancelResponse = cancelFuture.get(); + + Assertions.assertEquals(403, cancelResponse.getStatusCodeValue()); + + // verify that query status was created correctly + List queryStatusList = queryStorageCache.getQueryStatus(); + + // verify that none of the queries were canceled + for (QueryStatus queryStatus : queryStatusList) { + // @formatter:off + assertQueryStatus( + QueryStatus.QUERY_STATE.CREATE, + 0, + 0, + 0, + 0, + currentTimeMillis, + queryStatus); + // @formatter:on + + // verify that the query tasks are still present + assertTasksCreated(queryStatus.getQueryKey().getQueryId()); + } + + // verify that there are no more events + Assertions.assertEquals(0, queryRequestEvents.size()); + } +} diff --git a/service/src/test/java/datawave/microservice/query/QueryServiceCloseTest.java b/service/src/test/java/datawave/microservice/query/QueryServiceCloseTest.java new file mode 100644 index 00000000..b753809a --- /dev/null +++ b/service/src/test/java/datawave/microservice/query/QueryServiceCloseTest.java @@ -0,0 +1,482 @@ +package datawave.microservice.query; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.http.HttpMethod; +import org.springframework.http.RequestEntity; +import org.springframework.http.ResponseEntity; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit.jupiter.SpringExtension; +import org.springframework.util.LinkedMultiValueMap; +import org.springframework.util.MultiValueMap; +import org.springframework.web.util.UriComponents; + +import com.google.common.collect.Iterables; + +import datawave.microservice.authorization.service.RemoteAuthorizationServiceUserDetailsService; +import datawave.microservice.authorization.user.DatawaveUserDetails; +import datawave.microservice.query.remote.QueryRequest; +import datawave.microservice.query.storage.QueryStatus; +import datawave.webservice.result.DefaultEventQueryResponse; +import datawave.webservice.result.VoidResponse; + +@ExtendWith(SpringExtension.class) +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +@ActiveProfiles({"QueryStarterDefaults", "QueryStarterOverrides", "QueryServiceTest", RemoteAuthorizationServiceUserDetailsService.ACTIVATION_PROFILE}) +@ContextConfiguration(classes = {QueryService.class}) +public class QueryServiceCloseTest extends AbstractQueryServiceTest { + + @Test + public void testCloseSuccess() throws Exception { + DatawaveUserDetails authUser = createUserDetails(); + + // create a valid query + long currentTimeMillis = System.currentTimeMillis(); + String queryId = createQuery(authUser, createParams()); + + // close the query + Future> closeFuture = closeQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity closeResponse = closeFuture.get(); + + Assertions.assertEquals(200, closeResponse.getStatusCodeValue()); + + // verify that query status was created correctly + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + + // @formatter:off + assertQueryStatus( + QueryStatus.QUERY_STATE.CLOSE, + 0, + 0, + 0, + 0, + currentTimeMillis, + queryStatus); + // @formatter:on + + // verify that the query tasks are still present + assertTasksCreated(queryId); + + // verify that the close event was published + Assertions.assertEquals(2, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CLOSE, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @Test + public void testCloseSuccess_activeNextCall() throws Exception { + DatawaveUserDetails authUser = createUserDetails(); + + // create a valid query + long currentTimeMillis = System.currentTimeMillis(); + String queryId = createQuery(authUser, createParams()); + + // call next on the query + Future> nextFuture = nextQuery(authUser, queryId); + + boolean nextCallActive = queryStorageCache.getQueryStatus(queryId).getActiveNextCalls() > 0; + while (!nextCallActive) { + try { + nextFuture.get(500, TimeUnit.MILLISECONDS); + } catch (TimeoutException e) { + nextCallActive = queryStorageCache.getQueryStatus(queryId).getActiveNextCalls() > 0; + if ((System.currentTimeMillis() - currentTimeMillis) > TEST_WAIT_TIME_MILLIS) { + throw e; + } + } + } + + // close the query + Future> closeFuture = closeQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity closeResponse = closeFuture.get(); + + Assertions.assertEquals(200, closeResponse.getStatusCodeValue()); + + // verify that query status was created correctly + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + + // @formatter:off + assertQueryStatus( + QueryStatus.QUERY_STATE.CLOSE, + 0, + 0, + 1, + 0, + currentTimeMillis, + queryStatus); + // @formatter:on + + // send enough results to return a page + // pump enough results into the queue to trigger a complete page + int pageSize = queryStorageCache.getQueryStatus(queryId).getQuery().getPagesize(); + + // test field value pairings + MultiValueMap fieldValues = new LinkedMultiValueMap<>(); + fieldValues.add("LOKI", "ALLIGATOR"); + fieldValues.add("LOKI", "CLASSIC"); + + // @formatter:off + publishEventsToQueue( + queryId, + pageSize, + fieldValues, + "ALL"); + // @formatter:on + + // wait for the next call to return + nextFuture.get(); + + // verify that the query tasks are still present + assertTasksCreated(queryId); + + // verify that the close event was published + Assertions.assertEquals(3, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.NEXT, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CLOSE, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @Test + public void testCloseFailure_queryNotFound() throws Exception { + DatawaveUserDetails authUser = createUserDetails(); + + String queryId = UUID.randomUUID().toString(); + + // close the query + Future> closeFuture = closeQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity closeResponse = closeFuture.get(); + + Assertions.assertEquals(404, closeResponse.getStatusCodeValue()); + + // @formatter:off + assertQueryException( + "No query object matches this id. " + queryId, + "Exception with no cause caught", + "404-1", + Iterables.getOnlyElement(closeResponse.getBody().getExceptions())); + // @formatter:on + + } + + @Test + public void testCloseFailure_ownershipFailure() throws Exception { + DatawaveUserDetails authUser = createUserDetails(); + DatawaveUserDetails altAuthUser = createAltUserDetails(); + + // create a valid query + String queryId = createQuery(authUser, createParams()); + + // make the close call as an alternate user asynchronously + Future> future = closeQuery(altAuthUser, queryId); + + // the response should come back right away + ResponseEntity response = future.get(); + + Assertions.assertEquals(401, response.getStatusCodeValue()); + + // @formatter:off + assertQueryException( + "Current user does not match user that defined query. altuserdn != userdn", + "Exception with no cause caught", + "401-1", + Iterables.getOnlyElement(response.getBody().getExceptions())); + // @formatter:on + + // verify that the next events were published + Assertions.assertEquals(1, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @Test + public void testCloseFailure_queryNotRunning() throws Exception { + DatawaveUserDetails authUser = createUserDetails(); + + // create a valid query + String queryId = createQuery(authUser, createParams()); + + // close the query + Future> closeFuture = closeQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity closeResponse = closeFuture.get(); + + Assertions.assertEquals(200, closeResponse.getStatusCodeValue()); + + // try to close the query again + closeFuture = closeQuery(authUser, queryId); + + // the response should come back right away + closeResponse = closeFuture.get(); + + Assertions.assertEquals(400, closeResponse.getStatusCodeValue()); + + // @formatter:off + assertQueryException( + "Cannot call close on a query that is not running", + "Exception with no cause caught", + "400-1", + Iterables.getOnlyElement(closeResponse.getBody().getExceptions())); + // @formatter:on + + // verify that the next events were published + Assertions.assertEquals(2, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CLOSE, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @Test + public void testAdminCloseSuccess() throws Exception { + DatawaveUserDetails authUser = createUserDetails(); + DatawaveUserDetails adminUser = createAltUserDetails(Arrays.asList("AuthorizedUser", "Administrator"), null); + + // create a valid query + long currentTimeMillis = System.currentTimeMillis(); + String queryId = createQuery(authUser, createParams()); + + // close the query as the admin user + Future> closeFuture = adminCloseQuery(adminUser, queryId); + + // the response should come back right away + ResponseEntity closeResponse = closeFuture.get(); + + Assertions.assertEquals(200, closeResponse.getStatusCodeValue()); + + // verify that query status was created correctly + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + + // @formatter:off + assertQueryStatus( + QueryStatus.QUERY_STATE.CLOSE, + 0, + 0, + 0, + 0, + currentTimeMillis, + queryStatus); + // @formatter:on + + // verify that the query tasks are still present + assertTasksCreated(queryId); + + // verify that the close event was published + Assertions.assertEquals(2, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CLOSE, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @Test + public void testAdminCloseFailure_notAdminUser() throws Exception { + DatawaveUserDetails authUser = createUserDetails(); + + // create a valid query + long currentTimeMillis = System.currentTimeMillis(); + String queryId = createQuery(authUser, createParams()); + + UriComponents uri = createUri(queryId + "/adminClose"); + RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, null, null, HttpMethod.PUT, uri); + + // close the query + Future> closeFuture = Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, String.class)); + + // the response should come back right away + ResponseEntity closeResponse = closeFuture.get(); + + Assertions.assertEquals(403, closeResponse.getStatusCodeValue()); + + // verify that the create event was published + Assertions.assertEquals(1, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @Test + public void testAdminCloseAllSuccess() throws Exception { + DatawaveUserDetails adminUser = createUserDetails(Arrays.asList("AuthorizedUser", "Administrator"), null); + + // create a bunch of queries + long currentTimeMillis = System.currentTimeMillis(); + for (int i = 0; i < 10; i++) { + String queryId = createQuery(adminUser, createParams()); + mockServer.reset(); + + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + // close all queries as the admin user + Future> closeFuture = adminCloseAllQueries(adminUser); + + // the response should come back right away + ResponseEntity closeResponse = closeFuture.get(); + + Assertions.assertEquals(200, closeResponse.getStatusCodeValue()); + + // verify that query status was created correctly + List queryStatusList = queryStorageCache.getQueryStatus(); + + for (QueryStatus queryStatus : queryStatusList) { + // @formatter:off + assertQueryStatus( + QueryStatus.QUERY_STATE.CLOSE, + 0, + 0, + 0, + 0, + currentTimeMillis, + queryStatus); + // @formatter:on + + String queryId = queryStatus.getQueryKey().getQueryId(); + + // verify that the query tasks are still present + assertTasksCreated(queryStatus.getQueryKey().getQueryId()); + + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CLOSE, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + // verify that there are no more events + Assertions.assertEquals(0, queryRequestEvents.size()); + } + + @Test + public void testAdminCloseAllFailure_notAdminUser() throws Exception { + DatawaveUserDetails authUser = createUserDetails(); + + // create a bunch of queries + List queryIds = new ArrayList<>(); + long currentTimeMillis = System.currentTimeMillis(); + for (int i = 0; i < 10; i++) { + String queryId = createQuery(authUser, createParams()); + mockServer.reset(); + + queryIds.add(queryId); + + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + // close all queries as the admin user + UriComponents uri = createUri("/adminCloseAll"); + RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, null, null, HttpMethod.PUT, uri); + + // make the next call asynchronously + Future> closeFuture = Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, String.class)); + + // the response should come back right away + ResponseEntity closeResponse = closeFuture.get(); + + Assertions.assertEquals(403, closeResponse.getStatusCodeValue()); + + // verify that query status was created correctly + List queryStatusList = queryStorageCache.getQueryStatus(); + + // verify that none of the queries were canceled + for (QueryStatus queryStatus : queryStatusList) { + // @formatter:off + assertQueryStatus( + QueryStatus.QUERY_STATE.CREATE, + 0, + 0, + 0, + 0, + currentTimeMillis, + queryStatus); + // @formatter:on + + // verify that the query tasks are still present + assertTasksCreated(queryStatus.getQueryKey().getQueryId()); + } + + // verify that there are no more events + Assertions.assertEquals(0, queryRequestEvents.size()); + } +} diff --git a/service/src/test/java/datawave/microservice/query/QueryServiceCreateTest.java b/service/src/test/java/datawave/microservice/query/QueryServiceCreateTest.java new file mode 100644 index 00000000..cfe9a2d3 --- /dev/null +++ b/service/src/test/java/datawave/microservice/query/QueryServiceCreateTest.java @@ -0,0 +1,466 @@ +package datawave.microservice.query; + +import static datawave.microservice.query.QueryImpl.BEGIN_DATE; +import static datawave.microservice.query.QueryParameters.QUERY_MAX_CONCURRENT_TASKS; +import static datawave.microservice.query.QueryParameters.QUERY_MAX_RESULTS_OVERRIDE; +import static datawave.microservice.query.QueryParameters.QUERY_PAGESIZE; +import static datawave.webservice.common.audit.AuditParameters.QUERY_STRING; + +import java.io.IOException; +import java.text.ParseException; +import java.util.Arrays; +import java.util.Collections; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.http.HttpMethod; +import org.springframework.http.HttpStatus; +import org.springframework.http.RequestEntity; +import org.springframework.http.ResponseEntity; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit.jupiter.SpringExtension; +import org.springframework.util.MultiValueMap; +import org.springframework.web.util.UriComponents; + +import datawave.marking.ColumnVisibilitySecurityMarking; +import datawave.microservice.authorization.service.RemoteAuthorizationServiceUserDetailsService; +import datawave.microservice.authorization.user.DatawaveUserDetails; +import datawave.microservice.query.remote.QueryRequest; +import datawave.microservice.query.storage.QueryStatus; +import datawave.webservice.query.exception.QueryExceptionType; +import datawave.webservice.result.BaseResponse; +import datawave.webservice.result.GenericResponse; + +@ExtendWith(SpringExtension.class) +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +@ActiveProfiles({"QueryStarterDefaults", "QueryStarterOverrides", "QueryServiceTest", RemoteAuthorizationServiceUserDetailsService.ACTIVATION_PROFILE}) +@ContextConfiguration(classes = {QueryService.class}) +public class QueryServiceCreateTest extends AbstractQueryServiceTest { + + @Test + public void testCreateSuccess() throws ParseException, IOException { + DatawaveUserDetails authUser = createUserDetails(); + UriComponents uri = createUri("EventQuery/create"); + MultiValueMap map = createParams(); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); + + // setup a mock audit service + auditSentSetup(); + + long currentTimeMillis = System.currentTimeMillis(); + ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, GenericResponse.class); + // @formatter:off + GenericResponse genericResponse = assertGenericResponse( + true, + HttpStatus.Series.SUCCESSFUL, + resp); + // @formatter:on + + // verify that a query id was returned + String queryId = genericResponse.getResult(); + Assertions.assertNotNull(queryId); + + // verify that the create event was published + Assertions.assertEquals(1, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + + // verify that query status was created correctly + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + + // @formatter:off + assertQueryStatus( + QueryStatus.QUERY_STATE.CREATE, + 0, + 0, + 0, + 0, + currentTimeMillis, + queryStatus); + // @formatter:on + + // verify that they query was created correctly + Query query = queryStatus.getQuery(); + + // @formatter:off + assertQuery( + TEST_QUERY_STRING, + TEST_QUERY_NAME, + TEST_QUERY_AUTHORIZATIONS, + TEST_QUERY_BEGIN, + TEST_QUERY_END, + TEST_VISIBILITY_MARKING, + query); + // @formatter:on + + // verify that an audit message was sent and the the audit id matches the query id + assertAuditSent(queryId); + + // verify that query tasks were created + assertTasksCreated(queryId); + } + + @Test + public void testCreateFailure_paramValidation() { + DatawaveUserDetails authUser = createUserDetails(); + UriComponents uri = createUri("EventQuery/create"); + MultiValueMap map = createParams(); + + // remove the query param to induce a parameter validation failure + map.remove(QUERY_STRING); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); + + // setup a mock audit service + auditNotSentSetup(); + + ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, GenericResponse.class); + + // @formatter:off + BaseResponse baseResponse = assertBaseResponse( + false, + HttpStatus.Series.CLIENT_ERROR, + resp); + // @formatter:on + + // verify that there is no result + Assertions.assertFalse(baseResponse.getHasResults()); + + // verify that an exception was returned + Assertions.assertEquals(1, baseResponse.getExceptions().size()); + + QueryExceptionType queryException = baseResponse.getExceptions().get(0); + // @formatter:off + assertQueryException( + "Missing one or more required QueryParameters", + "java.lang.IllegalArgumentException: Missing one or more required QueryParameters", + "400-1", + queryException); + // @formatter:on + + // verify that there are no query statuses + Assertions.assertTrue(queryStorageCache.getQueryStatus().isEmpty()); + + // verify that no audit message was sent + assertAuditNotSent(); + } + + @Test + public void testCreateFailure_authValidation() { + DatawaveUserDetails authUser = createUserDetails(Collections.singleton("AuthorizedUser"), Collections.emptyList()); + UriComponents uri = createUri("EventQuery/create"); + MultiValueMap map = createParams(); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); + + // setup a mock audit service + auditSentSetup(); + + ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, GenericResponse.class); + + // @formatter:off + BaseResponse baseResponse = assertBaseResponse( + false, + HttpStatus.Series.CLIENT_ERROR, + resp); + // @formatter:on + + // verify that there is no result + Assertions.assertFalse(baseResponse.getHasResults()); + + // verify that an exception was returned + Assertions.assertEquals(1, baseResponse.getExceptions().size()); + + QueryExceptionType queryException = baseResponse.getExceptions().get(0); + // @formatter:off + assertQueryException( + "java.lang.IllegalArgumentException: User requested authorizations that they don't have. Missing: [ALL], Requested: [ALL], User: []", + "datawave.security.authorization.AuthorizationException: java.lang.IllegalArgumentException: User requested authorizations that they don't have. Missing: [ALL], Requested: [ALL], User: []", + "400-1", + queryException); + // @formatter:on + + // verify that there are no query statuses + Assertions.assertTrue(queryStorageCache.getQueryStatus().isEmpty()); + + // verify that no audit message was sent + assertAuditSent(null); + } + + @Test + public void testCreateFailure_queryLogicValidation() { + DatawaveUserDetails authUser = createUserDetails(); + UriComponents uri = createUri("EventQuery/create"); + MultiValueMap map = createParams(); + + // remove the beginDate param to induce a query logic validation failure + map.remove(BEGIN_DATE); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); + + // setup a mock audit service + auditNotSentSetup(); + + ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, GenericResponse.class); + + // @formatter:off + BaseResponse baseResponse = assertBaseResponse( + false, + HttpStatus.Series.CLIENT_ERROR, + resp); + // @formatter:on + + // verify that there is no result + Assertions.assertFalse(baseResponse.getHasResults()); + + // verify that an exception was returned + Assertions.assertEquals(1, baseResponse.getExceptions().size()); + + QueryExceptionType queryException = baseResponse.getExceptions().get(0); + // @formatter:off + assertQueryException( + "Required parameter begin not found", + "java.lang.IllegalArgumentException: Required parameter begin not found", + "400-1", + queryException); + // @formatter:on + + // verify that there are no query statuses + Assertions.assertTrue(queryStorageCache.getQueryStatus().isEmpty()); + + // verify that no audit message was sent + assertAuditNotSent(); + } + + @Test + public void testCreateFailure_maxPageSize() { + DatawaveUserDetails authUser = createUserDetails(Arrays.asList("AuthorizedUser", queryProperties.getPrivilegedRole()), null); + UriComponents uri = createUri("EventQuery/create"); + MultiValueMap map = createParams(); + + // set an invalid page size override + map.set(QUERY_PAGESIZE, Integer.toString(Integer.MAX_VALUE)); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); + + // setup a mock audit service + auditNotSentSetup(); + + ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, GenericResponse.class); + + // @formatter:off + BaseResponse baseResponse = assertBaseResponse( + false, + HttpStatus.Series.CLIENT_ERROR, + resp); + // @formatter:on + + // verify that there is no result + Assertions.assertFalse(baseResponse.getHasResults()); + + // verify that an exception was returned + Assertions.assertEquals(1, baseResponse.getExceptions().size()); + + QueryExceptionType queryException = baseResponse.getExceptions().get(0); + // @formatter:off + assertQueryException( + "Page size is larger than configured max. Max = 10,000.", + "Exception with no cause caught", + "400-6", + queryException); + // @formatter:on + + // verify that there are no query statuses + Assertions.assertTrue(queryStorageCache.getQueryStatus().isEmpty()); + + // verify that no audit message was sent + assertAuditNotSent(); + } + + @Test + public void testCreateFailure_maxResultsOverride() { + DatawaveUserDetails authUser = createUserDetails(); + UriComponents uri = createUri("EventQuery/create"); + MultiValueMap map = createParams(); + + // set an invalid max results override + map.set(QUERY_MAX_RESULTS_OVERRIDE, Long.toString(Long.MAX_VALUE)); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); + + // setup a mock audit service + auditNotSentSetup(); + + ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, GenericResponse.class); + + // @formatter:off + BaseResponse baseResponse = assertBaseResponse( + false, + HttpStatus.Series.CLIENT_ERROR, + resp); + // @formatter:on + + // verify that there is no result + Assertions.assertFalse(baseResponse.getHasResults()); + + // verify that an exception was returned + Assertions.assertEquals(1, baseResponse.getExceptions().size()); + + QueryExceptionType queryException = baseResponse.getExceptions().get(0); + // @formatter:off + assertQueryException( + "Invalid max results override value. Max = 369.", + "Exception with no cause caught", + "400-43", + queryException); + // @formatter:on + + // verify that there are no query statuses + Assertions.assertTrue(queryStorageCache.getQueryStatus().isEmpty()); + + // verify that no audit message was sent + assertAuditNotSent(); + } + + @Test + public void testCreateFailure_maxConcurrentTasksOverride() { + DatawaveUserDetails authUser = createUserDetails(); + UriComponents uri = createUri("EventQuery/create"); + MultiValueMap map = createParams(); + + // add an invalid max results override + map.set(QUERY_MAX_CONCURRENT_TASKS, Integer.toString(Integer.MAX_VALUE)); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); + + // setup a mock audit service + auditNotSentSetup(); + + ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, GenericResponse.class); + + // @formatter:off + BaseResponse baseResponse = assertBaseResponse( + false, + HttpStatus.Series.CLIENT_ERROR, + resp); + // @formatter:on + + // verify that there is no result + Assertions.assertFalse(baseResponse.getHasResults()); + + // verify that an exception was returned + Assertions.assertEquals(1, baseResponse.getExceptions().size()); + + QueryExceptionType queryException = baseResponse.getExceptions().get(0); + // @formatter:off + assertQueryException( + "Invalid max concurrent tasks override value. Max = 10.", + "Exception with no cause caught", + "400-44", + queryException); + // @formatter:on + + // verify that there are no query statuses + Assertions.assertTrue(queryStorageCache.getQueryStatus().isEmpty()); + + // verify that no audit message was sent + assertAuditNotSent(); + } + + @Test + public void testCreateFailure_roleValidation() { + // create a user without the required role + DatawaveUserDetails authUser = createUserDetails(Collections.emptyList(), Collections.singletonList("ALL")); + UriComponents uri = createUri("EventQuery/create"); + MultiValueMap map = createParams(); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); + + // setup a mock audit service + auditNotSentSetup(); + + ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, GenericResponse.class); + + // @formatter:off + BaseResponse baseResponse = assertBaseResponse( + false, + HttpStatus.Series.CLIENT_ERROR, + resp); + // @formatter:on + + // verify that there is no result + Assertions.assertFalse(baseResponse.getHasResults()); + + // verify that an exception was returned + Assertions.assertEquals(1, baseResponse.getExceptions().size()); + + QueryExceptionType queryException = baseResponse.getExceptions().get(0); + // @formatter:off + assertQueryException( + "User does not have the required roles.", + "datawave.webservice.query.exception.UnauthorizedQueryException: User does not have the required roles.", + "400-5", + queryException); + // @formatter:on + + // verify that there are no query statuses + Assertions.assertTrue(queryStorageCache.getQueryStatus().isEmpty()); + + // verify that no audit message was sent + assertAuditNotSent(); + } + + @Test + public void testCreateFailure_markingValidation() { + DatawaveUserDetails authUser = createUserDetails(); + UriComponents uri = createUri("EventQuery/create"); + MultiValueMap map = createParams(); + + // remove the column visibility param to induce a security marking validation failure + map.remove(ColumnVisibilitySecurityMarking.VISIBILITY_MARKING); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); + + // setup a mock audit service + auditNotSentSetup(); + + ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, GenericResponse.class); + + // @formatter:off + BaseResponse baseResponse = assertBaseResponse( + false, + HttpStatus.Series.CLIENT_ERROR, + resp); + // @formatter:on + + // verify that there is no result + Assertions.assertFalse(baseResponse.getHasResults()); + + // verify that an exception was returned + Assertions.assertEquals(1, baseResponse.getExceptions().size()); + + QueryExceptionType queryException = baseResponse.getExceptions().get(0); + // @formatter:off + assertQueryException( + "Required parameter columnVisibility not found", + "java.lang.IllegalArgumentException: Required parameter columnVisibility not found", + "400-4", + queryException); + // @formatter:on + + // verify that there are no query statuses + Assertions.assertTrue(queryStorageCache.getQueryStatus().isEmpty()); + + // verify that no audit message was sent + assertAuditNotSent(); + } +} diff --git a/service/src/test/java/datawave/microservice/query/QueryServiceDefineTest.java b/service/src/test/java/datawave/microservice/query/QueryServiceDefineTest.java new file mode 100644 index 00000000..1970a2b6 --- /dev/null +++ b/service/src/test/java/datawave/microservice/query/QueryServiceDefineTest.java @@ -0,0 +1,406 @@ +package datawave.microservice.query; + +import static datawave.microservice.query.QueryImpl.BEGIN_DATE; +import static datawave.microservice.query.QueryParameters.QUERY_MAX_CONCURRENT_TASKS; +import static datawave.microservice.query.QueryParameters.QUERY_MAX_RESULTS_OVERRIDE; +import static datawave.microservice.query.QueryParameters.QUERY_PAGESIZE; +import static datawave.webservice.common.audit.AuditParameters.QUERY_STRING; + +import java.io.IOException; +import java.text.ParseException; +import java.util.Arrays; +import java.util.Collections; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.http.HttpMethod; +import org.springframework.http.HttpStatus; +import org.springframework.http.RequestEntity; +import org.springframework.http.ResponseEntity; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit.jupiter.SpringExtension; +import org.springframework.util.MultiValueMap; +import org.springframework.web.util.UriComponents; + +import datawave.marking.ColumnVisibilitySecurityMarking; +import datawave.microservice.authorization.service.RemoteAuthorizationServiceUserDetailsService; +import datawave.microservice.authorization.user.DatawaveUserDetails; +import datawave.microservice.query.storage.QueryStatus; +import datawave.webservice.query.exception.QueryExceptionType; +import datawave.webservice.result.BaseResponse; +import datawave.webservice.result.GenericResponse; + +@ExtendWith(SpringExtension.class) +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +@ActiveProfiles({"QueryStarterDefaults", "QueryStarterOverrides", "QueryServiceTest", RemoteAuthorizationServiceUserDetailsService.ACTIVATION_PROFILE}) +@ContextConfiguration(classes = {QueryService.class}) +public class QueryServiceDefineTest extends AbstractQueryServiceTest { + + @Test + public void testDefineSuccess() throws ParseException, IOException { + DatawaveUserDetails authUser = createUserDetails(); + UriComponents uri = createUri("EventQuery/define"); + MultiValueMap map = createParams(); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); + + // setup a mock audit service + auditNotSentSetup(); + + long currentTimeMillis = System.currentTimeMillis(); + ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, GenericResponse.class); + // @formatter:off + GenericResponse genericResponse = assertGenericResponse( + false, + HttpStatus.Series.SUCCESSFUL, + resp); + // @formatter:on + + // verify that a query id was returned + String queryId = genericResponse.getResult(); + Assertions.assertNotNull(queryId); + + // verify that query status was created correctly + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + + // @formatter:off + assertQueryStatus( + QueryStatus.QUERY_STATE.DEFINE, + 0, + 0, + 0, + 0, + currentTimeMillis, + queryStatus); + // @formatter:on + + // verify that they query was created correctly + Query query = queryStatus.getQuery(); + // @formatter:off + assertQuery( + TEST_QUERY_STRING, + TEST_QUERY_NAME, + TEST_QUERY_AUTHORIZATIONS, + TEST_QUERY_BEGIN, + TEST_QUERY_END, + TEST_VISIBILITY_MARKING, + query); + // @formatter:on + + // verify that no audit message was sent + assertAuditNotSent(); + + // verify that query tasks weren't created + assertTasksNotCreated(queryId); + } + + @Test + public void testDefineFailure_paramValidation() { + DatawaveUserDetails authUser = createUserDetails(); + UriComponents uri = createUri("EventQuery/define"); + MultiValueMap map = createParams(); + + // remove the query param to induce a parameter validation failure + map.remove(QUERY_STRING); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); + + ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, GenericResponse.class); + + // @formatter:off + BaseResponse baseResponse = assertBaseResponse( + false, + HttpStatus.Series.CLIENT_ERROR, + resp); + // @formatter:on + + // verify that there is no result + Assertions.assertFalse(baseResponse.getHasResults()); + + // verify that an exception was returned + Assertions.assertEquals(1, baseResponse.getExceptions().size()); + + QueryExceptionType queryException = baseResponse.getExceptions().get(0); + // @formatter:off + assertQueryException( + "Missing one or more required QueryParameters", + "java.lang.IllegalArgumentException: Missing one or more required QueryParameters", + "400-1", + queryException); + // @formatter:on + + // verify that there are no query statuses + Assertions.assertTrue(queryStorageCache.getQueryStatus().isEmpty()); + } + + @Test + public void testDefineFailure_authValidation() { + DatawaveUserDetails authUser = createUserDetails(Collections.singleton("AuthorizedUser"), Collections.emptyList()); + UriComponents uri = createUri("EventQuery/define"); + MultiValueMap map = createParams(); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); + + ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, GenericResponse.class); + + // @formatter:off + BaseResponse baseResponse = assertBaseResponse( + false, + HttpStatus.Series.CLIENT_ERROR, + resp); + // @formatter:on + + // verify that there is no result + Assertions.assertFalse(baseResponse.getHasResults()); + + // verify that an exception was returned + Assertions.assertEquals(1, baseResponse.getExceptions().size()); + + QueryExceptionType queryException = baseResponse.getExceptions().get(0); + // @formatter:off + assertQueryException( + "java.lang.IllegalArgumentException: User requested authorizations that they don't have. Missing: [ALL], Requested: [ALL], User: []", + "datawave.security.authorization.AuthorizationException: java.lang.IllegalArgumentException: User requested authorizations that they don't have. Missing: [ALL], Requested: [ALL], User: []", + "400-1", + queryException); + // @formatter:on + + // verify that there are no query statuses + Assertions.assertTrue(queryStorageCache.getQueryStatus().isEmpty()); + } + + @Test + public void testDefineFailure_queryLogicValidation() { + DatawaveUserDetails authUser = createUserDetails(); + UriComponents uri = createUri("EventQuery/define"); + MultiValueMap map = createParams(); + + // remove the beginDate param to induce a query logic validation failure + map.remove(BEGIN_DATE); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); + + ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, GenericResponse.class); + + // @formatter:off + BaseResponse baseResponse = assertBaseResponse( + false, + HttpStatus.Series.CLIENT_ERROR, + resp); + // @formatter:on + + // verify that there is no result + Assertions.assertFalse(baseResponse.getHasResults()); + + // verify that an exception was returned + Assertions.assertEquals(1, baseResponse.getExceptions().size()); + + QueryExceptionType queryException = baseResponse.getExceptions().get(0); + // @formatter:off + assertQueryException( + "Required parameter begin not found", + "java.lang.IllegalArgumentException: Required parameter begin not found", + "400-1", + queryException); + // @formatter:on + + // verify that there are no query statuses + Assertions.assertTrue(queryStorageCache.getQueryStatus().isEmpty()); + } + + @Test + public void testDefineFailure_maxPageSize() { + DatawaveUserDetails authUser = createUserDetails(Arrays.asList("AuthorizedUser", queryProperties.getPrivilegedRole()), null); + UriComponents uri = createUri("EventQuery/define"); + MultiValueMap map = createParams(); + + // set an invalid page size override + map.set(QUERY_PAGESIZE, Integer.toString(Integer.MAX_VALUE)); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); + + ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, GenericResponse.class); + + // @formatter:off + BaseResponse baseResponse = assertBaseResponse( + false, + HttpStatus.Series.CLIENT_ERROR, + resp); + // @formatter:on + + // verify that there is no result + Assertions.assertFalse(baseResponse.getHasResults()); + + // verify that an exception was returned + Assertions.assertEquals(1, baseResponse.getExceptions().size()); + + QueryExceptionType queryException = baseResponse.getExceptions().get(0); + // @formatter:off + assertQueryException( + "Page size is larger than configured max. Max = 10,000.", + "Exception with no cause caught", + "400-6", + queryException); + // @formatter:on + + // verify that there are no query statuses + Assertions.assertTrue(queryStorageCache.getQueryStatus().isEmpty()); + } + + @Test + public void testDefineFailure_maxResultsOverride() { + DatawaveUserDetails authUser = createUserDetails(); + UriComponents uri = createUri("EventQuery/define"); + MultiValueMap map = createParams(); + + // set an invalid max results override + map.set(QUERY_MAX_RESULTS_OVERRIDE, Long.toString(Long.MAX_VALUE)); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); + + ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, GenericResponse.class); + + // @formatter:off + BaseResponse baseResponse = assertBaseResponse( + false, + HttpStatus.Series.CLIENT_ERROR, + resp); + // @formatter:on + + // verify that there is no result + Assertions.assertFalse(baseResponse.getHasResults()); + + // verify that an exception was returned + Assertions.assertEquals(1, baseResponse.getExceptions().size()); + + QueryExceptionType queryException = baseResponse.getExceptions().get(0); + // @formatter:off + assertQueryException( + "Invalid max results override value. Max = 369.", + "Exception with no cause caught", + "400-43", + queryException); + // @formatter:on + + // verify that there are no query statuses + Assertions.assertTrue(queryStorageCache.getQueryStatus().isEmpty()); + } + + @Test + public void testDefineFailure_maxConcurrentTasksOverride() { + DatawaveUserDetails authUser = createUserDetails(); + UriComponents uri = createUri("EventQuery/define"); + MultiValueMap map = createParams(); + + // add an invalid max results override + map.set(QUERY_MAX_CONCURRENT_TASKS, Integer.toString(Integer.MAX_VALUE)); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); + + ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, GenericResponse.class); + + // @formatter:off + BaseResponse baseResponse = assertBaseResponse( + false, + HttpStatus.Series.CLIENT_ERROR, + resp); + // @formatter:on + + // verify that there is no result + Assertions.assertFalse(baseResponse.getHasResults()); + + // verify that an exception was returned + Assertions.assertEquals(1, baseResponse.getExceptions().size()); + + QueryExceptionType queryException = baseResponse.getExceptions().get(0); + // @formatter:off + assertQueryException( + "Invalid max concurrent tasks override value. Max = 10.", + "Exception with no cause caught", + "400-44", + queryException); + // @formatter:on + + // verify that there are no query statuses + Assertions.assertTrue(queryStorageCache.getQueryStatus().isEmpty()); + } + + @Test + public void testDefineFailure_roleValidation() { + // create a user without the required role + DatawaveUserDetails authUser = createUserDetails(Collections.emptyList(), Collections.singletonList("ALL")); + UriComponents uri = createUri("EventQuery/define"); + MultiValueMap map = createParams(); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); + + ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, GenericResponse.class); + + // @formatter:off + BaseResponse baseResponse = assertBaseResponse( + false, + HttpStatus.Series.CLIENT_ERROR, + resp); + // @formatter:on + + // verify that there is no result + Assertions.assertFalse(baseResponse.getHasResults()); + + // verify that an exception was returned + Assertions.assertEquals(1, baseResponse.getExceptions().size()); + + QueryExceptionType queryException = baseResponse.getExceptions().get(0); + // @formatter:off + assertQueryException( + "User does not have the required roles.", + "datawave.webservice.query.exception.UnauthorizedQueryException: User does not have the required roles.", + "400-5", + queryException); + // @formatter:on + + // verify that there are no query statuses + Assertions.assertTrue(queryStorageCache.getQueryStatus().isEmpty()); + } + + @Test + public void testDefineFailure_markingValidation() { + DatawaveUserDetails authUser = createUserDetails(); + UriComponents uri = createUri("EventQuery/define"); + MultiValueMap map = createParams(); + + // remove the column visibility param to induce a security marking validation failure + map.remove(ColumnVisibilitySecurityMarking.VISIBILITY_MARKING); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); + + ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, GenericResponse.class); + + // @formatter:off + BaseResponse baseResponse = assertBaseResponse( + false, + HttpStatus.Series.CLIENT_ERROR, + resp); + // @formatter:on + + // verify that there is no result + Assertions.assertFalse(baseResponse.getHasResults()); + + // verify that an exception was returned + Assertions.assertEquals(1, baseResponse.getExceptions().size()); + + QueryExceptionType queryException = baseResponse.getExceptions().get(0); + // @formatter:off + assertQueryException( + "Required parameter columnVisibility not found", + "java.lang.IllegalArgumentException: Required parameter columnVisibility not found", + "400-4", + queryException); + // @formatter:on + + // verify that there are no query statuses + Assertions.assertTrue(queryStorageCache.getQueryStatus().isEmpty()); + } +} diff --git a/service/src/test/java/datawave/microservice/query/QueryServiceDuplicateTest.java b/service/src/test/java/datawave/microservice/query/QueryServiceDuplicateTest.java new file mode 100644 index 00000000..ba00b7bd --- /dev/null +++ b/service/src/test/java/datawave/microservice/query/QueryServiceDuplicateTest.java @@ -0,0 +1,581 @@ +package datawave.microservice.query; + +import static datawave.microservice.query.QueryImpl.BEGIN_DATE; +import static datawave.microservice.query.QueryImpl.END_DATE; +import static datawave.microservice.query.QueryImpl.QUERY; +import static datawave.microservice.query.QueryParameters.QUERY_LOGIC_NAME; +import static datawave.webservice.common.audit.AuditParameters.QUERY_AUTHORIZATIONS; + +import java.util.Arrays; +import java.util.UUID; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.http.HttpMethod; +import org.springframework.http.RequestEntity; +import org.springframework.http.ResponseEntity; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit.jupiter.SpringExtension; +import org.springframework.util.LinkedMultiValueMap; +import org.springframework.util.MultiValueMap; +import org.springframework.web.util.UriComponents; + +import com.google.common.collect.Iterables; + +import datawave.microservice.authorization.service.RemoteAuthorizationServiceUserDetailsService; +import datawave.microservice.authorization.user.DatawaveUserDetails; +import datawave.microservice.query.remote.QueryRequest; +import datawave.microservice.query.storage.QueryStatus; +import datawave.webservice.result.GenericResponse; +import datawave.webservice.result.VoidResponse; + +@ExtendWith(SpringExtension.class) +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +@ActiveProfiles({"QueryStarterDefaults", "QueryStarterOverrides", "QueryServiceTest", RemoteAuthorizationServiceUserDetailsService.ACTIVATION_PROFILE}) +@ContextConfiguration(classes = {QueryService.class}) +public class QueryServiceDuplicateTest extends AbstractQueryServiceTest { + + @Test + public void testDuplicateSuccess_duplicateOnDefined() throws Exception { + DatawaveUserDetails authUser = createUserDetails(); + + // define a valid query + long currentTimeMillis = System.currentTimeMillis(); + String queryId = defineQuery(authUser, createParams()); + + MultiValueMap updateParams = new LinkedMultiValueMap<>(); + + mockServer.reset(); + auditSentSetup(); + + // duplicate the query + Future> duplicateFuture = duplicateQuery(authUser, queryId, updateParams); + + // the response should come back right away + ResponseEntity response = duplicateFuture.get(); + + Assertions.assertEquals(200, response.getStatusCodeValue()); + + String dupeQueryId = (String) response.getBody().getResult(); + + // make sure an audit message was sent + assertAuditSent(dupeQueryId); + + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + // @formatter:off + assertQueryStatus( + QueryStatus.QUERY_STATE.DEFINE, + 0, + 0, + 0, + 0, + currentTimeMillis, + queryStatus); + // @formatter:on + + QueryStatus dupeQueryStatus = queryStorageCache.getQueryStatus(dupeQueryId); + // @formatter:off + assertQueryStatus( + QueryStatus.QUERY_STATE.CREATE, + 0, + 0, + 0, + 0, + currentTimeMillis, + dupeQueryStatus); + // @formatter:on + + // make sure the queries are identical + Assertions.assertEquals(queryStatus.getQuery().getQuery(), dupeQueryStatus.getQuery().getQuery()); + Assertions.assertEquals(queryStatus.getQuery().getQueryAuthorizations(), dupeQueryStatus.getQuery().getQueryAuthorizations()); + Assertions.assertEquals(DefaultQueryParameters.formatDate(queryStatus.getQuery().getBeginDate()), + DefaultQueryParameters.formatDate(dupeQueryStatus.getQuery().getBeginDate())); + Assertions.assertEquals(DefaultQueryParameters.formatDate(queryStatus.getQuery().getEndDate()), + DefaultQueryParameters.formatDate(dupeQueryStatus.getQuery().getEndDate())); + Assertions.assertEquals(queryStatus.getQuery().getQueryLogicName(), dupeQueryStatus.getQuery().getQueryLogicName()); + Assertions.assertEquals(queryStatus.getQuery().getPagesize(), dupeQueryStatus.getQuery().getPagesize()); + + // verify that no events were published + Assertions.assertEquals(1, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + dupeQueryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @Test + public void testDuplicateSuccess_duplicateOnCreated() throws Exception { + DatawaveUserDetails authUser = createUserDetails(); + + // create a valid query + long currentTimeMillis = System.currentTimeMillis(); + String queryId = createQuery(authUser, createParams()); + + MultiValueMap updateParams = new LinkedMultiValueMap<>(); + + mockServer.reset(); + auditSentSetup(); + + // duplicate the query + Future> duplicateFuture = duplicateQuery(authUser, queryId, updateParams); + + // the response should come back right away + ResponseEntity response = duplicateFuture.get(); + + Assertions.assertEquals(200, response.getStatusCodeValue()); + + String dupeQueryId = (String) response.getBody().getResult(); + + // make sure an audit message was sent + assertAuditSent(dupeQueryId); + + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + // @formatter:off + assertQueryStatus( + QueryStatus.QUERY_STATE.CREATE, + 0, + 0, + 0, + 0, + currentTimeMillis, + queryStatus); + // @formatter:on + + QueryStatus dupeQueryStatus = queryStorageCache.getQueryStatus(dupeQueryId); + // @formatter:off + assertQueryStatus( + QueryStatus.QUERY_STATE.CREATE, + 0, + 0, + 0, + 0, + currentTimeMillis, + dupeQueryStatus); + // @formatter:on + + // make sure the queries are identical + Assertions.assertEquals(queryStatus.getQuery().getQuery(), dupeQueryStatus.getQuery().getQuery()); + Assertions.assertEquals(queryStatus.getQuery().getQueryAuthorizations(), dupeQueryStatus.getQuery().getQueryAuthorizations()); + Assertions.assertEquals(DefaultQueryParameters.formatDate(queryStatus.getQuery().getBeginDate()), + DefaultQueryParameters.formatDate(dupeQueryStatus.getQuery().getBeginDate())); + Assertions.assertEquals(DefaultQueryParameters.formatDate(queryStatus.getQuery().getEndDate()), + DefaultQueryParameters.formatDate(dupeQueryStatus.getQuery().getEndDate())); + Assertions.assertEquals(queryStatus.getQuery().getQueryLogicName(), dupeQueryStatus.getQuery().getQueryLogicName()); + Assertions.assertEquals(queryStatus.getQuery().getPagesize(), dupeQueryStatus.getQuery().getPagesize()); + + // verify that no events were published + Assertions.assertEquals(2, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + dupeQueryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @Test + public void testDuplicateSuccess_duplicateOnCanceled() throws Exception { + DatawaveUserDetails authUser = createUserDetails(); + + // create a valid query + long currentTimeMillis = System.currentTimeMillis(); + String queryId = createQuery(authUser, createParams()); + + // cancel the query + Future> cancelFuture = cancelQuery(authUser, queryId); + + // this should return immediately + ResponseEntity cancelResponse = cancelFuture.get(); + + Assertions.assertEquals(200, cancelResponse.getStatusCodeValue()); + + MultiValueMap updateParams = new LinkedMultiValueMap<>(); + + mockServer.reset(); + auditSentSetup(); + + // duplicate the query + Future> duplicateFuture = duplicateQuery(authUser, queryId, updateParams); + + // the response should come back right away + ResponseEntity response = duplicateFuture.get(); + + Assertions.assertEquals(200, response.getStatusCodeValue()); + + String dupeQueryId = (String) response.getBody().getResult(); + + // make sure an audit message was sent + assertAuditSent(dupeQueryId); + + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + // @formatter:off + assertQueryStatus( + QueryStatus.QUERY_STATE.CANCEL, + 0, + 0, + 0, + 0, + currentTimeMillis, + queryStatus); + // @formatter:on + + QueryStatus dupeQueryStatus = queryStorageCache.getQueryStatus(dupeQueryId); + // @formatter:off + assertQueryStatus( + QueryStatus.QUERY_STATE.CREATE, + 0, + 0, + 0, + 0, + currentTimeMillis, + dupeQueryStatus); + // @formatter:on + + // make sure the queries are identical + Assertions.assertEquals(queryStatus.getQuery().getQuery(), dupeQueryStatus.getQuery().getQuery()); + Assertions.assertEquals(queryStatus.getQuery().getQueryAuthorizations(), dupeQueryStatus.getQuery().getQueryAuthorizations()); + Assertions.assertEquals(DefaultQueryParameters.formatDate(queryStatus.getQuery().getBeginDate()), + DefaultQueryParameters.formatDate(dupeQueryStatus.getQuery().getBeginDate())); + Assertions.assertEquals(DefaultQueryParameters.formatDate(queryStatus.getQuery().getEndDate()), + DefaultQueryParameters.formatDate(dupeQueryStatus.getQuery().getEndDate())); + Assertions.assertEquals(queryStatus.getQuery().getQueryLogicName(), dupeQueryStatus.getQuery().getQueryLogicName()); + Assertions.assertEquals(queryStatus.getQuery().getPagesize(), dupeQueryStatus.getQuery().getPagesize()); + + // verify that no events were published + Assertions.assertEquals(4, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "query:**", + QueryRequest.Method.CANCEL, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CANCEL, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + dupeQueryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @Test + public void testDuplicateSuccess_duplicateOnClosed() throws Exception { + DatawaveUserDetails authUser = createUserDetails(); + + // create a valid query + long currentTimeMillis = System.currentTimeMillis(); + String queryId = createQuery(authUser, createParams()); + + // close the query + Future> closeFuture = closeQuery(authUser, queryId); + + // this should return immediately + ResponseEntity closeResponse = closeFuture.get(); + + Assertions.assertEquals(200, closeResponse.getStatusCodeValue()); + + MultiValueMap updateParams = new LinkedMultiValueMap<>(); + + mockServer.reset(); + auditSentSetup(); + + // duplicate the query + Future> duplicateFuture = duplicateQuery(authUser, queryId, updateParams); + + // the response should come back right away + ResponseEntity response = duplicateFuture.get(); + + Assertions.assertEquals(200, response.getStatusCodeValue()); + + String dupeQueryId = (String) response.getBody().getResult(); + + // make sure an audit message was sent + assertAuditSent(dupeQueryId); + + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + // @formatter:off + assertQueryStatus( + QueryStatus.QUERY_STATE.CLOSE, + 0, + 0, + 0, + 0, + currentTimeMillis, + queryStatus); + // @formatter:on + + QueryStatus dupeQueryStatus = queryStorageCache.getQueryStatus(dupeQueryId); + // @formatter:off + assertQueryStatus( + QueryStatus.QUERY_STATE.CREATE, + 0, + 0, + 0, + 0, + currentTimeMillis, + dupeQueryStatus); + // @formatter:on + + // make sure the queries are identical + Assertions.assertEquals(queryStatus.getQuery().getQuery(), dupeQueryStatus.getQuery().getQuery()); + Assertions.assertEquals(queryStatus.getQuery().getQueryAuthorizations(), dupeQueryStatus.getQuery().getQueryAuthorizations()); + Assertions.assertEquals(DefaultQueryParameters.formatDate(queryStatus.getQuery().getBeginDate()), + DefaultQueryParameters.formatDate(dupeQueryStatus.getQuery().getBeginDate())); + Assertions.assertEquals(DefaultQueryParameters.formatDate(queryStatus.getQuery().getEndDate()), + DefaultQueryParameters.formatDate(dupeQueryStatus.getQuery().getEndDate())); + Assertions.assertEquals(queryStatus.getQuery().getQueryLogicName(), dupeQueryStatus.getQuery().getQueryLogicName()); + Assertions.assertEquals(queryStatus.getQuery().getPagesize(), dupeQueryStatus.getQuery().getPagesize()); + + // verify that no events were published + Assertions.assertEquals(3, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CLOSE, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + dupeQueryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @Test + public void testDuplicateSuccess_update() throws Exception { + DatawaveUserDetails authUser = createUserDetails(null, Arrays.asList("ALL", "NONE")); + + // define a valid query + long currentTimeMillis = System.currentTimeMillis(); + String queryId = defineQuery(authUser, createParams()); + + String newQuery = "SOME_OTHER_FIELD:SOME_OTHER_VALUE"; + String newAuths = "ALL,NONE"; + String newBegin = "20100101 000000.000"; + String newEnd = "20600101 000000.000"; + String newLogic = "AltEventQuery"; + + MultiValueMap updateParams = new LinkedMultiValueMap<>(); + updateParams.set(QUERY, newQuery); + updateParams.set(QUERY_AUTHORIZATIONS, newAuths); + updateParams.set(BEGIN_DATE, newBegin); + updateParams.set(END_DATE, newEnd); + updateParams.set(QUERY_LOGIC_NAME, newLogic); + + mockServer.reset(); + auditSentSetup(); + + // duplicate the query + Future> duplicateFuture = duplicateQuery(authUser, queryId, updateParams); + + // the response should come back right away + ResponseEntity response = duplicateFuture.get(); + + Assertions.assertEquals(200, response.getStatusCodeValue()); + + String dupeQueryId = (String) response.getBody().getResult(); + + // make sure an audit message was sent + assertAuditSent(dupeQueryId); + + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + // @formatter:off + assertQueryStatus( + QueryStatus.QUERY_STATE.DEFINE, + 0, + 0, + 0, + 0, + currentTimeMillis, + queryStatus); + // @formatter:on + + QueryStatus dupeQueryStatus = queryStorageCache.getQueryStatus(dupeQueryId); + // @formatter:off + assertQueryStatus( + QueryStatus.QUERY_STATE.CREATE, + 0, + 0, + 0, + 0, + currentTimeMillis, + dupeQueryStatus); + // @formatter:on + + // make sure the original query is unchanged + Assertions.assertEquals(TEST_QUERY_STRING, queryStatus.getQuery().getQuery()); + Assertions.assertEquals(TEST_QUERY_AUTHORIZATIONS, queryStatus.getQuery().getQueryAuthorizations()); + Assertions.assertEquals(TEST_QUERY_BEGIN, DefaultQueryParameters.formatDate(queryStatus.getQuery().getBeginDate())); + Assertions.assertEquals(TEST_QUERY_END, DefaultQueryParameters.formatDate(queryStatus.getQuery().getEndDate())); + Assertions.assertEquals("EventQuery", queryStatus.getQuery().getQueryLogicName()); + + // make sure the duplicated query is updated + Assertions.assertEquals(newQuery, dupeQueryStatus.getQuery().getQuery()); + Assertions.assertEquals(newAuths, dupeQueryStatus.getQuery().getQueryAuthorizations()); + Assertions.assertEquals(newBegin, DefaultQueryParameters.formatDate(dupeQueryStatus.getQuery().getBeginDate())); + Assertions.assertEquals(newEnd, DefaultQueryParameters.formatDate(dupeQueryStatus.getQuery().getEndDate())); + Assertions.assertEquals(newLogic, dupeQueryStatus.getQuery().getQueryLogicName()); + + // verify that no events were published + Assertions.assertEquals(1, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + dupeQueryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @Test + public void testDuplicateFailure_invalidUpdate() throws Exception { + DatawaveUserDetails authUser = createUserDetails(); + + // define a valid query + String queryId = defineQuery(authUser, createParams()); + + String newLogic = "SomeBogusLogic"; + + MultiValueMap updateParams = new LinkedMultiValueMap<>(); + updateParams.set(QUERY_LOGIC_NAME, newLogic); + + mockServer.reset(); + auditNotSentSetup(); + + // duplicate the query + UriComponents uri = createUri(queryId + "/duplicate"); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, updateParams, null, HttpMethod.POST, uri); + + // make the duplicate call asynchronously + Future> duplicateFuture = Executors.newSingleThreadExecutor() + .submit(() -> jwtRestTemplate.exchange(requestEntity, VoidResponse.class)); + + // the response should come back right away + ResponseEntity response = duplicateFuture.get(); + + Assertions.assertEquals(400, response.getStatusCodeValue()); + + // make sure an audit message wasn't sent + assertAuditNotSent(); + + // verify that no events were published + Assertions.assertEquals(0, queryRequestEvents.size()); + } + + @Test + public void testDuplicateFailure_queryNotFound() throws Exception { + DatawaveUserDetails authUser = createUserDetails(); + + String queryId = UUID.randomUUID().toString(); + + MultiValueMap updateParams = new LinkedMultiValueMap<>(); + + mockServer.reset(); + auditNotSentSetup(); + + // duplicate the query + UriComponents uri = createUri(queryId + "/duplicate"); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, updateParams, null, HttpMethod.POST, uri); + + // make the duplicate call asynchronously + Future> duplicateFuture = Executors.newSingleThreadExecutor() + .submit(() -> jwtRestTemplate.exchange(requestEntity, VoidResponse.class)); + + // the response should come back right away + ResponseEntity response = duplicateFuture.get(); + + Assertions.assertEquals(404, response.getStatusCodeValue()); + + // @formatter:off + assertQueryException( + "No query object matches this id. " + queryId, + "Exception with no cause caught", + "404-1", + Iterables.getOnlyElement(response.getBody().getExceptions())); + // @formatter:on + + // make sure an audit message wasn't sent + assertAuditNotSent(); + + // verify that no events were published + Assertions.assertEquals(0, queryRequestEvents.size()); + } + + @Test + public void testDuplicateFailure_ownershipFailure() throws Exception { + DatawaveUserDetails authUser = createUserDetails(); + DatawaveUserDetails altAuthUser = createAltUserDetails(); + + // define a valid query + String queryId = defineQuery(authUser, createParams()); + + MultiValueMap updateParams = new LinkedMultiValueMap<>(); + + mockServer.reset(); + auditNotSentSetup(); + + // duplicate the query + UriComponents uri = createUri(queryId + "/duplicate"); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(altAuthUser, updateParams, null, HttpMethod.POST, uri); + + // make the duplicate call asynchronously + Future> duplicateFuture = Executors.newSingleThreadExecutor() + .submit(() -> jwtRestTemplate.exchange(requestEntity, VoidResponse.class)); + + // the response should come back right away + ResponseEntity response = duplicateFuture.get(); + + Assertions.assertEquals(401, response.getStatusCodeValue()); + + // make sure an audit message wasn't sent + assertAuditNotSent(); + + // @formatter:off + assertQueryException( + "Current user does not match user that defined query. altuserdn != userdn", + "Exception with no cause caught", + "401-1", + Iterables.getOnlyElement(response.getBody().getExceptions())); + // @formatter:on + + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + + // make sure the query was not updated + Assertions.assertEquals(TEST_QUERY_STRING, queryStatus.getQuery().getQuery()); + + // verify that no events were published + Assertions.assertEquals(0, queryRequestEvents.size()); + } +} diff --git a/service/src/test/java/datawave/microservice/query/QueryServiceListTest.java b/service/src/test/java/datawave/microservice/query/QueryServiceListTest.java new file mode 100644 index 00000000..4883956a --- /dev/null +++ b/service/src/test/java/datawave/microservice/query/QueryServiceListTest.java @@ -0,0 +1,426 @@ +package datawave.microservice.query; + +import static datawave.microservice.query.QueryParameters.QUERY_NAME; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.stream.Collectors; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.http.HttpMethod; +import org.springframework.http.RequestEntity; +import org.springframework.http.ResponseEntity; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit.jupiter.SpringExtension; +import org.springframework.util.MultiValueMap; +import org.springframework.web.util.UriComponents; +import org.springframework.web.util.UriComponentsBuilder; + +import datawave.microservice.authorization.service.RemoteAuthorizationServiceUserDetailsService; +import datawave.microservice.authorization.user.DatawaveUserDetails; +import datawave.security.util.ProxiedEntityUtils; +import datawave.webservice.query.result.logic.QueryLogicDescription; +import datawave.webservice.result.QueryImplListResponse; +import datawave.webservice.result.QueryLogicResponse; + +@ExtendWith(SpringExtension.class) +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +@ActiveProfiles({"QueryStarterDefaults", "QueryStarterOverrides", "QueryServiceTest", RemoteAuthorizationServiceUserDetailsService.ACTIVATION_PROFILE}) +@ContextConfiguration(classes = {QueryService.class}) +public class QueryServiceListTest extends AbstractQueryServiceTest { + + @Test + public void testListSuccess() throws Exception { + DatawaveUserDetails authUser = createUserDetails(); + DatawaveUserDetails altAuthUser = createAltUserDetails(); + + // define a bunch of queries as the original user + List queryIds = new ArrayList<>(); + for (int i = 0; i < 5; i++) { + String queryId = createQuery(authUser, createParams()); + mockServer.reset(); + + queryIds.add(queryId); + } + + // define a bunch of queries as the alternate user + List altQueryIds = new ArrayList<>(); + for (int i = 0; i < 5; i++) { + String queryId = defineQuery(altAuthUser, createParams()); + mockServer.reset(); + + altQueryIds.add(queryId); + } + + // list queries as the original user + Future> listFuture = listQueries(authUser, null, null); + + // this should return immediately + ResponseEntity listResponse = listFuture.get(); + + Assertions.assertEquals(200, listResponse.getStatusCodeValue()); + + QueryImplListResponse result = listResponse.getBody(); + + Assertions.assertEquals(5, result.getNumResults()); + + List actualQueryIds = result.getQuery().stream().map(Query::getId).map(UUID::toString).collect(Collectors.toList()); + + Collections.sort(queryIds); + Collections.sort(actualQueryIds); + + Assertions.assertEquals(queryIds, actualQueryIds); + } + + @Test + public void testListSuccess_filterOnQueryId() throws Exception { + DatawaveUserDetails authUser = createUserDetails(); + + // define a bunch of queries as the original user + List queryIds = new ArrayList<>(); + for (int i = 0; i < 5; i++) { + String queryId = createQuery(authUser, createParams()); + mockServer.reset(); + + queryIds.add(queryId); + } + + // list queries + Future> listFuture = listQueries(authUser, queryIds.get(0), null); + + // this should return immediately + ResponseEntity listResponse = listFuture.get(); + + Assertions.assertEquals(200, listResponse.getStatusCodeValue()); + + QueryImplListResponse result = listResponse.getBody(); + + Assertions.assertEquals(1, result.getNumResults()); + + List actualQueryIds = result.getQuery().stream().map(Query::getId).map(UUID::toString).collect(Collectors.toList()); + + Assertions.assertEquals(queryIds.get(0), actualQueryIds.get(0)); + } + + @Test + public void testListSuccess_filterOnQueryName() throws Exception { + DatawaveUserDetails authUser = createUserDetails(); + + String uniqueQueryName = "Unique Query"; + + // define a bunch of queries as the original user + List queryIds = new ArrayList<>(); + for (int i = 0; i < 5; i++) { + MultiValueMap params = createParams(); + if (i == 0) { + params.set(QUERY_NAME, uniqueQueryName); + } + + String queryId = createQuery(authUser, params); + mockServer.reset(); + + queryIds.add(queryId); + } + + // list queries + Future> listFuture = listQueries(authUser, null, uniqueQueryName); + + // this should return immediately + ResponseEntity listResponse = listFuture.get(); + + Assertions.assertEquals(200, listResponse.getStatusCodeValue()); + + QueryImplListResponse result = listResponse.getBody(); + + Assertions.assertEquals(1, result.getNumResults()); + + List actualQueryIds = result.getQuery().stream().map(Query::getId).map(UUID::toString).collect(Collectors.toList()); + + Assertions.assertEquals(queryIds.get(0), actualQueryIds.get(0)); + } + + @Test + public void testListSuccess_filterOnMultiple() throws Exception { + DatawaveUserDetails authUser = createUserDetails(); + + String uniqueQueryName = "Unique Query"; + + // define a bunch of queries as the original user + List queryIds = new ArrayList<>(); + for (int i = 0; i < 5; i++) { + MultiValueMap params = createParams(); + if (i == 0) { + params.set(QUERY_NAME, uniqueQueryName); + } + + String queryId = createQuery(authUser, params); + mockServer.reset(); + + queryIds.add(queryId); + } + + // list queries with just the query ID and a bogus name + Future> listFuture = listQueries(authUser, queryIds.get(0), "bogus name"); + + // this should return immediately + ResponseEntity listResponse = listFuture.get(); + + Assertions.assertEquals(200, listResponse.getStatusCodeValue()); + + QueryImplListResponse result = listResponse.getBody(); + + Assertions.assertEquals(0, result.getNumResults()); + + // list queries with just the query name and a bogus ID + listFuture = listQueries(authUser, UUID.randomUUID().toString(), uniqueQueryName); + + // this should return immediately + listResponse = listFuture.get(); + + Assertions.assertEquals(200, listResponse.getStatusCodeValue()); + + result = listResponse.getBody(); + + Assertions.assertEquals(0, result.getNumResults()); + + // list queries with just the query name and a bogus ID + listFuture = listQueries(authUser, queryIds.get(0), uniqueQueryName); + + // this should return immediately + listResponse = listFuture.get(); + + Assertions.assertEquals(200, listResponse.getStatusCodeValue()); + + result = listResponse.getBody(); + + Assertions.assertEquals(1, result.getNumResults()); + + List actualQueryIds = result.getQuery().stream().map(Query::getId).map(UUID::toString).collect(Collectors.toList()); + + Assertions.assertEquals(queryIds.get(0), actualQueryIds.get(0)); + } + + @Test + public void testListFailure_ownershipFailure() throws Exception { + DatawaveUserDetails authUser = createUserDetails(); + DatawaveUserDetails altAuthUser = createAltUserDetails(); + + String uniqueQueryName = "Unique Query"; + + // define a bunch of queries as the original user + List queryIds = new ArrayList<>(); + for (int i = 0; i < 5; i++) { + MultiValueMap params = createParams(); + if (i == 0) { + params.set(QUERY_NAME, uniqueQueryName); + } + + String queryId = createQuery(authUser, params); + mockServer.reset(); + + queryIds.add(queryId); + } + + // list queries with just the query ID and a bogus name + Future> listFuture = listQueries(altAuthUser, queryIds.get(0), "bogus name"); + + // this should return immediately + ResponseEntity listResponse = listFuture.get(); + + Assertions.assertEquals(200, listResponse.getStatusCodeValue()); + + QueryImplListResponse result = listResponse.getBody(); + + Assertions.assertEquals(0, result.getNumResults()); + + // list queries with just the query name and a bogus ID + listFuture = listQueries(altAuthUser, UUID.randomUUID().toString(), uniqueQueryName); + + // this should return immediately + listResponse = listFuture.get(); + + Assertions.assertEquals(200, listResponse.getStatusCodeValue()); + + result = listResponse.getBody(); + + Assertions.assertEquals(0, result.getNumResults()); + + // list queries with the query name and query ID + listFuture = listQueries(altAuthUser, queryIds.get(0), uniqueQueryName); + + // this should return immediately + listResponse = listFuture.get(); + + Assertions.assertEquals(200, listResponse.getStatusCodeValue()); + + result = listResponse.getBody(); + + Assertions.assertEquals(0, result.getNumResults()); + } + + @Test + public void testAdminListSuccess() throws Exception { + DatawaveUserDetails authUser = createUserDetails(); + DatawaveUserDetails adminUser = createAltUserDetails(Arrays.asList("AuthorizedUser", "Administrator"), null); + + String user = ProxiedEntityUtils.getShortName(authUser.getPrimaryUser().getDn().subjectDN()); + + String uniqueQueryName = "Unique Query"; + + // define a bunch of queries as the original user + List queryIds = new ArrayList<>(); + for (int i = 0; i < 5; i++) { + MultiValueMap params = createParams(); + if (i == 0) { + params.set(QUERY_NAME, uniqueQueryName); + } + + String queryId = createQuery(authUser, params); + mockServer.reset(); + + queryIds.add(queryId); + } + + // list queries with just the query ID + Future> listFuture = adminListQueries(adminUser, queryIds.get(0), user, null); + + // this should return immediately + ResponseEntity listResponse = listFuture.get(); + + Assertions.assertEquals(200, listResponse.getStatusCodeValue()); + + QueryImplListResponse result = listResponse.getBody(); + + Assertions.assertEquals(1, result.getNumResults()); + + List actualQueryIds = result.getQuery().stream().map(Query::getId).map(UUID::toString).collect(Collectors.toList()); + + Assertions.assertEquals(queryIds.get(0), actualQueryIds.get(0)); + + // list queries with just the query name + listFuture = adminListQueries(adminUser, null, user, uniqueQueryName); + + // this should return immediately + listResponse = listFuture.get(); + + Assertions.assertEquals(200, listResponse.getStatusCodeValue()); + + result = listResponse.getBody(); + + Assertions.assertEquals(1, result.getNumResults()); + + actualQueryIds = result.getQuery().stream().map(Query::getId).map(UUID::toString).collect(Collectors.toList()); + + Assertions.assertEquals(queryIds.get(0), actualQueryIds.get(0)); + + // list queries with the query name and query ID + listFuture = adminListQueries(adminUser, queryIds.get(0), user, uniqueQueryName); + + // this should return immediately + listResponse = listFuture.get(); + + Assertions.assertEquals(200, listResponse.getStatusCodeValue()); + + result = listResponse.getBody(); + + Assertions.assertEquals(1, result.getNumResults()); + + actualQueryIds = result.getQuery().stream().map(Query::getId).map(UUID::toString).collect(Collectors.toList()); + + Assertions.assertEquals(queryIds.get(0), actualQueryIds.get(0)); + } + + @Test + public void testAdminListFailure_notAdminUser() throws Exception { + DatawaveUserDetails authUser = createUserDetails(); + DatawaveUserDetails altAuthUser = createAltUserDetails(); + + String user = ProxiedEntityUtils.getShortName(authUser.getPrimaryUser().getDn().subjectDN()); + + String uniqueQueryName = "Unique Query"; + + // define a bunch of queries as the original user + List queryIds = new ArrayList<>(); + for (int i = 0; i < 5; i++) { + MultiValueMap params = createParams(); + if (i == 0) { + params.set(QUERY_NAME, uniqueQueryName); + } + + String queryId = createQuery(authUser, params); + mockServer.reset(); + + queryIds.add(queryId); + } + + UriComponentsBuilder uriBuilder = uriBuilder("/adminList"); + UriComponents uri = uriBuilder.build(); + + RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(altAuthUser, null, null, HttpMethod.GET, uri); + + // make the next call asynchronously + Future> listFuture = Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, String.class)); + + ResponseEntity listResponse = listFuture.get(); + + Assertions.assertEquals(403, listResponse.getStatusCodeValue()); + } + + @Test + public void testGetQuerySuccess() throws Exception { + DatawaveUserDetails authUser = createUserDetails(); + + // define a query + String queryId = createQuery(authUser, createParams()); + mockServer.reset(); + + // get the query + Future> listFuture = getQuery(authUser, queryId); + + // this should return immediately + ResponseEntity listResponse = listFuture.get(); + + Assertions.assertEquals(200, listResponse.getStatusCodeValue()); + + QueryImplListResponse result = listResponse.getBody(); + + Assertions.assertEquals(1, result.getNumResults()); + + Assertions.assertEquals(queryId, result.getQuery().get(0).getId().toString()); + } + + @Test + public void testListQueryLogicSuccess() throws Exception { + DatawaveUserDetails authUser = createUserDetails(); + + Future> future = listQueryLogic(authUser); + + ResponseEntity response = future.get(); + + Assertions.assertEquals(200, response.getStatusCodeValue()); + + QueryLogicResponse qlResponse = response.getBody(); + + String[] expectedQueryLogics = new String[] {"AltEventQuery", "ContentQuery", "CountQuery", "DiscoveryQuery", "EdgeEventQuery", "EdgeQuery", + "ErrorCountQuery", "ErrorDiscoveryQuery", "ErrorEventQuery", "ErrorFieldIndexCountQuery", "EventQuery", "FacetedQuery", "FieldIndexCountQuery", + "HitHighlights", "IndexStatsQuery", "LuceneUUIDEventQuery", "QueryMetricsQuery", "InternalQueryMetricsQuery", "TermFrequencyQuery", + "FederatedEventQuery"}; + + Assertions.assertEquals(expectedQueryLogics.length, qlResponse.getQueryLogicList().size()); + + List qlNames = qlResponse.getQueryLogicList().stream().map(QueryLogicDescription::getName).sorted().collect(Collectors.toList()); + + qlNames.removeAll(Arrays.asList(expectedQueryLogics)); + + Assertions.assertTrue(qlNames.isEmpty()); + } +} diff --git a/service/src/test/java/datawave/microservice/query/QueryServiceNextTest.java b/service/src/test/java/datawave/microservice/query/QueryServiceNextTest.java new file mode 100644 index 00000000..f26f9bff --- /dev/null +++ b/service/src/test/java/datawave/microservice/query/QueryServiceNextTest.java @@ -0,0 +1,776 @@ +package datawave.microservice.query; + +import java.util.Arrays; +import java.util.Collections; +import java.util.Objects; +import java.util.UUID; +import java.util.concurrent.Future; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.http.ResponseEntity; +import org.springframework.test.annotation.DirtiesContext; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit.jupiter.SpringExtension; +import org.springframework.util.LinkedMultiValueMap; +import org.springframework.util.MultiValueMap; + +import com.google.common.collect.Iterables; + +import datawave.core.query.configuration.GenericQueryConfiguration; +import datawave.microservice.authorization.service.RemoteAuthorizationServiceUserDetailsService; +import datawave.microservice.authorization.user.DatawaveUserDetails; +import datawave.microservice.query.remote.QueryRequest; +import datawave.microservice.query.storage.QueryStatus; +import datawave.microservice.query.storage.TaskStates; +import datawave.webservice.query.result.event.DefaultEvent; +import datawave.webservice.result.DefaultEventQueryResponse; +import datawave.webservice.result.VoidResponse; + +@ExtendWith(SpringExtension.class) +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +@ActiveProfiles({"QueryStarterDefaults", "QueryStarterOverrides", "QueryServiceTest", RemoteAuthorizationServiceUserDetailsService.ACTIVATION_PROFILE}) +@ContextConfiguration(classes = {QueryService.class}) +public class QueryServiceNextTest extends AbstractQueryServiceTest { + + @Test + public void testNextSuccess() throws Exception { + DatawaveUserDetails authUser = createUserDetails(); + + // create a valid query + String queryId = createQuery(authUser, createParams()); + + // pump enough results into the queue to trigger a complete page + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + int pageSize = queryStatus.getQuery().getPagesize(); + + // test field value pairings + MultiValueMap fieldValues = new LinkedMultiValueMap<>(); + fieldValues.add("LOKI", "ALLIGATOR"); + fieldValues.add("LOKI", "CLASSIC"); + + // add a config object to the query status, which would normally be added by the executor service + queryStatus.setConfig(new GenericQueryConfiguration()); + queryStorageCache.updateQueryStatus(queryStatus); + + // @formatter:off + publishEventsToQueue( + queryId, + (int) (1.5 * pageSize), + fieldValues, + "ALL"); + // @formatter:on + + // make the next call asynchronously + Future> future = nextQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity response = future.get(); + + Assertions.assertEquals(200, response.getStatusCodeValue()); + + // verify some headers + Assertions.assertEquals("1", Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-query-page-number")))); + Assertions.assertEquals("false", Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-Partial-Results")))); + Assertions.assertEquals("false", Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-query-last-page")))); + + DefaultEventQueryResponse queryResponse = (DefaultEventQueryResponse) response.getBody(); + + // verify the query response + // @formatter:off + assertQueryResponse( + queryId, + "EventQuery", + 1, + false, + Long.parseLong(Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-OperationTimeInMS")))), + 1, + Collections.singletonList("LOKI"), + pageSize, + Objects.requireNonNull(queryResponse)); + // @formatter:on + + // validate one of the events + DefaultEvent event = (DefaultEvent) queryResponse.getEvents().get(0); + // @formatter:off + assertDefaultEvent( + Arrays.asList("LOKI", "LOKI"), + Arrays.asList("ALLIGATOR", "CLASSIC"), + event); + // @formatter:on + + // verify that the next event was published + Assertions.assertEquals(2, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.NEXT, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @Test + public void testNextSuccess_multiplePages() throws Exception { + DatawaveUserDetails authUser = createUserDetails(); + + // create a valid query + String queryId = createQuery(authUser, createParams()); + + // pump enough results into the queue to trigger two complete pages + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + int pageSize = queryStatus.getQuery().getPagesize(); + + // test field value pairings + MultiValueMap fieldValues = new LinkedMultiValueMap<>(); + fieldValues.add("LOKI", "ALLIGATOR"); + fieldValues.add("LOKI", "CLASSIC"); + + // verify that the create event was published + Assertions.assertEquals(1, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + // @formatter:off + + // add a config object to the query status, which would normally be added by the executor service + queryStatus.setConfig(new GenericQueryConfiguration()); + queryStorageCache.updateQueryStatus(queryStatus); + + for (int page = 1; page <= 2; page++) { + // NOTE: We have to generate the results in between next calls because the test queue manager does not handle requeueing of unused messages :( + // @formatter:off + publishEventsToQueue( + queryId, + pageSize, + fieldValues, + "ALL"); + // @formatter:on + + // make the next call asynchronously + Future> future = nextQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity response = future.get(); + + Assertions.assertEquals(200, response.getStatusCodeValue()); + + // verify some headers + Assertions.assertEquals(Integer.toString(page), Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-query-page-number")))); + Assertions.assertEquals("false", Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-Partial-Results")))); + Assertions.assertEquals("false", Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-query-last-page")))); + + DefaultEventQueryResponse queryResponse = (DefaultEventQueryResponse) response.getBody(); + + // verify the query response + // @formatter:off + assertQueryResponse( + queryId, + "EventQuery", + page, + false, + Long.parseLong(Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-OperationTimeInMS")))), + 1, + Collections.singletonList("LOKI"), + pageSize, + Objects.requireNonNull(queryResponse)); + // @formatter:on + + // validate one of the events + DefaultEvent event = (DefaultEvent) queryResponse.getEvents().get(0); + // @formatter:off + assertDefaultEvent( + Arrays.asList("LOKI", "LOKI"), + Arrays.asList("ALLIGATOR", "CLASSIC"), + event); + // @formatter:on + + // verify that the next event was published + Assertions.assertEquals(1, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.NEXT, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + } + + @Test + public void testNextSuccess_cancelPartialResults() throws Exception { + DatawaveUserDetails authUser = createUserDetails(); + + // create a valid query + String queryId = createQuery(authUser, createParams()); + + // pump enough results into the queue to trigger a complete page + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + int pageSize = queryStatus.getQuery().getPagesize(); + + // test field value pairings + MultiValueMap fieldValues = new LinkedMultiValueMap<>(); + fieldValues.add("LOKI", "ALLIGATOR"); + fieldValues.add("LOKI", "CLASSIC"); + + int numEvents = (int) (0.5 * pageSize); + + // add a config object to the query status, which would normally be added by the executor service + queryStatus.setConfig(new GenericQueryConfiguration()); + queryStorageCache.updateQueryStatus(queryStatus); + + // @formatter:off + publishEventsToQueue( + queryId, + numEvents, + fieldValues, + "ALL"); + // @formatter:on + + // make the next call asynchronously + Future> nextFuture = nextQuery(authUser, queryId); + + // make sure all events were consumed before canceling + while (queryQueueManager.getNumResultsRemaining(queryId) != 0) { + Thread.sleep(100); + } + + // cancel the query so that it returns partial results + Future> cancelFuture = cancelQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity cancelResponse = cancelFuture.get(); + + Assertions.assertEquals(200, cancelResponse.getStatusCodeValue()); + + // the response should come back right away + ResponseEntity nextResponse = nextFuture.get(); + + Assertions.assertEquals(200, nextResponse.getStatusCodeValue()); + + // verify some headers + Assertions.assertEquals("1", Iterables.getOnlyElement(Objects.requireNonNull(nextResponse.getHeaders().get("X-query-page-number")))); + Assertions.assertEquals("true", Iterables.getOnlyElement(Objects.requireNonNull(nextResponse.getHeaders().get("X-Partial-Results")))); + Assertions.assertEquals("false", Iterables.getOnlyElement(Objects.requireNonNull(nextResponse.getHeaders().get("X-query-last-page")))); + + DefaultEventQueryResponse queryResponse = (DefaultEventQueryResponse) nextResponse.getBody(); + + // verify the query response + // @formatter:off + assertQueryResponse( + queryId, + "EventQuery", + 1, + true, + Long.parseLong(Iterables.getOnlyElement(Objects.requireNonNull(nextResponse.getHeaders().get("X-OperationTimeInMS")))), + 1, + Collections.singletonList("LOKI"), + numEvents, + Objects.requireNonNull(queryResponse)); + // @formatter:on + + // validate one of the events + DefaultEvent event = (DefaultEvent) queryResponse.getEvents().get(0); + // @formatter:off + assertDefaultEvent( + Arrays.asList("LOKI", "LOKI"), + Arrays.asList("ALLIGATOR", "CLASSIC"), + event); + // @formatter:on + + // verify that the next events were published + Assertions.assertEquals(4, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.NEXT, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "query:**", + QueryRequest.Method.CANCEL, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CANCEL, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @Test + public void testNextSuccess_maxResults() throws Exception { + DatawaveUserDetails authUser = createUserDetails(); + + // create a valid query + String queryId = createQuery(authUser, createParams()); + + // pump enough results into the queue to trigger two complete pages + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + int pageSize = queryStatus.getQuery().getPagesize(); + + // test field value pairings + MultiValueMap fieldValues = new LinkedMultiValueMap<>(); + fieldValues.add("LOKI", "ALLIGATOR"); + fieldValues.add("LOKI", "CLASSIC"); + + // verify that the create event was published + Assertions.assertEquals(1, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + + // add a config object to the query status, which would normally be added by the executor service + queryStatus.setConfig(new GenericQueryConfiguration()); + queryStorageCache.updateQueryStatus(queryStatus); + + for (int page = 1; page <= 4; page++) { + // NOTE: We have to generate the results in between next calls because the test queue manager does not handle requeueing of unused messages :( + // @formatter:off + publishEventsToQueue( + queryId, + pageSize, + fieldValues, + "ALL"); + // @formatter:on + + // make the next call asynchronously + Future> future = nextQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity response = future.get(); + + if (page != 4) { + Assertions.assertEquals(200, response.getStatusCodeValue()); + } else { + Assertions.assertEquals(204, response.getStatusCodeValue()); + } + + if (page != 4) { + // verify some headers + Assertions.assertEquals("false", Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-Partial-Results")))); + Assertions.assertEquals(Integer.toString(page), + Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-query-page-number")))); + Assertions.assertEquals("false", Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-query-last-page")))); + + DefaultEventQueryResponse queryResponse = (DefaultEventQueryResponse) response.getBody(); + + // verify the query response + // @formatter:off + assertQueryResponse( + queryId, + "EventQuery", + page, + false, + Long.parseLong(Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-OperationTimeInMS")))), + 1, + Collections.singletonList("LOKI"), + pageSize, + Objects.requireNonNull(queryResponse)); + // @formatter:on + + // validate one of the events + DefaultEvent event = (DefaultEvent) queryResponse.getEvents().get(0); + // @formatter:off + assertDefaultEvent( + Arrays.asList("LOKI", "LOKI"), + Arrays.asList("ALLIGATOR", "CLASSIC"), + event); + // @formatter:on + + // verify that the next event was published + Assertions.assertEquals(1, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.NEXT, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } else { + Assertions.assertNull(response.getBody()); + + // verify that the next and close events were published + Assertions.assertEquals(2, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.NEXT, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CLOSE, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + } + + // make the next call asynchronously + Future> future = nextQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity response = future.get(); + + Assertions.assertEquals(400, response.getStatusCodeValue()); + + // @formatter:off + assertQueryException( + "Cannot call next on a query that is not running", + "Exception with no cause caught", + "400-1", + Iterables.getOnlyElement(response.getBody().getExceptions())); + // @formatter:on + } + + @Test + public void testNextSuccess_noResults() throws Exception { + DatawaveUserDetails authUser = createUserDetails(); + + // create a valid query + String queryId = createQuery(authUser, createParams()); + + // mark the task states as complete, and mark task creation as complete to make it appear that the executor has finished + TaskStates taskStates = queryStorageCache.getTaskStates(queryId); + for (int i = 0; i < taskStates.getNextTaskId(); i++) { + taskStates.setState(i, TaskStates.TASK_STATE.COMPLETED); + } + queryStorageCache.updateTaskStates(taskStates); + queryStorageCache.updateCreateStage(queryId, QueryStatus.CREATE_STAGE.RESULTS); + + // make the next call asynchronously + Future> future = nextQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity response = future.get(); + + Assertions.assertEquals(204, response.getStatusCodeValue()); + Assertions.assertNull(response.getBody()); + + // verify that the next event was published + Assertions.assertEquals(3, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.NEXT, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CLOSE, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @Test + public void testNextFailure_queryNotFound() throws Exception { + DatawaveUserDetails authUser = createUserDetails(); + + String queryId = UUID.randomUUID().toString(); + + // make the next call asynchronously + Future> future = nextQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity response = future.get(); + + Assertions.assertEquals(404, response.getStatusCodeValue()); + + // @formatter:off + assertQueryException( + "No query object matches this id. " + queryId, + "Exception with no cause caught", + "404-1", + Iterables.getOnlyElement(response.getBody().getExceptions())); + // @formatter:on + + // verify that no events were published + Assertions.assertEquals(0, queryRequestEvents.size()); + } + + @Test + public void testNextFailure_queryNotRunning() throws Exception { + DatawaveUserDetails authUser = createUserDetails(); + + // create a valid query + String queryId = createQuery(authUser, createParams()); + + // cancel the query so that it returns partial results + Future> cancelFuture = cancelQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity cancelResponse = cancelFuture.get(); + + Assertions.assertEquals(200, cancelResponse.getStatusCodeValue()); + + // make the next call asynchronously + Future> future = nextQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity response = future.get(); + + Assertions.assertEquals(400, response.getStatusCodeValue()); + + // @formatter:off + assertQueryException( + "Cannot call next on a query that is not running", + "Exception with no cause caught", + "400-1", + Iterables.getOnlyElement(response.getBody().getExceptions())); + // @formatter:on + + // verify that the next events were published + Assertions.assertEquals(3, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "query:**", + QueryRequest.Method.CANCEL, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CANCEL, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @Test + public void testNextFailure_ownershipFailure() throws Exception { + DatawaveUserDetails authUser = createUserDetails(); + DatawaveUserDetails altAuthUser = createAltUserDetails(); + + // create a valid query + String queryId = createQuery(authUser, createParams()); + + // make the next call as an alternate user asynchronously + Future> future = nextQuery(altAuthUser, queryId); + + // the response should come back right away + ResponseEntity response = future.get(); + + Assertions.assertEquals(401, response.getStatusCodeValue()); + + // @formatter:off + assertQueryException( + "Current user does not match user that defined query. altuserdn != userdn", + "Exception with no cause caught", + "401-1", + Iterables.getOnlyElement(response.getBody().getExceptions())); + // @formatter:on + + // verify that the next events were published + Assertions.assertEquals(1, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @DirtiesContext + @Test + public void testNextFailure_timeout() throws Exception { + DatawaveUserDetails authUser = createUserDetails(); + + // override the call timeout for this test + queryProperties.getExpiration().setCallTimeout(0); + + // create a valid query + String queryId = createQuery(authUser, createParams()); + + // make the next call asynchronously + Future> future = nextQuery(authUser, queryId); + + // the response should come back after the configured timeout (5 seconds) + ResponseEntity response = future.get(); + + Assertions.assertEquals(500, response.getStatusCodeValue()); + + // @formatter:off + assertQueryException( + "Query timed out. " + queryId + " timed out.", + "Exception with no cause caught", + "500-27", + Iterables.getOnlyElement(response.getBody().getExceptions())); + // @formatter:on + + // verify that the next events were published + Assertions.assertEquals(2, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.NEXT, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @Test + public void testNextFailure_nextOnDefined() throws Exception { + DatawaveUserDetails authUser = createUserDetails(); + + // define a valid query + String queryId = defineQuery(authUser, createParams()); + + // make the next call + Future> future = nextQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity response = future.get(); + + Assertions.assertEquals(400, response.getStatusCodeValue()); + + // @formatter:off + assertQueryException( + "Cannot call next on a query that is not running", + "Exception with no cause caught", + "400-1", + Iterables.getOnlyElement(response.getBody().getExceptions())); + // @formatter:on + + // verify that no events were published + Assertions.assertEquals(0, queryRequestEvents.size()); + } + + @Test + public void testNextFailure_nextOnClosed() throws Exception { + DatawaveUserDetails authUser = createUserDetails(); + + // create a valid query + String queryId = createQuery(authUser, createParams()); + + // close the query + Future> closeFuture = closeQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity closeResponse = closeFuture.get(); + + Assertions.assertEquals(200, closeResponse.getStatusCodeValue()); + + // make the next call asynchronously + Future> future = nextQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity response = future.get(); + + Assertions.assertEquals(400, response.getStatusCodeValue()); + + // @formatter:off + assertQueryException( + "Cannot call next on a query that is not running", + "Exception with no cause caught", + "400-1", + Iterables.getOnlyElement(response.getBody().getExceptions())); + // @formatter:on + + // verify that no events were published + Assertions.assertEquals(2, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CLOSE, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @Test + public void testNextFailure_nextOnCanceled() throws Exception { + DatawaveUserDetails authUser = createUserDetails(); + + // create a valid query + String queryId = createQuery(authUser, createParams()); + + // cancel the query + Future> cancelFuture = cancelQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity cancelResponse = cancelFuture.get(); + + Assertions.assertEquals(200, cancelResponse.getStatusCodeValue()); + + // make the next call asynchronously + Future> future = nextQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity response = future.get(); + + Assertions.assertEquals(400, response.getStatusCodeValue()); + + // @formatter:off + assertQueryException( + "Cannot call next on a query that is not running", + "Exception with no cause caught", + "400-1", + Iterables.getOnlyElement(response.getBody().getExceptions())); + // @formatter:on + + // verify that the cancel event was published + Assertions.assertEquals(3, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "query:**", + QueryRequest.Method.CANCEL, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CANCEL, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } +} diff --git a/service/src/test/java/datawave/microservice/query/QueryServicePlanTest.java b/service/src/test/java/datawave/microservice/query/QueryServicePlanTest.java new file mode 100644 index 00000000..2d1c9e9f --- /dev/null +++ b/service/src/test/java/datawave/microservice/query/QueryServicePlanTest.java @@ -0,0 +1,103 @@ +package datawave.microservice.query; + +import java.io.IOException; +import java.text.ParseException; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.cloud.bus.event.RemoteQueryRequestEvent; +import org.springframework.context.ApplicationEventPublisher; +import org.springframework.http.HttpMethod; +import org.springframework.http.HttpStatus; +import org.springframework.http.RequestEntity; +import org.springframework.http.ResponseEntity; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit.jupiter.SpringExtension; +import org.springframework.util.MultiValueMap; +import org.springframework.web.util.UriComponents; + +import datawave.microservice.authorization.service.RemoteAuthorizationServiceUserDetailsService; +import datawave.microservice.authorization.user.DatawaveUserDetails; +import datawave.microservice.query.remote.QueryRequest; +import datawave.microservice.query.storage.QueryStatus; +import datawave.webservice.result.GenericResponse; + +@ExtendWith(SpringExtension.class) +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +@ActiveProfiles({"QueryStarterDefaults", "QueryStarterOverrides", "QueryServiceTest", RemoteAuthorizationServiceUserDetailsService.ACTIVATION_PROFILE}) +@ContextConfiguration(classes = {QueryService.class}) +public class QueryServicePlanTest extends AbstractQueryServiceTest { + + @Autowired + public ApplicationEventPublisher eventPublisher; + + @Test + public void testPlanSuccess() throws ParseException, IOException, ExecutionException, InterruptedException { + DatawaveUserDetails authUser = createUserDetails(); + UriComponents uri = createUri("EventQuery/plan"); + MultiValueMap map = createParams(); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); + + // setup a mock audit service + auditSentSetup(); + + // make the plan call asynchronously + Future> futureResp = Executors.newSingleThreadExecutor() + .submit(() -> jwtRestTemplate.exchange(requestEntity, GenericResponse.class)); + + long startTime = System.currentTimeMillis(); + while (queryRequestEvents.size() == 0 && (System.currentTimeMillis() - startTime) < TEST_WAIT_TIME_MILLIS) { + Thread.sleep(500); + } + + // verify that the plan event was published + Assertions.assertEquals(1, queryRequestEvents.size()); + RemoteQueryRequestEvent requestEvent = queryRequestEvents.removeLast(); + + Assertions.assertEquals("executor-unassigned:**", requestEvent.getDestinationService()); + Assertions.assertEquals(QueryRequest.Method.PLAN, requestEvent.getRequest().getMethod()); + + String queryId = requestEvent.getRequest().getQueryId(); + String plan = "some plan"; + + // save the plan to the query status object + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + queryStatus.setPlan(plan); + queryStorageCache.updateQueryStatus(queryStatus); + + // send the plan response + eventPublisher.publishEvent(new RemoteQueryRequestEvent(this, "executor-unassigned:**", "query:**", QueryRequest.plan(queryId))); + + ResponseEntity resp = futureResp.get(); + + // @formatter:off + GenericResponse genericResponse = assertGenericResponse( + true, + HttpStatus.Series.SUCCESSFUL, + resp); + // @formatter:on + + String receivedPlan = genericResponse.getResult(); + Assertions.assertEquals(receivedPlan, plan); + + // @formatter:off + assertQueryRequestEvent( + "query:**", + QueryRequest.Method.PLAN, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + + // verify that the query status was deleted + queryStatus = queryStorageCache.getQueryStatus(queryId); + Assertions.assertNull(queryStatus); + } +} diff --git a/service/src/test/java/datawave/microservice/query/QueryServiceRemoveTest.java b/service/src/test/java/datawave/microservice/query/QueryServiceRemoveTest.java new file mode 100644 index 00000000..ce3c8861 --- /dev/null +++ b/service/src/test/java/datawave/microservice/query/QueryServiceRemoveTest.java @@ -0,0 +1,419 @@ +package datawave.microservice.query; + +import java.util.Arrays; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.http.HttpMethod; +import org.springframework.http.RequestEntity; +import org.springframework.http.ResponseEntity; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit.jupiter.SpringExtension; +import org.springframework.web.util.UriComponents; + +import com.google.common.collect.Iterables; + +import datawave.microservice.authorization.service.RemoteAuthorizationServiceUserDetailsService; +import datawave.microservice.authorization.user.DatawaveUserDetails; +import datawave.microservice.query.remote.QueryRequest; +import datawave.microservice.query.storage.QueryStatus; +import datawave.webservice.result.GenericResponse; +import datawave.webservice.result.VoidResponse; + +@ExtendWith(SpringExtension.class) +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +@ActiveProfiles({"QueryStarterDefaults", "QueryStarterOverrides", "QueryServiceTest", RemoteAuthorizationServiceUserDetailsService.ACTIVATION_PROFILE}) +@ContextConfiguration(classes = {QueryService.class}) +public class QueryServiceRemoveTest extends AbstractQueryServiceTest { + + @Test + public void testRemoveSuccess_removeOnDefined() throws Exception { + DatawaveUserDetails authUser = createUserDetails(); + + // define a valid query + String queryId = defineQuery(authUser, createParams()); + + // remove the query + Future> removeFuture = removeQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity response = removeFuture.get(); + + Assertions.assertEquals(200, response.getStatusCodeValue()); + + // verify that original query was removed + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + + Assertions.assertNull(queryStatus); + + // verify that events were published + Assertions.assertEquals(0, queryRequestEvents.size()); + } + + @Test + public void testRemoveSuccess_removeOnClosed() throws Exception { + DatawaveUserDetails authUser = createUserDetails(); + + // create a valid query + String queryId = createQuery(authUser, createParams()); + + // close the query + Future> closeFuture = closeQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity closeResponse = closeFuture.get(); + + Assertions.assertEquals(200, closeResponse.getStatusCodeValue()); + + // remove the query + Future> removeFuture = removeQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity response = removeFuture.get(); + + Assertions.assertEquals(200, response.getStatusCodeValue()); + + // verify that original query was removed + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + + Assertions.assertNull(queryStatus); + + // verify that events were published + Assertions.assertEquals(2, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CLOSE, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @Test + public void testRemoveSuccess_removeOnCanceled() throws Exception { + DatawaveUserDetails authUser = createUserDetails(); + + // create a valid query + String queryId = createQuery(authUser, createParams()); + + // cancel the query + Future> cancelFuture = cancelQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity cancelResponse = cancelFuture.get(); + + Assertions.assertEquals(200, cancelResponse.getStatusCodeValue()); + + // remove the query + Future> removeFuture = removeQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity response = removeFuture.get(); + + Assertions.assertEquals(200, response.getStatusCodeValue()); + + // verify that original query was removed + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + + Assertions.assertNull(queryStatus); + + // verify that events were published + Assertions.assertEquals(3, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "query:**", + QueryRequest.Method.CANCEL, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CANCEL, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @Test + public void testRemoveFailure_removeOnCreated() throws Exception { + DatawaveUserDetails authUser = createUserDetails(); + + // create a valid query + String queryId = createQuery(authUser, createParams()); + + // remove the query + Future> removeFuture = removeQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity response = removeFuture.get(); + + Assertions.assertEquals(400, response.getStatusCodeValue()); + + Assertions.assertEquals("Cannot remove a running query.", Iterables.getOnlyElement(response.getBody().getExceptions()).getMessage()); + + // verify that original query was not removed + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + + Assertions.assertNotNull(queryStatus); + + // verify that events were published + Assertions.assertEquals(1, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @Test + public void testRemoveFailure_removeOnClosedActiveNext() throws Exception { + DatawaveUserDetails authUser = createUserDetails(); + + // create a valid query + String queryId = createQuery(authUser, createParams()); + + // call next on the query + nextQuery(authUser, queryId); + + // sleep until the next call registers + long currentTimeMillis = System.currentTimeMillis(); + boolean nextCallActive = queryStorageCache.getQueryStatus(queryId).getActiveNextCalls() > 0; + while (!nextCallActive && (System.currentTimeMillis() - currentTimeMillis) < TEST_WAIT_TIME_MILLIS) { + Thread.sleep(500); + nextCallActive = queryStorageCache.getQueryStatus(queryId).getActiveNextCalls() > 0; + } + + // close the query + Future> closeFuture = closeQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity closeResponse = closeFuture.get(); + + Assertions.assertEquals(200, closeResponse.getStatusCodeValue()); + + // remove the query + Future> removeFuture = removeQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity response = removeFuture.get(); + + Assertions.assertEquals(400, response.getStatusCodeValue()); + + Assertions.assertEquals("Cannot remove a running query.", Iterables.getOnlyElement(response.getBody().getExceptions()).getMessage()); + + // verify that original query was not removed + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + + Assertions.assertNotNull(queryStatus); + + // verify that events were published + Assertions.assertEquals(3, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.NEXT, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CLOSE, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @Test + public void testRemoveFailure_queryNotFound() throws Exception { + DatawaveUserDetails authUser = createUserDetails(); + + String queryId = UUID.randomUUID().toString(); + + // remove the query + UriComponents uri = createUri(queryId + "/remove"); + RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, null, null, HttpMethod.DELETE, uri); + + // close the query + Future> resetFuture = Executors.newSingleThreadExecutor() + .submit(() -> jwtRestTemplate.exchange(requestEntity, GenericResponse.class)); + + // the response should come back right away + ResponseEntity response = resetFuture.get(); + + Assertions.assertEquals(404, response.getStatusCodeValue()); + + // @formatter:off + assertQueryException( + "No query object matches this id. " + queryId, + "Exception with no cause caught", + "404-1", + Iterables.getOnlyElement(response.getBody().getExceptions())); + // @formatter:on + + // verify that no events were published + Assertions.assertEquals(0, queryRequestEvents.size()); + } + + @Test + public void testRemoveFailure_ownershipFailure() throws Exception { + DatawaveUserDetails authUser = createUserDetails(); + DatawaveUserDetails altAuthUser = createAltUserDetails(); + + // define a valid query + String queryId = defineQuery(authUser, createParams()); + + // remove the query + UriComponents uri = createUri(queryId + "/remove"); + RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(altAuthUser, null, null, HttpMethod.DELETE, uri); + + // close the query + Future> resetFuture = Executors.newSingleThreadExecutor() + .submit(() -> jwtRestTemplate.exchange(requestEntity, GenericResponse.class)); + + // the response should come back right away + ResponseEntity response = resetFuture.get(); + + Assertions.assertEquals(401, response.getStatusCodeValue()); + + // @formatter:off + assertQueryException( + "Current user does not match user that defined query. altuserdn != userdn", + "Exception with no cause caught", + "401-1", + Iterables.getOnlyElement(response.getBody().getExceptions())); + // @formatter:on + + // verify that no events were published + Assertions.assertEquals(0, queryRequestEvents.size()); + } + + @Test + public void testAdminRemoveSuccess() throws Exception { + DatawaveUserDetails authUser = createUserDetails(); + DatawaveUserDetails adminUser = createAltUserDetails(Arrays.asList("AuthorizedUser", "Administrator"), null); + + // define a valid query + String queryId = defineQuery(authUser, createParams()); + + // remove the query + Future> removeFuture = adminRemoveQuery(adminUser, queryId); + + // the response should come back right away + ResponseEntity response = removeFuture.get(); + + Assertions.assertEquals(200, response.getStatusCodeValue()); + + // verify that original query was removed + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + + Assertions.assertNull(queryStatus); + + // verify that events were published + Assertions.assertEquals(0, queryRequestEvents.size()); + } + + @Test + public void testAdminRemoveFailure_notAdminUser() throws Exception { + DatawaveUserDetails authUser = createUserDetails(); + + // define a valid query + String queryId = defineQuery(authUser, createParams()); + + // remove the query + UriComponents uri = createUri(queryId + "/adminRemove"); + RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, null, null, HttpMethod.DELETE, uri); + + // remove the queries + Future> removeFuture = Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, String.class)); + + // the response should come back right away + ResponseEntity response = removeFuture.get(); + + Assertions.assertEquals(403, response.getStatusCodeValue()); + + // verify that original query was not removed + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + + Assertions.assertNotNull(queryStatus); + + // verify that events were published + Assertions.assertEquals(0, queryRequestEvents.size()); + } + + @Test + public void testAdminRemoveAllSuccess() throws Exception { + DatawaveUserDetails adminUser = createUserDetails(Arrays.asList("AuthorizedUser", "Administrator"), null); + + // define a bunch of queries + for (int i = 0; i < 10; i++) { + defineQuery(adminUser, createParams()); + } + + // remove all queries as the admin user + Future> removeFuture = adminRemoveAllQueries(adminUser); + + // the response should come back right away + ResponseEntity removeResponse = removeFuture.get(); + + Assertions.assertEquals(200, removeResponse.getStatusCodeValue()); + + // verify that query status was created correctly + List queryStatusList = queryStorageCache.getQueryStatus(); + + Assertions.assertEquals(0, queryStatusList.size()); + + // verify that there are no events + Assertions.assertEquals(0, queryRequestEvents.size()); + } + + @Test + public void testAdminRemoveAllFailure_notAdminUser() throws Exception { + DatawaveUserDetails authUser = createUserDetails(); + + // define a bunch of queries + for (int i = 0; i < 10; i++) { + defineQuery(authUser, createParams()); + } + + UriComponents uri = createUri("/adminRemoveAll"); + RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, null, null, HttpMethod.DELETE, uri); + + // remove the queries + Future> removeFuture = Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, String.class)); + + // the response should come back right away + ResponseEntity removeResponse = removeFuture.get(); + + Assertions.assertEquals(403, removeResponse.getStatusCodeValue()); + + // verify that query status was created correctly + List queryStatusList = queryStorageCache.getQueryStatus(); + + Assertions.assertEquals(10, queryStatusList.size()); + + // verify that there are no events + Assertions.assertEquals(0, queryRequestEvents.size()); + } +} diff --git a/service/src/test/java/datawave/microservice/query/QueryServiceResetTest.java b/service/src/test/java/datawave/microservice/query/QueryServiceResetTest.java new file mode 100644 index 00000000..bec40ae6 --- /dev/null +++ b/service/src/test/java/datawave/microservice/query/QueryServiceResetTest.java @@ -0,0 +1,480 @@ +package datawave.microservice.query; + +import java.util.UUID; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.http.HttpMethod; +import org.springframework.http.HttpStatus; +import org.springframework.http.RequestEntity; +import org.springframework.http.ResponseEntity; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit.jupiter.SpringExtension; +import org.springframework.web.util.UriComponents; + +import com.google.common.collect.Iterables; + +import datawave.microservice.authorization.service.RemoteAuthorizationServiceUserDetailsService; +import datawave.microservice.authorization.user.DatawaveUserDetails; +import datawave.microservice.query.remote.QueryRequest; +import datawave.microservice.query.storage.QueryStatus; +import datawave.webservice.result.GenericResponse; +import datawave.webservice.result.VoidResponse; + +@ExtendWith(SpringExtension.class) +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +@ActiveProfiles({"QueryStarterDefaults", "QueryStarterOverrides", "QueryServiceTest", RemoteAuthorizationServiceUserDetailsService.ACTIVATION_PROFILE}) +@ContextConfiguration(classes = {QueryService.class}) +public class QueryServiceResetTest extends AbstractQueryServiceTest { + + @Test + public void testResetSuccess_resetOnDefined() throws Exception { + DatawaveUserDetails authUser = createUserDetails(); + + // define a valid query + long currentTimeMillis = System.currentTimeMillis(); + String queryId = defineQuery(authUser, createParams()); + + mockServer.reset(); + auditSentSetup(); + + // reset the query + Future> resetFuture = resetQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity response = resetFuture.get(); + + Assertions.assertEquals(200, response.getStatusCodeValue()); + + // @formatter:off + assertGenericResponse( + true, + HttpStatus.Series.SUCCESSFUL, + response); + // @formatter:on + + String resetQueryId = (String) response.getBody().getResult(); + + // verify that a new query id was created + Assertions.assertNotEquals(queryId, resetQueryId); + + // verify that an audit record was sent + assertAuditSent(resetQueryId); + + // verify that original query was canceled + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + + // @formatter:off + assertQueryStatus( + QueryStatus.QUERY_STATE.DEFINE, + 0, + 0, + 0, + 0, + currentTimeMillis, + queryStatus); + // @formatter:on + + // verify that new query was created + QueryStatus resetQueryStatus = queryStorageCache.getQueryStatus(resetQueryId); + + // @formatter:off + assertQueryStatus( + QueryStatus.QUERY_STATE.CREATE, + 0, + 0, + 0, + 0, + currentTimeMillis, + resetQueryStatus); + // @formatter:on + + // make sure the queries are equal (ignoring the query id) + queryStatus.getQuery().setId(resetQueryStatus.getQuery().getId()); + Assertions.assertEquals(queryStatus.getQuery(), resetQueryStatus.getQuery()); + + // verify that events were published + Assertions.assertEquals(1, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + resetQueryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @Test + public void testResetSuccess_resetOnCreated() throws Exception { + DatawaveUserDetails authUser = createUserDetails(); + + // define a valid query + long currentTimeMillis = System.currentTimeMillis(); + String queryId = createQuery(authUser, createParams()); + + mockServer.reset(); + auditSentSetup(); + + // reset the query + Future> resetFuture = resetQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity response = resetFuture.get(); + + Assertions.assertEquals(200, response.getStatusCodeValue()); + + // @formatter:off + assertGenericResponse( + true, + HttpStatus.Series.SUCCESSFUL, + response); + // @formatter:on + + String resetQueryId = (String) response.getBody().getResult(); + + // verify that a new query id was created + Assertions.assertNotEquals(queryId, resetQueryId); + + // verify that an audit record was sent + assertAuditSent(resetQueryId); + + // verify that original query was canceled + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + + // @formatter:off + assertQueryStatus( + QueryStatus.QUERY_STATE.CANCEL, + 0, + 0, + 0, + 0, + currentTimeMillis, + queryStatus); + // @formatter:on + + // verify that new query was created + QueryStatus resetQueryStatus = queryStorageCache.getQueryStatus(resetQueryId); + + // @formatter:off + assertQueryStatus( + QueryStatus.QUERY_STATE.CREATE, + 0, + 0, + 0, + 0, + currentTimeMillis, + resetQueryStatus); + // @formatter:on + + // make sure the queries are equal (ignoring the query id) + queryStatus.getQuery().setId(resetQueryStatus.getQuery().getId()); + Assertions.assertEquals(queryStatus.getQuery(), resetQueryStatus.getQuery()); + + // verify that events were published + Assertions.assertEquals(4, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "query:**", + QueryRequest.Method.CANCEL, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CANCEL, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + resetQueryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @Test + public void testResetSuccess_resetOnClosed() throws Exception { + DatawaveUserDetails authUser = createUserDetails(); + + // define a valid query + long currentTimeMillis = System.currentTimeMillis(); + String queryId = createQuery(authUser, createParams()); + + // close the query + Future> closeFuture = closeQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity closeResponse = closeFuture.get(); + + Assertions.assertEquals(200, closeResponse.getStatusCodeValue()); + + mockServer.reset(); + auditSentSetup(); + + // reset the query + Future> resetFuture = resetQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity response = resetFuture.get(); + + Assertions.assertEquals(200, response.getStatusCodeValue()); + + // @formatter:off + assertGenericResponse( + true, + HttpStatus.Series.SUCCESSFUL, + response); + // @formatter:on + + String resetQueryId = (String) response.getBody().getResult(); + + // verify that a new query id was created + Assertions.assertNotEquals(queryId, resetQueryId); + + // verify that an audit record was sent + assertAuditSent(resetQueryId); + + // verify that original query was closed + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + + // @formatter:off + assertQueryStatus( + QueryStatus.QUERY_STATE.CLOSE, + 0, + 0, + 0, + 0, + currentTimeMillis, + queryStatus); + // @formatter:on + + // verify that new query was created + QueryStatus resetQueryStatus = queryStorageCache.getQueryStatus(resetQueryId); + + // @formatter:off + assertQueryStatus( + QueryStatus.QUERY_STATE.CREATE, + 0, + 0, + 0, + 0, + currentTimeMillis, + resetQueryStatus); + // @formatter:on + + // make sure the queries are equal (ignoring the query id) + queryStatus.getQuery().setId(resetQueryStatus.getQuery().getId()); + Assertions.assertEquals(queryStatus.getQuery(), resetQueryStatus.getQuery()); + + // verify that events were published + Assertions.assertEquals(3, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CLOSE, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + resetQueryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @Test + public void testResetSuccess_resetOnCanceled() throws Exception { + DatawaveUserDetails authUser = createUserDetails(); + + // define a valid query + long currentTimeMillis = System.currentTimeMillis(); + String queryId = createQuery(authUser, createParams()); + + // cancel the query + Future> cancelFuture = cancelQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity cancelResponse = cancelFuture.get(); + + Assertions.assertEquals(200, cancelResponse.getStatusCodeValue()); + + mockServer.reset(); + auditSentSetup(); + + // reset the query + Future> resetFuture = resetQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity response = resetFuture.get(); + + Assertions.assertEquals(200, response.getStatusCodeValue()); + + // @formatter:off + assertGenericResponse( + true, + HttpStatus.Series.SUCCESSFUL, + response); + // @formatter:on + + String resetQueryId = (String) response.getBody().getResult(); + + // verify that a new query id was created + Assertions.assertNotEquals(queryId, resetQueryId); + + // verify that an audit record was sent + assertAuditSent(resetQueryId); + + // verify that original query was canceled + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + + // @formatter:off + assertQueryStatus( + QueryStatus.QUERY_STATE.CANCEL, + 0, + 0, + 0, + 0, + currentTimeMillis, + queryStatus); + // @formatter:on + + // verify that new query was created + QueryStatus resetQueryStatus = queryStorageCache.getQueryStatus(resetQueryId); + + // @formatter:off + assertQueryStatus( + QueryStatus.QUERY_STATE.CREATE, + 0, + 0, + 0, + 0, + currentTimeMillis, + resetQueryStatus); + // @formatter:on + + // make sure the queries are equal (ignoring the query id) + queryStatus.getQuery().setId(resetQueryStatus.getQuery().getId()); + Assertions.assertEquals(queryStatus.getQuery(), resetQueryStatus.getQuery()); + + // verify that events were published + Assertions.assertEquals(4, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "query:**", + QueryRequest.Method.CANCEL, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CANCEL, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + resetQueryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @Test + public void testResetFailure_queryNotFound() throws Exception { + DatawaveUserDetails authUser = createUserDetails(); + + String queryId = UUID.randomUUID().toString(); + + auditNotSentSetup(); + + // reset the query + UriComponents uri = createUri(queryId + "/reset"); + RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, null, null, HttpMethod.PUT, uri); + + // close the query + Future> resetFuture = Executors.newSingleThreadExecutor() + .submit(() -> jwtRestTemplate.exchange(requestEntity, GenericResponse.class)); + + // the response should come back right away + ResponseEntity response = resetFuture.get(); + + Assertions.assertEquals(404, response.getStatusCodeValue()); + + // make sure no audits were sent + assertAuditNotSent(); + + // @formatter:off + assertQueryException( + "No query object matches this id. " + queryId, + "Exception with no cause caught", + "404-1", + Iterables.getOnlyElement(response.getBody().getExceptions())); + // @formatter:on + + // verify that no events were published + Assertions.assertEquals(0, queryRequestEvents.size()); + } + + @Test + public void testResetFailure_ownershipFailure() throws Exception { + DatawaveUserDetails authUser = createUserDetails(); + DatawaveUserDetails altAuthUser = createAltUserDetails(); + + // define a valid query + String queryId = createQuery(authUser, createParams()); + + mockServer.reset(); + auditNotSentSetup(); + + // reset the query + UriComponents uri = createUri(queryId + "/reset"); + RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(altAuthUser, null, null, HttpMethod.PUT, uri); + + // close the query + Future> resetFuture = Executors.newSingleThreadExecutor() + .submit(() -> jwtRestTemplate.exchange(requestEntity, GenericResponse.class)); + + // the response should come back right away + ResponseEntity response = resetFuture.get(); + + Assertions.assertEquals(401, response.getStatusCodeValue()); + + // make sure no audits were sent + assertAuditNotSent(); + + // @formatter:off + assertQueryException( + "Current user does not match user that defined query. altuserdn != userdn", + "Exception with no cause caught", + "401-1", + Iterables.getOnlyElement(response.getBody().getExceptions())); + // @formatter:on + + // verify that no events were published + Assertions.assertEquals(1, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } +} diff --git a/service/src/test/java/datawave/microservice/query/QueryServiceUpdateTest.java b/service/src/test/java/datawave/microservice/query/QueryServiceUpdateTest.java new file mode 100644 index 00000000..c1f3bcc6 --- /dev/null +++ b/service/src/test/java/datawave/microservice/query/QueryServiceUpdateTest.java @@ -0,0 +1,448 @@ +package datawave.microservice.query; + +import static datawave.microservice.query.QueryImpl.BEGIN_DATE; +import static datawave.microservice.query.QueryImpl.END_DATE; +import static datawave.microservice.query.QueryImpl.PAGESIZE; +import static datawave.microservice.query.QueryImpl.QUERY; +import static datawave.microservice.query.QueryParameters.QUERY_LOGIC_NAME; +import static datawave.webservice.common.audit.AuditParameters.QUERY_AUTHORIZATIONS; +import static datawave.webservice.common.audit.AuditParameters.QUERY_STRING; + +import java.util.Arrays; +import java.util.UUID; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.http.HttpMethod; +import org.springframework.http.RequestEntity; +import org.springframework.http.ResponseEntity; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit.jupiter.SpringExtension; +import org.springframework.util.LinkedMultiValueMap; +import org.springframework.util.MultiValueMap; +import org.springframework.web.util.UriComponents; + +import com.google.common.collect.Iterables; + +import datawave.microservice.authorization.service.RemoteAuthorizationServiceUserDetailsService; +import datawave.microservice.authorization.user.DatawaveUserDetails; +import datawave.microservice.query.remote.QueryRequest; +import datawave.microservice.query.storage.QueryStatus; +import datawave.webservice.result.GenericResponse; +import datawave.webservice.result.VoidResponse; + +@ExtendWith(SpringExtension.class) +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +@ActiveProfiles({"QueryStarterDefaults", "QueryStarterOverrides", "QueryServiceTest", RemoteAuthorizationServiceUserDetailsService.ACTIVATION_PROFILE}) +@ContextConfiguration(classes = {QueryService.class}) +public class QueryServiceUpdateTest extends AbstractQueryServiceTest { + + @Test + public void testUpdateSuccess_updateOnDefined() throws Exception { + DatawaveUserDetails authUser = createUserDetails(null, Arrays.asList("ALL", "NONE")); + + // define a valid query + String queryId = defineQuery(authUser, createParams()); + + String newQuery = "SOME_OTHER_FIELD:SOME_OTHER_VALUE"; + String newAuths = "ALL,NONE"; + String newBegin = "20100101 000000.000"; + String newEnd = "20600101 000000.000"; + String newLogic = "AltEventQuery"; + int newPageSize = 100; + + MultiValueMap updateParams = new LinkedMultiValueMap<>(); + updateParams.set(QUERY, newQuery); + updateParams.set(QUERY_AUTHORIZATIONS, newAuths); + updateParams.set(BEGIN_DATE, newBegin); + updateParams.set(END_DATE, newEnd); + updateParams.set(QUERY_LOGIC_NAME, newLogic); + updateParams.set(PAGESIZE, Integer.toString(newPageSize)); + + // update the query + Future> updateFuture = updateQuery(authUser, queryId, updateParams); + + // the response should come back right away + ResponseEntity response = updateFuture.get(); + + Assertions.assertEquals(200, response.getStatusCodeValue()); + + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + + // make sure the query was updated + Assertions.assertEquals(newQuery, queryStatus.getQuery().getQuery()); + Assertions.assertEquals(newAuths, queryStatus.getQuery().getQueryAuthorizations()); + Assertions.assertEquals(newBegin, DefaultQueryParameters.formatDate(queryStatus.getQuery().getBeginDate())); + Assertions.assertEquals(newEnd, DefaultQueryParameters.formatDate(queryStatus.getQuery().getEndDate())); + Assertions.assertEquals(newLogic, queryStatus.getQuery().getQueryLogicName()); + Assertions.assertEquals(newPageSize, queryStatus.getQuery().getPagesize()); + + // verify that no events were published + Assertions.assertEquals(0, queryRequestEvents.size()); + } + + @Test + public void testUpdateSuccess_updateOnCreated() throws Exception { + DatawaveUserDetails authUser = createUserDetails(); + + // create a valid query + String queryId = createQuery(authUser, createParams()); + + int newPageSize = 100; + + MultiValueMap updateParams = new LinkedMultiValueMap<>(); + updateParams.set(PAGESIZE, Integer.toString(newPageSize)); + + // update the query + Future> updateFuture = updateQuery(authUser, queryId, updateParams); + + // the response should come back right away + ResponseEntity response = updateFuture.get(); + + Assertions.assertEquals(200, response.getStatusCodeValue()); + + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + + // make sure the query was updated + Assertions.assertEquals(newPageSize, queryStatus.getQuery().getPagesize()); + + // verify that events were published + Assertions.assertEquals(1, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @Test + public void testUpdateFailure_unsafeParamUpdateQuery() throws Exception { + DatawaveUserDetails authUser = createUserDetails(); + + // create a valid query + String queryId = createQuery(authUser, createParams()); + + String newQuery = "SOME_OTHER_FIELD:SOME_OTHER_VALUE"; + + MultiValueMap updateParams = new LinkedMultiValueMap<>(); + updateParams.set(QUERY, newQuery); + + // update the query + UriComponents uri = createUri(queryId + "/update"); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, updateParams, null, HttpMethod.PUT, uri); + + // make the update call asynchronously + Future> updateFuture = Executors.newSingleThreadExecutor() + .submit(() -> jwtRestTemplate.exchange(requestEntity, VoidResponse.class)); + + // the response should come back right away + ResponseEntity response = updateFuture.get(); + + Assertions.assertEquals(400, response.getStatusCodeValue()); + + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + + // make sure the query was not updated + Assertions.assertEquals(TEST_QUERY_STRING, queryStatus.getQuery().getQuery()); + + // @formatter:off + assertQueryException( + "Cannot update the following parameters for a running query: query", + "Exception with no cause caught", + "400-1", + Iterables.getOnlyElement(response.getBody().getExceptions())); + // @formatter:on + + // verify that events were published + Assertions.assertEquals(1, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @Test + public void testUpdateFailure_unsafeParamUpdateDate() throws Exception { + DatawaveUserDetails authUser = createUserDetails(); + + // create a valid query + String queryId = createQuery(authUser, createParams()); + + String newBegin = "20100101 000000.000"; + String newEnd = "20600101 000000.000"; + + MultiValueMap updateParams = new LinkedMultiValueMap<>(); + updateParams.set(BEGIN_DATE, newBegin); + updateParams.set(END_DATE, newEnd); + + // update the query + UriComponents uri = createUri(queryId + "/update"); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, updateParams, null, HttpMethod.PUT, uri); + + // make the update call asynchronously + Future> updateFuture = Executors.newSingleThreadExecutor() + .submit(() -> jwtRestTemplate.exchange(requestEntity, VoidResponse.class)); + + // the response should come back right away + ResponseEntity response = updateFuture.get(); + + Assertions.assertEquals(400, response.getStatusCodeValue()); + + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + + // make sure the query was not updated + Assertions.assertEquals(TEST_QUERY_BEGIN, DefaultQueryParameters.formatDate(queryStatus.getQuery().getBeginDate())); + Assertions.assertEquals(TEST_QUERY_END, DefaultQueryParameters.formatDate(queryStatus.getQuery().getEndDate())); + + // @formatter:off + assertQueryException( + "Cannot update the following parameters for a running query: begin, end", + "Exception with no cause caught", + "400-1", + Iterables.getOnlyElement(response.getBody().getExceptions())); + // @formatter:on + + // verify that events were published + Assertions.assertEquals(1, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @Test + public void testUpdateFailure_unsafeParamUpdateLogic() throws Exception { + DatawaveUserDetails authUser = createUserDetails(); + + // create a valid query + String queryId = createQuery(authUser, createParams()); + + String newLogic = "AltEventQuery"; + + MultiValueMap updateParams = new LinkedMultiValueMap<>(); + updateParams.set(QUERY_LOGIC_NAME, newLogic); + + // update the query + UriComponents uri = createUri(queryId + "/update"); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, updateParams, null, HttpMethod.PUT, uri); + + // make the update call asynchronously + Future> updateFuture = Executors.newSingleThreadExecutor() + .submit(() -> jwtRestTemplate.exchange(requestEntity, VoidResponse.class)); + + // the response should come back right away + ResponseEntity response = updateFuture.get(); + + Assertions.assertEquals(400, response.getStatusCodeValue()); + + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + + // make sure the query was not updated + Assertions.assertEquals("EventQuery", queryStatus.getQuery().getQueryLogicName()); + + // @formatter:off + assertQueryException( + "Cannot update the following parameters for a running query: logicName", + "Exception with no cause caught", + "400-1", + Iterables.getOnlyElement(response.getBody().getExceptions())); + // @formatter:on + + // verify that events were published + Assertions.assertEquals(1, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @Test + public void testUpdateFailure_unsafeParamUpdateAuths() throws Exception { + DatawaveUserDetails authUser = createUserDetails(); + + // create a valid query + String queryId = createQuery(authUser, createParams()); + + String newAuths = "ALL,NONE"; + + MultiValueMap updateParams = new LinkedMultiValueMap<>(); + updateParams.set(QUERY_AUTHORIZATIONS, newAuths); + + // update the query + UriComponents uri = createUri(queryId + "/update"); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, updateParams, null, HttpMethod.PUT, uri); + + // make the update call asynchronously + Future> updateFuture = Executors.newSingleThreadExecutor() + .submit(() -> jwtRestTemplate.exchange(requestEntity, VoidResponse.class)); + + // the response should come back right away + ResponseEntity response = updateFuture.get(); + + Assertions.assertEquals(400, response.getStatusCodeValue()); + + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + + // make sure the query was not updated + Assertions.assertEquals(TEST_QUERY_AUTHORIZATIONS, queryStatus.getQuery().getQueryAuthorizations()); + + // @formatter:off + assertQueryException( + "Cannot update the following parameters for a running query: auths", + "Exception with no cause caught", + "400-1", + Iterables.getOnlyElement(response.getBody().getExceptions())); + // @formatter:on + + // verify that events were published + Assertions.assertEquals(1, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @Test + public void testUpdateFailure_nullParams() throws Exception { + DatawaveUserDetails authUser = createUserDetails(); + + // create a valid query + String queryId = createQuery(authUser, createParams()); + + MultiValueMap updateParams = new LinkedMultiValueMap<>(); + + // update the query + UriComponents uri = createUri(queryId + "/update"); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, updateParams, null, HttpMethod.PUT, uri); + + // make the update call asynchronously + Future> updateFuture = Executors.newSingleThreadExecutor() + .submit(() -> jwtRestTemplate.exchange(requestEntity, VoidResponse.class)); + + // the response should come back right away + ResponseEntity response = updateFuture.get(); + + Assertions.assertEquals(400, response.getStatusCodeValue()); + + // @formatter:off + assertQueryException( + "No parameters specified for update.", + "Exception with no cause caught", + "400-1", + Iterables.getOnlyElement(response.getBody().getExceptions())); + // @formatter:on + + // verify that events were published + Assertions.assertEquals(1, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @Test + public void testUpdateFailure_queryNotFound() throws Exception { + DatawaveUserDetails authUser = createUserDetails(); + + String queryId = UUID.randomUUID().toString(); + + MultiValueMap updateParams = new LinkedMultiValueMap<>(); + updateParams.set(QUERY_STRING, TEST_QUERY_STRING); + + // update the query + UriComponents uri = createUri(queryId + "/update"); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, updateParams, null, HttpMethod.PUT, uri); + + // make the update call asynchronously + Future> updateFuture = Executors.newSingleThreadExecutor() + .submit(() -> jwtRestTemplate.exchange(requestEntity, VoidResponse.class)); + + // the response should come back right away + ResponseEntity response = updateFuture.get(); + + Assertions.assertEquals(404, response.getStatusCodeValue()); + + // @formatter:off + assertQueryException( + "No query object matches this id. " + queryId, + "Exception with no cause caught", + "404-1", + Iterables.getOnlyElement(response.getBody().getExceptions())); + // @formatter:on + + // verify that no events were published + Assertions.assertEquals(0, queryRequestEvents.size()); + } + + @Test + public void testUpdateFailure_ownershipFailure() throws Exception { + DatawaveUserDetails authUser = createUserDetails(); + DatawaveUserDetails altAuthUser = createAltUserDetails(); + + // define a valid query + String queryId = defineQuery(authUser, createParams()); + + String newQuery = "SOME_OTHER_FIELD:SOME_OTHER_VALUE"; + + MultiValueMap updateParams = new LinkedMultiValueMap<>(); + updateParams.set(QUERY, newQuery); + + // update the query + UriComponents uri = createUri(queryId + "/update"); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(altAuthUser, updateParams, null, HttpMethod.PUT, uri); + + // make the update call asynchronously + Future> updateFuture = Executors.newSingleThreadExecutor() + .submit(() -> jwtRestTemplate.exchange(requestEntity, VoidResponse.class)); + + // the response should come back right away + ResponseEntity response = updateFuture.get(); + + Assertions.assertEquals(401, response.getStatusCodeValue()); + + // @formatter:off + assertQueryException( + "Current user does not match user that defined query. altuserdn != userdn", + "Exception with no cause caught", + "401-1", + Iterables.getOnlyElement(response.getBody().getExceptions())); + // @formatter:on + + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + + // make sure the query was not updated + Assertions.assertEquals(TEST_QUERY_STRING, queryStatus.getQuery().getQuery()); + + // verify that no events were published + Assertions.assertEquals(0, queryRequestEvents.size()); + } +} diff --git a/service/src/test/java/datawave/microservice/query/lookup/LookupServiceTest.java b/service/src/test/java/datawave/microservice/query/lookup/LookupServiceTest.java new file mode 100644 index 00000000..8a14ae6c --- /dev/null +++ b/service/src/test/java/datawave/microservice/query/lookup/LookupServiceTest.java @@ -0,0 +1,855 @@ +package datawave.microservice.query.lookup; + +import static datawave.microservice.query.QueryParameters.QUERY_MAX_CONCURRENT_TASKS; +import static datawave.microservice.query.QueryParameters.QUERY_MAX_RESULTS_OVERRIDE; +import static datawave.microservice.query.lookup.LookupService.LOOKUP_UUID_PAIRS; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.stream.Collectors; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.http.HttpMethod; +import org.springframework.http.RequestEntity; +import org.springframework.http.ResponseEntity; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.junit.jupiter.SpringExtension; +import org.springframework.util.LinkedMultiValueMap; +import org.springframework.util.MultiValueMap; +import org.springframework.web.util.UriComponents; + +import com.google.common.collect.Iterables; + +import datawave.core.query.configuration.GenericQueryConfiguration; +import datawave.core.query.logic.QueryKey; +import datawave.marking.ColumnVisibilitySecurityMarking; +import datawave.microservice.authorization.service.RemoteAuthorizationServiceUserDetailsService; +import datawave.microservice.authorization.user.DatawaveUserDetails; +import datawave.microservice.query.AbstractQueryServiceTest; +import datawave.microservice.query.DefaultQueryParameters; +import datawave.microservice.query.messaging.QueryResultsPublisher; +import datawave.microservice.query.messaging.Result; +import datawave.microservice.query.remote.QueryRequest; +import datawave.microservice.query.storage.QueryStatus; +import datawave.webservice.query.result.event.DefaultEvent; +import datawave.webservice.query.result.event.DefaultField; +import datawave.webservice.query.result.event.Metadata; +import datawave.webservice.result.DefaultEventQueryResponse; +import datawave.webservice.result.VoidResponse; + +@ExtendWith(SpringExtension.class) +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +@ActiveProfiles({"QueryStarterDefaults", "QueryStarterOverrides", "QueryServiceTest", RemoteAuthorizationServiceUserDetailsService.ACTIVATION_PROFILE}) +public class LookupServiceTest extends AbstractQueryServiceTest { + + @Autowired + public LookupProperties lookupProperties; + + @Test + public void testLookupUUIDSuccess() throws Exception { + DatawaveUserDetails authUser = createUserDetails(); + MultiValueMap uuidParams = createUUIDParams(); + + String uuidType = "PAGE_TITLE"; + String uuid = "anarchy"; + + Future> future = lookupUUID(authUser, uuidParams, uuidType, uuid); + + String queryId = null; + + // get the lookup query id + QueryStatus queryStatus = null; + long startTime = System.currentTimeMillis(); + while ((System.currentTimeMillis() - startTime) < TEST_WAIT_TIME_MILLIS && queryId == null) { + List queryStatuses = queryStorageCache.getQueryStatus(); + if (queryStatuses.size() > 0) { + queryStatus = queryStatuses.get(0); + queryId = queryStatuses.get(0).getQueryKey().getQueryId(); + } else { + Thread.sleep(500); + } + } + + // pump enough results into the queue to trigger a complete page + int pageSize = queryStatus.getQuery().getPagesize(); + + // test field value pairings + MultiValueMap fieldValues = new LinkedMultiValueMap<>(); + fieldValues.add(uuidType, uuid); + + // add a config object to the query status, which would normally be added by the executor service + queryStatus.setConfig(new GenericQueryConfiguration()); + queryStorageCache.updateQueryStatus(queryStatus); + + // @formatter:off + publishEventsToQueue( + queryId, + pageSize, + fieldValues, + "ALL"); + // @formatter:on + + ResponseEntity response = future.get(); + + Assertions.assertEquals(200, response.getStatusCodeValue()); + + // verify some headers + Assertions.assertEquals("1", Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-query-page-number")))); + Assertions.assertEquals("false", Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-Partial-Results")))); + Assertions.assertEquals("false", Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-query-last-page")))); + + DefaultEventQueryResponse queryResponse = (DefaultEventQueryResponse) response.getBody(); + + // verify the query response + // @formatter:off + assertQueryResponse( + queryId, + "LuceneUUIDEventQuery", + 1, + false, + Long.parseLong(Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-OperationTimeInMS")))), + 1, + Collections.singletonList(uuidType), + pageSize, + Objects.requireNonNull(queryResponse)); + // @formatter:on + + // validate one of the events + DefaultEvent event = (DefaultEvent) queryResponse.getEvents().get(0); + // @formatter:off + assertDefaultEvent( + Collections.singletonList(uuidType), + Collections.singletonList(uuid), + event); + // @formatter:on + + // verify that the correct events were published + Assertions.assertEquals(3, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.NEXT, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CLOSE, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @Test + public void testBatchLookupUUIDSuccess() throws Exception { + DatawaveUserDetails authUser = createUserDetails(); + + MultiValueMap uuidParams = createUUIDParams(); + uuidParams.add(LOOKUP_UUID_PAIRS, "PAGE_TITLE:anarchy OR PAGE_TITLE:accessiblecomputing"); + + Future> future = batchLookupUUID(authUser, uuidParams); + + String queryId = null; + + // get the lookup query id + QueryStatus queryStatus = null; + long startTime = System.currentTimeMillis(); + while ((System.currentTimeMillis() - startTime) < TEST_WAIT_TIME_MILLIS && queryId == null) { + List queryStatuses = queryStorageCache.getQueryStatus(); + if (queryStatuses.size() > 0) { + queryStatus = queryStatuses.get(0); + queryId = queryStatuses.get(0).getQueryKey().getQueryId(); + } else { + Thread.sleep(500); + } + } + + // pump enough results into the queue to trigger a complete page + int pageSize = queryStorageCache.getQueryStatus(queryId).getQuery().getPagesize(); + + // test field value pairings + MultiValueMap fieldValues = new LinkedMultiValueMap<>(); + fieldValues.add("PAGE_TITLE", "anarchy"); + fieldValues.add("PAGE_TITLE", "accessiblecomputing"); + + // add a config object to the query status, which would normally be added by the executor service + queryStatus.setConfig(new GenericQueryConfiguration()); + queryStorageCache.updateQueryStatus(queryStatus); + + // @formatter:off + publishEventsToQueue( + queryId, + pageSize, + fieldValues, + "ALL"); + // @formatter:on + + ResponseEntity response = future.get(); + + Assertions.assertEquals(200, response.getStatusCodeValue()); + + // verify some headers + Assertions.assertEquals("1", Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-query-page-number")))); + Assertions.assertEquals("false", Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-Partial-Results")))); + Assertions.assertEquals("false", Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-query-last-page")))); + + DefaultEventQueryResponse queryResponse = (DefaultEventQueryResponse) response.getBody(); + + // verify the query response + // @formatter:off + assertQueryResponse( + queryId, + "LuceneUUIDEventQuery", + 1, + false, + Long.parseLong(Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-OperationTimeInMS")))), + 1, + Collections.singletonList("PAGE_TITLE"), + pageSize, + Objects.requireNonNull(queryResponse)); + // @formatter:on + + // validate one of the events + DefaultEvent event = (DefaultEvent) queryResponse.getEvents().get(0); + // @formatter:off + assertDefaultEvent( + Arrays.asList("PAGE_TITLE", "PAGE_TITLE"), + Arrays.asList("anarchy", "accessiblecomputing"), + event); + // @formatter:on + + // verify that the correct events were published + Assertions.assertEquals(3, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.NEXT, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CLOSE, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @Test + public void testLookupContentUUIDSuccess() throws Exception { + DatawaveUserDetails authUser = createUserDetails(); + MultiValueMap uuidParams = createUUIDParams(); + + String uuidType = "PAGE_TITLE"; + String uuid = "anarchy"; + + Future> future = lookupContentUUID(authUser, uuidParams, uuidType, uuid); + + String queryId = null; + + // get the lookup query id + QueryStatus queryStatus = null; + long startTime = System.currentTimeMillis(); + while ((System.currentTimeMillis() - startTime) < TEST_WAIT_TIME_MILLIS && queryId == null) { + List queryStatuses = queryStorageCache.getQueryStatus(); + if (queryStatuses.size() > 0) { + queryStatus = queryStatuses.get(0); + queryId = queryStatuses.get(0).getQueryKey().getQueryId(); + } else { + Thread.sleep(500); + } + } + + // pump enough results into the queue to trigger a complete page + int pageSize = queryStatus.getQuery().getPagesize(); + + // test field value pairings + MultiValueMap fieldValues = new LinkedMultiValueMap<>(); + fieldValues.add(uuidType, uuid); + + // add a config object to the query status, which would normally be added by the executor service + queryStatus.setConfig(new GenericQueryConfiguration()); + queryStorageCache.updateQueryStatus(queryStatus); + + // @formatter:off + publishEventsToQueue( + queryId, + pageSize, + fieldValues, + "ALL"); + // @formatter:on + + Set contentQueryIds = null; + // wait for the initial event query to be closed + startTime = System.currentTimeMillis(); + while ((System.currentTimeMillis() - startTime) < TEST_WAIT_TIME_MILLIS && contentQueryIds == null) { + final String eventQueryId = queryId; + List queryStatuses = queryStorageCache.getQueryStatus(); + if (queryStatuses.size() == 1 + Math.ceil((double) pageSize / lookupProperties.getBatchLookupLimit())) { + contentQueryIds = queryStatuses.stream().map(QueryStatus::getQueryKey).map(QueryKey::getQueryId) + .filter(contentQueryId -> !contentQueryId.equals(eventQueryId)).collect(Collectors.toSet()); + } + // add a config object to the query status, which would normally be added by the executor service + for (QueryStatus status : queryStatuses) { + status.setConfig(new GenericQueryConfiguration()); + queryStorageCache.updateQueryStatus(status); + } + Thread.sleep(500); + } + + Assertions.assertNotNull(contentQueryIds); + for (String contentQueryId : contentQueryIds) { + MultiValueMap contentFieldValues = new LinkedMultiValueMap<>(); + contentFieldValues.add("CONTENT", "look I made you some content!"); + + // @formatter:off + publishEventsToQueue( + contentQueryId, + pageSize, + contentFieldValues, + "ALL"); + // @formatter:on + } + + ResponseEntity response = future.get(); + + Assertions.assertEquals(200, response.getStatusCodeValue()); + + // verify some headers + Assertions.assertEquals("1", Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-query-page-number")))); + Assertions.assertEquals("false", Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-Partial-Results")))); + Assertions.assertEquals("false", Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-query-last-page")))); + + DefaultEventQueryResponse queryResponse = (DefaultEventQueryResponse) response.getBody(); + + String responseQueryId = queryResponse.getQueryId(); + + Assertions.assertTrue(contentQueryIds.contains(responseQueryId)); + + // verify the query response + // @formatter:off + assertContentQueryResponse( + responseQueryId, + "ContentQuery", + 1, + false, + Long.parseLong(Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-OperationTimeInMS")))), + pageSize, + Objects.requireNonNull(queryResponse)); + // @formatter:on + + // validate one of the events + DefaultEvent event = (DefaultEvent) queryResponse.getEvents().get(0); + // @formatter:off + assertDefaultEvent( + Collections.singletonList("CONTENT"), + Collections.singletonList("look I made you some content!"), + event); + // @formatter:on + + // verify that the correct events were published + Assertions.assertEquals(7, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.NEXT, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CLOSE, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + responseQueryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.NEXT, + responseQueryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.NEXT, + responseQueryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CLOSE, + responseQueryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @Test + public void testBatchLookupContentUUIDSuccess() throws Exception { + DatawaveUserDetails authUser = createUserDetails(); + + MultiValueMap uuidParams = createUUIDParams(); + uuidParams.add(LOOKUP_UUID_PAIRS, "PAGE_TITLE:anarchy OR PAGE_TITLE:accessiblecomputing"); + uuidParams.add(QUERY_MAX_RESULTS_OVERRIDE, "10"); + + Future> future = batchLookupContentUUID(authUser, uuidParams); + + String queryId = null; + + // get the lookup query id + QueryStatus queryStatus = null; + long startTime = System.currentTimeMillis(); + while ((System.currentTimeMillis() - startTime) < TEST_WAIT_TIME_MILLIS && queryId == null) { + List queryStatuses = queryStorageCache.getQueryStatus(); + if (queryStatuses.size() > 0) { + queryStatus = queryStatuses.get(0); + queryId = queryStatuses.get(0).getQueryKey().getQueryId(); + } else { + Thread.sleep(500); + } + } + + // pump enough results into the queue to trigger a complete page + int pageSize = queryStatus.getQuery().getPagesize(); + + // test field value pairings + MultiValueMap fieldValues = new LinkedMultiValueMap<>(); + fieldValues.add("PAGE_TITLE", "anarchy"); + fieldValues.add("PAGE_TITLE", "accessiblecomputing"); + + // add a config object to the query status, which would normally be added by the executor service + queryStatus.setConfig(new GenericQueryConfiguration()); + queryStorageCache.updateQueryStatus(queryStatus); + + // @formatter:off + publishEventsToQueue( + queryId, + pageSize, + fieldValues, + "ALL"); + // @formatter:on + + Set contentQueryIds = null; + // wait for the initial event query to be closed + startTime = System.currentTimeMillis(); + while ((System.currentTimeMillis() - startTime) < TEST_WAIT_TIME_MILLIS && contentQueryIds == null) { + final String eventQueryId = queryId; + List queryStatuses = queryStorageCache.getQueryStatus(); + if (queryStatuses.size() == 1 + Math.ceil((double) pageSize / lookupProperties.getBatchLookupLimit())) { + contentQueryIds = queryStatuses.stream().map(QueryStatus::getQueryKey).map(QueryKey::getQueryId) + .filter(contentQueryId -> !contentQueryId.equals(eventQueryId)).collect(Collectors.toSet()); + } + // add a config object to the query status, which would normally be added by the executor service + for (QueryStatus status : queryStatuses) { + status.setConfig(new GenericQueryConfiguration()); + queryStorageCache.updateQueryStatus(status); + } + Thread.sleep(500); + } + + Assertions.assertNotNull(contentQueryIds); + for (String contentQueryId : contentQueryIds) { + MultiValueMap contentFieldValues = new LinkedMultiValueMap<>(); + contentFieldValues.add("CONTENT", "look I made you some content!"); + + // @formatter:off + publishEventsToQueue( + contentQueryId, + pageSize, + contentFieldValues, + "ALL"); + // @formatter:on + } + + // wait until each query has read its results, and then close it + for (String contentQueryId : contentQueryIds) { + QueryStatus status = queryStorageCache.getQueryStatus(contentQueryId); + startTime = System.currentTimeMillis(); + while ((System.currentTimeMillis() - startTime) < TEST_WAIT_TIME_MILLIS && status.getNumResultsConsumed() < pageSize) { + Thread.sleep(500); + status = queryStorageCache.getQueryStatus(contentQueryId); + } + } + + ResponseEntity response = future.get(); + + Assertions.assertEquals(200, response.getStatusCodeValue()); + + // verify some headers + Assertions.assertEquals("1", Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-query-page-number")))); + Assertions.assertEquals("false", Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-Partial-Results")))); + Assertions.assertEquals("false", Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-query-last-page")))); + + DefaultEventQueryResponse queryResponse = (DefaultEventQueryResponse) response.getBody(); + + String responseQueryId = queryResponse.getQueryId(); + + Assertions.assertTrue(contentQueryIds.contains(responseQueryId)); + + // verify the query response + // @formatter:off + assertContentQueryResponse( + responseQueryId, + "ContentQuery", + 1, + false, + Long.parseLong(Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-OperationTimeInMS")))), + pageSize, + Objects.requireNonNull(queryResponse)); + // @formatter:on + + // validate one of the events + DefaultEvent event = (DefaultEvent) queryResponse.getEvents().get(0); + // @formatter:off + assertDefaultEvent( + Collections.singletonList("CONTENT"), + Collections.singletonList("look I made you some content!"), + event); + // @formatter:on + + // verify that the correct events were published + Assertions.assertEquals(7, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.NEXT, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CLOSE, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + responseQueryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.NEXT, + responseQueryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.NEXT, + responseQueryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CLOSE, + responseQueryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @Test + public void testBatchLookupUUIDFailure_noLookupUUIDPairs() throws Exception { + DatawaveUserDetails authUser = createUserDetails(); + + MultiValueMap uuidParams = createUUIDParams(); + + UriComponents uri = createUri("lookupUUID"); + + // not testing audit with this method + auditIgnoreSetup(); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, uuidParams, null, HttpMethod.POST, uri); + + ResponseEntity response = jwtRestTemplate.exchange(requestEntity, VoidResponse.class); + + Assertions.assertEquals(400, response.getStatusCodeValue()); + + // @formatter:off + assertQueryException( + "Missing required parameter.", + "Exception with no cause caught", + "400-40", + Iterables.getOnlyElement(response.getBody().getExceptions())); + // @formatter:on + } + + @Test + public void testBatchLookupUUIDFailure_mixedQueryLogics() throws Exception { + DatawaveUserDetails authUser = createUserDetails(); + + MultiValueMap uuidParams = createUUIDParams(); + uuidParams.add(LOOKUP_UUID_PAIRS, "PAGE_TITLE:anarchy OR PAGE_NUMBER:accessiblecomputing"); + + UriComponents uri = createUri("lookupUUID"); + + // not testing audit with this method + auditIgnoreSetup(); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, uuidParams, null, HttpMethod.POST, uri); + + ResponseEntity response = jwtRestTemplate.exchange(requestEntity, VoidResponse.class); + + Assertions.assertEquals(400, response.getStatusCodeValue()); + + // @formatter:off + assertQueryException( + "Multiple UUID types 'LuceneUUIDEventQuery' and 'EventQuery' not supported within the same lookup request", + "java.lang.IllegalArgumentException: Multiple UUID types 'LuceneUUIDEventQuery' and 'EventQuery' not supported within the same lookup request", + "400-1", + Iterables.getOnlyElement(response.getBody().getExceptions())); + // @formatter:on + } + + @Test + public void testBatchLookupUUIDFailure_nullUUIDType() throws Exception { + DatawaveUserDetails authUser = createUserDetails(); + + MultiValueMap uuidParams = createUUIDParams(); + uuidParams.add(LOOKUP_UUID_PAIRS, "PAGE:anarchy"); + + UriComponents uri = createUri("lookupUUID"); + + // not testing audit with this method + auditIgnoreSetup(); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, uuidParams, null, HttpMethod.POST, uri); + + ResponseEntity response = jwtRestTemplate.exchange(requestEntity, VoidResponse.class); + + Assertions.assertEquals(400, response.getStatusCodeValue()); + + // @formatter:off + assertQueryException( + "Invalid type 'PAGE' for UUID anarchy not supported with the LuceneToJexlUUIDQueryParser", + "java.lang.IllegalArgumentException: Invalid type 'PAGE' for UUID anarchy not supported with the LuceneToJexlUUIDQueryParser", + "400-1", + Iterables.getOnlyElement(response.getBody().getExceptions())); + // @formatter:on + } + + @Test + public void testBatchLookupUUIDFailure_emptyUUIDFieldValue() throws Exception { + DatawaveUserDetails authUser = createUserDetails(); + + MultiValueMap uuidParams = createUUIDParams(); + uuidParams.add(LOOKUP_UUID_PAIRS, ":anarchy"); + + UriComponents uri = createUri("lookupUUID"); + + // not testing audit with this method + auditIgnoreSetup(); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, uuidParams, null, HttpMethod.POST, uri); + + ResponseEntity response = jwtRestTemplate.exchange(requestEntity, VoidResponse.class); + + Assertions.assertEquals(400, response.getStatusCodeValue()); + + // @formatter:off + assertQueryException( + "Empty UUID type or value extracted from uuidPair :anarchy", + "java.lang.IllegalArgumentException: Empty UUID type or value extracted from uuidPair :anarchy", + "400-1", + Iterables.getOnlyElement(response.getBody().getExceptions())); + // @formatter:on + } + + @Test + public void testBatchLookupUUIDFailure_invalidUUIDPair() throws Exception { + DatawaveUserDetails authUser = createUserDetails(); + + MultiValueMap uuidParams = createUUIDParams(); + uuidParams.add(LOOKUP_UUID_PAIRS, ":"); + + UriComponents uri = createUri("lookupUUID"); + + // not testing audit with this method + auditIgnoreSetup(); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, uuidParams, null, HttpMethod.POST, uri); + + ResponseEntity response = jwtRestTemplate.exchange(requestEntity, VoidResponse.class); + + Assertions.assertEquals(400, response.getStatusCodeValue()); + + // @formatter:off + assertQueryException( + "Unable to determine UUID type and value from uuidPair :", + "java.lang.IllegalArgumentException: Unable to determine UUID type and value from uuidPair :", + "400-1", + Iterables.getOnlyElement(response.getBody().getExceptions())); + // @formatter:on + } + + @Test + public void testBatchLookupUUIDFailure_tooManyTerms() throws Exception { + DatawaveUserDetails authUser = createUserDetails(); + + MultiValueMap uuidParams = createUUIDParams(); + + StringBuilder lookupUUIDPairs = new StringBuilder(); + for (int i = 0; i < lookupProperties.getBatchLookupLimit() + 1; i++) { + if (i > 0) { + lookupUUIDPairs.append(" OR "); + } + lookupUUIDPairs.append("PAGE_TITLE:anarchy-").append(i); + } + uuidParams.add(LOOKUP_UUID_PAIRS, lookupUUIDPairs.toString()); + + UriComponents uri = createUri("lookupUUID"); + + // not testing audit with this method + auditIgnoreSetup(); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, uuidParams, null, HttpMethod.POST, uri); + + ResponseEntity response = jwtRestTemplate.exchange(requestEntity, VoidResponse.class); + + Assertions.assertEquals(400, response.getStatusCodeValue()); + + // @formatter:off + assertQueryException( + "The " + (lookupProperties.getBatchLookupLimit() + 1) + " specified UUIDs exceed the maximum number of " + lookupProperties.getBatchLookupLimit() + " allowed for a given lookup request", + "java.lang.IllegalArgumentException: The " + (lookupProperties.getBatchLookupLimit() + 1) + " specified UUIDs exceed the maximum number of " + lookupProperties.getBatchLookupLimit() + " allowed for a given lookup request", + "400-1", + Iterables.getOnlyElement(response.getBody().getExceptions())); + // @formatter:on + } + + @Test + public void testBatchLookupUUIDFailure_nonLookupQueryLogic() throws Exception { + DatawaveUserDetails authUser = createUserDetails(); + + MultiValueMap uuidParams = createUUIDParams(); + uuidParams.add(LOOKUP_UUID_PAIRS, "PAGE_NUMBER:accessiblecomputing"); + + UriComponents uri = createUri("lookupUUID"); + + // not testing audit with this method + auditIgnoreSetup(); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, uuidParams, null, HttpMethod.POST, uri); + + ResponseEntity response = jwtRestTemplate.exchange(requestEntity, VoidResponse.class); + + Assertions.assertEquals(500, response.getStatusCodeValue()); + + // @formatter:off + assertQueryException( + "Error setting up query. Lookup UUID can only be run with a LookupQueryLogic", + "Exception with no cause caught", + "500-66", + Iterables.getOnlyElement(response.getBody().getExceptions())); + // @formatter:on + } + + protected MultiValueMap createUUIDParams() { + MultiValueMap map = new LinkedMultiValueMap<>(); + map.set(DefaultQueryParameters.QUERY_NAME, TEST_QUERY_NAME); + map.set(DefaultQueryParameters.QUERY_AUTHORIZATIONS, TEST_QUERY_AUTHORIZATIONS); + map.set(ColumnVisibilitySecurityMarking.VISIBILITY_MARKING, TEST_VISIBILITY_MARKING); + map.set(QUERY_MAX_CONCURRENT_TASKS, Integer.toString(1)); + return map; + } + + protected Future> batchLookupUUID(DatawaveUserDetails authUser, MultiValueMap map) { + UriComponents uri = createUri("lookupUUID"); + + // not testing audit with this method + auditIgnoreSetup(); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); + return Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, DefaultEventQueryResponse.class)); + } + + protected Future> lookupUUID(DatawaveUserDetails authUser, MultiValueMap map, String uuidType, + String uuid) { + UriComponents uri = createUri("lookupUUID/" + uuidType + "/" + uuid); + + // not testing audit with this method + auditIgnoreSetup(); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.GET, uri); + return Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, DefaultEventQueryResponse.class)); + } + + protected Future> batchLookupContentUUID(DatawaveUserDetails authUser, MultiValueMap map) { + UriComponents uri = createUri("lookupContentUUID"); + + // not testing audit with this method + auditIgnoreSetup(); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); + return Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, DefaultEventQueryResponse.class)); + } + + protected Future> lookupContentUUID(DatawaveUserDetails authUser, MultiValueMap map, + String uuidType, String uuid) { + UriComponents uri = createUri("lookupContentUUID/" + uuidType + "/" + uuid); + + // not testing audit with this method + auditIgnoreSetup(); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.GET, uri); + return Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, DefaultEventQueryResponse.class)); + } + + protected void publishEventsToQueue(String queryId, int numEvents, MultiValueMap fieldValues, String visibility) throws Exception { + QueryResultsPublisher publisher = queryQueueManager.createPublisher(queryId); + for (int resultId = 0; resultId < numEvents; resultId++) { + DefaultEvent event = new DefaultEvent(); + long currentTime = System.currentTimeMillis(); + List fields = new ArrayList<>(); + for (Map.Entry> entry : fieldValues.entrySet()) { + for (String value : entry.getValue()) { + fields.add(new DefaultField(entry.getKey(), visibility, new HashMap<>(), currentTime, value)); + } + } + event.setFields(fields); + + Metadata metadata = new Metadata(); + // tonight i'm gonna party like it's + metadata.setRow("19991231_0"); + metadata.setDataType("prince"); + metadata.setInternalId(UUID.randomUUID().toString()); + event.setMetadata(metadata); + publisher.publish(new Result(Integer.toString(resultId), event)); + } + } + + protected void assertContentQueryResponse(String queryId, String logicName, long pageNumber, boolean partialResults, long operationTimeInMS, int numEvents, + DefaultEventQueryResponse queryResponse) { + Assertions.assertEquals(queryId, queryResponse.getQueryId()); + Assertions.assertEquals(logicName, queryResponse.getLogicName()); + Assertions.assertEquals(pageNumber, queryResponse.getPageNumber()); + Assertions.assertEquals(partialResults, queryResponse.isPartialResults()); + Assertions.assertEquals(operationTimeInMS, queryResponse.getOperationTimeMS()); + Assertions.assertEquals(numEvents, queryResponse.getEvents().size()); + } +} diff --git a/service/src/test/java/datawave/microservice/query/stream/StreamingServiceTest.java b/service/src/test/java/datawave/microservice/query/stream/StreamingServiceTest.java new file mode 100644 index 00000000..8eb8e5de --- /dev/null +++ b/service/src/test/java/datawave/microservice/query/stream/StreamingServiceTest.java @@ -0,0 +1,337 @@ +package datawave.microservice.query.stream; + +import java.io.StringReader; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; + +import javax.xml.bind.JAXBContext; +import javax.xml.bind.JAXBException; +import javax.xml.bind.Unmarshaller; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpMethod; +import org.springframework.http.MediaType; +import org.springframework.http.RequestEntity; +import org.springframework.http.ResponseEntity; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.junit.jupiter.SpringExtension; +import org.springframework.util.LinkedMultiValueMap; +import org.springframework.util.MultiValueMap; +import org.springframework.web.util.UriComponents; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.MapperFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.module.jaxb.JaxbAnnotationModule; + +import datawave.core.query.configuration.GenericQueryConfiguration; +import datawave.microservice.authorization.service.RemoteAuthorizationServiceUserDetailsService; +import datawave.microservice.authorization.user.DatawaveUserDetails; +import datawave.microservice.query.AbstractQueryServiceTest; +import datawave.microservice.query.DefaultQueryParameters; +import datawave.microservice.query.remote.QueryRequest; +import datawave.microservice.query.storage.QueryStatus; +import datawave.webservice.query.result.event.DefaultEvent; +import datawave.webservice.result.DefaultEventQueryResponse; + +@ExtendWith(SpringExtension.class) +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +@ActiveProfiles({"QueryStarterDefaults", "QueryStarterOverrides", "QueryServiceTest", RemoteAuthorizationServiceUserDetailsService.ACTIVATION_PROFILE}) +public class StreamingServiceTest extends AbstractQueryServiceTest { + + @Test + public void testExecuteSuccess() throws Throwable { + DatawaveUserDetails authUser = createUserDetails(); + + // create a valid query + String queryId = createQuery(authUser, createParams()); + + // pump enough results into the queue to trigger a complete page + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + int pageSize = queryStatus.getQuery().getPagesize(); + + // test field value pairings + MultiValueMap fieldValues = new LinkedMultiValueMap<>(); + fieldValues.add("LOKI", "ALLIGATOR"); + fieldValues.add("LOKI", "CLASSIC"); + + // add a config object to the query status, which would normally be added by the executor service + queryStatus.setConfig(new GenericQueryConfiguration()); + queryStorageCache.updateQueryStatus(queryStatus); + + // @formatter:off + publishEventsToQueue( + queryId, + (int)TEST_MAX_RESULTS_OVERRIDE, + fieldValues, + "ALL"); + // @formatter:on + + // make the execute call asynchronously + Future> future = execute(authUser, queryId); + + // the response should come back right away + ResponseEntity response = future.get(); + + Assertions.assertEquals(200, response.getStatusCodeValue()); + + // verify some headers + Assertions.assertEquals(MediaType.APPLICATION_XML, response.getHeaders().getContentType()); + + int pageNumber = 1; + + List queryResponses = parseXMLBaseQueryResponses(response.getBody()); + for (DefaultEventQueryResponse queryResponse : queryResponses) { + // verify the query response + // @formatter:off + assertQueryResponse( + queryId, + "EventQuery", + pageNumber++, + false, + queryResponse.getOperationTimeMS(), + 1, + Collections.singletonList("LOKI"), + pageSize, + Objects.requireNonNull(queryResponse)); + // @formatter:on + + // validate one of the events + DefaultEvent event = (DefaultEvent) queryResponse.getEvents().get(0); + // @formatter:off + assertDefaultEvent( + Arrays.asList("LOKI", "LOKI"), + Arrays.asList("ALLIGATOR", "CLASSIC"), + event); + // @formatter:on + } + + // verify that the next event was published + Assertions.assertEquals(6, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.NEXT, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.NEXT, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.NEXT, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.NEXT, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CLOSE, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @Test + public void testCreateAndExecuteSuccess() throws Throwable { + DatawaveUserDetails authUser = createUserDetails(); + + final String query = TEST_QUERY_STRING + " CREATE_AND_NEXT:TEST"; + + MultiValueMap params = createParams(); + params.set(DefaultQueryParameters.QUERY_STRING, query); + + // make the execute call asynchronously + Future> future = createAndExecute(authUser, params); + + long startTime = System.currentTimeMillis(); + QueryStatus queryStatus = null; + while (queryStatus == null && (System.currentTimeMillis() - startTime) < TEST_WAIT_TIME_MILLIS) { + queryStatus = queryStorageCache.getQueryStatus().stream().filter(x -> x.getQuery().getQuery().equals(query)).findAny().orElse(null); + if (queryStatus == null) { + Thread.sleep(500); + } + } + + String queryId = queryStatus.getQueryKey().getQueryId(); + + // pump enough results into the queue to trigger a complete page + int pageSize = queryStatus.getQuery().getPagesize(); + + // test field value pairings + MultiValueMap fieldValues = new LinkedMultiValueMap<>(); + fieldValues.add("LOKI", "ALLIGATOR"); + fieldValues.add("LOKI", "CLASSIC"); + + // add a config object to the query status, which would normally be added by the executor service + queryStatus.setConfig(new GenericQueryConfiguration()); + queryStorageCache.updateQueryStatus(queryStatus); + + // @formatter:off + publishEventsToQueue( + queryId, + (int)TEST_MAX_RESULTS_OVERRIDE, + fieldValues, + "ALL"); + // @formatter:on + + // the response should come back right away + ResponseEntity response = future.get(); + + Assertions.assertEquals(200, response.getStatusCodeValue()); + + // verify some headers + Assertions.assertEquals(MediaType.APPLICATION_XML, response.getHeaders().getContentType()); + + int pageNumber = 1; + + List queryResponses = parseXMLBaseQueryResponses(response.getBody()); + for (DefaultEventQueryResponse queryResponse : queryResponses) { + // verify the query response + // @formatter:off + assertQueryResponse( + queryId, + "EventQuery", + pageNumber++, + false, + queryResponse.getOperationTimeMS(), + 1, + Collections.singletonList("LOKI"), + pageSize, + Objects.requireNonNull(queryResponse)); + // @formatter:on + + // validate one of the events + DefaultEvent event = (DefaultEvent) queryResponse.getEvents().get(0); + // @formatter:off + assertDefaultEvent( + Arrays.asList("LOKI", "LOKI"), + Arrays.asList("ALLIGATOR", "CLASSIC"), + event); + // @formatter:on + } + + // verify that the next event was published + Assertions.assertEquals(6, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.NEXT, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.NEXT, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.NEXT, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.NEXT, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CLOSE, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + protected Future> createAndExecute(DatawaveUserDetails authUser, MultiValueMap map) { + UriComponents uri = createUri("EventQuery/createAndExecute"); + + // not testing audit with this method + auditIgnoreSetup(); + + MultiValueMap headers = new LinkedMultiValueMap<>(); + headers.set(HttpHeaders.ACCEPT, MediaType.APPLICATION_XML_VALUE); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, headers, HttpMethod.POST, uri); + return Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, String.class)); + } + + protected Future> execute(DatawaveUserDetails authUser, String queryId) { + UriComponents uri = createUri(queryId + "/execute"); + + MultiValueMap headers = new LinkedMultiValueMap<>(); + headers.set(HttpHeaders.ACCEPT, MediaType.APPLICATION_XML_VALUE); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, null, headers, HttpMethod.GET, uri); + return Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, String.class)); + } + + private ObjectMapper createJSONObjectMapper() { + ObjectMapper mapper = new ObjectMapper(); + mapper.registerModule(new JaxbAnnotationModule()); + mapper.configure(MapperFeature.USE_WRAPPER_NAME_AS_PROPERTY_NAME, true); + return mapper; + } + + protected List parseJSONBaseQueryResponses(String responseBody) throws JsonProcessingException { + String delimiter = "}{"; + ObjectMapper mapper = createJSONObjectMapper(); + List baseResponses = new ArrayList<>(); + int start = 0; + int end = responseBody.indexOf(delimiter) + 1; + while (end > start) { + String stringResponse = responseBody.substring(start, end); + baseResponses.add(mapper.readValue(stringResponse, DefaultEventQueryResponse.class)); + start = end; + end = responseBody.indexOf(delimiter, start) + 1; + if (end == 0) { + end = responseBody.length(); + } + } + return baseResponses; + } + + protected List parseXMLBaseQueryResponses(String responseBody) throws JAXBException { + String delimiter = ""; + List baseResponses = new ArrayList<>(); + int start = responseBody.indexOf(delimiter); + int end = responseBody.indexOf(delimiter, start + delimiter.length()); + while (end > start) { + String stringResponse = responseBody.substring(start, end); + + JAXBContext jaxbContext = JAXBContext.newInstance(DefaultEventQueryResponse.class); + Unmarshaller unmarshaller = jaxbContext.createUnmarshaller(); + baseResponses.add((DefaultEventQueryResponse) unmarshaller.unmarshal(new StringReader(stringResponse))); + + start = end; + end = responseBody.indexOf(delimiter, start + delimiter.length()); + if (end == -1) { + end = responseBody.length(); + } + } + return baseResponses; + } +} diff --git a/service/src/test/resources/TestQueryLogicFactory.xml b/service/src/test/resources/TestQueryLogicFactory.xml new file mode 100644 index 00000000..86c1bc97 --- /dev/null +++ b/service/src/test/resources/TestQueryLogicFactory.xml @@ -0,0 +1,15 @@ + + + + + + + + + + + + diff --git a/service/src/test/resources/config/application-QueryStarterOverrides.yml b/service/src/test/resources/config/application-QueryStarterOverrides.yml new file mode 100644 index 00000000..a3445181 --- /dev/null +++ b/service/src/test/resources/config/application-QueryStarterOverrides.yml @@ -0,0 +1,51 @@ +# Use this file to override properties from the QueryStarterDefaults profile (found in the datawave query starter). +# Make sure that you add the QueryStarterOverrides profile to your tests after the QueryStarterDefaults profile. + +datawave: + query: + cachedResults: + enabled: false + awaitExecutorCreateResponse: false + nextCall: + resultPollInterval: 500 + statusUpdateInterval: 0 + logic: + factory: + # Uncomment the following line to override the query logic beans to load + xmlBeansPath: "classpath:TestQueryLogicFactory.xml" + queryLogicsByName: + "AltEventQuery": "AltEventQuery" + logics: + BaseEventQuery: + maxResults: 369 + auditType: "ACTIVE" + ContentQuery: + maxResults: 10 + + lookup: + types: + 'EVENT_ID': + fieldName: 'EVENT_ID' + queryLogics: + 'default': 'LuceneUUIDEventQuery' + allowedWildcardAfter: 28 + 'UUID': + fieldName: 'UUID' + queryLogics: + 'default': 'LuceneUUIDEventQuery' + 'PARENT_UUID': + fieldName: 'PARENT_UUID' + queryLogics: + 'default': 'LuceneUUIDEventQuery' + 'PAGE_ID': + fieldName: 'PAGE_ID' + queryLogics: + 'default': 'LuceneUUIDEventQuery' + 'PAGE_TITLE': + fieldName: 'PAGE_TITLE' + queryLogics: + 'default': 'LuceneUUIDEventQuery' + 'PAGE_NUMBER': + fieldName: 'PAGE_NUMBER' + queryLogics: + 'default': 'EventQuery' diff --git a/service/src/test/resources/config/application.yml b/service/src/test/resources/config/application.yml new file mode 100644 index 00000000..71ad1f4e --- /dev/null +++ b/service/src/test/resources/config/application.yml @@ -0,0 +1,54 @@ +spring: + application: + name: query + + autoconfigure: + exclude: org.springframework.boot.autoconfigure.web.servlet.error.ErrorMvcAutoConfiguration + + security: + user.password: passwordNotUsed + datawave: + jwt.ttl: 3600 + issuers-required: true + enforce-allowed-callers: false + allowed-callers: + - "cn=test keystore, ou=my department, o=my company, st=some-state, c=us" + +hazelcast.client.enabled: false + +server: + port: 0 + non-secure-port: 0 + servlet.context-path: /query + ssl: + client-auth: NEED + trust-store: 'classpath:testCA.p12' + trust-store-type: PKCS12 + trust-store-password: 'ChangeIt' + key-store: 'classpath:testServer.p12' + key-store-type: PKCS12 + key-store-password: 'ChangeIt' + outbound-ssl: + key-store: ${server.ssl.key-store} + key-store-password: ${server.ssl.key-store-password} + key-store-type: ${server.ssl.key-store-type} + trust-store: ${server.ssl.trust-store} + trust-store-password: ${server.ssl.trust-store-password} + trust-store-type: ${server.ssl.trust-store-type} + +management: + endpoints: + web: + base-path: "/mgmt" + +datawave: + query: + messaging: + backend: hazelcast + metric: + client: + confirmAckEnabled: false + +logging: + level: + root: FATAL diff --git a/service/src/test/resources/config/bootstrap.yml b/service/src/test/resources/config/bootstrap.yml new file mode 100644 index 00000000..9188151a --- /dev/null +++ b/service/src/test/resources/config/bootstrap.yml @@ -0,0 +1,21 @@ +spring: + cloud: + bus: + enabled: true + consul: + enabled: false + config: + enabled: false + discovery: + enabled: false + main: + banner-mode: "OFF" + # Starting with spring-boot 2.6, circular references are disabled by default + # This is still needed for the evaluation-only function + allow-circular-references: true + +datawave: + table: + cache: + enabled: false + diff --git a/service/src/test/resources/log4j2-test.xml b/service/src/test/resources/log4j2-test.xml new file mode 100644 index 00000000..0600e42c --- /dev/null +++ b/service/src/test/resources/log4j2-test.xml @@ -0,0 +1,17 @@ + + + + ???? + %clr{%d{yyyy-MM-dd HH:mm:ss.SSS}}{faint} %clr{%5p} %clr{${sys:PID}}{magenta} %clr{---}{faint} %clr{[%15.15t]}{faint} %clr{%-40.40c{1.}}{cyan} %clr{:}{faint} %m%n%wEx + + + + + + + + + + + + \ No newline at end of file