Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add ability to configure date types for no expansion #2662

Open
wants to merge 1 commit into
base: integration
Choose a base branch
from
Open

Add ability to configure date types for no expansion #2662

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 42 additions & 5 deletions warehouse/query-core/src/main/java/datawave/query/config/ShardQueryConfiguration.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -515,6 +515,18 @@ public class ShardQueryConfiguration extends GenericQueryConfiguration implement
*/
private double fieldIndexHoleMinThreshold = 1.0d;

/**
* The set of date types that, if the query's end date is the current date, will NOT result in any date range adjustments or the addition of a
* SHARDS_AND_DAYS hint.
*/
private Set<String> noExpansionIfCurrent = Collections.emptySet();

/**
* Whether the SHARDS_AND_DAYS hint should be allowed for the query. This will be set to false iff the query specified a date type, and the date type is
* present in {@link #noExpansionIfCurrent}, and the query's end date is the current date.
*/
private boolean shardsAndDaysHintAllowed = true;

/**
* Default constructor
*/
Expand Down Expand Up @@ -753,6 +765,8 @@ public void copyFrom(ShardQueryConfiguration other) {
this.setUseQueryTreeScanHintRules(other.isUseQueryTreeScanHintRules());
this.setQueryTreeScanHintRules(other.getQueryTreeScanHintRules());
this.setFieldIndexHoleMinThreshold(other.getFieldIndexHoleMinThreshold());
this.setNoExpansionIfCurrent(other.getNoExpansionIfCurrent() == null ? null : Sets.newHashSet(other.getNoExpansionIfCurrent()));
this.setShardsAndDaysHintAllowed(other.isShardsAndDaysHintAllowed());
}

/**
Expand Down Expand Up @@ -2791,12 +2805,15 @@ public void setSortQueryPostIndexWithTermCounts(boolean sortQueryPostIndexWithTe

@Override
public boolean equals(Object o) {
if (this == o)
if (this == o) {
return true;
if (o == null || getClass() != o.getClass())
}
if (o == null || getClass() != o.getClass()) {
return false;
if (!super.equals(o))
}
if (!super.equals(o)) {
return false;
}
// @formatter:off
ShardQueryConfiguration that = (ShardQueryConfiguration) o;
return isTldQuery() == that.isTldQuery() &&
Expand Down Expand Up @@ -2995,7 +3012,9 @@ public boolean equals(Object o) {
isSortQueryPreIndexWithImpliedCounts() == isSortQueryPreIndexWithImpliedCounts() &&
isSortQueryPreIndexWithFieldCounts() == isSortQueryPreIndexWithFieldCounts() &&
isSortQueryPostIndexWithTermCounts() == isSortQueryPostIndexWithTermCounts() &&
isSortQueryPostIndexWithFieldCounts() == isSortQueryPostIndexWithFieldCounts();
isSortQueryPostIndexWithFieldCounts() == isSortQueryPostIndexWithFieldCounts() &&
Objects.equals(getNoExpansionIfCurrent(), that.getNoExpansionIfCurrent()) &&
isShardsAndDaysHintAllowed() == that.isShardsAndDaysHintAllowed();
// @formatter:on
}

Expand Down Expand Up @@ -3199,7 +3218,9 @@ public int hashCode() {
isSortQueryPreIndexWithImpliedCounts(),
isSortQueryPreIndexWithFieldCounts(),
isSortQueryPostIndexWithTermCounts(),
isSortQueryPostIndexWithFieldCounts()
isSortQueryPostIndexWithFieldCounts(),
getNoExpansionIfCurrent(),
isShardsAndDaysHintAllowed()
);
// @formatter:on
}
Expand Down Expand Up @@ -3234,4 +3255,20 @@ public long getMaxAnyFieldScanTimeMillis() {
public void setMaxAnyFieldScanTimeMillis(long maxAnyFieldScanTimeMillis) {
this.maxAnyFieldScanTimeMillis = maxAnyFieldScanTimeMillis;
}

public Set<String> getNoExpansionIfCurrent() {
return noExpansionIfCurrent;
}

public void setNoExpansionIfCurrent(Set<String> noExpansionIfCurrent) {
this.noExpansionIfCurrent = noExpansionIfCurrent;
}

public boolean isShardsAndDaysHintAllowed() {
return shardsAndDaysHintAllowed;
}

public void setShardsAndDaysHintAllowed(boolean shardsAndDaysHintAllowed) {
this.shardsAndDaysHintAllowed = shardsAndDaysHintAllowed;
}
}
135 changes: 78 additions & 57 deletions warehouse/query-core/src/main/java/datawave/query/planner/DefaultQueryPlanner.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -834,15 +834,18 @@ protected ASTJexlScript updateQueryTree(ScannerFactory scannerFactory, MetadataH
throw new DatawaveFatalQueryException("Found incorrectly marked bounded ranges");
}

if (optionsMap.containsKey(QueryParameters.SHARDS_AND_DAYS)) {
config.setQueryTree(timedAddShardsAndDaysFromOptions(timers, config.getQueryTree(), optionsMap));
} else {
// look for the shards and days hint in the query settings
// the shards and days hint cannot always be specified in the query string when using certain query parsers
Parameter parameter = settings.findParameter(QueryParameters.SHARDS_AND_DAYS);
if (StringUtils.isNotBlank(parameter.getParameterValue())) {
optionsMap.put(QueryParameters.SHARDS_AND_DAYS, parameter.getParameterValue());
// Do not add a SHARDS_AND_DAYS hint if it is specifically not allowed. This was checked and updated when timedIncludeDateFilters was called.
if (config.isShardsAndDaysHintAllowed()) {
if (optionsMap.containsKey(QueryParameters.SHARDS_AND_DAYS)) {
config.setQueryTree(timedAddShardsAndDaysFromOptions(timers, config.getQueryTree(), optionsMap));
} else {
// look for the shards and days hint in the query settings
// the shards and days hint cannot always be specified in the query string when using certain query parsers
Parameter parameter = settings.findParameter(QueryParameters.SHARDS_AND_DAYS);
if (StringUtils.isNotBlank(parameter.getParameterValue())) {
optionsMap.put(QueryParameters.SHARDS_AND_DAYS, parameter.getParameterValue());
config.setQueryTree(timedAddShardsAndDaysFromOptions(timers, config.getQueryTree(), optionsMap));
}
}
}

Expand Down Expand Up @@ -2113,57 +2116,73 @@ public ASTJexlScript addDateFilters(final ASTJexlScript queryTree, ScannerFactor
}
}

// if we are using something other than the default of EVENT date
// time, then we need to modify the query
if (!dateType.equals(defaultDateType)) {
// Get the set of date types that should not be expanded if the end date is the current date.
// @formatter:off
Set<String> noExpansionIfCurrent = config.getNoExpansionIfCurrent() == null ? Collections.emptySet() :
config.getNoExpansionIfCurrent().stream()
.map(String::trim)
.map(String::toUpperCase)
.collect(Collectors.toSet());
// @formatter:on

log.info("Using the date index for " + dateType);
// if no date index helper configured, then we are in error
if (dateIndexHelper == null) {
throw new DatawaveQueryException("Requested date range of type " + dateType + " but no date index is configured");
}
// get all of the fields used for this date type
DateIndexHelper.DateTypeDescription dateIndexData = dateIndexHelper.getTypeDescription(dateType, config.getBeginDate(), config.getEndDate(),
config.getDatatypeFilter());
if (dateIndexData.getFields().isEmpty()) {
log.warn("The specified date type: " + dateType + " is unknown for the specified data types");
// If this is the case, then essentially we have no dates to search. Adding the filter function with _NO_FIELD_ will have the desired effect.
// Also it will be understandable from the plan as to why no results were returned.
dateIndexData.getFields().add(Constants.NO_FIELD);
}
log.info("Adding date filters for the following fields: " + dateIndexData.getFields());
// now for each field, add an expression to filter that date
List<JexlNode> andChildren = new ArrayList<>();
for (int i = 0; i < queryTree.jjtGetNumChildren(); i++) {
if (queryTree.jjtGetChild(i) instanceof ASTAndNode) {
andChildren.add(queryTree.jjtGetChild(i));
// If the date type is one marked for no expansion if current, and the query's end date is the current date, do not add any date filters, and do not
// allow a SHARDS_AND_DAYS hint to be added later.
if (config.getNoExpansionIfCurrent().contains(dateType) && DateUtils.isSameDay(new Date(), config.getEndDate())) {
log.info("Query end date equals current date and date type " + dateType
+ " is marked for no expansion if current. SHARDS_AND_DAYS hint will be forbidden.");
config.setShardsAndDaysHintAllowed(false);
} else {
// If we are using something other than the default of EVENT date time, then we need to modify the query.
if (!dateType.equals(defaultDateType)) {
log.info("Using the date index for " + dateType);
// if no date index helper configured, then we are in error
if (dateIndexHelper == null) {
throw new DatawaveQueryException("Requested date range of type " + dateType + " but no date index is configured");
}
// get all of the fields used for this date type
DateIndexHelper.DateTypeDescription dateIndexData = dateIndexHelper.getTypeDescription(dateType, config.getBeginDate(), config.getEndDate(),
config.getDatatypeFilter());
if (dateIndexData.getFields().isEmpty()) {
log.warn("The specified date type: " + dateType + " is unknown for the specified data types");
// If this is the case, then essentially we have no dates to search. Adding the filter function with _NO_FIELD_ will have the desired
// effect.
// Also it will be understandable from the plan as to why no results were returned.
dateIndexData.getFields().add(Constants.NO_FIELD);
}
log.info("Adding date filters for the following fields: " + dateIndexData.getFields());
// now for each field, add an expression to filter that date
List<JexlNode> andChildren = new ArrayList<>();
for (int i = 0; i < queryTree.jjtGetNumChildren(); i++) {
if (queryTree.jjtGetChild(i) instanceof ASTAndNode) {
andChildren.add(queryTree.jjtGetChild(i));
} else {
andChildren.add(JexlNodeFactory.createExpression(queryTree.jjtGetChild(i)));
}
}
List<JexlNode> orChildren = new ArrayList<>();
for (String field : dateIndexData.getFields()) {
orChildren.add(createDateFilter(dateType, field, config.getBeginDate(), config.getEndDate()));
}
if (orChildren.size() > 1) {
andChildren.add(JexlNodeFactory.createOrNode(orChildren));
} else {
andChildren.add(JexlNodeFactory.createExpression(queryTree.jjtGetChild(i)));
andChildren.addAll(orChildren);
}
}
List<JexlNode> orChildren = new ArrayList<>();
for (String field : dateIndexData.getFields()) {
orChildren.add(createDateFilter(dateType, field, config.getBeginDate(), config.getEndDate()));
}
if (orChildren.size() > 1) {
andChildren.add(JexlNodeFactory.createOrNode(orChildren));
JexlNode andNode = JexlNodeFactory.createAndNode(andChildren);
JexlNodeFactory.setChildren(queryTree, Collections.singleton(andNode));

// now lets update the query parameters with the correct start and
// end dates
log.info("Remapped " + dateType + " dates [" + config.getBeginDate() + "," + config.getEndDate() + "] to EVENT dates "
+ dateIndexData.getBeginDate() + "," + dateIndexData.getEndDate());

// reset the dates in the configuration, no need to reset then in
// the Query settings object
config.setBeginDate(dateIndexData.getBeginDate());
config.setEndDate(dateIndexData.getEndDate());
} else {
andChildren.addAll(orChildren);
log.info("Date index not needed for this query");
}
JexlNode andNode = JexlNodeFactory.createAndNode(andChildren);
JexlNodeFactory.setChildren(queryTree, Collections.singleton(andNode));

// now lets update the query parameters with the correct start and
// end dates
log.info("Remapped " + dateType + " dates [" + config.getBeginDate() + "," + config.getEndDate() + "] to EVENT dates "
+ dateIndexData.getBeginDate() + "," + dateIndexData.getEndDate());

// reset the dates in the configuration, no need to reset then in
// the Query settings object
config.setBeginDate(dateIndexData.getBeginDate());
config.setEndDate(dateIndexData.getEndDate());
} else {
log.info("Date index not needed for this query");
}

return queryTree;
Expand Down Expand Up @@ -2447,16 +2466,18 @@ public List<IvaratorCacheDirConfig> getShuffledIvaratoCacheDirConfigs(ShardQuery
*/
protected IteratorSetting getQueryIterator(MetadataHelper metadataHelper, ShardQueryConfiguration config, String queryString, Boolean isFullTable,
boolean isPreload) throws DatawaveQueryException {
if (null == settingFuture)
if (null == settingFuture) {
settingFuture = loadQueryIterator(metadataHelper, config, isFullTable, isPreload);
if (settingFuture.isDone())
}
if (settingFuture.isDone()) {
try {
return settingFuture.get();
} catch (InterruptedException | ExecutionException e) {
throw new RuntimeException(e.getCause());
}
else
} else {
return null;
}
}

public void configureTypeMappings(ShardQueryConfiguration config, IteratorSetting cfg, MetadataHelper metadataHelper, boolean compressMappings)
Expand Down
8 changes: 8 additions & 0 deletions warehouse/query-core/src/main/java/datawave/query/tables/ShardQueryLogic.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2990,4 +2990,12 @@ public void setFieldIndexHoleMinThreshold(double fieldIndexHoleMinThreshold) {
public double getFieldIndexHoleMinThreshold(int fieldIndexHoleMinThreshold) {
return getConfig().getFieldIndexHoleMinThreshold();
}

public Set<String> getNoExpansionIfCurrent() {
return getConfig().getNoExpansionIfCurrent();
}

public void setNoExpansionIfCurrent(Set<String> noExpansionIfCurrent) {
getConfig().setNoExpansionIfCurrent(noExpansionIfCurrent);
}
}
6 changes: 6 additions & 0 deletions warehouse/query-core/src/test/java/datawave/query/config/ShardQueryConfigurationTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -596,6 +596,12 @@ public void setUp() throws Exception {
updatedValues.put("useQueryTreeScanHintRules", true);
defaultValues.put("queryTreeScanHintRules", Collections.emptyList());
updatedValues.put("queryTreeScanHintRules", Collections.singletonList(new IvaratorScanHint()));

defaultValues.put("noExpansionIfCurrent", Collections.emptySet());
updatedValues.put("noExpansionIfCurrent", Collections.singleton("EVENT"));

defaultValues.put("shardsAndDaysHintAllowed", true);
updatedValues.put("shardsAndDaysHintAllowed", false);
}

private Query createQuery(String query) {
Expand Down
Loading
Loading