Skip to content

Commit

Permalink
Add to QueryOptions
Browse files Browse the repository at this point in the history
  • Loading branch information
@SethSmucker
SethSmucker committed Nov 26, 2024
1 parent dd97c1e commit 94ce5ee
Show file tree
Hide file tree
Showing 3 changed files with 46 additions and 70 deletions.
14 changes: 2 additions & 12 deletions warehouse/query-core/src/main/java/datawave/query/config/IvaratorConfig.java
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package datawave.query.config;

import datawave.query.iterator.ivarator.IvaratorCacheDirConfig;
import datawave.query.util.sortedset.FileSortedSet;

import java.util.Collections;
import java.util.List;
Expand All @@ -19,7 +20,7 @@ public class IvaratorConfig {
private int ivaratorPersistVerifyCount = 100;
private int maxIvaratorSources = 33;
private long maxIvaratorSourceWait = 1000L * 60 * 30;
private long maxIvaratorResults = -1;


public IvaratorConfig(){};

Expand All @@ -39,8 +40,6 @@ public void copyFrom(IvaratorConfig other) {
this.setIvaratorPersistVerify(other.isIvaratorPersistVerify());
this.setIvaratorPersistVerifyCount(other.getIvaratorPersistVerifyCount());
this.setMaxIvaratorSources(other.getMaxIvaratorSources());
this.setMaxIvaratorSourceWait(other.getMaxIvaratorSourceWait());
this.setMaxIvaratorResults(other.getMaxIvaratorResults());
}

public List<IvaratorCacheDirConfig> getIvaratorCacheDirConfigs() {
Expand Down Expand Up @@ -138,13 +137,4 @@ public long getMaxIvaratorSourceWait() {
public void setMaxIvaratorSourceWait(long maxIvaratorSourceWait) {
this.maxIvaratorSourceWait = maxIvaratorSourceWait;
}

public long getMaxIvaratorResults() {
return maxIvaratorResults;
}

public void setMaxIvaratorResults(long maxIvaratorResults) {
this.maxIvaratorResults = maxIvaratorResults;
}

}
10 changes: 5 additions & 5 deletions warehouse/query-core/src/main/java/datawave/query/iterator/QueryIterator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -288,12 +288,12 @@ public void init(SortedKeyValueIterator<Key,Value> source, Map<String,String> op
// this method will prune any ivarator cache directories that do not have a valid configuration.
private void pruneIvaratorCacheDirs() throws InterruptedIOException {
List<IvaratorCacheDirConfig> configsToRemove = new ArrayList<>();
for (IvaratorCacheDirConfig config : ivaratorCacheDirConfigs) {
for (IvaratorCacheDirConfig config : getIvaratorCacheDirConfigs()) {
if (!hasValidBasePath(config)) {
configsToRemove.add(config);
}
}
ivaratorCacheDirConfigs.removeAll(configsToRemove);
getIvaratorCacheDirConfigs().removeAll(configsToRemove);
}

private boolean hasValidBasePath(IvaratorCacheDirConfig config) throws InterruptedIOException {
Expand Down Expand Up @@ -1442,12 +1442,12 @@ protected IteratorBuildingVisitor createIteratorBuildingVisitor(Class<? extends
.setIvaratorCacheBufferSize(this.getIvaratorCacheBufferSize())
.setIvaratorCacheScanPersistThreshold(this.getIvaratorCacheScanPersistThreshold())
.setIvaratorCacheScanTimeout(this.getIvaratorCacheScanTimeout())
.setMaxRangeSplit(this.getMaxIndexRangeSplit())
.setMaxRangeSplit(this.getMaxFieldIndexRangeSplit())
.setIvaratorMaxOpenFiles(this.getIvaratorMaxOpenFiles())
.setIvaratorNumRetries(this.getIvaratorNumRetries())
.setIvaratorPersistOptions(this.getIvaratorPersistOptions())
.setUnsortedIvaratorSource(this.sourceForDeepCopies)
.setIvaratorSourcePool(createIvaratorSourcePool(this.maxIvaratorSources, this.ivaratorCacheScanTimeout))
.setIvaratorSourcePool(createIvaratorSourcePool(getMaxIvaratorSources(), getIvaratorCacheScanTimeout()))
.setMaxIvaratorResults(this.getMaxIvaratorResults())
.setIncludes(indexedFields)
.setUnindexedFields(nonIndexedFields)
Expand Down Expand Up @@ -1570,7 +1570,7 @@ protected UniqueTransform getUniqueTransform() {
if (uniqueTransform == null && getUniqueFields() != null && !getUniqueFields().isEmpty()) {
synchronized (getUniqueFields()) {
if (uniqueTransform == null) {
uniqueTransform = new UniqueTransform(getUniqueFields(), getResultTimeout());
uniqueTransform = new UniqueTransform(getUniqueFields(), getIvaratorCacheScanTimeout());
}
}
}
Expand Down
92 changes: 39 additions & 53 deletions warehouse/query-core/src/main/java/datawave/query/iterator/QueryOptions.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;

import datawave.query.config.IvaratorConfig;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.PartialKey;
import org.apache.accumulo.core.data.Range;
Expand Down Expand Up @@ -362,20 +363,9 @@ public class QueryOptions implements OptionDescriber {

protected String zookeeperConfig = null;

protected List<IvaratorCacheDirConfig> ivaratorCacheDirConfigs = Collections.emptyList();
protected long ivaratorCacheScanPersistThreshold = 100000L;
protected long ivaratorCacheScanTimeout = 1000L * 60 * 60;
protected int ivaratorCacheBufferSize = 10000;

protected long resultTimeout = 1000L * 60 * 60;
protected int maxIndexRangeSplit = 11;
protected int ivaratorMaxOpenFiles = 100;
protected int ivaratorNumRetries = 2;
protected FileSortedSet.PersistOptions ivaratorPersistOptions = new FileSortedSet.PersistOptions();

protected int maxIvaratorSources = 33;
protected long maxIvaratorSourceWait = 1000L * 60 * 30;

protected IvaratorConfig ivaratorConfig = new IvaratorConfig();
private FileSortedSet.PersistOptions ivaratorPersistOptions = new FileSortedSet.PersistOptions();

protected long maxIvaratorResults = -1;

protected long yieldThresholdMs = Long.MAX_VALUE;
Expand Down Expand Up @@ -508,17 +498,17 @@ public void deepCopy(QueryOptions other) {
this.fiEvaluationFilter = other.fiEvaluationFilter;
this.eventEvaluationFilter = other.eventEvaluationFilter;

this.ivaratorCacheDirConfigs = (other.ivaratorCacheDirConfigs == null) ? null : new ArrayList<>(other.ivaratorCacheDirConfigs);
setIvaratorCacheDirConfigs((other.getIvaratorCacheDirConfigs() == null) ? null : new ArrayList<>(other.getIvaratorCacheDirConfigs()));
this.hdfsSiteConfigURLs = other.hdfsSiteConfigURLs;
this.ivaratorCacheBufferSize = other.ivaratorCacheBufferSize;
this.ivaratorCacheScanPersistThreshold = other.ivaratorCacheScanPersistThreshold;
this.ivaratorCacheScanTimeout = other.ivaratorCacheScanTimeout;
setIvaratorCacheBufferSize(other.getIvaratorCacheBufferSize());
setIvaratorCacheScanPersistThreshold(other.getIvaratorCacheScanPersistThreshold());
setIvaratorCacheScanTimeout(other.getIvaratorCacheScanTimeout());
this.hdfsFileCompressionCodec = other.hdfsFileCompressionCodec;
this.maxIndexRangeSplit = other.maxIndexRangeSplit;
this.ivaratorMaxOpenFiles = other.ivaratorMaxOpenFiles;
this.maxIvaratorSources = other.maxIvaratorSources;
this.maxIvaratorSourceWait = other.maxIvaratorSourceWait;
this.maxIvaratorResults = other.maxIvaratorResults;
setMaxFieldIndexRangeSplit(other.getMaxFieldIndexRangeSplit());
setIvaratorMaxOpenFiles(other.getIvaratorMaxOpenFiles());
setMaxIvaratorSources(other.getMaxIvaratorSources());
setMaxIvaratorSourceWait(other.getMaxIvaratorSourceWait());


this.yieldThresholdMs = other.yieldThresholdMs;

Expand Down Expand Up @@ -1015,7 +1005,7 @@ public FileSystemCache getFileSystemCache() throws MalformedURLException {

public QueryLock getQueryLock() throws MalformedURLException, ConfigException {
return new QueryLock.Builder().forQueryId(getQueryId()).forFSCache(getFileSystemCache())
.forIvaratorDirs(ivaratorCacheDirConfigs.stream().map(IvaratorCacheDirConfig::getBasePathURI).collect(Collectors.joining(",")))
.forIvaratorDirs(ivaratorConfig.getIvaratorCacheDirConfigs().stream().map(IvaratorCacheDirConfig::getBasePathURI).collect(Collectors.joining(",")))
.forZookeeper(getZookeeperConfig(), HdfsBackedControl.CANCELLED_CHECK_INTERVAL * 2).build();
}

Expand All @@ -1034,69 +1024,64 @@ public String getZookeeperConfig() {
public void setZookeeperConfig(String zookeeperConfig) {
this.zookeeperConfig = zookeeperConfig;
}

public List<IvaratorCacheDirConfig> getIvaratorCacheDirConfigs() {
return ivaratorCacheDirConfigs;
return ivaratorConfig.getIvaratorCacheDirConfigs();
}

public void setIvaratorCacheDirConfigs(List<IvaratorCacheDirConfig> ivaratorCacheDirConfigs) {
this.ivaratorCacheDirConfigs = ivaratorCacheDirConfigs;
ivaratorConfig.setIvaratorCacheDirConfigs(ivaratorCacheDirConfigs);
}

public int getIvaratorCacheBufferSize() {
return ivaratorCacheBufferSize;
return ivaratorConfig.getIvaratorCacheBufferSize();
}

public void setIvaratorCacheBufferSize(int ivaratorCacheBufferSize) {
this.ivaratorCacheBufferSize = ivaratorCacheBufferSize;
ivaratorConfig.setIvaratorCacheBufferSize(ivaratorCacheBufferSize);
}

public long getIvaratorCacheScanPersistThreshold() {
return ivaratorCacheScanPersistThreshold;
return ivaratorConfig.getIvaratorCacheScanPersistThreshold();
}

public void setIvaratorCacheScanPersistThreshold(long ivaratorCacheScanPersistThreshold) {
this.ivaratorCacheScanPersistThreshold = ivaratorCacheScanPersistThreshold;
ivaratorConfig.setIvaratorCacheScanPersistThreshold(ivaratorCacheScanPersistThreshold);
}

public long getIvaratorCacheScanTimeout() {
return ivaratorCacheScanTimeout;
return ivaratorConfig.getIvaratorCacheScanTimeout();
}

public void setIvaratorCacheScanTimeout(long ivaratorCacheScanTimeout) {
this.ivaratorCacheScanTimeout = ivaratorCacheScanTimeout;
}

public long getResultTimeout() {
return resultTimeout;
ivaratorConfig.setIvaratorCacheScanTimeout(ivaratorCacheScanTimeout);
}

public void setResultTimeout(long resultTimeout) {
this.resultTimeout = resultTimeout;
public void setivaratorCacheScanPersistThreshold(long ivaratorCacheScanPersistThreshold) {
ivaratorConfig.setIvaratorCacheScanPersistThreshold(ivaratorCacheScanPersistThreshold);
}

public int getMaxIndexRangeSplit() {
return maxIndexRangeSplit;
public int getMaxFieldIndexRangeSplit() {
return ivaratorConfig.getMaxFieldIndexRangeSplit();
}

public void setMaxIndexRangeSplit(int maxIndexRangeSplit) {
this.maxIndexRangeSplit = maxIndexRangeSplit;
public void setMaxFieldIndexRangeSplit(int maxFieldIndexRangeSplit) {
ivaratorConfig.setMaxFieldIndexRangeSplit(maxFieldIndexRangeSplit);
}

public int getIvaratorMaxOpenFiles() {
return ivaratorMaxOpenFiles;
return ivaratorConfig.getIvaratorMaxOpenFiles();
}

public void setIvaratorMaxOpenFiles(int ivaratorMaxOpenFiles) {
this.ivaratorMaxOpenFiles = ivaratorMaxOpenFiles;
ivaratorConfig.setIvaratorMaxOpenFiles(ivaratorMaxOpenFiles);
}

public int getIvaratorNumRetries() {
return ivaratorNumRetries;
return ivaratorConfig.getIvaratorNumRetries();
}

public void setIvaratorNumRetries(int ivaratorNumRetries) {
this.ivaratorNumRetries = ivaratorNumRetries;
ivaratorConfig.setIvaratorNumRetries(ivaratorNumRetries);
}

public FileSortedSet.PersistOptions getIvaratorPersistOptions() {
Expand All @@ -1108,21 +1093,22 @@ public void setIvaratorPersistOptions(FileSortedSet.PersistOptions ivaratorPersi
}

public int getMaxIvaratorSources() {
return maxIvaratorSources;
return ivaratorConfig.getMaxIvaratorSources();
}

public void setMaxIvaratorSources(int maxIvaratorSources) {
this.maxIvaratorSources = maxIvaratorSources;
ivaratorConfig.setMaxIvaratorSources(maxIvaratorSources);
}

public long getMaxIvaratorSourceWait() {
return maxIvaratorSourceWait;
return ivaratorConfig.getMaxIvaratorSourceWait();
}

public void setMaxIvaratorSourceWait(long maxIvaratorSourceWait) {
this.maxIvaratorSourceWait = maxIvaratorSourceWait;
ivaratorConfig.setMaxIvaratorSourceWait(maxIvaratorSourceWait);
}


public long getMaxIvaratorResults() {
return maxIvaratorResults;
}
Expand Down Expand Up @@ -1765,11 +1751,11 @@ public boolean validateOptions(Map<String,String> options) {
}

if (options.containsKey(RESULT_TIMEOUT)) {
this.setResultTimeout(Long.parseLong(options.get(RESULT_TIMEOUT)));
this.setivaratorCacheScanPersistThreshold(Long.parseLong(options.get(RESULT_TIMEOUT)));
}

if (options.containsKey(MAX_INDEX_RANGE_SPLIT)) {
this.setMaxIndexRangeSplit(Integer.parseInt(options.get(MAX_INDEX_RANGE_SPLIT)));
this.setMaxFieldIndexRangeSplit(Integer.parseInt(options.get(MAX_INDEX_RANGE_SPLIT)));
}

if (options.containsKey(MAX_IVARATOR_OPEN_FILES)) {
Expand Down

0 comments on commit 94ce5ee

Please sign in to comment.