diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 8d4c0e5d30f..a507c1026f2 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -100,6 +100,11 @@ New Features --------------------- * SOLR-13350: Multithreaded search execution (Ishan Chattopadhyaya, Mark Miller, Christine Poerschke, David Smiley, noble) +* SOLR-17192: Put an UpdateRequestProcessor-enforced soft-limit on the number of fields allowed in a core. The `NumFieldLimitingUpdateRequestProcessorFactory` + limit may be adjusted by raising the factory's `maxFields` setting, toggled in and out of "warning-only" mode using the `warnOnly` setting, or disabled entirely + by removing it solrconfig.xml. The limit is set at 1000 fields in the "_default" configset, but left in warning-only mode. (David Smiley, Eric Pugh, + Jason Gerlowski) + Improvements --------------------- * SOLR-16921: use -solrUrl to derive the zk host connection for bin/solr zk subcommands (Eric Pugh) @@ -125,6 +130,9 @@ Other Changes --------------------- * SOLR-17248: Refactor ZK related SolrCli tools to separate SolrZkClient and CloudSolrClient instantiation/usage (Lamine Idjeraoui via Eric Pugh) +* SOLR-16505: Use Jetty HTTP2 for index replication and other "recovery" operations + (Sanjay Dutt, David Smiley) + ================== 9.6.0 ================== New Features --------------------- diff --git a/solr/core/src/java/org/apache/solr/cloud/RecoveryStrategy.java b/solr/core/src/java/org/apache/solr/cloud/RecoveryStrategy.java index 9ea837378d7..d9453757dcc 100644 --- a/solr/core/src/java/org/apache/solr/cloud/RecoveryStrategy.java +++ b/solr/core/src/java/org/apache/solr/cloud/RecoveryStrategy.java @@ -22,17 +22,17 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Optional; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; +import java.util.concurrent.FutureTask; import java.util.concurrent.TimeUnit; -import org.apache.http.client.methods.HttpUriRequest; import org.apache.lucene.index.IndexCommit; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.store.Directory; import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrServerException; -import org.apache.solr.client.solrj.impl.HttpSolrClient; -import org.apache.solr.client.solrj.impl.HttpSolrClient.HttpUriRequestResponse; +import org.apache.solr.client.solrj.impl.Http2SolrClient; import org.apache.solr.client.solrj.request.AbstractUpdateRequest; import org.apache.solr.client.solrj.request.CoreAdminRequest.WaitForState; import org.apache.solr.client.solrj.request.UpdateRequest; @@ -124,7 +124,7 @@ public static interface RecoveryListener { private int retries; private boolean recoveringAfterStartup; private CoreContainer cc; - private volatile HttpUriRequest prevSendPreRecoveryHttpUriRequest; + private volatile FutureTask> prevSendPreRecoveryHttpUriRequest; private final Replica.Type replicaType; private CoreDescriptor coreDescriptor; @@ -175,25 +175,18 @@ public final void setRecoveringAfterStartup(boolean recoveringAfterStartup) { this.recoveringAfterStartup = recoveringAfterStartup; } - /** Builds a new HttpSolrClient for use in recovery. Caller must close */ - private HttpSolrClient.Builder recoverySolrClientBuilder(String baseUrl, String leaderCoreName) { - // workaround for SOLR-13605: get the configured timeouts & set them directly - // (even though getRecoveryOnlyHttpClient() already has them set) + private Http2SolrClient.Builder recoverySolrClientBuilder(String baseUrl, String leaderCoreName) { final UpdateShardHandlerConfig cfg = cc.getConfig().getUpdateShardHandlerConfig(); - return (new HttpSolrClient.Builder(baseUrl) + return new Http2SolrClient.Builder(baseUrl) .withDefaultCollection(leaderCoreName) - .withConnectionTimeout(cfg.getDistributedConnectionTimeout(), TimeUnit.MILLISECONDS) - .withSocketTimeout(cfg.getDistributedSocketTimeout(), TimeUnit.MILLISECONDS) - .withHttpClient(cc.getUpdateShardHandler().getRecoveryOnlyHttpClient())); + .withHttpClient(cc.getUpdateShardHandler().getRecoveryOnlyHttpClient()); } // make sure any threads stop retrying @Override public final void close() { close = true; - if (prevSendPreRecoveryHttpUriRequest != null) { - prevSendPreRecoveryHttpUriRequest.abort(); - } + cancelPrepRecoveryCmd(); log.warn("Stopping recovery for core=[{}] coreNodeName=[{}]", coreName, coreZkNodeName); } @@ -634,11 +627,7 @@ public final void doSyncOrReplicateRecovery(SolrCore core) throws Exception { .getCollection(cloudDesc.getCollectionName()) .getSlice(cloudDesc.getShardId()); - try { - prevSendPreRecoveryHttpUriRequest.abort(); - } catch (NullPointerException e) { - // okay - } + cancelPrepRecoveryCmd(); if (isClosed()) { log.info("RecoveryStrategy has been closed"); @@ -894,7 +883,6 @@ public final boolean isClosed() { private final void sendPrepRecoveryCmd(String leaderBaseUrl, String leaderCoreName, Slice slice) throws SolrServerException, IOException, InterruptedException, ExecutionException { - WaitForState prepCmd = new WaitForState(); prepCmd.setCoreName(leaderCoreName); prepCmd.setNodeName(zkController.getNodeName()); @@ -915,18 +903,19 @@ private final void sendPrepRecoveryCmd(String leaderBaseUrl, String leaderCoreNa int readTimeout = conflictWaitMs + Integer.parseInt(System.getProperty("prepRecoveryReadTimeoutExtraWait", "8000")); - try (HttpSolrClient client = + try (SolrClient client = recoverySolrClientBuilder( leaderBaseUrl, null) // leader core omitted since client only used for 'admin' request - .withSocketTimeout(readTimeout, TimeUnit.MILLISECONDS) + .withIdleTimeout(readTimeout, TimeUnit.MILLISECONDS) .build()) { - HttpUriRequestResponse mrr = client.httpUriRequest(prepCmd); - prevSendPreRecoveryHttpUriRequest = mrr.httpUriRequest; - + prevSendPreRecoveryHttpUriRequest = new FutureTask<>(() -> client.request(prepCmd)); log.info("Sending prep recovery command to [{}]; [{}]", leaderBaseUrl, prepCmd); - - mrr.future.get(); + prevSendPreRecoveryHttpUriRequest.run(); } } + + private void cancelPrepRecoveryCmd() { + Optional.ofNullable(prevSendPreRecoveryHttpUriRequest).ifPresent(req -> req.cancel(true)); + } } diff --git a/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java b/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java index b347565450a..69a2ddda8bb 100644 --- a/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java +++ b/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java @@ -84,7 +84,6 @@ import java.util.zip.Adler32; import java.util.zip.Checksum; import java.util.zip.InflaterInputStream; -import org.apache.http.client.HttpClient; import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.index.IndexCommit; import org.apache.lucene.index.IndexWriter; @@ -97,9 +96,9 @@ import org.apache.lucene.store.IndexOutput; import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.impl.Http2SolrClient; import org.apache.solr.client.solrj.impl.HttpClientUtil; -import org.apache.solr.client.solrj.impl.HttpSolrClient; -import org.apache.solr.client.solrj.impl.HttpSolrClient.Builder; +import org.apache.solr.client.solrj.impl.InputStreamResponseParser; import org.apache.solr.client.solrj.request.QueryRequest; import org.apache.solr.cloud.CloudDescriptor; import org.apache.solr.cloud.ZkController; @@ -128,12 +127,12 @@ import org.apache.solr.search.SolrIndexSearcher; import org.apache.solr.security.AllowListUrlChecker; import org.apache.solr.update.CommitUpdateCommand; +import org.apache.solr.update.UpdateShardHandler; import org.apache.solr.util.FileUtils; import org.apache.solr.util.IndexOutputOutputStream; import org.apache.solr.util.RTimer; import org.apache.solr.util.RefCounted; import org.apache.solr.util.TestInjection; -import org.apache.solr.util.stats.InstrumentedHttpRequestExecutor; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -186,7 +185,7 @@ public class IndexFetcher { boolean fetchFromLeader = false; - private final HttpClient myHttpClient; + private final SolrClient solrClient; private Integer connTimeout; @@ -261,22 +260,22 @@ public String getMessage() { } } - private static HttpClient createHttpClient( - SolrCore core, - String httpBasicAuthUser, - String httpBasicAuthPassword, - boolean useCompression) { - final ModifiableSolrParams httpClientParams = new ModifiableSolrParams(); - httpClientParams.set(HttpClientUtil.PROP_BASIC_AUTH_USER, httpBasicAuthUser); - httpClientParams.set(HttpClientUtil.PROP_BASIC_AUTH_PASS, httpBasicAuthPassword); - httpClientParams.set(HttpClientUtil.PROP_ALLOW_COMPRESSION, useCompression); - // no metrics, just tracing - InstrumentedHttpRequestExecutor executor = new InstrumentedHttpRequestExecutor(null); - return HttpClientUtil.createClient( - httpClientParams, - core.getCoreContainer().getUpdateShardHandler().getRecoveryOnlyConnectionManager(), - true, - executor); + // It's crucial not to remove the authentication credentials as they are essential for User + // managed replication. + // GitHub PR #2276 + private SolrClient createSolrClient( + SolrCore core, String httpBasicAuthUser, String httpBasicAuthPassword, String leaderBaseUrl) { + final UpdateShardHandler updateShardHandler = core.getCoreContainer().getUpdateShardHandler(); + Http2SolrClient httpClient = + new Http2SolrClient.Builder(leaderBaseUrl) + .withHttpClient(updateShardHandler.getRecoveryOnlyHttpClient()) + .withListenerFactory( + updateShardHandler.getRecoveryOnlyHttpClient().getListenerFactory()) + .withBasicAuthCredentials(httpBasicAuthUser, httpBasicAuthPassword) + .withIdleTimeout(soTimeout, TimeUnit.MILLISECONDS) + .withConnectionTimeout(connTimeout, TimeUnit.MILLISECONDS) + .build(); + return httpClient; } public IndexFetcher( @@ -318,12 +317,10 @@ public IndexFetcher( if (soTimeout == -1) { soTimeout = getParameter(initArgs, HttpClientUtil.PROP_SO_TIMEOUT, 120000, null); } - String httpBasicAuthUser = (String) initArgs.get(HttpClientUtil.PROP_BASIC_AUTH_USER); String httpBasicAuthPassword = (String) initArgs.get(HttpClientUtil.PROP_BASIC_AUTH_PASS); - myHttpClient = - createHttpClient( - solrCore, httpBasicAuthUser, httpBasicAuthPassword, useExternalCompression); + solrClient = + createSolrClient(solrCore, httpBasicAuthUser, httpBasicAuthPassword, leaderBaseUrl); } private void setLeaderCoreUrl(String leaderCoreUrl) { @@ -381,16 +378,10 @@ public NamedList getLatestVersion() throws IOException { params.set(CommonParams.WT, JAVABIN); params.set(CommonParams.QT, ReplicationHandler.PATH); QueryRequest req = new QueryRequest(params); - + req.setBasePath(leaderBaseUrl); // TODO modify to use shardhandler - try (SolrClient client = - new Builder(leaderBaseUrl) - .withHttpClient(myHttpClient) - .withConnectionTimeout(connTimeout, TimeUnit.MILLISECONDS) - .withSocketTimeout(soTimeout, TimeUnit.MILLISECONDS) - .build()) { - - return client.request(req, leaderCoreName); + try { + return solrClient.request(req, leaderCoreName); } catch (SolrServerException e) { throw new SolrException(ErrorCode.SERVER_ERROR, e.getMessage(), e); } @@ -408,15 +399,10 @@ private void fetchFileList(long gen) throws IOException { params.set(CommonParams.WT, JAVABIN); params.set(CommonParams.QT, ReplicationHandler.PATH); QueryRequest req = new QueryRequest(params); - + req.setBasePath(leaderBaseUrl); // TODO modify to use shardhandler - try (SolrClient client = - new HttpSolrClient.Builder(leaderBaseUrl) - .withHttpClient(myHttpClient) - .withConnectionTimeout(connTimeout, TimeUnit.MILLISECONDS) - .withSocketTimeout(soTimeout, TimeUnit.MILLISECONDS) - .build()) { - NamedList response = client.request(req, leaderCoreName); + try { + NamedList response = solrClient.request(req, leaderCoreName); List> files = (List>) response.get(CMD_GET_FILE_LIST); if (files != null) filesToDownload = Collections.synchronizedList(files); @@ -1805,10 +1791,10 @@ public void fetchFile() throws Exception { private void fetch() throws Exception { try { while (true) { - int result; - try (FastInputStream is = getStream()) { + try (FastInputStream fis = getStream()) { + int result; // fetch packets one by one in a single request - result = fetchPackets(is); + result = fetchPackets(fis); if (result == 0 || result == NO_CONTENT) { return; } @@ -1834,18 +1820,25 @@ private int fetchPackets(FastInputStream fis) throws Exception { byte[] longbytes = new byte[8]; try { while (true) { + if (fis.peek() == -1) { + if (bytesDownloaded == 0) { + log.warn("No content received for file: {}", fileName); + return NO_CONTENT; + } + return 0; + } if (stop) { stop = false; aborted = true; throw new ReplicationHandlerException("User aborted replication"); } long checkSumServer = -1; + fis.readFully(intbytes); // read the size of the packet int packetSize = readInt(intbytes); if (packetSize <= 0) { - log.warn("No content received for file: {}", fileName); - return NO_CONTENT; + continue; } // TODO consider recoding the remaining logic to not use/need buf[]; instead use the // internal buffer of fis @@ -1879,7 +1872,6 @@ private int fetchPackets(FastInputStream fis) throws Exception { log.debug("Fetched and wrote {} bytes of file: {}", bytesDownloaded, fileName); // errorCount is always set to zero after a successful packet errorCount = 0; - if (bytesDownloaded >= size) return 0; } } catch (ReplicationHandlerException e) { throw e; @@ -1968,7 +1960,7 @@ private void cleanup() { private FastInputStream getStream() throws IOException { ModifiableSolrParams params = new ModifiableSolrParams(); - // //the method is command=filecontent + // the method is command=filecontent params.set(COMMAND, CMD_GET_FILE); params.set(GENERATION, Long.toString(indexGen)); params.set(CommonParams.QT, ReplicationHandler.PATH); @@ -1991,17 +1983,13 @@ private FastInputStream getStream() throws IOException { NamedList response; InputStream is = null; - // TODO use shardhandler - try (SolrClient client = - new Builder(leaderBaseUrl) - .withHttpClient(myHttpClient) - .withResponseParser(null) - .withConnectionTimeout(connTimeout, TimeUnit.MILLISECONDS) - .withSocketTimeout(soTimeout, TimeUnit.MILLISECONDS) - .build()) { + try { QueryRequest req = new QueryRequest(params); - response = client.request(req, leaderCoreName); + req.setResponseParser(new InputStreamResponseParser(FILE_STREAM)); + req.setBasePath(leaderBaseUrl); + if (useExternalCompression) req.addHeader("Accept-Encoding", "gzip"); + response = solrClient.request(req, leaderCoreName); is = (InputStream) response.get("stream"); if (useInternalCompression) { is = new InflaterInputStream(is); @@ -2125,21 +2113,15 @@ NamedList getDetails() throws IOException, SolrServerException { params.set("follower", false); params.set(CommonParams.QT, ReplicationHandler.PATH); + QueryRequest request = new QueryRequest(params); + request.setBasePath(leaderBaseUrl); // TODO use shardhandler - try (SolrClient client = - new HttpSolrClient.Builder(leaderBaseUrl) - .withHttpClient(myHttpClient) - .withConnectionTimeout(connTimeout, TimeUnit.MILLISECONDS) - .withSocketTimeout(soTimeout, TimeUnit.MILLISECONDS) - .build()) { - QueryRequest request = new QueryRequest(params); - return client.request(request, leaderCoreName); - } + return solrClient.request(request, leaderCoreName); } public void destroy() { abortFetch(); - HttpClientUtil.close(myHttpClient); + IOUtils.closeQuietly(solrClient); } String getLeaderCoreUrl() { diff --git a/solr/core/src/java/org/apache/solr/update/UpdateShardHandler.java b/solr/core/src/java/org/apache/solr/update/UpdateShardHandler.java index b3ab8cb9156..650a61dc41d 100644 --- a/solr/core/src/java/org/apache/solr/update/UpdateShardHandler.java +++ b/solr/core/src/java/org/apache/solr/update/UpdateShardHandler.java @@ -75,17 +75,15 @@ public class UpdateShardHandler implements SolrInfoBean { private final Http2SolrClient updateOnlyClient; - private final CloseableHttpClient recoveryOnlyClient; + private final Http2SolrClient recoveryOnlyClient; private final CloseableHttpClient defaultClient; - private final InstrumentedPoolingHttpClientConnectionManager recoveryOnlyConnectionManager; - private final InstrumentedPoolingHttpClientConnectionManager defaultConnectionManager; private final InstrumentedHttpRequestExecutor httpRequestExecutor; - private final InstrumentedHttpListenerFactory updateHttpListenerFactory; + private final InstrumentedHttpListenerFactory trackHttpSolrMetrics; private SolrMetricsContext solrMetricsContext; @@ -93,16 +91,11 @@ public class UpdateShardHandler implements SolrInfoBean { private int connectionTimeout = HttpClientUtil.DEFAULT_CONNECT_TIMEOUT; public UpdateShardHandler(UpdateShardHandlerConfig cfg) { - recoveryOnlyConnectionManager = - new InstrumentedPoolingHttpClientConnectionManager( - HttpClientUtil.getSocketFactoryRegistryProvider().getSocketFactoryRegistry()); defaultConnectionManager = new InstrumentedPoolingHttpClientConnectionManager( HttpClientUtil.getSocketFactoryRegistryProvider().getSocketFactoryRegistry()); ModifiableSolrParams clientParams = new ModifiableSolrParams(); if (cfg != null) { - recoveryOnlyConnectionManager.setMaxTotal(cfg.getMaxUpdateConnections()); - recoveryOnlyConnectionManager.setDefaultMaxPerRoute(cfg.getMaxUpdateConnectionsPerHost()); defaultConnectionManager.setMaxTotal(cfg.getMaxUpdateConnections()); defaultConnectionManager.setDefaultMaxPerRoute(cfg.getMaxUpdateConnectionsPerHost()); clientParams.set(HttpClientUtil.PROP_SO_TIMEOUT, cfg.getDistributedSocketTimeout()); @@ -120,10 +113,8 @@ public UpdateShardHandler(UpdateShardHandlerConfig cfg) { log.debug("Created default UpdateShardHandler HTTP client with params: {}", clientParams); httpRequestExecutor = new InstrumentedHttpRequestExecutor(getMetricNameStrategy(cfg)); - updateHttpListenerFactory = new InstrumentedHttpListenerFactory(getNameStrategy(cfg)); - recoveryOnlyClient = - HttpClientUtil.createClient( - clientParams, recoveryOnlyConnectionManager, false, httpRequestExecutor); + trackHttpSolrMetrics = new InstrumentedHttpListenerFactory(getNameStrategy(cfg)); + defaultClient = HttpClientUtil.createClient( clientParams, defaultConnectionManager, false, httpRequestExecutor); @@ -133,15 +124,24 @@ public UpdateShardHandler(UpdateShardHandlerConfig cfg) { DistributedUpdateProcessor.DISTRIB_FROM, DistributingUpdateProcessorFactory.DISTRIB_UPDATE_PARAM); Http2SolrClient.Builder updateOnlyClientBuilder = new Http2SolrClient.Builder(); + Http2SolrClient.Builder recoveryOnlyClientBuilder = new Http2SolrClient.Builder(); if (cfg != null) { updateOnlyClientBuilder .withConnectionTimeout(cfg.getDistributedConnectionTimeout(), TimeUnit.MILLISECONDS) .withIdleTimeout(cfg.getDistributedSocketTimeout(), TimeUnit.MILLISECONDS) .withMaxConnectionsPerHost(cfg.getMaxUpdateConnectionsPerHost()); + recoveryOnlyClientBuilder + .withConnectionTimeout(cfg.getDistributedConnectionTimeout(), TimeUnit.MILLISECONDS) + .withIdleTimeout(cfg.getDistributedSocketTimeout(), TimeUnit.MILLISECONDS) + .withMaxConnectionsPerHost(cfg.getMaxUpdateConnectionsPerHost()); } + updateOnlyClientBuilder.withTheseParamNamesInTheUrl(urlParamNames); updateOnlyClient = updateOnlyClientBuilder.build(); - updateOnlyClient.addListenerFactory(updateHttpListenerFactory); + updateOnlyClient.addListenerFactory(trackHttpSolrMetrics); + + recoveryOnlyClient = recoveryOnlyClientBuilder.build(); + recoveryOnlyClient.addListenerFactory(trackHttpSolrMetrics); ThreadFactory recoveryThreadFactory = new SolrNamedThreadFactory("recoveryExecutor"); if (cfg != null && cfg.getMaxRecoveryThreads() > 0) { @@ -205,7 +205,7 @@ public String getName() { public void initializeMetrics(SolrMetricsContext parentContext, String scope) { solrMetricsContext = parentContext.getChildContext(this); String expandedScope = SolrMetricManager.mkName(scope, getCategory().name()); - updateHttpListenerFactory.initializeMetrics(solrMetricsContext, expandedScope); + trackHttpSolrMetrics.initializeMetrics(solrMetricsContext, expandedScope); defaultConnectionManager.initializeMetrics(solrMetricsContext, expandedScope); updateExecutor = MetricUtils.instrumentedExecutorService( @@ -247,7 +247,7 @@ public Http2SolrClient getUpdateOnlyHttpClient() { } // don't introduce a bug, this client is for recovery ops only! - public HttpClient getRecoveryOnlyHttpClient() { + public Http2SolrClient getRecoveryOnlyHttpClient() { return recoveryOnlyClient; } @@ -264,10 +264,6 @@ public PoolingHttpClientConnectionManager getDefaultConnectionManager() { return defaultConnectionManager; } - public PoolingHttpClientConnectionManager getRecoveryOnlyConnectionManager() { - return recoveryOnlyConnectionManager; - } - /** * @return executor for recovery operations */ @@ -290,10 +286,9 @@ public void close() { // do nothing } IOUtils.closeQuietly(updateOnlyClient); - HttpClientUtil.close(recoveryOnlyClient); + IOUtils.closeQuietly(recoveryOnlyClient); HttpClientUtil.close(defaultClient); defaultConnectionManager.close(); - recoveryOnlyConnectionManager.close(); } } @@ -309,5 +304,6 @@ public int getConnectionTimeout() { public void setSecurityBuilder(HttpClientBuilderPlugin builder) { builder.setup(updateOnlyClient); + builder.setup(recoveryOnlyClient); } } diff --git a/solr/core/src/java/org/apache/solr/update/processor/NumFieldLimitingUpdateRequestProcessorFactory.java b/solr/core/src/java/org/apache/solr/update/processor/NumFieldLimitingUpdateRequestProcessorFactory.java new file mode 100644 index 00000000000..9382d25a499 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/update/processor/NumFieldLimitingUpdateRequestProcessorFactory.java @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.update.processor; + +import java.io.IOException; +import java.lang.invoke.MethodHandles; +import java.util.Locale; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.response.SolrQueryResponse; +import org.apache.solr.search.SolrIndexSearcher; +import org.apache.solr.update.AddUpdateCommand; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * This factory generates an UpdateRequestProcessor which fails update requests once a core has + * exceeded a configurable maximum number of fields. Meant as a safeguard to help users notice + * potentially-dangerous schema design before performance and stability problems start to occur. + * + *

The URP uses the core's {@link SolrIndexSearcher} to judge the current number of fields. + * Accordingly, it undercounts the number of fields in the core - missing all fields added since the + * previous searcher was opened. As such, the URP's request-blocking is "best effort" - it cannot be + * relied on as a precise limit on the number of fields. + * + *

Additionally, the field-counting includes all documents present in the index, including any + * deleted docs that haven't yet been purged via segment merging. Note that this can differ + * significantly from the number of fields defined in managed-schema.xml - especially when dynamic + * fields are enabled. The only way to reduce this field count is to delete documents and wait until + * the deleted documents have been removed by segment merges. Users may of course speed up this + * process by tweaking Solr's segment-merging, triggering an "optimize" operation, etc. + * + *

{@link NumFieldLimitingUpdateRequestProcessorFactory} accepts two configuration parameters: + * + *

    + *
  • maxFields - (required) The maximum number of fields before update requests + * should be aborted. Once this limit has been exceeded, additional update requests will fail + * until fields have been removed or the "maxFields" is increased. + *
  • warnOnly - (optional) If true then the URP logs verbose warnings + * about the limit being exceeded but doesn't abort update requests. Defaults to false + * if not specified + *
+ * + * @since 9.7.0 + */ +public class NumFieldLimitingUpdateRequestProcessorFactory extends UpdateRequestProcessorFactory { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + private static final String MAXIMUM_FIELDS_PARAM = "maxFields"; + private static final String WARN_ONLY_PARAM = "warnOnly"; + + // package visibility for tests + int maximumFields; + boolean warnOnly; + + @Override + public void init(NamedList args) { + warnOnly = args.indexOf(WARN_ONLY_PARAM, 0) > 0 ? args.getBooleanArg(WARN_ONLY_PARAM) : false; + + if (args.indexOf(MAXIMUM_FIELDS_PARAM, 0) < 0) { + throw new IllegalArgumentException( + "The " + + MAXIMUM_FIELDS_PARAM + + " parameter is required for " + + getClass().getName() + + ", but no value was provided."); + } + final Object rawMaxFields = args.get(MAXIMUM_FIELDS_PARAM); + if (!(rawMaxFields instanceof Integer)) { + throw new IllegalArgumentException( + MAXIMUM_FIELDS_PARAM + " must be configured as a non-null "); + } + maximumFields = (Integer) rawMaxFields; + if (maximumFields <= 0) { + throw new IllegalArgumentException(MAXIMUM_FIELDS_PARAM + " must be a positive integer"); + } + } + + @Override + public UpdateRequestProcessor getInstance( + SolrQueryRequest req, SolrQueryResponse rsp, UpdateRequestProcessor next) { + // note: it's unusual to call req.getSearcher in a /update request but it should be fine + final int currentNumFields = req.getSearcher().getFieldInfos().size(); + if (currentNumFields <= maximumFields) { + // great; no need to insert an URP to block or log anything + return next; + } + + // Block indexing new documents + return new UpdateRequestProcessor(next) { + @Override + public void processAdd(AddUpdateCommand cmd) throws IOException { + String id = cmd.getPrintableId(); + final String messageSuffix = warnOnly ? "Blocking update of document " + id : ""; + final String message = + String.format( + Locale.ROOT, + "Current core has %d fields, exceeding the max-fields limit of %d. %s", + currentNumFields, + maximumFields, + messageSuffix); + if (warnOnly) { + log.warn(message); + } else { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, message); + } + } + }; + } +} diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-follower-auth.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-follower-auth.xml new file mode 100644 index 00000000000..1635cfb099b --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-follower-auth.xml @@ -0,0 +1,61 @@ + + + + + + ${tests.luceneMatchVersion:LATEST} + + + ${solr.data.dir:} + + + + + + + + true + + + + + + + + + + + + + + http://127.0.0.1:TEST_PORT/solr/collection1 + 00:00:01 + COMPRESSION + solr + SolrRocks + + + + + + + max-age=30, public + + + + diff --git a/solr/core/src/test-files/solr/configsets/cloud-minimal-field-limiting/conf/schema.xml b/solr/core/src/test-files/solr/configsets/cloud-minimal-field-limiting/conf/schema.xml new file mode 100644 index 00000000000..d6a2fa7a916 --- /dev/null +++ b/solr/core/src/test-files/solr/configsets/cloud-minimal-field-limiting/conf/schema.xml @@ -0,0 +1,30 @@ + + + + + + + + + + + + + + id + diff --git a/solr/core/src/test-files/solr/configsets/cloud-minimal-field-limiting/conf/solrconfig.xml b/solr/core/src/test-files/solr/configsets/cloud-minimal-field-limiting/conf/solrconfig.xml new file mode 100644 index 00000000000..00f1ab3714b --- /dev/null +++ b/solr/core/src/test-files/solr/configsets/cloud-minimal-field-limiting/conf/solrconfig.xml @@ -0,0 +1,60 @@ + + + + + + + + + ${solr.data.dir:} + + + + + ${tests.luceneMatchVersion:LATEST} + + + + ${solr.commitwithin.softcommit:true} + + + + + + + explicit + true + text + + + + + + ${solr.test.maxFields:1234} + + + + + + + + + + + diff --git a/solr/core/src/test/org/apache/solr/cloud/RecoveryStrategyStressTest.java b/solr/core/src/test/org/apache/solr/cloud/RecoveryStrategyStressTest.java new file mode 100644 index 00000000000..06c03971a96 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/cloud/RecoveryStrategyStressTest.java @@ -0,0 +1,135 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.cloud; + +import com.carrotsearch.randomizedtesting.annotations.Nightly; +import java.io.IOException; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.SolrQuery; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.impl.CloudLegacySolrClient; +import org.apache.solr.client.solrj.impl.HttpSolrClient; +import org.apache.solr.client.solrj.request.CollectionAdminRequest; +import org.apache.solr.client.solrj.request.UpdateRequest; +import org.apache.solr.common.cloud.DocCollection; +import org.apache.solr.common.cloud.Replica; +import org.apache.solr.common.cloud.Slice; +import org.apache.solr.common.util.SolrNamedThreadFactory; +import org.apache.solr.embedded.JettySolrRunner; +import org.junit.BeforeClass; +import org.junit.Test; + +@Nightly +public class RecoveryStrategyStressTest extends SolrCloudTestCase { + + @BeforeClass + public static void setupCluster() throws Exception { + cluster = configureCluster(4).addConfig("conf", configset("cloud-minimal")).configure(); + } + + @Test + public void stressTestRecovery() throws Exception { + final String collection = "recoveryStressTest"; + CollectionAdminRequest.createCollection(collection, "conf", 1, 4) + .process(cluster.getSolrClient()); + waitForState( + "Expected a collection with one shard and two replicas", collection, clusterShape(1, 4)); + final var scheduledExecutorService = + Executors.newScheduledThreadPool(1, new SolrNamedThreadFactory("stressTestRecovery")); + try (SolrClient solrClient = + cluster.basicSolrClientBuilder().withDefaultCollection(collection).build()) { + final StoppableIndexingThread indexThread = + new StoppableIndexingThread(null, solrClient, "1", true, 10, 1, true); + + final var startAndStopCount = new CountDownLatch(50); + final Thread startAndStopRandomReplicas = + new Thread( + () -> { + try { + while (startAndStopCount.getCount() > 0) { + DocCollection state = getCollectionState(collection); + Replica leader = state.getLeader("shard1"); + Replica replica = + getRandomReplica(state.getSlice("shard1"), (r) -> !leader.equals(r)); + + JettySolrRunner jetty = cluster.getReplicaJetty(replica); + jetty.stop(); + Thread.sleep(100); + jetty.start(); + startAndStopCount.countDown(); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + startAndStopRandomReplicas.start(); + // index and commit doc after fixed interval of 10 sec + scheduledExecutorService.scheduleWithFixedDelay( + indexThread, 1000, 10000, TimeUnit.MILLISECONDS); + scheduledExecutorService.scheduleWithFixedDelay( + () -> { + try { + new UpdateRequest().commit(solrClient, collection); + } catch (IOException e) { + throw new RuntimeException(e); + } catch (SolrServerException e) { + throw new RuntimeException(e); + } + }, + 100, + 10000, + TimeUnit.MILLISECONDS); + + startAndStopCount.await(); + scheduledExecutorService.shutdownNow(); + // final commit to make documents visible for replicas + new UpdateRequest().commit(solrClient, collection); + } + cluster.getZkStateReader().waitForState(collection, 120, TimeUnit.SECONDS, clusterShape(1, 4)); + + // test that leader and replica have same doc count + DocCollection state = getCollectionState(collection); + assertShardConsistency(state.getSlice("shard1"), true); + } + + private void assertShardConsistency(Slice shard, boolean expectDocs) throws Exception { + List replicas = shard.getReplicas(r -> r.getState() == Replica.State.ACTIVE); + long[] numCounts = new long[replicas.size()]; + int i = 0; + for (Replica replica : replicas) { + try (var client = + new HttpSolrClient.Builder(replica.getBaseUrl()) + .withDefaultCollection(replica.getCoreName()) + .withHttpClient(((CloudLegacySolrClient) cluster.getSolrClient()).getHttpClient()) + .build()) { + numCounts[i] = + client.query(new SolrQuery("*:*").add("distrib", "false")).getResults().getNumFound(); + i++; + } + } + for (int j = 1; j < replicas.size(); j++) { + if (numCounts[j] != numCounts[j - 1]) + fail("Mismatch in counts between replicas"); // TODO improve this! + if (numCounts[j] == 0 && expectDocs) + fail("Expected docs on shard " + shard.getName() + " but found none"); + } + } +} diff --git a/solr/core/src/test/org/apache/solr/handler/ReplicationTestHelper.java b/solr/core/src/test/org/apache/solr/handler/ReplicationTestHelper.java index 892dc679d3f..a9c81429467 100644 --- a/solr/core/src/test/org/apache/solr/handler/ReplicationTestHelper.java +++ b/solr/core/src/test/org/apache/solr/handler/ReplicationTestHelper.java @@ -103,7 +103,8 @@ private static void copyFile(File src, File dst, Integer port, boolean internalC if (null != port) { line = line.replace("TEST_PORT", port.toString()); } - line = line.replace("COMPRESSION", internalCompression ? "internal" : "false"); + String externalCompression = LuceneTestCase.random().nextBoolean() ? "external" : "false"; + line = line.replace("COMPRESSION", internalCompression ? "internal" : externalCompression); out.write(line); } } diff --git a/solr/core/src/test/org/apache/solr/handler/TestUserManagedReplicationWithAuth.java b/solr/core/src/test/org/apache/solr/handler/TestUserManagedReplicationWithAuth.java new file mode 100644 index 00000000000..b230aad2021 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/handler/TestUserManagedReplicationWithAuth.java @@ -0,0 +1,267 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.handler; + +import static org.apache.solr.common.params.CommonParams.JAVABIN; +import static org.apache.solr.handler.ReplicationHandler.CMD_DISABLE_POLL; +import static org.apache.solr.handler.ReplicationHandler.CMD_FETCH_INDEX; +import static org.apache.solr.handler.ReplicationHandler.COMMAND; +import static org.apache.solr.handler.ReplicationTestHelper.createAndStartJetty; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.SolrTestCaseJ4.SuppressSSL; +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.SolrQuery; +import org.apache.solr.client.solrj.SolrRequest; +import org.apache.solr.client.solrj.SolrResponse; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.request.HealthCheckRequest; +import org.apache.solr.client.solrj.request.QueryRequest; +import org.apache.solr.client.solrj.request.UpdateRequest; +import org.apache.solr.client.solrj.response.QueryResponse; +import org.apache.solr.common.SolrInputDocument; +import org.apache.solr.common.params.CommonParams; +import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.embedded.JettySolrRunner; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +@SuppressSSL +public class TestUserManagedReplicationWithAuth extends SolrTestCaseJ4 { + JettySolrRunner leaderJetty, followerJetty, followerJettyWithAuth; + SolrClient leaderClient, followerClient, followerClientWithAuth; + ReplicationTestHelper.SolrInstance leader = null, follower = null, followerWithAuth = null; + + private static String user = "solr"; + private static String pass = "SolrRocks"; + private static String securityJson = + "{\n" + + "\"authentication\":{ \n" + + " \"blockUnknown\": true, \n" + + " \"class\":\"solr.BasicAuthPlugin\",\n" + + " \"credentials\":{\"solr\":\"IV0EHq1OnNrj6gvRCwvFwTrZ1+z1oBbnQdiVC3otuq0= Ndd7LKvVBAaZIF0QAVi1ekCfAJXr1GGfLtRUXhgrF8c=\"}, \n" + + " \"realm\":\"My Solr users\", \n" + + " \"forwardCredentials\": false \n" + + "},\n" + + "\"authorization\":{\n" + + " \"class\":\"solr.RuleBasedAuthorizationPlugin\",\n" + + " \"permissions\":[{\"name\":\"security-edit\",\n" + + " \"role\":\"admin\"}],\n" + + " \"user-role\":{\"solr\":\"admin\"}\n" + + "}}"; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + systemSetPropertySolrDisableUrlAllowList("true"); + // leader with Basic auth enabled via security.json + leader = + new ReplicationTestHelper.SolrInstance( + createTempDir("solr-instance").toFile(), "leader", null); + leader.setUp(); + // Configuring basic auth for Leader + Path solrLeaderHome = Path.of(leader.getHomeDir()); + Files.write( + solrLeaderHome.resolve("security.json"), securityJson.getBytes(StandardCharsets.UTF_8)); + leaderJetty = ReplicationTestHelper.createAndStartJetty(leader); + leaderClient = + ReplicationTestHelper.createNewSolrClient( + buildUrl(leaderJetty.getLocalPort()), DEFAULT_TEST_CORENAME); + + // follower with no basic auth credentials for leader configured. + follower = + new ReplicationTestHelper.SolrInstance( + createTempDir("solr-instance").toFile(), "follower", leaderJetty.getLocalPort()); + follower.setUp(); + followerJetty = createAndStartJetty(follower); + followerClient = + ReplicationTestHelper.createNewSolrClient( + buildUrl(followerJetty.getLocalPort()), DEFAULT_TEST_CORENAME); + + // follower with basic auth credentials for leader configured in solrconfig.xml. + followerWithAuth = + new ReplicationTestHelper.SolrInstance( + createTempDir("solr-instance").toFile(), "follower-auth", leaderJetty.getLocalPort()); + followerWithAuth.setUp(); + followerJettyWithAuth = createAndStartJetty(followerWithAuth); + followerClientWithAuth = + ReplicationTestHelper.createNewSolrClient( + buildUrl(followerJettyWithAuth.getLocalPort()), DEFAULT_TEST_CORENAME); + } + + @Override + @After + public void tearDown() throws Exception { + super.tearDown(); + if (null != leaderJetty) { + leaderJetty.stop(); + leaderJetty = null; + } + if (null != followerJetty) { + followerJetty.stop(); + followerJetty = null; + } + if (null != followerJettyWithAuth) { + followerJettyWithAuth.stop(); + followerJettyWithAuth = null; + } + if (null != leaderClient) { + leaderClient.close(); + leaderClient = null; + } + if (null != followerClient) { + followerClient.close(); + followerClient = null; + } + if (null != followerClientWithAuth) { + followerClientWithAuth.close(); + followerClientWithAuth = null; + } + } + + private > T withBasicAuth(T req) { + req.setBasicAuthCredentials(user, pass); + return req; + } + + @Test + public void doTestManualFetchIndexWithAuthEnabled() throws Exception { + disablePoll(followerJetty, followerClient); + int nDocs = 500; + int docsAdded = 0; + + UpdateRequest commitReq = new UpdateRequest(); + withBasicAuth(commitReq); + for (int i = 0; docsAdded < nDocs / 2; i++, docsAdded++) { + SolrInputDocument doc = new SolrInputDocument(); + String[] fields = {"id", i + "", "name", "name = " + i}; + for (int j = 0; j < fields.length; j += 2) { + doc.addField(fields[j], fields[j + 1]); + } + UpdateRequest req = new UpdateRequest(); + withBasicAuth(req).add(doc); + req.process(leaderClient, DEFAULT_TEST_CORENAME); + if (i % 10 == 0) { + commitReq.commit(leaderClient, DEFAULT_TEST_CORENAME); + } + } + commitReq.commit(leaderClient, DEFAULT_TEST_CORENAME); + + assertEquals( + docsAdded, + queryWithBasicAuth(leaderClient, new SolrQuery("*:*")).getResults().getNumFound()); + + // Without Auth credentials fetchIndex will fail + pullIndexFromTo(leaderJetty, followerJetty, false); + assertNotEquals( + docsAdded, + queryWithBasicAuth(followerClient, new SolrQuery("*:*")).getResults().getNumFound()); + + // With Auth credentials + pullIndexFromTo(leaderJetty, followerJetty, true); + assertEquals( + docsAdded, + queryWithBasicAuth(followerClient, new SolrQuery("*:*")).getResults().getNumFound()); + } + + @Test + public void doTestAutoReplicationWithAuthEnabled() throws Exception { + int nDocs = 250; + UpdateRequest commitReq = new UpdateRequest(); + withBasicAuth(commitReq); + for (int i = 0; i < nDocs; i++) { + SolrInputDocument doc = new SolrInputDocument(); + String[] fields = {"id", i + "", "name", "name = " + i}; + for (int j = 0; j < fields.length; j += 2) { + doc.addField(fields[j], fields[j + 1]); + } + UpdateRequest req = new UpdateRequest(); + withBasicAuth(req).add(doc); + req.process(leaderClient, DEFAULT_TEST_CORENAME); + if (i % 10 == 0) { + commitReq.commit(leaderClient, DEFAULT_TEST_CORENAME); + } + } + commitReq.commit(leaderClient, DEFAULT_TEST_CORENAME); + // wait for followers to fetchIndex + Thread.sleep(5000); + // follower with auth should be healthy + HealthCheckRequest healthCheckRequestFollower = new HealthCheckRequest(); + healthCheckRequestFollower.setMaxGenerationLag(2); + assertEquals( + CommonParams.OK, + healthCheckRequestFollower + .process(followerClientWithAuth) + .getResponse() + .get(CommonParams.STATUS)); + // follower with auth should be unhealthy + healthCheckRequestFollower = new HealthCheckRequest(); + healthCheckRequestFollower.setMaxGenerationLag(2); + assertEquals( + CommonParams.FAILURE, + healthCheckRequestFollower.process(followerClient).getResponse().get(CommonParams.STATUS)); + } + + private QueryResponse queryWithBasicAuth(SolrClient client, SolrQuery q) + throws IOException, SolrServerException { + return withBasicAuth(new QueryRequest(q)).process(client); + } + + private void disablePoll(JettySolrRunner Jetty, SolrClient solrClient) + throws SolrServerException, IOException { + ModifiableSolrParams disablePollParams = new ModifiableSolrParams(); + disablePollParams.set(COMMAND, CMD_DISABLE_POLL); + disablePollParams.set(CommonParams.WT, JAVABIN); + disablePollParams.set(CommonParams.QT, ReplicationHandler.PATH); + QueryRequest req = new QueryRequest(disablePollParams); + withBasicAuth(req); + req.setBasePath(buildUrl(Jetty.getLocalPort())); + + solrClient.request(req, DEFAULT_TEST_CORENAME); + } + + private void pullIndexFromTo( + JettySolrRunner srcSolr, JettySolrRunner destSolr, boolean authEnabled) + throws SolrServerException, IOException { + String srcUrl = buildUrl(srcSolr.getLocalPort()) + "/" + DEFAULT_TEST_CORENAME; + String destUrl = buildUrl(destSolr.getLocalPort()) + "/" + DEFAULT_TEST_CORENAME; + QueryRequest req = getQueryRequestForFetchIndex(authEnabled, srcUrl); + req.setBasePath(buildUrl(destSolr.getLocalPort())); + followerClient.request(req, DEFAULT_TEST_CORENAME); + } + + private QueryRequest getQueryRequestForFetchIndex(boolean authEnabled, String srcUrl) { + ModifiableSolrParams solrParams = new ModifiableSolrParams(); + solrParams.set(COMMAND, CMD_FETCH_INDEX); + solrParams.set(CommonParams.WT, JAVABIN); + solrParams.set(CommonParams.QT, ReplicationHandler.PATH); + solrParams.set("leaderUrl", srcUrl); + solrParams.set("wait", "true"); + if (authEnabled) { + solrParams.set("httpBasicAuthUser", user); + solrParams.set("httpBasicAuthPassword", pass); + } + QueryRequest req = new QueryRequest(solrParams); + return req; + } +} diff --git a/solr/core/src/test/org/apache/solr/update/processor/NumFieldLimitingUpdateRequestProcessorFactoryTest.java b/solr/core/src/test/org/apache/solr/update/processor/NumFieldLimitingUpdateRequestProcessorFactoryTest.java new file mode 100644 index 00000000000..eae9fe0e7c9 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/update/processor/NumFieldLimitingUpdateRequestProcessorFactoryTest.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.update.processor; + +import org.apache.solr.SolrTestCase; +import org.apache.solr.common.util.NamedList; +import org.hamcrest.Matchers; +import org.junit.Before; +import org.junit.Test; + +public class NumFieldLimitingUpdateRequestProcessorFactoryTest extends SolrTestCase { + + private NumFieldLimitingUpdateRequestProcessorFactory factory = null; + + @Before + public void initFactory() { + factory = new NumFieldLimitingUpdateRequestProcessorFactory(); + } + + @Test + public void testReportsErrorIfMaximumFieldsNotProvided() { + final var initArgs = new NamedList<>(); + final IllegalArgumentException thrown = + expectThrows( + IllegalArgumentException.class, + () -> { + factory.init(initArgs); + }); + assertThat(thrown.getMessage(), Matchers.containsString("maxFields parameter is required")); + assertThat(thrown.getMessage(), Matchers.containsString("no value was provided")); + } + + @Test + public void testReportsErrorIfMaximumFieldsIsInvalid() { + final var initArgs = new NamedList<>(); + initArgs.add("maxFields", "nonIntegerValue"); + IllegalArgumentException thrown = + expectThrows( + IllegalArgumentException.class, + () -> { + factory.init(initArgs); + }); + assertThat( + thrown.getMessage(), + Matchers.containsString("maxFields must be configured as a non-null ")); + + initArgs.clear(); + initArgs.add("maxFields", Integer.valueOf(-5)); + thrown = + expectThrows( + IllegalArgumentException.class, + () -> { + factory.init(initArgs); + }); + assertThat( + thrown.getMessage(), Matchers.containsString("maxFields must be a positive integer")); + } + + @Test + public void testCorrectlyParsesAllConfigurationParams() { + final var initArgs = new NamedList<>(); + initArgs.add("maxFields", 123); + initArgs.add("warnOnly", Boolean.TRUE); + + factory.init(initArgs); + + assertEquals(123, factory.maximumFields); + assertEquals(true, factory.warnOnly); + } +} diff --git a/solr/core/src/test/org/apache/solr/update/processor/NumFieldLimitingUpdateRequestProcessorIntegrationTest.java b/solr/core/src/test/org/apache/solr/update/processor/NumFieldLimitingUpdateRequestProcessorIntegrationTest.java new file mode 100644 index 00000000000..0ebaba57215 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/update/processor/NumFieldLimitingUpdateRequestProcessorIntegrationTest.java @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.update.processor; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.UUID; +import java.util.concurrent.TimeUnit; +import org.apache.solr.client.solrj.request.CollectionAdminRequest; +import org.apache.solr.cloud.SolrCloudTestCase; +import org.apache.solr.common.SolrInputDocument; +import org.hamcrest.Matchers; +import org.junit.BeforeClass; +import org.junit.Test; + +public class NumFieldLimitingUpdateRequestProcessorIntegrationTest extends SolrCloudTestCase { + + private static String COLLECTION_NAME = "collName"; + private static String FIELD_LIMITING_CS_NAME = "fieldLimitingConfig"; + + @BeforeClass + public static void setupCluster() throws Exception { + final var configPath = + TEST_PATH().resolve("configsets").resolve("cloud-minimal-field-limiting").resolve("conf"); + configureCluster(1).addConfig(FIELD_LIMITING_CS_NAME, configPath).configure(); + + final var createRequest = + CollectionAdminRequest.createCollection(COLLECTION_NAME, FIELD_LIMITING_CS_NAME, 1, 1); + createRequest.process(cluster.getSolrClient()); + cluster.waitForActiveCollection(COLLECTION_NAME, 20, TimeUnit.SECONDS, 1, 1); + } + + private void setFieldLimitTo(int value) throws Exception { + System.setProperty("solr.test.maxFields", String.valueOf(value)); + + final var reloadRequest = CollectionAdminRequest.reloadCollection(COLLECTION_NAME); + final var reloadResponse = reloadRequest.process(cluster.getSolrClient()); + assertEquals(0, reloadResponse.getStatus()); + } + + @Test + public void test() throws Exception { + setFieldLimitTo(100); + + // Add 100 new fields - should all succeed since we're under the limit until the final commit + for (int i = 0; i < 5; i++) { + addNewFieldsAndCommit(20); + } + + // Adding any additional docs should fail because we've exceeded the field limit + final var thrown = + expectThrows( + Exception.class, + () -> { + addNewFieldsAndCommit(10); + }); + assertThat( + thrown.getMessage(), Matchers.containsString("exceeding the max-fields limit of 100")); + + // After raising the limit, updates succeed again + setFieldLimitTo(150); + for (int i = 0; i < 3; i++) { + addNewFieldsAndCommit(10); + } + } + + private void addNewFieldsAndCommit(int numFields) throws Exception { + final var docList = getDocumentListToAddFields(numFields); + final var updateResponse = cluster.getSolrClient(COLLECTION_NAME).add(docList); + assertEquals(0, updateResponse.getStatus()); + cluster.getSolrClient(COLLECTION_NAME).commit(); + } + + private Collection getDocumentListToAddFields(int numFieldsToAdd) { + int fieldsAdded = 0; + final var docList = new ArrayList(); + while (fieldsAdded < numFieldsToAdd) { + final var doc = new SolrInputDocument(); + doc.addField("id", randomFieldValue()); + + final int fieldsForDoc = Math.min(numFieldsToAdd - fieldsAdded, 5); + for (int fieldCount = 0; fieldCount < fieldsForDoc; fieldCount++) { + doc.addField(randomFieldName(), randomFieldValue()); + } + fieldsAdded += fieldsForDoc; + docList.add(doc); + } + + return docList; + } + + private String randomFieldName() { + return UUID.randomUUID().toString().replace("-", "_") + "_s"; + } + + private String randomFieldValue() { + return UUID.randomUUID().toString(); + } +} diff --git a/solr/core/src/test/org/apache/solr/util/TimeZoneUtilsTest.java b/solr/core/src/test/org/apache/solr/util/TimeZoneUtilsTest.java index 4b102a4bdda..5d085be3153 100644 --- a/solr/core/src/test/org/apache/solr/util/TimeZoneUtilsTest.java +++ b/solr/core/src/test/org/apache/solr/util/TimeZoneUtilsTest.java @@ -62,8 +62,8 @@ public void testValidIds() { // Hack: Why do some timezones have useDaylightTime() as true, but DST as 0? // It causes an exception during String.valueOf(actual/expected) - if (expected.useDaylightTime() && expected.getDSTSavings() == 0 - || actual.useDaylightTime() && actual.getDSTSavings() == 0) { + if ((expected.useDaylightTime() && expected.getDSTSavings() == 0) + || (actual.useDaylightTime() && actual.getDSTSavings() == 0)) { if (log.isWarnEnabled()) { log.warn( "Not expecting DST to be 0 for {} " + " (actual: {})", diff --git a/solr/server/solr/configsets/_default/conf/solrconfig.xml b/solr/server/solr/configsets/_default/conf/solrconfig.xml index e04a4cb9a9b..5b24094179b 100644 --- a/solr/server/solr/configsets/_default/conf/solrconfig.xml +++ b/solr/server/solr/configsets/_default/conf/solrconfig.xml @@ -894,6 +894,9 @@ [^\w-\.] _ + + 1000 + @@ -937,10 +940,11 @@ pdoubles + + processor="uuid,remove-blank,field-name-mutating,max-fields,parse-boolean,parse-long,parse-double,parse-date,add-schema-fields"> diff --git a/solr/solr-ref-guide/modules/configuration-guide/pages/update-request-processors.adoc b/solr/solr-ref-guide/modules/configuration-guide/pages/update-request-processors.adoc index ee160af4136..8ac9faf031f 100644 --- a/solr/solr-ref-guide/modules/configuration-guide/pages/update-request-processors.adoc +++ b/solr/solr-ref-guide/modules/configuration-guide/pages/update-request-processors.adoc @@ -337,6 +337,12 @@ Documents processed prior to the offender are indexed by Solr; documents followi + Alternatively, the processor offers a "permissive" mode (`permissiveMode=true`) which skips the offending document and logs a warning, but doesn't abort the remainder of the batch or return an error to users. +{solr-javadocs}/core/org/apache/solr/update/processor/NumFieldLimitingUpdateRequestProcessorFactory.html[NumFieldLimitingUpdateRequestProcessorFactory]:: Fails update requests once a core has exceeded a configurable "maximum" number of fields. ++ +Solr performance can degrade and even become unstable if cores accumulate too many (e.g. more than 500) fields. The "NumFieldLimiting" URP is offered as a safeguard that helps users notice potentially-dangerous schema design and/or mis-use of dynamic fields, before these performance and stability problems would manifest. +Note that the field count an index reports can be influenced by deleted (but not yet purged) documents, and may vary from replica to replica. +In order to avoid these sort of discrepancies between replicas, use of this URP should almost always precede DistributedUpdateProcessor in when running in SolrCloud mode. + {solr-javadocs}/core/org/apache/solr/update/processor/RegexpBoostProcessorFactory.html[RegexpBoostProcessorFactory]:: A processor which will match content of "inputField" against regular expressions found in "boostFilename", and if it matches will return the corresponding boost value from the file and output this to "boostField" as a double value. {solr-javadocs}/core/org/apache/solr/update/processor/SignatureUpdateProcessorFactory.html[SignatureUpdateProcessorFactory]:: Uses a defined set of fields to generate a hash "signature" for the document. diff --git a/solr/solr-ref-guide/modules/deployment-guide/pages/basic-authentication-plugin.adoc b/solr/solr-ref-guide/modules/deployment-guide/pages/basic-authentication-plugin.adoc index bfc998021ab..e878e902459 100644 --- a/solr/solr-ref-guide/modules/deployment-guide/pages/basic-authentication-plugin.adoc +++ b/solr/solr-ref-guide/modules/deployment-guide/pages/basic-authentication-plugin.adoc @@ -29,7 +29,7 @@ This file and where to put it is described in detail in the section xref:authent If running in cloud mode, you can use the `bin/solr auth` command-line utility to enable security for a new installation, see: `bin/solr auth --help` for more details. For Basic authentication, `security.json` must have an `authentication` block which defines the class being used for authentication. -Usernames and passwords (Format: `base64(sha256(sha256(salt+password)) base64(salt)`) could be added when the file is created, or can be added later with the Authentication API, described below. +Usernames and passwords (Format: `base64(sha256(sha256(salt+password))) base64(salt)`) could be added when the file is created, or can be added later with the Authentication API, described below. An example `security.json` showing `authentication` and `authorization` blocks is shown below to show how authentication and authorization plugins can work together: diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Http2SolrClient.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Http2SolrClient.java index 40fd27b7c8e..e437e2f0a18 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Http2SolrClient.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Http2SolrClient.java @@ -135,7 +135,9 @@ protected Http2SolrClient(String serverBaseUrl, Builder builder) { this.httpClient = createHttpClient(builder); this.closeClient = true; } - + if (builder.listenerFactory != null) { + this.listenerFactory.addAll(builder.listenerFactory); + } updateDefaultMimeTypeForParser(); this.httpClient.setFollowRedirects(Boolean.TRUE.equals(builder.followRedirects)); @@ -147,6 +149,10 @@ public void addListenerFactory(HttpListenerFactory factory) { this.listenerFactory.add(factory); } + public List getListenerFactory() { + return listenerFactory; + } + // internal usage only HttpClient getHttpClient() { return httpClient; @@ -845,6 +851,13 @@ public static class Builder protected Long keyStoreReloadIntervalSecs; + public Http2SolrClient.Builder withListenerFactory(List listenerFactory) { + this.listenerFactory = listenerFactory; + return this; + } + + private List listenerFactory; + public Builder() { super(); } diff --git a/solr/solrj/src/java/org/apache/solr/common/util/StrUtils.java b/solr/solrj/src/java/org/apache/solr/common/util/StrUtils.java index 6c209770e42..e2992204856 100644 --- a/solr/solrj/src/java/org/apache/solr/common/util/StrUtils.java +++ b/solr/solrj/src/java/org/apache/solr/common/util/StrUtils.java @@ -388,4 +388,19 @@ public static String stringFromReader(Reader inReader) throws IOException { return stringWriter.toString(); } } + + @SuppressWarnings("ReferenceEquality") + public static boolean equalsIgnoreCase(String left, String right) { + if (left == right) { + return true; + } + if (left == null || right == null) { + return false; + } + if (left.length() != right.length()) { + return false; + } + + return left.equalsIgnoreCase(right); + } } diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/Http2SolrClientTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/Http2SolrClientTest.java index a8831fd4842..b6642955c4a 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/Http2SolrClientTest.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/Http2SolrClientTest.java @@ -587,5 +587,27 @@ public void testBuilder() { } } + @Test + public void testIdleTimeoutWithHttpClient() { + try (Http2SolrClient oldClient = + new Http2SolrClient.Builder("baseSolrUrl") + .withIdleTimeout(5000, TimeUnit.MILLISECONDS) + .build()) { + try (Http2SolrClient onlyBaseUrlChangedClient = + new Http2SolrClient.Builder("newBaseSolrUrl").withHttpClient(oldClient).build()) { + assertEquals(oldClient.getIdleTimeout(), onlyBaseUrlChangedClient.getIdleTimeout()); + assertEquals(oldClient.getHttpClient(), onlyBaseUrlChangedClient.getHttpClient()); + } + try (Http2SolrClient idleTimeoutChangedClient = + new Http2SolrClient.Builder("baseSolrUrl") + .withHttpClient(oldClient) + .withIdleTimeout(3000, TimeUnit.MILLISECONDS) + .build()) { + assertFalse(oldClient.getIdleTimeout() == idleTimeoutChangedClient.getIdleTimeout()); + assertEquals(3000, idleTimeoutChangedClient.getIdleTimeout()); + } + } + } + /* Missed tests : - set cookies via interceptor - invariant params - compression */ }