From de7d91e5a17346d8305ec6985bd7a4a77781dd33 Mon Sep 17 00:00:00 2001 From: Matt Pearce Date: Sat, 31 Oct 2020 11:55:10 +0000 Subject: [PATCH 1/5] #91: Implement platform availability check in search platform. --- .../search/ArgConstructorSearchPlatform.java | 5 + .../sease/rre/search/api/SearchPlatform.java | 13 +- .../search/api/SearchPlatformException.java | 40 ++ .../pom.xml | 12 + .../rre/search/api/impl/Elasticsearch.java | 7 +- .../api/impl/ExternalElasticsearch.java | 28 +- .../search/api/impl/ElasticsearchTest.java | 53 ++- .../impl/ExternalElasticsearchIndexTest.java | 80 ++++ .../configuration_sets/v1.0/index-shape.json | 29 ++ .../corpora/electric_basses.bulk | 8 + .../search/api/impl/ExternalApacheSolr.java | 29 +- .../sease/rre/search/api/impl/ApacheSolr.java | 450 +++++++++--------- .../rre/search/api/impl/ApacheSolrTest.java | 25 + .../v1.0/core1/conf/schema.xml | 37 ++ .../v1.0/core1/conf/solrconfig.xml | 66 +++ .../configuration_sets/v1.0/solr.xml | 20 + .../resources/corpora/electric_basses.json | 22 + 17 files changed, 701 insertions(+), 223 deletions(-) create mode 100644 rre-search-platform/rre-search-platform-api/src/main/java/io/sease/rre/search/api/SearchPlatformException.java create mode 100644 rre-search-platform/rre-search-platform-elastic-search-impl/src/test/java/io/sease/rre/search/api/impl/ExternalElasticsearchIndexTest.java create mode 100644 rre-search-platform/rre-search-platform-elastic-search-impl/src/test/resources/elasticsearch/configuration_sets/v1.0/index-shape.json create mode 100644 rre-search-platform/rre-search-platform-elastic-search-impl/src/test/resources/elasticsearch/corpora/electric_basses.bulk create mode 100644 rre-search-platform/rre-search-platform-solr-impl/src/test/resources/configuration_sets/v1.0/core1/conf/schema.xml create mode 100644 rre-search-platform/rre-search-platform-solr-impl/src/test/resources/configuration_sets/v1.0/core1/conf/solrconfig.xml create mode 100644 rre-search-platform/rre-search-platform-solr-impl/src/test/resources/configuration_sets/v1.0/solr.xml create mode 100644 rre-search-platform/rre-search-platform-solr-impl/src/test/resources/corpora/electric_basses.json diff --git a/rre-maven-plugin/rre-maven-generic-search-plugin/src/test/java/io/sease/rre/maven/plugin/search/ArgConstructorSearchPlatform.java b/rre-maven-plugin/rre-maven-generic-search-plugin/src/test/java/io/sease/rre/maven/plugin/search/ArgConstructorSearchPlatform.java index 801419b2..d6f0a58e 100644 --- a/rre-maven-plugin/rre-maven-generic-search-plugin/src/test/java/io/sease/rre/maven/plugin/search/ArgConstructorSearchPlatform.java +++ b/rre-maven-plugin/rre-maven-generic-search-plugin/src/test/java/io/sease/rre/maven/plugin/search/ArgConstructorSearchPlatform.java @@ -70,4 +70,9 @@ public boolean isCorporaRequired() { @Override public void close() { } + + @Override + public boolean checkCollection(String collection, String version) { + return true; + } } diff --git a/rre-search-platform/rre-search-platform-api/src/main/java/io/sease/rre/search/api/SearchPlatform.java b/rre-search-platform/rre-search-platform-api/src/main/java/io/sease/rre/search/api/SearchPlatform.java index 9a859812..e59d1963 100644 --- a/rre-search-platform/rre-search-platform-api/src/main/java/io/sease/rre/search/api/SearchPlatform.java +++ b/rre-search-platform/rre-search-platform-api/src/main/java/io/sease/rre/search/api/SearchPlatform.java @@ -57,7 +57,7 @@ public interface SearchPlatform extends Closeable { default String getFullyQualifiedDomainName(final String indexName, final String version) { return (indexName + "_" + version).toLowerCase(); } - + /** * Starts this search platform. */ @@ -119,4 +119,15 @@ default String getFullyQualifiedDomainName(final String indexName, final String * loaded in order to run. */ boolean isCorporaRequired(); + + /** + * Check whether the collection is available on this search platform. + * Expects the platform to have been initialised (ie. started, data + * loaded, etc.) + * + * @param collection the name of the collection to check for. + * @param version the version of the collection to check for. + * @return {@code true} if this collection can be reached. + */ + boolean checkCollection(String collection, String version); } diff --git a/rre-search-platform/rre-search-platform-api/src/main/java/io/sease/rre/search/api/SearchPlatformException.java b/rre-search-platform/rre-search-platform-api/src/main/java/io/sease/rre/search/api/SearchPlatformException.java new file mode 100644 index 00000000..9e132a17 --- /dev/null +++ b/rre-search-platform/rre-search-platform-api/src/main/java/io/sease/rre/search/api/SearchPlatformException.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.sease.rre.search.api; + +/** + * Exception class referencing a specific issue with a search platform + * implementation. + * + * @author Matt Pearce (matt@elysiansoftware.co.uk) + */ +public class SearchPlatformException extends Exception { + public SearchPlatformException() { + } + + public SearchPlatformException(String s) { + super(s); + } + + public SearchPlatformException(String s, Throwable throwable) { + super(s, throwable); + } + + public SearchPlatformException(Throwable throwable) { + super(throwable); + } +} diff --git a/rre-search-platform/rre-search-platform-elastic-search-impl/pom.xml b/rre-search-platform/rre-search-platform-elastic-search-impl/pom.xml index 8dcbdcc1..91426a57 100644 --- a/rre-search-platform/rre-search-platform-elastic-search-impl/pom.xml +++ b/rre-search-platform/rre-search-platform-elastic-search-impl/pom.xml @@ -71,5 +71,17 @@ analysis-common ${project.version} + + org.mock-server + mockserver-junit-rule + 5.11.1 + test + + + org.apache.logging.log4j + log4j-core + 2.10.0 + test + \ No newline at end of file diff --git a/rre-search-platform/rre-search-platform-elastic-search-impl/src/main/java/io/sease/rre/search/api/impl/Elasticsearch.java b/rre-search-platform/rre-search-platform-elastic-search-impl/src/main/java/io/sease/rre/search/api/impl/Elasticsearch.java index c38e69bb..35674e32 100644 --- a/rre-search-platform/rre-search-platform-elastic-search-impl/src/main/java/io/sease/rre/search/api/impl/Elasticsearch.java +++ b/rre-search-platform/rre-search-platform-elastic-search-impl/src/main/java/io/sease/rre/search/api/impl/Elasticsearch.java @@ -110,7 +110,7 @@ public void beforeStart(final Map configuration) { } nodeConfigFolder = new File((String) configuration.get("path.home"), "config"); - nodeConfigFolder.mkdir(); + nodeConfigFolder.mkdirs(); final Settings.Builder settings = Settings.builder() .put("path.home", (String) configuration.get("path.home")) @@ -333,4 +333,9 @@ public boolean isSearchPlatformConfiguration(String indexName, File file) { public boolean isCorporaRequired() { return true; } + + @Override + public boolean checkCollection(String collection, String version) { + return proxy.admin().indices().exists(indicesExistsRequest(getFullyQualifiedDomainName(collection, version))).actionGet().isExists(); + } } diff --git a/rre-search-platform/rre-search-platform-elastic-search-impl/src/main/java/io/sease/rre/search/api/impl/ExternalElasticsearch.java b/rre-search-platform/rre-search-platform-elastic-search-impl/src/main/java/io/sease/rre/search/api/impl/ExternalElasticsearch.java index ea3b0be9..350ab3b0 100644 --- a/rre-search-platform/rre-search-platform-elastic-search-impl/src/main/java/io/sease/rre/search/api/impl/ExternalElasticsearch.java +++ b/rre-search-platform/rre-search-platform-elastic-search-impl/src/main/java/io/sease/rre/search/api/impl/ExternalElasticsearch.java @@ -34,6 +34,7 @@ import org.elasticsearch.client.RestClient; import org.elasticsearch.client.RestClientBuilder; import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.client.indices.GetIndexRequest; import java.io.File; import java.io.IOException; @@ -78,12 +79,17 @@ public void load(File dataToBeIndexed, File settingsFile, String collection, Str if (indexClients.get(version) == null) { indexClients.put(version, initialiseClient(settings.getHostUrls(), settings.getUser(), settings.getPassword())); } - } catch (IOException e) { LOGGER.error("Could not read settings from " + settingsFile.getName() + " :: " + e.getMessage()); } } + void setSettings(IndexSettings settings, String version) { + if (indexClients.get(version) == null) { + indexClients.put(version, initialiseClient(settings.getHostUrls(), settings.getUser(), settings.getPassword())); + } + } + private RestHighLevelClient initialiseClient(List hosts, String user, String password) { // Convert hosts to HTTP host objects HttpHost[] httpHosts = hosts.stream() @@ -155,6 +161,16 @@ private void closeClient(RestHighLevelClient client) { } } + @Override + public boolean checkCollection(String collection, String version) { + try { + return indexClients.get(version).indices().exists(new GetIndexRequest(collection), RequestOptions.DEFAULT); + } catch (IOException e) { + LOGGER.error("Caught IOException checking collection {} version {}: {}", collection, version, e.getMessage()); + LOGGER.error(e); + return false; + } + } public static class IndexSettings { @JsonProperty("hostUrls") @@ -174,16 +190,20 @@ public IndexSettings(@JsonProperty("hostUrls") List hostUrls, this.password = password; } - List getHostUrls() { + public List getHostUrls() { return hostUrls; } - String getUser() { + public String getUser() { return user; } - String getPassword() { + public String getPassword() { return password; } } + + void addClient(String version, RestHighLevelClient client) { + indexClients.put(version, client); + } } diff --git a/rre-search-platform/rre-search-platform-elastic-search-impl/src/test/java/io/sease/rre/search/api/impl/ElasticsearchTest.java b/rre-search-platform/rre-search-platform-elastic-search-impl/src/test/java/io/sease/rre/search/api/impl/ElasticsearchTest.java index 71e7732a..6efdd5fd 100644 --- a/rre-search-platform/rre-search-platform-elastic-search-impl/src/test/java/io/sease/rre/search/api/impl/ElasticsearchTest.java +++ b/rre-search-platform/rre-search-platform-elastic-search-impl/src/test/java/io/sease/rre/search/api/impl/ElasticsearchTest.java @@ -17,12 +17,17 @@ package io.sease.rre.search.api.impl; import io.sease.rre.search.api.SearchPlatform; +import org.junit.After; import org.junit.Before; +import org.junit.BeforeClass; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; import java.io.File; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -30,17 +35,30 @@ public class ElasticsearchTest { private static final String INDEX_NAME = "test"; + private static final String VERSION = "1.0"; @Rule public TemporaryFolder tempFolder = new TemporaryFolder(); - private SearchPlatform platform; + private static SearchPlatform platform; + + @BeforeClass + public static void configureEsNettySettings() { + // This is a set-once property - when running the tests through Maven, + // it gets set multiple times unless we disable that behaviour here + System.setProperty("es.set.netty.runtime.available.processors", "false"); + } @Before public void setupPlatform() { platform = new Elasticsearch(); } + @After + public void tearDownPlatform() { + platform = null; + } + @Test public void isSearchPlatformFile_returnsFalseWhenDirectory() throws Exception { File dummyFile = tempFolder.newFolder(); @@ -58,4 +76,37 @@ public void isSearchPlatformFile_returnsTrueWhenDirectoryContainsConfig() throws File configFile = tempFolder.newFile("index-shape.json"); assertTrue(platform.isSearchPlatformConfiguration(INDEX_NAME, configFile)); } + + + @Test + public void checkCollection_returnsFalseWhenNotLoaded() throws Exception { + Map configuration = buildConfiguration(); + platform.beforeStart(configuration); + platform.start(); + assertFalse(platform.checkCollection(INDEX_NAME, VERSION)); + platform.close(); + } + + @Test + public void checkCollection_returnsTrueWhenInitialised() throws Exception { + Map configuration = buildConfiguration(); + platform.beforeStart(configuration); + platform.start(); + platform.load( + new File(this.getClass().getResource("/elasticsearch/corpora/electric_basses.bulk").getPath()), + new File(this.getClass().getResource("/elasticsearch/configuration_sets/v1.0/index-shape.json").getPath()), + INDEX_NAME, VERSION); + assertTrue(platform.checkCollection(INDEX_NAME, VERSION)); + platform.close(); + } + + private Map buildConfiguration() throws IOException { + Map configuration = new HashMap<>(); + File homeFolder = tempFolder.newFolder(); + File dataFolder = tempFolder.newFolder(); + configuration.put("path.home", homeFolder.getAbsolutePath()); + configuration.put("path.data", dataFolder.getAbsolutePath()); + configuration.put("forceRefresh", Boolean.FALSE); + return configuration; + } } diff --git a/rre-search-platform/rre-search-platform-elastic-search-impl/src/test/java/io/sease/rre/search/api/impl/ExternalElasticsearchIndexTest.java b/rre-search-platform/rre-search-platform-elastic-search-impl/src/test/java/io/sease/rre/search/api/impl/ExternalElasticsearchIndexTest.java new file mode 100644 index 00000000..c0cf0326 --- /dev/null +++ b/rre-search-platform/rre-search-platform-elastic-search-impl/src/test/java/io/sease/rre/search/api/impl/ExternalElasticsearchIndexTest.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.sease.rre.search.api.impl; + +import io.sease.rre.search.api.SearchPlatform; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.mockserver.client.MockServerClient; +import org.mockserver.junit.MockServerRule; + +import java.util.Collections; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockserver.model.HttpRequest.request; +import static org.mockserver.model.HttpResponse.response; + + +/** + * An integration test for checking ExternalElasticsearch methods that rely on + * an existing Elasticsearch cluster. + * + * @author Matt Pearce (matt@elysiansoftware.co.uk) + */ +public class ExternalElasticsearchIndexTest { + + private static final String INDEX_NAME = "test"; + private static final String VERSION = "1.0"; + + private static final String ES_DOCKER_IMAGE = "docker.elastic.co/elasticsearch/elasticsearch"; + private static final String ES_VERSION = "7.5.0"; + + @Rule + public TemporaryFolder tempFolder = new TemporaryFolder(); + @Rule + public MockServerRule mockServerRule = new MockServerRule(this); + + private MockServerClient mockServerClient; + + private SearchPlatform platform; + + @Before + public void setupPlatform() throws Exception { + platform = new ExternalElasticsearch(); + // Setting these explicitly, rather than using load() + ((ExternalElasticsearch) platform).setSettings( + new ExternalElasticsearch.IndexSettings(Collections.singletonList("http://localhost:" + mockServerClient.getPort()), null, null), + VERSION); + } + + @Test + public void checkCollection_returnsFalseWhenNotLoaded() throws Exception { + mockServerClient.when(request().withMethod("HEAD").withPath("/" + INDEX_NAME)) + .respond(response().withStatusCode(404)); + assertFalse(platform.checkCollection(INDEX_NAME, VERSION)); + } + + @Test + public void checkCollection_returnsTrueWhenAvailable() throws Exception { + mockServerClient.when(request().withMethod("HEAD").withPath("/" + INDEX_NAME)) + .respond(response().withStatusCode(200)); + assertTrue(platform.checkCollection(INDEX_NAME, VERSION)); + } +} diff --git a/rre-search-platform/rre-search-platform-elastic-search-impl/src/test/resources/elasticsearch/configuration_sets/v1.0/index-shape.json b/rre-search-platform/rre-search-platform-elastic-search-impl/src/test/resources/elasticsearch/configuration_sets/v1.0/index-shape.json new file mode 100644 index 00000000..5a96358a --- /dev/null +++ b/rre-search-platform/rre-search-platform-elastic-search-impl/src/test/resources/elasticsearch/configuration_sets/v1.0/index-shape.json @@ -0,0 +1,29 @@ +{ + "settings": { + "number_of_shards": 1, + "number_of_replicas": 0, + "analysis": { + "analyzer": { + "raw_text": { + "tokenizer": "standard", + "filter": [ + "lowercase" + ] + } + } + } + }, + "mappings": { + "doc": { + "properties": { + "name": { + "type": "text", + "analyzer": "raw_text" + }, + "number_of_strings": { + "type": "integer" + } + } + } + } +} diff --git a/rre-search-platform/rre-search-platform-elastic-search-impl/src/test/resources/elasticsearch/corpora/electric_basses.bulk b/rre-search-platform/rre-search-platform-elastic-search-impl/src/test/resources/elasticsearch/corpora/electric_basses.bulk new file mode 100644 index 00000000..6fa05410 --- /dev/null +++ b/rre-search-platform/rre-search-platform-elastic-search-impl/src/test/resources/elasticsearch/corpora/electric_basses.bulk @@ -0,0 +1,8 @@ +{"index" : {"_index":"dataset1","_type":"doc","_id":"1"}} +{"name":"Fender Jazz Bass", "number_of_strings": 4} +{"index" : {"_index":"dataset1","_type":"doc","_id":"2"}} +{"name":"Fender Precision", "number_of_strings": 4} +{"index" : {"_index":"dataset1","_type":"doc","_id":"3"}} +{"name":"Warwick Corvette Bass", "number_of_strings": 5} +{"index" : {"_index":"dataset1","_type":"doc","_id":"4"}} +{"name":"Warwick Thumb", "number_of_strings": 6} \ No newline at end of file diff --git a/rre-search-platform/rre-search-platform-external-solr-impl/src/main/java/io/sease/rre/search/api/impl/ExternalApacheSolr.java b/rre-search-platform/rre-search-platform-external-solr-impl/src/main/java/io/sease/rre/search/api/impl/ExternalApacheSolr.java index ea15e070..ed9e58b0 100644 --- a/rre-search-platform/rre-search-platform-external-solr-impl/src/main/java/io/sease/rre/search/api/impl/ExternalApacheSolr.java +++ b/rre-search-platform/rre-search-platform-external-solr-impl/src/main/java/io/sease/rre/search/api/impl/ExternalApacheSolr.java @@ -22,15 +22,23 @@ import com.fasterxml.jackson.databind.ObjectMapper; import io.sease.rre.search.api.QueryOrSearchResponse; import io.sease.rre.search.api.SearchPlatform; +import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.client.solrj.SolrRequest; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.common.SolrException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; import java.io.IOException; -import java.util.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Optional; import static java.util.Optional.of; @@ -154,6 +162,25 @@ public void close() { clientManager.close(); } + @Override + public boolean checkCollection(String collection, String version) { + try { + SolrQuery query = new SolrQuery("*:*").setRows(0); + SolrClient client = clientManager.getSolrClient(version); + if (client != null) { + QueryResponse response = client.query(collection, query); + return response.getStatus() == 0; + } + } catch (SolrException e) { + // If index doesn't exist, we'll get a SolrException (a RuntimeException) + LOGGER.warn("Caught SolrException checking for collection {} version {}: {}", + collection, version, e.getMessage()); + } catch (SolrServerException | IOException e) { + LOGGER.warn("Caught exception checking platform for collection {} version {}: {}", + collection, version, e.getMessage()); + } + return false; + } public static class SolrSettings { diff --git a/rre-search-platform/rre-search-platform-solr-impl/src/main/java/io/sease/rre/search/api/impl/ApacheSolr.java b/rre-search-platform/rre-search-platform-solr-impl/src/main/java/io/sease/rre/search/api/impl/ApacheSolr.java index fc85bfa8..f447a278 100644 --- a/rre-search-platform/rre-search-platform-solr-impl/src/main/java/io/sease/rre/search/api/impl/ApacheSolr.java +++ b/rre-search-platform/rre-search-platform-solr-impl/src/main/java/io/sease/rre/search/api/impl/ApacheSolr.java @@ -24,7 +24,9 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.solr.client.solrj.SolrQuery; +import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.embedded.EmbeddedSolrServer; +import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.client.solrj.response.UpdateResponse; import org.apache.solr.common.SolrException; import org.apache.solr.core.CoreContainer; @@ -52,219 +54,237 @@ * @since 1.0 */ public class ApacheSolr implements SearchPlatform { - private final static Logger LOGGER = LogManager.getLogger(ApacheSolr.class); - - private EmbeddedSolrServer proxy; - private File solrHome; - private File coreProperties; - private File renamedCoreProperties; - - private boolean refreshRequired = false; - private boolean defaultSolrHome = false; - - @Override - public void beforeStart(final Map configuration) { - if (configuration.containsKey("solr.home")) { - // Use external, configured Solr home directory - solrHome = new File((String) configuration.get("solr.home")); - } else { - // Use tmp directory (will be deleted after processing) - solrHome = new File(System.getProperty("java.io.tmpdir"), String.valueOf(System.currentTimeMillis())); - defaultSolrHome = true; - } - - if ((Boolean) configuration.get("forceRefresh") && solrHome.exists()) { - try { - DirectoryUtils.deleteDirectory(solrHome); - } catch (IOException e) { - LOGGER.error("Could not delete data directory - expect data to be stale!", e); - } - } - if (!solrHome.exists()) { - // If no data directory, refresh is required even if nothing else has changed - prepareSolrHome(solrHome); - refreshRequired = true; - } - - File dataDir = new File(solrHome, "data"); - dataDir.mkdirs(); - - System.setProperty("solr.data.dir", dataDir.getAbsolutePath()); - - proxy = new EmbeddedSolrServer(solrHome.toPath(), "dummy"); - } - - @Override - public void load(final File dataToBeIndexed, final File configFolder, final String collection, String version) { - coreProperties = new File(configFolder, "core.properties"); - if (coreProperties.exists()) { - renamedCoreProperties = new File(configFolder, "core.properties.ignore"); - coreProperties.renameTo(renamedCoreProperties); - } - - // Copy files from configFolder into solrHome/targetIndexName - String coreName = getFullyQualifiedDomainName(collection, version); - File targetIndexDir = new File(solrHome, coreName); - try { - // Make sure the directory is deleted before copying to it - DirectoryUtils.deleteDirectory(targetIndexDir); - DirectoryUtils.copyDirectory(configFolder, targetIndexDir); - } catch (IOException e) { - throw new RuntimeException(e); - } - - try { - // Using absolute path for the targetIndexDir, otherwise Solr can put the core.properties in the wrong place. - proxy.getCoreContainer().create(coreName, targetIndexDir.toPath().toAbsolutePath(), emptyMap(), true); - } catch (SolrException e) { - if (e.code() == SolrException.ErrorCode.SERVER_ERROR.code) { - // Core already exists - ignore - LOGGER.debug("Core " + coreName + " already exists - skipping index creation"); - } else { - LOGGER.error("Caught Solr exception creating core :: " + e.getMessage()); - } - } - - try { - UpdateResponse response = new JsonUpdateRequest(new FileInputStream(dataToBeIndexed)).process(proxy, coreName); - if (response.getStatus() != 0) { - throw new IllegalArgumentException("Received an error status from Solr: " + response.getStatus()); - } - } catch (final Exception exception) { - throw new RuntimeException(exception); - } - } - - @Override - public void start() { - // Nothing to be done here, the embedded server doesn't need an explicit start command. - } - - @Override - public void afterStart() { - // Nothing to be done here. - } - - @Override - public void beforeStop() { - // If using the default Solr home (eg. /tmp), clear the index in - // preparation for deleting the tmp directory later. - if (defaultSolrHome) { - ofNullable(proxy).ifPresent(solr -> { - try { - solr.deleteByQuery("*:*"); - solr.commit(); - } catch (final Exception exception) { - exception.printStackTrace(); - } - }); - } - } - - @Override - public void close() { - ofNullable(proxy).ifPresent(solr -> { - try { - solr.close(); - } catch (final Exception exception) { - exception.printStackTrace(); - } - }); - - if (defaultSolrHome) { - // If using the default Solr home (eg. /tmp), unload and set - // directory for deletion. - ofNullable(proxy) - .map(EmbeddedSolrServer::getCoreContainer) - .map(CoreContainer::getAllCoreNames) - .orElse(Collections.emptyList()) - .forEach(coreName -> - proxy.getCoreContainer().unload(coreName, true, true, false)); - solrHome.deleteOnExit(); - } - - ofNullable(renamedCoreProperties).ifPresent(file -> file.renameTo(coreProperties)); - } - - @Override - public QueryOrSearchResponse executeQuery(final String collection, final String version, final String queryString, final String[] fields, final int maxRows) { - String coreName = getFullyQualifiedDomainName(collection, version); - try { - final SolrQuery query = - new SolrQuery() - .setRows(maxRows) - .setFields(fields); - final ObjectMapper mapper = new ObjectMapper(); - final JsonNode queryDef = mapper.readTree(queryString); - - for (final Iterator> iterator = queryDef.fields(); iterator.hasNext(); ) { - final Map.Entry field = iterator.next(); - final String value; - if (field.getValue().isValueNode()) { - value = field.getValue().asText(); - } else { - // Either an array or an object - use writeValueAsString() instead - // to convert to a string. Useful for writing JSON queries without escaping them. - value = mapper.writeValueAsString(field.getValue()); - } - query.add(field.getKey(), value); - } - - return of(proxy.query(coreName, query)) - .map(response -> - new QueryOrSearchResponse( - response.getResults().getNumFound(), - new ArrayList>(response.getResults()))) - .get(); - } catch (SolrException e) { - LOGGER.error("Caught Solr exception :: " + e.getMessage()); - return new QueryOrSearchResponse(0, Collections.emptyList()); - } catch (final Exception exception) { - throw new RuntimeException(exception); - } - } - - @Override - public String getName() { - return "Apache Solr"; - } - - @Override - public boolean isRefreshRequired() { - return refreshRequired; - } - - /** - * Setup the Solr instance by preparing a minimal solr.home directory. - * - * @param folder the folder where the temporary solr.home will be created. - */ - private void prepareSolrHome(final File folder) { - folder.mkdirs(); - try (final BufferedWriter writer = new BufferedWriter(new FileWriter(new File(folder, "solr.xml")))) { - writer.write(""); - - final File dummyCoreHome = new File(folder, "dummy"); - final File dummyCoreConf = new File(dummyCoreHome, "conf"); - dummyCoreConf.mkdirs(); - - Files.copy(getClass().getResourceAsStream("/schema.xml"), new File(dummyCoreConf, "schema.xml").toPath(), StandardCopyOption.REPLACE_EXISTING); - Files.copy(getClass().getResourceAsStream("/solrconfig.xml"), new File(dummyCoreConf, "solrconfig.xml").toPath(), StandardCopyOption.REPLACE_EXISTING); - Files.copy(getClass().getResourceAsStream("/core.properties"), new File(dummyCoreHome, "core.properties").toPath(), StandardCopyOption.REPLACE_EXISTING); - - } catch (final Exception exception) { - folder.deleteOnExit(); - throw new RuntimeException(exception); - } - } - - @Override - public boolean isSearchPlatformConfiguration(String indexName, File file) { - return file.isDirectory() && file.getName().equals(indexName); - } - - @Override - public boolean isCorporaRequired() { - return true; - } + private final static Logger LOGGER = LogManager.getLogger(ApacheSolr.class); + + private EmbeddedSolrServer proxy; + private File solrHome; + private File coreProperties; + private File renamedCoreProperties; + + private boolean refreshRequired = false; + private boolean defaultSolrHome = false; + + @Override + public void beforeStart(final Map configuration) { + if (configuration.containsKey("solr.home")) { + // Use external, configured Solr home directory + solrHome = new File((String) configuration.get("solr.home")); + } else { + // Use tmp directory (will be deleted after processing) + solrHome = new File(System.getProperty("java.io.tmpdir"), String.valueOf(System.currentTimeMillis())); + defaultSolrHome = true; + } + + if ((Boolean) configuration.get("forceRefresh") && solrHome.exists()) { + try { + DirectoryUtils.deleteDirectory(solrHome); + } catch (IOException e) { + LOGGER.error("Could not delete data directory - expect data to be stale!", e); + } + } + if (!solrHome.exists()) { + // If no data directory, refresh is required even if nothing else has changed + prepareSolrHome(solrHome); + refreshRequired = true; + } + + File dataDir = new File(solrHome, "data"); + dataDir.mkdirs(); + + System.setProperty("solr.data.dir", dataDir.getAbsolutePath()); + + proxy = new EmbeddedSolrServer(solrHome.toPath(), "dummy"); + } + + @Override + public void load(final File dataToBeIndexed, final File configFolder, final String collection, String version) { + coreProperties = new File(configFolder, "core.properties"); + if (coreProperties.exists()) { + renamedCoreProperties = new File(configFolder, "core.properties.ignore"); + coreProperties.renameTo(renamedCoreProperties); + } + + // Copy files from configFolder into solrHome/targetIndexName + String coreName = getFullyQualifiedDomainName(collection, version); + File targetIndexDir = new File(solrHome, coreName); + try { + // Make sure the directory is deleted before copying to it + DirectoryUtils.deleteDirectory(targetIndexDir); + DirectoryUtils.copyDirectory(configFolder, targetIndexDir); + } catch (IOException e) { + throw new RuntimeException(e); + } + + try { + // Using absolute path for the targetIndexDir, otherwise Solr can put the core.properties in the wrong place. + proxy.getCoreContainer().create(coreName, targetIndexDir.toPath().toAbsolutePath(), emptyMap(), true); + } catch (SolrException e) { + if (e.code() == SolrException.ErrorCode.SERVER_ERROR.code) { + // Core already exists - ignore + LOGGER.debug("Core " + coreName + " already exists - skipping index creation"); + } else { + LOGGER.error("Caught Solr exception creating core :: " + e.getMessage()); + } + } + + try { + UpdateResponse response = new JsonUpdateRequest(new FileInputStream(dataToBeIndexed)).process(proxy, coreName); + if (response.getStatus() != 0) { + throw new IllegalArgumentException("Received an error status from Solr: " + response.getStatus()); + } + } catch (final Exception exception) { + throw new RuntimeException(exception); + } + } + + @Override + public void start() { + // Nothing to be done here, the embedded server doesn't need an explicit start command. + } + + @Override + public void afterStart() { + // Nothing to be done here. + } + + @Override + public void beforeStop() { + // If using the default Solr home (eg. /tmp), clear the index in + // preparation for deleting the tmp directory later. + if (defaultSolrHome) { + ofNullable(proxy).ifPresent(solr -> { + try { + solr.deleteByQuery("*:*"); + solr.commit(); + } catch (final Exception exception) { + exception.printStackTrace(); + } + }); + } + } + + @Override + public void close() { + ofNullable(proxy).ifPresent(solr -> { + try { + solr.close(); + } catch (final Exception exception) { + exception.printStackTrace(); + } + }); + + if (defaultSolrHome) { + // If using the default Solr home (eg. /tmp), unload and set + // directory for deletion. + ofNullable(proxy) + .map(EmbeddedSolrServer::getCoreContainer) + .map(CoreContainer::getAllCoreNames) + .orElse(Collections.emptyList()) + .forEach(coreName -> + proxy.getCoreContainer().unload(coreName, true, true, false)); + solrHome.deleteOnExit(); + } + + ofNullable(renamedCoreProperties).ifPresent(file -> file.renameTo(coreProperties)); + } + + @Override + public QueryOrSearchResponse executeQuery(final String collection, final String version, final String queryString, final String[] fields, final int maxRows) { + String coreName = getFullyQualifiedDomainName(collection, version); + try { + final SolrQuery query = + new SolrQuery() + .setRows(maxRows) + .setFields(fields); + final ObjectMapper mapper = new ObjectMapper(); + final JsonNode queryDef = mapper.readTree(queryString); + + for (final Iterator> iterator = queryDef.fields(); iterator.hasNext(); ) { + final Map.Entry field = iterator.next(); + final String value; + if (field.getValue().isValueNode()) { + value = field.getValue().asText(); + } else { + // Either an array or an object - use writeValueAsString() instead + // to convert to a string. Useful for writing JSON queries without escaping them. + value = mapper.writeValueAsString(field.getValue()); + } + query.add(field.getKey(), value); + } + + return of(proxy.query(coreName, query)) + .map(response -> + new QueryOrSearchResponse( + response.getResults().getNumFound(), + new ArrayList>(response.getResults()))) + .get(); + } catch (SolrException e) { + LOGGER.error("Caught Solr exception :: " + e.getMessage()); + return new QueryOrSearchResponse(0, Collections.emptyList()); + } catch (final Exception exception) { + throw new RuntimeException(exception); + } + } + + @Override + public String getName() { + return "Apache Solr"; + } + + @Override + public boolean isRefreshRequired() { + return refreshRequired; + } + + /** + * Setup the Solr instance by preparing a minimal solr.home directory. + * + * @param folder the folder where the temporary solr.home will be created. + */ + private void prepareSolrHome(final File folder) { + folder.mkdirs(); + try (final BufferedWriter writer = new BufferedWriter(new FileWriter(new File(folder, "solr.xml")))) { + writer.write(""); + + final File dummyCoreHome = new File(folder, "dummy"); + final File dummyCoreConf = new File(dummyCoreHome, "conf"); + dummyCoreConf.mkdirs(); + + Files.copy(getClass().getResourceAsStream("/schema.xml"), new File(dummyCoreConf, "schema.xml").toPath(), StandardCopyOption.REPLACE_EXISTING); + Files.copy(getClass().getResourceAsStream("/solrconfig.xml"), new File(dummyCoreConf, "solrconfig.xml").toPath(), StandardCopyOption.REPLACE_EXISTING); + Files.copy(getClass().getResourceAsStream("/core.properties"), new File(dummyCoreHome, "core.properties").toPath(), StandardCopyOption.REPLACE_EXISTING); + + } catch (final Exception exception) { + folder.deleteOnExit(); + throw new RuntimeException(exception); + } + } + + @Override + public boolean isSearchPlatformConfiguration(String indexName, File file) { + return file.isDirectory() && file.getName().equals(indexName); + } + + @Override + public boolean isCorporaRequired() { + return true; + } + + @Override + public boolean checkCollection(String collection, String version) { + String coreName = getFullyQualifiedDomainName(collection, version); + try { + SolrQuery query = new SolrQuery("*:*").setRows(0); + QueryResponse response = proxy.query(coreName, query); + return response.getStatus() == 0; + } catch (SolrException e) { + // If index doesn't exist, we'll get a SolrException (a RuntimeException) + LOGGER.warn("Caught SolrException checking for collection {} version {}: {}", + collection, version, e.getMessage()); + } catch (SolrServerException | IOException e) { + LOGGER.warn("Caught exception checking platform for collection {} version {}: {}", + collection, version, e.getMessage()); + } + return false; + } } diff --git a/rre-search-platform/rre-search-platform-solr-impl/src/test/java/io/sease/rre/search/api/impl/ApacheSolrTest.java b/rre-search-platform/rre-search-platform-solr-impl/src/test/java/io/sease/rre/search/api/impl/ApacheSolrTest.java index e5dced64..30390bda 100644 --- a/rre-search-platform/rre-search-platform-solr-impl/src/test/java/io/sease/rre/search/api/impl/ApacheSolrTest.java +++ b/rre-search-platform/rre-search-platform-solr-impl/src/test/java/io/sease/rre/search/api/impl/ApacheSolrTest.java @@ -23,6 +23,8 @@ import org.junit.rules.TemporaryFolder; import java.io.File; +import java.util.HashMap; +import java.util.Map; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -57,4 +59,27 @@ public void isSearchPlatformFile_returnsTrueWhenDirectoryContainsSolrConfig() th File configFile = tempFolder.newFolder(INDEX_NAME); assertTrue(platform.isSearchPlatformConfiguration(INDEX_NAME, configFile)); } + + + @Test + public void checkPlatform_returnsFalseWhenNoIndexLoaded() throws Exception { + Map config = new HashMap<>(); + config.put("forceRefresh", Boolean.FALSE); + platform.beforeStart(config); + + assertFalse(platform.checkCollection(INDEX_NAME, "v1.0")); + } + + @Test + public void checkPlatform_returnsTrueWhenIndexLoaded() throws Exception { + Map config = new HashMap<>(); + config.put("forceRefresh", Boolean.FALSE); + platform.beforeStart(config); + + File dataFile = new File(ApacheSolrTest.class.getResource("/corpora/electric_basses.json").getPath()); + File configFolder = new File(ApacheSolrTest.class.getResource("/configuration_sets/v1.0/core1").getPath()); + platform.load(dataFile, configFolder, INDEX_NAME, "v1.0"); + + assertTrue(platform.checkCollection(INDEX_NAME, "v1.0")); + } } diff --git a/rre-search-platform/rre-search-platform-solr-impl/src/test/resources/configuration_sets/v1.0/core1/conf/schema.xml b/rre-search-platform/rre-search-platform-solr-impl/src/test/resources/configuration_sets/v1.0/core1/conf/schema.xml new file mode 100644 index 00000000..206916e8 --- /dev/null +++ b/rre-search-platform/rre-search-platform-solr-impl/src/test/resources/configuration_sets/v1.0/core1/conf/schema.xml @@ -0,0 +1,37 @@ + + + + + + + + + + + + + + + + + + + id + \ No newline at end of file diff --git a/rre-search-platform/rre-search-platform-solr-impl/src/test/resources/configuration_sets/v1.0/core1/conf/solrconfig.xml b/rre-search-platform/rre-search-platform-solr-impl/src/test/resources/configuration_sets/v1.0/core1/conf/solrconfig.xml new file mode 100644 index 00000000..6d1336d1 --- /dev/null +++ b/rre-search-platform/rre-search-platform-solr-impl/src/test/resources/configuration_sets/v1.0/core1/conf/solrconfig.xml @@ -0,0 +1,66 @@ + + + + LATEST + ${solr.data.dir:}/${solr.core.name:} + + + + + ${solr.lock.type:native} + + + + + 1024 + + + + + true + 20 + 200 + false + 2 + + + + + + + + + name + + + + *:* + + \ No newline at end of file diff --git a/rre-search-platform/rre-search-platform-solr-impl/src/test/resources/configuration_sets/v1.0/solr.xml b/rre-search-platform/rre-search-platform-solr-impl/src/test/resources/configuration_sets/v1.0/solr.xml new file mode 100644 index 00000000..9056527b --- /dev/null +++ b/rre-search-platform/rre-search-platform-solr-impl/src/test/resources/configuration_sets/v1.0/solr.xml @@ -0,0 +1,20 @@ + + + \ No newline at end of file diff --git a/rre-search-platform/rre-search-platform-solr-impl/src/test/resources/corpora/electric_basses.json b/rre-search-platform/rre-search-platform-solr-impl/src/test/resources/corpora/electric_basses.json new file mode 100644 index 00000000..9b30e197 --- /dev/null +++ b/rre-search-platform/rre-search-platform-solr-impl/src/test/resources/corpora/electric_basses.json @@ -0,0 +1,22 @@ +[ + { + "id": 1, + "name": "Fender Jazz Bass", + "number_of_strings": 4 + }, + { + "id": 2, + "name": "Fender Precision Bass", + "number_of_strings": 4 + }, + { + "id": 3, + "name": "Warwick Corvette Bass", + "number_of_strings": 5 + }, + { + "id": 4, + "name": "Warwick Thumb", + "number_of_strings": 6 + } +] \ No newline at end of file From ca5fbe89a18609a573afa6e53d76b0f50380ce9f Mon Sep 17 00:00:00 2001 From: Matt Pearce Date: Sat, 31 Oct 2020 11:55:59 +0000 Subject: [PATCH 2/5] #91: Add platform check to evaluation process. --- .../main/java/io/sease/rre/core/Engine.java | 180 ++++++++++-------- 1 file changed, 104 insertions(+), 76 deletions(-) diff --git a/rre-core/src/main/java/io/sease/rre/core/Engine.java b/rre-core/src/main/java/io/sease/rre/core/Engine.java index a369a647..a5189f8c 100644 --- a/rre-core/src/main/java/io/sease/rre/core/Engine.java +++ b/rre-core/src/main/java/io/sease/rre/core/Engine.java @@ -38,6 +38,7 @@ import io.sease.rre.persistence.PersistenceHandler; import io.sease.rre.persistence.PersistenceManager; import io.sease.rre.search.api.SearchPlatform; +import io.sease.rre.search.api.SearchPlatformException; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -47,9 +48,11 @@ import java.io.FileOutputStream; import java.io.IOException; import java.util.ArrayList; +import java.util.Collection; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.stream.Collectors; import java.util.stream.Stream; import java.util.stream.StreamSupport; import java.util.zip.ZipEntry; @@ -243,68 +246,10 @@ public Evaluation evaluate(final Map configuration) { final Evaluation evaluation = new Evaluation(); - ratings().forEach(ratingsNode -> { - LOGGER.info("RRE: Ratings Set processing starts"); - - final String indexName = - requireNonNull( - ratingsNode.get(INDEX_NAME), - "WARNING!!! \"" + INDEX_NAME + "\" attribute not found!").asText(); - final String idFieldName = - requireNonNull( - ratingsNode.get(ID_FIELD_NAME), - "WARNING!!! \"" + ID_FIELD_NAME + "\" attribute not found!") - .asText(DEFAULT_ID_FIELD_NAME); - - final Optional data = data(ratingsNode); - final String queryPlaceholder = ofNullable(ratingsNode.get("query_placeholder")).map(JsonNode::asText).orElse("$query"); - - LOGGER.info(""); - LOGGER.info("*********************************"); - LOGGER.info("RRE: Index name => " + indexName); - LOGGER.info("RRE: ID Field name => " + idFieldName); - - data.ifPresent(file -> LOGGER.info("RRE: Test Collection => " + file.getAbsolutePath())); - prepareData(indexName, data.orElse(null)); - - final Corpus corpus = evaluation.findOrCreate(data.map(File::getName).orElse(indexName), Corpus::new); - all(ratingsNode, TOPICS) - .forEach(topicNode -> { - final Topic topic = corpus.findOrCreate(name(topicNode), Topic::new); - - LOGGER.info("TOPIC: " + topic.getName()); - - all(topicNode, QUERY_GROUPS) - .forEach(groupNode -> { - final QueryGroup group = topic.findOrCreate(name(groupNode), QueryGroup::new); - - LOGGER.info("\tQUERY GROUP: " + group.getName()); - - final String sharedTemplate = ofNullable(groupNode.get("template")).map(JsonNode::asText).orElse(null); - all(groupNode, QUERIES) - .forEach(queryNode -> { - final String queryString = queryNode.findValue(queryPlaceholder).asText(); - - LOGGER.info("\t\tQUERY: " + queryString); - - final JsonNode relevantDocuments = relevantDocuments( - Optional.ofNullable(queryNode.get(RELEVANT_DOCUMENTS)) - .orElse(groupNode.get(RELEVANT_DOCUMENTS))); - final Query queryEvaluation = group.findOrCreate(queryString, Query::new); - queryEvaluation.setIdFieldName(idFieldName); - queryEvaluation.setRelevantDocuments(relevantDocuments); - - List metrics = availableMetrics(idFieldName, relevantDocuments, - new ArrayList<>(versionManager.getConfigurationVersions())); - queryEvaluation.prepare(metrics); - - evaluationManager.evaluateQuery(queryEvaluation, indexName, queryNode, sharedTemplate, - Math.max(relevantDocuments.size(), minimumRequiredResults(metrics))); - }); - }); - }); - }); + // Start the evaluation process for all of the ratings nodes + ratings().forEach(ratingsNode -> evaluateRatings(evaluation, ratingsNode)); + // Wait for the evaluations to complete while (evaluationManager.isRunning()) { LOGGER.info(" ... completed {} / {} evaluations ...", (evaluationManager.getTotalQueries() - evaluationManager.getQueriesRemaining()), @@ -314,7 +259,12 @@ public Evaluation evaluate(final Map configuration) { } catch (InterruptedException ignore) { } } - LOGGER.info(" ... completed all {} evaluations.", evaluationManager.getTotalQueries()); + + if (evaluationManager.getTotalQueries() > 0) { + LOGGER.info(" ... completed all {} evaluations.", evaluationManager.getTotalQueries()); + } else { + LOGGER.warn(" ... no queries evaluated!"); + } return evaluation; } finally { @@ -327,6 +277,80 @@ public Evaluation evaluate(final Map configuration) { } } + /** + * Evaluate a single ratings set, updating the evaluation with the results. + * + * @param evaluation the evaluation holding the query results. + * @param ratingsNode the contents of the ratings set. + */ + private void evaluateRatings(Evaluation evaluation, JsonNode ratingsNode) { + LOGGER.info("RRE: Ratings Set processing starts"); + + final String indexName = + requireNonNull( + ratingsNode.get(INDEX_NAME), + "WARNING!!! \"" + INDEX_NAME + "\" attribute not found!").asText(); + final String idFieldName = + requireNonNull( + ratingsNode.get(ID_FIELD_NAME), + "WARNING!!! \"" + ID_FIELD_NAME + "\" attribute not found!") + .asText(DEFAULT_ID_FIELD_NAME); + + final Optional data = data(ratingsNode); + final String queryPlaceholder = ofNullable(ratingsNode.get("query_placeholder")).map(JsonNode::asText).orElse("$query"); + + LOGGER.info(""); + LOGGER.info("*********************************"); + LOGGER.info("RRE: Index name => " + indexName); + LOGGER.info("RRE: ID Field name => " + idFieldName); + data.ifPresent(file -> LOGGER.info("RRE: Test Collection => " + file.getAbsolutePath())); + + try { + // Load the data. If the collection being loaded cannot be reached, + // this will fail. + prepareData(indexName, data.orElse(null)); + + final Corpus corpus = evaluation.findOrCreate(data.map(File::getName).orElse(indexName), Corpus::new); + all(ratingsNode, TOPICS) + .forEach(topicNode -> { + final Topic topic = corpus.findOrCreate(name(topicNode), Topic::new); + + LOGGER.info("TOPIC: " + topic.getName()); + + all(topicNode, QUERY_GROUPS) + .forEach(groupNode -> { + final QueryGroup group = topic.findOrCreate(name(groupNode), QueryGroup::new); + + LOGGER.info("\tQUERY GROUP: " + group.getName()); + + final String sharedTemplate = ofNullable(groupNode.get("template")).map(JsonNode::asText).orElse(null); + all(groupNode, QUERIES) + .forEach(queryNode -> { + final String queryString = queryNode.findValue(queryPlaceholder).asText(); + + LOGGER.info("\t\tQUERY: " + queryString); + + final JsonNode relevantDocuments = relevantDocuments( + Optional.ofNullable(queryNode.get(RELEVANT_DOCUMENTS)) + .orElse(groupNode.get(RELEVANT_DOCUMENTS))); + final Query queryEvaluation = group.findOrCreate(queryString, Query::new); + queryEvaluation.setIdFieldName(idFieldName); + queryEvaluation.setRelevantDocuments(relevantDocuments); + + List metrics = availableMetrics(idFieldName, relevantDocuments, + new ArrayList<>(versionManager.getConfigurationVersions())); + queryEvaluation.prepare(metrics); + + evaluationManager.evaluateQuery(queryEvaluation, indexName, queryNode, sharedTemplate, + Math.max(relevantDocuments.size(), minimumRequiredResults(metrics))); + }); + }); + }); + } catch (SearchPlatformException spe) { + LOGGER.error("SearchPlatform error while evaluating ratings: {}", spe.getMessage()); + } + } + private Optional data(final JsonNode ratingsNode) { final File retFile; @@ -480,28 +504,32 @@ private Stream all(final JsonNode source, final String name) { /** * Prepares the search platform with the given index name and dataset. * - * @param collection the index name. - * @param dataToBeIndexed the dataset. + * @param collection the index name. + * @param dataToBeIndexed the dataset. + * @throws SearchPlatformException if problems occur loading data to the + * search platform. */ - private void prepareData(final String collection, final File dataToBeIndexed) { + private void prepareData(final String collection, final File dataToBeIndexed) throws SearchPlatformException { if (dataToBeIndexed != null) { LOGGER.info("Preparing dataToBeIndexed for " + collection + " from " + dataToBeIndexed.getAbsolutePath()); } else { LOGGER.info("Preparing platform for " + collection); } - Stream searchCollectionsConfigs = versionManager.getConfigurationVersionFolders().stream() - .filter(versionFolder -> isConfigurationReloadNecessary(versionFolder)) - .flatMap(versionFolder -> stream(safe(versionFolder.listFiles(ONLY_NON_HIDDEN_FILES)))); - //each one of searchCollectionsConfigs stream elements is a full configuration for a search collection - searchCollectionsConfigs + Collection configFiles = versionManager.getConfigurationVersionFolders().stream() + .filter(this::isConfigurationReloadNecessary) + .flatMap(versionFolder -> stream(safe(versionFolder.listFiles(ONLY_NON_HIDDEN_FILES)))) .filter(file -> platform.isSearchPlatformConfiguration(collection, file)) .sorted() - .peek(searchPlatformConfiguration -> LOGGER.info("RRE: Loading the Search Engine " + platform.getName() + ", configuration version " + searchPlatformConfiguration.getParentFile().getName())) - .forEach(searchPlatformConfiguration -> { - String version = searchPlatformConfiguration.getParentFile().getName(); - platform.load(dataToBeIndexed, searchPlatformConfiguration, collection, version); - }); + .collect(Collectors.toList()); + for (File searchPlatformConfiguration : configFiles) { + LOGGER.info("RRE: Loading the Search Engine " + platform.getName() + ", configuration version " + searchPlatformConfiguration.getParentFile().getName()); + String version = searchPlatformConfiguration.getParentFile().getName(); + platform.load(dataToBeIndexed, searchPlatformConfiguration, collection, version); + if (!platform.checkCollection(collection, version)) { + throw new SearchPlatformException("Collection check failed for " + collection + " version " + version); + } + } LOGGER.info("RRE: " + platform.getName() + " has been correctly loaded."); @@ -510,7 +538,7 @@ private void prepareData(final String collection, final File dataToBeIndexed) { LOGGER.info("RRE: target versions are " + String.join(",", versionManager.getConfigurationVersions())); } - private boolean isConfigurationReloadNecessary( File versionFolder) { + private boolean isConfigurationReloadNecessary(File versionFolder) { boolean corporaChanged = folderHasChanged(corporaFolder); return folderHasChanged(versionFolder) || corporaChanged || platform.isRefreshRequired(); } From 5f55cbba830317b804c9c2dc94d689d695735e9a Mon Sep 17 00:00:00 2001 From: Matt Pearce Date: Sun, 1 Nov 2020 11:55:53 +0000 Subject: [PATCH 3/5] Add integration tests for the checkPlatform method. --- .../pom.xml | 51 ++++++++++ .../search/api/impl/ExternalApacheSolrIT.java | 95 +++++++++++++++++++ 2 files changed, 146 insertions(+) create mode 100644 rre-search-platform/rre-search-platform-external-solr-impl/src/test/java/io/sease/rre/search/api/impl/ExternalApacheSolrIT.java diff --git a/rre-search-platform/rre-search-platform-external-solr-impl/pom.xml b/rre-search-platform/rre-search-platform-external-solr-impl/pom.xml index dcaf2a25..c3f45da7 100644 --- a/rre-search-platform/rre-search-platform-external-solr-impl/pom.xml +++ b/rre-search-platform/rre-search-platform-external-solr-impl/pom.xml @@ -14,6 +14,51 @@ 8.3.0 + + + + org.apache.maven.plugins + maven-surefire-plugin + 3.0.0-M5 + + + **/*IT.java + + + + + + + + + + integration + + + + org.apache.maven.plugins + maven-failsafe-plugin + 3.0.0-M5 + + + ${project.version} + + + + + + integration-test + verify + + + + + + + + + io.sease @@ -51,5 +96,11 @@ + + org.testcontainers + solr + 1.15.0-rc2 + test + \ No newline at end of file diff --git a/rre-search-platform/rre-search-platform-external-solr-impl/src/test/java/io/sease/rre/search/api/impl/ExternalApacheSolrIT.java b/rre-search-platform/rre-search-platform-external-solr-impl/src/test/java/io/sease/rre/search/api/impl/ExternalApacheSolrIT.java new file mode 100644 index 00000000..3142d3de --- /dev/null +++ b/rre-search-platform/rre-search-platform-external-solr-impl/src/test/java/io/sease/rre/search/api/impl/ExternalApacheSolrIT.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.sease.rre.search.api.impl; + +import io.sease.rre.search.api.SearchPlatform; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.testcontainers.containers.SolrContainer; +import org.testcontainers.utility.DockerImageName; + +import java.io.File; +import java.io.FileWriter; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * Integration tests for the Solr implementation. + *

+ * These won't be run as part of the main test phase, only as part of the + * "integration" profile. + *

+ * These use the TestContainers framework, which spins up Docker containers + * to allow testing against a Solr instance. + * + * @author Matt Pearce (matt@elysiansoftware.co.uk) + */ +public class ExternalApacheSolrIT { + + private static String SOLR_CONTAINER_BASE = "solr"; + private static String DEFAULT_SOLR_VERSION = "8.6"; + + private static final String INDEX_NAME = "test"; + private static final String INDEX_VERSION = "v1.0"; + + private static final DockerImageName DOCKER_IMAGE = DockerImageName.parse(SOLR_CONTAINER_BASE + ":" + System.getProperty("solr.version", DEFAULT_SOLR_VERSION)); + + @Rule + public TemporaryFolder tempFolder = new TemporaryFolder(); + + private SearchPlatform platform; + + @Before + public void setupPlatform() { + platform = new ExternalApacheSolr(); + } + + @Test + public void checkCollectionReturnsFalse_whenNoSuchCollection() throws Exception { + final SolrContainer solrContainer = new SolrContainer(DOCKER_IMAGE); + solrContainer.start(); + + final File settingsFile = tempFolder.newFile("ccrf_settings.json"); + FileWriter fw = new FileWriter(settingsFile); + fw.write("{ \"baseUrls\": [ \"http://" + solrContainer.getHost() + ":" + solrContainer.getSolrPort() + "\" ]}"); + fw.close(); + + platform.load(null, settingsFile, INDEX_NAME, INDEX_VERSION); + + assertFalse(platform.checkCollection(INDEX_NAME, INDEX_VERSION)); + solrContainer.close(); + } + + @Test + public void checkCollectionReturnsTrue_whenCollectionExists() throws Exception { + final SolrContainer solrContainer = new SolrContainer(DOCKER_IMAGE).withCollection(INDEX_NAME); + solrContainer.start(); + + final File settingsFile = tempFolder.newFile("ccrt_settings.json"); + FileWriter fw = new FileWriter(settingsFile); + fw.write("{ \"baseUrls\": [ \"http://" + solrContainer.getHost() + ":" + solrContainer.getSolrPort() + "/solr\" ]}"); + fw.close(); + + platform.load(null, settingsFile, INDEX_NAME, INDEX_VERSION); + + assertTrue(platform.checkCollection(INDEX_NAME, INDEX_VERSION)); + solrContainer.close(); + } +} From 91903da969f7eb03ef29a6c658ad77455285aafb Mon Sep 17 00:00:00 2001 From: Matt Pearce Date: Sun, 1 Nov 2020 12:21:16 +0000 Subject: [PATCH 4/5] Add integration tests for the Elasticsearch checkPlatform method. --- .../pom.xml | 52 +++++++++ .../api/impl/ExternalElasticSearchIT.java | 106 ++++++++++++++++++ .../impl/ExternalElasticsearchIndexTest.java | 80 ------------- 3 files changed, 158 insertions(+), 80 deletions(-) create mode 100644 rre-search-platform/rre-search-platform-elastic-search-impl/src/test/java/io/sease/rre/search/api/impl/ExternalElasticSearchIT.java delete mode 100644 rre-search-platform/rre-search-platform-elastic-search-impl/src/test/java/io/sease/rre/search/api/impl/ExternalElasticsearchIndexTest.java diff --git a/rre-search-platform/rre-search-platform-elastic-search-impl/pom.xml b/rre-search-platform/rre-search-platform-elastic-search-impl/pom.xml index 91426a57..4da352c1 100644 --- a/rre-search-platform/rre-search-platform-elastic-search-impl/pom.xml +++ b/rre-search-platform/rre-search-platform-elastic-search-impl/pom.xml @@ -29,6 +29,52 @@ rre-search-platform-elasticsearch-impl 7.5.0 RRE - Elasticsearch platform binding + + + + + org.apache.maven.plugins + maven-surefire-plugin + 3.0.0-M5 + + + **/*IT.java + + + + + + + + + + integration + + + + org.apache.maven.plugins + maven-failsafe-plugin + 3.0.0-M5 + + + ${project.version} + + + + + + integration-test + verify + + + + + + + + + io.sease @@ -83,5 +129,11 @@ 2.10.0 test + + org.testcontainers + elasticsearch + 1.15.0-rc2 + test + \ No newline at end of file diff --git a/rre-search-platform/rre-search-platform-elastic-search-impl/src/test/java/io/sease/rre/search/api/impl/ExternalElasticSearchIT.java b/rre-search-platform/rre-search-platform-elastic-search-impl/src/test/java/io/sease/rre/search/api/impl/ExternalElasticSearchIT.java new file mode 100644 index 00000000..8c07a3eb --- /dev/null +++ b/rre-search-platform/rre-search-platform-elastic-search-impl/src/test/java/io/sease/rre/search/api/impl/ExternalElasticSearchIT.java @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.sease.rre.search.api.impl; + +import io.sease.rre.search.api.SearchPlatform; +import org.apache.http.HttpHost; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.client.indices.CreateIndexRequest; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.testcontainers.elasticsearch.ElasticsearchContainer; +import org.testcontainers.utility.DockerImageName; + +import java.io.File; +import java.io.FileWriter; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * Integration test for the External Elasticsearch implementation. + *

+ * These won't be run as part of the main test phase, only as part of the + * "integration" profile. + *

+ * These use the TestContainers framework, which spins up Docker containers + * to allow testing against a Solr instance. + * + * @author Matt Pearce (matt@elysiansoftware.co.uk) + */ +public class ExternalElasticSearchIT { + + private static String ELASTICSEARCH_CONTAINER_BASE = "docker.elastic.co/elasticsearch/elasticsearch"; + private static String DEFAULT_ELASTICSEARCH_VERSION = "7.5.0"; + + private static final String INDEX_NAME = "test"; + private static final String INDEX_VERSION = "v1.0"; + + private static final DockerImageName DOCKER_IMAGE = DockerImageName.parse(ELASTICSEARCH_CONTAINER_BASE + ":" + System.getProperty("elasticsearch.version", DEFAULT_ELASTICSEARCH_VERSION)); + + @Rule + public TemporaryFolder tempFolder = new TemporaryFolder(); + + private SearchPlatform platform; + + @Before + public void setupPlatform() throws Exception { + platform = new ExternalElasticsearch(); + } + + @Test + public void checkCollection_returnsFalseWhenNotLoaded() throws Exception { + final ElasticsearchContainer es = new ElasticsearchContainer(DOCKER_IMAGE); + es.start(); + + final File settingsFile = tempFolder.newFile("ccrf_settings.json"); + FileWriter fw = new FileWriter(settingsFile); + fw.write("{ \"hostUrls\": [ \"" + es.getHttpHostAddress() + "\" ]}"); + fw.close(); + + platform.load(null, settingsFile, INDEX_NAME, INDEX_VERSION); + + assertFalse(platform.checkCollection(INDEX_NAME, INDEX_VERSION)); + es.close(); + } + + @Test + public void checkCollection_returnsTrueWhenAvailable() throws Exception { + final ElasticsearchContainer es = new ElasticsearchContainer(DOCKER_IMAGE); + es.start(); + + // Create an index + final RestHighLevelClient hlClient = new RestHighLevelClient(RestClient.builder(HttpHost.create(es.getHttpHostAddress()))); + hlClient.indices().create(new CreateIndexRequest(INDEX_NAME), RequestOptions.DEFAULT); + + final File settingsFile = tempFolder.newFile("ccrt_settings.json"); + FileWriter fw = new FileWriter(settingsFile); + fw.write("{ \"hostUrls\": [ \"" + es.getHttpHostAddress() + "\" ]}"); + fw.close(); + + platform.load(null, settingsFile, INDEX_NAME, INDEX_VERSION); + + assertTrue(platform.checkCollection(INDEX_NAME, INDEX_VERSION)); + hlClient.close(); + es.close(); + } + +} diff --git a/rre-search-platform/rre-search-platform-elastic-search-impl/src/test/java/io/sease/rre/search/api/impl/ExternalElasticsearchIndexTest.java b/rre-search-platform/rre-search-platform-elastic-search-impl/src/test/java/io/sease/rre/search/api/impl/ExternalElasticsearchIndexTest.java deleted file mode 100644 index c0cf0326..00000000 --- a/rre-search-platform/rre-search-platform-elastic-search-impl/src/test/java/io/sease/rre/search/api/impl/ExternalElasticsearchIndexTest.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.sease.rre.search.api.impl; - -import io.sease.rre.search.api.SearchPlatform; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; -import org.mockserver.client.MockServerClient; -import org.mockserver.junit.MockServerRule; - -import java.util.Collections; - -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.mockserver.model.HttpRequest.request; -import static org.mockserver.model.HttpResponse.response; - - -/** - * An integration test for checking ExternalElasticsearch methods that rely on - * an existing Elasticsearch cluster. - * - * @author Matt Pearce (matt@elysiansoftware.co.uk) - */ -public class ExternalElasticsearchIndexTest { - - private static final String INDEX_NAME = "test"; - private static final String VERSION = "1.0"; - - private static final String ES_DOCKER_IMAGE = "docker.elastic.co/elasticsearch/elasticsearch"; - private static final String ES_VERSION = "7.5.0"; - - @Rule - public TemporaryFolder tempFolder = new TemporaryFolder(); - @Rule - public MockServerRule mockServerRule = new MockServerRule(this); - - private MockServerClient mockServerClient; - - private SearchPlatform platform; - - @Before - public void setupPlatform() throws Exception { - platform = new ExternalElasticsearch(); - // Setting these explicitly, rather than using load() - ((ExternalElasticsearch) platform).setSettings( - new ExternalElasticsearch.IndexSettings(Collections.singletonList("http://localhost:" + mockServerClient.getPort()), null, null), - VERSION); - } - - @Test - public void checkCollection_returnsFalseWhenNotLoaded() throws Exception { - mockServerClient.when(request().withMethod("HEAD").withPath("/" + INDEX_NAME)) - .respond(response().withStatusCode(404)); - assertFalse(platform.checkCollection(INDEX_NAME, VERSION)); - } - - @Test - public void checkCollection_returnsTrueWhenAvailable() throws Exception { - mockServerClient.when(request().withMethod("HEAD").withPath("/" + INDEX_NAME)) - .respond(response().withStatusCode(200)); - assertTrue(platform.checkCollection(INDEX_NAME, VERSION)); - } -} From c90e3a0c7352cf939152d415e46aa1cf98ecf3c9 Mon Sep 17 00:00:00 2001 From: Matt Pearce Date: Sun, 1 Nov 2020 12:25:02 +0000 Subject: [PATCH 5/5] Modify Solr checkPlatform() to use ping call rather than query. --- .../io/sease/rre/search/api/impl/ExternalApacheSolr.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/rre-search-platform/rre-search-platform-external-solr-impl/src/main/java/io/sease/rre/search/api/impl/ExternalApacheSolr.java b/rre-search-platform/rre-search-platform-external-solr-impl/src/main/java/io/sease/rre/search/api/impl/ExternalApacheSolr.java index ed9e58b0..dd94f6de 100644 --- a/rre-search-platform/rre-search-platform-external-solr-impl/src/main/java/io/sease/rre/search/api/impl/ExternalApacheSolr.java +++ b/rre-search-platform/rre-search-platform-external-solr-impl/src/main/java/io/sease/rre/search/api/impl/ExternalApacheSolr.java @@ -26,7 +26,7 @@ import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.client.solrj.SolrRequest; import org.apache.solr.client.solrj.SolrServerException; -import org.apache.solr.client.solrj.response.QueryResponse; +import org.apache.solr.client.solrj.response.SolrPingResponse; import org.apache.solr.common.SolrException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -165,10 +165,9 @@ public void close() { @Override public boolean checkCollection(String collection, String version) { try { - SolrQuery query = new SolrQuery("*:*").setRows(0); SolrClient client = clientManager.getSolrClient(version); if (client != null) { - QueryResponse response = client.query(collection, query); + SolrPingResponse response = client.ping(collection); return response.getStatus() == 0; } } catch (SolrException e) {