From b9495a06b5adf70b2ab7719dd95acf767d4578b9 Mon Sep 17 00:00:00 2001 From: FineAndDandy Date: Wed, 13 Nov 2024 00:20:23 +0000 Subject: [PATCH 01/12] Example metadataHelper caching unit test --- .../query/util/MetadataHelperCacheTest.java | 215 ++++++++++++++++++ 1 file changed, 215 insertions(+) create mode 100644 warehouse/query-core/src/test/java/datawave/query/util/MetadataHelperCacheTest.java diff --git a/warehouse/query-core/src/test/java/datawave/query/util/MetadataHelperCacheTest.java b/warehouse/query-core/src/test/java/datawave/query/util/MetadataHelperCacheTest.java new file mode 100644 index 0000000000..375a36cc0c --- /dev/null +++ b/warehouse/query-core/src/test/java/datawave/query/util/MetadataHelperCacheTest.java @@ -0,0 +1,215 @@ +package datawave.query.util; + +import datawave.query.composite.CompositeMetadataHelper; +import org.apache.accumulo.core.client.AccumuloClient; +import org.apache.accumulo.core.client.Scanner; +import org.apache.accumulo.core.client.TableNotFoundException; +import org.apache.accumulo.core.data.Key; +import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.core.security.Authorizations; +import org.apache.deltaspike.core.api.config.Configuration; +import org.apache.hadoop.io.Text; +import org.easymock.EasyMock; +import org.easymock.EasyMockSupport; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.cache.CacheManager; +import org.springframework.cache.annotation.Cacheable; +import org.springframework.cache.annotation.EnableCaching; +import org.springframework.cache.concurrent.ConcurrentMapCacheManager; +import org.springframework.context.annotation.Bean; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.easymock.EasyMock.anyObject; +import static org.easymock.EasyMock.eq; +import static org.easymock.EasyMock.expect; +import static org.junit.Assert.assertNotNull; + +@RunWith(SpringJUnit4ClassRunner.class) +@ContextConfiguration(classes ={MetadataHelperCacheTest.Config.class}) +public class MetadataHelperCacheTest extends EasyMockSupport { + + // will fake this part out + private static AccumuloClient accumuloClient = EasyMock.createMock(AccumuloClient.class); + + // wire up all the spring beans to get a workable cache + @Configuration + @EnableCaching + static class Config { + @Autowired + private TypeMetadataHelper typeMetadataHelper; + + @Autowired + private CompositeMetadataHelper compositeMetadataHelper; + + @Autowired + private AllFieldMetadataHelper allFieldMetadataHelper; + + @Bean + CacheManager metadataHelperCacheManager() { + return new ConcurrentMapCacheManager(); + } + + @Bean + AccumuloClient accumuloClient() { + return accumuloClient; + } + + @Bean + Set allMetadataAuths() { + Set auths = new HashSet<>(); + auths.add(new Authorizations("ALL")); + + return auths; + } + + @Bean + TypeMetadataHelper typeMetadataHelper() { + return new TypeMetadataHelper(Collections.EMPTY_MAP, allMetadataAuths(), accumuloClient(), "table", allMetadataAuths(), false); + } + + @Bean + CompositeMetadataHelper compositeMetadataHelper() { + return new CompositeMetadataHelper(accumuloClient(), "table", allMetadataAuths()); + } + + @Bean + AllFieldMetadataHelper allFieldMetadataHelper() { + return new AllFieldMetadataHelper(typeMetadataHelper, compositeMetadataHelper, accumuloClient(), "table", allMetadataAuths(), allMetadataAuths()); + } + + @Bean + MetadataHelper metadataHelper() { + return new MetadataHelper(allFieldMetadataHelper, allMetadataAuths(), accumuloClient(), "table", allMetadataAuths(), allMetadataAuths()); + } + + @Bean + TestCache testCache() { + return new TestCache(); + } + } + + @Autowired + private MetadataHelper metadataHelper; + + @Test + public void metadataHelperTest() throws TableNotFoundException { + Scanner s = createMock(Scanner.class); + + List> entries = new ArrayList<>(); + + expect(accumuloClient.createScanner(eq("table"), EasyMock.anyObject())).andReturn(s).anyTimes(); + s.setRange(anyObject()); + s.fetchColumnFamily(new Text("i")); + expect(s.iterator()).andReturn(entries.iterator()); + s.close(); + + EasyMock.replay(accumuloClient); + replayAll(); + + assertNotNull(metadataHelper); + metadataHelper.isIndexed("myField", Collections.emptySet()); + // this call should hit the cache + metadataHelper.isIndexed("myField", Collections.emptySet()); + + EasyMock.verify(accumuloClient); + verifyAll(); + } + + @Test + public void getModelTableNamesTest() throws TableNotFoundException { + Scanner s = createMock(Scanner.class); + + List> entries = new ArrayList<>(); + + expect(accumuloClient.createScanner(eq("t1"), EasyMock.anyObject())).andReturn(s).anyTimes(); + s.setRange(anyObject()); + expect(s.iterator()).andReturn(entries.iterator()); + s.close(); + + expect(accumuloClient.createScanner(eq("t2"), EasyMock.anyObject())).andReturn(s).anyTimes(); + s.setRange(anyObject()); + expect(s.iterator()).andReturn(entries.iterator()); + s.close(); + + + EasyMock.replay(accumuloClient); + replayAll(); + + assertNotNull(metadataHelper); + metadataHelper.getQueryModelNames("t1"); + // this call should *NOT* hit cache + metadataHelper.getQueryModelNames("t2"); + + EasyMock.verify(accumuloClient); + verifyAll(); + } + + @Test + public void getTermFrequencyFieldsTest() throws TableNotFoundException { + Scanner s = createMock(Scanner.class); + + List> entries = new ArrayList<>(); + + // call 1 + expect(accumuloClient.createScanner(eq("table"), EasyMock.anyObject())).andReturn(s); + s.setRange(anyObject()); + s.fetchColumnFamily(new Text("tf")); + expect(s.iterator()).andReturn(entries.iterator()); + s.close(); + + // call 2 + expect(accumuloClient.createScanner(eq("table"), EasyMock.anyObject())).andReturn(s); + s.setRange(anyObject()); + s.fetchColumnFamily(new Text("tf")); + expect(s.iterator()).andReturn(entries.iterator()); + s.close(); + + // call 3 + expect(accumuloClient.createScanner(eq("table"), EasyMock.anyObject())).andReturn(s); + s.setRange(anyObject()); + s.fetchColumnFamily(new Text("tf")); + expect(s.iterator()).andReturn(entries.iterator()); + s.close(); + + EasyMock.replay(accumuloClient); + replayAll(); + + assertNotNull(metadataHelper); + metadataHelper.getTermFrequencyFields(null); + // this call should *NOT* hit cache + metadataHelper.getTermFrequencyFields(Collections.emptySet()); + // this call should *NOT* hit cache + metadataHelper.getTermFrequencyFields(Collections.singleton("mega")); + + EasyMock.verify(accumuloClient); + verifyAll(); + } + + // just to prove out how to wire up a spring cache + public static class TestCache { + int i = 0; + @Cacheable(value = "someCache") + public String getData(String key) { + return key + i++; + } + } + + @Autowired + private TestCache c; + + @Test + public void testCache() { + System.out.println(c.getData("a")); + System.out.println(c.getData("a")); + } +} From 442ad699748f91cb2261a3ca869d0ba6fd8deeb9 Mon Sep 17 00:00:00 2001 From: FineAndDandy Date: Wed, 13 Nov 2024 21:12:55 +0000 Subject: [PATCH 02/12] extend test coverage --- .../query/util/MetadataHelperCacheTest.java | 578 ++++++++++++++++-- 1 file changed, 542 insertions(+), 36 deletions(-) diff --git a/warehouse/query-core/src/test/java/datawave/query/util/MetadataHelperCacheTest.java b/warehouse/query-core/src/test/java/datawave/query/util/MetadataHelperCacheTest.java index 375a36cc0c..3bd0e28cc8 100644 --- a/warehouse/query-core/src/test/java/datawave/query/util/MetadataHelperCacheTest.java +++ b/warehouse/query-core/src/test/java/datawave/query/util/MetadataHelperCacheTest.java @@ -2,8 +2,11 @@ import datawave.query.composite.CompositeMetadataHelper; import org.apache.accumulo.core.client.AccumuloClient; +import org.apache.accumulo.core.client.AccumuloException; +import org.apache.accumulo.core.client.AccumuloSecurityException; import org.apache.accumulo.core.client.Scanner; import org.apache.accumulo.core.client.TableNotFoundException; +import org.apache.accumulo.core.client.admin.SecurityOperations; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.security.Authorizations; @@ -11,9 +14,11 @@ import org.apache.hadoop.io.Text; import org.easymock.EasyMock; import org.easymock.EasyMockSupport; +import org.junit.After; import org.junit.Test; import org.junit.runner.RunWith; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.cache.CacheManager; import org.springframework.cache.annotation.Cacheable; import org.springframework.cache.annotation.EnableCaching; @@ -28,11 +33,16 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.ExecutionException; import static org.easymock.EasyMock.anyObject; import static org.easymock.EasyMock.eq; import static org.easymock.EasyMock.expect; +import static org.easymock.EasyMock.expectLastCall; +import static org.easymock.EasyMock.isA; +import static org.easymock.EasyMock.reset; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; @RunWith(SpringJUnit4ClassRunner.class) @ContextConfiguration(classes ={MetadataHelperCacheTest.Config.class}) @@ -72,6 +82,14 @@ Set allMetadataAuths() { return auths; } + @Bean + Set alternateAllMetadataAuths() { + Set auths = new HashSet<>(); + auths.add(new Authorizations("ALTERNATE")); + + return auths; + } + @Bean TypeMetadataHelper typeMetadataHelper() { return new TypeMetadataHelper(Collections.EMPTY_MAP, allMetadataAuths(), accumuloClient(), "table", allMetadataAuths(), false); @@ -92,6 +110,21 @@ MetadataHelper metadataHelper() { return new MetadataHelper(allFieldMetadataHelper, allMetadataAuths(), accumuloClient(), "table", allMetadataAuths(), allMetadataAuths()); } + @Bean + MetadataHelper alternateTableMetadataHelper() { + return new MetadataHelper(allFieldMetadataHelper, allMetadataAuths(), accumuloClient(), "table2", allMetadataAuths(), allMetadataAuths()); + } + + @Bean + MetadataHelper alternateAuthsHelper() { + return new MetadataHelper(allFieldMetadataHelper, alternateAllMetadataAuths(), accumuloClient(), "table", alternateAllMetadataAuths(), alternateAllMetadataAuths()); + } + + @Bean + MetadataHelper alternateHelper() { + return new MetadataHelper(allFieldMetadataHelper, alternateAllMetadataAuths(), accumuloClient(), "table2", alternateAllMetadataAuths(), alternateAllMetadataAuths()); + } + @Bean TestCache testCache() { return new TestCache(); @@ -99,87 +132,222 @@ TestCache testCache() { } @Autowired + @Qualifier("metadataHelper") private MetadataHelper metadataHelper; + @Autowired + @Qualifier("alternateTableMetadataHelper") + private MetadataHelper alternateTableMetadataHelper; + + @Autowired + @Qualifier("alternateAuthsHelper") + private MetadataHelper alternateAuthsHelper; + + @Autowired + @Qualifier("alternateHelper") + private MetadataHelper alternateHelper; + + @After + public void cleanup() { + reset(accumuloClient); + } + + // this test will fail until MetadataHelper.getQueryModelNames uses the right key @Test - public void metadataHelperTest() throws TableNotFoundException { + public void getModelTableNames_param_test() throws TableNotFoundException { Scanner s = createMock(Scanner.class); List> entries = new ArrayList<>(); - expect(accumuloClient.createScanner(eq("table"), EasyMock.anyObject())).andReturn(s).anyTimes(); - s.setRange(anyObject()); - s.fetchColumnFamily(new Text("i")); - expect(s.iterator()).andReturn(entries.iterator()); - s.close(); + expectScanner("t1", s, entries); + + expectScanner("t2", s, entries); EasyMock.replay(accumuloClient); replayAll(); assertNotNull(metadataHelper); - metadataHelper.isIndexed("myField", Collections.emptySet()); - // this call should hit the cache - metadataHelper.isIndexed("myField", Collections.emptySet()); + metadataHelper.getQueryModelNames("t1"); + // this call should *NOT* hit cache + metadataHelper.getQueryModelNames("t2"); + // this call should hit cache + metadataHelper.getQueryModelNames("t2"); EasyMock.verify(accumuloClient); verifyAll(); } @Test - public void getModelTableNamesTest() throws TableNotFoundException { + public void getFacets_param_test() throws TableNotFoundException, InstantiationException, IllegalAccessException { Scanner s = createMock(Scanner.class); List> entries = new ArrayList<>(); - expect(accumuloClient.createScanner(eq("t1"), EasyMock.anyObject())).andReturn(s).anyTimes(); - s.setRange(anyObject()); - expect(s.iterator()).andReturn(entries.iterator()); - s.close(); + // call 1 + expectScanner("t1", s, entries); - expect(accumuloClient.createScanner(eq("t2"), EasyMock.anyObject())).andReturn(s).anyTimes(); - s.setRange(anyObject()); - expect(s.iterator()).andReturn(entries.iterator()); - s.close(); + // call 2 + expectScanner("t2", s, entries); + // call 3 cached EasyMock.replay(accumuloClient); replayAll(); assertNotNull(metadataHelper); - metadataHelper.getQueryModelNames("t1"); + metadataHelper.getFacets("t1"); // this call should *NOT* hit cache - metadataHelper.getQueryModelNames("t2"); + metadataHelper.getFacets("t2"); + // this call should hit cache t2 + metadataHelper.getFacets("t2"); + + EasyMock.verify(accumuloClient); + verifyAll(); + } + + @Test + public void getTermCounts_param_test() throws TableNotFoundException, InstantiationException, IllegalAccessException { + Scanner s = createMock(Scanner.class); + + List> entries = new ArrayList<>(); + + // call 1 + expectScanner("table", s, entries); + + // call 2 cached + + EasyMock.replay(accumuloClient); + replayAll(); + + assertNotNull(metadataHelper); + metadataHelper.getTermCounts(); + // this call should hit cache + metadataHelper.getTermCounts(); + + EasyMock.verify(accumuloClient); + verifyAll(); + } + + @Test + public void getTermCountsWithRootAuths_param_test() throws TableNotFoundException, InstantiationException, IllegalAccessException, AccumuloException, AccumuloSecurityException { + Scanner s = createMock(Scanner.class); + + List> entries = new ArrayList<>(); + + // call 1 + expectScanner("table", s, entries); + + // call 2 cached + + // mock the security check for call 1 + SecurityOperations so = createMock(SecurityOperations.class); + expect(accumuloClient.securityOperations()).andReturn(so); + expect(accumuloClient.whoami()).andReturn("dwv"); + expect(so.getUserAuthorizations("dwv")).andReturn(new Authorizations("steve")); + + EasyMock.replay(accumuloClient); + replayAll(); + + assertNotNull(metadataHelper); + metadataHelper.getTermCountsWithRootAuths(); + // this call should hit cache + metadataHelper.getTermCountsWithRootAuths(); + + EasyMock.verify(accumuloClient); + verifyAll(); + } + + @Test + public void getTermCounts_getTermCountsWithRootAuths_notShared_test() throws TableNotFoundException, InstantiationException, IllegalAccessException, AccumuloException, AccumuloSecurityException { + Scanner s = createMock(Scanner.class); + + List> entries = new ArrayList<>(); + + // call 1 + expectScanner("table", s, entries); + + // call 2 + expectScanner("table", s, entries); + SecurityOperations so = createMock(SecurityOperations.class); + expect(accumuloClient.securityOperations()).andReturn(so); + expect(accumuloClient.whoami()).andReturn("dwv"); + expect(so.getUserAuthorizations("dwv")).andReturn(new Authorizations("steve")); + + EasyMock.replay(accumuloClient); + replayAll(); + + assertNotNull(metadataHelper); + metadataHelper.getTermCounts(); + // this call should *not* hit cache + metadataHelper.getTermCountsWithRootAuths(); + + EasyMock.verify(accumuloClient); + verifyAll(); + } + + @Test + public void getAllNormalized_param_test() throws TableNotFoundException, InstantiationException, IllegalAccessException { + Scanner s = createMock(Scanner.class); + + List> entries = new ArrayList<>(); + + // call 1 + expectScanner("table", s, entries); + + // call 2 cached + + EasyMock.replay(accumuloClient); + replayAll(); + + assertNotNull(metadataHelper); + metadataHelper.getAllNormalized(); + // this call should hit cache + metadataHelper.getAllNormalized(); + + EasyMock.verify(accumuloClient); + verifyAll(); + } + + @Test + public void getEdges_param_test() throws TableNotFoundException, InstantiationException, IllegalAccessException, ExecutionException { + Scanner s = createMock(Scanner.class); + + List> entries = new ArrayList<>(); + + // call 1 + expectScanner("table", s, entries); + // extra scanner expectations + s.addScanIterator(anyObject()); + s.addScanIterator(anyObject()); + + // call 2 cached + + EasyMock.replay(accumuloClient); + replayAll(); + + assertNotNull(metadataHelper); + metadataHelper.getEdges(); + // this call should hit cache + metadataHelper.getEdges(); EasyMock.verify(accumuloClient); verifyAll(); } @Test - public void getTermFrequencyFieldsTest() throws TableNotFoundException { + public void getTermFrequencyFields_param_test() throws TableNotFoundException { Scanner s = createMock(Scanner.class); List> entries = new ArrayList<>(); // call 1 - expect(accumuloClient.createScanner(eq("table"), EasyMock.anyObject())).andReturn(s); - s.setRange(anyObject()); - s.fetchColumnFamily(new Text("tf")); - expect(s.iterator()).andReturn(entries.iterator()); - s.close(); + expectScanner("table", s, entries); // call 2 - expect(accumuloClient.createScanner(eq("table"), EasyMock.anyObject())).andReturn(s); - s.setRange(anyObject()); - s.fetchColumnFamily(new Text("tf")); - expect(s.iterator()).andReturn(entries.iterator()); - s.close(); + expectScanner("table", s, entries); // call 3 - expect(accumuloClient.createScanner(eq("table"), EasyMock.anyObject())).andReturn(s); - s.setRange(anyObject()); - s.fetchColumnFamily(new Text("tf")); - expect(s.iterator()).andReturn(entries.iterator()); - s.close(); + expectScanner("table", s, entries); EasyMock.replay(accumuloClient); replayAll(); @@ -195,6 +363,344 @@ public void getTermFrequencyFieldsTest() throws TableNotFoundException { verifyAll(); } + @Test + public void getTermFrequencyFields_alternateKey_test() throws TableNotFoundException { + Scanner s = createMock(Scanner.class); + + List> entries = new ArrayList<>(); + + // call 1 + expectScanner("table", s, entries); + + // call 2 + expectScanner("table2", s, entries); + + // call 3 + expectScanner("table", s, entries); + + // call 4 + expectScanner("table2", s, entries); + + // call 5-8 cached + + EasyMock.replay(accumuloClient); + replayAll(); + + assertNotNull(metadataHelper); + assertNotNull(alternateTableMetadataHelper); + assertNotNull(alternateAuthsHelper); + assertNotNull(alternateHelper); + + metadataHelper.getTermFrequencyFields(null); + // this call should *NOT* hit cache + alternateTableMetadataHelper.getTermFrequencyFields(null); + // this call should *NOT* hit cache + alternateAuthsHelper.getTermFrequencyFields(null); + // this call should *NOT* hit cache + alternateHelper.getTermFrequencyFields(null); + + // cached calls + metadataHelper.getTermFrequencyFields(null); + alternateTableMetadataHelper.getTermFrequencyFields(null); + alternateAuthsHelper.getTermFrequencyFields(null); + alternateHelper.getTermFrequencyFields(null); + + EasyMock.verify(accumuloClient); + verifyAll(); + } + + @Test + public void getAllFields_param_test() throws TableNotFoundException { + Scanner s = createMock(Scanner.class); + + List> entries = new ArrayList<>(); + + // call 1 + expectScanner("table", s, entries); + + // call 2 cached + + // call 3 cached + + EasyMock.replay(accumuloClient); + replayAll(); + + assertNotNull(metadataHelper); + metadataHelper.getAllFields(null); + // this call should hit cache + metadataHelper.getAllFields(Collections.emptySet()); + // this call should hit cache + metadataHelper.getAllFields(Collections.singleton("mega")); + + EasyMock.verify(accumuloClient); + verifyAll(); + } + + // this test will fail until AllFieldMetadataHelper.getIndexOnlyFields() properly closes its scanner + @Test + public void getNonEventFields_param_test() throws TableNotFoundException { + Scanner s = createMock(Scanner.class); + + List> entries = new ArrayList<>(); + + // call 1 + // getIndexOnlyFields + expectScanner("table", s, entries); + // getTermFrequencyFields + expectScanner("table", s, entries); + // getCompositeToFieldMap + expectScanner("table", s, entries); + + // call 2 + // getIndexOnlyFields cached + // getTermFrequencyFields + expectScanner("table", s, entries); + // getCompositeToFieldMap + expectScanner("table", s, entries); + + // call 3 + // getIndexOnlyFields cached + // getTermFrequencyFields + expectScanner("table", s, entries); + // getCompositeToFieldMap + expectScanner("table", s, entries); + + EasyMock.replay(accumuloClient); + replayAll(); + + assertNotNull(metadataHelper); + metadataHelper.getNonEventFields(null); + // this call should *NOT* hit cache + metadataHelper.getNonEventFields(Collections.emptySet()); + // this call should *NOT* hit cache + metadataHelper.getNonEventFields(Collections.singleton("mega")); + + EasyMock.verify(accumuloClient); + verifyAll(); + } + + @Test + public void getNonEventFields_key_test() throws TableNotFoundException { + // TODO + assertTrue(false); + } + + // this test will fail until AllFieldMetadataHelper.getIndexOnlyFields() properly closes its scanner + @Test + public void getIndexOnlyFields_param_test() throws TableNotFoundException { + Scanner s = createMock(Scanner.class); + + List> entries = new ArrayList<>(); + + // call 1 + // getIndexOnlyFields + expectScanner("table", s, entries); + + // call 2 cached + + // call 3 cached + + EasyMock.replay(accumuloClient); + replayAll(); + + assertNotNull(metadataHelper); + metadataHelper.getIndexOnlyFields(null); + // this call should hit cache + metadataHelper.getIndexOnlyFields(Collections.emptySet()); + // this call should hit cache + metadataHelper.getIndexOnlyFields(Collections.singleton("mega")); + + EasyMock.verify(accumuloClient); + verifyAll(); + } + + @Test + public void isReverseIndexed_param_test() throws TableNotFoundException { + Scanner s = createMock(Scanner.class); + + List> entries = new ArrayList<>(); + + // call 1 + expectScanner("table", s, entries); + // call 2 + expectScanner("table", s, entries); + // call 3 cached + // call 4 + expectScanner("table", s, entries); + + EasyMock.replay(accumuloClient); + replayAll(); + + assertNotNull(metadataHelper); + metadataHelper.isReverseIndexed("myField", Collections.emptySet()); + // this call should *NOT* hit the cache + metadataHelper.isReverseIndexed("myField2", Collections.emptySet()); + // this call should hit the cache + metadataHelper.isReverseIndexed("myField2", Collections.emptySet()); + // this call should *NOT* hit the cache + metadataHelper.isReverseIndexed("myField2", Collections.singleton("ingestTypeFilter1")); + + EasyMock.verify(accumuloClient); + verifyAll(); + } + + @Test + public void isIndexed_param_test() throws TableNotFoundException { + Scanner s = createMock(Scanner.class); + + List> entries = new ArrayList<>(); + + // call 1 + expectScanner("table", s, entries); + // call 2 cached + // call 3 + expectScanner("table", s, entries); + // call 4 + expectScanner("table", s, entries); + + EasyMock.replay(accumuloClient); + replayAll(); + + assertNotNull(metadataHelper); + metadataHelper.isIndexed("myField", Collections.emptySet()); + // this call should hit the cache + metadataHelper.isIndexed("myField", Collections.emptySet()); + // this call should *NOT* hit the cache + metadataHelper.isIndexed("myField2", Collections.emptySet()); + // this call should *NOT* hit the cache + metadataHelper.isIndexed("myField2", Collections.singleton("ingestTypeFilter1")); + + EasyMock.verify(accumuloClient); + verifyAll(); + } + + @Test + public void isTokenized_param_test() throws TableNotFoundException { + Scanner s = createMock(Scanner.class); + + List> entries = new ArrayList<>(); + + // call 1 + expectScanner("table", s, entries); + // call 2 cached + // call 3 + expectScanner("table", s, entries); + // call 4 + expectScanner("table", s, entries); + + EasyMock.replay(accumuloClient); + replayAll(); + + assertNotNull(metadataHelper); + metadataHelper.isTokenized("myField", Collections.emptySet()); + // this call should hit the cache + metadataHelper.isTokenized("myField", Collections.emptySet()); + // this call should *NOT* hit the cache + metadataHelper.isTokenized("myField2", Collections.emptySet()); + // this call should *NOT* hit the cache + metadataHelper.isTokenized("myField2", Collections.singleton("ingestTypeFilter1")); + + EasyMock.verify(accumuloClient); + verifyAll(); + } + + @Test + public void getAllDatatypes_param_test() throws TableNotFoundException, InstantiationException, IllegalAccessException { + Scanner s = createMock(Scanner.class); + + List> entries = new ArrayList<>(); + + // call 1 + expectScanner("table", s, entries); + // call 2 cached + + EasyMock.replay(accumuloClient); + replayAll(); + + assertNotNull(metadataHelper); + metadataHelper.getAllDatatypes(); + // this call should hit the cache + metadataHelper.getAllDatatypes(); + + EasyMock.verify(accumuloClient); + verifyAll(); + } + + @Test + public void getCompositeToFieldMap_param_test() throws TableNotFoundException, InstantiationException, IllegalAccessException { + Scanner s = createMock(Scanner.class); + + List> entries = new ArrayList<>(); + + // call 1 + expectScanner("table", s, entries); + // call 2 cached + + EasyMock.replay(accumuloClient); + replayAll(); + + assertNotNull(metadataHelper); + metadataHelper.getCompositeToFieldMap(); + // this call should hit the cache + metadataHelper.getCompositeToFieldMap(); + + EasyMock.verify(accumuloClient); + verifyAll(); + } + + @Test + public void getCompositeToFieldMap_param2_test() throws TableNotFoundException, InstantiationException, IllegalAccessException { + Scanner s = createMock(Scanner.class); + + List> entries = new ArrayList<>(); + + // call 1 + expectScanner("table", s, entries); + // call 2 + expectScanner("table", s, entries); + // call 3 + expectScanner("table", s, entries); + // call 4 + expectScanner("table", s, entries); + // call 5 cached + // call 6 cached + + EasyMock.replay(accumuloClient); + replayAll(); + + assertNotNull(metadataHelper); + metadataHelper.getCompositeToFieldMap(); + // this call should *NOT* hit the cache + metadataHelper.getCompositeToFieldMap(null); + // this call should *NOT* hit the cache + metadataHelper.getCompositeToFieldMap(Collections.emptySet()); + // this call should *NOT* hit the cache + metadataHelper.getCompositeToFieldMap(Collections.singleton("test")); + // this call should hit the cache + metadataHelper.getCompositeToFieldMap(null); + // this call should hit the cache + metadataHelper.getCompositeToFieldMap(Collections.singleton("test")); + + EasyMock.verify(accumuloClient); + verifyAll(); + } + + /** + * expect/mock the scanner creation and return of results from an iterator + * @param table + * @param mockScanner + * @param entries + * @throws TableNotFoundException + */ + private void expectScanner(String table, Scanner mockScanner, List> entries) throws TableNotFoundException { + expect(accumuloClient.createScanner(eq(table), EasyMock.anyObject())).andReturn(mockScanner); + mockScanner.setRange(anyObject()); + mockScanner.fetchColumnFamily(isA(Text.class)); + expectLastCall().anyTimes(); + expect(mockScanner.iterator()).andReturn(entries.iterator()); + mockScanner.close(); + } + // just to prove out how to wire up a spring cache public static class TestCache { int i = 0; From e411365a377efc376ac0bf8c4303057dc4d510b8 Mon Sep 17 00:00:00 2001 From: FineAndDandy Date: Wed, 13 Nov 2024 21:40:00 +0000 Subject: [PATCH 03/12] cleanup static accumuloClient reference --- .../query/util/MetadataHelperCacheTest.java | 28 +++++++++++-------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/warehouse/query-core/src/test/java/datawave/query/util/MetadataHelperCacheTest.java b/warehouse/query-core/src/test/java/datawave/query/util/MetadataHelperCacheTest.java index 3bd0e28cc8..91df721ee1 100644 --- a/warehouse/query-core/src/test/java/datawave/query/util/MetadataHelperCacheTest.java +++ b/warehouse/query-core/src/test/java/datawave/query/util/MetadataHelperCacheTest.java @@ -14,7 +14,7 @@ import org.apache.hadoop.io.Text; import org.easymock.EasyMock; import org.easymock.EasyMockSupport; -import org.junit.After; +import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.springframework.beans.factory.annotation.Autowired; @@ -24,6 +24,7 @@ import org.springframework.cache.annotation.EnableCaching; import org.springframework.cache.concurrent.ConcurrentMapCacheManager; import org.springframework.context.annotation.Bean; +import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; @@ -40,17 +41,14 @@ import static org.easymock.EasyMock.expect; import static org.easymock.EasyMock.expectLastCall; import static org.easymock.EasyMock.isA; -import static org.easymock.EasyMock.reset; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; @RunWith(SpringJUnit4ClassRunner.class) @ContextConfiguration(classes ={MetadataHelperCacheTest.Config.class}) +@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_EACH_TEST_METHOD) public class MetadataHelperCacheTest extends EasyMockSupport { - - // will fake this part out - private static AccumuloClient accumuloClient = EasyMock.createMock(AccumuloClient.class); - // wire up all the spring beans to get a workable cache @Configuration @EnableCaching @@ -64,6 +62,8 @@ static class Config { @Autowired private AllFieldMetadataHelper allFieldMetadataHelper; + public AccumuloClient accumuloClient = null; + @Bean CacheManager metadataHelperCacheManager() { return new ConcurrentMapCacheManager(); @@ -71,6 +71,10 @@ CacheManager metadataHelperCacheManager() { @Bean AccumuloClient accumuloClient() { + if (accumuloClient == null) { + accumuloClient = EasyMock.createMock(AccumuloClient.class); + } + return accumuloClient; } @@ -147,9 +151,12 @@ TestCache testCache() { @Qualifier("alternateHelper") private MetadataHelper alternateHelper; - @After - public void cleanup() { - reset(accumuloClient); + @Autowired + private AccumuloClient accumuloClient; + + @Before + public void setup() { + assertNotNull(accumuloClient); } // this test will fail until MetadataHelper.getQueryModelNames uses the right key @@ -715,7 +722,6 @@ public String getData(String key) { @Test public void testCache() { - System.out.println(c.getData("a")); - System.out.println(c.getData("a")); + assertEquals(c.getData("a"), c.getData("a")); } } From ac934bc005cfe7acef675f91db257b8f51934ed2 Mon Sep 17 00:00:00 2001 From: FineAndDandy Date: Thu, 14 Nov 2024 00:29:27 +0000 Subject: [PATCH 04/12] Add framework to test all methods with reflection --- .../query/util/MetadataHelperCacheTest.java | 161 +++++++++++++----- 1 file changed, 120 insertions(+), 41 deletions(-) diff --git a/warehouse/query-core/src/test/java/datawave/query/util/MetadataHelperCacheTest.java b/warehouse/query-core/src/test/java/datawave/query/util/MetadataHelperCacheTest.java index 91df721ee1..cc8ed550ed 100644 --- a/warehouse/query-core/src/test/java/datawave/query/util/MetadataHelperCacheTest.java +++ b/warehouse/query-core/src/test/java/datawave/query/util/MetadataHelperCacheTest.java @@ -1,6 +1,26 @@ package datawave.query.util; -import datawave.query.composite.CompositeMetadataHelper; +import static org.easymock.EasyMock.anyObject; +import static org.easymock.EasyMock.eq; +import static org.easymock.EasyMock.expect; +import static org.easymock.EasyMock.expectLastCall; +import static org.easymock.EasyMock.isA; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ExecutionException; + import org.apache.accumulo.core.client.AccumuloClient; import org.apache.accumulo.core.client.AccumuloException; import org.apache.accumulo.core.client.AccumuloSecurityException; @@ -28,25 +48,10 @@ import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ExecutionException; - -import static org.easymock.EasyMock.anyObject; -import static org.easymock.EasyMock.eq; -import static org.easymock.EasyMock.expect; -import static org.easymock.EasyMock.expectLastCall; -import static org.easymock.EasyMock.isA; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; +import datawave.query.composite.CompositeMetadataHelper; @RunWith(SpringJUnit4ClassRunner.class) -@ContextConfiguration(classes ={MetadataHelperCacheTest.Config.class}) +@ContextConfiguration(classes = {MetadataHelperCacheTest.Config.class}) @DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_EACH_TEST_METHOD) public class MetadataHelperCacheTest extends EasyMockSupport { // wire up all the spring beans to get a workable cache @@ -69,7 +74,8 @@ CacheManager metadataHelperCacheManager() { return new ConcurrentMapCacheManager(); } - @Bean + // do not attempt to close this on destroy + @Bean(destroyMethod = "") AccumuloClient accumuloClient() { if (accumuloClient == null) { accumuloClient = EasyMock.createMock(AccumuloClient.class); @@ -121,12 +127,14 @@ MetadataHelper alternateTableMetadataHelper() { @Bean MetadataHelper alternateAuthsHelper() { - return new MetadataHelper(allFieldMetadataHelper, alternateAllMetadataAuths(), accumuloClient(), "table", alternateAllMetadataAuths(), alternateAllMetadataAuths()); + return new MetadataHelper(allFieldMetadataHelper, alternateAllMetadataAuths(), accumuloClient(), "table", alternateAllMetadataAuths(), + alternateAllMetadataAuths()); } @Bean MetadataHelper alternateHelper() { - return new MetadataHelper(allFieldMetadataHelper, alternateAllMetadataAuths(), accumuloClient(), "table2", alternateAllMetadataAuths(), alternateAllMetadataAuths()); + return new MetadataHelper(allFieldMetadataHelper, alternateAllMetadataAuths(), accumuloClient(), "table2", alternateAllMetadataAuths(), + alternateAllMetadataAuths()); } @Bean @@ -164,7 +172,7 @@ public void setup() { public void getModelTableNames_param_test() throws TableNotFoundException { Scanner s = createMock(Scanner.class); - List> entries = new ArrayList<>(); + List> entries = new ArrayList<>(); expectScanner("t1", s, entries); @@ -188,7 +196,7 @@ public void getModelTableNames_param_test() throws TableNotFoundException { public void getFacets_param_test() throws TableNotFoundException, InstantiationException, IllegalAccessException { Scanner s = createMock(Scanner.class); - List> entries = new ArrayList<>(); + List> entries = new ArrayList<>(); // call 1 expectScanner("t1", s, entries); @@ -216,7 +224,7 @@ public void getFacets_param_test() throws TableNotFoundException, InstantiationE public void getTermCounts_param_test() throws TableNotFoundException, InstantiationException, IllegalAccessException { Scanner s = createMock(Scanner.class); - List> entries = new ArrayList<>(); + List> entries = new ArrayList<>(); // call 1 expectScanner("table", s, entries); @@ -236,10 +244,11 @@ public void getTermCounts_param_test() throws TableNotFoundException, Instantiat } @Test - public void getTermCountsWithRootAuths_param_test() throws TableNotFoundException, InstantiationException, IllegalAccessException, AccumuloException, AccumuloSecurityException { + public void getTermCountsWithRootAuths_param_test() + throws TableNotFoundException, InstantiationException, IllegalAccessException, AccumuloException, AccumuloSecurityException { Scanner s = createMock(Scanner.class); - List> entries = new ArrayList<>(); + List> entries = new ArrayList<>(); // call 1 expectScanner("table", s, entries); @@ -265,10 +274,11 @@ public void getTermCountsWithRootAuths_param_test() throws TableNotFoundExceptio } @Test - public void getTermCounts_getTermCountsWithRootAuths_notShared_test() throws TableNotFoundException, InstantiationException, IllegalAccessException, AccumuloException, AccumuloSecurityException { + public void getTermCounts_getTermCountsWithRootAuths_notShared_test() + throws TableNotFoundException, InstantiationException, IllegalAccessException, AccumuloException, AccumuloSecurityException { Scanner s = createMock(Scanner.class); - List> entries = new ArrayList<>(); + List> entries = new ArrayList<>(); // call 1 expectScanner("table", s, entries); @@ -296,7 +306,7 @@ public void getTermCounts_getTermCountsWithRootAuths_notShared_test() throws Tab public void getAllNormalized_param_test() throws TableNotFoundException, InstantiationException, IllegalAccessException { Scanner s = createMock(Scanner.class); - List> entries = new ArrayList<>(); + List> entries = new ArrayList<>(); // call 1 expectScanner("table", s, entries); @@ -319,7 +329,7 @@ public void getAllNormalized_param_test() throws TableNotFoundException, Instant public void getEdges_param_test() throws TableNotFoundException, InstantiationException, IllegalAccessException, ExecutionException { Scanner s = createMock(Scanner.class); - List> entries = new ArrayList<>(); + List> entries = new ArrayList<>(); // call 1 expectScanner("table", s, entries); @@ -345,7 +355,7 @@ public void getEdges_param_test() throws TableNotFoundException, InstantiationEx public void getTermFrequencyFields_param_test() throws TableNotFoundException { Scanner s = createMock(Scanner.class); - List> entries = new ArrayList<>(); + List> entries = new ArrayList<>(); // call 1 expectScanner("table", s, entries); @@ -374,7 +384,7 @@ public void getTermFrequencyFields_param_test() throws TableNotFoundException { public void getTermFrequencyFields_alternateKey_test() throws TableNotFoundException { Scanner s = createMock(Scanner.class); - List> entries = new ArrayList<>(); + List> entries = new ArrayList<>(); // call 1 expectScanner("table", s, entries); @@ -420,7 +430,7 @@ public void getTermFrequencyFields_alternateKey_test() throws TableNotFoundExcep public void getAllFields_param_test() throws TableNotFoundException { Scanner s = createMock(Scanner.class); - List> entries = new ArrayList<>(); + List> entries = new ArrayList<>(); // call 1 expectScanner("table", s, entries); @@ -448,7 +458,7 @@ public void getAllFields_param_test() throws TableNotFoundException { public void getNonEventFields_param_test() throws TableNotFoundException { Scanner s = createMock(Scanner.class); - List> entries = new ArrayList<>(); + List> entries = new ArrayList<>(); // call 1 // getIndexOnlyFields @@ -497,7 +507,7 @@ public void getNonEventFields_key_test() throws TableNotFoundException { public void getIndexOnlyFields_param_test() throws TableNotFoundException { Scanner s = createMock(Scanner.class); - List> entries = new ArrayList<>(); + List> entries = new ArrayList<>(); // call 1 // getIndexOnlyFields @@ -525,7 +535,7 @@ public void getIndexOnlyFields_param_test() throws TableNotFoundException { public void isReverseIndexed_param_test() throws TableNotFoundException { Scanner s = createMock(Scanner.class); - List> entries = new ArrayList<>(); + List> entries = new ArrayList<>(); // call 1 expectScanner("table", s, entries); @@ -555,7 +565,7 @@ public void isReverseIndexed_param_test() throws TableNotFoundException { public void isIndexed_param_test() throws TableNotFoundException { Scanner s = createMock(Scanner.class); - List> entries = new ArrayList<>(); + List> entries = new ArrayList<>(); // call 1 expectScanner("table", s, entries); @@ -585,7 +595,7 @@ public void isIndexed_param_test() throws TableNotFoundException { public void isTokenized_param_test() throws TableNotFoundException { Scanner s = createMock(Scanner.class); - List> entries = new ArrayList<>(); + List> entries = new ArrayList<>(); // call 1 expectScanner("table", s, entries); @@ -615,7 +625,7 @@ public void isTokenized_param_test() throws TableNotFoundException { public void getAllDatatypes_param_test() throws TableNotFoundException, InstantiationException, IllegalAccessException { Scanner s = createMock(Scanner.class); - List> entries = new ArrayList<>(); + List> entries = new ArrayList<>(); // call 1 expectScanner("table", s, entries); @@ -637,7 +647,7 @@ public void getAllDatatypes_param_test() throws TableNotFoundException, Instanti public void getCompositeToFieldMap_param_test() throws TableNotFoundException, InstantiationException, IllegalAccessException { Scanner s = createMock(Scanner.class); - List> entries = new ArrayList<>(); + List> entries = new ArrayList<>(); // call 1 expectScanner("table", s, entries); @@ -659,7 +669,7 @@ public void getCompositeToFieldMap_param_test() throws TableNotFoundException, I public void getCompositeToFieldMap_param2_test() throws TableNotFoundException, InstantiationException, IllegalAccessException { Scanner s = createMock(Scanner.class); - List> entries = new ArrayList<>(); + List> entries = new ArrayList<>(); // call 1 expectScanner("table", s, entries); @@ -692,8 +702,76 @@ public void getCompositeToFieldMap_param2_test() throws TableNotFoundException, verifyAll(); } + // test all methods against all objects + @Test + public void allTest() throws Exception { + Map methodToParams = new HashMap<>(); + methodToParams.put("getQueryModelNames", + new MethodParamsAndExpectations[] {new MethodParamsAndExpectations(metadataHelper, new Object[] {"modelName"}) { + protected void expect() throws TableNotFoundException { + Scanner s = createMock(Scanner.class); + List> entries = new ArrayList<>(); + expectScanner("modelName", s, entries); + } + }, new MethodParamsAndExpectations(alternateTableMetadataHelper, new Object[] {"modelName"}), + new MethodParamsAndExpectations(alternateAuthsHelper, new Object[] {"modelName"}) { + protected void expect() throws TableNotFoundException { + Scanner s = createMock(Scanner.class); + List> entries = new ArrayList<>(); + expectScanner("modelName", s, entries); + } + }, new MethodParamsAndExpectations(alternateHelper, new Object[] {"modelName"}),}); + + // test that all methods are tested + for (Method method : MetadataHelper.class.getDeclaredMethods()) { + if (Modifier.isStatic(method.getModifiers())) { + // no need to verify statics + continue; + } + // verify all non static methods are tested + assertNotNull("MetadataHelper method '" + method.getName() + "' not tested", methodToParams.get(method.getName())); + + // setup expectations + for (MethodParamsAndExpectations expectation : methodToParams.get(method.getName())) { + expectation.expect(); + } + + // replay expectations + EasyMock.replay(accumuloClient); + replayAll(); + + // invoke method on all objects in order + for (MethodParamsAndExpectations expectation : methodToParams.get(method.getName())) { + method.invoke(expectation.get(), expectation.args); + } + + verifyAll(); + + // reset mocks to use again + EasyMock.reset(accumuloClient); + resetAll(); + } + } + + private static class MethodParamsAndExpectations { + public MetadataHelper metadataHelper; + public Object[] args; + + private MethodParamsAndExpectations(MetadataHelper helper, Object[] args) { + metadataHelper = helper; + this.args = args; + } + + protected void expect() throws Exception {} + + public Object get() { + return metadataHelper; + } + } + /** * expect/mock the scanner creation and return of results from an iterator + * * @param table * @param mockScanner * @param entries @@ -711,6 +789,7 @@ private void expectScanner(String table, Scanner mockScanner, List Date: Fri, 15 Nov 2024 16:14:27 +0000 Subject: [PATCH 05/12] Validate cached results and start adding method coverage --- .../query/util/MetadataHelperCacheTest.java | 486 +++++++++++++++++- 1 file changed, 459 insertions(+), 27 deletions(-) diff --git a/warehouse/query-core/src/test/java/datawave/query/util/MetadataHelperCacheTest.java b/warehouse/query-core/src/test/java/datawave/query/util/MetadataHelperCacheTest.java index cc8ed550ed..a981c246e5 100644 --- a/warehouse/query-core/src/test/java/datawave/query/util/MetadataHelperCacheTest.java +++ b/warehouse/query-core/src/test/java/datawave/query/util/MetadataHelperCacheTest.java @@ -9,7 +9,6 @@ import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; -import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.lang.reflect.Modifier; import java.util.ArrayList; @@ -50,23 +49,58 @@ import datawave.query.composite.CompositeMetadataHelper; +// enable springification @RunWith(SpringJUnit4ClassRunner.class) +// use the embedded Config class for all bean definitions @ContextConfiguration(classes = {MetadataHelperCacheTest.Config.class}) +// clear the context after each test so that the accumulo client expectations don't bleed between tests @DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_EACH_TEST_METHOD) public class MetadataHelperCacheTest extends EasyMockSupport { // wire up all the spring beans to get a workable cache @Configuration @EnableCaching static class Config { + // autowiring the resources ensures the @Cacheable interface will activate, using new will not activate the spring cache @Autowired + @Qualifier("typeMetadataHelper") private TypeMetadataHelper typeMetadataHelper; - @Autowired + @Qualifier("compositeMetadataHelper") private CompositeMetadataHelper compositeMetadataHelper; - @Autowired + @Qualifier("allFieldMetadataHelper") private AllFieldMetadataHelper allFieldMetadataHelper; + @Autowired + @Qualifier("alternateTableTypeHelper") + private TypeMetadataHelper alternateTableTypeHelper; + @Autowired + @Qualifier("alternateTableCompositeHelper") + private CompositeMetadataHelper alternateTableCompositeHelper; + @Autowired + @Qualifier("alternateTableAllFieldHelper") + private AllFieldMetadataHelper alternateTableAllFieldHelper; + + @Autowired + @Qualifier("alternateAuthsTypeHelper") + private TypeMetadataHelper alternateAuthsTypeHelper; + @Autowired + @Qualifier("alternateAuthsCompositeHelper") + private CompositeMetadataHelper alternateAuthsCompositeHelper; + @Autowired + @Qualifier("alternateAuthsAllFieldHelper") + private AllFieldMetadataHelper alternateAuthsAllFieldHelper; + + @Autowired + @Qualifier("alternateTypeHelper") + private TypeMetadataHelper alternateTypeHelper; + @Autowired + @Qualifier("alternateCompositeHelper") + private CompositeMetadataHelper alternateCompositeHelper; + @Autowired + @Qualifier("alternateAllFieldHelper") + private AllFieldMetadataHelper alternateAllFieldHelper; + public AccumuloClient accumuloClient = null; @Bean @@ -74,10 +108,12 @@ CacheManager metadataHelperCacheManager() { return new ConcurrentMapCacheManager(); } - // do not attempt to close this on destroy + // do not attempt to close this on destroy otherwise it triggers unexpected method calls on the mock @Bean(destroyMethod = "") - AccumuloClient accumuloClient() { + // synchronized to ensure only one client is created per test + synchronized AccumuloClient accumuloClient() { if (accumuloClient == null) { + // since the context will be torn down with @DirtiesContext create a new mock for each new set of tests that will have clean expectations accumuloClient = EasyMock.createMock(AccumuloClient.class); } @@ -105,16 +141,61 @@ TypeMetadataHelper typeMetadataHelper() { return new TypeMetadataHelper(Collections.EMPTY_MAP, allMetadataAuths(), accumuloClient(), "table", allMetadataAuths(), false); } + @Bean + TypeMetadataHelper alternateTableTypeHelper() { + return new TypeMetadataHelper(Collections.EMPTY_MAP, allMetadataAuths(), accumuloClient(), "table2", allMetadataAuths(), false); + } + + @Bean + TypeMetadataHelper alternateAuthsTypeHelper() { + return new TypeMetadataHelper(Collections.EMPTY_MAP, alternateAllMetadataAuths(), accumuloClient(), "table", alternateAllMetadataAuths(), false); + } + + @Bean + TypeMetadataHelper alternateTypeHelper() { + return new TypeMetadataHelper(Collections.EMPTY_MAP, alternateAllMetadataAuths(), accumuloClient(), "table2", alternateAllMetadataAuths(), false); + } + @Bean CompositeMetadataHelper compositeMetadataHelper() { return new CompositeMetadataHelper(accumuloClient(), "table", allMetadataAuths()); } + @Bean + CompositeMetadataHelper alternateTableCompositeHelper() { + return new CompositeMetadataHelper(accumuloClient(), "table2", allMetadataAuths()); + } + + @Bean + CompositeMetadataHelper alternateAuthsCompositeHelper() { + return new CompositeMetadataHelper(accumuloClient(), "table", alternateAllMetadataAuths()); + } + + @Bean + CompositeMetadataHelper alternateCompositeHelper() { + return new CompositeMetadataHelper(accumuloClient(), "table2", alternateAllMetadataAuths()); + } + @Bean AllFieldMetadataHelper allFieldMetadataHelper() { return new AllFieldMetadataHelper(typeMetadataHelper, compositeMetadataHelper, accumuloClient(), "table", allMetadataAuths(), allMetadataAuths()); } + @Bean + AllFieldMetadataHelper alternateTableAllFieldHelper() { + return new AllFieldMetadataHelper(alternateTableTypeHelper, alternateTableCompositeHelper, accumuloClient(), "table", allMetadataAuths(), allMetadataAuths()); + } + + @Bean + AllFieldMetadataHelper alternateAuthsAllFieldHelper() { + return new AllFieldMetadataHelper(alternateAuthsTypeHelper, alternateAuthsCompositeHelper, accumuloClient(), "table", allMetadataAuths(), allMetadataAuths()); + } + + @Bean + AllFieldMetadataHelper alternateAllFieldHelper() { + return new AllFieldMetadataHelper(alternateTypeHelper, alternateCompositeHelper, accumuloClient(), "table", allMetadataAuths(), allMetadataAuths()); + } + @Bean MetadataHelper metadataHelper() { return new MetadataHelper(allFieldMetadataHelper, allMetadataAuths(), accumuloClient(), "table", allMetadataAuths(), allMetadataAuths()); @@ -122,18 +203,18 @@ MetadataHelper metadataHelper() { @Bean MetadataHelper alternateTableMetadataHelper() { - return new MetadataHelper(allFieldMetadataHelper, allMetadataAuths(), accumuloClient(), "table2", allMetadataAuths(), allMetadataAuths()); + return new MetadataHelper(alternateTableAllFieldHelper, allMetadataAuths(), accumuloClient(), "table2", allMetadataAuths(), allMetadataAuths()); } @Bean MetadataHelper alternateAuthsHelper() { - return new MetadataHelper(allFieldMetadataHelper, alternateAllMetadataAuths(), accumuloClient(), "table", alternateAllMetadataAuths(), + return new MetadataHelper(alternateAuthsAllFieldHelper, alternateAllMetadataAuths(), accumuloClient(), "table", alternateAllMetadataAuths(), alternateAllMetadataAuths()); } @Bean MetadataHelper alternateHelper() { - return new MetadataHelper(allFieldMetadataHelper, alternateAllMetadataAuths(), accumuloClient(), "table2", alternateAllMetadataAuths(), + return new MetadataHelper(alternateAllFieldHelper, alternateAllMetadataAuths(), accumuloClient(), "table2", alternateAllMetadataAuths(), alternateAllMetadataAuths()); } @@ -143,6 +224,7 @@ TestCache testCache() { } } + // inject all beans into the test class so they will be spring cache enabled @Autowired @Qualifier("metadataHelper") private MetadataHelper metadataHelper; @@ -162,9 +244,31 @@ TestCache testCache() { @Autowired private AccumuloClient accumuloClient; + @Autowired + private CacheManager cacheManager; + @Before public void setup() { + // ensure @Autowired is working assertNotNull(accumuloClient); + assertNotNull(metadataHelper); + assertNotNull(alternateTableMetadataHelper); + assertNotNull(alternateAuthsHelper); + assertNotNull(alternateHelper); + assertNotNull(cacheManager); + + // TODO delete this later +// expect(accumuloClient.instanceOperations()).andAnswer(new IAnswer() { +// @Override +// public InstanceOperations answer() throws Throwable { +// return null; +// } +// }).anyTimes(); + } + + // wrapper so I can track access while testing + private AccumuloClient getAccumuloClient() { + return accumuloClient; } // this test will fail until MetadataHelper.getQueryModelNames uses the right key @@ -702,25 +806,298 @@ public void getCompositeToFieldMap_param2_test() throws TableNotFoundException, verifyAll(); } + // key = "{#root.target.auths,#modelTableName}" + private MethodParamsAndExpectations[] getExpectedQueryModelNames() { + MethodParamsAndExpectations base = new MethodParamsAndExpectations(metadataHelper, new Object[] {"modelName"}) { + protected void expect() throws TableNotFoundException { + Scanner s = createMock(Scanner.class); + List> entries = new ArrayList<>(); + expectScanner("modelName", s, entries); + } + }; + + MethodParamsAndExpectations newModelName = new MethodParamsAndExpectations(metadataHelper, new Object[] {"newModelName"}) { + protected void expect() throws TableNotFoundException { + Scanner s = createMock(Scanner.class); + List> entries = new ArrayList<>(); + expectScanner("newModelName", s, entries); + } + }; + + MethodParamsAndExpectations altAuths = new MethodParamsAndExpectations(alternateAuthsHelper, new Object[] {"modelName"}) { + protected void expect() throws TableNotFoundException { + Scanner s = createMock(Scanner.class); + List> entries = new ArrayList<>(); + expectScanner("modelName", s, entries); + } + }; + + // @formatter:off + return new MethodParamsAndExpectations[] { + // base case + base, + // different metadataTable + new MethodParamsAndExpectations(alternateTableMetadataHelper, new Object[] {"modelName"}, base), + // different auths, not cached + altAuths, + // different table and auths, but should be cached by the previous alt auths call + new MethodParamsAndExpectations(alternateHelper, new Object[] {"modelName"}, altAuths), + // different modelTableName + newModelName, + // same modelTableName repeated + new MethodParamsAndExpectations(metadataHelper, new Object[] {"newModelName"}, newModelName) + }; + // @formatter:on + } + + // key = "{#root.target.auths,#table}" + private MethodParamsAndExpectations[] getExpectedFacets() { + MethodParamsAndExpectations base = new MethodParamsAndExpectations(metadataHelper, new Object[] {"t1"}) { + protected void expect() throws TableNotFoundException { + Scanner s = createMock(Scanner.class); + List> entries = new ArrayList<>(); + expectScanner("t1", s, entries); + } + }; + + MethodParamsAndExpectations newTableName = new MethodParamsAndExpectations(metadataHelper, new Object[] {"t2"}) { + protected void expect() throws TableNotFoundException { + Scanner s = createMock(Scanner.class); + List> entries = new ArrayList<>(); + expectScanner("t2", s, entries); + } + }; + + MethodParamsAndExpectations altAuths = new MethodParamsAndExpectations(alternateAuthsHelper, new Object[] {"t1"}) { + protected void expect() throws TableNotFoundException { + Scanner s = createMock(Scanner.class); + List> entries = new ArrayList<>(); + expectScanner("t1", s, entries); + } + }; + + // @formatter:off + return new MethodParamsAndExpectations[] { + // base case + base, + // different metadataTable + new MethodParamsAndExpectations(alternateTableMetadataHelper, new Object[] {"t1"}, base), + // different auths, not cached + altAuths, + // different table and auths, but should be cached by the previous alt auths call + new MethodParamsAndExpectations(alternateHelper, new Object[] {"t1"}, altAuths), + // different modelTableName + newTableName, + // same modelTableName repeated + new MethodParamsAndExpectations(metadataHelper, new Object[] {"t2"}, newTableName) + }; + // @formatter:on + } + + // validate key = "{#root.target.metadataTableName}" + private MethodParamsAndExpectations[] getNoArgMetadataTableNameVariations () { + return getNoArgMetadataTableNameVariations(null); + } + + private MethodParamsAndExpectations[] getNoArgMetadataTableNameVariations (ExtraExpectation extra) { + MethodParamsAndExpectations base = new MethodParamsAndExpectations(metadataHelper, new Object[] {}) { + protected void expect() throws Exception { + Scanner s = createMock(Scanner.class); + List> entries = new ArrayList<>(); + expectScanner("table", s, entries); + + if (extra != null) { + extra.extraExpectation(s); + } + } + }; + + MethodParamsAndExpectations altTable = new MethodParamsAndExpectations(alternateTableMetadataHelper, new Object[] {}) { + protected void expect() throws Exception { + Scanner s = createMock(Scanner.class); + List> entries = new ArrayList<>(); + expectScanner("table2", s, entries); + + if (extra != null) { + extra.extraExpectation(s); + } + } + }; + + // @formatter:off + return new MethodParamsAndExpectations[] { + // base case + base, + // different metadataTable + altTable, + // different auths + new MethodParamsAndExpectations(alternateAuthsHelper, new Object[] {}, base), + // different table and auths, + new MethodParamsAndExpectations(alternateHelper, new Object[] {}, altTable), + new MethodParamsAndExpectations(metadataHelper, new Object[] {}, base), + new MethodParamsAndExpectations(alternateTableMetadataHelper, new Object[] {}, altTable), + new MethodParamsAndExpectations(alternateAuthsHelper, new Object[] {}, base), + new MethodParamsAndExpectations(alternateHelper, new Object[] {}, altTable), + }; + // @formatter:on + } + + // validate key = "{#root.target.auths,#root.target.metadataTableName}" + private MethodParamsAndExpectations[] getNoArgAuthMetadataTableNameVariations (ExtraExpectation extra) { + MethodParamsAndExpectations base = new MethodParamsAndExpectations(metadataHelper, new Object[] {}) { + protected void expect() throws Exception { + Scanner s = createMock(Scanner.class); + List> entries = new ArrayList<>(); + expectScanner("table", s, entries); + + if (extra != null) { + extra.extraExpectation(s); + } + } + }; + + MethodParamsAndExpectations altAuths = new MethodParamsAndExpectations(alternateAuthsHelper, new Object[] {}) { + protected void expect() throws Exception { + Scanner s = createMock(Scanner.class); + List> entries = new ArrayList<>(); + expectScanner("table", s, entries); + + if (extra != null) { + extra.extraExpectation(s); + } + } + }; + + MethodParamsAndExpectations altTable = new MethodParamsAndExpectations(alternateTableMetadataHelper, new Object[] {}) { + protected void expect() throws Exception { + Scanner s = createMock(Scanner.class); + List> entries = new ArrayList<>(); + expectScanner("table2", s, entries); + + if (extra != null) { + extra.extraExpectation(s); + } + } + }; + + MethodParamsAndExpectations altTableAndAuths = new MethodParamsAndExpectations(alternateHelper, new Object[] {}) { + protected void expect() throws Exception { + Scanner s = createMock(Scanner.class); + List> entries = new ArrayList<>(); + expectScanner("table2", s, entries); + + if (extra != null) { + extra.extraExpectation(s); + } + } + }; + + // @formatter:off + return new MethodParamsAndExpectations[] { + // base case + base, + // different metadataTable + altTable, + // different auths + altAuths, + // different table and auths, + altTableAndAuths, + new MethodParamsAndExpectations(metadataHelper, new Object[] {}, base), + new MethodParamsAndExpectations(alternateTableMetadataHelper, new Object[] {}, altTable), + new MethodParamsAndExpectations(alternateAuthsHelper, new Object[] {}, altAuths), + new MethodParamsAndExpectations(alternateHelper, new Object[] {}, altTableAndAuths), + }; + // @formatter:on + } + + // validate key = "{#root.target.auths,#root.target.metadataTableName}" + private MethodParamsAndExpectations[] getAuthMetadataTableNameVariations (Args[] args, boolean argsInKey) { + if (args == null) { + args = new Args[] {new Args(new Object[]{})}; + } + List results = new ArrayList<>(); + + boolean first = true; + + MethodParamsAndExpectations base = null; + MethodParamsAndExpectations altAuths = null; + MethodParamsAndExpectations altTable = null; + MethodParamsAndExpectations altTableAndAuths = null; + + for (Args a : args ) { + if (first || argsInKey) { + base = new MethodParamsAndExpectations(metadataHelper, a.args) { + protected void expect() throws TableNotFoundException { + Scanner s = createMock(Scanner.class); + List> entries = new ArrayList<>(); + expectScanner("table", s, entries); + } + }; + + altAuths = new MethodParamsAndExpectations(alternateAuthsHelper, a.args) { + protected void expect() throws TableNotFoundException { + Scanner s = createMock(Scanner.class); + List> entries = new ArrayList<>(); + expectScanner("table", s, entries); + } + }; + + altTable = new MethodParamsAndExpectations(alternateTableMetadataHelper, a.args) { + protected void expect() throws TableNotFoundException { + Scanner s = createMock(Scanner.class); + List> entries = new ArrayList<>(); + expectScanner("table2", s, entries); + } + }; + + altTableAndAuths = new MethodParamsAndExpectations(alternateHelper, a.args) { + protected void expect() throws TableNotFoundException { + Scanner s = createMock(Scanner.class); + List> entries = new ArrayList<>(); + expectScanner("table2", s, entries); + } + }; + + results.add(base); + results.add(altTable); + results.add(altAuths); + results.add(altTableAndAuths); + + first = false; + } + + results.add(new MethodParamsAndExpectations(metadataHelper, a.args, base)); + results.add(new MethodParamsAndExpectations(alternateTableMetadataHelper, a.args, altTable)); + results.add(new MethodParamsAndExpectations(alternateAuthsHelper, a.args, altAuths)); + results.add(new MethodParamsAndExpectations(alternateHelper, a.args, altTableAndAuths)); + } + + return results.toArray(new MethodParamsAndExpectations[0]); + } + // test all methods against all objects @Test public void allTest() throws Exception { Map methodToParams = new HashMap<>(); - methodToParams.put("getQueryModelNames", - new MethodParamsAndExpectations[] {new MethodParamsAndExpectations(metadataHelper, new Object[] {"modelName"}) { - protected void expect() throws TableNotFoundException { - Scanner s = createMock(Scanner.class); - List> entries = new ArrayList<>(); - expectScanner("modelName", s, entries); - } - }, new MethodParamsAndExpectations(alternateTableMetadataHelper, new Object[] {"modelName"}), - new MethodParamsAndExpectations(alternateAuthsHelper, new Object[] {"modelName"}) { - protected void expect() throws TableNotFoundException { - Scanner s = createMock(Scanner.class); - List> entries = new ArrayList<>(); - expectScanner("modelName", s, entries); - } - }, new MethodParamsAndExpectations(alternateHelper, new Object[] {"modelName"}),}); + methodToParams.put("getQueryModelNames", getExpectedQueryModelNames()); + methodToParams.put("getFacets", getExpectedFacets()); + methodToParams.put("getTermCounts", getNoArgAuthMetadataTableNameVariations(null)); + methodToParams.put("getTermCountsWithRootAuths", getNoArgMetadataTableNameVariations((s)-> { + SecurityOperations so = createMock(SecurityOperations.class); + expect(getAccumuloClient().securityOperations()).andReturn(so); + expect(getAccumuloClient().whoami()).andReturn("dwv"); + expect(so.getUserAuthorizations("dwv")).andReturn(new Authorizations("steve")); + })); + methodToParams.put("getAllNormalized", getNoArgAuthMetadataTableNameVariations(null)); + methodToParams.put("getEdges", getNoArgAuthMetadataTableNameVariations((s) -> { + s.addScanIterator(anyObject()); + s.addScanIterator(anyObject()); + })); + methodToParams.put("getIndexOnlyFields", getAuthMetadataTableNameVariations(new Args[]{new Args(new Object[] {null}), new Args(new Object[] {Collections.emptySet()}), new Args(new Object[] {Collections.singleton("filter")})}, false)); + methodToParams.put("getTermFrequencyFields", getAuthMetadataTableNameVariations(new Args[] {new Args(new Object[] {null})}, true)); + // no-op, but is a public method so needs an entry + methodToParams.put("toString", new MethodParamsAndExpectations[]{}); + methodToParams.put("getAllFields", getAuthMetadataTableNameVariations(new Args[]{new Args(new Object[] {null}), new Args(new Object[] {Collections.emptySet()}), new Args(new Object[] {Collections.singleton("filter")})}, false)); // test that all methods are tested for (Method method : MetadataHelper.class.getDeclaredMethods()) { @@ -728,8 +1105,20 @@ protected void expect() throws TableNotFoundException { // no need to verify statics continue; } + + // TODO delete this after all methods are added + // isolate a single test +// if (!method.getName().equals("getIndexOnlyFields")) { +// continue; +// } + // verify all non static methods are tested assertNotNull("MetadataHelper method '" + method.getName() + "' not tested", methodToParams.get(method.getName())); + // TODO delete this after testing + System.out.println(method.getName() + ": Testing"); + + // clear spring caches to prevent cross contamination between calls + cleanupCache(); // setup expectations for (MethodParamsAndExpectations expectation : methodToParams.get(method.getName())) { @@ -737,29 +1126,64 @@ protected void expect() throws TableNotFoundException { } // replay expectations - EasyMock.replay(accumuloClient); + EasyMock.replay(getAccumuloClient()); replayAll(); // invoke method on all objects in order for (MethodParamsAndExpectations expectation : methodToParams.get(method.getName())) { - method.invoke(expectation.get(), expectation.args); + Object result = method.invoke(expectation.get(), expectation.args); + expectation.setResult(result); + + // test cache value matches if it was supposed to be a cache hit + if (expectation.cachedOf != null) { + assertEquals(expectation.cachedOf.getResult(), result); + } } + EasyMock.verify(getAccumuloClient()); verifyAll(); + System.out.println(method.getName() + ": OK"); + // reset mocks to use again - EasyMock.reset(accumuloClient); + EasyMock.reset(getAccumuloClient()); resetAll(); } } + // force cache clear + private void cleanupCache() { + for(String cacheName : cacheManager.getCacheNames()) { + cacheManager.getCache(cacheName).clear(); + } + } + + private static class Args { + public Object[] args; + + public Args(Object[] args) { + this.args = args; + } + } + + private interface ExtraExpectation { + void extraExpectation(Scanner s) throws Exception; + } + private static class MethodParamsAndExpectations { public MetadataHelper metadataHelper; public Object[] args; + public MethodParamsAndExpectations cachedOf; + public Object result = null; private MethodParamsAndExpectations(MetadataHelper helper, Object[] args) { + this(helper, args, null); + } + + private MethodParamsAndExpectations(MetadataHelper helper, Object[] args, MethodParamsAndExpectations cachedOf) { metadataHelper = helper; this.args = args; + this.cachedOf = cachedOf; } protected void expect() throws Exception {} @@ -767,6 +1191,14 @@ protected void expect() throws Exception {} public Object get() { return metadataHelper; } + + public void setResult(Object result) { + this.result = result; + } + + public Object getResult() { + return this.result; + } } /** @@ -778,7 +1210,7 @@ public Object get() { * @throws TableNotFoundException */ private void expectScanner(String table, Scanner mockScanner, List> entries) throws TableNotFoundException { - expect(accumuloClient.createScanner(eq(table), EasyMock.anyObject())).andReturn(mockScanner); + expect(getAccumuloClient().createScanner(eq(table), EasyMock.anyObject())).andReturn(mockScanner); mockScanner.setRange(anyObject()); mockScanner.fetchColumnFamily(isA(Text.class)); expectLastCall().anyTimes(); From c4046fd94c061f4a5ee9addcde360226ecc414ab Mon Sep 17 00:00:00 2001 From: FineAndDandy Date: Fri, 15 Nov 2024 16:28:18 +0000 Subject: [PATCH 06/12] assign correct table and auths to allFieldMetadataHelper for proper cache functionality --- .../java/datawave/query/util/MetadataHelperCacheTest.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/warehouse/query-core/src/test/java/datawave/query/util/MetadataHelperCacheTest.java b/warehouse/query-core/src/test/java/datawave/query/util/MetadataHelperCacheTest.java index a981c246e5..6945dea000 100644 --- a/warehouse/query-core/src/test/java/datawave/query/util/MetadataHelperCacheTest.java +++ b/warehouse/query-core/src/test/java/datawave/query/util/MetadataHelperCacheTest.java @@ -183,17 +183,17 @@ AllFieldMetadataHelper allFieldMetadataHelper() { @Bean AllFieldMetadataHelper alternateTableAllFieldHelper() { - return new AllFieldMetadataHelper(alternateTableTypeHelper, alternateTableCompositeHelper, accumuloClient(), "table", allMetadataAuths(), allMetadataAuths()); + return new AllFieldMetadataHelper(alternateTableTypeHelper, alternateTableCompositeHelper, accumuloClient(), "table2", allMetadataAuths(), allMetadataAuths()); } @Bean AllFieldMetadataHelper alternateAuthsAllFieldHelper() { - return new AllFieldMetadataHelper(alternateAuthsTypeHelper, alternateAuthsCompositeHelper, accumuloClient(), "table", allMetadataAuths(), allMetadataAuths()); + return new AllFieldMetadataHelper(alternateAuthsTypeHelper, alternateAuthsCompositeHelper, accumuloClient(), "table", alternateAllMetadataAuths(), alternateAllMetadataAuths()); } @Bean AllFieldMetadataHelper alternateAllFieldHelper() { - return new AllFieldMetadataHelper(alternateTypeHelper, alternateCompositeHelper, accumuloClient(), "table", allMetadataAuths(), allMetadataAuths()); + return new AllFieldMetadataHelper(alternateTypeHelper, alternateCompositeHelper, accumuloClient(), "table2", alternateAllMetadataAuths(), alternateAllMetadataAuths()); } @Bean From d93de3e9e485a87f7830e16937b1a37852b43dbb Mon Sep 17 00:00:00 2001 From: FineAndDandy Date: Fri, 15 Nov 2024 20:38:00 +0000 Subject: [PATCH 07/12] add getNonEventFields test --- .../query/util/MetadataHelperCacheTest.java | 160 +++++++++++++++--- 1 file changed, 132 insertions(+), 28 deletions(-) diff --git a/warehouse/query-core/src/test/java/datawave/query/util/MetadataHelperCacheTest.java b/warehouse/query-core/src/test/java/datawave/query/util/MetadataHelperCacheTest.java index 6945dea000..e3808c613b 100644 --- a/warehouse/query-core/src/test/java/datawave/query/util/MetadataHelperCacheTest.java +++ b/warehouse/query-core/src/test/java/datawave/query/util/MetadataHelperCacheTest.java @@ -809,7 +809,7 @@ public void getCompositeToFieldMap_param2_test() throws TableNotFoundException, // key = "{#root.target.auths,#modelTableName}" private MethodParamsAndExpectations[] getExpectedQueryModelNames() { MethodParamsAndExpectations base = new MethodParamsAndExpectations(metadataHelper, new Object[] {"modelName"}) { - protected void expect() throws TableNotFoundException { + public void expect() throws TableNotFoundException { Scanner s = createMock(Scanner.class); List> entries = new ArrayList<>(); expectScanner("modelName", s, entries); @@ -817,7 +817,7 @@ protected void expect() throws TableNotFoundException { }; MethodParamsAndExpectations newModelName = new MethodParamsAndExpectations(metadataHelper, new Object[] {"newModelName"}) { - protected void expect() throws TableNotFoundException { + public void expect() throws TableNotFoundException { Scanner s = createMock(Scanner.class); List> entries = new ArrayList<>(); expectScanner("newModelName", s, entries); @@ -825,7 +825,7 @@ protected void expect() throws TableNotFoundException { }; MethodParamsAndExpectations altAuths = new MethodParamsAndExpectations(alternateAuthsHelper, new Object[] {"modelName"}) { - protected void expect() throws TableNotFoundException { + public void expect() throws TableNotFoundException { Scanner s = createMock(Scanner.class); List> entries = new ArrayList<>(); expectScanner("modelName", s, entries); @@ -853,7 +853,7 @@ protected void expect() throws TableNotFoundException { // key = "{#root.target.auths,#table}" private MethodParamsAndExpectations[] getExpectedFacets() { MethodParamsAndExpectations base = new MethodParamsAndExpectations(metadataHelper, new Object[] {"t1"}) { - protected void expect() throws TableNotFoundException { + public void expect() throws TableNotFoundException { Scanner s = createMock(Scanner.class); List> entries = new ArrayList<>(); expectScanner("t1", s, entries); @@ -861,7 +861,7 @@ protected void expect() throws TableNotFoundException { }; MethodParamsAndExpectations newTableName = new MethodParamsAndExpectations(metadataHelper, new Object[] {"t2"}) { - protected void expect() throws TableNotFoundException { + public void expect() throws TableNotFoundException { Scanner s = createMock(Scanner.class); List> entries = new ArrayList<>(); expectScanner("t2", s, entries); @@ -869,7 +869,7 @@ protected void expect() throws TableNotFoundException { }; MethodParamsAndExpectations altAuths = new MethodParamsAndExpectations(alternateAuthsHelper, new Object[] {"t1"}) { - protected void expect() throws TableNotFoundException { + public void expect() throws TableNotFoundException { Scanner s = createMock(Scanner.class); List> entries = new ArrayList<>(); expectScanner("t1", s, entries); @@ -901,7 +901,7 @@ private MethodParamsAndExpectations[] getNoArgMetadataTableNameVariations () { private MethodParamsAndExpectations[] getNoArgMetadataTableNameVariations (ExtraExpectation extra) { MethodParamsAndExpectations base = new MethodParamsAndExpectations(metadataHelper, new Object[] {}) { - protected void expect() throws Exception { + public void expect() throws Exception { Scanner s = createMock(Scanner.class); List> entries = new ArrayList<>(); expectScanner("table", s, entries); @@ -913,7 +913,7 @@ protected void expect() throws Exception { }; MethodParamsAndExpectations altTable = new MethodParamsAndExpectations(alternateTableMetadataHelper, new Object[] {}) { - protected void expect() throws Exception { + public void expect() throws Exception { Scanner s = createMock(Scanner.class); List> entries = new ArrayList<>(); expectScanner("table2", s, entries); @@ -945,7 +945,7 @@ protected void expect() throws Exception { // validate key = "{#root.target.auths,#root.target.metadataTableName}" private MethodParamsAndExpectations[] getNoArgAuthMetadataTableNameVariations (ExtraExpectation extra) { MethodParamsAndExpectations base = new MethodParamsAndExpectations(metadataHelper, new Object[] {}) { - protected void expect() throws Exception { + public void expect() throws Exception { Scanner s = createMock(Scanner.class); List> entries = new ArrayList<>(); expectScanner("table", s, entries); @@ -957,7 +957,7 @@ protected void expect() throws Exception { }; MethodParamsAndExpectations altAuths = new MethodParamsAndExpectations(alternateAuthsHelper, new Object[] {}) { - protected void expect() throws Exception { + public void expect() throws Exception { Scanner s = createMock(Scanner.class); List> entries = new ArrayList<>(); expectScanner("table", s, entries); @@ -969,7 +969,7 @@ protected void expect() throws Exception { }; MethodParamsAndExpectations altTable = new MethodParamsAndExpectations(alternateTableMetadataHelper, new Object[] {}) { - protected void expect() throws Exception { + public void expect() throws Exception { Scanner s = createMock(Scanner.class); List> entries = new ArrayList<>(); expectScanner("table2", s, entries); @@ -981,7 +981,7 @@ protected void expect() throws Exception { }; MethodParamsAndExpectations altTableAndAuths = new MethodParamsAndExpectations(alternateHelper, new Object[] {}) { - protected void expect() throws Exception { + public void expect() throws Exception { Scanner s = createMock(Scanner.class); List> entries = new ArrayList<>(); expectScanner("table2", s, entries); @@ -1010,8 +1010,12 @@ protected void expect() throws Exception { // @formatter:on } - // validate key = "{#root.target.auths,#root.target.metadataTableName}" private MethodParamsAndExpectations[] getAuthMetadataTableNameVariations (Args[] args, boolean argsInKey) { + return getAuthMetadataTableNameVariations(args, argsInKey, true); + } + + // validate key = "{#root.target.auths,#root.target.metadataTableName}" + private MethodParamsAndExpectations[] getAuthMetadataTableNameVariations (Args[] args, boolean argsInKey, boolean exactMatch) { if (args == null) { args = new Args[] {new Args(new Object[]{})}; } @@ -1027,7 +1031,7 @@ private MethodParamsAndExpectations[] getAuthMetadataTableNameVariations (Args[] for (Args a : args ) { if (first || argsInKey) { base = new MethodParamsAndExpectations(metadataHelper, a.args) { - protected void expect() throws TableNotFoundException { + public void expect() throws TableNotFoundException { Scanner s = createMock(Scanner.class); List> entries = new ArrayList<>(); expectScanner("table", s, entries); @@ -1035,7 +1039,7 @@ protected void expect() throws TableNotFoundException { }; altAuths = new MethodParamsAndExpectations(alternateAuthsHelper, a.args) { - protected void expect() throws TableNotFoundException { + public void expect() throws TableNotFoundException { Scanner s = createMock(Scanner.class); List> entries = new ArrayList<>(); expectScanner("table", s, entries); @@ -1043,7 +1047,7 @@ protected void expect() throws TableNotFoundException { }; altTable = new MethodParamsAndExpectations(alternateTableMetadataHelper, a.args) { - protected void expect() throws TableNotFoundException { + public void expect() throws TableNotFoundException { Scanner s = createMock(Scanner.class); List> entries = new ArrayList<>(); expectScanner("table2", s, entries); @@ -1051,7 +1055,7 @@ protected void expect() throws TableNotFoundException { }; altTableAndAuths = new MethodParamsAndExpectations(alternateHelper, a.args) { - protected void expect() throws TableNotFoundException { + public void expect() throws TableNotFoundException { Scanner s = createMock(Scanner.class); List> entries = new ArrayList<>(); expectScanner("table2", s, entries); @@ -1066,10 +1070,65 @@ protected void expect() throws TableNotFoundException { first = false; } - results.add(new MethodParamsAndExpectations(metadataHelper, a.args, base)); - results.add(new MethodParamsAndExpectations(alternateTableMetadataHelper, a.args, altTable)); - results.add(new MethodParamsAndExpectations(alternateAuthsHelper, a.args, altAuths)); - results.add(new MethodParamsAndExpectations(alternateHelper, a.args, altTableAndAuths)); + results.add(new MethodParamsAndExpectations(metadataHelper, a.args, base, exactMatch)); + results.add(new MethodParamsAndExpectations(alternateTableMetadataHelper, a.args, altTable, exactMatch)); + results.add(new MethodParamsAndExpectations(alternateAuthsHelper, a.args, altAuths, exactMatch)); + results.add(new MethodParamsAndExpectations(alternateHelper, a.args, altTableAndAuths, exactMatch)); + } + + return results.toArray(new MethodParamsAndExpectations[0]); + } + + private MethodParamsAndExpectations getWithExpectation(MetadataHelper helper, Object[] args, MethodParamsAndExpectations cachedObject, boolean exactMatch, Expectation expectation) { + return new MethodParamsAndExpectations(helper, args, cachedObject, exactMatch) { + public void expect() throws Exception { + expectation.expect(); + } + }; + } + + // this is a special case because the cache layers are different on the dependent methods if they come into line this can be folded into the generic expectation setter + // getTermFrequencyFields and getCompositeToFieldMap cache on key = auths,metadataTable,ingestTypeFilers + // getIndexOnlyFields caches on key = auths,metadataTable + private MethodParamsAndExpectations[] getNonEventFieldExpectations(Args[] args) { + List results = new ArrayList<>(); + + MethodParamsAndExpectations base = getWithExpectation(metadataHelper, args[0].args, null, false, new ScannerExpectation("table", 3)); + MethodParamsAndExpectations baseAltTable = getWithExpectation(alternateTableMetadataHelper, args[0].args, null, false, new ScannerExpectation("table2", 3)); + MethodParamsAndExpectations baseAltAuths = getWithExpectation(alternateAuthsHelper, args[0].args, null, false, new ScannerExpectation("table", 3)); + MethodParamsAndExpectations baseAlt = getWithExpectation(alternateHelper, args[0].args, null, false, new ScannerExpectation("table2", 3)); + + // no cache hits across any variation here + results.add(base); + results.add(baseAltTable); + results.add(baseAltAuths); + results.add(baseAlt); + + // should have cache hits that match values on all replays, but getTermFrequencyFields() is a special case + // spring doesn't intercept cache calls within the same class. See https://spring.io/blog/2012/05/23/transactions-caching-and-aop-understanding-proxy-usage-in-spring + results.add(getWithExpectation(metadataHelper, args[0].args, base, false, new ScannerExpectation("table", 1))); + results.add(getWithExpectation(alternateTableMetadataHelper, args[0].args, baseAltTable, false, new ScannerExpectation("table2", 1))); + results.add(getWithExpectation(alternateAuthsHelper, args[0].args, baseAltAuths, false, new ScannerExpectation("table", 1))); + results.add(getWithExpectation(alternateHelper, args[0].args, baseAlt, false, new ScannerExpectation("table2", 1))); + + // after the first pass all the others should be the same + for (int i = 1; i < args.length; i++) { + // for each object test first call and cached call + base = getWithExpectation(metadataHelper, args[i].args, null, false, new ScannerExpectation("table", 2)); + results.add(base); + results.add(getWithExpectation(metadataHelper, args[i].args, base, false, new ScannerExpectation("table", 1))); + + baseAltTable = getWithExpectation(alternateTableMetadataHelper, args[i].args, null, false, new ScannerExpectation("table2", 2)); + results.add(baseAltTable); + results.add(getWithExpectation(alternateTableMetadataHelper, args[i].args, baseAltTable, false, new ScannerExpectation("table2", 1))); + + baseAltAuths = getWithExpectation(alternateAuthsHelper, args[i].args, null, false, new ScannerExpectation("table", 2)); + results.add(baseAltAuths); + results.add(getWithExpectation(alternateAuthsHelper, args[i].args, baseAltAuths, false, new ScannerExpectation("table", 1))); + + baseAlt = getWithExpectation(alternateHelper, args[i].args, null, false, new ScannerExpectation("table2", 2)); + results.add(baseAlt); + results.add(getWithExpectation(alternateHelper, args[i].args, baseAlt, false, new ScannerExpectation("table2", 1))); } return results.toArray(new MethodParamsAndExpectations[0]); @@ -1093,11 +1152,12 @@ public void allTest() throws Exception { s.addScanIterator(anyObject()); s.addScanIterator(anyObject()); })); - methodToParams.put("getIndexOnlyFields", getAuthMetadataTableNameVariations(new Args[]{new Args(new Object[] {null}), new Args(new Object[] {Collections.emptySet()}), new Args(new Object[] {Collections.singleton("filter")})}, false)); + methodToParams.put("getIndexOnlyFields", getAuthMetadataTableNameVariations(new Args[]{new Args(new Object[] {null}), new Args(new Object[] {Collections.emptySet()}), new Args(new Object[] {Collections.singleton("filter")})}, false, false)); methodToParams.put("getTermFrequencyFields", getAuthMetadataTableNameVariations(new Args[] {new Args(new Object[] {null})}, true)); // no-op, but is a public method so needs an entry methodToParams.put("toString", new MethodParamsAndExpectations[]{}); - methodToParams.put("getAllFields", getAuthMetadataTableNameVariations(new Args[]{new Args(new Object[] {null}), new Args(new Object[] {Collections.emptySet()}), new Args(new Object[] {Collections.singleton("filter")})}, false)); + methodToParams.put("getAllFields", getAuthMetadataTableNameVariations(new Args[]{new Args(new Object[] {null}), new Args(new Object[] {Collections.emptySet()}), new Args(new Object[] {Collections.singleton("filter")})}, false, false)); + methodToParams.put("getNonEventFields", getNonEventFieldExpectations(new Args[]{new Args(new Object[] {null}), new Args(new Object[] {Collections.emptySet()}), new Args(new Object[] {Collections.singleton("filter")})})); // test that all methods are tested for (Method method : MetadataHelper.class.getDeclaredMethods()) { @@ -1108,7 +1168,7 @@ public void allTest() throws Exception { // TODO delete this after all methods are added // isolate a single test -// if (!method.getName().equals("getIndexOnlyFields")) { +// if (!method.getName().equals("getNonEventFields")) { // continue; // } @@ -1136,7 +1196,11 @@ public void allTest() throws Exception { // test cache value matches if it was supposed to be a cache hit if (expectation.cachedOf != null) { - assertEquals(expectation.cachedOf.getResult(), result); + if (expectation.exactMatch) { + assertTrue("didn't get exact expected cached value", expectation.cachedOf.getResult() == result); + } else { + assertEquals("didn't get equivalent value", expectation.cachedOf.getResult(), result); + } } } @@ -1170,23 +1234,63 @@ private interface ExtraExpectation { void extraExpectation(Scanner s) throws Exception; } - private static class MethodParamsAndExpectations { + private interface Expectation { + void expect() throws Exception; + } + + private class ScannerExpectation implements Expectation { + + private final String table; + private final int count; + + private ScannerExpectation(String table, int count) { + this.table = table; + this.count = count; + } + + @Override + public void expect() throws Exception { + for (int i = 0; i < count; i++) { + Scanner s = createMock(Scanner.class); + List> entries = new ArrayList<>(); + expectScanner(table, s, entries); + } + } + } + + private static class MethodParamsAndExpectations implements Expectation { public MetadataHelper metadataHelper; public Object[] args; public MethodParamsAndExpectations cachedOf; public Object result = null; + public boolean exactMatch; + private Expectation expectation; private MethodParamsAndExpectations(MetadataHelper helper, Object[] args) { this(helper, args, null); } private MethodParamsAndExpectations(MetadataHelper helper, Object[] args, MethodParamsAndExpectations cachedOf) { - metadataHelper = helper; + this(helper, args, cachedOf, true); + } + + private MethodParamsAndExpectations(MetadataHelper helper, Object[] args, MethodParamsAndExpectations cachedOf, boolean exactMatch) { + this(helper, args, cachedOf, exactMatch, null); + } + + private MethodParamsAndExpectations(MetadataHelper helper, Object[] args, MethodParamsAndExpectations cachedOf, boolean exactMatch, Expectation expectation) { + this.metadataHelper = helper; this.args = args; this.cachedOf = cachedOf; + this.exactMatch = exactMatch; + this.expectation = expectation; } - protected void expect() throws Exception {} + public void expect() throws Exception { + if (expectation != null) { + expectation.expect(); + } + } public Object get() { return metadataHelper; From 74e1bab0d507d466e6a52b188d614e35b547f1c3 Mon Sep 17 00:00:00 2001 From: FineAndDandy Date: Fri, 15 Nov 2024 21:15:01 +0000 Subject: [PATCH 08/12] added test coverage --- .../query/util/MetadataHelperCacheTest.java | 97 ++++++++++++------- 1 file changed, 62 insertions(+), 35 deletions(-) diff --git a/warehouse/query-core/src/test/java/datawave/query/util/MetadataHelperCacheTest.java b/warehouse/query-core/src/test/java/datawave/query/util/MetadataHelperCacheTest.java index e3808c613b..5fb15feabf 100644 --- a/warehouse/query-core/src/test/java/datawave/query/util/MetadataHelperCacheTest.java +++ b/warehouse/query-core/src/test/java/datawave/query/util/MetadataHelperCacheTest.java @@ -183,17 +183,20 @@ AllFieldMetadataHelper allFieldMetadataHelper() { @Bean AllFieldMetadataHelper alternateTableAllFieldHelper() { - return new AllFieldMetadataHelper(alternateTableTypeHelper, alternateTableCompositeHelper, accumuloClient(), "table2", allMetadataAuths(), allMetadataAuths()); + return new AllFieldMetadataHelper(alternateTableTypeHelper, alternateTableCompositeHelper, accumuloClient(), "table2", allMetadataAuths(), + allMetadataAuths()); } @Bean AllFieldMetadataHelper alternateAuthsAllFieldHelper() { - return new AllFieldMetadataHelper(alternateAuthsTypeHelper, alternateAuthsCompositeHelper, accumuloClient(), "table", alternateAllMetadataAuths(), alternateAllMetadataAuths()); + return new AllFieldMetadataHelper(alternateAuthsTypeHelper, alternateAuthsCompositeHelper, accumuloClient(), "table", alternateAllMetadataAuths(), + alternateAllMetadataAuths()); } @Bean AllFieldMetadataHelper alternateAllFieldHelper() { - return new AllFieldMetadataHelper(alternateTypeHelper, alternateCompositeHelper, accumuloClient(), "table2", alternateAllMetadataAuths(), alternateAllMetadataAuths()); + return new AllFieldMetadataHelper(alternateTypeHelper, alternateCompositeHelper, accumuloClient(), "table2", alternateAllMetadataAuths(), + alternateAllMetadataAuths()); } @Bean @@ -258,12 +261,12 @@ public void setup() { assertNotNull(cacheManager); // TODO delete this later -// expect(accumuloClient.instanceOperations()).andAnswer(new IAnswer() { -// @Override -// public InstanceOperations answer() throws Throwable { -// return null; -// } -// }).anyTimes(); + // expect(accumuloClient.instanceOperations()).andAnswer(new IAnswer() { + // @Override + // public InstanceOperations answer() throws Throwable { + // return null; + // } + // }).anyTimes(); } // wrapper so I can track access while testing @@ -895,11 +898,11 @@ public void expect() throws TableNotFoundException { } // validate key = "{#root.target.metadataTableName}" - private MethodParamsAndExpectations[] getNoArgMetadataTableNameVariations () { + private MethodParamsAndExpectations[] getNoArgMetadataTableNameVariations() { return getNoArgMetadataTableNameVariations(null); } - private MethodParamsAndExpectations[] getNoArgMetadataTableNameVariations (ExtraExpectation extra) { + private MethodParamsAndExpectations[] getNoArgMetadataTableNameVariations(ExtraExpectation extra) { MethodParamsAndExpectations base = new MethodParamsAndExpectations(metadataHelper, new Object[] {}) { public void expect() throws Exception { Scanner s = createMock(Scanner.class); @@ -943,7 +946,7 @@ public void expect() throws Exception { } // validate key = "{#root.target.auths,#root.target.metadataTableName}" - private MethodParamsAndExpectations[] getNoArgAuthMetadataTableNameVariations (ExtraExpectation extra) { + private MethodParamsAndExpectations[] getNoArgAuthMetadataTableNameVariations(ExtraExpectation extra) { MethodParamsAndExpectations base = new MethodParamsAndExpectations(metadataHelper, new Object[] {}) { public void expect() throws Exception { Scanner s = createMock(Scanner.class); @@ -1010,14 +1013,14 @@ public void expect() throws Exception { // @formatter:on } - private MethodParamsAndExpectations[] getAuthMetadataTableNameVariations (Args[] args, boolean argsInKey) { + private MethodParamsAndExpectations[] getAuthMetadataTableNameVariations(Args[] args, boolean argsInKey) { return getAuthMetadataTableNameVariations(args, argsInKey, true); } // validate key = "{#root.target.auths,#root.target.metadataTableName}" - private MethodParamsAndExpectations[] getAuthMetadataTableNameVariations (Args[] args, boolean argsInKey, boolean exactMatch) { + private MethodParamsAndExpectations[] getAuthMetadataTableNameVariations(Args[] args, boolean argsInKey, boolean exactMatch) { if (args == null) { - args = new Args[] {new Args(new Object[]{})}; + args = new Args[] {new Args(new Object[] {})}; } List results = new ArrayList<>(); @@ -1028,12 +1031,12 @@ private MethodParamsAndExpectations[] getAuthMetadataTableNameVariations (Args[] MethodParamsAndExpectations altTable = null; MethodParamsAndExpectations altTableAndAuths = null; - for (Args a : args ) { + for (Args a : args) { if (first || argsInKey) { base = new MethodParamsAndExpectations(metadataHelper, a.args) { public void expect() throws TableNotFoundException { Scanner s = createMock(Scanner.class); - List> entries = new ArrayList<>(); + List> entries = new ArrayList<>(); expectScanner("table", s, entries); } }; @@ -1041,7 +1044,7 @@ public void expect() throws TableNotFoundException { altAuths = new MethodParamsAndExpectations(alternateAuthsHelper, a.args) { public void expect() throws TableNotFoundException { Scanner s = createMock(Scanner.class); - List> entries = new ArrayList<>(); + List> entries = new ArrayList<>(); expectScanner("table", s, entries); } }; @@ -1049,7 +1052,7 @@ public void expect() throws TableNotFoundException { altTable = new MethodParamsAndExpectations(alternateTableMetadataHelper, a.args) { public void expect() throws TableNotFoundException { Scanner s = createMock(Scanner.class); - List> entries = new ArrayList<>(); + List> entries = new ArrayList<>(); expectScanner("table2", s, entries); } }; @@ -1057,7 +1060,7 @@ public void expect() throws TableNotFoundException { altTableAndAuths = new MethodParamsAndExpectations(alternateHelper, a.args) { public void expect() throws TableNotFoundException { Scanner s = createMock(Scanner.class); - List> entries = new ArrayList<>(); + List> entries = new ArrayList<>(); expectScanner("table2", s, entries); } }; @@ -1079,7 +1082,8 @@ public void expect() throws TableNotFoundException { return results.toArray(new MethodParamsAndExpectations[0]); } - private MethodParamsAndExpectations getWithExpectation(MetadataHelper helper, Object[] args, MethodParamsAndExpectations cachedObject, boolean exactMatch, Expectation expectation) { + private MethodParamsAndExpectations getWithExpectation(MetadataHelper helper, Object[] args, MethodParamsAndExpectations cachedObject, boolean exactMatch, + Expectation expectation) { return new MethodParamsAndExpectations(helper, args, cachedObject, exactMatch) { public void expect() throws Exception { expectation.expect(); @@ -1087,14 +1091,16 @@ public void expect() throws Exception { }; } - // this is a special case because the cache layers are different on the dependent methods if they come into line this can be folded into the generic expectation setter + // this is a special case because the cache layers are different on the dependent methods if they come into line this can be folded into the generic + // expectation setter // getTermFrequencyFields and getCompositeToFieldMap cache on key = auths,metadataTable,ingestTypeFilers // getIndexOnlyFields caches on key = auths,metadataTable private MethodParamsAndExpectations[] getNonEventFieldExpectations(Args[] args) { List results = new ArrayList<>(); MethodParamsAndExpectations base = getWithExpectation(metadataHelper, args[0].args, null, false, new ScannerExpectation("table", 3)); - MethodParamsAndExpectations baseAltTable = getWithExpectation(alternateTableMetadataHelper, args[0].args, null, false, new ScannerExpectation("table2", 3)); + MethodParamsAndExpectations baseAltTable = getWithExpectation(alternateTableMetadataHelper, args[0].args, null, false, + new ScannerExpectation("table2", 3)); MethodParamsAndExpectations baseAltAuths = getWithExpectation(alternateAuthsHelper, args[0].args, null, false, new ScannerExpectation("table", 3)); MethodParamsAndExpectations baseAlt = getWithExpectation(alternateHelper, args[0].args, null, false, new ScannerExpectation("table2", 3)); @@ -1105,7 +1111,8 @@ private MethodParamsAndExpectations[] getNonEventFieldExpectations(Args[] args) results.add(baseAlt); // should have cache hits that match values on all replays, but getTermFrequencyFields() is a special case - // spring doesn't intercept cache calls within the same class. See https://spring.io/blog/2012/05/23/transactions-caching-and-aop-understanding-proxy-usage-in-spring + // spring doesn't intercept cache calls within the same class. See + // https://spring.io/blog/2012/05/23/transactions-caching-and-aop-understanding-proxy-usage-in-spring results.add(getWithExpectation(metadataHelper, args[0].args, base, false, new ScannerExpectation("table", 1))); results.add(getWithExpectation(alternateTableMetadataHelper, args[0].args, baseAltTable, false, new ScannerExpectation("table2", 1))); results.add(getWithExpectation(alternateAuthsHelper, args[0].args, baseAltAuths, false, new ScannerExpectation("table", 1))); @@ -1137,11 +1144,12 @@ private MethodParamsAndExpectations[] getNonEventFieldExpectations(Args[] args) // test all methods against all objects @Test public void allTest() throws Exception { + // @formatter: off Map methodToParams = new HashMap<>(); methodToParams.put("getQueryModelNames", getExpectedQueryModelNames()); methodToParams.put("getFacets", getExpectedFacets()); methodToParams.put("getTermCounts", getNoArgAuthMetadataTableNameVariations(null)); - methodToParams.put("getTermCountsWithRootAuths", getNoArgMetadataTableNameVariations((s)-> { + methodToParams.put("getTermCountsWithRootAuths", getNoArgMetadataTableNameVariations((s) -> { SecurityOperations so = createMock(SecurityOperations.class); expect(getAccumuloClient().securityOperations()).andReturn(so); expect(getAccumuloClient().whoami()).andReturn("dwv"); @@ -1152,12 +1160,30 @@ public void allTest() throws Exception { s.addScanIterator(anyObject()); s.addScanIterator(anyObject()); })); - methodToParams.put("getIndexOnlyFields", getAuthMetadataTableNameVariations(new Args[]{new Args(new Object[] {null}), new Args(new Object[] {Collections.emptySet()}), new Args(new Object[] {Collections.singleton("filter")})}, false, false)); + methodToParams.put("getIndexOnlyFields", getAuthMetadataTableNameVariations(new Args[] {new Args(new Object[] {null}), + new Args(new Object[] {Collections.emptySet()}), new Args(new Object[] {Collections.singleton("filter")})}, false, false)); methodToParams.put("getTermFrequencyFields", getAuthMetadataTableNameVariations(new Args[] {new Args(new Object[] {null})}, true)); // no-op, but is a public method so needs an entry - methodToParams.put("toString", new MethodParamsAndExpectations[]{}); - methodToParams.put("getAllFields", getAuthMetadataTableNameVariations(new Args[]{new Args(new Object[] {null}), new Args(new Object[] {Collections.emptySet()}), new Args(new Object[] {Collections.singleton("filter")})}, false, false)); - methodToParams.put("getNonEventFields", getNonEventFieldExpectations(new Args[]{new Args(new Object[] {null}), new Args(new Object[] {Collections.emptySet()}), new Args(new Object[] {Collections.singleton("filter")})})); + methodToParams.put("toString", new MethodParamsAndExpectations[] {}); + methodToParams.put("getAllFields", getAuthMetadataTableNameVariations(new Args[] {new Args(new Object[] {null}), + new Args(new Object[] {Collections.emptySet()}), new Args(new Object[] {Collections.singleton("filter")})}, false, false)); + methodToParams.put("getNonEventFields", getNonEventFieldExpectations(new Args[] {new Args(new Object[] {null}), + new Args(new Object[] {Collections.emptySet()}), new Args(new Object[] {Collections.singleton("filter")})})); + methodToParams.put("isReverseIndexed", + getAuthMetadataTableNameVariations(new Args[] {new Args(new Object[] {"f1", Collections.emptySet()}), + new Args(new Object[] {"f1", Collections.singleton("filter")}), new Args(new Object[] {"f2", Collections.singleton("filter")})}, + true, false)); + methodToParams.put("isIndexed", + getAuthMetadataTableNameVariations(new Args[] {new Args(new Object[] {"f1", Collections.emptySet()}), + new Args(new Object[] {"f1", Collections.singleton("filter")}), new Args(new Object[] {"f2", Collections.singleton("filter")})}, + true, false)); + methodToParams.put("isTokenized", + getAuthMetadataTableNameVariations(new Args[] {new Args(new Object[] {"f1", Collections.emptySet()}), + new Args(new Object[] {"f1", Collections.singleton("filter")}), new Args(new Object[] {"f2", Collections.singleton("filter")})}, + true, false)); + methodToParams.put("getAllDatatypes", getNoArgAuthMetadataTableNameVariations(null)); + // methodToParams.put("getCompositeToFieldMap", getNoArgAuthMetadataTableNameVariations(null)); + // @formatter: on // test that all methods are tested for (Method method : MetadataHelper.class.getDeclaredMethods()) { @@ -1168,9 +1194,9 @@ public void allTest() throws Exception { // TODO delete this after all methods are added // isolate a single test -// if (!method.getName().equals("getNonEventFields")) { -// continue; -// } + // if (!method.getName().equals("isReverseIndexed")) { + // continue; + // } // verify all non static methods are tested assertNotNull("MetadataHelper method '" + method.getName() + "' not tested", methodToParams.get(method.getName())); @@ -1217,7 +1243,7 @@ public void allTest() throws Exception { // force cache clear private void cleanupCache() { - for(String cacheName : cacheManager.getCacheNames()) { + for (String cacheName : cacheManager.getCacheNames()) { cacheManager.getCache(cacheName).clear(); } } @@ -1252,7 +1278,7 @@ private ScannerExpectation(String table, int count) { public void expect() throws Exception { for (int i = 0; i < count; i++) { Scanner s = createMock(Scanner.class); - List> entries = new ArrayList<>(); + List> entries = new ArrayList<>(); expectScanner(table, s, entries); } } @@ -1278,7 +1304,8 @@ private MethodParamsAndExpectations(MetadataHelper helper, Object[] args, Method this(helper, args, cachedOf, exactMatch, null); } - private MethodParamsAndExpectations(MetadataHelper helper, Object[] args, MethodParamsAndExpectations cachedOf, boolean exactMatch, Expectation expectation) { + private MethodParamsAndExpectations(MetadataHelper helper, Object[] args, MethodParamsAndExpectations cachedOf, boolean exactMatch, + Expectation expectation) { this.metadataHelper = helper; this.args = args; this.cachedOf = cachedOf; From 6fe629414e7c8ecbd25a4f173b2f99fbbb1574b5 Mon Sep 17 00:00:00 2001 From: FineAndDandy Date: Fri, 15 Nov 2024 21:21:42 +0000 Subject: [PATCH 09/12] handle overloaded methods --- .../query/util/MetadataHelperCacheTest.java | 40 ++++++++++--------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/warehouse/query-core/src/test/java/datawave/query/util/MetadataHelperCacheTest.java b/warehouse/query-core/src/test/java/datawave/query/util/MetadataHelperCacheTest.java index 5fb15feabf..0c3f2d145b 100644 --- a/warehouse/query-core/src/test/java/datawave/query/util/MetadataHelperCacheTest.java +++ b/warehouse/query-core/src/test/java/datawave/query/util/MetadataHelperCacheTest.java @@ -1146,43 +1146,43 @@ private MethodParamsAndExpectations[] getNonEventFieldExpectations(Args[] args) public void allTest() throws Exception { // @formatter: off Map methodToParams = new HashMap<>(); - methodToParams.put("getQueryModelNames", getExpectedQueryModelNames()); - methodToParams.put("getFacets", getExpectedFacets()); - methodToParams.put("getTermCounts", getNoArgAuthMetadataTableNameVariations(null)); - methodToParams.put("getTermCountsWithRootAuths", getNoArgMetadataTableNameVariations((s) -> { + methodToParams.put("1.getQueryModelNames", getExpectedQueryModelNames()); + methodToParams.put("1.getFacets", getExpectedFacets()); + methodToParams.put("0.getTermCounts", getNoArgAuthMetadataTableNameVariations(null)); + methodToParams.put("0.getTermCountsWithRootAuths", getNoArgMetadataTableNameVariations((s) -> { SecurityOperations so = createMock(SecurityOperations.class); expect(getAccumuloClient().securityOperations()).andReturn(so); expect(getAccumuloClient().whoami()).andReturn("dwv"); expect(so.getUserAuthorizations("dwv")).andReturn(new Authorizations("steve")); })); - methodToParams.put("getAllNormalized", getNoArgAuthMetadataTableNameVariations(null)); - methodToParams.put("getEdges", getNoArgAuthMetadataTableNameVariations((s) -> { + methodToParams.put("0.getAllNormalized", getNoArgAuthMetadataTableNameVariations(null)); + methodToParams.put("0.getEdges", getNoArgAuthMetadataTableNameVariations((s) -> { s.addScanIterator(anyObject()); s.addScanIterator(anyObject()); })); - methodToParams.put("getIndexOnlyFields", getAuthMetadataTableNameVariations(new Args[] {new Args(new Object[] {null}), + methodToParams.put("1.getIndexOnlyFields", getAuthMetadataTableNameVariations(new Args[] {new Args(new Object[] {null}), new Args(new Object[] {Collections.emptySet()}), new Args(new Object[] {Collections.singleton("filter")})}, false, false)); - methodToParams.put("getTermFrequencyFields", getAuthMetadataTableNameVariations(new Args[] {new Args(new Object[] {null})}, true)); + methodToParams.put("1.getTermFrequencyFields", getAuthMetadataTableNameVariations(new Args[] {new Args(new Object[] {null})}, true)); // no-op, but is a public method so needs an entry - methodToParams.put("toString", new MethodParamsAndExpectations[] {}); - methodToParams.put("getAllFields", getAuthMetadataTableNameVariations(new Args[] {new Args(new Object[] {null}), + methodToParams.put("0.toString", new MethodParamsAndExpectations[] {}); + methodToParams.put("1.getAllFields", getAuthMetadataTableNameVariations(new Args[] {new Args(new Object[] {null}), new Args(new Object[] {Collections.emptySet()}), new Args(new Object[] {Collections.singleton("filter")})}, false, false)); - methodToParams.put("getNonEventFields", getNonEventFieldExpectations(new Args[] {new Args(new Object[] {null}), + methodToParams.put("1.getNonEventFields", getNonEventFieldExpectations(new Args[] {new Args(new Object[] {null}), new Args(new Object[] {Collections.emptySet()}), new Args(new Object[] {Collections.singleton("filter")})})); - methodToParams.put("isReverseIndexed", + methodToParams.put("2.isReverseIndexed", getAuthMetadataTableNameVariations(new Args[] {new Args(new Object[] {"f1", Collections.emptySet()}), new Args(new Object[] {"f1", Collections.singleton("filter")}), new Args(new Object[] {"f2", Collections.singleton("filter")})}, true, false)); - methodToParams.put("isIndexed", + methodToParams.put("2.isIndexed", getAuthMetadataTableNameVariations(new Args[] {new Args(new Object[] {"f1", Collections.emptySet()}), new Args(new Object[] {"f1", Collections.singleton("filter")}), new Args(new Object[] {"f2", Collections.singleton("filter")})}, true, false)); - methodToParams.put("isTokenized", + methodToParams.put("2.isTokenized", getAuthMetadataTableNameVariations(new Args[] {new Args(new Object[] {"f1", Collections.emptySet()}), new Args(new Object[] {"f1", Collections.singleton("filter")}), new Args(new Object[] {"f2", Collections.singleton("filter")})}, true, false)); - methodToParams.put("getAllDatatypes", getNoArgAuthMetadataTableNameVariations(null)); - // methodToParams.put("getCompositeToFieldMap", getNoArgAuthMetadataTableNameVariations(null)); + methodToParams.put("0.getAllDatatypes", getNoArgAuthMetadataTableNameVariations(null)); + methodToParams.put("0.getCompositeToFieldMap", getNoArgAuthMetadataTableNameVariations(null)); // @formatter: on // test that all methods are tested @@ -1198,8 +1198,10 @@ public void allTest() throws Exception { // continue; // } + String lookupName = method.getParameterCount() + "." + method.getName(); + // verify all non static methods are tested - assertNotNull("MetadataHelper method '" + method.getName() + "' not tested", methodToParams.get(method.getName())); + assertNotNull("MetadataHelper method '" + method.getName() + "' not tested", methodToParams.get(lookupName)); // TODO delete this after testing System.out.println(method.getName() + ": Testing"); @@ -1207,7 +1209,7 @@ public void allTest() throws Exception { cleanupCache(); // setup expectations - for (MethodParamsAndExpectations expectation : methodToParams.get(method.getName())) { + for (MethodParamsAndExpectations expectation : methodToParams.get(lookupName)) { expectation.expect(); } @@ -1216,7 +1218,7 @@ public void allTest() throws Exception { replayAll(); // invoke method on all objects in order - for (MethodParamsAndExpectations expectation : methodToParams.get(method.getName())) { + for (MethodParamsAndExpectations expectation : methodToParams.get(lookupName)) { Object result = method.invoke(expectation.get(), expectation.args); expectation.setResult(result); From 6fef7a090fe9defb75c93f60013940755c98328e Mon Sep 17 00:00:00 2001 From: FineAndDandy Date: Fri, 15 Nov 2024 22:35:51 +0000 Subject: [PATCH 10/12] Increase test coverage --- .../query/util/MetadataHelperCacheTest.java | 151 ++++++++++++++++-- 1 file changed, 138 insertions(+), 13 deletions(-) diff --git a/warehouse/query-core/src/test/java/datawave/query/util/MetadataHelperCacheTest.java b/warehouse/query-core/src/test/java/datawave/query/util/MetadataHelperCacheTest.java index 0c3f2d145b..4dc4a094ed 100644 --- a/warehouse/query-core/src/test/java/datawave/query/util/MetadataHelperCacheTest.java +++ b/warehouse/query-core/src/test/java/datawave/query/util/MetadataHelperCacheTest.java @@ -20,6 +20,9 @@ import java.util.Set; import java.util.concurrent.ExecutionException; +import datawave.IdentityDataType; +import datawave.data.type.DateType; +import datawave.data.type.GeoType; import org.apache.accumulo.core.client.AccumuloClient; import org.apache.accumulo.core.client.AccumuloException; import org.apache.accumulo.core.client.AccumuloSecurityException; @@ -1144,7 +1147,7 @@ private MethodParamsAndExpectations[] getNonEventFieldExpectations(Args[] args) // test all methods against all objects @Test public void allTest() throws Exception { - // @formatter: off + // @formatter:off Map methodToParams = new HashMap<>(); methodToParams.put("1.getQueryModelNames", getExpectedQueryModelNames()); methodToParams.put("1.getFacets", getExpectedFacets()); @@ -1163,8 +1166,6 @@ public void allTest() throws Exception { methodToParams.put("1.getIndexOnlyFields", getAuthMetadataTableNameVariations(new Args[] {new Args(new Object[] {null}), new Args(new Object[] {Collections.emptySet()}), new Args(new Object[] {Collections.singleton("filter")})}, false, false)); methodToParams.put("1.getTermFrequencyFields", getAuthMetadataTableNameVariations(new Args[] {new Args(new Object[] {null})}, true)); - // no-op, but is a public method so needs an entry - methodToParams.put("0.toString", new MethodParamsAndExpectations[] {}); methodToParams.put("1.getAllFields", getAuthMetadataTableNameVariations(new Args[] {new Args(new Object[] {null}), new Args(new Object[] {Collections.emptySet()}), new Args(new Object[] {Collections.singleton("filter")})}, false, false)); methodToParams.put("1.getNonEventFields", getNonEventFieldExpectations(new Args[] {new Args(new Object[] {null}), @@ -1177,13 +1178,133 @@ public void allTest() throws Exception { getAuthMetadataTableNameVariations(new Args[] {new Args(new Object[] {"f1", Collections.emptySet()}), new Args(new Object[] {"f1", Collections.singleton("filter")}), new Args(new Object[] {"f2", Collections.singleton("filter")})}, true, false)); - methodToParams.put("2.isTokenized", - getAuthMetadataTableNameVariations(new Args[] {new Args(new Object[] {"f1", Collections.emptySet()}), - new Args(new Object[] {"f1", Collections.singleton("filter")}), new Args(new Object[] {"f2", Collections.singleton("filter")})}, - true, false)); + methodToParams.put("2.isTokenized", getAuthMetadataTableNameVariations(new Args[] { + new Args(new Object[] {"f1", Collections.emptySet()}), + new Args(new Object[] {"f1", Collections.singleton("filter")}), + new Args(new Object[] {"f2", Collections.singleton("filter")})}, true, false)); methodToParams.put("0.getAllDatatypes", getNoArgAuthMetadataTableNameVariations(null)); methodToParams.put("0.getCompositeToFieldMap", getNoArgAuthMetadataTableNameVariations(null)); - // @formatter: on + methodToParams.put("1.getCompositeToFieldMap", getAuthMetadataTableNameVariations(new Args[] { + new Args(new Object[] {null}), + new Args(new Object[] {Collections.emptySet()}), + new Args(new Object[] {Collections.singleton("filter")})}, true, false)); + methodToParams.put("0.getCompositeTransitionDateMap", getNoArgAuthMetadataTableNameVariations(null)); + methodToParams.put("1.getCompositeTransitionDateMap", getAuthMetadataTableNameVariations(new Args[] { + new Args(new Object[] {null}), + new Args(new Object[] {Collections.emptySet()}), + new Args(new Object[] {Collections.singleton("filter")})}, true, false)); + methodToParams.put("0.getWhindexCreationDateMap", getNoArgAuthMetadataTableNameVariations(null)); + methodToParams.put("1.getWhindexCreationDateMap", getAuthMetadataTableNameVariations(new Args[] { + new Args(new Object[] {null}), + new Args(new Object[] {Collections.emptySet()}), + new Args(new Object[] {Collections.singleton("filter")})}, true, false)); + methodToParams.put("0.getCompositeFieldSeparatorMap", getNoArgAuthMetadataTableNameVariations(null)); + methodToParams.put("1.getCompositeFieldSeparatorMap", getAuthMetadataTableNameVariations(new Args[] { + new Args(new Object[] {null}), + new Args(new Object[] {Collections.emptySet()}), + new Args(new Object[] {Collections.singleton("filter")})}, true, false)); + methodToParams.put("1.getDatatypesForField", getAuthMetadataTableNameVariations(new Args[] { + new Args(new Object[] {"f1"}), + new Args(new Object[] {"f2"}), + new Args(new Object[] {"f3"})},false, false)); + methodToParams.put("2.getDatatypesForField", getAuthMetadataTableNameVariations(new Args[] { + new Args(new Object[] {"f1", Collections.emptySet()}), + new Args(new Object[] {"f1", Collections.singleton("filter")}), + // TODO this case should be verified since its a partial key +// new Args(new Object[] {"f2", Collections.singleton("filter")})}, true, false)); + new Args(new Object[] {"f2", Collections.singleton("filter2")})}, true, false)); + methodToParams.put("0.getTypeMetadata", getNoArgAuthMetadataTableNameVariations(null)); + methodToParams.put("1.getTypeMetadata", getAuthMetadataTableNameVariations(new Args[] { + new Args(new Object[] {null}), + new Args(new Object[] {Collections.emptySet()}), + new Args(new Object[] {Collections.singleton("filter")})}, true, false)); + methodToParams.put("0.getCompositeMetadata", getNoArgAuthMetadataTableNameVariations(null)); + methodToParams.put("1.getCompositeMetadata", getAuthMetadataTableNameVariations(new Args[] { + new Args(new Object[] {null}), + new Args(new Object[] {Collections.emptySet()}), + new Args(new Object[] {Collections.singleton("filter")})}, true, false)); + methodToParams.put("1.getFieldsToDatatypes", getAuthMetadataTableNameVariations(new Args[] { + new Args(new Object[] {null}), + new Args(new Object[] {Collections.emptySet()}), + new Args(new Object[] {Collections.singleton("filter")})}, true, true)); + // TODO verifying filters would require even more test setup to mock the type metadata that matches the classes being passed + methodToParams.put("2.getFieldsForDatatype", getAuthMetadataTableNameVariations(new Args[] { + new Args(new Object[] {IdentityDataType.class, Collections.emptySet()}), + new Args(new Object[] {DateType.class, Collections.emptySet()}), + new Args(new Object[] {GeoType.class, Collections.emptySet()})}, false, false)); + methodToParams.put("1.getFieldsForDatatype", getAuthMetadataTableNameVariations(new Args[] { + new Args(new Object[] {IdentityDataType.class}), + new Args(new Object[] {DateType.class}), + new Args(new Object[] {GeoType.class})}, false, false)); + methodToParams.put("1.getIndexedFields", getAuthMetadataTableNameVariations(new Args[] { + new Args(new Object[] {null}), + new Args(new Object[] {Collections.emptySet()}), + new Args(new Object[] {Collections.singleton("filter")})}, false, false)); + methodToParams.put("1.getReverseIndexedFields", getAuthMetadataTableNameVariations(new Args[] { + new Args(new Object[] {null}), + new Args(new Object[] {Collections.emptySet()}), + new Args(new Object[] {Collections.singleton("filter")})}, false, false)); + methodToParams.put("1.getExpansionFields", getAuthMetadataTableNameVariations(new Args[] { + new Args(new Object[] {null}), + new Args(new Object[] {Collections.emptySet()}), + new Args(new Object[] {Collections.singleton("filter")})}, false, false)); + methodToParams.put("1.getContentFields", getAuthMetadataTableNameVariations(new Args[] { + new Args(new Object[] {null}), + new Args(new Object[] {Collections.emptySet()}), + new Args(new Object[] {Collections.singleton("filter")})}, false, false)); + methodToParams.put("1.getDatatypes", getAuthMetadataTableNameVariations(new Args[] { + new Args(new Object[] {null}), + new Args(new Object[] {Collections.emptySet()}), + new Args(new Object[] {Collections.singleton("filter")})}, false, false)); + methodToParams.put("0.loadIndexOnlyFields", getNoArgAuthMetadataTableNameVariations(null)); + + // uncached methods that can safely be skipped + methodToParams.put("0.toString", new MethodParamsAndExpectations[] {}); + methodToParams.put("1.getDatatype", new MethodParamsAndExpectations[] {}); + methodToParams.put("0.getUsersMetadataAuthorizationSubset", new MethodParamsAndExpectations[] {}); + methodToParams.put("2.getUsersMetadataAuthorizationSubset", new MethodParamsAndExpectations[] {}); + methodToParams.put("1.getAllMetadataAuthsPowerSet", new MethodParamsAndExpectations[] {}); + methodToParams.put("0.getAllMetadataAuths", new MethodParamsAndExpectations[] {}); + methodToParams.put("0.getAuths", new MethodParamsAndExpectations[] {}); + methodToParams.put("0.getFullUserAuths", new MethodParamsAndExpectations[] {}); + methodToParams.put("0.getAllFieldMetadataHelper", new MethodParamsAndExpectations[] {}); + methodToParams.put("0.getMetadata", new MethodParamsAndExpectations[] {}); + methodToParams.put("1.getMetadata", new MethodParamsAndExpectations[] {}); + methodToParams.put("0.getEvaluationOnlyFields", new MethodParamsAndExpectations[] {}); + methodToParams.put("1.setEvaluationOnlyFields", new MethodParamsAndExpectations[] {}); + methodToParams.put("1.getDatatypeFromClass", new MethodParamsAndExpectations[] {}); + methodToParams.put("3.getCountsByFieldForDays", new MethodParamsAndExpectations[] {}); + methodToParams.put("4.getCountsByFieldForDays", new MethodParamsAndExpectations[] {}); + methodToParams.put("4.createFieldCountRanges", new MethodParamsAndExpectations[] {}); + methodToParams.put("1.readLongFromValue", new MethodParamsAndExpectations[] {}); + methodToParams.put("0.getMetadataTableName", new MethodParamsAndExpectations[] {}); + methodToParams.put("0.getTypeCacheSize", new MethodParamsAndExpectations[] {}); + methodToParams.put("1.setTypeCacheSize", new MethodParamsAndExpectations[] {}); + + // not currently cached but should probably cover with tests + methodToParams.put("0.getTypeMetadataMap", new MethodParamsAndExpectations[] {}); + methodToParams.put("2.getQueryModel", new MethodParamsAndExpectations[] {}); + methodToParams.put("3.getQueryModel", new MethodParamsAndExpectations[] {}); + methodToParams.put("4.getQueryModel", new MethodParamsAndExpectations[] {}); + methodToParams.put("3.getCardinalityForField", new MethodParamsAndExpectations[] {}); + methodToParams.put("4.getCardinalityForField", new MethodParamsAndExpectations[] {}); + methodToParams.put("1.getCountsByFieldInDayWithTypes", new MethodParamsAndExpectations[] {}); + methodToParams.put("2.getCountsByFieldInDayWithTypes", new MethodParamsAndExpectations[] {}); + methodToParams.put("3.getCountsByFieldInDayWithTypes", new MethodParamsAndExpectations[] {}); + methodToParams.put("4.getCountsByFieldInDayWithTypes", new MethodParamsAndExpectations[] {}); + methodToParams.put("3.getCountsForFieldsInDateRange", new MethodParamsAndExpectations[] {}); + methodToParams.put("4.getCountsForFieldsInDateRange", new MethodParamsAndExpectations[] {}); + methodToParams.put("1.getEarliestOccurrenceOfField", new MethodParamsAndExpectations[] {}); + methodToParams.put("2.getEarliestOccurrenceOfFieldWithType", new MethodParamsAndExpectations[] {}); + methodToParams.put("4.getEarliestOccurrenceOfFieldWithType", new MethodParamsAndExpectations[] {}); + methodToParams.put("3.getFieldIndexHoles", new MethodParamsAndExpectations[] {}); + methodToParams.put("3.getReversedFieldIndexHoles", new MethodParamsAndExpectations[] {}); + methodToParams.put("0.loadAllFields", new MethodParamsAndExpectations[] {}); + methodToParams.put("0.loadTermFrequencyFields", new MethodParamsAndExpectations[] {}); + // TBD + methodToParams.put("2.getCountsByFieldInDay", new MethodParamsAndExpectations[] {}); + + // @formatter:on // test that all methods are tested for (Method method : MetadataHelper.class.getDeclaredMethods()) { @@ -1191,19 +1312,23 @@ public void allTest() throws Exception { // no need to verify statics continue; } + if (Modifier.isPrivate(method.getModifiers())) { + // no need to verify private + continue; + } // TODO delete this after all methods are added // isolate a single test - // if (!method.getName().equals("isReverseIndexed")) { - // continue; - // } +// if (!method.getName().equals("getFieldsForDatatype")) { +// continue; +// } String lookupName = method.getParameterCount() + "." + method.getName(); // verify all non static methods are tested - assertNotNull("MetadataHelper method '" + method.getName() + "' not tested", methodToParams.get(lookupName)); + assertNotNull("MetadataHelper method '" + method.getName() + "()' with " + method.getParameterCount() + " args not tested", methodToParams.get(lookupName)); // TODO delete this after testing - System.out.println(method.getName() + ": Testing"); +// System.out.println(method.getName() + ": Testing"); // clear spring caches to prevent cross contamination between calls cleanupCache(); From 6aeed3106aa6029e099bde764a2d0c2de70eb6c9 Mon Sep 17 00:00:00 2001 From: FineAndDandy Date: Sat, 16 Nov 2024 00:16:00 +0000 Subject: [PATCH 11/12] more tests --- .../query/util/MetadataHelperCacheTest.java | 183 ++++++++++++++---- 1 file changed, 147 insertions(+), 36 deletions(-) diff --git a/warehouse/query-core/src/test/java/datawave/query/util/MetadataHelperCacheTest.java b/warehouse/query-core/src/test/java/datawave/query/util/MetadataHelperCacheTest.java index 4dc4a094ed..d609f5d75b 100644 --- a/warehouse/query-core/src/test/java/datawave/query/util/MetadataHelperCacheTest.java +++ b/warehouse/query-core/src/test/java/datawave/query/util/MetadataHelperCacheTest.java @@ -606,12 +606,6 @@ public void getNonEventFields_param_test() throws TableNotFoundException { verifyAll(); } - @Test - public void getNonEventFields_key_test() throws TableNotFoundException { - // TODO - assertTrue(false); - } - // this test will fail until AllFieldMetadataHelper.getIndexOnlyFields() properly closes its scanner @Test public void getIndexOnlyFields_param_test() throws TableNotFoundException { @@ -1085,6 +1079,86 @@ public void expect() throws TableNotFoundException { return results.toArray(new MethodParamsAndExpectations[0]); } + private MethodParamsAndExpectations[] getAuthMetadataTableNameVariations(Args[] args, boolean argsInKey, boolean exactMatch, Expectation alwaysExpect) { + if (args == null) { + args = new Args[] {new Args(new Object[] {})}; + } + List results = new ArrayList<>(); + + boolean first = true; + + MethodParamsAndExpectations base = null; + MethodParamsAndExpectations altAuths = null; + MethodParamsAndExpectations altTable = null; + MethodParamsAndExpectations altTableAndAuths = null; + + for (Args a : args) { + if (first || argsInKey) { + base = new MethodParamsAndExpectations(metadataHelper, a.args) { + public void expect() throws Exception { + Scanner s = createMock(Scanner.class); + List> entries = new ArrayList<>(); + expectScanner("table", s, entries); + + if (alwaysExpect != null) { + alwaysExpect.expect(); + } + } + }; + + altAuths = new MethodParamsAndExpectations(alternateAuthsHelper, a.args) { + public void expect() throws Exception { + Scanner s = createMock(Scanner.class); + List> entries = new ArrayList<>(); + expectScanner("table", s, entries); + + if (alwaysExpect != null) { + alwaysExpect.expect(); + } + } + }; + + altTable = new MethodParamsAndExpectations(alternateTableMetadataHelper, a.args) { + public void expect() throws Exception { + Scanner s = createMock(Scanner.class); + List> entries = new ArrayList<>(); + expectScanner("table2", s, entries); + + if (alwaysExpect != null) { + alwaysExpect.expect(); + } + } + }; + + altTableAndAuths = new MethodParamsAndExpectations(alternateHelper, a.args) { + public void expect() throws Exception { + Scanner s = createMock(Scanner.class); + List> entries = new ArrayList<>(); + expectScanner("table2", s, entries); + + if (alwaysExpect != null) { + alwaysExpect.expect(); + } + } + }; + + results.add(base); + results.add(altTable); + results.add(altAuths); + results.add(altTableAndAuths); + + first = false; + } + + results.add(new MethodParamsAndExpectations(metadataHelper, a.args, base, exactMatch, alwaysExpect)); + results.add(new MethodParamsAndExpectations(alternateTableMetadataHelper, a.args, altTable, exactMatch, alwaysExpect)); + results.add(new MethodParamsAndExpectations(alternateAuthsHelper, a.args, altAuths, exactMatch, alwaysExpect)); + results.add(new MethodParamsAndExpectations(alternateHelper, a.args, altTableAndAuths, exactMatch, alwaysExpect)); + } + + return results.toArray(new MethodParamsAndExpectations[0]); + } + private MethodParamsAndExpectations getWithExpectation(MetadataHelper helper, Object[] args, MethodParamsAndExpectations cachedObject, boolean exactMatch, Expectation expectation) { return new MethodParamsAndExpectations(helper, args, cachedObject, exactMatch) { @@ -1144,9 +1218,35 @@ private MethodParamsAndExpectations[] getNonEventFieldExpectations(Args[] args) return results.toArray(new MethodParamsAndExpectations[0]); } - // test all methods against all objects + private MethodParamsAndExpectations[] getScansPerCall(Args[] args, int scansPerTable) { + List results = new ArrayList<>(); + + for (int i = 0; i < args.length; i++) { + MethodParamsAndExpectations base = getWithExpectation(metadataHelper, args[i].args, null, false, new ScannerExpectation("table", scansPerTable)); + MethodParamsAndExpectations baseAltTable = getWithExpectation(alternateTableMetadataHelper, args[i].args, null, false, + new ScannerExpectation("table2", scansPerTable)); + MethodParamsAndExpectations baseAltAuths = getWithExpectation(alternateAuthsHelper, args[i].args, null, false, new ScannerExpectation("table", scansPerTable)); + MethodParamsAndExpectations baseAlt = getWithExpectation(alternateHelper, args[i].args, null, false, new ScannerExpectation("table2", scansPerTable)); + + // no cache hits across any variation here + results.add(base); + results.add(baseAltTable); + results.add(baseAltAuths); + results.add(baseAlt); + + // no caching is happening so should be exactly the same for any follow up calls + results.add(getWithExpectation(metadataHelper, args[i].args, base, false, new ScannerExpectation("table", scansPerTable))); + results.add(getWithExpectation(alternateTableMetadataHelper, args[i].args, baseAltTable, false, new ScannerExpectation("table2", scansPerTable))); + results.add(getWithExpectation(alternateAuthsHelper, args[i].args, baseAltAuths, false, new ScannerExpectation("table", scansPerTable))); + results.add(getWithExpectation(alternateHelper, args[i].args, baseAlt, false, new ScannerExpectation("table2", scansPerTable))); + } + + return results.toArray(new MethodParamsAndExpectations[0]); + } + + // test all public, non-static caching methods for consistency @Test - public void allTest() throws Exception { + public void cachingConsistencyTest() throws Exception { // @formatter:off Map methodToParams = new HashMap<>(); methodToParams.put("1.getQueryModelNames", getExpectedQueryModelNames()); @@ -1163,21 +1263,27 @@ public void allTest() throws Exception { s.addScanIterator(anyObject()); s.addScanIterator(anyObject()); })); - methodToParams.put("1.getIndexOnlyFields", getAuthMetadataTableNameVariations(new Args[] {new Args(new Object[] {null}), - new Args(new Object[] {Collections.emptySet()}), new Args(new Object[] {Collections.singleton("filter")})}, false, false)); - methodToParams.put("1.getTermFrequencyFields", getAuthMetadataTableNameVariations(new Args[] {new Args(new Object[] {null})}, true)); - methodToParams.put("1.getAllFields", getAuthMetadataTableNameVariations(new Args[] {new Args(new Object[] {null}), - new Args(new Object[] {Collections.emptySet()}), new Args(new Object[] {Collections.singleton("filter")})}, false, false)); + methodToParams.put("1.getIndexOnlyFields", getAuthMetadataTableNameVariations(new Args[] { + new Args(new Object[] {null}), + new Args(new Object[] {Collections.emptySet()}), + new Args(new Object[] {Collections.singleton("filter")})}, false, false)); + methodToParams.put("1.getTermFrequencyFields", getAuthMetadataTableNameVariations( + new Args[] {new Args(new Object[] {null})}, true)); + methodToParams.put("1.getAllFields", getAuthMetadataTableNameVariations(new Args[] { + new Args(new Object[] {null}), + new Args(new Object[] {Collections.emptySet()}), + new Args(new Object[] {Collections.singleton("filter")})}, false, false)); methodToParams.put("1.getNonEventFields", getNonEventFieldExpectations(new Args[] {new Args(new Object[] {null}), - new Args(new Object[] {Collections.emptySet()}), new Args(new Object[] {Collections.singleton("filter")})})); - methodToParams.put("2.isReverseIndexed", - getAuthMetadataTableNameVariations(new Args[] {new Args(new Object[] {"f1", Collections.emptySet()}), - new Args(new Object[] {"f1", Collections.singleton("filter")}), new Args(new Object[] {"f2", Collections.singleton("filter")})}, - true, false)); - methodToParams.put("2.isIndexed", - getAuthMetadataTableNameVariations(new Args[] {new Args(new Object[] {"f1", Collections.emptySet()}), - new Args(new Object[] {"f1", Collections.singleton("filter")}), new Args(new Object[] {"f2", Collections.singleton("filter")})}, - true, false)); + new Args(new Object[] {Collections.emptySet()}), + new Args(new Object[] {Collections.singleton("filter")})})); + methodToParams.put("2.isReverseIndexed", getAuthMetadataTableNameVariations(new Args[] { + new Args(new Object[] {"f1", Collections.emptySet()}), + new Args(new Object[] {"f1", Collections.singleton("filter")}), + new Args(new Object[] {"f2", Collections.singleton("filter")})}, true, false)); + methodToParams.put("2.isIndexed", getAuthMetadataTableNameVariations(new Args[] { + new Args(new Object[] {"f1", Collections.emptySet()}), + new Args(new Object[] {"f1", Collections.singleton("filter")}), + new Args(new Object[] {"f2", Collections.singleton("filter")})}, true, false)); methodToParams.put("2.isTokenized", getAuthMetadataTableNameVariations(new Args[] { new Args(new Object[] {"f1", Collections.emptySet()}), new Args(new Object[] {"f1", Collections.singleton("filter")}), @@ -1257,6 +1363,10 @@ public void allTest() throws Exception { new Args(new Object[] {Collections.emptySet()}), new Args(new Object[] {Collections.singleton("filter")})}, false, false)); methodToParams.put("0.loadIndexOnlyFields", getNoArgAuthMetadataTableNameVariations(null)); + // TODO special cases + methodToParams.put("2.getQueryModel", new MethodParamsAndExpectations[] {}); + methodToParams.put("3.getQueryModel", new MethodParamsAndExpectations[] {}); + methodToParams.put("4.getQueryModel", new MethodParamsAndExpectations[] {}); // uncached methods that can safely be skipped methodToParams.put("0.toString", new MethodParamsAndExpectations[] {}); @@ -1280,14 +1390,20 @@ public void allTest() throws Exception { methodToParams.put("0.getMetadataTableName", new MethodParamsAndExpectations[] {}); methodToParams.put("0.getTypeCacheSize", new MethodParamsAndExpectations[] {}); methodToParams.put("1.setTypeCacheSize", new MethodParamsAndExpectations[] {}); - - // not currently cached but should probably cover with tests - methodToParams.put("0.getTypeMetadataMap", new MethodParamsAndExpectations[] {}); - methodToParams.put("2.getQueryModel", new MethodParamsAndExpectations[] {}); - methodToParams.put("3.getQueryModel", new MethodParamsAndExpectations[] {}); - methodToParams.put("4.getQueryModel", new MethodParamsAndExpectations[] {}); + methodToParams.put("0.getTypeCacheExpirationInMinutes", new MethodParamsAndExpectations[] {}); + methodToParams.put("1.setTypeCacheExpirationInMinutes", new MethodParamsAndExpectations[] {}); + + // not currently cached, verify scanners so if caching is added things will fail + methodToParams.put("0.getTypeMetadataMap", getScansPerCall(new Args[] { + new Args(new Object[] {})}, 2)); + methodToParams.put("0.loadTermFrequencyFields", getScansPerCall(new Args[] { + new Args(new Object[] {})}, 1)); + methodToParams.put("0.loadAllFields", getScansPerCall(new Args[] { + new Args(new Object[] {})}, 1)); + // TODO from here down for scan verification methodToParams.put("3.getCardinalityForField", new MethodParamsAndExpectations[] {}); methodToParams.put("4.getCardinalityForField", new MethodParamsAndExpectations[] {}); + methodToParams.put("2.getCountsByFieldInDay", new MethodParamsAndExpectations[] {}); methodToParams.put("1.getCountsByFieldInDayWithTypes", new MethodParamsAndExpectations[] {}); methodToParams.put("2.getCountsByFieldInDayWithTypes", new MethodParamsAndExpectations[] {}); methodToParams.put("3.getCountsByFieldInDayWithTypes", new MethodParamsAndExpectations[] {}); @@ -1299,11 +1415,6 @@ public void allTest() throws Exception { methodToParams.put("4.getEarliestOccurrenceOfFieldWithType", new MethodParamsAndExpectations[] {}); methodToParams.put("3.getFieldIndexHoles", new MethodParamsAndExpectations[] {}); methodToParams.put("3.getReversedFieldIndexHoles", new MethodParamsAndExpectations[] {}); - methodToParams.put("0.loadAllFields", new MethodParamsAndExpectations[] {}); - methodToParams.put("0.loadTermFrequencyFields", new MethodParamsAndExpectations[] {}); - // TBD - methodToParams.put("2.getCountsByFieldInDay", new MethodParamsAndExpectations[] {}); - // @formatter:on // test that all methods are tested @@ -1319,9 +1430,9 @@ public void allTest() throws Exception { // TODO delete this after all methods are added // isolate a single test -// if (!method.getName().equals("getFieldsForDatatype")) { -// continue; -// } + if (!method.getName().equals("loadAllFields")) { + continue; + } String lookupName = method.getParameterCount() + "." + method.getName(); From acbb4db18d43e3804f440caa817f966d54424252 Mon Sep 17 00:00:00 2001 From: FineAndDandy Date: Wed, 20 Nov 2024 01:55:06 +0000 Subject: [PATCH 12/12] Add comments and final tests --- .../query/util/MetadataHelperCacheTest.java | 240 ++++++++++++------ 1 file changed, 166 insertions(+), 74 deletions(-) diff --git a/warehouse/query-core/src/test/java/datawave/query/util/MetadataHelperCacheTest.java b/warehouse/query-core/src/test/java/datawave/query/util/MetadataHelperCacheTest.java index d609f5d75b..c42c092379 100644 --- a/warehouse/query-core/src/test/java/datawave/query/util/MetadataHelperCacheTest.java +++ b/warehouse/query-core/src/test/java/datawave/query/util/MetadataHelperCacheTest.java @@ -13,6 +13,7 @@ import java.lang.reflect.Modifier; import java.util.ArrayList; import java.util.Collections; +import java.util.Date; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -20,12 +21,10 @@ import java.util.Set; import java.util.concurrent.ExecutionException; -import datawave.IdentityDataType; -import datawave.data.type.DateType; -import datawave.data.type.GeoType; import org.apache.accumulo.core.client.AccumuloClient; import org.apache.accumulo.core.client.AccumuloException; import org.apache.accumulo.core.client.AccumuloSecurityException; +import org.apache.accumulo.core.client.BatchScanner; import org.apache.accumulo.core.client.Scanner; import org.apache.accumulo.core.client.TableNotFoundException; import org.apache.accumulo.core.client.admin.SecurityOperations; @@ -50,6 +49,9 @@ import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; +import datawave.IdentityDataType; +import datawave.data.type.DateType; +import datawave.data.type.GeoType; import datawave.query.composite.CompositeMetadataHelper; // enable springification @@ -262,17 +264,8 @@ public void setup() { assertNotNull(alternateAuthsHelper); assertNotNull(alternateHelper); assertNotNull(cacheManager); - - // TODO delete this later - // expect(accumuloClient.instanceOperations()).andAnswer(new IAnswer() { - // @Override - // public InstanceOperations answer() throws Throwable { - // return null; - // } - // }).anyTimes(); } - // wrapper so I can track access while testing private AccumuloClient getAccumuloClient() { return accumuloClient; } @@ -894,11 +887,11 @@ public void expect() throws TableNotFoundException { // @formatter:on } - // validate key = "{#root.target.metadataTableName}" - private MethodParamsAndExpectations[] getNoArgMetadataTableNameVariations() { - return getNoArgMetadataTableNameVariations(null); - } - + /** + * Create expectations for no argument method calls for each of the metadataHelper objects. Verify a second call returns the same result as the first without additional accumulo calls. When extra is set, also expect this on the initial call + * @param extra + * @return + */ private MethodParamsAndExpectations[] getNoArgMetadataTableNameVariations(ExtraExpectation extra) { MethodParamsAndExpectations base = new MethodParamsAndExpectations(metadataHelper, new Object[] {}) { public void expect() throws Exception { @@ -1080,6 +1073,11 @@ public void expect() throws TableNotFoundException { } private MethodParamsAndExpectations[] getAuthMetadataTableNameVariations(Args[] args, boolean argsInKey, boolean exactMatch, Expectation alwaysExpect) { + return getAuthMetadataTableNameVariations(args, argsInKey, exactMatch, alwaysExpect, 1); + } + + private MethodParamsAndExpectations[] getAuthMetadataTableNameVariations(Args[] args, boolean argsInKey, boolean exactMatch, Expectation alwaysExpect, + int callsPerCache) { if (args == null) { args = new Args[] {new Args(new Object[] {})}; } @@ -1096,9 +1094,11 @@ private MethodParamsAndExpectations[] getAuthMetadataTableNameVariations(Args[] if (first || argsInKey) { base = new MethodParamsAndExpectations(metadataHelper, a.args) { public void expect() throws Exception { - Scanner s = createMock(Scanner.class); - List> entries = new ArrayList<>(); - expectScanner("table", s, entries); + for (int i = 0; i < callsPerCache; i++) { + Scanner s = createMock(Scanner.class); + List> entries = new ArrayList<>(); + expectScanner("table", s, entries); + } if (alwaysExpect != null) { alwaysExpect.expect(); @@ -1108,9 +1108,11 @@ public void expect() throws Exception { altAuths = new MethodParamsAndExpectations(alternateAuthsHelper, a.args) { public void expect() throws Exception { - Scanner s = createMock(Scanner.class); - List> entries = new ArrayList<>(); - expectScanner("table", s, entries); + for (int i = 0; i < callsPerCache; i++) { + Scanner s = createMock(Scanner.class); + List> entries = new ArrayList<>(); + expectScanner("table", s, entries); + } if (alwaysExpect != null) { alwaysExpect.expect(); @@ -1120,9 +1122,11 @@ public void expect() throws Exception { altTable = new MethodParamsAndExpectations(alternateTableMetadataHelper, a.args) { public void expect() throws Exception { - Scanner s = createMock(Scanner.class); - List> entries = new ArrayList<>(); - expectScanner("table2", s, entries); + for (int i = 0; i < callsPerCache; i++) { + Scanner s = createMock(Scanner.class); + List> entries = new ArrayList<>(); + expectScanner("table2", s, entries); + } if (alwaysExpect != null) { alwaysExpect.expect(); @@ -1132,9 +1136,11 @@ public void expect() throws Exception { altTableAndAuths = new MethodParamsAndExpectations(alternateHelper, a.args) { public void expect() throws Exception { - Scanner s = createMock(Scanner.class); - List> entries = new ArrayList<>(); - expectScanner("table2", s, entries); + for (int i = 0; i < callsPerCache; i++) { + Scanner s = createMock(Scanner.class); + List> entries = new ArrayList<>(); + expectScanner("table2", s, entries); + } if (alwaysExpect != null) { alwaysExpect.expect(); @@ -1219,14 +1225,21 @@ private MethodParamsAndExpectations[] getNonEventFieldExpectations(Args[] args) } private MethodParamsAndExpectations[] getScansPerCall(Args[] args, int scansPerTable) { + return getScansPerCall(args, scansPerTable, null); + } + + private MethodParamsAndExpectations[] getScansPerCall(Args[] args, int scansPerTable, ExtraExpectation extra) { List results = new ArrayList<>(); for (int i = 0; i < args.length; i++) { - MethodParamsAndExpectations base = getWithExpectation(metadataHelper, args[i].args, null, false, new ScannerExpectation("table", scansPerTable)); + MethodParamsAndExpectations base = getWithExpectation(metadataHelper, args[i].args, null, false, + new ScannerExpectation("table", scansPerTable, extra)); MethodParamsAndExpectations baseAltTable = getWithExpectation(alternateTableMetadataHelper, args[i].args, null, false, - new ScannerExpectation("table2", scansPerTable)); - MethodParamsAndExpectations baseAltAuths = getWithExpectation(alternateAuthsHelper, args[i].args, null, false, new ScannerExpectation("table", scansPerTable)); - MethodParamsAndExpectations baseAlt = getWithExpectation(alternateHelper, args[i].args, null, false, new ScannerExpectation("table2", scansPerTable)); + new ScannerExpectation("table2", scansPerTable, extra)); + MethodParamsAndExpectations baseAltAuths = getWithExpectation(alternateAuthsHelper, args[i].args, null, false, + new ScannerExpectation("table", scansPerTable, extra)); + MethodParamsAndExpectations baseAlt = getWithExpectation(alternateHelper, args[i].args, null, false, + new ScannerExpectation("table2", scansPerTable, extra)); // no cache hits across any variation here results.add(base); @@ -1235,16 +1248,17 @@ private MethodParamsAndExpectations[] getScansPerCall(Args[] args, int scansPerT results.add(baseAlt); // no caching is happening so should be exactly the same for any follow up calls - results.add(getWithExpectation(metadataHelper, args[i].args, base, false, new ScannerExpectation("table", scansPerTable))); - results.add(getWithExpectation(alternateTableMetadataHelper, args[i].args, baseAltTable, false, new ScannerExpectation("table2", scansPerTable))); - results.add(getWithExpectation(alternateAuthsHelper, args[i].args, baseAltAuths, false, new ScannerExpectation("table", scansPerTable))); - results.add(getWithExpectation(alternateHelper, args[i].args, baseAlt, false, new ScannerExpectation("table2", scansPerTable))); + results.add(getWithExpectation(metadataHelper, args[i].args, base, false, new ScannerExpectation("table", scansPerTable, extra))); + results.add(getWithExpectation(alternateTableMetadataHelper, args[i].args, baseAltTable, false, + new ScannerExpectation("table2", scansPerTable, extra))); + results.add(getWithExpectation(alternateAuthsHelper, args[i].args, baseAltAuths, false, new ScannerExpectation("table", scansPerTable, extra))); + results.add(getWithExpectation(alternateHelper, args[i].args, baseAlt, false, new ScannerExpectation("table2", scansPerTable, extra))); } return results.toArray(new MethodParamsAndExpectations[0]); } - // test all public, non-static caching methods for consistency + // test all public, non-static caching methods for cache consistency. Ensure caching is defined and works as expected for all methods. This test will fail if cache keys are wrong leading to unexpected accumulo calls @Test public void cachingConsistencyTest() throws Exception { // @formatter:off @@ -1316,8 +1330,6 @@ public void cachingConsistencyTest() throws Exception { methodToParams.put("2.getDatatypesForField", getAuthMetadataTableNameVariations(new Args[] { new Args(new Object[] {"f1", Collections.emptySet()}), new Args(new Object[] {"f1", Collections.singleton("filter")}), - // TODO this case should be verified since its a partial key -// new Args(new Object[] {"f2", Collections.singleton("filter")})}, true, false)); new Args(new Object[] {"f2", Collections.singleton("filter2")})}, true, false)); methodToParams.put("0.getTypeMetadata", getNoArgAuthMetadataTableNameVariations(null)); methodToParams.put("1.getTypeMetadata", getAuthMetadataTableNameVariations(new Args[] { @@ -1333,7 +1345,7 @@ public void cachingConsistencyTest() throws Exception { new Args(new Object[] {null}), new Args(new Object[] {Collections.emptySet()}), new Args(new Object[] {Collections.singleton("filter")})}, true, true)); - // TODO verifying filters would require even more test setup to mock the type metadata that matches the classes being passed + // verifying filters would require even more test setup to mock the type metadata that matches the classes being passed methodToParams.put("2.getFieldsForDatatype", getAuthMetadataTableNameVariations(new Args[] { new Args(new Object[] {IdentityDataType.class, Collections.emptySet()}), new Args(new Object[] {DateType.class, Collections.emptySet()}), @@ -1363,10 +1375,43 @@ public void cachingConsistencyTest() throws Exception { new Args(new Object[] {Collections.emptySet()}), new Args(new Object[] {Collections.singleton("filter")})}, false, false)); methodToParams.put("0.loadIndexOnlyFields", getNoArgAuthMetadataTableNameVariations(null)); - // TODO special cases - methodToParams.put("2.getQueryModel", new MethodParamsAndExpectations[] {}); - methodToParams.put("3.getQueryModel", new MethodParamsAndExpectations[] {}); - methodToParams.put("4.getQueryModel", new MethodParamsAndExpectations[] {}); + // 1 scanner + getAllFields + getIndexOnlyFields caching + methodToParams.put("2.getQueryModel", getAuthMetadataTableNameVariations(new Args[]{ + new Args(new Object[] {"table", "model"}), + new Args(new Object[] {"table", "model2"}), + new Args(new Object[] {"table", "model3"})}, false, false, new ScannerExpectation("table", 1), 2)); + // 1 scanner + getAllFields caching + methodToParams.put("3.getQueryModel", getAuthMetadataTableNameVariations(new Args[]{ + new Args(new Object[] {"table", "model", Collections.emptySet()}), + new Args(new Object[] {"table", "model", Collections.singleton("f1")}), + new Args(new Object[] {"table", "model2", Collections.singleton("f1")}), + new Args(new Object[] {"table", "model2", Collections.singleton("f2")}), + new Args(new Object[] {"table", "model3", Collections.singleton("f1")}), + new Args(new Object[] {"t3", "model3", Collections.singleton("f1")})}, false, false, new ScannerExpectation(null, 1))); + // 1 scanner + getAllFields caching + methodToParams.put("4.getQueryModel", getAuthMetadataTableNameVariations(new Args[]{ + new Args(new Object[] {"table", "model", Collections.emptySet(), Collections.emptySet()}), + new Args(new Object[] {"table", "model", Collections.singleton("f1"), Collections.emptySet()}), + new Args(new Object[] {"table", "model2", Collections.singleton("f1"), Collections.singleton("filter")}), + new Args(new Object[] {"table", "model2", Collections.singleton("f2"), Collections.singleton("filter")}), + new Args(new Object[] {"table", "model3", Collections.singleton("f1"), Collections.singleton("filter")}), + new Args(new Object[] {"t3", "model3", Collections.singleton("f1"), Collections.singleton("filter")})}, false, false, new ScannerExpectation(null, 1))); + // loadIndexedFields, but loadIndexedFields is call from within AllFieldsMetadataHelper so is NOT cached + methodToParams.put("3.getFieldIndexHoles", getScansPerCall(new Args[]{ + // empty set of fields does an extra scan (always) so can't be tested in this test +// new Args(new Object[] {Collections.emptySet(), Collections.emptySet(), 3}), + new Args(new Object[] {Collections.singleton("f1"), Collections.emptySet(), 3}), + new Args(new Object[] {Collections.singleton("f1"), Collections.singleton("filter"), 3}), + new Args(new Object[] {Collections.singleton("f1"), Collections.singleton("filter2"), 3}), + new Args(new Object[] {Collections.singleton("f2"), Collections.singleton("filter2"), 3})}, 1)); + // loadIndexedFields, but loadIndexedFields is call from within AllFieldsMetadataHelper so is NOT cached + methodToParams.put("3.getReversedFieldIndexHoles", getScansPerCall(new Args[]{ + // empty set of fields does an extra scan (always) so can't be tested in this test +// new Args(new Object[] {Collections.emptySet(), Collections.emptySet(), 3}), + new Args(new Object[] {Collections.singleton("f1"), Collections.emptySet(), 3}), + new Args(new Object[] {Collections.singleton("f1"), Collections.singleton("filter"), 3}), + new Args(new Object[] {Collections.singleton("f1"), Collections.singleton("filter2"), 3}), + new Args(new Object[] {Collections.singleton("f2"), Collections.singleton("filter2"), 3})}, 1)); // uncached methods that can safely be skipped methodToParams.put("0.toString", new MethodParamsAndExpectations[] {}); @@ -1400,21 +1445,42 @@ public void cachingConsistencyTest() throws Exception { new Args(new Object[] {})}, 1)); methodToParams.put("0.loadAllFields", getScansPerCall(new Args[] { new Args(new Object[] {})}, 1)); - // TODO from here down for scan verification - methodToParams.put("3.getCardinalityForField", new MethodParamsAndExpectations[] {}); - methodToParams.put("4.getCardinalityForField", new MethodParamsAndExpectations[] {}); - methodToParams.put("2.getCountsByFieldInDay", new MethodParamsAndExpectations[] {}); - methodToParams.put("1.getCountsByFieldInDayWithTypes", new MethodParamsAndExpectations[] {}); - methodToParams.put("2.getCountsByFieldInDayWithTypes", new MethodParamsAndExpectations[] {}); - methodToParams.put("3.getCountsByFieldInDayWithTypes", new MethodParamsAndExpectations[] {}); - methodToParams.put("4.getCountsByFieldInDayWithTypes", new MethodParamsAndExpectations[] {}); - methodToParams.put("3.getCountsForFieldsInDateRange", new MethodParamsAndExpectations[] {}); - methodToParams.put("4.getCountsForFieldsInDateRange", new MethodParamsAndExpectations[] {}); - methodToParams.put("1.getEarliestOccurrenceOfField", new MethodParamsAndExpectations[] {}); - methodToParams.put("2.getEarliestOccurrenceOfFieldWithType", new MethodParamsAndExpectations[] {}); - methodToParams.put("4.getEarliestOccurrenceOfFieldWithType", new MethodParamsAndExpectations[] {}); - methodToParams.put("3.getFieldIndexHoles", new MethodParamsAndExpectations[] {}); - methodToParams.put("3.getReversedFieldIndexHoles", new MethodParamsAndExpectations[] {}); + methodToParams.put("3.getCardinalityForField", getScansPerCall(new Args[] { + new Args(new Object[] {"f1", new Date(), new Date()})}, 1)); + methodToParams.put("4.getCardinalityForField", getScansPerCall(new Args[] { + new Args(new Object[] {"f1", "type1", new Date(), new Date()})}, 1)); + methodToParams.put("2.getCountsByFieldInDay", getScansPerCall(new Args[] { + new Args(new Object[] {"f1", "20241119"}), + new Args(new Object[] {"f2", "20241119"}), + new Args(new Object[] {"f1", "20241118"})}, 1, (s)-> { + s.addScanIterator(anyObject()); + })); + methodToParams.put("3.getCountsByFieldInDayWithTypes", getScansPerCall(new Args[] { + new Args(new Object[] {"f1", "20241119", Collections.emptySet()}), + new Args(new Object[] {"f2", "20241119", Collections.emptySet()}), + new Args(new Object[] {"f1", "20241119", Collections.singleton("filter")})}, 1, (s)-> { + s.addScanIterator(anyObject()); + })); + // These use batch scanners... + methodToParams.put("3.getCountsForFieldsInDateRange", getScansPerCall(new Args[] { + new Args(new Object[] {Collections.emptySet(), new Date(), new Date()})}, 0, (s)-> { + BatchScanner bs = createMock(BatchScanner.class); + expect(getAccumuloClient().createBatchScanner(anyObject(), anyObject())).andReturn(bs); + bs.close(); + })); + // can't do this one, doesn't support overloaded params for same param count + methodToParams.put("4.getCountsForFieldsInDateRange", new MethodParamsAndExpectations[] {}); + methodToParams.put("1.getEarliestOccurrenceOfField", getScansPerCall(new Args[] { + new Args(new Object[] {"f1"}), + new Args(new Object[] {"f2"}), + new Args(new Object[] {"f3"})}, 1)); + methodToParams.put("2.getEarliestOccurrenceOfFieldWithType", getScansPerCall(new Args[] { + new Args(new Object[] {"f1", "type1"}), + new Args(new Object[] {"f2", "type1"}), + new Args(new Object[] {"f1", "type2"}), + new Args(new Object[] {"f2", "type2"})}, 1, (s)-> { + s.addScanIterator(anyObject()); + })); // @formatter:on // test that all methods are tested @@ -1427,19 +1493,24 @@ public void cachingConsistencyTest() throws Exception { // no need to verify private continue; } + if (Modifier.isProtected(method.getModifiers())) { + // no need to verify protected + continue; + } - // TODO delete this after all methods are added // isolate a single test - if (!method.getName().equals("loadAllFields")) { - continue; - } + // if (!method.getName().equals("getCountsByFieldInDayWithTypes")) { + // continue; + // } + // if (method.getParameterCount() != 2) { + // continue; + // } String lookupName = method.getParameterCount() + "." + method.getName(); // verify all non static methods are tested - assertNotNull("MetadataHelper method '" + method.getName() + "()' with " + method.getParameterCount() + " args not tested", methodToParams.get(lookupName)); - // TODO delete this after testing -// System.out.println(method.getName() + ": Testing"); + assertNotNull("MetadataHelper method '" + method.getName() + "()' with " + method.getParameterCount() + " args not tested", + methodToParams.get(lookupName)); // clear spring caches to prevent cross contamination between calls cleanupCache(); @@ -1461,9 +1532,9 @@ public void cachingConsistencyTest() throws Exception { // test cache value matches if it was supposed to be a cache hit if (expectation.cachedOf != null) { if (expectation.exactMatch) { - assertTrue("didn't get exact expected cached value", expectation.cachedOf.getResult() == result); + assertTrue(method.getName() + "didn't get exact expected cached value", expectation.cachedOf.getResult() == result); } else { - assertEquals("didn't get equivalent value", expectation.cachedOf.getResult(), result); + assertEquals(method.getName() + "didn't get equivalent value", expectation.cachedOf.getResult(), result); } } } @@ -1471,15 +1542,15 @@ public void cachingConsistencyTest() throws Exception { EasyMock.verify(getAccumuloClient()); verifyAll(); - System.out.println(method.getName() + ": OK"); - // reset mocks to use again EasyMock.reset(getAccumuloClient()); resetAll(); } } - // force cache clear + /** + * Reset the spring cache's to prevent contamination + */ private void cleanupCache() { for (String cacheName : cacheManager.getCacheNames()) { cacheManager.getCache(cacheName).clear(); @@ -1506,22 +1577,36 @@ private class ScannerExpectation implements Expectation { private final String table; private final int count; + private ExtraExpectation extra; private ScannerExpectation(String table, int count) { this.table = table; this.count = count; } + private ScannerExpectation(String table, int count, ExtraExpectation extra) { + this.table = table; + this.count = count; + this.extra = extra; + } + @Override public void expect() throws Exception { for (int i = 0; i < count; i++) { Scanner s = createMock(Scanner.class); List> entries = new ArrayList<>(); expectScanner(table, s, entries); + + if (extra != null) { + extra.extraExpectation(s); + } } } } + /** + * Used to set expectations and cache settings of a method call. Used by cachingConsistencyTest() + */ private static class MethodParamsAndExpectations implements Expectation { public MetadataHelper metadataHelper; public Object[] args; @@ -1573,13 +1658,17 @@ public Object getResult() { /** * expect/mock the scanner creation and return of results from an iterator * - * @param table + * @param table if not null expects a specific table, otherwise any table * @param mockScanner * @param entries * @throws TableNotFoundException */ private void expectScanner(String table, Scanner mockScanner, List> entries) throws TableNotFoundException { - expect(getAccumuloClient().createScanner(eq(table), EasyMock.anyObject())).andReturn(mockScanner); + if (table != null) { + expect(getAccumuloClient().createScanner(eq(table), anyObject())).andReturn(mockScanner); + } else { + expect(getAccumuloClient().createScanner(isA(String.class), anyObject())).andReturn(mockScanner); + } mockScanner.setRange(anyObject()); mockScanner.fetchColumnFamily(isA(Text.class)); expectLastCall().anyTimes(); @@ -1600,6 +1689,9 @@ public String getData(String key) { @Autowired private TestCache c; + /** + * If this test fails the spring cache wiring is broken + */ @Test public void testCache() { assertEquals(c.getData("a"), c.getData("a"));