Update the rule engine to never save the KB blob to the database.

The KB from the database is not required to be saved to function. This will save a large data-call to the database as the rule blob is saved as a large text file. #2020
US-CBP · Mar 12, 2021 · 705fbd2 · 705fbd2
1 parent 9222ea5
commit 705fbd2
Show file tree

Hide file tree

Showing 7 changed files with 74 additions and 28 deletions.
diff --git a/gtas-parent/gtas-commons/src/main/java/gov/gtas/model/udr/KnowledgeBase.java b/gtas-parent/gtas-commons/src/main/java/gov/gtas/model/udr/KnowledgeBase.java
@@ -20,6 +20,7 @@
 import javax.persistence.Table;
 import javax.persistence.Temporal;
 import javax.persistence.TemporalType;
+import javax.persistence.Transient;
 import javax.persistence.UniqueConstraint;
 import javax.persistence.Version;
 
@@ -42,14 +43,10 @@ public class KnowledgeBase extends BaseEntity {
 	@Column(name = "KB_NAME", nullable = false, length = 20)
 	private String kbName;
 
-	@Lob
-	@Basic(fetch = FetchType.EAGER)
-	@Column(name = "KB_BLOB", nullable = false, length = 2000000)
+	@Transient
 	private byte[] kbBlob;
 
-	@Lob
-	@Basic(fetch = FetchType.EAGER)
-	@Column(name = "RL_BLOB", nullable = false, length = 1000000)
+	@Transient
 	private byte[] rulesBlob;
 
 	@Temporal(TemporalType.TIMESTAMP)

diff --git a/...rent/gtas-job-scheduler-war/src/main/java/gov/gtas/job/scheduler/RuleRunnerScheduler.java b/...rent/gtas-job-scheduler-war/src/main/java/gov/gtas/job/scheduler/RuleRunnerScheduler.java
@@ -6,18 +6,25 @@
 package gov.gtas.job.scheduler;
 
 import gov.gtas.constant.RuleConstants;
+import gov.gtas.constant.WatchlistConstants;
 import gov.gtas.job.config.JobSchedulerConfig;
 import gov.gtas.model.Message;
 import gov.gtas.model.MessageStatus;
 import gov.gtas.model.MessageStatusEnum;
 import gov.gtas.model.lookup.AppConfiguration;
 import gov.gtas.model.udr.KnowledgeBase;
+import gov.gtas.model.udr.UdrRule;
+import gov.gtas.model.watchlist.WatchlistItem;
 import gov.gtas.repository.AppConfigurationRepository;
 import gov.gtas.repository.KnowledgeBaseRepository;
 import gov.gtas.repository.MessageStatusRepository;
 import gov.gtas.repository.PendingHitDetailRepository;
+import gov.gtas.repository.watchlist.WatchlistItemRepository;
 import gov.gtas.rule.KIEAndLastUpdate;
 import gov.gtas.rule.RuleUtils;
+import gov.gtas.rule.builder.DrlRuleFileBuilder;
+import gov.gtas.services.udr.RulePersistenceService;
+import gov.gtas.svc.RuleManagementService;
 import gov.gtas.svc.UdrService;
 import gov.gtas.svc.WatchlistService;
 import org.kie.api.KieBase;
@@ -28,6 +35,7 @@
 import org.springframework.context.ApplicationContext;
 import org.springframework.scheduling.annotation.Scheduled;
 import org.springframework.stereotype.Component;
+import org.springframework.util.CollectionUtils;
 
 import java.io.IOException;
 import java.util.*;
@@ -109,26 +117,35 @@ public void ruleEngine() throws InterruptedException, IOException {
 
 		AppConfiguration recompileRulesAndWatchlist = appConfigurationRepository.findByOption(RECOMPILE_RULES);
 		if (!isBlank(recompileRulesAndWatchlist.getOption()) && Boolean.parseBoolean(recompileRulesAndWatchlist.getValue())) {
-			logger.info("RECOMPILING KBS!");
-			watchlistService.activateAllWatchlists();
-			udrService.recompileRules(RuleConstants.UDR_KNOWLEDGE_BASE_NAME, "RULE_SCHEDULER");
+			logger.info("RECOMPILING WL KB!");
+			updateWatchlistKb();
+			logger.info("RECOMPILING RULE KB!");
+			updateRuleKb();
+			logger.info("RECOMPILING FINISHED!");
 			recompileRulesAndWatchlist.setValue("false");
 			appConfigurationRepository.save(recompileRulesAndWatchlist);
 		}
 
 		Iterable<KnowledgeBase> kbs = knowledgeBaseRepository.findAll();
-		for (KnowledgeBase kb : kbs ) {
+		for (KnowledgeBase kb : kbs) {
 			if (rules.containsKey(kb.getKbName())) {
 				KIEAndLastUpdate kau = rules.get(kb.getKbName());
 				if (kau.getUpdated().before(kb.getCreationDt())) {
-					logger.info("updating rule runner kie for " + kb.getKbName());
-					addOrUpdateNameAndKie(kb);
-					logger.info("Done updating rule runner kie for " + kb.getKbName());
+					if (WatchlistConstants.WL_KNOWLEDGE_BASE_NAME.equalsIgnoreCase(kb.getKbName())) {
+						logger.info("Updating Watchlist KB");
+						updateWatchlistKb();
+					} else if (RuleConstants.UDR_KNOWLEDGE_BASE_NAME.equalsIgnoreCase(kb.getKbName())) {
+						logger.info("Updated UDR KB");
+						updateRuleKb();					}
 				}
 			} else {
-				logger.info("making new rule kie for " + kb.getKbName());
-				addOrUpdateNameAndKie(kb);
-				logger.info("Done creating rule kie for " + kb.getKbName());
+				if (WatchlistConstants.WL_KNOWLEDGE_BASE_NAME.equalsIgnoreCase(kb.getKbName())) {
+					logger.info("Creating new WL KB");
+					updateWatchlistKb();
+				} else if (RuleConstants.UDR_KNOWLEDGE_BASE_NAME.equalsIgnoreCase(kb.getKbName())) {
+					logger.info("Creating new UDR KB");
+					updateRuleKb();					
+				}
 			}
 		}
 
@@ -238,12 +255,27 @@ public void ruleEngine() throws InterruptedException, IOException {
 		}
 	}
 
+	private void updateWatchlistKb() throws IOException {
+		KnowledgeBase wlKb = watchlistService.createAKnowledgeBase(WatchlistConstants.WL_KNOWLEDGE_BASE_NAME);
+		if (wlKb != null) {
+			addOrUpdateNameAndKie(wlKb);
+		}
+	}
+
+	private void updateRuleKb() throws IOException {
+		KnowledgeBase udrKnowledgeBase = udrService.recompileRules(RuleConstants.UDR_KNOWLEDGE_BASE_NAME, "RULE_SCHEDULER");
+		if (udrKnowledgeBase != null) {
+			addOrUpdateNameAndKie(udrKnowledgeBase);
+		}
+	}
+
+
 	private void addOrUpdateNameAndKie(KnowledgeBase kb) throws IOException {
-		KieBase kieBase = RuleUtils.createKieBaseFromDrlString(new String(kb.getRulesBlob()));
-		Date updatedDate = kb.getCreationDt();
-		KIEAndLastUpdate KIEAndLastUpdate = new KIEAndLastUpdate();
-		KIEAndLastUpdate.setKieBase(kieBase);
-		KIEAndLastUpdate.setUpdated(updatedDate);
-		rules.put(kb.getKbName(), KIEAndLastUpdate);
+		 		KieBase kieBase = RuleUtils.createKieBaseFromDrlString(new String(kb.getRulesBlob()));
+		 		Date updatedDate = kb.getCreationDt();
+		 		KIEAndLastUpdate KIEAndLastUpdate = new KIEAndLastUpdate();
+		 		KIEAndLastUpdate.setKieBase(kieBase);
+		 		KIEAndLastUpdate.setUpdated(updatedDate);
+		 		rules.put(kb.getKbName(), KIEAndLastUpdate);
 	}
 }
diff --git a/gtas-parent/gtas-rulesvc/src/main/java/gov/gtas/svc/RuleManagementServiceImpl.java b/gtas-parent/gtas-rulesvc/src/main/java/gov/gtas/svc/RuleManagementServiceImpl.java
@@ -95,7 +95,7 @@ public String fetchDefaultDrlRulesFromKnowledgeBase() {
 	}
 
 	@Override
-	@Transactional()
+	@Transactional
 	public KnowledgeBase createKnowledgeBaseFromUdrRules(String kbName, Collection<UdrRule> rules, String userId) {
 		if (!CollectionUtils.isEmpty(rules)) {
 			DrlRuleFileBuilder ruleFileBuilder = new DrlRuleFileBuilder();

diff --git a/gtas-parent/gtas-rulesvc/src/main/java/gov/gtas/svc/UdrService.java b/gtas-parent/gtas-rulesvc/src/main/java/gov/gtas/svc/UdrService.java
@@ -7,6 +7,7 @@
 
 import static gov.gtas.constant.GtasSecurityConstants.PRIVILEGES_ADMIN_AND_MANAGE_RULES;
 import gov.gtas.json.JsonServiceResponse;
+import gov.gtas.model.udr.KnowledgeBase;
 import gov.gtas.model.udr.json.JsonUdrListElement;
 import gov.gtas.model.udr.json.UdrSpecification;
 
@@ -111,5 +112,5 @@ public interface UdrService {
 	@PreAuthorize(PRIVILEGES_ADMIN_AND_MANAGE_RULES)
 	JsonServiceResponse deleteUdr(String userId, Long id);
 
-	public void recompileRules(final String kbName, String userId);
+	public KnowledgeBase recompileRules(final String kbName, String userId);
 }
diff --git a/gtas-parent/gtas-rulesvc/src/main/java/gov/gtas/svc/UdrServiceImpl.java b/gtas-parent/gtas-rulesvc/src/main/java/gov/gtas/svc/UdrServiceImpl.java
@@ -277,12 +277,13 @@ public JsonServiceResponse createUdr(String userId, UdrSpecification udrToCreate
 	 * @param kbName
 	 * @param userId
 	 */
-	public void recompileRules(final String kbName, String userId) {
+	public KnowledgeBase recompileRules(final String kbName, String userId) {
 		List<UdrRule> ruleList = rulePersistenceService.findAll();
+		KnowledgeBase kb = null;
 		if (!CollectionUtils.isEmpty(ruleList)) {
-			ruleManagementService.createKnowledgeBaseFromUdrRules(kbName, ruleList, userId);
+			kb = ruleManagementService.createKnowledgeBaseFromUdrRules(kbName, ruleList, userId);
 		} else {
-			KnowledgeBase kb = rulePersistenceService.findUdrKnowledgeBase(kbName);
+			kb = rulePersistenceService.findUdrKnowledgeBase(kbName);
 			if (kb != null) {
 				List<Rule> rules = rulePersistenceService.findRulesByKnowledgeBaseId(kb.getId());
 				if (CollectionUtils.isEmpty(rules)) {
@@ -291,6 +292,7 @@ public void recompileRules(final String kbName, String userId) {
 				}
 			}
 		}
+		return kb;
 	}
 
 	private User fetchRuleAuthor(final String userId, final String authorUserId) {

diff --git a/gtas-parent/gtas-rulesvc/src/main/java/gov/gtas/svc/WatchlistService.java b/gtas-parent/gtas-rulesvc/src/main/java/gov/gtas/svc/WatchlistService.java
@@ -11,6 +11,7 @@
 import gov.gtas.json.JsonLookupData;
 import gov.gtas.json.JsonServiceResponse;
 import gov.gtas.model.lookup.HitCategory;
+import gov.gtas.model.udr.KnowledgeBase;
 import gov.gtas.model.watchlist.WatchlistItem;
 import gov.gtas.model.watchlist.json.WatchlistSpec;
 
@@ -79,6 +80,11 @@ JsonServiceResponse createUpdateDeleteWatchlistItems(String userId, WatchlistSpe
 	 */
 	JsonServiceResponse activateAllWatchlists(String knowledgeBaseName);
 
+
+	/**
+	 * Compiles all watch lists into a named knowledge base
+	 */
+	public KnowledgeBase createAKnowledgeBase(String knowledgeBaseName);
 	/**
 	 * Compiles all watch lists into the default knowledge base for watch lists.
 	 * 

diff --git a/gtas-parent/gtas-rulesvc/src/main/java/gov/gtas/svc/WatchlistServiceImpl.java b/gtas-parent/gtas-rulesvc/src/main/java/gov/gtas/svc/WatchlistServiceImpl.java
@@ -115,12 +115,20 @@ public List<WatchlistSpec> fetchAllWatchlists() {
 	@Override
 	@Transactional
 	public JsonServiceResponse activateAllWatchlists(String knowledgeBaseName) {
+		KnowledgeBase kb = createAKnowledgeBase(knowledgeBaseName);
+		return WatchlistServiceJsonResponseHelper.createKnowledBaseResponse(kb, null);
+	}
+
+
+	@Transactional
+	@Override
+	public KnowledgeBase createAKnowledgeBase(String knowledgeBaseName) {
 		Iterable<WatchlistItem> items = watchlistPersistenceService.findAllWatchlistItems();
 		if (StringUtils.isEmpty(knowledgeBaseName)) {
 			knowledgeBaseName = WatchlistConstants.WL_KNOWLEDGE_BASE_NAME;
 		}
 		KnowledgeBase kb = ruleManagementService.createKnowledgeBaseFromWatchlistItems(knowledgeBaseName, items);
-		return WatchlistServiceJsonResponseHelper.createKnowledBaseResponse(kb, null);
+		return kb;
 	}
 
 	@Override