Bugfix / fix dcs search

.github/workflows/node.js.yml

@@ -16,7 +16,7 @@ jobs:
 
     strategy:
       matrix:
-        node-version: [16.4.2]
+        node-version: [16.20.2]
         # See supported Node.js release schedule at https://nodejs.org/en/about/releases/
 
     steps:

__tests__/apiHelpers.test.js

@@ -25,7 +25,7 @@ describe('apiHelpers.getCatalog', () => {
         expectMemberType(item, idx, 'downloadUrl', 'string');
       });
     });
-  }, 60000);
+  }, 180000);
 });
 
 /**

package.json

@@ -1,6 +1,6 @@
 {
   "name": "tc-source-content-updater",
-  "version": "1.4.25",
+  "version": "1.4.26",
   "description": "Module that updates source content for the desktop application translationCore.",
   "main": "lib/index.js",
   "display": "library",

src/helpers/apiHelpers.js

@@ -2,7 +2,7 @@
 import {SORT, STAGE, SUBJECT} from '../index';
 import fs from 'fs-extra';
 import path from 'path-extra';
 import * as Bible from '../resources/bible';
 import {
   cleanReaddirSync,
   getLatestVersionsAndOwners,
@@ -13,6 +13,7 @@
 const request = require('request');
 export const DOOR43_CATALOG = `Door43-Catalog`;
 export const CN_CATALOG = `unfoldingWord`;
+export const UNFOLDING_WORD = `unfoldingWord`;
 export const DEFAULT_OWNER = DOOR43_CATALOG;
 export const TRANSLATION_HELPS = 'translationHelps';
 export const EMPTY_TIME = '0001-01-01T00:00:00+00:00';
@@ -173,7 +174,7 @@
  * @return {*}
  */
 function findResource(resourceList, resource) {
   const foundResource = resourceList.find(item => {
     return (item.owner === resource.owner) &&
       (item.language === resource.language) &&
       (item.resourceId === resource.resourceId);
@@ -187,7 +188,7 @@
  * @return {*}
  */
 export function combineTwords(catalogReleases) {
   return catalogReleases.filter(resource => {
     if (resource.owner !== 'Door43-Catalog') {
       switch (resource.resourceId) {
         case 'twl': {
@@ -214,29 +215,50 @@
   });
 }
 
+/**
+ * check if tag name is valid for release version
+ * @param {string} tagName
+ * @return {boolean}
+ */
+function isValidVersionTag(tagName) {
+  const firstChar = tagName && tagName[0];
+  const isDigit = (firstChar >= '0') && (firstChar <= '9');
+  const isValidVersionTag = firstChar && ((firstChar === 'v') || isDigit);
+  return isValidVersionTag;
+}
+
 /**
  * filter list of items and ignore master branch
  * @param {[object]} catalog - array of resources
  * @param {[string]} ignoredResources
+ * @param {[object]} newCatalog - new catalog to search for matches to Door43-Catalog matches in unfoldingWord
  * @return {[object]}
  */
-function filterOutMasterBranch(catalog, ignoredResources = []) {
+function filterOutMasterBranch(catalog, ignoredResources = [], newCatalog = []) {
   let catalog_ = catalog.filter(resource => {
     if (ignoredResources.includes(resource.resource)) {
       return false; // reject ignored resources
     }
 
     const tagName = resource.branch_or_tag_name;
     if (tagName) { // check for version
-      const firstChar = tagName[0];
-      const isDigit = (firstChar >= '0') && (firstChar <= '9');
+      const _isValidVersionTag = isValidVersionTag(tagName);
       const isD43Master = (tagName === 'master') && (resource.owner === DOOR43_CATALOG);
-      if (isD43Master || (firstChar !== 'v' && !isDigit)) {
-        if (!isD43Master) {
-          console.log(`filterOutMasterBranch - invalid version: ${tagName} in ${getResourceInfo(resource)}`);
+      if (isD43Master) {
+        // find the latest release in unfoldingWord org to get version info
+        const sourceResource = findResource(newCatalog, {...resource, owner: UNFOLDING_WORD});
+        if (sourceResource) {
+          const _tagName = sourceResource.branch_or_tag_name;
+          if (isValidVersionTag(_tagName)) {
+            resource.branch_or_tag_name = _tagName;
+            return true;
+          }
         }
-        return false; // reject if tag is not a version
+      } else if (_isValidVersionTag) {
+        return true;
       }
+      console.log(`filterOutMasterBranch - invalid version: ${tagName} in ${getResourceInfo(resource)}`);
+      return false; // reject if tag is not a version
     }
 
     return true;
@@ -260,8 +282,6 @@
   };
   const catalogReleases = await searchCatalogNext(searchParams);
   console.log(`getCatalog - found ${catalogReleases.length} items in old Door43-Catalog`);
-  let catalogReleases_ = filterOutMasterBranch(catalogReleases, ['obs', 'obs-tn']);
-  console.log(`getCatalog - found ${catalogReleases_.length} items in old Door43-Catalog after filter`);
   searchParams = {
     subject: SUBJECT.ALL_TC_RESOURCES,
     stage: config.stage || STAGE.PROD,
@@ -270,12 +290,16 @@
   };
   const newCatalogReleases = await searchCatalogNext(searchParams);
   console.log(`getCatalog - found ${newCatalogReleases.length} items in catalog next`);
+
+  let catalogReleases_ = filterOutMasterBranch(catalogReleases, ['obs', 'obs-tn'], newCatalogReleases);
+  console.log(`getCatalog - found ${catalogReleases_.length} items in old Door43-Catalog after filter`);
+
   let newCatalogReleases_ = filterOutMasterBranch(newCatalogReleases, ['obs', 'obs-tn']);
   console.log(`getCatalog - found ${newCatalogReleases_.length} items in catalog next after filter`);
 
   // merge catalogs together - catalog new takes precedence
   for (const item of newCatalogReleases_) {
     const index = catalogReleases_.findIndex(oldItem => (item.full_name === oldItem.full_name) && (item.full_name === oldItem.full_name));
     if (index >= 0) {
       catalogReleases_[index] = item; // overwrite item in old catalog
       catalogReleases_[index].foundInCatalog = 'NEW+OLD';

src/helpers/parseHelpers.js

@@ -280,7 +280,7 @@
   }
 
   if (filterByOwner) { // we need to remove resources that are not in owner list
     const filteredResources = tCoreResources.filter(resource => (filterByOwner.includes(resource.owner)));
     const itemsRemoved = tCoreResources.length - filteredResources.length;
     console.log(`${itemsRemoved} items removed from filtered list, new length is ${filteredResources.length}`);
     tCoreResources = filteredResources;
@@ -390,6 +390,9 @@
     const isDesiredSubject = !config.subjectFilters ||
       config.subjectFilters.includes(subject);
     let version = catalogItem.version;
+    if (!version) {
+      version = catalogItem.branch_or_tag_name;
+    }
     if (!version) {
       version = 'master'; // we are on latest
     } else if (version[0].toLowerCase() === 'v') { // trim leading v