From c86881694263092b530a9a036a0b8a8ad5ea9a3e Mon Sep 17 00:00:00 2001 From: Ivan Bella <347158+ivakegg@users.noreply.github.com> Date: Mon, 25 Nov 2024 15:54:10 +0000 Subject: [PATCH] Updated profiles to optionally add former submodules into build/test Updated test workflow to add all former submodules into build/.test Changed README.md instructions from submodules to subrepo Updated build readme --- .github/workflows/tests.yml | 16 +-- BUILDME.md | 2 +- README.md | 97 +++++++++----- core/pom.xml | 37 ++++-- core/utils/pom.xml | 46 ++++--- microservices/pom.xml | 24 +--- microservices/services/accumulo/pom.xml | 2 +- microservices/services/audit/pom.xml | 2 +- microservices/services/authorization/pom.xml | 2 +- microservices/services/dictionary/pom.xml | 2 +- microservices/services/file-provider/pom.xml | 4 +- microservices/services/hazelcast/pom.xml | 2 +- .../services/mapreduce-query/pom.xml | 2 +- microservices/services/modification/pom.xml | 2 +- microservices/services/pom.xml | 118 +++++++++++------- microservices/services/query-executor/pom.xml | 2 +- microservices/services/query-metric/pom.xml | 2 +- microservices/services/query/pom.xml | 2 +- microservices/starters/pom.xml | 73 ++++++----- 19 files changed, 257 insertions(+), 180 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 0e38efb41c1..37ba7e67163 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -74,7 +74,7 @@ jobs: USER_NAME: ${{ secrets.USER_NAME }} ACCESS_TOKEN: ${{ secrets.ACCESS_TOKEN }} run: | - RUN_TESTS="mvn -s $GITHUB_WORKSPACE/.github/workflows/settings.xml -V -B -e -ntp "-Dstyle.color=always" -Pdev,examples,assemble,spotbugs -Dmaven.build.cache.enabled=false -Ddeploy -Ddist -T1C clean verify" + RUN_TESTS="mvn -s $GITHUB_WORKSPACE/.github/workflows/settings.xml -V -B -e -ntp "-Dstyle.color=always" -Pdev,examples,assemble,spotbugs -Dmaven.build.cache.enabled=false -Ddeploy -Ddist -T1C clean verify -Dutils -Dservices -Dstarters" $RUN_TESTS \ || { echo "***** TESTS FAILED. Attempting retry."; $RUN_TESTS; } \ || { echo "***** TESTS FAILED. Attempting final retry."; $RUN_TESTS; } @@ -110,14 +110,14 @@ jobs: # USER_NAME: ${{ secrets.USER_NAME }} # ACCESS_TOKEN: ${{ secrets.ACCESS_TOKEN }} # run: | - # BUILD="mvn -s $GITHUB_WORKSPACE/.github/workflows/settings.xml -V -B -e -Pdev,examples,assemble,spotbugs -Dmaven.build.cache.enabled=false -DskipServices -Ddeploy -Ddist -T1C -pl "-:config-service" clean install -DskipTests" + # BUILD="mvn -s $GITHUB_WORKSPACE/.github/workflows/settings.xml -V -B -e -Dstarters -Dservices -Dutils -Pdev,examples,assemble,spotbugs -Dmaven.build.cache.enabled=false -DonlyServiceApis -Ddeploy -Ddist -T1C -pl "-:config-service" clean install -DskipTests" # $BUILD # - name: Run Microservice Unit Tests # env: # USER_NAME: ${{ secrets.USER_NAME }} # ACCESS_TOKEN: ${{ secrets.ACCESS_TOKEN }} # run: | - # RUN_TESTS="mvn -s $GITHUB_WORKSPACE/.github/workflows/settings.xml -V -B -e -Dmaven.build.cache.enabled=false verify" + # RUN_TESTS="mvn -s $GITHUB_WORKSPACE/.github/workflows/settings.xml -V -B -e -Dstarters -Dservices -Dutils -Dmaven.build.cache.enabled=false verify" # cd microservices # $RUN_TESTS \ # || { echo "***** TESTS FAILED. Attempting retry."; $RUN_TESTS; } \ @@ -183,13 +183,13 @@ jobs: ACCESS_TOKEN: ${{ secrets.ACCESS_TOKEN }} run: | # update datawave dependencies to use the latest snapshots - mvn -s $GITHUB_WORKSPACE/.github/workflows/settings.xml -B -V -e versions:update-properties versions:update-parent -DallowSnapshots=true -Dincludes=gov.nsa.* + mvn -s $GITHUB_WORKSPACE/.github/workflows/settings.xml -B -V -e -Dstarters -Dservices -Dutils versions:update-properties versions:update-parent -DallowSnapshots=true -Dincludes=gov.nsa.* - mvn -s $GITHUB_WORKSPACE/.github/workflows/settings.xml -B -V -e -Pcompose -Dmicroservice-docker -Dquickstart-docker -Ddeploy -Dtar -DskipTests clean install + mvn -s $GITHUB_WORKSPACE/.github/workflows/settings.xml -B -V -e -Dstarters -Dservices -Dutils -Pcompose -Dmicroservice-docker -Dquickstart-docker -Ddeploy -Dtar -DskipTests clean install # free up some space so that we don't run out docker system prune -f - mvn -s $GITHUB_WORKSPACE/.github/workflows/settings.xml -B -V -e -Pcompose -Dmicroservice-docker -Dquickstart-docker -Ddeploy -Dtar -DskipTests clean + mvn -s $GITHUB_WORKSPACE/.github/workflows/settings.xml -B -V -e -Dstarters -Dservices -Dutils -Pcompose -Dmicroservice-docker -Dquickstart-docker -Ddeploy -Dtar -DskipTests clean cd docker ./bootstrap.sh @@ -264,10 +264,10 @@ jobs: -Durl.hadoop=https://bogus.apache.org/hadoop/common/hadoop-3.3.6/hadoop-3.3.6.tar.gz \ -Durl.maven=https://bogus.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz" - mvn -s $GITHUB_WORKSPACE/.github/workflows/settings.xml -B -V -e -Pcompose -Dmicroservice-docker -Dquickstart-docker -Dquickstart-maven ${DIST_URLS} -Ddeploy -Dtar -DskipTests -Dmaven.build.cache.enabled=false clean install + mvn -s $GITHUB_WORKSPACE/.github/workflows/settings.xml -B -V -e -Dstarters -Dservices -Dutils -Pcompose -Dmicroservice-docker -Dquickstart-docker -Dquickstart-maven ${DIST_URLS} -Ddeploy -Dtar -DskipTests -Dmaven.build.cache.enabled=false clean install # free up some space so that we don't run out docker system prune -f - mvn -s $GITHUB_WORKSPACE/.github/workflows/settings.xml -B -V -e -Pcompose -Dmicroservice-docker -Dquickstart-docker -Dquickstart-maven ${DIST_URLS} -Ddeploy -Dtar -DskipTests -Dmaven.build.cache.enabled=false clean + mvn -s $GITHUB_WORKSPACE/.github/workflows/settings.xml -B -V -e -Dstarters -Dservices -Dutils -Pcompose -Dmicroservice-docker -Dquickstart-docker -Dquickstart-maven ${DIST_URLS} -Ddeploy -Dtar -DskipTests -Dmaven.build.cache.enabled=false clean cd docker ./bootstrap.sh attempt=0 diff --git a/BUILDME.md b/BUILDME.md index fbd6744675f..8cdb019042b 100644 --- a/BUILDME.md +++ b/BUILDME.md @@ -65,7 +65,7 @@ mvn -Pdev,assemble,rpm -Ddeploy -Dtar -Ddist -DskipTests clean install Datawave web services utilize several microservices at runtime (currently authorization and auditing, although that list will expand soon). Datawave depends on api modules for some of these services, and the dependencies are set in the parent pom (see `version.datawave.*` properties) to released versions. If you wish to build the microservices -for some reason, you can simply add `-Dservices` to your maven build command. +for some reason, you can simply add `-Dservices` to your maven build command. If you wish to build the starters you can add `-Dstarters` and for the utility modules add `-Dutils`. ### Releasing Microservices diff --git a/README.md b/README.md index 2b154209b7b..3530fb0a2c3 100644 --- a/README.md +++ b/README.md @@ -19,42 +19,73 @@ Basic build instructions are [here](BUILDME.md) ## How to Use this Repository -The microservices and associated utility projects are intended to be -developed, versioned, and released independently and as such are stored -in separate repositories. This repository includes them all as submodules -in order to provide an easy way to import them all in an IDE for viewing -the code, or refactoring. [Git submodules](https://git-scm.com/book/en/v2/Git-Tools-Submodules) -require some extra commands over the normal ones that one may be familiar -with. - -### Cloning with all submodules -Cloning with all of the submodules is not required; however, if you are interested in checking -out and building all of the datawave projects under one repo, read this! - -It's easiest to clone the repository pointing the submodules at the same branch -```bash -# Start out by cloning the project as you normally would. -git clone git@github.com:NationalSecurityAgency/datawave.git - -# Now, use git to retrieve all of the datawave submodules. -# This will leave your submodules in a detached head state. -cd datawave -git submodule update --init --recursive - -# You can checkout the main branch for each submodule so that you are no longer in a detached head state. -# The addition of `|| :` will ensure that the command is executed for each submodule, -# ignoring failures for submodules that don't have a main branch. -git submodule foreach 'git checkout main || :' - -# It is recommended to build the project using multiple threads. -mvn -Pdocker,dist clean install -T 1C +The microservices and associated utility projects are intended to be developed, versioned, +and released independently. The following subdirectories contain those independently +versioned modules: + +``` +core/utils/type-utils +contrib/datawave-utils +core/base-rest-responses +core/in-memory-accumulo +core/metrics-reporter +core/utils/accumulo-utils +core/utils/common-utils +core/utils/metadata-utils +microservices/microservice-parent +microservices/microservice-service-parent +microservices/starters/audit +microservices/starters/cache +microservices/starters/cached-results +microservices/starters/datawave +microservices/starters/metadata +microservices/starters/query +microservices/starters/query-metric +microservices/services/accumulo +microservices/services/audit +microservices/services/authorization +microservices/services/config +microservices/services/dictionary +microservices/services/file-provider +microservices/services/hazelcast +microservices/services/map +microservices/services/mapreduce-query +microservices/services/modification +microservices/services/query +microservices/services/query-executor +microservices/services/query-metric +``` -# If you don't want to build the microservices, you can skip them. -mvn -Pdocker,dist -DskipMicroservices clean install -T 1C +Each of those subdirectories contain a .gitrepo file that keeps track of where the code came from. -# If you decide that you no longer need the submodules, you can remove them. -git submodule deinit --all +### Updating one of the datawave sub-repositories +At one point we used submodules to link in a all of the sub-repositories. We have now switched +to including the submodules' code directly into the main datawave repository. The git subrepo +mechanism (https://github.com/ingydotnet/git-subrepo) was used to facilitate the transition. +That same mechanism can be used to pull in changes from the other repositories as needed until +they can be removed altogether. The original cloning of the sub repositories was done using +the subrepo command as follows: ``` +git subrepo clone +``` +If changes need to be pulled in, then the following process can be used: +``` +git subrepo pull +``` +### Building + +It is recommended to build the project using multiple threads. +``` +mvn -Pdocker,dist clean install -T 1C +``` + + If you don't want to build the starters, util modules, and services then try this +``` +mvn -Pdocker,dist -Dstarters -Dservices -Dutils clean install -T 1C +``` + +NOTE: The util modules, starters, and services are actually tagged and deployed separately. + Hence the snapshot versions within those sub repos are not connected together. ### DataWave Microservices diff --git a/core/pom.xml b/core/pom.xml index cd285f42816..ff9f4618bc0 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -87,37 +87,50 @@ - submodule-base-rest-responses + base-rest-responses - - base-rest-responses/pom.xml - + + base-rest-responses + base-rest-responses - submodule-in-memory-accumulo + in-memory-accumulo - - in-memory-accumulo/pom.xml - + + in-memory-accumulo + in-memory-accumulo - submodule-metrics-reporter + metrics-reporter - - metrics-reporter/pom.xml - + + metrics-reporter + metrics-reporter + + utils + + + utils + + + + base-rest-responses + in-memory-accumulo + metrics-reporter + + diff --git a/core/utils/pom.xml b/core/utils/pom.xml index ba22ec0dd00..b61a48bec10 100644 --- a/core/utils/pom.xml +++ b/core/utils/pom.xml @@ -13,48 +13,62 @@ - submodule-accumulo-utils + accumulo-utils - - accumulo-utils/pom.xml - + + accumulo-utils + accumulo-utils - submodule-common-utils + common-utils - - common-utils/pom.xml - + + common-utils + common-utils - submodule-metadata-utils + metadata-utils - - metadata-utils/pom.xml - + + metadata-utils + metadata-utils - submodule-type-utils + type-utils - - type-utils/pom.xml - + + type-utils + type-utils + + utils + + + utils + + + + accumulo-utils + common-utils + metadata-utils + type-utils + + diff --git a/microservices/pom.xml b/microservices/pom.xml index 722195928eb..dc871b5ad7d 100644 --- a/microservices/pom.xml +++ b/microservices/pom.xml @@ -11,32 +11,12 @@ pom ${project.artifactId} + microservice-parent + microservice-service-parent services starters - - submodule-microservice-parent - - - microservice-parent/pom.xml - - - - microservice-parent - - - - submodule-microservice-service-parent - - - microservice-service-parent/pom.xml - - - - microservice-service-parent - - configcheck diff --git a/microservices/services/accumulo/pom.xml b/microservices/services/accumulo/pom.xml index 11c4f203860..56254ad67e8 100644 --- a/microservices/services/accumulo/pom.xml +++ b/microservices/services/accumulo/pom.xml @@ -37,7 +37,7 @@ services - !skipServices + !onlyServiceApis diff --git a/microservices/services/audit/pom.xml b/microservices/services/audit/pom.xml index f21b2ba8a36..17dfe3e6679 100644 --- a/microservices/services/audit/pom.xml +++ b/microservices/services/audit/pom.xml @@ -37,7 +37,7 @@ services - !skipServices + !onlyServiceApis diff --git a/microservices/services/authorization/pom.xml b/microservices/services/authorization/pom.xml index 60df2ddfa70..31bffb2aa93 100644 --- a/microservices/services/authorization/pom.xml +++ b/microservices/services/authorization/pom.xml @@ -37,7 +37,7 @@ services - !skipServices + !onlyServiceApis diff --git a/microservices/services/dictionary/pom.xml b/microservices/services/dictionary/pom.xml index d58b8ef32a1..a9a7e0b987a 100644 --- a/microservices/services/dictionary/pom.xml +++ b/microservices/services/dictionary/pom.xml @@ -37,7 +37,7 @@ services - !skipServices + !onlyServiceApis diff --git a/microservices/services/file-provider/pom.xml b/microservices/services/file-provider/pom.xml index cfbd15858d7..651ee05022d 100644 --- a/microservices/services/file-provider/pom.xml +++ b/microservices/services/file-provider/pom.xml @@ -37,7 +37,7 @@ services - !skipServices + !onlyServiceApis @@ -45,4 +45,4 @@ - \ No newline at end of file + diff --git a/microservices/services/hazelcast/pom.xml b/microservices/services/hazelcast/pom.xml index 0345b832c64..a8de82f8928 100644 --- a/microservices/services/hazelcast/pom.xml +++ b/microservices/services/hazelcast/pom.xml @@ -38,7 +38,7 @@ services - !skipServices + !onlyServiceApis diff --git a/microservices/services/mapreduce-query/pom.xml b/microservices/services/mapreduce-query/pom.xml index cbf266701df..cf70edf8eda 100644 --- a/microservices/services/mapreduce-query/pom.xml +++ b/microservices/services/mapreduce-query/pom.xml @@ -36,7 +36,7 @@ services - !skipServices + !onlyServiceApis diff --git a/microservices/services/modification/pom.xml b/microservices/services/modification/pom.xml index 57b79217c3a..932e2c025df 100644 --- a/microservices/services/modification/pom.xml +++ b/microservices/services/modification/pom.xml @@ -37,7 +37,7 @@ services - !skipServices + !onlyServiceApis diff --git a/microservices/services/pom.xml b/microservices/services/pom.xml index 209db1ba076..4235258f684 100644 --- a/microservices/services/pom.xml +++ b/microservices/services/pom.xml @@ -12,136 +12,158 @@ - submodule-service-accumulo + service-accumulo - - accumulo/pom.xml - + + service-accumulo + accumulo - submodule-service-audit + service-audit - - audit/pom.xml - + + service-audit + audit - submodule-service-authorization + service-authorization - - authorization/pom.xml - + + service-authorization + authorization - submodule-service-config + service-config - - config/pom.xml - + + service-config + config - submodule-service-dictionary + service-dictionary - - dictionary/pom.xml - + + service-dictionary + dictionary - submodule-service-file-provider + service-file-provider - - file-provider/pom.xml - + + service-file-provider + file-provider - submodule-service-hazelcast + service-hazelcast - - hazelcast/pom.xml - + + service-hazelcast + hazelcast - submodule-service-mapreduce-query + service-mapreduce-query - - mapreduce-query/pom.xml - + + service-mapreduce-query + mapreduce-query - submodule-service-modification + service-modification - - modification/pom.xml - + + service-modification + modification - submodule-service-query + service-query - - query/pom.xml - + + service-query + query - submodule-service-query-executor + service-query-executor - - query-executor/pom.xml - + + service-query-executor + query-executor - submodule-service-query-metric + service-query-metric - - query-metric/pom.xml - + + service-query-metric + query-metric + + services + + + services + + + + accumulo + audit + authorization + config + dictionary + file-provider + hazelcast + mapreduce-query + modification + query + query-executor + query-metric + + diff --git a/microservices/services/query-executor/pom.xml b/microservices/services/query-executor/pom.xml index c44f5f3ef41..54513a00657 100644 --- a/microservices/services/query-executor/pom.xml +++ b/microservices/services/query-executor/pom.xml @@ -36,7 +36,7 @@ services - !skipServices + !onlyServiceApis diff --git a/microservices/services/query-metric/pom.xml b/microservices/services/query-metric/pom.xml index e51cb2214d7..e97b4ae8341 100644 --- a/microservices/services/query-metric/pom.xml +++ b/microservices/services/query-metric/pom.xml @@ -37,7 +37,7 @@ services - !skipServices + !onlyServiceApis diff --git a/microservices/services/query/pom.xml b/microservices/services/query/pom.xml index 2f8b3f005b1..9f5a0869424 100644 --- a/microservices/services/query/pom.xml +++ b/microservices/services/query/pom.xml @@ -36,7 +36,7 @@ services - !skipServices + !onlyServiceApis diff --git a/microservices/starters/pom.xml b/microservices/starters/pom.xml index d3c602386bc..9f098e9c281 100644 --- a/microservices/starters/pom.xml +++ b/microservices/starters/pom.xml @@ -12,81 +12,98 @@ - submodule-starter-audit + starter-audit - - audit/pom.xml - + + starter-audit + audit - submodule-starter-cache + starter-cache - - cache/pom.xml - + + starter-cache + cache - submodule-starter-cached-results + starter-cached-results - - cached-results/pom.xml - + + starter-cached-results + cached-results - submodule-starter-datawave + starter-datawave - - datawave/pom.xml - + + starter-datawave + datawave - submodule-starter-metadata + starter-metadata - - metadata/pom.xml - + + starter-metadata + metadata - submodule-starter-query + starter-query - - query/pom.xml - + + starter-query + query - submodule-starter-query-metric + query-metric - - query-metric/pom.xml - + + starter-query-metric + query-metric + + starters + + + starters + + + + audit + cache + cached-results + datawave + metadata + query + query-metric + +