Merging develop into main (#74)

* Changes to add message parser and support for service bus * Adding java wrapper * fixing wrapper test * Adding mesage parser unit test * Adding test cases with status of success, failute and warning to get the correct status * Adding test cases for hl7 report parsing * code fix * Remove function kotlin wrapper * Removing unneeded file * Adding missing fields in other reports * Adding test case for file copy report format * Code refactoring * Adding test case for metadata verify file * Adding changes for rule-engine * Fixing changes as per code review * Fixing test cases * Adding rule engine * Adding test for rules engine * Adding code to test ruleEngine * Adding Servicebus endpoint * Adding regex for validation of websocket url * code fixes for string matching for valid web socket url * testing web socket string * testing web socket string * fixing statustype * Adding logs * Making log statments implement debug method than info and adding more info to test * reverting local changes in build.gradle * DASB-289 - Static cosmosdb connection for the reports manager (#64) * DASB-289 - Static cosmosdb connection for the reports manager (#66) * DASB-289 - Static cosmosdb connection for the reports manager * Fixed logic where reports container can be null, like in the case of unit tests where there is no actual cosmos db connection --------- Co-authored-by: Matt B Krystof <[email protected]> * DASB-291 - Added feature to disable tracing using app setting (#67) * Code cleanup * Added kill switch for tracing --------- Co-authored-by: Matt B Krystof <[email protected]> * DASB-296 - Updates to report counts (#68) * Include timestamp with the counts reports * Added new query param for getting counts to specify days interval --------- Co-authored-by: Matt B Krystof <[email protected]> * Added retries to getting traces and added test scripts (#69) Co-authored-by: Matt B Krystof <[email protected]> * DASB-230 - Meta V2 Implementation (#70) * DASB-230 - Meta V2 Implementation * DASB-230 - Meta V2 Implementation corrections * DASB-269 - Throw exception for Malformed request body * DASB-295 - Add custom metrics for App Insights, cosmos exception handling (#72) * Added initial app insights metrics * Add exception handling to cosmos createItem --------- Co-authored-by: Matt B Krystof <[email protected]> * Updated schema names for HL7v2 debatcher and structure validator (#73) Co-authored-by: Matt B Krystof <[email protected]> --------- Co-authored-by: Nagpal <[email protected]> Co-authored-by: Purnima Nagpal <[email protected]> Co-authored-by: Matt B Krystof <[email protected]> Co-authored-by: Surya Talla <[email protected]>
CDCgov · Mar 5, 2024 · 1dcfba0 · 1dcfba0
1 parent 5b466d5
commit 1dcfba0
Show file tree

Hide file tree

Showing 99 changed files with 3,049 additions and 495 deletions.
diff --git a/docs/processing-status-openapi-spec.yaml b/docs/processing-status-openapi-spec.yaml
@@ -61,18 +61,18 @@ paths:
           schema:
             type: string
           description: Upload identifier.
-        - name: destinationId
+        - name: dataStreamId
           in: query
           required: true
           schema:
             type: string
-          description: Destination identifier.
-        - name: eventType
+          description: Data Stream identifier.
+        - name: dataStreamRoute
           in: query
           required: true
           schema:
             type: string
-          description: Event Type.
+          description: Data Stream Route.
       responses:
         '200':
           description: Trace and parent span information for the trace created
@@ -81,7 +81,7 @@ paths:
               schema:
                 $ref: '#/components/schemas/CreateTrace'
         '400':
-          description: uploadId, destinationId and eventType are required.
+          description: uploadId, dataStreamId and dataStreamRoute are required.
         '401':
           description: Authorization information is missing or invalid.
   '/trace/startSpan/{traceId}/{parentSpanId}':
@@ -283,18 +283,18 @@ paths:
           schema:
             type: string
           description: Upload ID.
-        - name: destinationId
+        - name: dataStreamId
           in: query
           required: true
           schema:
             type: string
-          description: Destination ID.
-        - name: eventType
+          description: Data Stream ID.
+        - name: dataStreamRoute
           in: query
           required: true
           schema:
             type: string
-          description: Event Type.
+          description: Data Stream Route.
         - name: stageName
           in: query
           required: true
@@ -323,18 +323,18 @@ paths:
           schema:
             type: string
           description: Upload ID.
-        - name: destinationId
+        - name: dataStreamId
           in: query
           required: true
           schema:
             type: string
-          description: Destination ID.
-        - name: eventType
+          description: Data Stream ID.
+        - name: dataStreamRoute
           in: query
           required: true
           schema:
             type: string
-          description: Event Type.
+          description: Data Stream Route.
         - name: stageName
           in: query
           required: true
@@ -364,7 +364,7 @@ paths:
           required: true
           schema:
             type: string
-          description: Destination identifier to retrieve all the upload statuses for.
+          description: Upload identifier to retrieve all the upload statuses for.
       responses:
         '200':
           description: Report for the given identifier
@@ -415,7 +415,7 @@ paths:
           required: true
           schema:
             type: string
-          description: Destination identifier to retrieve all the upload statuses for.
+          description: Trace identifier to retrieve all the upload statuses for.
       responses:
         '200':
           description: Report for the given identifier
@@ -429,27 +429,27 @@ paths:
           description: Bad request.  The identifier provided was not found.
         '401':
           description: Authorization information is missing or invalid.
-  '/report/ids/{destinationId}':
+  '/report/ids/{dataStreamId}':
     get:
       tags:
         - DEX Reports
       summary: Provides list of report identifiers for given criteria. (Not yet implemented)
       description: >-
         Provides list of report identifiers for the given criteria. **(Not yet implemented)**
       parameters:
-        - name: ext_event
+        - name: dataStreamRoute
           in: query
           schema:
             type: string
           description: >-
-            If specified, also filter by meta_ext_event that was provided with the upload.
+            If specified, also filter by dataStreamRoute that was provided with the upload.
           example: RoutineImmunization
-        - name: destinationId
+        - name: dataStreamId
           in: path
           required: true
           schema:
             type: string
-          description: Destination identifier to retrieve all the upload statuses for.
+          description: Data Stream identifier to retrieve all the upload statuses for.
         - name: date_start
           in: query
           schema:
@@ -490,17 +490,17 @@ paths:
           description: Bad request.  The identifier provided was not found.
         '401':
           description: Authorization information is missing or invalid.
-  '/info/destinationId':
+  '/info/dataStreamId':
     get:
       tags:
         - DEX Info
       summary: >-
-        Provides a list of all known destination IDs. (Not yet implemented)
+        Provides a list of all known data stream IDs. (Not yet implemented)
       description: >-
-        Provides a list of all known destination IDs. **(Not yet implemented)**
+        Provides a list of all known data stream IDs. **(Not yet implemented)**
       responses:
         '200':
-          description: All of the known destination identifiers.
+          description: All of the known data stream identifiers.
           content:
             application/json:
               schema:
@@ -509,31 +509,31 @@ paths:
                   type: string
         '401':
           description: Authorization information is missing or invalid.
-  '/info/{destinationId}/eventTypes':
+  '/info/{dataStreamId}/routes':
     get:
       tags:
         - DEX Info
       summary: >-
-        Provides a list of all known event types. (Not yet implemented)
+        Provides a list of all known routes. (Not yet implemented)
       description: >-
-        Provides a list of all known event types for the given destination identifier. **(Not yet implemented)**
+        Provides a list of all known routes for the given dataStreamId identifier. **(Not yet implemented)**
       parameters:
-        - name: destinationId
+        - name: dataStreamId
           in: path
           required: true
           schema:
             type: string
       responses:
         '200':
-          description: All of the known event types
+          description: All of the known routes
           content:
             application/json:
               schema:
                 type: array
                 items:
                   type: string
         '400':
-          description: Bad request.  The destination identifier provided was not found.
+          description: Bad request.  The data stream identifier provided was not found.
         '401':
           description: Authorization information is missing or invalid.
   '/info/reportSchemas':
@@ -587,28 +587,28 @@ paths:
           description: Bad request.  The upload identifier provided was not found.
         '401':
           description: Authorization information is missing or invalid.
-  '/upload/{destinationId}':
+  '/upload/{dataStreamId}':
     get:
       tags:
         - DEX Status
       summary: Provides all upload statuses for the given destination identifier, further filtered by the optional ext_event and date range parameters.
       description: >-
         Provides report for the given identifier.
       parameters:
-        - name: ext_event
+        - name: data_stream_route
           in: query
           schema:
             type: string
           description: >-
-            If specified, also filter by meta_ext_event that was provided with
+            If specified, also filter by data_stream_route that was provided with
             the upload.
           example: RoutineImmunization
-        - name: destinationId
+        - name: dataStreamId
           in: path
           required: true
           schema:
             type: string
-          description: Destination identifier to retrieve all the upload statuses for.
+          description: Data stream identifier to retrieve all the upload statuses for.
         - name: date_start
           in: query
           schema:
@@ -641,7 +641,7 @@ paths:
           in: query
           schema:
             type: string
-          description: Can sort by date or event
+          description: Can sort by date or route
           example: date
         - name: sort_order
           in: query
@@ -695,18 +695,18 @@ paths:
       description: >-
         Provides report counts for the given criteria.
       parameters:
-        - name: destination_id
+        - name: data_stream_id
           in: query
           required: true
           schema:
             type: string
-          description: Filter by the destination identifier to retrieve the report counts for.
-        - name: ext_event
+          description: Filter by the data stream identifier to retrieve the report counts for.
+        - name: data_stream_route
           in: query
           required: true
           schema:
             type: string
-          description: Filter by the ext event that was provided with the upload.
+          description: Filter by the data stream route that was provided with the upload.
           example: routineImmunization
         - name: date_start
           in: query
@@ -753,26 +753,26 @@ paths:
                   timestamp:
                     type: string
                     format: date
-  '/notifications/subscribe/email/{destinationId}/{eventType}':
+  '/notifications/subscribe/email/{dataStreamId}/{dataStreamRoute}':
     post:
       tags:
         - DEX Notifications
-      summary: Create a subscription on the given destinationId, stageNames and event types for the provided email.
+      summary: Create a subscription on the given destinationId, stageNames and routes for the provided email.
       description: >-
         Creates a new email subscription for the notification of any type of status updates(warnings, errors or success) of pipeline.
       parameters:
-        - name: destinationId
+        - name: dataStreamId
           in: path
           required: true
           schema:
             type: string
-          description: Destination Id or Program Id to subscribe for notifications.
-        - name: eventType
+          description: Data Stream Id or Program Id to subscribe for notifications.
+        - name: dataStreamRoute
           in: path
           required: true
           schema:
             type: string
-          description: Event type within the program to subscribe.
+          description: Data Stream Route within the program to subscribe.
         - name: email
           in: query
           required: true
@@ -805,26 +805,26 @@ paths:
           description: Authorization information is missing or invalid.
         '403':
           description: Unauthorized access.
-  '/notifications/subscribe/websocket/{destinationId}/{eventType}':
+  '/notifications/subscribe/websocket/{dataStreamId}/{dataStreamRoute}':
     post:
       tags:
         - DEX Notifications
-      summary: Create a subscription on the given destinationId, stageNames and event types for the provided web socket url.
+      summary: Create a subscription on the given dataStreamId, stageNames and dataStreamRoute for the provided web socket url.
       description: >-
         Creates a new websocket subscription for the notification of any type of status updates(warnings, errors or success) of pipeline.
       parameters:
-        - name: destinationId
+        - name: dataStreamId
           in: path
           required: true
           schema:
             type: string
-          description: Destination ID
-        - name: eventType
+          description: Data Stream Id
+        - name: dataStreamRoute
           in: path
           required: true
           schema:
             type: string
-          description: Event type to subscribe.
+          description: DataStreamRoute to subscribe.
         - name: url
           in: query
           required: true
@@ -917,9 +917,9 @@ components:
           type: number
           description: length of time in microseconds the stage ran or has run if status is in_progress
           example: 765024
-        destination_id:
+        data_stream_id:
           type: string
-        event_type:
+        data_stream_route:
           type: string
         metadata:
           type: array
@@ -994,10 +994,10 @@ components:
         upload_id:
           type: string
           format: uuid
-        destination_id:
+        data_stream_id:
           type: string
           example: ndlp
-        event_type:
+        data_stream_route:
           type: string
           example: routineImmunization
         stages:

diff --git a/processing-status-api-function-app/build.gradle b/processing-status-api-function-app/build.gradle
@@ -30,6 +30,7 @@ configurations {
 
 dependencies {
     implementation "com.microsoft.azure.functions:azure-functions-java-library:3.0.0"
+    implementation 'com.microsoft.azure:applicationinsights-core:3.4.19'
     implementation 'com.azure:azure-cosmos:4.55.0'
     implementation "com.fasterxml.jackson.module:jackson-module-kotlin:2.14.2"
     implementation 'io.opentelemetry:opentelemetry-api:1.29.0'

diff --git a/...api-function-app/src/main/java/gov/cdc/ocio/processingstatusapi/FunctionJavaWrappers.java b/...api-function-app/src/main/java/gov/cdc/ocio/processingstatusapi/FunctionJavaWrappers.java
@@ -175,26 +175,26 @@ public HttpResponseMessage getUploadStatus(
             @HttpTrigger(
                     name = "req",
                     methods = {HttpMethod.GET},
-                    route = "upload/{destinationId}",
+                    route = "upload/{dataStreamId}",
                     authLevel = AuthorizationLevel.ANONYMOUS
             ) HttpRequestMessage<Optional<String>> request,
-            @BindingName("destinationId") String destinationId) {
-        return new GetUploadStatusFunction(request).uploadStatus(destinationId, "dex-upload");
+            @BindingName("dataStreamId") String dataStreamId) {
+        return new GetUploadStatusFunction(request).uploadStatus(dataStreamId, "dex-upload");
     }
 
     @FunctionName("GetReportForStage")
     public HttpResponseMessage getReportByStage(
             @HttpTrigger(
                     name = "req",
                     methods = {HttpMethod.GET},
-                    route = "report/{destinationId}/{stageName}",
+                    route = "report/{dataStreamId}/{stageName}",
                     authLevel = AuthorizationLevel.ANONYMOUS
             ) HttpRequestMessage<Optional<String>> request,
-            @BindingName("destinationId") String destinationId,
+            @BindingName("dataStreamId") String dataStreamId,
             @BindingName("stageName") String stageName,
             final ExecutionContext context) {
-        context.getLogger().info("getReportByStage: destinationId=" + destinationId + ", stageName=" + stageName);
-        return new GetReportFunction(request).withDestinationId(destinationId, stageName);
+        context.getLogger().info("getReportByStage: dataStreamId=" + dataStreamId + ", stageName=" + stageName);
+        return new GetReportFunction(request).withDataStreamId(dataStreamId, stageName);
     }
 
     @FunctionName("GetReportCountsByUploadId")