Merge branch 'master' into CMR-9878

ingest-app/docs/api.md

@@ -159,8 +159,10 @@ The following fields are validated:
 * [Data Format](https://gcmd.earthdata.nasa.gov/kms/concepts/concept_scheme/dataformat?format=csv) - Archival and Distribution File Format, and GetData Format
 * [ProcessingLevel] (https://gcmd.earthdata.nasa.gov/kms/concepts/concept_scheme/productlevelid?format=csv) - productlevelid
 
-**Note**: cmr-validate-keywords is set to TRUE by default
+**Note**: if cmr-validate-keywords header is not set explicitly, it will behave as if it was set to FALSE by default behind the scenes
+
 **Note**: that when multiple fields are present the combination of keywords are validated to match a known combination.
+
 **Note**: Among the validation fields above, [Platforms], [Instruments], [Projects], [Science Keywords], [Location Keywords] and [Data Centers] are also validated when the `Cmr-Validate-Keywords` header is not set to `true` except that  validation errors will be returned to users as warnings.
 
 **Note**: the following fields are always checked:

ingest-app/src/cmr/ingest/api/collections.clj

@@ -7,18 +7,26 @@
    [cmr.common.log :refer [info]]
    [cmr.common.util :as util]
    [cmr.ingest.api.core :as api-core]
+   [cmr.ingest.config :as ingest-config]
    [cmr.ingest.services.ingest-service :as ingest]))
 
 (def VALIDATE_KEYWORDS_HEADER "cmr-validate-keywords")
 (def ENABLE_UMM_C_VALIDATION_HEADER "cmr-validate-umm-c")
 (def TESTING_EXISTING_ERRORS_HEADER "cmr-test-existing-errors")
 
+(def validate-keywords-default-true-enabled?
+  "Checks to see if the feature toggle for validate-keywords-default-true is enabled."
+  (ingest-config/validate-keywords-default-true-enabled))
+
 (defn get-validation-options
   "Returns a map of validation options with boolean values"
   [headers]
-  {:validate-keywords? (if (= "false" (get headers VALIDATE_KEYWORDS_HEADER)) false true)
-   :validate-umm? (= "true" (get headers ENABLE_UMM_C_VALIDATION_HEADER))
-   :test-existing-errors? (= "true" (get headers TESTING_EXISTING_ERRORS_HEADER))})
+  (let [validate-keywords-value (if validate-keywords-default-true-enabled?
+                                  (if (= "false" (get headers VALIDATE_KEYWORDS_HEADER)) false true)
+                                  (= "true" (get headers VALIDATE_KEYWORDS_HEADER)))]
+    {:validate-keywords? validate-keywords-value
+     :validate-umm? (= "true" (get headers ENABLE_UMM_C_VALIDATION_HEADER))
+     :test-existing-errors? (= "true" (get headers TESTING_EXISTING_ERRORS_HEADER))}))
 
 (defn validate-collection
   [provider-id native-id request]

ingest-app/src/cmr/ingest/config.clj

@@ -7,6 +7,10 @@
    [cmr.oracle.config :as oracle-config]
    [cmr.oracle.connection :as conn]))
 
+(defconfig validate-keywords-default-true-enabled
+   "Flag for whether or not cmr-validate-keywords value is defaulted to true or false in backend when missing in ingest api headers"
+   {:default true :type Boolean})
+
 (defconfig progressive-update-enabled
   "Flag for whether or not collection progressive update is enabled."
   {:default true :type Boolean})

search-app/src/cmr/search/api/association.clj

@@ -15,29 +15,48 @@
   [headers]
   (mt/extract-header-mime-type #{mt/json} headers "content-type" true))
 
+(defn add-individual-statuses
+  "When given a list of response entities, will set the http response status of each response entity based on its content.
+  Ex) If the response entity contains an error message, will set the status as 400."
+  [list]
+  (map #(assoc % :status (if (or (:errors %) (:warning %)) 400 200))
+       list))
+
 (defn- api-response
-  "Creates a successful association response with the given data response"
-  ([data]
-   (api-response 200 data))
+  "Creates an association response with the given data response"
   ([status-code data]
-   {:status status-code
-    :body (json/generate-string (util/snake-case-data data))
-    :headers {"Content-Type" mt/json}}))
+   ;; For association responses that partially fail we want to return a 207 and
+   ;; detail which associations failed and/or succeeded in the given body
+   (let [data-value (if (= 207 status-code)
+                      (add-individual-statuses data)
+                      data)]
+     {:status status-code
+      :body (json/generate-string (util/snake-case-data data-value))
+      :headers {"Content-Type" mt/json}})))
 
-(defn- results-contain-errors?
-  "Returns true if the results contain :errors"
+(defn num-errors-in-assoc-results
+  "Counts the number of errors in association-results"
   [results]
-  (seq (filter #(some? (:errors %)) results)))
+  (count (filter :errors results)))
 
 (defn association-results->status-code
-  "Check for concept-types requiring error status to be returned. This is currently :service and :variable
-  If the concept-type is error-sensitive the function will check for any errors in the results, and will return 400 if
-  any are errors are present. Otherwise it will return 200"
+  "Check for concept-types requiring error status to be returned.
+  If the concept-type is error-sensitive the function will check for any errors in the results.
+  Will return:
+  - 200 OK -- if response has no errors
+  - 207 MULTI-STATUS -- if response has some errors and some successes
+  - 400 BAD REQUEST -- if response has all errors"
   [concept-type results]
   (if (some #{concept-type} '(:variable :service :tool))
-    (if (results-contain-errors? results)
-      400
-      200)
+    (let [result-count (count results)
+          num-errors (num-errors-in-assoc-results results)]
+      (if (= 0 result-count)
+        200
+        (if (= num-errors result-count)
+          400
+          (if (> num-errors 0)
+            207
+            200))))
     200))
 
 (defn associate-concept-to-collections

search-app/src/cmr/search/api/generic_association.clj

@@ -1,14 +1,15 @@
 (ns cmr.search.api.generic-association
   "Defines common functions used by associations among generic concepts in the CMR."
   (:require
-   [cheshire.core :as json]
-   [cmr.common-app.api.enabled :as common-enabled]
-   [cmr.common.concepts :as common-concepts]
-   [cmr.common.log :refer (info)]
-   [cmr.common.mime-types :as mt]
-   [cmr.common.util :as util]
-   [cmr.search.services.generic-association-service :as generic-assoc-service]
-   [compojure.core :refer :all]))
+    [cheshire.core :as json]
+    [cmr.common-app.api.enabled :as common-enabled]
+    [cmr.common.concepts :as common-concepts]
+    [cmr.common.log :refer (info)]
+    [cmr.common.mime-types :as mt]
+    [cmr.common.util :as util]
+    [cmr.search.api.association :as assoc]
+    [cmr.search.services.generic-association-service :as generic-assoc-service]
+    [compojure.core :refer :all]))
 
 (defn- validate-association-content-type
   "Validates that content type sent with a association is JSON."
@@ -17,24 +18,31 @@
 
 (defn- api-response
   "Creates a successful association response with the given data response"
-  ([data]
-   (api-response 200 data))
-  ([status-code data]
-   {:status status-code
-    :body (json/generate-string (util/snake-case-data data))
-    :headers {"Content-Type" mt/json}}))
+  [status-code data]
+  (if (= 207 status-code)
+    {:status status-code
+     :body (json/generate-string (util/snake-case-data (assoc/add-individual-statuses data)))
+     :headers {"Content-Type" mt/json}}
+    {:status status-code
+     :body (json/generate-string (util/snake-case-data data))
+     :headers {"Content-Type" mt/json}}))
 
-(defn- results-contain-errors?
-  "Returns true if the results contain :errors"
+(defn generic-assoc-results->status-code
+  "Return status code depending on if results contains error.
+  Check for concept-types requiring error status to be returned.
+  If the concept-type is error-sensitive the function will check for any errors in the results.
+  Will return:
+  - 200 OK -- if response has no errors
+  - 207 MULTI-STATUS -- if response has some errors and some successes
+  - 400 BAD REQUEST -- if response has all errors"
   [results]
-  (seq (filter #(some? (:errors %)) results)))
-
-(defn- results->status-code
-  "Return status code depending on if results contains error."
-  [results]
-  (if (results-contain-errors? results)
-    400
-    200)) 
+  (let [result-count (count results)
+        num-errors (assoc/num-errors-in-assoc-results results)]
+    (cond
+      (zero? result-count) 200
+      (= num-errors result-count) 400
+      (pos? num-errors) 207
+      :else 200)))
 
 (defn associate-concept-to-concepts
   "Associate the given concept by concept type and concept id to a list of
@@ -47,7 +55,7 @@
                 concept-id revision-id body (:client-id context)))
   (let [concept-type (common-concepts/concept-id->type concept-id)
         results (generic-assoc-service/associate-to-concepts context concept-type concept-id revision-id body)
-        status-code (results->status-code results)]
+        status-code (generic-assoc-results->status-code results)]
     (api-response status-code results)))
 
 (defn dissociate-concept-from-concepts
@@ -60,5 +68,5 @@
                 concept-id revision-id body (:client-id context)))
   (let [concept-type (common-concepts/concept-id->type concept-id)
         results (generic-assoc-service/dissociate-from-concepts context concept-type concept-id revision-id body)
-        status-code (results->status-code results)]
+        status-code (generic-assoc-results->status-code results)]
     (api-response status-code results)))

search-app/src/cmr/search/api/tags.clj

@@ -5,7 +5,8 @@
    [clojure.string :as string]
    [cmr.acl.core :as acl]
    [cmr.common-app.api.enabled :as common-enabled]
-   [cmr.common.log :refer (debug info warn error)]
+   [cmr.common.log :refer (debug)]
+   [cmr.search.api.association :as assoc]
    [cmr.common.mime-types :as mt]
    [cmr.common.services.errors :as errors]
    [cmr.common.util :as util]
@@ -23,16 +24,35 @@
   [data]
   (some #(contains? % :errors) data))
 
+(defn tag-association-results->status-code
+  "Check for concept-types requiring error status to be returned.
+  If the concept-type is error-sensitive the function will check for any errors in the results.
+  Will return:
+  - 200 OK -- if response has no errors
+  - 207 MULTI-STATUS -- if response has some errors and some successes
+  - 400 BAD REQUEST -- if response has all errors"
+  [results]
+    (let [result-count (count results)
+          num-errors (assoc/num-errors-in-assoc-results results)]
+      (cond
+        (zero? result-count) 200
+        (= num-errors result-count) 400
+        (pos? num-errors) 207
+        :else 200)))
+
 (defn tag-api-response
   "Creates a successful tag response with the given data response"
   ([data]
    (if (has-error? data)
      (tag-api-response 400 data)
      (tag-api-response 200 data)))
   ([status-code data]
+   (let [data-val (if (= 207 status-code)
+                    (assoc/add-individual-statuses data)
+                    data)]
    {:status status-code
-    :body (json/generate-string (util/snake-case-data data))
-    :headers {"Content-Type" mt/json}}))
+    :body (json/generate-string (util/snake-case-data data-val))
+    :headers {"Content-Type" mt/json}})))
 
 (defn- verify-tag-modification-permission
   "Verifies the current user has been granted permission to modify tags in ECHO ACLs"
@@ -81,7 +101,9 @@
   (validate-tag-content-type headers)
   (debug (format "Tagging [%s] on collections: %s by client: %s."
                 tag-key body (:client-id context)))
-  (tag-api-response (tagging-service/associate-tag-to-collections context tag-key body)))
+  (let [result (tagging-service/associate-tag-to-collections context tag-key body)
+        status-code (tag-association-results->status-code result)]
+      (tag-api-response status-code result)))
 
 (defn dissociate-tag-to-collections
   "Dissociate the tag to a list of collections."
@@ -91,7 +113,9 @@
   (validate-tag-content-type headers)
   (debug (format "Dissociating tag [%s] from collections: %s by client: %s."
                 tag-key body (:client-id context)))
-  (tag-api-response (tagging-service/dissociate-tag-to-collections context tag-key body)))
+  (let [result (tagging-service/dissociate-tag-to-collections context tag-key body)
+        status-code (tag-association-results->status-code result)]
+    (tag-api-response status-code result)))
 
 (defn associate-tag-by-query
   "Processes a request to associate a tag."