support new has/nothas filter operator (#200)

docs/go-c8y-cli/docs/concepts/filtering.md

@@ -35,6 +35,8 @@ The `filter` parameter uses a query language which supports the following operat
 |dategt|match date greater than (newer) to value of (datetime/relative)|`--filter "creationTime dategt 2022-01-02T12:00"`|
 |dategte (or 'newerthan')|match date greater than (newer) or equal to value of (datetime/relative)|`--filter "creationTime dategte 2022-01-02T12:00"`|
 |version|match a version or version range|`--filter "c8y_Firmware.version version >1.0.1, <=2.0.0"`|
+|has|match when an object has a given key/property|`--filter "has name`|
+|nothas|match when an object does not have a given key/property|`--filter "nothas name`|
 
 ## Examples
 

go.mod

@@ -32,7 +32,6 @@ require (
 	github.com/spf13/pflag v1.0.5
 	github.com/spf13/viper v1.13.0
 	github.com/stretchr/testify v1.8.0
-	github.com/thedevsaddam/gojsonq v2.3.0+incompatible
 	github.com/tidwall/gjson v1.14.3
 	github.com/tidwall/pretty v1.2.1
 	github.com/tidwall/sjson v1.2.5
@@ -46,7 +45,10 @@ require (
 
 require github.com/hashicorp/go-version v1.6.0
 
-require github.com/cli/browser v1.1.0
+require (
+	github.com/cli/browser v1.1.0
+	github.com/reubenmiller/gojsonq/v2 v2.0.0-20221119213524-0fd921ac20a3
+)
 
 require (
 	github.com/VividCortex/ewma v1.2.0 // indirect

go.sum

@@ -445,6 +445,8 @@ github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1
 github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
 github.com/reubenmiller/go-c8y v0.12.0-rc.10 h1:fhqraGWY+hWSPMXx3jW/S+Qjw0ipN8WX2d7wNYmj8+I=
 github.com/reubenmiller/go-c8y v0.12.0-rc.10/go.mod h1:t+biKGzNJKWx0A8inMl6xkmVWKuRyTe36+Mq/kFLBkM=
+github.com/reubenmiller/gojsonq/v2 v2.0.0-20221119213524-0fd921ac20a3 h1:v8Q77ObTxkm0Wj9iAjcc0VMLxqEzKIdAnaTNPzSiw8Q=
+github.com/reubenmiller/gojsonq/v2 v2.0.0-20221119213524-0fd921ac20a3/go.mod h1:QidmUT4ebNVwyjKXAQgx9VFHxpOxBKWs32EEXaXnEfE=
 github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
@@ -502,8 +504,6 @@ github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69
 github.com/subosito/gotenv v1.4.1 h1:jyEFiXpy21Wm81FBN71l9VoMMV8H8jG+qIK3GCpY6Qs=
 github.com/subosito/gotenv v1.4.1/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0=
 github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502/go.mod h1:p9lPsd+cx33L3H9nNoecRRxPssFKUwwI50I3pZ0yT+8=
-github.com/thedevsaddam/gojsonq v2.3.0+incompatible h1:i2lFTvGY4LvoZ2VUzedsFlRiyaWcJm3Uh6cQ9+HyQA8=
-github.com/thedevsaddam/gojsonq v2.3.0+incompatible/go.mod h1:RBcQaITThgJAAYKH7FNp2onYodRz8URfsuEGpAch0NA=
 github.com/tidwall/gjson v1.13.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
 github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
 github.com/tidwall/gjson v1.14.3 h1:9jvXn7olKEHU1S9vwoMGliaT8jq1vJ7IH/n9zD9Dnlw=

pkg/jsonfilter/jsonfilter.go

@@ -17,7 +17,7 @@ import (
 	"github.com/reubenmiller/go-c8y-cli/v2/pkg/sortorder"
 	"github.com/reubenmiller/go-c8y-cli/v2/pkg/timestamp"
 	"github.com/reubenmiller/go-c8y/pkg/c8y"
-	"github.com/thedevsaddam/gojsonq"
+	"github.com/reubenmiller/gojsonq/v2"
 	"github.com/tidwall/gjson"
 	"go.uber.org/zap/zapcore"
 )
@@ -52,7 +52,7 @@ func (f *JSONFilters) AddSelectors(props ...string) {
 // AddRawFilters add list of raw filters
 func (f *JSONFilters) AddRawFilters(rawFilters []string) error {
 	for _, item := range rawFilters {
-		sepPattern := regexp.MustCompile(`(\s+[\-]?(like|match|notlike|notmatch|newerthan|olderthan|datelte|datelt|dategt|dategte|version|eq|neq|lt|lte|gt|gte|notIn|in|startsWith|endsWith|contains|len[n]?eq|lengt[e]?|lenlt[e]?)\s+|(!?=|[<>]=?))`)
+		sepPattern := regexp.MustCompile(`(\s*[\-]?(has|nothas|hasnot|missing|keyIn|keyNotIn|like|match|notlike|notmatch|newerthan|olderthan|datelte|datelt|dategt|dategte|version|eq|neq|lt|lte|gt|gte|notIn|in|startsWith|endsWith|contains|len[n]?eq|lengt[e]?|lenlt[e]?)\s+|(!?=|[<>]=?))`)
 
 		parts := sepPattern.Split(item, 2)
 
@@ -72,6 +72,17 @@ func (f *JSONFilters) AddRawFilters(rawFilters []string) error {
 			continue
 		}
 
+		operatorAliases := map[string]string{
+			"has":     "keyIn",
+			"hasnot":  "keyNotIn",
+			"nothas":  "keyNotIn",
+			"missing": "keyNotIn",
+		}
+
+		if realName, ok := operatorAliases[operator]; ok {
+			operator = realName
+		}
+
 		if v, err := strconv.ParseFloat(value, 64); err == nil {
 			if strings.Contains(value, ".") {
 				// use float
@@ -84,6 +95,18 @@ func (f *JSONFilters) AddRawFilters(rawFilters []string) error {
 			// Check boolean values
 			f.Add(strings.TrimSpace(parts[0]), operator, bool(v))
 		} else {
+			if parts[0] == "" {
+				// Support keyIn and keyNotIn operators which don't take
+				if strings.Contains(value, ".") {
+					lastIdx := strings.LastIndex(value, ".")
+					parts[0] = value[0:lastIdx]
+					value = value[lastIdx+1:]
+				} else {
+					// Default to root element
+					parts[0] = "."
+				}
+			}
+
 			f.Add(strings.TrimSpace(parts[0]), operator, value)
 		}
 	}

tests/manual/common/filter/filter_options.yml

@@ -79,3 +79,30 @@ tests:
       exactly: |
         {"value":"2.0.0"}
         {"value":""}
+
+  It supports filtering by key presence:
+    command: |
+      c8y util repeat 2 | c8y template execute --template "{value: if input.index == 1 then '2.0.0' else {nested:'item'}}" | c8y util show --filter "has value.nested" -o json
+    exit-code: 0
+    stdout:
+      exactly: |
+        {"value":{"nested":"item"}}
+
+  It supports filtering by key non-existence:
+    command: |
+      c8y util repeat 2 | c8y template execute --template "{value: if input.index == 1 then '2.0.0' else {nested:'item'}}" | c8y util show --filter "nothas value.nested" -o json
+      c8y util repeat 2 | c8y template execute --template "{value: if input.index == 1 then '2.0.0' else {nested:'item'}}" | c8y util show --filter "missing value.nested" -o json
+    exit-code: 0
+    stdout:
+      exactly: |
+        {"value":"2.0.0"}
+        {"value":"2.0.0"}
+
+  It supports filtering by key presence using different data types:
+    command: |
+      c8y util repeat 2 | c8y template execute --template "{value: if input.index == 1 then '2.0.0' else {nested:'item'}}" | c8y util show --filter "has value" -o json
+    exit-code: 0
+    stdout:
+      exactly: |
+        {"value":"2.0.0"}
+        {"value":{"nested":"item"}}