Skip to content

Commit

Permalink
1.4.1 toml yml policy diff (#116)
Browse files Browse the repository at this point in the history
  • Loading branch information
@xfhg
xfhg authored Oct 10, 2023
1 parent 7c7510e commit 233c840
Show file tree
Hide file tree
Showing 6 changed files with 242 additions and 95 deletions.
21 changes: 21 additions & 0 deletions cmd/aux.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,27 @@ func cleanupFiles() {

}

func isSubsetOrEqual(a, b map[string]interface{}) bool {
for k, v := range a {
if bv, exists := b[k]; exists {
switch vv := v.(type) {
case map[string]interface{}:
bvv, ok := bv.(map[string]interface{})
if !ok || !isSubsetOrEqual(vv, bvv) {
return false
}
default:
if v != bv {
return false
}
}
} else {
return false
}
}
return true
}

func detectFormat(data string) string {
// Try to unmarshal as JSON
var js map[string]interface{}
Expand Down
58 changes: 57 additions & 1 deletion cmd/type_toml.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ import (

"cuelang.org/go/cue"
"github.com/BurntSushi/toml"
xdiff "github.com/yudai/gojsondiff"
"github.com/yudai/gojsondiff/formatter"
)

func validateTOMLAndCUEContent(tomlContent string, cueContent string) (bool, string) {
Expand Down Expand Up @@ -37,10 +39,64 @@ func validateTOMLAndCUEContent(tomlContent string, cueContent string) (bool, str
return false, fmt.Sprintf("error compiling JSON data to CUE value: %v", err)
}

fmt.Printf("│ DEBUG tomlCueValue : ", tomlCueValue.Value())

err = cueValue.Unify(tomlCueValue.Value()).Validate(cue.Concrete(true))
if err != nil {
return false, fmt.Sprintf("error validating toml data against CUE schema: %v", err)
}

return true, ""
cuepolicy, err := cueValue.Value().MarshalJSON()
tomlcontent, err := tomlCueValue.Value().MarshalJSON()

err = cueValue.UnifyAccept(cueValue.Value(), tomlCueValue.Value()).Validate(cue.Concrete(true))
if err != nil {
return false, fmt.Sprintf("error validating toml accept data against CUE schema: %v", err)
}

// subset

var a, b map[string]interface{}

json.Unmarshal(cuepolicy, &a)
json.Unmarshal(tomlcontent, &b)

if isSubsetOrEqual(a, b) {
return true, ""
}

// diff

differ := xdiff.New()
d, err := differ.Compare(cuepolicy, tomlcontent)
if err != nil {
return false, fmt.Sprintf("error unmarshaling content: %s\n", err.Error())
}

if d.Modified() {

var diffString string

var aJson map[string]interface{}
json.Unmarshal(cuepolicy, &aJson)

config := formatter.AsciiFormatterConfig{
ShowArrayIndex: true,
Coloring: true,
}

zformatter := formatter.NewAsciiFormatter(aJson, config)
diffString, err = zformatter.Format(d)
if err != nil {
return false, fmt.Sprintf("Internal error: %v", err)
}

fmt.Println(diffString)

return false, fmt.Sprintf("Missing required keys \n")

} else {
return true, ""
}

}
98 changes: 50 additions & 48 deletions cmd/type_yml.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,56 +5,11 @@ import (
"fmt"

"cuelang.org/go/cue"
xdiff "github.com/yudai/gojsondiff"
"github.com/yudai/gojsondiff/formatter"
"gopkg.in/yaml.v3"
)

// func validateYAMLAgainstCUE(yamlFile string, cueFile string) (bool, string) {
// yamlData, err := os.ReadFile(yamlFile)
// if err != nil {
// return false, fmt.Sprintf("error reading YAML file: %v", err)
// }

// var yamlObj interface{}
// err = yaml.Unmarshal(yamlData, &yamlObj)
// if err != nil {
// return false, fmt.Sprintf("error unmarshaling YAML data: %v", err)
// }

// var r cue.Runtime
// binst := load.Instances([]string{cueFile}, &load.Config{})
// if len(binst) != 1 || binst[0].Err != nil {
// return false, fmt.Sprintf("error loading CUE file: %v", binst[0].Err)
// }

// cueInstance, err := r.Build(binst[0])
// if err != nil {
// return false, fmt.Sprintf("error building CUE instance: %v", err)
// }

// cueValue := cueInstance.Value()
// err = cueValue.Validate(cue.Concrete(true))
// if err != nil {
// return false, fmt.Sprintf("error validating CUE value: %v", err)
// }

// jsonData, err := json.Marshal(yamlObj)
// if err != nil {
// return false, fmt.Sprintf("error marshaling YAML object to JSON: %v", err)
// }

// yamlCueValue, err := r.Compile("", string(jsonData))
// if err != nil {
// return false, fmt.Sprintf("error compiling JSON data to CUE value: %v", err)
// }

// err = cueValue.Unify(yamlCueValue.Value()).Validate(cue.Concrete(true))
// if err != nil {
// return false, fmt.Sprintf("error validating YAML data against CUE schema: %v", err)
// }

// return true, ""
// }

func validateYAMLAndCUEContent(yamlContent string, cueContent string) (bool, string) {
var yamlObj interface{}
err := yaml.Unmarshal([]byte(yamlContent), &yamlObj)
Expand Down Expand Up @@ -84,10 +39,57 @@ func validateYAMLAndCUEContent(yamlContent string, cueContent string) (bool, str
return false, fmt.Sprintf("error compiling JSON data to CUE value: %v", err)
}

cuepolicy, err := cueValue.Value().MarshalJSON()
yamlcontent, err := yamlCueValue.Value().MarshalJSON()

err = cueValue.Unify(yamlCueValue.Value()).Validate(cue.Concrete(true))
if err != nil {
return false, fmt.Sprintf("error validating YAML data against CUE schema: %v", err)
}

return true, ""
// subset

var a, b map[string]interface{}

json.Unmarshal(cuepolicy, &a)
json.Unmarshal(yamlcontent, &b)

if isSubsetOrEqual(a, b) {
return true, ""
}

// diff

differ := xdiff.New()
d, err := differ.Compare(cuepolicy, yamlcontent)
if err != nil {
return false, fmt.Sprintf("error unmarshaling content: %s\n", err.Error())
}

if d.Modified() {

var diffString string

var aJson map[string]interface{}
json.Unmarshal(cuepolicy, &aJson)

config := formatter.AsciiFormatterConfig{
ShowArrayIndex: true,
Coloring: true,
}

zformatter := formatter.NewAsciiFormatter(aJson, config)
diffString, err = zformatter.Format(d)
if err != nil {
return false, fmt.Sprintf("Internal error: %v", err)
}

fmt.Println(diffString)

return false, fmt.Sprintf("Missing required keys \n")

} else {
return true, ""
}

}
25 changes: 20 additions & 5 deletions examples/policy/filetype.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ Banner: |
Rules:

- name: YML ASSURE ingress enabled
id: 101
id: 151
description: Assure ingress enabled
error: Misconfiguration or omission is fatal
tags: KEY
Expand All @@ -21,7 +21,7 @@ Rules:
- name: YML ASSURE ingress disabled
id: 102
id: 152
description: Assure ingress disabled
error: Misconfiguration or omission is fatal
tags: KEY
Expand All @@ -36,7 +36,7 @@ Rules:
...
- name: YML ASSURE memory limit
id: 103
id: 153
description: Assure resources
error: Misconfiguration or omission is fatal
tags: KEY
Expand All @@ -47,12 +47,12 @@ Rules:
confidence: high
yml_filepattern: "^development-\\d+\\.yml$"
yml_structure: |
logging: { level: "info" }
resources: {limits: { memory: "128Mi" }}
logging { "alert" | "info" }
...
- name: TOML ASSURE database port
id: 103
id: 133
description: Assure database port
error: Misconfiguration or omission is fatal
tags: KEY
Expand All @@ -67,6 +67,21 @@ Rules:
logging: {level : "info"}
...
- name: TOML ASSURE missing database port
id: 134
description: Assure database port
error: Misconfiguration or omission is fatal
tags: KEY
type: toml
fatal: false
enforcement: true
environment: all
confidence: high
toml_filepattern: "^development-\\d+\\.toml$"
toml_structure: |
db: {port: 666 }
...
ExitCritical: "Critical irregularities found in your configuration"
ExitWarning: "Irregularities found in your configuration"
ExitClean: "Clean report"
31 changes: 19 additions & 12 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,18 +6,19 @@ require (
cuelang.org/go v0.6.0
github.com/BurntSushi/toml v1.3.2
github.com/blang/semver v3.5.1+incompatible
github.com/go-resty/resty/v2 v2.7.0
github.com/go-resty/resty/v2 v2.9.1
github.com/gookit/color v1.5.4
github.com/gosuri/uitable v0.0.4
github.com/itchyny/gojq v0.12.13
github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0
github.com/lens-vm/jsonmerge v0.0.0-20210328131518-91ae88d1bfff
github.com/mitchellh/go-homedir v1.1.0
github.com/owenrumney/go-sarif/v2 v2.2.0
github.com/owenrumney/go-sarif/v2 v2.3.0
github.com/rhysd/go-github-selfupdate v1.2.3
github.com/spf13/cobra v1.7.0
github.com/spf13/viper v1.16.0
github.com/spf13/viper v1.17.0
github.com/theckman/yacspin v0.13.12
github.com/yudai/gojsondiff v1.0.0
gopkg.in/yaml.v2 v2.4.0
)

Expand All @@ -38,33 +39,39 @@ require (
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/pelletier/go-toml/v2 v2.1.0 // indirect
github.com/rivo/uniseg v0.4.4 // indirect
github.com/spf13/afero v1.9.5 // indirect
github.com/spf13/afero v1.10.0 // indirect
github.com/spf13/cast v1.5.1 // indirect
github.com/spf13/jwalterweatherman v1.1.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/tcnksm/go-gitconfig v0.1.2 // indirect
github.com/ulikunitz/xz v0.5.11 // indirect
github.com/valyala/fastjson v1.6.4 // indirect
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
golang.org/x/crypto v0.12.0 // indirect
golang.org/x/net v0.14.0 // indirect
golang.org/x/oauth2 v0.11.0 // indirect
golang.org/x/sys v0.12.0 // indirect
golang.org/x/crypto v0.14.0 // indirect
golang.org/x/net v0.16.0 // indirect
golang.org/x/oauth2 v0.13.0 // indirect
golang.org/x/sys v0.13.0 // indirect
golang.org/x/text v0.13.0 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/appengine v1.6.8 // indirect
google.golang.org/protobuf v1.31.0 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
gopkg.in/yaml.v3 v3.0.1
)

require (
github.com/cockroachdb/apd/v3 v3.2.0 // indirect
github.com/cockroachdb/apd/v3 v3.2.1 // indirect
github.com/emicklei/proto v1.12.1 // indirect
github.com/google/uuid v1.3.1 // indirect
github.com/mpvl/unique v0.0.0-20150818121801-cbe035fff7de // indirect
github.com/protocolbuffers/txtpbfmt v0.0.0-20230730201308-0c31dbd32b9f // indirect
github.com/sagikazarmark/locafero v0.3.0 // indirect
github.com/sagikazarmark/slog-shim v0.1.0 // indirect
github.com/sergi/go-diff v1.3.1 // indirect
github.com/sourcegraph/conc v0.3.0 // indirect
github.com/subosito/gotenv v1.6.0 // indirect
golang.org/x/mod v0.12.0 // indirect
github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82 // indirect
github.com/yudai/pp v2.0.1+incompatible // indirect
go.uber.org/multierr v1.11.0 // indirect
golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect
)

replace github.com/subosito/gotenv => github.com/subosito/gotenv v1.4.2
Loading

0 comments on commit 233c840

Please sign in to comment.