From aa0e998b4d8b57eb4248470e7854f43b68ef4839 Mon Sep 17 00:00:00 2001 From: Jakob Hahn Date: Wed, 16 Oct 2024 11:40:58 +0200 Subject: [PATCH 1/8] config: add json tags to config structs Signed-off-by: Jakob Hahn --- config/config.go | 142 +++++++++++++++++++++++------------------------ 1 file changed, 71 insertions(+), 71 deletions(-) diff --git a/config/config.go b/config/config.go index 559dbf56..4c4a60dc 100644 --- a/config/config.go +++ b/config/config.go @@ -79,7 +79,7 @@ var ( ) type Config struct { - Modules map[string]Module `yaml:"modules"` + Modules map[string]Module `yaml:"modules" json:"modules"` } type SafeConfig struct { @@ -193,104 +193,104 @@ func MustNewRegexp(s string) Regexp { } type Module struct { - Prober string `yaml:"prober,omitempty"` - Timeout time.Duration `yaml:"timeout,omitempty"` - HTTP HTTPProbe `yaml:"http,omitempty"` - TCP TCPProbe `yaml:"tcp,omitempty"` - ICMP ICMPProbe `yaml:"icmp,omitempty"` - DNS DNSProbe `yaml:"dns,omitempty"` - GRPC GRPCProbe `yaml:"grpc,omitempty"` + Prober string `yaml:"prober,omitempty" json:"prober,omitempty"` + Timeout time.Duration `yaml:"timeout,omitempty" json:"timeout,omitempty"` + HTTP HTTPProbe `yaml:"http,omitempty" json:"http,omitempty"` + TCP TCPProbe `yaml:"tcp,omitempty" json:"tcp,omitempty"` + ICMP ICMPProbe `yaml:"icmp,omitempty" json:"icmp,omitempty"` + DNS DNSProbe `yaml:"dns,omitempty" json:"dns,omitempty"` + GRPC GRPCProbe `yaml:"grpc,omitempty" json:"grpc,omitempty"` } type HTTPProbe struct { // Defaults to 2xx. - ValidStatusCodes []int `yaml:"valid_status_codes,omitempty"` - ValidHTTPVersions []string `yaml:"valid_http_versions,omitempty"` - IPProtocol string `yaml:"preferred_ip_protocol,omitempty"` - IPProtocolFallback bool `yaml:"ip_protocol_fallback,omitempty"` - SkipResolvePhaseWithProxy bool `yaml:"skip_resolve_phase_with_proxy,omitempty"` - NoFollowRedirects *bool `yaml:"no_follow_redirects,omitempty"` - FailIfSSL bool `yaml:"fail_if_ssl,omitempty"` - FailIfNotSSL bool `yaml:"fail_if_not_ssl,omitempty"` - Method string `yaml:"method,omitempty"` - Headers map[string]string `yaml:"headers,omitempty"` - FailIfBodyMatchesRegexp []Regexp `yaml:"fail_if_body_matches_regexp,omitempty"` - FailIfBodyNotMatchesRegexp []Regexp `yaml:"fail_if_body_not_matches_regexp,omitempty"` - FailIfHeaderMatchesRegexp []HeaderMatch `yaml:"fail_if_header_matches,omitempty"` - FailIfHeaderNotMatchesRegexp []HeaderMatch `yaml:"fail_if_header_not_matches,omitempty"` - Body string `yaml:"body,omitempty"` - BodyFile string `yaml:"body_file,omitempty"` - HTTPClientConfig config.HTTPClientConfig `yaml:"http_client_config,inline"` - Compression string `yaml:"compression,omitempty"` - BodySizeLimit units.Base2Bytes `yaml:"body_size_limit,omitempty"` + ValidStatusCodes []int `yaml:"valid_status_codes,omitempty" json:"valid_status_codes,omitempty"` + ValidHTTPVersions []string `yaml:"valid_http_versions,omitempty" json:"valid_http_versions,omitempty"` + IPProtocol string `yaml:"preferred_ip_protocol,omitempty" json:"preferred_ip_protocol,omitempty"` + IPProtocolFallback bool `yaml:"ip_protocol_fallback,omitempty" json:"ip_protocol_fallback,omitempty"` + SkipResolvePhaseWithProxy bool `yaml:"skip_resolve_phase_with_proxy,omitempty" json:"skip_resolve_phase_with_proxy,omitempty"` + NoFollowRedirects *bool `yaml:"no_follow_redirects,omitempty" json:"no_follow_redirects,omitempty"` + FailIfSSL bool `yaml:"fail_if_ssl,omitempty" json:"fail_if_ssl,omitempty"` + FailIfNotSSL bool `yaml:"fail_if_not_ssl,omitempty" json:"fail_if_not_ssl,omitempty"` + Method string `yaml:"method,omitempty" json:"method,omitempty"` + Headers map[string]string `yaml:"headers,omitempty" json:"headers,omitempty"` + FailIfBodyMatchesRegexp []Regexp `yaml:"fail_if_body_matches_regexp,omitempty" json:"fail_if_body_matches_regexp,omitempty"` + FailIfBodyNotMatchesRegexp []Regexp `yaml:"fail_if_body_not_matches_regexp,omitempty" json:"fail_if_body_not_matches_regexp,omitempty"` + FailIfHeaderMatchesRegexp []HeaderMatch `yaml:"fail_if_header_matches,omitempty" json:"fail_if_header_matches,omitempty"` + FailIfHeaderNotMatchesRegexp []HeaderMatch `yaml:"fail_if_header_not_matches,omitempty" json:"fail_if_header_not_matches,omitempty"` + Body string `yaml:"body,omitempty" json:"body,omitempty"` + BodyFile string `yaml:"body_file,omitempty" json:"body_file,omitempty"` + HTTPClientConfig config.HTTPClientConfig `yaml:"http_client_config,inline" json:"http_client_config,inline"` + Compression string `yaml:"compression,omitempty" json:"compression,omitempty"` + BodySizeLimit units.Base2Bytes `yaml:"body_size_limit,omitempty" json:"body_size_limit,omitempty"` } type GRPCProbe struct { - Service string `yaml:"service,omitempty"` - TLS bool `yaml:"tls,omitempty"` - TLSConfig config.TLSConfig `yaml:"tls_config,omitempty"` - IPProtocolFallback bool `yaml:"ip_protocol_fallback,omitempty"` - PreferredIPProtocol string `yaml:"preferred_ip_protocol,omitempty"` + Service string `yaml:"service,omitempty" json:"service,omitempty"` + TLS bool `yaml:"tls,omitempty" json:"tls,omitempty"` + TLSConfig config.TLSConfig `yaml:"tls_config,omitempty" json:"tls_config,omitempty"` + IPProtocolFallback bool `yaml:"ip_protocol_fallback,omitempty" json:"ip_protocol_fallback,omitempty"` + PreferredIPProtocol string `yaml:"preferred_ip_protocol,omitempty" json:"preferred_ip_protocol,omitempty"` } type HeaderMatch struct { - Header string `yaml:"header,omitempty"` - Regexp Regexp `yaml:"regexp,omitempty"` - AllowMissing bool `yaml:"allow_missing,omitempty"` + Header string `yaml:"header,omitempty" json:"header,omitempty"` + Regexp Regexp `yaml:"regexp,omitempty" json:"regexp,omitempty"` + AllowMissing bool `yaml:"allow_missing,omitempty" json:"allow_missing,omitempty"` } type Label struct { - Name string `yaml:"name,omitempty"` - Value string `yaml:"value,omitempty"` + Name string `yaml:"name,omitempty" json:"name,omitempty"` + Value string `yaml:"value,omitempty" json:"value,omitempty"` } type QueryResponse struct { - Expect Regexp `yaml:"expect,omitempty"` - Labels []Label `yaml:"labels,omitempty"` - Send string `yaml:"send,omitempty"` - StartTLS bool `yaml:"starttls,omitempty"` + Expect Regexp `yaml:"expect,omitempty" json:"expect,omitempty"` + Labels []Label `yaml:"labels,omitempty" json:"labels,omitempty"` + Send string `yaml:"send,omitempty" json:"send,omitempty"` + StartTLS bool `yaml:"starttls,omitempty" json:"starttls,omitempty"` } type TCPProbe struct { - IPProtocol string `yaml:"preferred_ip_protocol,omitempty"` - IPProtocolFallback bool `yaml:"ip_protocol_fallback,omitempty"` - SourceIPAddress string `yaml:"source_ip_address,omitempty"` - QueryResponse []QueryResponse `yaml:"query_response,omitempty"` - TLS bool `yaml:"tls,omitempty"` - TLSConfig config.TLSConfig `yaml:"tls_config,omitempty"` + IPProtocol string `yaml:"preferred_ip_protocol,omitempty" json:"preferred_ip_protocol,omitempty"` + IPProtocolFallback bool `yaml:"ip_protocol_fallback,omitempty" json:"ip_protocol_fallback,omitempty"` + SourceIPAddress string `yaml:"source_ip_address,omitempty" json:"source_ip_address,omitempty"` + QueryResponse []QueryResponse `yaml:"query_response,omitempty" json:"query_response,omitempty"` + TLS bool `yaml:"tls,omitempty" json:"tls,omitempty"` + TLSConfig config.TLSConfig `yaml:"tls_config,omitempty" json:"tls_config,omitempty"` } type ICMPProbe struct { - IPProtocol string `yaml:"preferred_ip_protocol,omitempty"` // Defaults to "ip6". - IPProtocolFallback bool `yaml:"ip_protocol_fallback,omitempty"` - SourceIPAddress string `yaml:"source_ip_address,omitempty"` - PayloadSize int `yaml:"payload_size,omitempty"` - DontFragment bool `yaml:"dont_fragment,omitempty"` - TTL int `yaml:"ttl,omitempty"` + IPProtocol string `yaml:"preferred_ip_protocol,omitempty" json:"preferred_ip_protocol,omitempty"` // Defaults to "ip6". + IPProtocolFallback bool `yaml:"ip_protocol_fallback,omitempty" json:"ip_protocol_fallback,omitempty"` + SourceIPAddress string `yaml:"source_ip_address,omitempty" json:"source_ip_address,omitempty"` + PayloadSize int `yaml:"payload_size,omitempty" json:"payload_size,omitempty"` + DontFragment bool `yaml:"dont_fragment,omitempty" json:"dont_fragment,omitempty"` + TTL int `yaml:"ttl,omitempty" json:"ttl,omitempty"` } type DNSProbe struct { - IPProtocol string `yaml:"preferred_ip_protocol,omitempty"` - IPProtocolFallback bool `yaml:"ip_protocol_fallback,omitempty"` - DNSOverTLS bool `yaml:"dns_over_tls,omitempty"` - TLSConfig config.TLSConfig `yaml:"tls_config,omitempty"` - SourceIPAddress string `yaml:"source_ip_address,omitempty"` - TransportProtocol string `yaml:"transport_protocol,omitempty"` - QueryClass string `yaml:"query_class,omitempty"` // Defaults to IN. - QueryName string `yaml:"query_name,omitempty"` - QueryType string `yaml:"query_type,omitempty"` // Defaults to ANY. - Recursion bool `yaml:"recursion_desired,omitempty"` // Defaults to true. - ValidRcodes []string `yaml:"valid_rcodes,omitempty"` // Defaults to NOERROR. - ValidateAnswer DNSRRValidator `yaml:"validate_answer_rrs,omitempty"` - ValidateAuthority DNSRRValidator `yaml:"validate_authority_rrs,omitempty"` - ValidateAdditional DNSRRValidator `yaml:"validate_additional_rrs,omitempty"` + IPProtocol string `yaml:"preferred_ip_protocol,omitempty" json:"preferred_ip_protocol,omitempty"` + IPProtocolFallback bool `yaml:"ip_protocol_fallback,omitempty" json:"ip_protocol_fallback,omitempty"` + DNSOverTLS bool `yaml:"dns_over_tls,omitempty" json:"dns_over_tls,omitempty"` + TLSConfig config.TLSConfig `yaml:"tls_config,omitempty" json:"tls_config,omitempty"` + SourceIPAddress string `yaml:"source_ip_address,omitempty" json:"source_ip_address,omitempty"` + TransportProtocol string `yaml:"transport_protocol,omitempty" json:"transport_protocol,omitempty"` + QueryClass string `yaml:"query_class,omitempty" json:"query_class,omitempty"` // Defaults to IN. + QueryName string `yaml:"query_name,omitempty" json:"query_name,omitempty"` + QueryType string `yaml:"query_type,omitempty" json:"query_type,omitempty"` // Defaults to ANY. + Recursion bool `yaml:"recursion_desired,omitempty" json:"recursion_desired,omitempty"` // Defaults to true. + ValidRcodes []string `yaml:"valid_rcodes,omitempty" json:"valid_rcodes,omitempty"` // Defaults to NOERROR. + ValidateAnswer DNSRRValidator `yaml:"validate_answer_rrs,omitempty" json:"validate_answer_rrs,omitempty"` + ValidateAuthority DNSRRValidator `yaml:"validate_authority_rrs,omitempty" json:"validate_authority_rrs,omitempty"` + ValidateAdditional DNSRRValidator `yaml:"validate_additional_rrs,omitempty" json:"validate_additional_rrs,omitempty"` } type DNSRRValidator struct { - FailIfMatchesRegexp []string `yaml:"fail_if_matches_regexp,omitempty"` - FailIfAllMatchRegexp []string `yaml:"fail_if_all_match_regexp,omitempty"` - FailIfNotMatchesRegexp []string `yaml:"fail_if_not_matches_regexp,omitempty"` - FailIfNoneMatchesRegexp []string `yaml:"fail_if_none_matches_regexp,omitempty"` + FailIfMatchesRegexp []string `yaml:"fail_if_matches_regexp,omitempty" json:"fail_if_matches_regexp,omitempty"` + FailIfAllMatchRegexp []string `yaml:"fail_if_all_match_regexp,omitempty" json:"fail_if_all_match_regexp,omitempty"` + FailIfNotMatchesRegexp []string `yaml:"fail_if_not_matches_regexp,omitempty" json:"fail_if_not_matches_regexp,omitempty"` + FailIfNoneMatchesRegexp []string `yaml:"fail_if_none_matches_regexp,omitempty" json:"fail_if_none_matches_regexp,omitempty"` } // UnmarshalYAML implements the yaml.Unmarshaler interface. From 86f85707df0f14d7a17f8ef2b0453368111c8db6 Mon Sep 17 00:00:00 2001 From: Jakob Hahn Date: Thu, 17 Oct 2024 11:29:44 +0200 Subject: [PATCH 2/8] config: add custom json un-/marshal functions Signed-off-by: Jakob Hahn --- config/config.go | 236 +++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 229 insertions(+), 7 deletions(-) diff --git a/config/config.go b/config/config.go index 4c4a60dc..12c81fa6 100644 --- a/config/config.go +++ b/config/config.go @@ -14,11 +14,14 @@ package config import ( + "bytes" + "encoding/json" "errors" "fmt" "math" "net/textproto" "os" + "reflect" "regexp" "runtime" "sort" @@ -115,16 +118,25 @@ func (sc *SafeConfig) ReloadConfig(confFile string, logger log.Logger) (err erro } }() - yamlReader, err := os.Open(confFile) + fileReader, err := os.Open(confFile) if err != nil { return fmt.Errorf("error reading config file: %s", err) } - defer yamlReader.Close() - decoder := yaml.NewDecoder(yamlReader) - decoder.KnownFields(true) + defer fileReader.Close() + if strings.HasSuffix(confFile, ".json") { + decoder := json.NewDecoder(fileReader) + decoder.DisallowUnknownFields() - if err = decoder.Decode(c); err != nil { - return fmt.Errorf("error parsing config file: %s", err) + if err = decoder.Decode(c); err != nil { + return fmt.Errorf("error parsing config file: %s", err) + } + } else { + decoder := yaml.NewDecoder(fileReader) + decoder.KnownFields(true) + + if err = decoder.Decode(c); err != nil { + return fmt.Errorf("error parsing config file: %s", err) + } } for name, module := range c.Modules { @@ -175,6 +187,22 @@ func (re *Regexp) UnmarshalYAML(unmarshal func(interface{}) error) error { return nil } +// UnmarshalJSON implements the json.Unmarshaler interface. +func (re *Regexp) UnmarshalJSON(data []byte) error { + var s string + decoder := json.NewDecoder(bytes.NewReader(data)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&s); err != nil { + return err + } + r, err := NewRegexp(s) + if err != nil { + return fmt.Errorf("\"Could not compile regular expression\" regexp=\"%s\"", s) + } + *re = r + return nil +} + // MarshalYAML implements the yaml.Marshaler interface. func (re Regexp) MarshalYAML() (interface{}, error) { if re.original != "" { @@ -183,6 +211,14 @@ func (re Regexp) MarshalYAML() (interface{}, error) { return nil, nil } +// MarshalJSON implements the json.Marshaler interface. +func (re Regexp) MarshalJSOn() ([]byte, error) { + if re.original != "" { + return []byte(re.original), nil + } + return nil, nil +} + // MustNewRegexp works like NewRegexp, but panics if the regular expression does not compile. func MustNewRegexp(s string) Regexp { re, err := NewRegexp(s) @@ -302,6 +338,14 @@ func (s *Config) UnmarshalYAML(unmarshal func(interface{}) error) error { return nil } +// UnmarshalJSON implements the json.Unmarshaler interface. +func (s *Config) UnmarshalJSON(data []byte) error { + type plain Config + decoder := json.NewDecoder(bytes.NewReader(data)) + decoder.DisallowUnknownFields() + return decoder.Decode((*plain)(s)) +} + // UnmarshalYAML implements the yaml.Unmarshaler interface. func (s *Module) UnmarshalYAML(unmarshal func(interface{}) error) error { *s = DefaultModule @@ -312,6 +356,58 @@ func (s *Module) UnmarshalYAML(unmarshal func(interface{}) error) error { return nil } +// UnmarshalJSON implements the json.Unmarshaler interface. +func (s *Module) UnmarshalJSON(data []byte) error { + // The json lib does not support to unmarshal into time.Duration + // We duplicate the module type with Timeout set to any, so we can parse it + type tmpType struct { + Prober string `json:"prober,omitempty"` + Timeout any `json:"timeout,omitempty"` + HTTP HTTPProbe `json:"http,omitempty"` + TCP TCPProbe `json:"tcp,omitempty"` + ICMP ICMPProbe `json:"icmp,omitempty"` + DNS DNSProbe `json:"dns,omitempty"` + GRPC GRPCProbe `json:"grpc,omitempty"` + } + tmp := tmpType{ + HTTP: DefaultModule.HTTP, + TCP: DefaultModule.TCP, + ICMP: DefaultModule.ICMP, + DNS: DefaultModule.DNS, + } + decoder := json.NewDecoder(bytes.NewReader(data)) + decoder.DisallowUnknownFields() + err := decoder.Decode(&tmp) + if err != nil { + return err + } + // Duration can be of type string or float64 + var duration time.Duration + if tmp.Timeout != nil { + switch value := tmp.Timeout.(type) { + case float64: + duration = time.Duration(value) + case string: + duration, err = time.ParseDuration(value) + if err != nil { + return err + } + default: + return fmt.Errorf("invalid duration: %#v", tmp) + } + } + *s = Module{ + Prober: tmp.Prober, + Timeout: duration, + HTTP: tmp.HTTP, + TCP: tmp.TCP, + ICMP: tmp.ICMP, + DNS: tmp.DNS, + GRPC: tmp.GRPC, + } + return nil +} + // UnmarshalYAML implements the yaml.Unmarshaler interface. func (s *HTTPProbe) UnmarshalYAML(unmarshal func(interface{}) error) error { *s = DefaultHTTPProbe @@ -320,6 +416,64 @@ func (s *HTTPProbe) UnmarshalYAML(unmarshal func(interface{}) error) error { return err } + return s.setDefaults() +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (s *HTTPProbe) UnmarshalJSON(data []byte) error { + // The currentl json lib can not handle inline, we therefore need to separate the HTTPClientConf fields from the HTTPProbe fields + var tmp config.HTTPClientConfig + var input map[string]any + httpClientInput := make(map[string]any) + + // Parse the data into the generic map[string]any so we can get the json keys + if err := json.Unmarshal(data, &input); err != nil { + return err + } + + // Use reflect to get all json keys of the config.HTTPClientConfig + typ := reflect.TypeOf(tmp) + for idx := 0; idx < typ.NumField(); idx++ { + field := typ.Field(idx) + tag := strings.Split(field.Tag.Get("json"), ",") + if len(tag) == 0 || tag[0] == "" { + continue + } + // Separate the data + if _, ok := input[tag[0]]; ok { + httpClientInput[tag[0]] = input[tag[0]] + delete(input, tag[0]) + } + } + + // Marshal the data for the decoder + httpClientData, err := json.Marshal(httpClientInput) + if err != nil { + return err + } + httpData, err := json.Marshal(input) + if err != nil { + return err + } + + *s = DefaultHTTPProbe + type plain HTTPProbe + decoder := json.NewDecoder(bytes.NewReader(httpClientData)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&tmp); err != nil { + return err + } + + decoder = json.NewDecoder(bytes.NewReader(httpData)) + decoder.DisallowUnknownFields() + if err := decoder.Decode((*plain)(s)); err != nil { + return err + } + s.HTTPClientConfig = tmp + return s.setDefaults() +} + +func (s *HTTPProbe) setDefaults() error { // BodySizeLimit == 0 means no limit. By leaving it at 0 we // avoid setting up the limiter. if s.BodySizeLimit < 0 || s.BodySizeLimit == math.MaxInt64 { @@ -350,7 +504,6 @@ func (s *HTTPProbe) UnmarshalYAML(unmarshal func(interface{}) error) error { } } } - return nil } @@ -364,6 +517,18 @@ func (s *GRPCProbe) UnmarshalYAML(unmarshal func(interface{}) error) error { return nil } +// UnmarshalJSON implements the json.Unmarshaler interface. +func (s *GRPCProbe) UnmarshalJSON(data []byte) error { + *s = DefaultGRPCProbe + type plain GRPCProbe + decoder := json.NewDecoder(bytes.NewReader(data)) + decoder.DisallowUnknownFields() + if err := decoder.Decode((*plain)(s)); err != nil { + return err + } + return nil +} + // UnmarshalYAML implements the yaml.Unmarshaler interface. func (s *DNSProbe) UnmarshalYAML(unmarshal func(interface{}) error) error { *s = DefaultDNSProbe @@ -371,6 +536,22 @@ func (s *DNSProbe) UnmarshalYAML(unmarshal func(interface{}) error) error { if err := unmarshal((*plain)(s)); err != nil { return err } + return s.verifyFields() +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (s *DNSProbe) UnmarshalJSON(data []byte) error { + *s = DefaultDNSProbe + type plain DNSProbe + decoder := json.NewDecoder(bytes.NewReader(data)) + decoder.DisallowUnknownFields() + if err := decoder.Decode((*plain)(s)); err != nil { + return err + } + return s.verifyFields() +} + +func (s *DNSProbe) verifyFields() error { if s.QueryName == "" { return errors.New("query name must be set for DNS module") } @@ -398,6 +579,18 @@ func (s *TCPProbe) UnmarshalYAML(unmarshal func(interface{}) error) error { return nil } +// UnmarshalJSON implements the json.Unmarshaler interface. +func (s *TCPProbe) UnmarshalJSON(data []byte) error { + *s = DefaultTCPProbe + type plain TCPProbe + decoder := json.NewDecoder(bytes.NewReader(data)) + decoder.DisallowUnknownFields() + if err := decoder.Decode((*plain)(s)); err != nil { + return err + } + return nil +} + // UnmarshalYAML implements the yaml.Unmarshaler interface. func (s *DNSRRValidator) UnmarshalYAML(unmarshal func(interface{}) error) error { type plain DNSRRValidator @@ -414,7 +607,22 @@ func (s *ICMPProbe) UnmarshalYAML(unmarshal func(interface{}) error) error { if err := unmarshal((*plain)(s)); err != nil { return err } + return s.verifyFields() +} +// UnmarshalJSON implements the json.Unmarshaler interface. +func (s *ICMPProbe) UnmarshalJSON(data []byte) error { + *s = DefaultICMPProbe + type plain ICMPProbe + decoder := json.NewDecoder(bytes.NewReader(data)) + decoder.DisallowUnknownFields() + if err := decoder.Decode((*plain)(s)); err != nil { + return err + } + return s.verifyFields() +} + +func (s *ICMPProbe) verifyFields() error { if runtime.GOOS == "windows" && s.DontFragment { return errors.New("\"dont_fragment\" is not supported on windows platforms") } @@ -444,7 +652,21 @@ func (s *HeaderMatch) UnmarshalYAML(unmarshal func(interface{}) error) error { if err := unmarshal((*plain)(s)); err != nil { return err } + return s.verifyFields() +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (s *HeaderMatch) UnmarshalJSON(data []byte) error { + type plain HeaderMatch + decoder := json.NewDecoder(bytes.NewReader(data)) + decoder.DisallowUnknownFields() + if err := decoder.Decode((*plain)(s)); err != nil { + return err + } + return s.verifyFields() +} +func (s *HeaderMatch) verifyFields() error { if s.Header == "" { return errors.New("header name must be set for HTTP header matchers") } From aa0d49c87341185903b5190c267770ec05258cda Mon Sep 17 00:00:00 2001 From: Jakob Hahn Date: Fri, 18 Oct 2024 12:52:11 +0200 Subject: [PATCH 3/8] config: add json config tests Signed-off-by: Jakob Hahn --- config/config_test.go | 120 +++++++++------ config/testdata/blackbox-bad.json | 86 +++++++++++ config/testdata/blackbox-bad2.json | 17 +++ config/testdata/blackbox-good.json | 144 ++++++++++++++++++ config/testdata/invalid-dns-class.json | 13 ++ config/testdata/invalid-dns-module.json | 16 ++ config/testdata/invalid-dns-type.json | 13 ++ config/testdata/invalid-http-body-config.json | 12 ++ .../invalid-http-body-match-regexp.json | 13 ++ .../invalid-http-body-not-match-regexp.json | 13 ++ ...ttp-compression-mismatch-special-case.json | 14 ++ .../invalid-http-compression-mismatch.json | 14 ++ .../invalid-http-header-match-regexp.json | 17 +++ .../testdata/invalid-http-header-match.json | 16 ++ ...uest-compression-reject-all-encodings.json | 14 ++ .../testdata/invalid-icmp-ttl-overflow.json | 10 ++ config/testdata/invalid-icmp-ttl.json | 10 ++ .../invalid-tcp-query-response-regexp.json | 18 +++ 18 files changed, 516 insertions(+), 44 deletions(-) create mode 100644 config/testdata/blackbox-bad.json create mode 100644 config/testdata/blackbox-bad2.json create mode 100644 config/testdata/blackbox-good.json create mode 100644 config/testdata/invalid-dns-class.json create mode 100644 config/testdata/invalid-dns-module.json create mode 100644 config/testdata/invalid-dns-type.json create mode 100644 config/testdata/invalid-http-body-config.json create mode 100644 config/testdata/invalid-http-body-match-regexp.json create mode 100644 config/testdata/invalid-http-body-not-match-regexp.json create mode 100644 config/testdata/invalid-http-compression-mismatch-special-case.json create mode 100644 config/testdata/invalid-http-compression-mismatch.json create mode 100644 config/testdata/invalid-http-header-match-regexp.json create mode 100644 config/testdata/invalid-http-header-match.json create mode 100644 config/testdata/invalid-http-request-compression-reject-all-encodings.json create mode 100644 config/testdata/invalid-icmp-ttl-overflow.json create mode 100644 config/testdata/invalid-icmp-ttl.json create mode 100644 config/testdata/invalid-tcp-query-response-regexp.json diff --git a/config/config_test.go b/config/config_test.go index 974e6f35..dbb5db9c 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -14,100 +14,132 @@ package config import ( + "fmt" "strings" "testing" "github.com/prometheus/client_golang/prometheus" + "github.com/stretchr/testify/require" yaml "gopkg.in/yaml.v3" ) func TestLoadConfig(t *testing.T) { - sc := NewSafeConfig(prometheus.NewRegistry()) + diff := make([]*SafeConfig, 2) + for idx, file := range []string{"testdata/blackbox-good.yml", "testdata/blackbox-good.json"} { + sc := NewSafeConfig(prometheus.NewRegistry()) - err := sc.ReloadConfig("testdata/blackbox-good.yml", nil) - if err != nil { - t.Errorf("Error loading config %v: %v", "blackbox.yml", err) + err := sc.ReloadConfig(file, nil) + if err != nil { + t.Errorf("Error loading config %v: %v", file, err) + } + diff[idx] = sc } + require.EqualExportedValues(t, diff[0], diff[1]) } func TestLoadBadConfigs(t *testing.T) { sc := NewSafeConfig(prometheus.NewRegistry()) tests := []struct { - input string - want string + input string + want string + format []string }{ { - input: "testdata/blackbox-bad.yml", - want: "error parsing config file: yaml: unmarshal errors:\n line 50: field invalid_extra_field not found in type config.plain", + input: "testdata/blackbox-bad", + want: "error parsing config file: yaml: unmarshal errors:\n line 50: field invalid_extra_field not found in type config.plain", + format: []string{"yml"}, }, { - input: "testdata/blackbox-bad2.yml", - want: "error parsing config file: at most one of bearer_token & bearer_token_file must be configured", + input: "testdata/blackbox-bad", + want: "error parsing config file: json: unknown field \"invalid_extra_field\"", + format: []string{"json"}, }, { - input: "testdata/invalid-dns-module.yml", - want: "error parsing config file: query name must be set for DNS module", + input: "testdata/blackbox-bad2", + want: "error parsing config file: at most one of bearer_token & bearer_token_file must be configured", + format: []string{"yml", "json"}, }, { - input: "testdata/invalid-dns-class.yml", - want: "error parsing config file: query class 'X' is not valid", + input: "testdata/invalid-dns-module", + want: "error parsing config file: query name must be set for DNS module", + format: []string{"yml", "json"}, }, { - input: "testdata/invalid-dns-type.yml", - want: "error parsing config file: query type 'X' is not valid", + input: "testdata/invalid-dns-class", + want: "error parsing config file: query class 'X' is not valid", + format: []string{"yml", "json"}, }, { - input: "testdata/invalid-http-header-match.yml", - want: "error parsing config file: regexp must be set for HTTP header matchers", + input: "testdata/invalid-dns-type", + want: "error parsing config file: query type 'X' is not valid", + format: []string{"yml", "json"}, }, { - input: "testdata/invalid-http-body-match-regexp.yml", - want: `error parsing config file: "Could not compile regular expression" regexp=":["`, + input: "testdata/invalid-http-header-match", + want: "error parsing config file: regexp must be set for HTTP header matchers", + format: []string{"yml", "json"}, }, { - input: "testdata/invalid-http-body-not-match-regexp.yml", - want: `error parsing config file: "Could not compile regular expression" regexp=":["`, + input: "testdata/invalid-http-body-match-regexp", + want: `error parsing config file: "Could not compile regular expression" regexp=":["`, + format: []string{"yml", "json"}, }, { - input: "testdata/invalid-http-header-match-regexp.yml", - want: `error parsing config file: "Could not compile regular expression" regexp=":["`, + input: "testdata/invalid-http-body-not-match-regexp", + want: `error parsing config file: "Could not compile regular expression" regexp=":["`, + format: []string{"yml", "json"}, }, { - input: "testdata/invalid-http-compression-mismatch.yml", - want: `error parsing config file: invalid configuration "Accept-Encoding: deflate", "compression: gzip"`, + input: "testdata/invalid-http-header-match-regexp", + want: `error parsing config file: "Could not compile regular expression" regexp=":["`, + format: []string{"yml", "json"}, }, { - input: "testdata/invalid-http-compression-mismatch-special-case.yml", - want: `error parsing config file: invalid configuration "accEpt-enCoding: deflate", "compression: gzip"`, + input: "testdata/invalid-http-compression-mismatch", + want: `error parsing config file: invalid configuration "Accept-Encoding: deflate", "compression: gzip"`, + format: []string{"yml", "json"}, }, { - input: "testdata/invalid-http-request-compression-reject-all-encodings.yml", - want: `error parsing config file: invalid configuration "Accept-Encoding: *;q=0.0", "compression: gzip"`, + input: "testdata/invalid-http-compression-mismatch-special-case", + want: `error parsing config file: invalid configuration "accEpt-enCoding: deflate", "compression: gzip"`, + format: []string{"yml", "json"}, }, { - input: "testdata/invalid-icmp-ttl.yml", - want: "error parsing config file: \"ttl\" cannot be negative", + input: "testdata/invalid-http-request-compression-reject-all-encodings", + want: `error parsing config file: invalid configuration "Accept-Encoding: *;q=0.0", "compression: gzip"`, + format: []string{"yml", "json"}, }, { - input: "testdata/invalid-icmp-ttl-overflow.yml", - want: "error parsing config file: \"ttl\" cannot exceed 255", + input: "testdata/invalid-icmp-ttl", + want: "error parsing config file: \"ttl\" cannot be negative", + format: []string{"yml", "json"}, }, { - input: "testdata/invalid-tcp-query-response-regexp.yml", - want: `error parsing config file: "Could not compile regular expression" regexp=":["`, + input: "testdata/invalid-icmp-ttl-overflow", + want: "error parsing config file: \"ttl\" cannot exceed 255", + format: []string{"yml", "json"}, }, { - input: "testdata/invalid-http-body-config.yml", - want: `error parsing config file: setting body and body_file both are not allowed`, + input: "testdata/invalid-tcp-query-response-regexp", + want: `error parsing config file: "Could not compile regular expression" regexp=":["`, + format: []string{"yml", "json"}, + }, + { + input: "testdata/invalid-http-body-config", + want: `error parsing config file: setting body and body_file both are not allowed`, + format: []string{"yml", "json"}, }, } for _, test := range tests { - t.Run(test.input, func(t *testing.T) { - got := sc.ReloadConfig(test.input, nil) - if got == nil || got.Error() != test.want { - t.Fatalf("ReloadConfig(%q) = %v; want %q", test.input, got, test.want) - } - }) + for _, format := range test.format { + path := fmt.Sprintf("%s.%s", test.input, format) + t.Run(path, func(t *testing.T) { + got := sc.ReloadConfig(path, nil) + if got == nil || got.Error() != test.want { + t.Fatalf("ReloadConfig(%q) = %v; want %q", path, got, test.want) + } + }) + } } } diff --git a/config/testdata/blackbox-bad.json b/config/testdata/blackbox-bad.json new file mode 100644 index 00000000..2fe0bb00 --- /dev/null +++ b/config/testdata/blackbox-bad.json @@ -0,0 +1,86 @@ +{ + "modules": { + "http_2xx": { + "prober": "http", + "timeout": "5s", + "http": null + }, + "http_post_2xx": { + "prober": "http", + "timeout": "5s", + "http": { + "method": "POST", + "invalid_extra_field": "value" + } + }, + "tcp_connect": { + "prober": "tcp", + "timeout": "5s" + }, + "pop3s_banner": { + "prober": "tcp", + "tcp": { + "query_response": [ + { + "expect": "^+OK" + } + ], + "tls": true, + "tls_config": { + "insecure_skip_verify": false + } + } + }, + "ssh_banner": { + "prober": "tcp", + "timeout": "5s", + "tcp": { + "query_response": [ + { + "expect": "^SSH-2.0-" + } + ] + } + }, + "irc_banner": { + "prober": "tcp", + "timeout": "5s", + "tcp": { + "query_response": [ + { + "send": "NICK prober" + }, + { + "send": "USER prober prober prober :prober" + }, + { + "expect": "PING :([^ ]+)", + "send": "PONG ${1}" + }, + { + "expect": "^:[^ ]+ 001" + } + ] + } + }, + "icmp_test": { + "prober": "icmp", + "timeout": "5s", + "icmp": { + "preferred_ip_protocol": "ip4" + } + }, + "dns_test": { + "prober": "dns", + "timeout": "5s", + "dns": { + "preferred_ip_protocol": "ip6", + "validate_answer_rrs": { + "fail_if_matches_regexp": [ + "test" + ] + } + } + } + } +} diff --git a/config/testdata/blackbox-bad2.json b/config/testdata/blackbox-bad2.json new file mode 100644 index 00000000..a9bbfd5b --- /dev/null +++ b/config/testdata/blackbox-bad2.json @@ -0,0 +1,17 @@ +{ + "modules": { + "http_post_2xx": { + "prober": "http", + "timeout": "5s", + "http": { + "method": "POST", + "bearer_token": "foo", + "bearer_token_file": "foo", + "basic_auth": { + "username": "username", + "password": "mysecret" + } + } + } + } +} diff --git a/config/testdata/blackbox-good.json b/config/testdata/blackbox-good.json new file mode 100644 index 00000000..25864289 --- /dev/null +++ b/config/testdata/blackbox-good.json @@ -0,0 +1,144 @@ +{ + "modules": { + "http_2xx": { + "prober": "http", + "timeout": "5s", + "http": null + }, + "http_post_2xx": { + "prober": "http", + "timeout": "5s", + "http": { + "method": "POST", + "basic_auth": { + "username": "username", + "password": "mysecret" + }, + "body_size_limit": "1MB" + } + }, + "tcp_connect": { + "prober": "tcp", + "timeout": "5s" + }, + "pop3s_banner": { + "prober": "tcp", + "tcp": { + "query_response": [ + { + "expect": "^+OK" + } + ], + "tls": true, + "tls_config": { + "insecure_skip_verify": false + } + } + }, + "ssh_banner": { + "prober": "tcp", + "timeout": "5s", + "tcp": { + "query_response": [ + { + "expect": "^SSH-2.0-" + } + ] + } + }, + "smtp_starttls": { + "prober": "tcp", + "timeout": "5s", + "tcp": { + "query_response": [ + { + "expect": "^220 " + }, + { + "send": "EHLO prober\r" + }, + { + "expect": "^250-STARTTLS" + }, + { + "send": "STARTTLS\r" + }, + { + "expect": "^220" + }, + { + "starttls": true + }, + { + "send": "EHLO prober\r" + }, + { + "expect": "^250-AUTH" + }, + { + "send": "QUIT\r" + } + ] + } + }, + "irc_banner": { + "prober": "tcp", + "timeout": "5s", + "tcp": { + "query_response": [ + { + "send": "NICK prober" + }, + { + "send": "USER prober prober prober :prober" + }, + { + "expect": "PING :([^ ]+)", + "send": "PONG ${1}" + }, + { + "expect": "^:[^ ]+ 001" + } + ] + } + }, + "icmp_test": { + "prober": "icmp", + "timeout": "5s", + "icmp": { + "preferred_ip_protocol": "ip4" + } + }, + "dns_test": { + "prober": "dns", + "timeout": "5s", + "dns": { + "query_name": "example.com", + "preferred_ip_protocol": "ip4", + "ip_protocol_fallback": false, + "validate_answer_rrs": { + "fail_if_matches_regexp": [ + "test" + ] + } + } + }, + "http_header_match_origin": { + "prober": "http", + "timeout": "5s", + "http": { + "method": "GET", + "headers": { + "Origin": "example.com" + }, + "fail_if_header_not_matches": [ + { + "header": "Access-Control-Allow-Origin", + "regexp": "(\\*|example\\.com)", + "allow_missing": false + } + ] + } + } + } +} diff --git a/config/testdata/invalid-dns-class.json b/config/testdata/invalid-dns-class.json new file mode 100644 index 00000000..fb0ad221 --- /dev/null +++ b/config/testdata/invalid-dns-class.json @@ -0,0 +1,13 @@ +{ + "modules": { + "dns_test": { + "prober": "dns", + "timeout": "5s", + "dns": { + "query_name": "example.com", + "query_class": "X", + "query_type": "A" + } + } + } +} diff --git a/config/testdata/invalid-dns-module.json b/config/testdata/invalid-dns-module.json new file mode 100644 index 00000000..851bbb58 --- /dev/null +++ b/config/testdata/invalid-dns-module.json @@ -0,0 +1,16 @@ +{ + "modules": { + "dns_test": { + "prober": "dns", + "timeout": "5s", + "dns": { + "preferred_ip_protocol": "ip6", + "validate_answer_rrs": { + "fail_if_matches_regexp": [ + "test" + ] + } + } + } + } +} diff --git a/config/testdata/invalid-dns-type.json b/config/testdata/invalid-dns-type.json new file mode 100644 index 00000000..7cf52b1e --- /dev/null +++ b/config/testdata/invalid-dns-type.json @@ -0,0 +1,13 @@ +{ + "modules": { + "dns_test": { + "prober": "dns", + "timeout": "5s", + "dns": { + "query_name": "example.com", + "query_class": "CH", + "query_type": "X" + } + } + } +} diff --git a/config/testdata/invalid-http-body-config.json b/config/testdata/invalid-http-body-config.json new file mode 100644 index 00000000..eb4fcbe1 --- /dev/null +++ b/config/testdata/invalid-http-body-config.json @@ -0,0 +1,12 @@ +{ + "modules": { + "http_test": { + "prober": "http", + "timeout": "5s", + "http": { + "body": "Test body", + "body_file": "test_body.txt" + } + } + } +} diff --git a/config/testdata/invalid-http-body-match-regexp.json b/config/testdata/invalid-http-body-match-regexp.json new file mode 100644 index 00000000..0f74cb84 --- /dev/null +++ b/config/testdata/invalid-http-body-match-regexp.json @@ -0,0 +1,13 @@ +{ + "modules": { + "http_headers": { + "prober": "http", + "timeout": "5s", + "http": { + "fail_if_body_matches_regexp": [ + ":[" + ] + } + } + } +} diff --git a/config/testdata/invalid-http-body-not-match-regexp.json b/config/testdata/invalid-http-body-not-match-regexp.json new file mode 100644 index 00000000..2e3279ce --- /dev/null +++ b/config/testdata/invalid-http-body-not-match-regexp.json @@ -0,0 +1,13 @@ +{ + "modules": { + "http_headers": { + "prober": "http", + "timeout": "5s", + "http": { + "fail_if_body_not_matches_regexp": [ + ":[" + ] + } + } + } +} diff --git a/config/testdata/invalid-http-compression-mismatch-special-case.json b/config/testdata/invalid-http-compression-mismatch-special-case.json new file mode 100644 index 00000000..4792ac1a --- /dev/null +++ b/config/testdata/invalid-http-compression-mismatch-special-case.json @@ -0,0 +1,14 @@ +{ + "modules": { + "http_headers": { + "prober": "http", + "timeout": "5s", + "http": { + "compression": "gzip", + "headers": { + "accEpt-enCoding": "deflate" + } + } + } + } +} diff --git a/config/testdata/invalid-http-compression-mismatch.json b/config/testdata/invalid-http-compression-mismatch.json new file mode 100644 index 00000000..8ee590e6 --- /dev/null +++ b/config/testdata/invalid-http-compression-mismatch.json @@ -0,0 +1,14 @@ +{ + "modules": { + "http_headers": { + "prober": "http", + "timeout": "5s", + "http": { + "compression": "gzip", + "headers": { + "Accept-Encoding": "deflate" + } + } + } + } +} diff --git a/config/testdata/invalid-http-header-match-regexp.json b/config/testdata/invalid-http-header-match-regexp.json new file mode 100644 index 00000000..233e2ac0 --- /dev/null +++ b/config/testdata/invalid-http-header-match-regexp.json @@ -0,0 +1,17 @@ +{ + "modules": { + "http_headers": { + "prober": "http", + "timeout": "5s", + "http": { + "fail_if_header_not_matches": [ + { + "header": "Access-Control-Allow-Origin", + "allow_missing": false, + "regexp": ":[" + } + ] + } + } + } +} diff --git a/config/testdata/invalid-http-header-match.json b/config/testdata/invalid-http-header-match.json new file mode 100644 index 00000000..a23d6c58 --- /dev/null +++ b/config/testdata/invalid-http-header-match.json @@ -0,0 +1,16 @@ +{ + "modules": { + "http_headers": { + "prober": "http", + "timeout": "5s", + "http": { + "fail_if_header_not_matches": [ + { + "header": "Access-Control-Allow-Origin", + "allow_missing": false + } + ] + } + } + } +} diff --git a/config/testdata/invalid-http-request-compression-reject-all-encodings.json b/config/testdata/invalid-http-request-compression-reject-all-encodings.json new file mode 100644 index 00000000..5a64f155 --- /dev/null +++ b/config/testdata/invalid-http-request-compression-reject-all-encodings.json @@ -0,0 +1,14 @@ +{ + "modules": { + "http_headers": { + "prober": "http", + "timeout": "5s", + "http": { + "compression": "gzip", + "headers": { + "Accept-Encoding": "*;q=0.0" + } + } + } + } +} diff --git a/config/testdata/invalid-icmp-ttl-overflow.json b/config/testdata/invalid-icmp-ttl-overflow.json new file mode 100644 index 00000000..bea59347 --- /dev/null +++ b/config/testdata/invalid-icmp-ttl-overflow.json @@ -0,0 +1,10 @@ +{ + "modules": { + "icmp_test": { + "prober": "icmp", + "icmp": { + "ttl": 256 + } + } + } +} diff --git a/config/testdata/invalid-icmp-ttl.json b/config/testdata/invalid-icmp-ttl.json new file mode 100644 index 00000000..92f75967 --- /dev/null +++ b/config/testdata/invalid-icmp-ttl.json @@ -0,0 +1,10 @@ +{ + "modules": { + "icmp_test": { + "prober": "icmp", + "icmp": { + "ttl": -1 + } + } + } +} diff --git a/config/testdata/invalid-tcp-query-response-regexp.json b/config/testdata/invalid-tcp-query-response-regexp.json new file mode 100644 index 00000000..297b406b --- /dev/null +++ b/config/testdata/invalid-tcp-query-response-regexp.json @@ -0,0 +1,18 @@ +{ + "modules": { + "tcp_test": { + "prober": "tcp", + "timeout": "5s", + "tcp": { + "query_response": [ + { + "expect": ":[" + }, + { + "send": ". STARTTLS" + } + ] + } + } + } +} From 1e430e4ab132ba4f6654dbf45e33de114e275aa0 Mon Sep 17 00:00:00 2001 From: Jakob Hahn Date: Fri, 18 Oct 2024 12:52:44 +0200 Subject: [PATCH 4/8] go: add modules for json unmarshal and testing Signed-off-by: Jakob Hahn --- go.mod | 3 +++ 1 file changed, 3 insertions(+) diff --git a/go.mod b/go.mod index 7e62fb90..6c108b27 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,7 @@ require ( github.com/prometheus/client_model v0.6.1 github.com/prometheus/common v0.57.0 github.com/prometheus/exporter-toolkit v0.11.0 + github.com/stretchr/testify v1.9.0 golang.org/x/net v0.28.0 google.golang.org/grpc v1.67.0 gopkg.in/yaml.v2 v2.4.0 @@ -22,11 +23,13 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/coreos/go-systemd/v22 v22.5.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect github.com/go-logfmt/logfmt v0.5.1 // indirect github.com/jpillora/backoff v1.0.0 // indirect github.com/klauspost/compress v1.17.9 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/procfs v0.15.1 // indirect github.com/xhit/go-str2duration/v2 v2.1.0 // indirect golang.org/x/crypto v0.26.0 // indirect From 17128dcdb74b624af28739a16092ee0b25f7f8cb Mon Sep 17 00:00:00 2001 From: Jakob Hahn Date: Fri, 18 Oct 2024 16:00:20 +0200 Subject: [PATCH 5/8] config: fix Regexp Marshal, add more tests Signed-off-by: Jakob Hahn --- config/config.go | 11 +++----- config/config_test.go | 66 +++++++++++++++++++++++++++++++++++++++---- 2 files changed, 64 insertions(+), 13 deletions(-) diff --git a/config/config.go b/config/config.go index 12c81fa6..59eeee9f 100644 --- a/config/config.go +++ b/config/config.go @@ -159,8 +159,8 @@ func (sc *SafeConfig) ReloadConfig(confFile string, logger log.Logger) (err erro // Regexp encapsulates a regexp.Regexp and makes it YAML marshalable. type Regexp struct { - *regexp.Regexp - original string + *regexp.Regexp `json:"-"` + original string } // NewRegexp creates a new anchored Regexp and returns an error if the @@ -212,11 +212,8 @@ func (re Regexp) MarshalYAML() (interface{}, error) { } // MarshalJSON implements the json.Marshaler interface. -func (re Regexp) MarshalJSOn() ([]byte, error) { - if re.original != "" { - return []byte(re.original), nil - } - return nil, nil +func (re Regexp) MarshalJSON() ([]byte, error) { + return json.Marshal(re.original) } // MustNewRegexp works like NewRegexp, but panics if the regular expression does not compile. diff --git a/config/config_test.go b/config/config_test.go index dbb5db9c..29bea6c5 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -14,6 +14,7 @@ package config import ( + "encoding/json" "fmt" "strings" "testing" @@ -26,17 +27,70 @@ import ( func TestLoadConfig(t *testing.T) { diff := make([]*SafeConfig, 2) for idx, file := range []string{"testdata/blackbox-good.yml", "testdata/blackbox-good.json"} { - sc := NewSafeConfig(prometheus.NewRegistry()) + t.Run(file, func(t *testing.T) { + sc := NewSafeConfig(prometheus.NewRegistry()) - err := sc.ReloadConfig(file, nil) - if err != nil { - t.Errorf("Error loading config %v: %v", file, err) - } - diff[idx] = sc + err := sc.ReloadConfig(file, nil) + if err != nil { + t.Errorf("Error loading config %v: %v", file, err) + } + diff[idx] = sc + }) } require.EqualExportedValues(t, diff[0], diff[1]) } +// Testing the Marshal and Unmarshal functions of the Regexp type +func TestRegexpMarshal(t *testing.T) { + var regexp struct { + Test Regexp `yaml:"test" json:"test"` + } + t.Run("JSON", func(t *testing.T) { + data := []byte(`{"test":"(\\w+.+)"}`) + err := json.Unmarshal(data, ®exp) + require.NoError(t, err) + marshaled, err := json.Marshal(®exp) + require.NoError(t, err) + require.Equal(t, string(data), string(marshaled)) + + _, err = json.Marshal(Regexp{}) + require.NoError(t, err) + }) + + t.Run("YAML", func(t *testing.T) { + data := []byte("test: (\\w+.+)\n") + err := yaml.Unmarshal(data, ®exp) + require.NoError(t, err) + marshaled, err := yaml.Marshal(®exp) + require.NoError(t, err) + require.Equal(t, string(data), string(marshaled)) + + _, err = yaml.Marshal(Regexp{}) + require.NoError(t, err) + }) +} + +// Testing the capability of Marsheling the config without errors +func TestConfigMarshal(t *testing.T) { + for _, file := range []string{"testdata/blackbox-good.yml", "testdata/blackbox-good.json"} { + t.Run(file, func(t *testing.T) { + sc := NewSafeConfig(prometheus.NewRegistry()) + err := sc.ReloadConfig(file, nil) + if err != nil { + t.Errorf("Error loading config %v: %v", file, err) + } + + if strings.HasSuffix(file, ".json") { + _, err := json.Marshal(sc.C) + require.NoError(t, err) + } else { + _, err := yaml.Marshal(sc.C) + require.NoError(t, err) + } + }) + } +} + func TestLoadBadConfigs(t *testing.T) { sc := NewSafeConfig(prometheus.NewRegistry()) tests := []struct { From cd379e3d01525733f8885f2c30d52d90daa53b5c Mon Sep 17 00:00:00 2001 From: Jakob Hahn Date: Mon, 21 Oct 2024 10:22:22 +0200 Subject: [PATCH 6/8] config: remove json test data Signed-off-by: Jakob Hahn --- config/testdata/blackbox-bad.json | 86 ----------- config/testdata/blackbox-bad2.json | 17 --- config/testdata/blackbox-good.json | 144 ------------------ config/testdata/invalid-dns-class.json | 13 -- config/testdata/invalid-dns-module.json | 16 -- config/testdata/invalid-dns-type.json | 13 -- config/testdata/invalid-http-body-config.json | 12 -- .../invalid-http-body-match-regexp.json | 13 -- .../invalid-http-body-not-match-regexp.json | 13 -- ...ttp-compression-mismatch-special-case.json | 14 -- .../invalid-http-compression-mismatch.json | 14 -- .../invalid-http-header-match-regexp.json | 17 --- .../testdata/invalid-http-header-match.json | 16 -- ...uest-compression-reject-all-encodings.json | 14 -- .../testdata/invalid-icmp-ttl-overflow.json | 10 -- config/testdata/invalid-icmp-ttl.json | 10 -- .../invalid-tcp-query-response-regexp.json | 18 --- 17 files changed, 440 deletions(-) delete mode 100644 config/testdata/blackbox-bad.json delete mode 100644 config/testdata/blackbox-bad2.json delete mode 100644 config/testdata/blackbox-good.json delete mode 100644 config/testdata/invalid-dns-class.json delete mode 100644 config/testdata/invalid-dns-module.json delete mode 100644 config/testdata/invalid-dns-type.json delete mode 100644 config/testdata/invalid-http-body-config.json delete mode 100644 config/testdata/invalid-http-body-match-regexp.json delete mode 100644 config/testdata/invalid-http-body-not-match-regexp.json delete mode 100644 config/testdata/invalid-http-compression-mismatch-special-case.json delete mode 100644 config/testdata/invalid-http-compression-mismatch.json delete mode 100644 config/testdata/invalid-http-header-match-regexp.json delete mode 100644 config/testdata/invalid-http-header-match.json delete mode 100644 config/testdata/invalid-http-request-compression-reject-all-encodings.json delete mode 100644 config/testdata/invalid-icmp-ttl-overflow.json delete mode 100644 config/testdata/invalid-icmp-ttl.json delete mode 100644 config/testdata/invalid-tcp-query-response-regexp.json diff --git a/config/testdata/blackbox-bad.json b/config/testdata/blackbox-bad.json deleted file mode 100644 index 2fe0bb00..00000000 --- a/config/testdata/blackbox-bad.json +++ /dev/null @@ -1,86 +0,0 @@ -{ - "modules": { - "http_2xx": { - "prober": "http", - "timeout": "5s", - "http": null - }, - "http_post_2xx": { - "prober": "http", - "timeout": "5s", - "http": { - "method": "POST", - "invalid_extra_field": "value" - } - }, - "tcp_connect": { - "prober": "tcp", - "timeout": "5s" - }, - "pop3s_banner": { - "prober": "tcp", - "tcp": { - "query_response": [ - { - "expect": "^+OK" - } - ], - "tls": true, - "tls_config": { - "insecure_skip_verify": false - } - } - }, - "ssh_banner": { - "prober": "tcp", - "timeout": "5s", - "tcp": { - "query_response": [ - { - "expect": "^SSH-2.0-" - } - ] - } - }, - "irc_banner": { - "prober": "tcp", - "timeout": "5s", - "tcp": { - "query_response": [ - { - "send": "NICK prober" - }, - { - "send": "USER prober prober prober :prober" - }, - { - "expect": "PING :([^ ]+)", - "send": "PONG ${1}" - }, - { - "expect": "^:[^ ]+ 001" - } - ] - } - }, - "icmp_test": { - "prober": "icmp", - "timeout": "5s", - "icmp": { - "preferred_ip_protocol": "ip4" - } - }, - "dns_test": { - "prober": "dns", - "timeout": "5s", - "dns": { - "preferred_ip_protocol": "ip6", - "validate_answer_rrs": { - "fail_if_matches_regexp": [ - "test" - ] - } - } - } - } -} diff --git a/config/testdata/blackbox-bad2.json b/config/testdata/blackbox-bad2.json deleted file mode 100644 index a9bbfd5b..00000000 --- a/config/testdata/blackbox-bad2.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "modules": { - "http_post_2xx": { - "prober": "http", - "timeout": "5s", - "http": { - "method": "POST", - "bearer_token": "foo", - "bearer_token_file": "foo", - "basic_auth": { - "username": "username", - "password": "mysecret" - } - } - } - } -} diff --git a/config/testdata/blackbox-good.json b/config/testdata/blackbox-good.json deleted file mode 100644 index 25864289..00000000 --- a/config/testdata/blackbox-good.json +++ /dev/null @@ -1,144 +0,0 @@ -{ - "modules": { - "http_2xx": { - "prober": "http", - "timeout": "5s", - "http": null - }, - "http_post_2xx": { - "prober": "http", - "timeout": "5s", - "http": { - "method": "POST", - "basic_auth": { - "username": "username", - "password": "mysecret" - }, - "body_size_limit": "1MB" - } - }, - "tcp_connect": { - "prober": "tcp", - "timeout": "5s" - }, - "pop3s_banner": { - "prober": "tcp", - "tcp": { - "query_response": [ - { - "expect": "^+OK" - } - ], - "tls": true, - "tls_config": { - "insecure_skip_verify": false - } - } - }, - "ssh_banner": { - "prober": "tcp", - "timeout": "5s", - "tcp": { - "query_response": [ - { - "expect": "^SSH-2.0-" - } - ] - } - }, - "smtp_starttls": { - "prober": "tcp", - "timeout": "5s", - "tcp": { - "query_response": [ - { - "expect": "^220 " - }, - { - "send": "EHLO prober\r" - }, - { - "expect": "^250-STARTTLS" - }, - { - "send": "STARTTLS\r" - }, - { - "expect": "^220" - }, - { - "starttls": true - }, - { - "send": "EHLO prober\r" - }, - { - "expect": "^250-AUTH" - }, - { - "send": "QUIT\r" - } - ] - } - }, - "irc_banner": { - "prober": "tcp", - "timeout": "5s", - "tcp": { - "query_response": [ - { - "send": "NICK prober" - }, - { - "send": "USER prober prober prober :prober" - }, - { - "expect": "PING :([^ ]+)", - "send": "PONG ${1}" - }, - { - "expect": "^:[^ ]+ 001" - } - ] - } - }, - "icmp_test": { - "prober": "icmp", - "timeout": "5s", - "icmp": { - "preferred_ip_protocol": "ip4" - } - }, - "dns_test": { - "prober": "dns", - "timeout": "5s", - "dns": { - "query_name": "example.com", - "preferred_ip_protocol": "ip4", - "ip_protocol_fallback": false, - "validate_answer_rrs": { - "fail_if_matches_regexp": [ - "test" - ] - } - } - }, - "http_header_match_origin": { - "prober": "http", - "timeout": "5s", - "http": { - "method": "GET", - "headers": { - "Origin": "example.com" - }, - "fail_if_header_not_matches": [ - { - "header": "Access-Control-Allow-Origin", - "regexp": "(\\*|example\\.com)", - "allow_missing": false - } - ] - } - } - } -} diff --git a/config/testdata/invalid-dns-class.json b/config/testdata/invalid-dns-class.json deleted file mode 100644 index fb0ad221..00000000 --- a/config/testdata/invalid-dns-class.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "modules": { - "dns_test": { - "prober": "dns", - "timeout": "5s", - "dns": { - "query_name": "example.com", - "query_class": "X", - "query_type": "A" - } - } - } -} diff --git a/config/testdata/invalid-dns-module.json b/config/testdata/invalid-dns-module.json deleted file mode 100644 index 851bbb58..00000000 --- a/config/testdata/invalid-dns-module.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "modules": { - "dns_test": { - "prober": "dns", - "timeout": "5s", - "dns": { - "preferred_ip_protocol": "ip6", - "validate_answer_rrs": { - "fail_if_matches_regexp": [ - "test" - ] - } - } - } - } -} diff --git a/config/testdata/invalid-dns-type.json b/config/testdata/invalid-dns-type.json deleted file mode 100644 index 7cf52b1e..00000000 --- a/config/testdata/invalid-dns-type.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "modules": { - "dns_test": { - "prober": "dns", - "timeout": "5s", - "dns": { - "query_name": "example.com", - "query_class": "CH", - "query_type": "X" - } - } - } -} diff --git a/config/testdata/invalid-http-body-config.json b/config/testdata/invalid-http-body-config.json deleted file mode 100644 index eb4fcbe1..00000000 --- a/config/testdata/invalid-http-body-config.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "modules": { - "http_test": { - "prober": "http", - "timeout": "5s", - "http": { - "body": "Test body", - "body_file": "test_body.txt" - } - } - } -} diff --git a/config/testdata/invalid-http-body-match-regexp.json b/config/testdata/invalid-http-body-match-regexp.json deleted file mode 100644 index 0f74cb84..00000000 --- a/config/testdata/invalid-http-body-match-regexp.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "modules": { - "http_headers": { - "prober": "http", - "timeout": "5s", - "http": { - "fail_if_body_matches_regexp": [ - ":[" - ] - } - } - } -} diff --git a/config/testdata/invalid-http-body-not-match-regexp.json b/config/testdata/invalid-http-body-not-match-regexp.json deleted file mode 100644 index 2e3279ce..00000000 --- a/config/testdata/invalid-http-body-not-match-regexp.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "modules": { - "http_headers": { - "prober": "http", - "timeout": "5s", - "http": { - "fail_if_body_not_matches_regexp": [ - ":[" - ] - } - } - } -} diff --git a/config/testdata/invalid-http-compression-mismatch-special-case.json b/config/testdata/invalid-http-compression-mismatch-special-case.json deleted file mode 100644 index 4792ac1a..00000000 --- a/config/testdata/invalid-http-compression-mismatch-special-case.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "modules": { - "http_headers": { - "prober": "http", - "timeout": "5s", - "http": { - "compression": "gzip", - "headers": { - "accEpt-enCoding": "deflate" - } - } - } - } -} diff --git a/config/testdata/invalid-http-compression-mismatch.json b/config/testdata/invalid-http-compression-mismatch.json deleted file mode 100644 index 8ee590e6..00000000 --- a/config/testdata/invalid-http-compression-mismatch.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "modules": { - "http_headers": { - "prober": "http", - "timeout": "5s", - "http": { - "compression": "gzip", - "headers": { - "Accept-Encoding": "deflate" - } - } - } - } -} diff --git a/config/testdata/invalid-http-header-match-regexp.json b/config/testdata/invalid-http-header-match-regexp.json deleted file mode 100644 index 233e2ac0..00000000 --- a/config/testdata/invalid-http-header-match-regexp.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "modules": { - "http_headers": { - "prober": "http", - "timeout": "5s", - "http": { - "fail_if_header_not_matches": [ - { - "header": "Access-Control-Allow-Origin", - "allow_missing": false, - "regexp": ":[" - } - ] - } - } - } -} diff --git a/config/testdata/invalid-http-header-match.json b/config/testdata/invalid-http-header-match.json deleted file mode 100644 index a23d6c58..00000000 --- a/config/testdata/invalid-http-header-match.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "modules": { - "http_headers": { - "prober": "http", - "timeout": "5s", - "http": { - "fail_if_header_not_matches": [ - { - "header": "Access-Control-Allow-Origin", - "allow_missing": false - } - ] - } - } - } -} diff --git a/config/testdata/invalid-http-request-compression-reject-all-encodings.json b/config/testdata/invalid-http-request-compression-reject-all-encodings.json deleted file mode 100644 index 5a64f155..00000000 --- a/config/testdata/invalid-http-request-compression-reject-all-encodings.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "modules": { - "http_headers": { - "prober": "http", - "timeout": "5s", - "http": { - "compression": "gzip", - "headers": { - "Accept-Encoding": "*;q=0.0" - } - } - } - } -} diff --git a/config/testdata/invalid-icmp-ttl-overflow.json b/config/testdata/invalid-icmp-ttl-overflow.json deleted file mode 100644 index bea59347..00000000 --- a/config/testdata/invalid-icmp-ttl-overflow.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "modules": { - "icmp_test": { - "prober": "icmp", - "icmp": { - "ttl": 256 - } - } - } -} diff --git a/config/testdata/invalid-icmp-ttl.json b/config/testdata/invalid-icmp-ttl.json deleted file mode 100644 index 92f75967..00000000 --- a/config/testdata/invalid-icmp-ttl.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "modules": { - "icmp_test": { - "prober": "icmp", - "icmp": { - "ttl": -1 - } - } - } -} diff --git a/config/testdata/invalid-tcp-query-response-regexp.json b/config/testdata/invalid-tcp-query-response-regexp.json deleted file mode 100644 index 297b406b..00000000 --- a/config/testdata/invalid-tcp-query-response-regexp.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "modules": { - "tcp_test": { - "prober": "tcp", - "timeout": "5s", - "tcp": { - "query_response": [ - { - "expect": ":[" - }, - { - "send": ". STARTTLS" - } - ] - } - } - } -} From af13c69bc836e248238346fb378f51e4866271d8 Mon Sep 17 00:00:00 2001 From: Jakob Hahn Date: Mon, 21 Oct 2024 10:27:27 +0200 Subject: [PATCH 7/8] config: adjust test to generate json files Signed-off-by: Jakob Hahn --- config/config_test.go | 46 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 44 insertions(+), 2 deletions(-) diff --git a/config/config_test.go b/config/config_test.go index 29bea6c5..60cd54f7 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -16,6 +16,7 @@ package config import ( "encoding/json" "fmt" + "os" "strings" "testing" @@ -24,9 +25,45 @@ import ( yaml "gopkg.in/yaml.v3" ) +func yamlToJson(t *testing.T, path string) error { + t.Helper() + data := make(map[string]any) + fileReader, err := os.Open(fmt.Sprintf("%s.yml", path)) + if err != nil { + return fmt.Errorf("error reading config file: %s", err) + } + defer fileReader.Close() + + decoder := yaml.NewDecoder(fileReader) + if err := decoder.Decode(&data); err != nil { + return err + } + + jsonData, err := json.Marshal(&data) + if err != nil { + return err + } + + file, err := os.Create(fmt.Sprintf("%s.json", path)) + if err != nil { + return err + } + defer file.Close() + t.Cleanup(func() { os.Remove(fmt.Sprintf("%s.json", path)) }) + + if _, err = file.Write(jsonData); err != nil { + return err + } + return nil +} + func TestLoadConfig(t *testing.T) { diff := make([]*SafeConfig, 2) - for idx, file := range []string{"testdata/blackbox-good.yml", "testdata/blackbox-good.json"} { + path := "testdata/blackbox-good" + require.NoError(t, yamlToJson(t, path)) + + for idx, format := range []string{"yml", "json"} { + file := fmt.Sprintf("%s.%s", path, format) t.Run(file, func(t *testing.T) { sc := NewSafeConfig(prometheus.NewRegistry()) @@ -72,7 +109,11 @@ func TestRegexpMarshal(t *testing.T) { // Testing the capability of Marsheling the config without errors func TestConfigMarshal(t *testing.T) { - for _, file := range []string{"testdata/blackbox-good.yml", "testdata/blackbox-good.json"} { + path := "testdata/blackbox-good" + require.NoError(t, yamlToJson(t, path)) + + for _, format := range []string{"yml", "json"} { + file := fmt.Sprintf("%s.%s", path, format) t.Run(file, func(t *testing.T) { sc := NewSafeConfig(prometheus.NewRegistry()) err := sc.ReloadConfig(file, nil) @@ -185,6 +226,7 @@ func TestLoadBadConfigs(t *testing.T) { }, } for _, test := range tests { + require.NoError(t, yamlToJson(t, test.input)) for _, format := range test.format { path := fmt.Sprintf("%s.%s", test.input, format) t.Run(path, func(t *testing.T) { From 682f9bd45081e4abf491d3227973b03748f3dc58 Mon Sep 17 00:00:00 2001 From: Jakob Hahn Date: Tue, 22 Oct 2024 14:50:02 +0200 Subject: [PATCH 8/8] config: validate module probe on unmarshal Signed-off-by: Jakob Hahn --- config/config.go | 12 ++++++++++-- config/config_test.go | 5 +++++ config/testdata/invalid-module-prober.yml | 5 +++++ 3 files changed, 20 insertions(+), 2 deletions(-) create mode 100644 config/testdata/invalid-module-prober.yml diff --git a/config/config.go b/config/config.go index 59eeee9f..b749833c 100644 --- a/config/config.go +++ b/config/config.go @@ -24,6 +24,7 @@ import ( "reflect" "regexp" "runtime" + "slices" "sort" "strconv" "strings" @@ -350,7 +351,7 @@ func (s *Module) UnmarshalYAML(unmarshal func(interface{}) error) error { if err := unmarshal((*plain)(s)); err != nil { return err } - return nil + return s.validate() } // UnmarshalJSON implements the json.Unmarshaler interface. @@ -390,7 +391,7 @@ func (s *Module) UnmarshalJSON(data []byte) error { return err } default: - return fmt.Errorf("invalid duration: %#v", tmp) + return fmt.Errorf("invalid duration '%#v'", tmp) } } *s = Module{ @@ -402,6 +403,13 @@ func (s *Module) UnmarshalJSON(data []byte) error { DNS: tmp.DNS, GRPC: tmp.GRPC, } + return s.validate() +} + +func (s *Module) validate() error { + if !slices.Contains([]string{"http", "tcp", "icmp", "dns", "grpc"}, s.Prober) { + return fmt.Errorf("prober '%s' is invalid", s.Prober) + } return nil } diff --git a/config/config_test.go b/config/config_test.go index 60cd54f7..46080a1f 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -224,6 +224,11 @@ func TestLoadBadConfigs(t *testing.T) { want: `error parsing config file: setting body and body_file both are not allowed`, format: []string{"yml", "json"}, }, + { + input: "testdata/invalid-module-prober", + want: `error parsing config file: prober 'hTTp' is invalid`, + format: []string{"yml", "json"}, + }, } for _, test := range tests { require.NoError(t, yamlToJson(t, test.input)) diff --git a/config/testdata/invalid-module-prober.yml b/config/testdata/invalid-module-prober.yml new file mode 100644 index 00000000..0b8e17b6 --- /dev/null +++ b/config/testdata/invalid-module-prober.yml @@ -0,0 +1,5 @@ +modules: + http_2xx: + prober: hTTp + timeout: 5s + http: