feat: Formating Module (#71)

* chore: Harmonize the specs field of storages

* docs: Use template correctly on configuration example

* feat: Implement the last big brick of Webhooked: Formating module

* fix: Prevent segfault when no global is defined

* test: Implement test suite for formatter

* chore(github/actions): Add golang 1.18 to test suite

* clean: Remove template file from source code and move it to documentation

* chore: Use same naming for all url fields

* test: Update test file to follow the new Payload variable

* fix: Use Payload instead of RequestBody

* test: Add tests for configuration loader

* chore: Fix misspell

* chore(githun/actions): Add Codecov to pipeline

* docs: Add coverage badge

* docs: Add badges

* docs: Update pictures on README.md

* chore: Fix misspelling about formatting

* docs: Add formatting feature to the README

* docs: Add Formatting wiki Link to README

* docs: Update links in README

* chore: Rename formatting package

* docs: Add formatting docs

* docs: Update kubernetes example for 0.6

.github/workflows/tests.yaml

@@ -16,7 +16,7 @@ jobs:
     strategy:
       fail-fast: true
       matrix:
-        goVersion: [ '1.16', '1.17' ]
+        goVersion: [ '1.16', '1.17', '1.18' ]
     steps:
     - name: Checkout project
       uses: actions/checkout@v3
@@ -66,6 +66,7 @@ jobs:
             echo "Failed"
             exit 1
         fi
+    - uses: codecov/codecov-action@v2
     - name: Run Go Build
       run: |
         go build -o /tmp/applications-test-units

README.md

@@ -2,6 +2,8 @@
 
 <p align="center"><a href="https://github.com/42Atomys/webhooked/actions/workflows/release.yaml"><img src="https://github.com/42Atomys/webhooked/actions/workflows/release.yaml/badge.svg" alt="Release 🎉"></a>
 <a href="https://goreportcard.com/report/atomys.codes/webhooked"><img src="https://goreportcard.com/badge/atomys.codes/webhooked" /></a>
+<a href="https://codeclimate.com/github/42Atomys/webhooked"><img alt="Code Climate maintainability" src="https://img.shields.io/codeclimate/maintainability/42Atomys/webhooked"></a>
+<a href="https://codecov.io/gh/42Atomys/webhooked"><img alt="Codecov" src="https://img.shields.io/codecov/c/gh/42Atomys/webhooked?token=NSUZMDT9M9"></a>
 <img src="https://img.shields.io/github/v/release/42atomys/webhooked?label=last%20release" alt="GitHub release (latest by date)">
 <img src="https://img.shields.io/github/contributors/42Atomys/webhooked?color=blueviolet" alt="GitHub contributors">
 <img src="https://img.shields.io/github/stars/42atomys/webhooked?color=blueviolet" alt="GitHub Repo stars">
@@ -21,7 +23,7 @@ This is exactly what `Webhooked` does !
 
 ## Roadmap
 
-I am actively working on this project to release a stable version by the **end of March 2022**
+I am actively working on this project in order to release a stable version for **mid-2022**
 
 ![Roadmap](/.github/profile/roadmap.png)
 
@@ -60,6 +62,28 @@ specs:
         values: ['foo', 'bar']
         valueFrom:
           envRef: SECRET_TOKEN
+
+  # Formatting allows you to apply a custom format to the payload received
+  # before send it to the storage. You can use built-in helper function to
+  # format it as you want. (Optional)
+  #
+  # Per default the format applied is: "{{ .Payload }}"
+  # 
+  # THIS IS AN ADVANCED FEATURE :
+  # Be careful when using this feature, the slightest error in format can
+  # result in DEFFINITIVE loss of the collected data. Make sure your template is
+  # correct before applying it in production.
+  formatting:
+    templateString: |
+      {
+        "config": "{{ toJson .Config }}",
+        "metadata": {
+          "specName": "{{ .Spec.Name }}",
+          "deliveryID": "{{ .Request.Header | getHeader "X-Delivery" | default "unknown" }}"
+        },
+        "payload": {{ .Payload }}
+      }
+
   # Storage allows you to list where you want to store the raw payloads
   # received by webhooked. You can add an unlimited number of storages, webhooked
   # will store in **ALL** the listed storages
@@ -68,6 +92,9 @@ specs:
   # on the `example-webhook` Redis Key on the Database 0
   storage:
   - type: redis
+    # You can apply a specific formatting per storage (Optional)
+    formatting: {}
+    # Storage specification
     specs:
       host: redis.default.svc.cluster.local
       port: 6379
@@ -77,7 +104,9 @@ specs:
 
 More informations about security pipeline available on wiki : [Configuration/Security](https://github.com/42Atomys/webhooked/wiki/Security)
 
-More informations about storages available on wiki : [Configuration/Storages](https://github.com/42Atomys/webhooked/wiki/Configuration-Storages)
+More informations about storages available on wiki : [Configuration/Storages](https://github.com/42Atomys/webhooked/wiki/Storages)
+
+More informations about formatting available on wiki : [Configuration/Formatting](https://github.com/42Atomys/webhooked/wiki/Formatting)
 
 ### Step 2 : Launch it 🚀
 ### With Kubernetes

config/webhooks.example.yml

@@ -12,7 +12,7 @@ specs:
   - compare:
       inputs:
       - name: first
-        value: '{{ Outputs.header.value }}'
+        value: '{{ .Outputs.header.value }}'
       - name: second
         valueFrom:
           envRef: SECRET_TOKEN

examples/kubernetes/deployment.yaml

@@ -22,7 +22,7 @@ data:
       - compare:
           inputs:
           - name: first
-            value: '{{ Outputs.header.value }}'
+            value: '{{ .Outputs.header.value }}'
           - name: second
             valueFrom:
               envRef: SECRET_TOKEN
@@ -40,7 +40,7 @@ metadata:
   name: webhooked
   labels:
     app.kubernetes.io/name: webhooked
-    app.kubernetes.io/version: '0.4'
+    app.kubernetes.io/version: '0.6'
 spec:
   selector:
     matchLabels:
@@ -52,7 +52,7 @@ spec:
     spec:
       containers:
       - name: webhooked
-        image: atomys/webhooked:0.4
+        image: atomys/webhooked:0.6
         imagePullPolicy: IfNotPresent
         env:
           - name: SECRET_TOKEN

go.mod

@@ -1,6 +1,6 @@
 module atomys.codes/webhooked
 
-go 1.17
+go 1.18
 
 require (
 	github.com/go-redis/redis/v8 v8.11.5

internal/config/configuration.go

@@ -1,8 +1,11 @@
 package config
 
 import (
+	"bytes"
 	"errors"
 	"fmt"
+	"io"
+	"os"
 
 	"github.com/rs/zerolog/log"
 	"github.com/spf13/viper"
@@ -15,6 +18,9 @@ var (
 	currentConfig = &Configuration{}
 	// ErrSpecNotFound is returned when the spec is not found
 	ErrSpecNotFound = errors.New("spec not found")
+	// defaultTemplate is the default template for the payload
+	// when no template is defined
+	defaultTemplate = `{{ .Payload }}`
 )
 
 // Load loads the configuration from the viper configuration file
@@ -30,6 +36,10 @@ func Load() error {
 			return err
 		}
 
+		if spec.Formatting, err = loadTemplate(spec.Formatting, nil); err != nil {
+			return fmt.Errorf("configured storage for %s received an error: %s", spec.Name, err.Error())
+		}
+
 		if err = loadStorage(spec); err != nil {
 			return fmt.Errorf("configured storage for %s received an error: %s", spec.Name, err.Error())
 		}
@@ -95,12 +105,62 @@ func loadStorage(spec *WebhookSpec) (err error) {
 		if err != nil {
 			return fmt.Errorf("storage %s cannot be loaded properly: %s", s.Type, err.Error())
 		}
+
+		if s.Formatting, err = loadTemplate(s.Formatting, spec.Formatting); err != nil {
+			return fmt.Errorf("storage %s cannot be loaded properly: %s", s.Type, err.Error())
+		}
 	}
 
 	log.Debug().Msgf("%d storages loaded for spec %s", len(spec.Storage), spec.Name)
 	return
 }
 
+// loadTemplate loads the template for the given `spec`. When no spec is defined
+// we try to load the template from the parentSpec and fallback to the default
+// template if parentSpec is not given.
+func loadTemplate(spec, parentSpec *FormattingSpec) (*FormattingSpec, error) {
+	if spec == nil {
+		spec = &FormattingSpec{}
+	}
+
+	if spec.TemplateString != "" {
+		spec.Template = spec.TemplateString
+		return spec, nil
+	}
+
+	if spec.TemplatePath != "" {
+		file, err := os.OpenFile(spec.TemplatePath, os.O_RDONLY, 0666)
+		if err != nil {
+			return spec, err
+		}
+		defer file.Close()
+
+		var buffer bytes.Buffer
+		_, err = io.Copy(&buffer, file)
+		if err != nil {
+			return spec, err
+		}
+
+		spec.Template = buffer.String()
+		return spec, nil
+	}
+
+	if parentSpec != nil {
+		if parentSpec.Template == "" {
+			var err error
+			parentSpec, err = loadTemplate(parentSpec, nil)
+			if err != nil {
+				return spec, err
+			}
+		}
+		spec.Template = parentSpec.Template
+	} else {
+		spec.Template = defaultTemplate
+	}
+
+	return spec, nil
+}
+
 // Current returns the aftual configuration
 func Current() *Configuration {
 	return currentConfig

internal/config/configuration_test.go

@@ -170,11 +170,11 @@ func TestLoadSecurityFactory(t *testing.T) {
 	for _, test := range tests {
 		err := loadSecurityFactory(test.input)
 		if test.wantErr {
-			assert.Error(err)
+			assert.Error(err, test.name)
 		} else {
-			assert.NoError(err)
+			assert.NoError(err, test.name)
 		}
-		assert.Equal(test.input.SecurityPipeline.FactoryCount(), test.wantLen)
+		assert.Equal(test.input.SecurityPipeline.FactoryCount(), test.wantLen, test.name)
 	}
 }
 
@@ -183,15 +183,13 @@ func TestLoadStorage(t *testing.T) {
 
 	tests := []struct {
 		name        string
-		storageName string
 		input       *WebhookSpec
 		wantErr     bool
 		wantStorage bool
 	}{
-		{"no spec", "", &WebhookSpec{Name: "test"}, false, false},
+		{"no spec", &WebhookSpec{Name: "test"}, false, false},
 		{
 			"full valid storage",
-			"connection invalid must return an error",
 			&WebhookSpec{
 				Name: "test",
 				Storage: []*StorageSpec{
@@ -201,6 +199,7 @@ func TestLoadStorage(t *testing.T) {
 							"host": "localhost",
 							"port": 0,
 						},
+						Formatting: &FormattingSpec{TemplateString: "null"},
 					},
 				},
 			},
@@ -209,7 +208,6 @@ func TestLoadStorage(t *testing.T) {
 		},
 		{
 			"empty storage configuration",
-			"",
 			&WebhookSpec{
 				Name:    "test",
 				Storage: []*StorageSpec{},
@@ -219,7 +217,6 @@ func TestLoadStorage(t *testing.T) {
 		},
 		{
 			"invalid storage name in configuration",
-			"",
 			&WebhookSpec{
 				Name: "test",
 				Storage: []*StorageSpec{
@@ -234,14 +231,92 @@ func TestLoadStorage(t *testing.T) {
 	for _, test := range tests {
 		err := loadStorage(test.input)
 		if test.wantErr {
-			assert.Error(err)
+			assert.Error(err, test.name)
 		} else {
-			assert.NoError(err)
+			assert.NoError(err, test.name)
 		}
 
 		if test.wantStorage && assert.Len(test.input.Storage, 1, "no storage is loaded for test %s", test.name) {
 			s := test.input.Storage[0]
-			assert.NotNil(s)
+			assert.NotNil(s, test.name)
+		}
+	}
+}
+
+func Test_loadTemplate(t *testing.T) {
+	tests := []struct {
+		name         string
+		input        *FormattingSpec
+		parentSpec   *FormattingSpec
+		wantErr      bool
+		wantTemplate string
+	}{
+		{
+			"no template",
+			nil,
+			nil,
+			false,
+			defaultTemplate,
+		},
+		{
+			"template string",
+			&FormattingSpec{TemplateString: "{{ .Request.Method }}"},
+			nil,
+			false,
+			"{{ .Request.Method }}",
+		},
+		{
+			"template file",
+			&FormattingSpec{TemplatePath: "../../tests/simple_template.tpl"},
+			nil,
+			false,
+			"{{ .Request.Method }}",
+		},
+		{
+			"template file with template string",
+			&FormattingSpec{TemplatePath: "../../tests/simple_template.tpl", TemplateString: "{{ .Request.Path }}"},
+			nil,
+			false,
+			"{{ .Request.Path }}",
+		},
+		{
+			"no template with not loaded parent",
+			nil,
+			&FormattingSpec{TemplateString: "{{ .Request.Method }}"},
+			false,
+			"{{ .Request.Method }}",
+		},
+		{
+			"no template with loaded parent",
+			nil,
+			&FormattingSpec{Template: "{{ .Request.Method }}", TemplateString: "{{ .Request.Path }}"},
+			false,
+			"{{ .Request.Method }}",
+		},
+		{
+			"no template with unloaded parent and error",
+			nil,
+			&FormattingSpec{TemplatePath: "//invalid//path//"},
+			true,
+			"",
+		},
+		{
+			"template file not found",
+			&FormattingSpec{TemplatePath: "//invalid//path//"},
+			nil,
+			true,
+			"",
+		},
+	}
+
+	for _, test := range tests {
+		tmpl, err := loadTemplate(test.input, test.parentSpec)
+		if test.wantErr {
+			assert.Error(t, err, test.name)
+		} else {
+			assert.NoError(t, err, test.name)
 		}
+		assert.NotNil(t, tmpl, test.name)
+		assert.Equal(t, test.wantTemplate, tmpl.Template, test.name)
 	}
 }

internal/config/specification.go

@@ -4,3 +4,13 @@ package config
 func (s WebhookSpec) HasSecurity() bool {
 	return s.SecurityPipeline != nil && s.SecurityPipeline.HasFactories()
 }
+
+// HasGlobalFormatting returns true if the spec has a global formatting
+func (s WebhookSpec) HasGlobalFormatting() bool {
+	return s.Formatting != nil && (s.Formatting.TemplatePath != "" || s.Formatting.TemplateString != "")
+}
+
+// HasFormatting returns true if the storage spec has a formatting
+func (s StorageSpec) HasFormatting() bool {
+	return s.Formatting != nil && (s.Formatting.TemplatePath != "" || s.Formatting.TemplateString != "")
+}