From 31a60a1ba69eca526fd1605e247f6db583c7ecb1 Mon Sep 17 00:00:00 2001 From: Romain Beuque <556072+rbeuque74@users.noreply.github.com> Date: Tue, 21 Dec 2021 17:15:02 +0000 Subject: [PATCH 1/2] feat: api: add a route to resolve a templating expression When debugging a failed task, we often need to resolve templated value that we struggle to inspect. POST /resolution/:id/templating will execute a templating expression given as input, and returns the resolved expression as output. This route can only be used by admins. Signed-off-by: Romain Beuque <556072+rbeuque74@users.noreply.github.com> --- api/api_test.go | 99 ++++++++++++++++++++++----------------- api/handler/resolution.go | 78 ++++++++++++++++++++++++++++++ api/server.go | 8 ++++ engine/engine.go | 56 ++-------------------- utask.go | 74 +++++++++++++++++++++++++++++ 5 files changed, 221 insertions(+), 94 deletions(-) diff --git a/api/api_test.go b/api/api_test.go index 9ae50c35..8db9707e 100644 --- a/api/api_test.go +++ b/api/api_test.go @@ -558,40 +558,6 @@ func waitChecker(dur time.Duration) iffy.Checker { } } -func templatesWithInvalidInputs() []tasktemplate.TaskTemplate { - var tt []tasktemplate.TaskTemplate - for _, inp := range []input.Input{ - { - Name: "input-with-redundant-regex", - LegalValues: []interface{}{"a", "b", "c"}, - Regex: strPtr("^d.+$"), - }, - { - Name: "input-with-bad-regex", - Regex: strPtr("^^[d.+$"), - }, - { - Name: "input-with-bad-type", - Type: "bad-type", - }, - { - Name: "input-with-bad-legal-values", - Type: "number", - LegalValues: []interface{}{"a", "b", "c"}, - }, - } { - tt = append(tt, tasktemplate.TaskTemplate{ - Name: "invalid-template", - Description: "Invalid template", - TitleFormat: "Invalid template", - Inputs: []input.Input{ - inp, - }, - }) - } - return tt -} - func templateWithPasswordInput() tasktemplate.TaskTemplate { return tasktemplate.TaskTemplate{ Name: "input-password", @@ -706,6 +672,63 @@ func dummyTemplate() tasktemplate.TaskTemplate { } } +func clientErrorTemplate() tasktemplate.TaskTemplate { + return tasktemplate.TaskTemplate{ + Name: "client-error-template", + Description: "does nothing", + TitleFormat: "this task does nothing at all", + Inputs: []input.Input{ + { + Name: "id", + }, + }, + Variables: []values.Variable{ + { + Name: "var1", + Value: "hello id {{.input.id }} for {{ .step.step1.output.foo }} and {{ .step.this.state | default \"BROKEN_TEMPLATING\" }}", + }, + { + Name: "var2", + Expression: "var a = 3+2; a;", + }, + }, + Steps: map[string]*step.Step{ + "step1": { + Action: executor.Executor{ + Type: "echo", + Configuration: json.RawMessage(`{ + "output": {"foo":"bar"} + }`), + }, + }, + "step2": { + Action: executor.Executor{ + Type: "echo", + Configuration: json.RawMessage(`{ + "output": {"foo":"bar"} + }`), + }, + Dependencies: []string{"step1"}, + Conditions: []*condition.Condition{ + { + If: []*condition.Assert{ + { + Expected: "1", + Value: "1", + Operator: "EQ", + }, + }, + Then: map[string]string{ + "this": "CLIENT_ERROR", + }, + Type: "skip", + }, + }, + }, + }, + } +} + func blockedHidden(name string, blocked, hidden bool) tasktemplate.TaskTemplate { return tasktemplate.TaskTemplate{ Name: name, @@ -759,12 +782,4 @@ func expectStringPresent(value string) iffy.Checker { } } -func marshalJSON(t *testing.T, i interface{}) string { - jsonBytes, err := json.Marshal(i) - if err != nil { - t.Fatal(err) - } - return string(jsonBytes) -} - func strPtr(s string) *string { return &s } diff --git a/api/handler/resolution.go b/api/handler/resolution.go index f94c09d9..d651d51a 100644 --- a/api/handler/resolution.go +++ b/api/handler/resolution.go @@ -9,6 +9,7 @@ import ( "github.com/loopfz/gadgeto/zesty" "github.com/sirupsen/logrus" + "github.com/ovh/configstore" "github.com/ovh/utask" "github.com/ovh/utask/engine" "github.com/ovh/utask/engine/step" @@ -932,3 +933,80 @@ func UpdateResolutionStepState(c *gin.Context, in *updateResolutionStepStateIn) return nil } + +type resolveTemplatingResolutionIn struct { + PublicID string `path:"id" validate:"required"` + TemplatingExpression string `json:"templating_expression" validate:"required"` + StepName string `json:"step_name"` +} + +// ResolveTemplatingResolutionOut is the output of the HTTP route +// for ResolveTemplatingResolution +type ResolveTemplatingResolutionOut struct { + Result string `json:"result"` + Error *string `json:"error,omitempty"` +} + +// ResolveTemplatingResolution will use µtask templating engine for a given resolution +// to validate a given template. Action is restricted to admin only, as it could be used +// to exfiltrate configuration. +func ResolveTemplatingResolution(c *gin.Context, in *resolveTemplatingResolutionIn) (*ResolveTemplatingResolutionOut, error) { + metadata.AddActionMetadata(c, metadata.ResolutionID, in.PublicID) + + dbp, err := zesty.NewDBProvider(utask.DBName) + if err != nil { + return nil, err + } + + r, err := resolution.LoadFromPublicID(dbp, in.PublicID) + if err != nil { + return nil, err + } + + t, err := task.LoadFromID(dbp, r.TaskID) + if err != nil { + return nil, err + } + + metadata.AddActionMetadata(c, metadata.TaskID, t.PublicID) + + tt, err := tasktemplate.LoadFromID(dbp, t.TemplateID) + if err != nil { + return nil, err + } + + metadata.AddActionMetadata(c, metadata.TemplateName, tt.Name) + + admin := auth.IsAdmin(c) == nil + + if !admin { + return nil, errors.Forbiddenf("You are not allowed to resolve resolution variables") + } + + metadata.SetSUDO(c) + + // provide the resolution with values + t.ExportTaskInfos(r.Values) + r.Values.SetInput(t.Input) + r.Values.SetResolverInput(r.ResolverInput) + r.Values.SetVariables(tt.Variables) + + config, err := utask.GetTemplatingConfig(configstore.DefaultStore) + if err != nil { + return nil, err + } + + r.Values.SetConfig(config) + + output, err := r.Values.Apply(in.TemplatingExpression, nil, in.StepName) + if err != nil { + errStr := err.Error() + return &ResolveTemplatingResolutionOut{ + Error: &errStr, + }, nil + } + + return &ResolveTemplatingResolutionOut{ + Result: string(output), + }, nil +} diff --git a/api/server.go b/api/server.go index 414eda3b..e7f8c6af 100644 --- a/api/server.go +++ b/api/server.go @@ -401,6 +401,14 @@ func (s *Server) build(ctx context.Context) { }, maintenanceMode, tonic.Handler(handler.UpdateResolutionStepState, 204)) + resolutionRoutes.POST("/resolution/:id/templating", + []fizz.OperationOption{ + fizz.ID("ResolveTemplatingResolution"), + fizz.Summary("Resolve templating of a resolution"), + fizz.Description("Resolve the templating of a string, within a task resolution. Admin users only."), + }, + maintenanceMode, + tonic.Handler(handler.ResolveTemplatingResolution, 200)) // resolutionRoutes.POST("/resolution/:id/rollback", // []fizz.OperationOption{ diff --git a/engine/engine.go b/engine/engine.go index c0991373..a33e9ffb 100644 --- a/engine/engine.go +++ b/engine/engine.go @@ -3,14 +3,12 @@ package engine import ( "bytes" "context" - "encoding/json" "fmt" "strings" "sync" "time" "github.com/cenkalti/backoff" - "github.com/ghodss/yaml" expbk "github.com/jpillora/backoff" "github.com/juju/errors" "github.com/loopfz/gadgeto/zesty" @@ -64,38 +62,13 @@ func Init(ctx context.Context, wg *sync.WaitGroup, store *configstore.Store) err if err != nil { return err } - // get all configuration items - itemList, err := store.GetItemList() - if err != nil { - return err - } - // Squash to ensure that secrets with lower priority - // are dismissed. - itemList = configstore.Filter().Squash().Apply(itemList) - // drop those that shouldnt be available for task execution - // (don't let DB credentials leak, for instance...) - config, err := filteredConfig(itemList, cfg.ConcealedSecrets...) - if err != nil { + var engineCfg map[string]interface{} + if engineCfg, err = utask.GetTemplatingConfig(store); err != nil { return err } - // attempt to deserialize json formatted config items - // -> make it easier to access internal nodes/values when templating - eng.config = make(map[string]interface{}) - for k, v := range config { - var i interface{} - if v != nil { - err := yaml.Unmarshal([]byte(*v), &i, func(dec *json.Decoder) *json.Decoder { - dec.UseNumber() - return dec - }) - if err != nil { - eng.config[k] = v - } else { - eng.config[k] = i - } - } - } + + eng.config = engineCfg // channels for handling graceful shutdown shutdownCtx = ctx @@ -150,27 +123,6 @@ func Init(ctx context.Context, wg *sync.WaitGroup, store *configstore.Store) err return nil } -// filteredConfig takes a configstore item list, drops some items by key -// then reduces the result into a map of key->values -func filteredConfig(list *configstore.ItemList, dropAlias ...string) (map[string]*string, error) { - cfg := make(map[string]*string) - for _, i := range list.Items { - if !utils.ListContainsString(dropAlias, i.Key()) { - // assume only one value per alias - if _, ok := cfg[i.Key()]; !ok { - v, err := i.Value() - if err != nil { - return nil, err - } - if len(v) > 0 { - cfg[i.Key()] = &v - } - } - } - } - return cfg, nil -} - // GetEngine returns the singleton instance of Engine func GetEngine() Engine { return eng diff --git a/utask.go b/utask.go index c786d747..4ce2921d 100644 --- a/utask.go +++ b/utask.go @@ -7,6 +7,7 @@ import ( "fmt" "time" + "github.com/ghodss/yaml" "golang.org/x/sync/semaphore" "github.com/ovh/configstore" @@ -386,3 +387,76 @@ func Config(store *configstore.Store) (*Cfg, error) { return global, nil } + +// GetTemplatingConfig returns the µTask configuration without sensible piece of configuration +// such as encryption key, ... to be used by templating functions. +func GetTemplatingConfig(store *configstore.Store) (map[string]interface{}, error) { + cfg, err := Config(store) + if err != nil { + return nil, err + } + // get all configuration items + itemList, err := store.GetItemList() + if err != nil { + return nil, err + } + // Squash to ensure that secrets with lower priority + // are dismissed. + itemList = configstore.Filter().Squash().Apply(itemList) + + // drop those that shouldnt be available for task execution + // (don't let DB credentials leak, for instance...) + config, err := filteredConfig(itemList, cfg.ConcealedSecrets...) + if err != nil { + return nil, err + } + // attempt to deserialize json formatted config items + // -> make it easier to access internal nodes/values when templating + c := make(map[string]interface{}) + for k, v := range config { + var i interface{} + if v != nil { + err := yaml.Unmarshal([]byte(*v), &i, func(dec *json.Decoder) *json.Decoder { + dec.UseNumber() + return dec + }) + if err != nil { + c[k] = v + } else { + c[k] = i + } + } + } + return c, nil +} + +// filteredConfig takes a configstore item list, drops some items by key +// then reduces the result into a map of key->values +func filteredConfig(list *configstore.ItemList, dropAlias ...string) (map[string]*string, error) { + cfg := make(map[string]*string) + for _, i := range list.Items { + if !listContainsString(dropAlias, i.Key()) { + // assume only one value per alias + if _, ok := cfg[i.Key()]; !ok { + v, err := i.Value() + if err != nil { + return nil, err + } + if len(v) > 0 { + cfg[i.Key()] = &v + } + } + } + } + return cfg, nil +} + +// listContainsString asserts that a string slice contains a given string +func listContainsString(list []string, item string) bool { + for _, i := range list { + if i == item { + return true + } + } + return false +} From 7c898db9a4716ae62ff4243e3fcea7629a10c42b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20B=C3=A9trancourt?= Date: Fri, 6 May 2022 10:22:42 +0000 Subject: [PATCH 2/2] feat: ui: add template expression widget MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Thomas Bétrancourt --- .../resolution-expression.component.ts | 71 +++++++++++++++++++ .../resolution-expression.html | 25 +++++++ .../resolution-expression.sass | 13 ++++ .../src/lib/@models/resolution.model.ts | 7 +- .../utask-lib/src/lib/@routes/task/task.html | 11 +++ .../src/lib/@services/api.service.ts | 11 +++ .../utask-lib/src/lib/utask-lib.module.ts | 2 + 7 files changed, 139 insertions(+), 1 deletion(-) create mode 100644 ui/dashboard/projects/utask-lib/src/lib/@components/resolution-expression/resolution-expression.component.ts create mode 100644 ui/dashboard/projects/utask-lib/src/lib/@components/resolution-expression/resolution-expression.html create mode 100644 ui/dashboard/projects/utask-lib/src/lib/@components/resolution-expression/resolution-expression.sass diff --git a/ui/dashboard/projects/utask-lib/src/lib/@components/resolution-expression/resolution-expression.component.ts b/ui/dashboard/projects/utask-lib/src/lib/@components/resolution-expression/resolution-expression.component.ts new file mode 100644 index 00000000..62c024c9 --- /dev/null +++ b/ui/dashboard/projects/utask-lib/src/lib/@components/resolution-expression/resolution-expression.component.ts @@ -0,0 +1,71 @@ +import { ChangeDetectionStrategy, Component, Input } from "@angular/core"; +import { FormBuilder, FormGroup, Validators } from "@angular/forms"; +import { BehaviorSubject } from "rxjs"; +import Resolution from "../../@models/resolution.model"; +import { ApiService } from "../../@services/api.service"; + +@Component({ + selector: "lib-utask-resoution-expression", + templateUrl: "./resolution-expression.html", + styleUrls: ["./resolution-expression.sass"], + changeDetection: ChangeDetectionStrategy.OnPush, +}) +export class ResolutionExpressionComponent { + private _resolution: Resolution; + private _steps$ = new BehaviorSubject([]); + private _result$ = new BehaviorSubject(null); + private _error$ = new BehaviorSubject(null); + + readonly formGroup: FormGroup; + + readonly steps$ = this._steps$.asObservable(); + readonly result$ = this._result$.asObservable(); + readonly error$ = this._error$.asObservable(); + + @Input("resolution") set resolution(r: Resolution) { + if ((this._resolution = r)) { + this._steps$.next(Object.keys(r.steps)); + } else { + this._steps$.next([]); + } + } + + get resolution(): Resolution { + return this._resolution; + } + + constructor(private _api: ApiService, _builder: FormBuilder) { + this.formGroup = _builder.group({ + step: ["", [Validators.required]], + expression: ["", [Validators.required]], + }); + } + + reset(): void { + this.formGroup.reset(); + this._result$.next(null); + this._error$.next(null); + } + + submit(): void { + const { step, expression } = this.formGroup.value; + + this._api.resolution + .templating(this._resolution, step, expression) + .subscribe( + (result) => { + if (result.error) { + this._result$.next(null); + this._error$.next(result.error); + } else { + this._result$.next(result.result); + this._error$.next(null); + } + }, + (e) => { + this._result$.next(null); + this._error$.next(e.error.error); + } + ); + } +} diff --git a/ui/dashboard/projects/utask-lib/src/lib/@components/resolution-expression/resolution-expression.html b/ui/dashboard/projects/utask-lib/src/lib/@components/resolution-expression/resolution-expression.html new file mode 100644 index 00000000..d43e3878 --- /dev/null +++ b/ui/dashboard/projects/utask-lib/src/lib/@components/resolution-expression/resolution-expression.html @@ -0,0 +1,25 @@ +
+ + Step + + + + + + + + Template expression + + + + +
+ + +
+ + +
+
+ 
{{ result }}
+
diff --git a/ui/dashboard/projects/utask-lib/src/lib/@components/resolution-expression/resolution-expression.sass b/ui/dashboard/projects/utask-lib/src/lib/@components/resolution-expression/resolution-expression.sass new file mode 100644 index 00000000..e200cc3e --- /dev/null +++ b/ui/dashboard/projects/utask-lib/src/lib/@components/resolution-expression/resolution-expression.sass @@ -0,0 +1,13 @@ +.buttons + margin-top: 1em + + button + margin-right: 0.5em + +.result + margin-top: 1em + border: 1px solid #d9d9d9 + padding: 1em + + pre + margin: 0 diff --git a/ui/dashboard/projects/utask-lib/src/lib/@models/resolution.model.ts b/ui/dashboard/projects/utask-lib/src/lib/@models/resolution.model.ts index b6141999..aed64136 100644 --- a/ui/dashboard/projects/utask-lib/src/lib/@models/resolution.model.ts +++ b/ui/dashboard/projects/utask-lib/src/lib/@models/resolution.model.ts @@ -14,4 +14,9 @@ export default class Resolution { task_id: string; task_title: string; steps: { [key: string]: Step }; -} \ No newline at end of file +} + +export class TemplateExpression { + result: string; + error?: string; +} diff --git a/ui/dashboard/projects/utask-lib/src/lib/@routes/task/task.html b/ui/dashboard/projects/utask-lib/src/lib/@routes/task/task.html index 64a3dd80..7f3f8636 100644 --- a/ui/dashboard/projects/utask-lib/src/lib/@routes/task/task.html +++ b/ui/dashboard/projects/utask-lib/src/lib/@routes/task/task.html @@ -272,6 +272,17 @@ + + +
+ Template expression +   +
+
+ +
+ diff --git a/ui/dashboard/projects/utask-lib/src/lib/@services/api.service.ts b/ui/dashboard/projects/utask-lib/src/lib/@services/api.service.ts index c1dc5a10..4ecbabef 100644 --- a/ui/dashboard/projects/utask-lib/src/lib/@services/api.service.ts +++ b/ui/dashboard/projects/utask-lib/src/lib/@services/api.service.ts @@ -5,6 +5,7 @@ import { Observable } from 'rxjs'; import { HttpClient, HttpResponse } from '@angular/common/http'; import Meta from '../@models/meta.model'; import Template from '../@models/template.model'; +import Resolution, { TemplateExpression } from '../@models/resolution.model'; export class ParamsListTasks { page_size?: number; @@ -314,6 +315,16 @@ export class ApiServiceResolution { resolution ); } + + templating(resolution: Resolution, step: string, expression: string) { + return this.http.post( + `${this.base}resolution/${resolution.id}/templating`, + { + step_name: step, + templating_expression: expression, + } + ); + } } @Injectable({ diff --git a/ui/dashboard/projects/utask-lib/src/lib/utask-lib.module.ts b/ui/dashboard/projects/utask-lib/src/lib/utask-lib.module.ts index f7def510..0dfa4c73 100644 --- a/ui/dashboard/projects/utask-lib/src/lib/utask-lib.module.ts +++ b/ui/dashboard/projects/utask-lib/src/lib/utask-lib.module.ts @@ -15,6 +15,7 @@ import { InputsFormComponent } from './@components/inputs-form/inputs-form.compo import { InputTagsComponent } from './@components/input-tags/input-tags.component'; import { InputEditorComponent } from './@components/input-editor/input-editor.component'; import { LoaderComponent } from './@components/loader/loader.component'; +import { ResolutionExpressionComponent } from './@components/resolution-expression/resolution-expression.component'; import { MetaResolve } from './@resolves/meta.resolve'; import { ModalApiYamlComponent } from './@modals/modal-api-yaml/modal-api-yaml.component'; import { ModalEditResolutionStepStateComponent } from './@modals/modal-edit-resolution-step-state/modal-edit-resolution-step-state.component'; @@ -77,6 +78,7 @@ const components: any[] = [ ModalEditResolutionStepStateComponent, ModalApiYamlEditComponent, NzModalContentWithErrorComponent, + ResolutionExpressionComponent, StepNodeComponent, StepsListComponent, StepsViewerComponent,