Improve input/output transformation configuration (#504)

Signed-off-by: Tyler Ohlsen <[email protected]>

common/constants.ts

@@ -4,6 +4,7 @@
  */
 
 import {
+  InputMapEntry,
   MapEntry,
   PromptPreset,
   QueryPreset,
@@ -468,6 +469,13 @@ export enum TRANSFORM_CONTEXT {
   INPUT = 'input',
   OUTPUT = 'output',
 }
+export enum TRANSFORM_TYPE {
+  STRING = 'String',
+  FIELD = 'Field',
+  EXPRESSION = 'Expression',
+  TEMPLATE = 'Template',
+}
+export const NO_TRANSFORMATION = 'No transformation';
 export const START_FROM_SCRATCH_WORKFLOW_NAME = 'Start From Scratch';
 export const DEFAULT_NEW_WORKFLOW_NAME = 'new_workflow';
 export const DEFAULT_NEW_WORKFLOW_DESCRIPTION = 'My new workflow';
@@ -490,13 +498,21 @@ export enum SORT_ORDER {
 export const MAX_DOCS = 1000;
 export const MAX_STRING_LENGTH = 100;
 export const MAX_JSON_STRING_LENGTH = 10000;
+export const MAX_TEMPLATE_STRING_LENGTH = 10000;
 export const MAX_WORKFLOW_NAME_TO_DISPLAY = 40;
 export const WORKFLOW_NAME_REGEXP = RegExp('^[a-zA-Z0-9_-]*$');
 export const EMPTY_MAP_ENTRY = { key: '', value: '' } as MapEntry;
+export const EMPTY_INPUT_MAP_ENTRY = {
+  key: '',
+  value: {
+    transformType: '' as TRANSFORM_TYPE,
+    value: '',
+  },
+} as InputMapEntry;
+export const EMPTY_OUTPUT_MAP_ENTRY = EMPTY_INPUT_MAP_ENTRY;
 export const MODEL_OUTPUT_SCHEMA_NESTED_PATH =
   'output.properties.inference_results.items.properties.output.items.properties.dataAsMap.properties';
 export const MODEL_OUTPUT_SCHEMA_FULL_PATH = 'output.properties';
-export const PROMPT_FIELD = 'prompt'; // TODO: likely expand to support a pattern and/or multiple (e.g., "prompt", "prompt_template", etc.)
 export enum CONFIG_STEP {
   INGEST = 'Ingestion pipeline',
   SEARCH = 'Search pipeline',

common/interfaces.ts

@@ -9,7 +9,7 @@ import { ObjectSchema } from 'yup';
 import {
   COMPONENT_CLASS,
   PROCESSOR_TYPE,
-  PROMPT_FIELD,
+  TRANSFORM_TYPE,
   WORKFLOW_TYPE,
 } from './constants';
 
@@ -35,7 +35,9 @@ export type ConfigFieldType =
   | 'map'
   | 'mapArray'
   | 'boolean'
-  | 'number';
+  | 'number'
+  | 'inputMapArray'
+  | 'outputMapArray';
 
 export type ConfigFieldValue = string | {};
 
@@ -100,6 +102,31 @@ export type MapFormValue = MapEntry[];
 
 export type MapArrayFormValue = MapFormValue[];
 
+export type ExpressionVar = {
+  name: string;
+  transform: string;
+};
+
+export type Transform = {
+  transformType: TRANSFORM_TYPE;
+  value?: string;
+  // Templates may persist their own set of nested transforms
+  // to be dynamically injected into the template
+  nestedVars?: ExpressionVar[];
+};
+
+export type InputMapEntry = {
+  key: string;
+  value: Transform;
+};
+export type OutputMapEntry = InputMapEntry;
+
+export type InputMapFormValue = InputMapEntry[];
+export type OutputMapFormValue = OutputMapEntry[];
+
+export type InputMapArrayFormValue = InputMapFormValue[];
+export type OutputMapArrayFormValue = OutputMapFormValue[];
+
 export type WorkflowFormValues = {
   ingest: FormikValues;
   search: FormikValues;
@@ -110,31 +137,48 @@ export type WorkflowSchemaObj = {
 };
 export type WorkflowSchema = ObjectSchema<WorkflowSchemaObj>;
 
-// Form / schema interfaces for the ingest docs sub-form
+/**
+ ********** MODAL SUB-FORM TYPES/INTERFACES **********
+ We persist sub-forms in the form modals s.t. data is only
+ saved back to the parent form if the user explicitly saves.
+
+ We define the structure of the forms here.
+ */
+
+// Ingest docs modal
 export type IngestDocsFormValues = {
   docs: FormikValues;
 };
-export type IngestDocsSchema = WorkflowSchema;
 
-// Form / schema interfaces for the request query sub-form
+// Search request modal
 export type RequestFormValues = {
   request: ConfigFieldValue;
 };
-export type RequestSchema = WorkflowSchema;
 
-// Form / schema interfaces for the input transform sub-form
+// Input transform modal
 export type InputTransformFormValues = {
-  input_map: MapArrayFormValue;
+  input_map: InputMapArrayFormValue;
   one_to_one: ConfigFieldValue;
 };
-export type InputTransformSchema = WorkflowSchema;
 
-// Form / schema interfaces for the output transform sub-form
+// Output transform modal
 export type OutputTransformFormValues = {
   output_map: MapArrayFormValue;
   full_response_path: ConfigFieldValue;
 };
-export type OutputTransformSchema = WorkflowSchema;
+
+// Configure template modal
+export type TemplateFormValues = Omit<Transform, 'transformType'>;
+
+// Configure expression modal
+export type ExpressionFormValues = {
+  expression: string;
+};
+
+// Configure multi-expression modal
+export type MultiExpressionFormValues = {
+  expressions: ExpressionVar[];
+};
 
 /**
  ********** WORKSPACE TYPES/INTERFACES **********
@@ -429,7 +473,6 @@ export type ModelInterface = {
 export type ConnectorParameters = {
   model?: string;
   dimensions?: number;
-  [PROMPT_FIELD]?: string;
 };
 
 export type Model = {

public/configs/ml_processor.ts

@@ -22,11 +22,11 @@ export abstract class MLProcessor extends Processor {
       },
       {
         id: 'input_map',
-        type: 'mapArray',
+        type: 'inputMapArray',
       },
       {
         id: 'output_map',
-        type: 'mapArray',
+        type: 'outputMapArray',
       },
     ];
     this.optionalFields = [

public/pages/workflow_detail/workflow_inputs/ingest_inputs/source_data_modal.tsx

@@ -31,7 +31,6 @@ import {
   IConfigField,
   IndexMappings,
   IngestDocsFormValues,
-  IngestDocsSchema,
   isVectorSearchUseCase,
   SearchHit,
   SOURCE_OPTIONS,
@@ -78,7 +77,7 @@ export function SourceDataModal(props: SourceDataProps) {
     docs: getFieldSchema({
       type: 'jsonArray',
     } as IConfigField),
-  }) as IngestDocsSchema;
+  }) as yup.Schema;
 
   // persist standalone values. update / initialize when it is first opened
   const [tempDocs, setTempDocs] = useState<string>('[]');

public/pages/workflow_detail/workflow_inputs/input_fields/index.ts

@@ -11,3 +11,4 @@ export { MapArrayField } from './map_array_field';
 export { BooleanField } from './boolean_field';
 export { SelectField } from './select_field';
 export { NumberField } from './number_field';
+export { SelectWithCustomOptions } from './select_with_custom_options';

public/pages/workflow_detail/workflow_inputs/input_fields/map_field.tsx

@@ -185,6 +185,7 @@ export function MapField(props: MapFieldProps) {
                                     placeholder={
                                       props.keyPlaceholder || 'Input'
                                     }
+                                    allowCreate={true}
                                   />
                                 ) : (
                                   <TextField
@@ -221,6 +222,7 @@ export function MapField(props: MapFieldProps) {
                                     placeholder={
                                       props.valuePlaceholder || 'Output'
                                     }
+                                    allowCreate={true}
                                   />
                                 ) : (
                                   <TextField

public/pages/workflow_detail/workflow_inputs/input_fields/select_with_custom_options.tsx

@@ -13,6 +13,8 @@ interface SelectWithCustomOptionsProps {
   fieldPath: string;
   placeholder: string;
   options: { label: string }[];
+  allowCreate?: boolean;
+  onChange?: () => void;
 }
 
 /**
@@ -31,6 +33,8 @@ export function SelectWithCustomOptions(props: SelectWithCustomOptionsProps) {
     const formValue = getIn(values, props.fieldPath);
     if (!isEmpty(formValue)) {
       setSelectedOption([{ label: formValue }]);
+    } else {
+      setSelectedOption([]);
     }
   }, [getIn(values, props.fieldPath)]);
 
@@ -76,9 +80,14 @@ export function SelectWithCustomOptions(props: SelectWithCustomOptionsProps) {
       onChange={(options) => {
         setFieldTouched(props.fieldPath, true);
         setFieldValue(props.fieldPath, get(options, '0.label'));
+        if (props.onChange) {
+          props.onChange();
+        }
       }}
-      onCreateOption={onCreateOption}
-      customOptionText="Add {searchValue} as a custom option"
+      onCreateOption={props.allowCreate ? onCreateOption : undefined}
+      customOptionText={
+        props.allowCreate ? 'Add {searchValue} as a custom option' : undefined
+      }
     />
   );
 }