From a148ea52244d3d50e972a218968337a3fcbd1830 Mon Sep 17 00:00:00 2001 From: Tyler Ohlsen Date: Wed, 27 Nov 2024 13:26:59 -0800 Subject: [PATCH] Improve input/output transformation configuration (#504) Signed-off-by: Tyler Ohlsen --- common/constants.ts | 18 +- common/interfaces.ts | 67 +- public/configs/ml_processor.ts | 4 +- .../ingest_inputs/source_data_modal.tsx | 3 +- .../workflow_inputs/input_fields/index.ts | 1 + .../input_fields/map_field.tsx | 2 + .../select_with_custom_options.tsx | 13 +- .../ml_processor_inputs.tsx | 285 +----- .../modals/configure_expression_modal.tsx | 573 ++++++++++++ .../configure_multi_expression_modal.tsx | 588 ++++++++++++ .../modals/configure_prompt_modal.tsx | 305 ------- .../modals/configure_template_modal.tsx | 777 ++++++++++++++++ .../ml_processor_inputs/modals/index.ts | 6 +- .../modals/input_transform_modal.tsx | 836 ------------------ .../modals/output_transform_modal.tsx | 656 -------------- .../modals/override_query_modal.tsx | 6 +- .../ml_processor_inputs/model_inputs.tsx | 511 ++++++++++- .../ml_processor_inputs/model_outputs.tsx | 428 ++++++++- .../search_inputs/edit_query_modal.tsx | 3 +- .../new_workflow/quick_configure_modal.tsx | 127 ++- public/utils/config_to_form_utils.ts | 4 +- public/utils/config_to_schema_utils.ts | 94 +- public/utils/config_to_template_utils.ts | 143 ++- public/utils/utils.ts | 18 +- server/routes/helpers.ts | 2 - 25 files changed, 3269 insertions(+), 2201 deletions(-) create mode 100644 public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs/modals/configure_expression_modal.tsx create mode 100644 public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs/modals/configure_multi_expression_modal.tsx delete mode 100644 public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs/modals/configure_prompt_modal.tsx create mode 100644 public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs/modals/configure_template_modal.tsx delete mode 100644 public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs/modals/input_transform_modal.tsx delete mode 100644 public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs/modals/output_transform_modal.tsx diff --git a/common/constants.ts b/common/constants.ts index 69826bc7..2d8787ff 100644 --- a/common/constants.ts +++ b/common/constants.ts @@ -4,6 +4,7 @@ */ import { + InputMapEntry, MapEntry, PromptPreset, QueryPreset, @@ -468,6 +469,13 @@ export enum TRANSFORM_CONTEXT { INPUT = 'input', OUTPUT = 'output', } +export enum TRANSFORM_TYPE { + STRING = 'String', + FIELD = 'Field', + EXPRESSION = 'Expression', + TEMPLATE = 'Template', +} +export const NO_TRANSFORMATION = 'No transformation'; export const START_FROM_SCRATCH_WORKFLOW_NAME = 'Start From Scratch'; export const DEFAULT_NEW_WORKFLOW_NAME = 'new_workflow'; export const DEFAULT_NEW_WORKFLOW_DESCRIPTION = 'My new workflow'; @@ -490,13 +498,21 @@ export enum SORT_ORDER { export const MAX_DOCS = 1000; export const MAX_STRING_LENGTH = 100; export const MAX_JSON_STRING_LENGTH = 10000; +export const MAX_TEMPLATE_STRING_LENGTH = 10000; export const MAX_WORKFLOW_NAME_TO_DISPLAY = 40; export const WORKFLOW_NAME_REGEXP = RegExp('^[a-zA-Z0-9_-]*$'); export const EMPTY_MAP_ENTRY = { key: '', value: '' } as MapEntry; +export const EMPTY_INPUT_MAP_ENTRY = { + key: '', + value: { + transformType: '' as TRANSFORM_TYPE, + value: '', + }, +} as InputMapEntry; +export const EMPTY_OUTPUT_MAP_ENTRY = EMPTY_INPUT_MAP_ENTRY; export const MODEL_OUTPUT_SCHEMA_NESTED_PATH = 'output.properties.inference_results.items.properties.output.items.properties.dataAsMap.properties'; export const MODEL_OUTPUT_SCHEMA_FULL_PATH = 'output.properties'; -export const PROMPT_FIELD = 'prompt'; // TODO: likely expand to support a pattern and/or multiple (e.g., "prompt", "prompt_template", etc.) export enum CONFIG_STEP { INGEST = 'Ingestion pipeline', SEARCH = 'Search pipeline', diff --git a/common/interfaces.ts b/common/interfaces.ts index f9bda443..1189cfa8 100644 --- a/common/interfaces.ts +++ b/common/interfaces.ts @@ -9,7 +9,7 @@ import { ObjectSchema } from 'yup'; import { COMPONENT_CLASS, PROCESSOR_TYPE, - PROMPT_FIELD, + TRANSFORM_TYPE, WORKFLOW_TYPE, } from './constants'; @@ -35,7 +35,9 @@ export type ConfigFieldType = | 'map' | 'mapArray' | 'boolean' - | 'number'; + | 'number' + | 'inputMapArray' + | 'outputMapArray'; export type ConfigFieldValue = string | {}; @@ -100,6 +102,31 @@ export type MapFormValue = MapEntry[]; export type MapArrayFormValue = MapFormValue[]; +export type ExpressionVar = { + name: string; + transform: string; +}; + +export type Transform = { + transformType: TRANSFORM_TYPE; + value?: string; + // Templates may persist their own set of nested transforms + // to be dynamically injected into the template + nestedVars?: ExpressionVar[]; +}; + +export type InputMapEntry = { + key: string; + value: Transform; +}; +export type OutputMapEntry = InputMapEntry; + +export type InputMapFormValue = InputMapEntry[]; +export type OutputMapFormValue = OutputMapEntry[]; + +export type InputMapArrayFormValue = InputMapFormValue[]; +export type OutputMapArrayFormValue = OutputMapFormValue[]; + export type WorkflowFormValues = { ingest: FormikValues; search: FormikValues; @@ -110,31 +137,48 @@ export type WorkflowSchemaObj = { }; export type WorkflowSchema = ObjectSchema; -// Form / schema interfaces for the ingest docs sub-form +/** + ********** MODAL SUB-FORM TYPES/INTERFACES ********** + We persist sub-forms in the form modals s.t. data is only + saved back to the parent form if the user explicitly saves. + + We define the structure of the forms here. + */ + +// Ingest docs modal export type IngestDocsFormValues = { docs: FormikValues; }; -export type IngestDocsSchema = WorkflowSchema; -// Form / schema interfaces for the request query sub-form +// Search request modal export type RequestFormValues = { request: ConfigFieldValue; }; -export type RequestSchema = WorkflowSchema; -// Form / schema interfaces for the input transform sub-form +// Input transform modal export type InputTransformFormValues = { - input_map: MapArrayFormValue; + input_map: InputMapArrayFormValue; one_to_one: ConfigFieldValue; }; -export type InputTransformSchema = WorkflowSchema; -// Form / schema interfaces for the output transform sub-form +// Output transform modal export type OutputTransformFormValues = { output_map: MapArrayFormValue; full_response_path: ConfigFieldValue; }; -export type OutputTransformSchema = WorkflowSchema; + +// Configure template modal +export type TemplateFormValues = Omit; + +// Configure expression modal +export type ExpressionFormValues = { + expression: string; +}; + +// Configure multi-expression modal +export type MultiExpressionFormValues = { + expressions: ExpressionVar[]; +}; /** ********** WORKSPACE TYPES/INTERFACES ********** @@ -429,7 +473,6 @@ export type ModelInterface = { export type ConnectorParameters = { model?: string; dimensions?: number; - [PROMPT_FIELD]?: string; }; export type Model = { diff --git a/public/configs/ml_processor.ts b/public/configs/ml_processor.ts index 623981ae..e570dd0c 100644 --- a/public/configs/ml_processor.ts +++ b/public/configs/ml_processor.ts @@ -22,11 +22,11 @@ export abstract class MLProcessor extends Processor { }, { id: 'input_map', - type: 'mapArray', + type: 'inputMapArray', }, { id: 'output_map', - type: 'mapArray', + type: 'outputMapArray', }, ]; this.optionalFields = [ diff --git a/public/pages/workflow_detail/workflow_inputs/ingest_inputs/source_data_modal.tsx b/public/pages/workflow_detail/workflow_inputs/ingest_inputs/source_data_modal.tsx index 54ba6e33..97c31b61 100644 --- a/public/pages/workflow_detail/workflow_inputs/ingest_inputs/source_data_modal.tsx +++ b/public/pages/workflow_detail/workflow_inputs/ingest_inputs/source_data_modal.tsx @@ -31,7 +31,6 @@ import { IConfigField, IndexMappings, IngestDocsFormValues, - IngestDocsSchema, isVectorSearchUseCase, SearchHit, SOURCE_OPTIONS, @@ -78,7 +77,7 @@ export function SourceDataModal(props: SourceDataProps) { docs: getFieldSchema({ type: 'jsonArray', } as IConfigField), - }) as IngestDocsSchema; + }) as yup.Schema; // persist standalone values. update / initialize when it is first opened const [tempDocs, setTempDocs] = useState('[]'); diff --git a/public/pages/workflow_detail/workflow_inputs/input_fields/index.ts b/public/pages/workflow_detail/workflow_inputs/input_fields/index.ts index 3928800a..8de788b2 100644 --- a/public/pages/workflow_detail/workflow_inputs/input_fields/index.ts +++ b/public/pages/workflow_detail/workflow_inputs/input_fields/index.ts @@ -11,3 +11,4 @@ export { MapArrayField } from './map_array_field'; export { BooleanField } from './boolean_field'; export { SelectField } from './select_field'; export { NumberField } from './number_field'; +export { SelectWithCustomOptions } from './select_with_custom_options'; diff --git a/public/pages/workflow_detail/workflow_inputs/input_fields/map_field.tsx b/public/pages/workflow_detail/workflow_inputs/input_fields/map_field.tsx index 784823dd..e1795e36 100644 --- a/public/pages/workflow_detail/workflow_inputs/input_fields/map_field.tsx +++ b/public/pages/workflow_detail/workflow_inputs/input_fields/map_field.tsx @@ -185,6 +185,7 @@ export function MapField(props: MapFieldProps) { placeholder={ props.keyPlaceholder || 'Input' } + allowCreate={true} /> ) : ( ) : ( void; } /** @@ -31,6 +33,8 @@ export function SelectWithCustomOptions(props: SelectWithCustomOptionsProps) { const formValue = getIn(values, props.fieldPath); if (!isEmpty(formValue)) { setSelectedOption([{ label: formValue }]); + } else { + setSelectedOption([]); } }, [getIn(values, props.fieldPath)]); @@ -76,9 +80,14 @@ export function SelectWithCustomOptions(props: SelectWithCustomOptionsProps) { onChange={(options) => { setFieldTouched(props.fieldPath, true); setFieldValue(props.fieldPath, get(options, '0.label')); + if (props.onChange) { + props.onChange(); + } }} - onCreateOption={onCreateOption} - customOptionText="Add {searchValue} as a custom option" + onCreateOption={props.allowCreate ? onCreateOption : undefined} + customOptionText={ + props.allowCreate ? 'Add {searchValue} as a custom option' : undefined + } /> ); } diff --git a/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs/ml_processor_inputs.tsx b/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs/ml_processor_inputs.tsx index 90c525e7..995f39f6 100644 --- a/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs/ml_processor_inputs.tsx +++ b/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs/ml_processor_inputs.tsx @@ -7,17 +7,15 @@ import React, { useState, useEffect } from 'react'; import { getIn, useFormikContext } from 'formik'; import { isEmpty } from 'lodash'; import { useSelector } from 'react-redux'; -import { flattie } from 'flattie'; import { EuiAccordion, - EuiSmallButtonEmpty, EuiCallOut, EuiFlexGroup, EuiFlexItem, EuiSpacer, EuiText, - EuiToolTip, EuiSmallButton, + EuiIconTip, } from '@elastic/eui'; import { IProcessorConfig, @@ -26,27 +24,26 @@ import { WorkflowConfig, WorkflowFormValues, ModelInterface, - IndexMappings, - PROMPT_FIELD, - MapArrayFormValue, - MapEntry, - MapFormValue, + EMPTY_INPUT_MAP_ENTRY, + REQUEST_PREFIX, + REQUEST_PREFIX_WITH_JSONPATH_ROOT_SELECTOR, + OutputMapEntry, + OutputMapFormValue, + OutputMapArrayFormValue, + EMPTY_OUTPUT_MAP_ENTRY, } from '../../../../../../common'; import { ModelField } from '../../input_fields'; import { - ConfigurePromptModal, - InputTransformModal, - OutputTransformModal, - OverrideQueryModal, -} from './modals'; + InputMapFormValue, + InputMapArrayFormValue, +} from '../../../../../../common'; +import { OverrideQueryModal } from './modals'; import { ModelInputs } from './model_inputs'; -import { AppState, getMappings, useAppDispatch } from '../../../../../store'; +import { AppState } from '../../../../../store'; import { formikToPartialPipeline, - getDataSourceId, parseModelInputs, parseModelOutputs, - sanitizeJSONPath, } from '../../../../../utils'; import { ConfigFieldList } from '../../config_field_list'; import { ModelOutputs } from './model_outputs'; @@ -65,10 +62,7 @@ interface MLProcessorInputsProps { * output map configuration forms, respectively. */ export function MLProcessorInputs(props: MLProcessorInputsProps) { - const dispatch = useAppDispatch(); - const dataSourceId = getDataSourceId(); - const { models, connectors } = useSelector((state: AppState) => state.ml); - const indices = useSelector((state: AppState) => state.opensearch.indices); + const { models } = useSelector((state: AppState) => state.ml); const { values, setFieldValue, setFieldTouched } = useFormikContext< WorkflowFormValues >(); @@ -84,12 +78,6 @@ export function MLProcessorInputs(props: MLProcessorInputsProps) { const outputMapFieldPath = `${props.baseConfigPath}.${props.config.id}.output_map`; const outputMapValue = getIn(values, outputMapFieldPath); - // contains a configurable prompt field or not. if so, expose some extra - // dedicated UI - const [containsPromptField, setContainsPromptField] = useState( - false - ); - // preview availability states // if there are preceding search request processors, we cannot fetch and display the interim transformed query. // additionally, cannot preview output transforms for search request processors because output_maps need to be defined @@ -115,13 +103,6 @@ export function MLProcessorInputs(props: MLProcessorInputsProps) { }, [props.uiConfig.search.enrichRequest.processors]); // various modal states - const [isInputTransformModalOpen, setIsInputTransformModalOpen] = useState< - boolean - >(false); - const [isOutputTransformModalOpen, setIsOutputTransformModalOpen] = useState< - boolean - >(false); - const [isPromptModalOpen, setIsPromptModalOpen] = useState(false); const [isQueryModalOpen, setIsQueryModalOpen] = useState(false); // model interface state @@ -138,19 +119,19 @@ export function MLProcessorInputs(props: MLProcessorInputsProps) { const modelInputsAsForm = [ parseModelInputs(newModelInterface).map((modelInput) => { return { + ...EMPTY_INPUT_MAP_ENTRY, key: modelInput.label, - value: '', - } as MapEntry; - }) as MapFormValue, - ] as MapArrayFormValue; + }; + }) as InputMapFormValue, + ] as InputMapArrayFormValue; const modelOutputsAsForm = [ parseModelOutputs(newModelInterface).map((modelOutput) => { return { + ...EMPTY_OUTPUT_MAP_ENTRY, key: modelOutput.label, - value: '', - } as MapEntry; - }) as MapFormValue, - ] as MapArrayFormValue; + } as OutputMapEntry; + }) as OutputMapFormValue, + ] as OutputMapArrayFormValue; setFieldValue(inputMapFieldPath, modelInputsAsForm); setFieldValue(outputMapFieldPath, modelOutputsAsForm); @@ -168,143 +149,8 @@ export function MLProcessorInputs(props: MLProcessorInputsProps) { } }, [models]); - // persisting doc/query/index mapping fields to collect a list - // of options to display in the dropdowns when configuring input / output maps - const [docFields, setDocFields] = useState<{ label: string }[]>([]); - const [queryFields, setQueryFields] = useState<{ label: string }[]>([]); - const [indexMappingFields, setIndexMappingFields] = useState< - { label: string }[] - >([]); - useEffect(() => { - try { - const docObjKeys = Object.keys( - flattie((JSON.parse(values.ingest.docs) as {}[])[0]) - ); - if (docObjKeys.length > 0) { - setDocFields( - docObjKeys.map((key) => { - return { - label: - // ingest inputs can handle dot notation, and hence don't need - // sanitizing to handle JSONPath edge cases. The other contexts - // only support JSONPath, and hence need some post-processing/sanitizing. - props.context === PROCESSOR_CONTEXT.INGEST - ? key - : sanitizeJSONPath(key), - }; - }) - ); - } - } catch {} - }, [values?.ingest?.docs]); - useEffect(() => { - try { - const queryObjKeys = Object.keys( - flattie(JSON.parse(values.search.request)) - ); - if (queryObjKeys.length > 0) { - setQueryFields( - queryObjKeys.map((key) => { - return { - label: - // ingest inputs can handle dot notation, and hence don't need - // sanitizing to handle JSONPath edge cases. The other contexts - // only support JSONPath, and hence need some post-processing/sanitizing. - props.context === PROCESSOR_CONTEXT.INGEST - ? key - : sanitizeJSONPath(key), - }; - }) - ); - } - } catch {} - }, [values?.search?.request]); - useEffect(() => { - const indexName = values?.search?.index?.name as string | undefined; - if (indexName !== undefined && indices[indexName] !== undefined) { - dispatch( - getMappings({ - index: indexName, - dataSourceId, - }) - ) - .unwrap() - .then((resp: IndexMappings) => { - const mappingsObjKeys = Object.keys(resp.properties); - if (mappingsObjKeys.length > 0) { - setIndexMappingFields( - mappingsObjKeys.map((key) => { - return { - label: key, - type: resp.properties[key]?.type, - }; - }) - ); - } - }); - } - }, [values?.search?.index?.name]); - - // Check if there is an exposed prompt field users can override. Need to navigate - // to the associated connector details to view the connector parameters list. - useEffect(() => { - const selectedModel = Object.values(models).find( - (model) => model.id === getIn(values, modelIdFieldPath) - ); - if (selectedModel?.connectorId !== undefined) { - const connectorParameters = - connectors[selectedModel.connectorId]?.parameters; - if (connectorParameters !== undefined) { - if (connectorParameters[PROMPT_FIELD] !== undefined) { - setContainsPromptField(true); - } else { - setContainsPromptField(false); - } - } else { - setContainsPromptField(false); - } - } - }, [models, connectors, getIn(values, modelIdFieldPath)]); - return ( <> - {isInputTransformModalOpen && ( - setIsInputTransformModalOpen(false)} - /> - )} - {isOutputTransformModalOpen && ( - setIsOutputTransformModalOpen(false)} - /> - )} - {isPromptModalOpen && ( - setIsPromptModalOpen(false)} - /> - )} {isQueryModalOpen && ( )} - {containsPromptField && ( - <> - {`Configure prompt (Optional)`} - - setIsPromptModalOpen(true)} - data-testid="configurePromptButton" - > - Configure - - - - )} - {`Inputs`} - - - - { - setIsInputTransformModalOpen(true); - }} - > - Preview inputs - - + + + Inputs + + + + + @@ -399,31 +229,14 @@ export function MLProcessorInputs(props: MLProcessorInputsProps) { style={{ marginTop: '4px' }} >{`Outputs`} - - - { - setIsOutputTransformModalOpen(true); - }} - > - Preview outputs - - - {inputMapValue.length !== outputMapValue.length && diff --git a/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs/modals/configure_expression_modal.tsx b/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs/modals/configure_expression_modal.tsx new file mode 100644 index 00000000..c8a0b561 --- /dev/null +++ b/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs/modals/configure_expression_modal.tsx @@ -0,0 +1,573 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import React, { useEffect, useState } from 'react'; +import { useFormikContext, getIn, Formik } from 'formik'; +import { isEmpty } from 'lodash'; +import * as yup from 'yup'; +import Ajv from 'ajv'; +import { + EuiCodeEditor, + EuiFlexGroup, + EuiFlexItem, + EuiModal, + EuiModalBody, + EuiModalFooter, + EuiModalHeader, + EuiModalHeaderTitle, + EuiSmallButton, + EuiText, + EuiSmallButtonEmpty, + EuiSpacer, + EuiIconTip, +} from '@elastic/eui'; +import { + customStringify, + ExpressionFormValues, + IngestPipelineConfig, + InputMapEntry, + IProcessorConfig, + MAX_STRING_LENGTH, + ModelInterface, + PROCESSOR_CONTEXT, + SearchHit, + SimulateIngestPipelineResponse, + TRANSFORM_CONTEXT, + WorkflowConfig, + WorkflowFormValues, +} from '../../../../../../../common'; +import { + formikToPartialPipeline, + generateArrayTransform, + generateTransform, + getDataSourceId, + getInitialValue, + prepareDocsForSimulate, + unwrapTransformedDocs, +} from '../../../../../../utils'; +import { TextField } from '../../../input_fields'; +import { + searchIndex, + simulatePipeline, + useAppDispatch, +} from '../../../../../../store'; +import { getCore } from '../../../../../../services'; + +interface ConfigureExpressionModalProps { + uiConfig: WorkflowConfig; + config: IProcessorConfig; + context: PROCESSOR_CONTEXT; + baseConfigPath: string; + modelInputFieldName: string; + fieldPath: string; + modelInterface: ModelInterface | undefined; + isDataFetchingAvailable: boolean; + onClose: () => void; +} + +// Spacing between the input field columns +const KEY_FLEX_RATIO = 7; +const VALUE_FLEX_RATIO = 4; + +// the max number of input docs we use to display & test transforms with (search response hits) +const MAX_INPUT_DOCS = 10; + +/** + * A modal to configure a JSONPath expression / transform. Used for configuring model input transforms. + * Performs field-level validation, if the configured field is available in the model interface. + */ +export function ConfigureExpressionModal(props: ConfigureExpressionModalProps) { + const dispatch = useAppDispatch(); + const dataSourceId = getDataSourceId(); + const { values, setFieldValue, setFieldTouched } = useFormikContext< + WorkflowFormValues + >(); + + // sub-form values/schema + const expressionFormValues = { + expression: getInitialValue('string'), + } as ExpressionFormValues; + const expressionFormSchema = yup.object({ + expression: yup + .string() + .trim() + .min(1, 'Too short') + .max(MAX_STRING_LENGTH, 'Too long') + .required('Required') as yup.Schema, + }) as yup.Schema; + + // persist standalone values. update / initialize when it is first opened + const [tempExpression, setTempExpression] = useState(''); + const [tempErrors, setTempErrors] = useState(false); + + // get some current form values + const oneToOne = getIn( + values, + `${props.baseConfigPath}.${props.config.id}.one_to_one` + ); + const docs = getIn(values, 'ingest.docs'); + let docObjs = [] as {}[] | undefined; + try { + docObjs = JSON.parse(docs); + } catch {} + const query = getIn(values, 'search.request'); + let queryObj = {} as {} | undefined; + try { + queryObj = JSON.parse(query); + } catch {} + const onIngestAndNoDocs = + props.context === PROCESSOR_CONTEXT.INGEST && isEmpty(docObjs); + const onSearchAndNoQuery = + (props.context === PROCESSOR_CONTEXT.SEARCH_REQUEST || + props.context === PROCESSOR_CONTEXT.SEARCH_RESPONSE) && + isEmpty(queryObj); + + // button updating state + const [isUpdating, setIsUpdating] = useState(false); + + // validation state utilizing the model interface, if applicable. undefined if + // there is no model interface and/or no source input + const [isValid, setIsValid] = useState(undefined); + + // source input / transformed input state + const [sourceInput, setSourceInput] = useState('{}'); + const [transformedInput, setTransformedInput] = useState('{}'); + + // fetching input data state + const [isFetching, setIsFetching] = useState(false); + + // hook to re-generate the transform when any inputs to the transform are updated + useEffect(() => { + const tempExpressionAsInputMap = [ + { + key: props.modelInputFieldName, + value: { + value: tempExpression, + }, + } as InputMapEntry, + ]; + if (!isEmpty(tempExpression) && !isEmpty(JSON.parse(sourceInput))) { + let sampleSourceInput = {} as {} | []; + try { + sampleSourceInput = JSON.parse(sourceInput); + const output = + // Edge case: users are collapsing input docs into a single input field when many-to-one is selected + // fo input transforms on search response processors. + oneToOne === false && + props.context === PROCESSOR_CONTEXT.SEARCH_RESPONSE && + Array.isArray(sampleSourceInput) + ? generateArrayTransform( + sampleSourceInput as [], + tempExpressionAsInputMap, + props.context, + TRANSFORM_CONTEXT.INPUT, + queryObj + ) + : generateTransform( + sampleSourceInput, + tempExpressionAsInputMap, + props.context, + TRANSFORM_CONTEXT.INPUT, + queryObj + ); + + setTransformedInput(customStringify(output)); + } catch {} + } else { + setTransformedInput('{}'); + } + }, [tempExpression, sourceInput]); + + // hook to re-determine validity when the generated output changes + // utilize Ajv JSON schema validator library. For more info/examples, see + // https://www.npmjs.com/package/ajv + useEffect(() => { + if ( + !isEmpty(JSON.parse(sourceInput)) && + !isEmpty(props.modelInterface?.input?.properties?.parameters) && + !isEmpty( + getIn( + props.modelInterface?.input?.properties?.parameters?.properties, + props.modelInputFieldName + ) + ) + ) { + // we customize the model interface JSON schema to just + // include the field we are transforming. Overriding any + // other config fields that could make this unnecessarily fail + // (required, additionalProperties, etc.) + try { + const customJSONSchema = { + ...props.modelInterface?.input?.properties?.parameters, + properties: { + [props.modelInputFieldName]: getIn( + props.modelInterface?.input?.properties?.parameters.properties, + props.modelInputFieldName + ), + }, + required: [], + additionalProperties: true, + }; + + const validateFn = new Ajv().compile(customJSONSchema); + setIsValid(validateFn(JSON.parse(transformedInput))); + } catch { + setIsValid(undefined); + } + } else { + setIsValid(undefined); + } + }, [transformedInput]); + + // if updating, take the temp vars and assign it to the parent form + function onUpdate() { + setIsUpdating(true); + setFieldValue(`${props.fieldPath}.value`, tempExpression); + setFieldTouched(props.fieldPath, true); + props.onClose(); + } + + return ( + {}} + validate={(values) => {}} + > + {(formikProps) => { + // override to parent form values when changes detected + useEffect(() => { + formikProps.setFieldValue( + 'expression', + getIn(values, `${props.fieldPath}.value`) + ); + }, [getIn(values, props.fieldPath)]); + + // update temp vars when form changes are detected + useEffect(() => { + setTempExpression(getIn(formikProps.values, 'expression')); + }, [getIn(formikProps.values, 'expression')]); + + // update tempErrors if errors detected + useEffect(() => { + setTempErrors(!isEmpty(formikProps.errors)); + }, [formikProps.errors]); + + return ( + + + +

{`Extract data with expression`}

+ + + + + + + + + + + {`Expression`} + + + + + {`Model input name`} + + + + + + + + + + {props.modelInputFieldName} + + + + + + + + + + + + Preview + + + { + setIsFetching(true); + switch (props.context) { + case PROCESSOR_CONTEXT.INGEST: { + // get the current ingest pipeline up to, but not including, this processor + const curIngestPipeline = formikToPartialPipeline( + values, + props.uiConfig, + props.config.id, + false, + PROCESSOR_CONTEXT.INGEST + ); + // if there are preceding processors, we need to simulate the partial ingest pipeline, + // in order to get the latest transformed version of the docs + if (curIngestPipeline !== undefined) { + const curDocs = prepareDocsForSimulate( + values.ingest.docs, + values.ingest.index.name + ); + await dispatch( + simulatePipeline({ + apiBody: { + pipeline: curIngestPipeline as IngestPipelineConfig, + docs: [curDocs[0]], + }, + dataSourceId, + }) + ) + .unwrap() + .then( + ( + resp: SimulateIngestPipelineResponse + ) => { + const docObjs = unwrapTransformedDocs( + resp + ); + if (docObjs.length > 0) { + setSourceInput( + customStringify(docObjs[0]) + ); + } + } + ) + .catch((error: any) => { + getCore().notifications.toasts.addDanger( + `Failed to fetch input data` + ); + }) + .finally(() => { + setIsFetching(false); + }); + } else { + try { + const docObjs = JSON.parse( + values.ingest.docs + ) as {}[]; + if (docObjs.length > 0) { + setSourceInput( + customStringify(docObjs[0]) + ); + } + } catch { + } finally { + setIsFetching(false); + } + } + break; + } + case PROCESSOR_CONTEXT.SEARCH_REQUEST: { + // get the current search pipeline up to, but not including, this processor + const curSearchPipeline = formikToPartialPipeline( + values, + props.uiConfig, + props.config.id, + false, + PROCESSOR_CONTEXT.SEARCH_REQUEST + ); + // if there are preceding processors, we cannot generate. The button to render + // this modal should be disabled if the search pipeline would be enabled. We add + // this if check as an extra layer of checking, and if mechanism for gating + // this is changed in the future. + if (curSearchPipeline === undefined) { + setSourceInput(values.search.request); + } + setIsFetching(false); + break; + } + case PROCESSOR_CONTEXT.SEARCH_RESPONSE: { + // get the current search pipeline up to, but not including, this processor + const curSearchPipeline = formikToPartialPipeline( + values, + props.uiConfig, + props.config.id, + false, + PROCESSOR_CONTEXT.SEARCH_RESPONSE + ); + // Execute search. If there are preceding processors, augment the existing query with + // the partial search pipeline (inline) to get the latest transformed version of the response. + dispatch( + searchIndex({ + apiBody: { + index: values.search.index.name, + body: JSON.stringify({ + ...JSON.parse( + values.search.request as string + ), + search_pipeline: + curSearchPipeline || {}, + }), + }, + dataSourceId, + }) + ) + .unwrap() + .then(async (resp) => { + const hits = resp?.hits?.hits + ?.map((hit: SearchHit) => hit._source) + .slice(0, MAX_INPUT_DOCS); + if (hits.length > 0) { + setSourceInput( + // if one-to-one, treat the source input as a single retrieved document + // else, treat it as all of the returned documents + customStringify( + oneToOne ? hits[0] : hits + ) + ); + } + }) + .catch((error: any) => { + getCore().notifications.toasts.addDanger( + `Failed to fetch source input data` + ); + }) + .finally(() => { + setIsFetching(false); + }); + break; + } + } + }} + > + Run preview + + + + + + Source data + + + + + + + + Extracted data + + {isValid !== undefined && ( + + + + )} + + + + + + + + + + + + Cancel + + { + formikProps + .submitForm() + .then((value: any) => { + onUpdate(); + }) + .catch((err: any) => {}); + }} + isLoading={isUpdating} + isDisabled={tempErrors} // blocking update until valid input is given + fill={true} + color="primary" + data-testid="updateExpressionButton" + > + Save + + + + ); + }} + + ); +} diff --git a/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs/modals/configure_multi_expression_modal.tsx b/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs/modals/configure_multi_expression_modal.tsx new file mode 100644 index 00000000..22cb3a42 --- /dev/null +++ b/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs/modals/configure_multi_expression_modal.tsx @@ -0,0 +1,588 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import React, { useEffect, useState } from 'react'; +import { useFormikContext, getIn, Formik } from 'formik'; +import { cloneDeep, isEmpty, set } from 'lodash'; +import * as yup from 'yup'; +import { + EuiCodeEditor, + EuiFlexGroup, + EuiFlexItem, + EuiModal, + EuiModalBody, + EuiModalFooter, + EuiModalHeader, + EuiModalHeaderTitle, + EuiSmallButton, + EuiText, + EuiSmallButtonEmpty, + EuiSpacer, + EuiSmallButtonIcon, +} from '@elastic/eui'; +import { + customStringify, + ExpressionVar, + IngestPipelineConfig, + IProcessorConfig, + MAX_STRING_LENGTH, + ModelInterface, + MultiExpressionFormValues, + OutputMapEntry, + PROCESSOR_CONTEXT, + SearchHit, + SearchPipelineConfig, + SimulateIngestPipelineResponse, + TRANSFORM_CONTEXT, + WorkflowConfig, + WorkflowFormValues, +} from '../../../../../../../common'; +import { + formikToPartialPipeline, + generateTransform, + getDataSourceId, + prepareDocsForSimulate, + unwrapTransformedDocs, +} from '../../../../../../utils'; +import { TextField } from '../../../input_fields'; +import { + searchIndex, + simulatePipeline, + useAppDispatch, +} from '../../../../../../store'; +import { getCore } from '../../../../../../services'; + +interface ConfigureMultiExpressionModalProps { + uiConfig: WorkflowConfig; + config: IProcessorConfig; + context: PROCESSOR_CONTEXT; + baseConfigPath: string; + modelInputFieldName: string; + fieldPath: string; + modelInterface: ModelInterface | undefined; + outputMapFieldPath: string; + isDataFetchingAvailable: boolean; + onClose: () => void; +} + +// Spacing between the input field columns +const KEY_FLEX_RATIO = 6; +const VALUE_FLEX_RATIO = 4; + +/** + * A modal to configure multiple JSONPath expression / transforms. Used for parsing out + * model outputs, which can be arbitrarily complex / nested. + */ +export function ConfigureMultiExpressionModal( + props: ConfigureMultiExpressionModalProps +) { + const dispatch = useAppDispatch(); + const dataSourceId = getDataSourceId(); + const { values, setFieldValue, setFieldTouched } = useFormikContext< + WorkflowFormValues + >(); + + // sub-form values/schema + const expressionsFormValues = { + expressions: [], + } as MultiExpressionFormValues; + const expressionsFormSchema = yup.object({ + expressions: yup + .array() + .of( + yup.object({ + name: yup + .string() + .trim() + .min(1, 'Too short') + .max(MAX_STRING_LENGTH, 'Too long') + .required('Required') as yup.Schema, + transform: yup + .string() + .trim() + .min(1, 'Too short') + .max(MAX_STRING_LENGTH, 'Too long') + .required('Required') as yup.Schema, + }) + ) + .min(1), + }) as yup.Schema; + + // persist standalone values. update / initialize when it is first opened + const [tempExpressions, setTempExpressions] = useState([]); + const [tempErrors, setTempErrors] = useState(false); + + // get some current form values + const fullResponsePathPath = `${props.baseConfigPath}.${props.config.id}.full_response_path`; + const docs = getIn(values, 'ingest.docs'); + let docObjs = [] as {}[] | undefined; + try { + docObjs = JSON.parse(docs); + } catch {} + const query = getIn(values, 'search.request'); + let queryObj = {} as {} | undefined; + try { + queryObj = JSON.parse(query); + } catch {} + const onIngestAndNoDocs = + props.context === PROCESSOR_CONTEXT.INGEST && isEmpty(docObjs); + const onSearchAndNoQuery = + (props.context === PROCESSOR_CONTEXT.SEARCH_REQUEST || + props.context === PROCESSOR_CONTEXT.SEARCH_RESPONSE) && + isEmpty(queryObj); + + // button updating state + const [isUpdating, setIsUpdating] = useState(false); + + // source input / transformed input state + const [sourceInput, setSourceInput] = useState('{}'); + const [transformedInput, setTransformedInput] = useState('{}'); + + // fetching input data state + const [isFetching, setIsFetching] = useState(false); + + // hook to re-generate the transform when any inputs to the transform are updated + useEffect(() => { + const tempExpressionsAsOutputMap = tempExpressions.map( + (expressionVar) => + ({ + key: expressionVar.name || '', + value: { + value: expressionVar.transform || '', + }, + } as OutputMapEntry) + ); + if ( + !isEmpty(tempExpressionsAsOutputMap) && + !isEmpty(JSON.parse(sourceInput)) + ) { + let sampleSourceInput = {} as {} | []; + try { + sampleSourceInput = JSON.parse(sourceInput); + const output = generateTransform( + sampleSourceInput, + tempExpressionsAsOutputMap, + props.context, + TRANSFORM_CONTEXT.OUTPUT + ); + setTransformedInput(customStringify(output)); + } catch {} + } else { + setTransformedInput('{}'); + } + }, [tempExpressions, sourceInput]); + + // if updating, take the temp vars and assign it to the parent form + function onUpdate() { + setIsUpdating(true); + setFieldValue(`${props.fieldPath}.nestedVars`, tempExpressions); + setFieldTouched(props.fieldPath, true); + props.onClose(); + } + + return ( + {}} + validate={(values) => {}} + > + {(formikProps) => { + // override to parent form values when changes detected + useEffect(() => { + formikProps.setFieldValue( + 'expressions', + getIn(values, `${props.fieldPath}.nestedVars`) + ); + }, [getIn(values, props.fieldPath)]); + + // update temp vars when form changes are detected + useEffect(() => { + setTempExpressions(getIn(formikProps.values, 'expressions')); + }, [getIn(formikProps.values, 'expressions')]); + + // update tempErrors if errors detected + useEffect(() => { + setTempErrors(!isEmpty(formikProps.errors)); + }, [formikProps.errors]); + + // Adding an input var to the end of the existing arr + function addExpression(curExpressions: ExpressionVar[]): void { + const updatedExpressions = [ + ...curExpressions, + { name: '', transform: '' } as ExpressionVar, + ]; + formikProps.setFieldValue(`expressions`, updatedExpressions); + formikProps.setFieldTouched(`expressions`, true); + } + + // Deleting an input var + function deleteExpression( + expressions: ExpressionVar[], + idxToDelete: number + ): void { + const updatedExpressions = [...expressions]; + updatedExpressions.splice(idxToDelete, 1); + formikProps.setFieldValue('expressions', updatedExpressions); + formikProps.setFieldTouched('expressions', true); + } + + return ( + + + +

{`Extract data with expression`}

+ + + + + + + + + + + {`Expressions`} + + + + + {props.context === PROCESSOR_CONTEXT.SEARCH_REQUEST + ? `New query field` + : `New document field`} + + + + + {formikProps.values.expressions?.map( + (expression, idx) => { + return ( +
+ + + + + + + + + + {idx > 0 && ( + + { + deleteExpression( + formikProps.values.expressions || + [], + idx + ); + }} + /> + + )} + + + + +
+ ); + } + )} + { + addExpression(formikProps.values.expressions || []); + }} + > + {`Add expression`} + + + + + + + + + + + Preview + + + { + setIsFetching(true); + switch (props.context) { + // note we skip search request processor context. that is because empty output maps are not supported. + // for more details, see comment in ml_processor_inputs.tsx + case PROCESSOR_CONTEXT.INGEST: { + // get the current ingest pipeline up to, and including this processor. + // remove any currently-configured output map since we only want the transformation + // up to, and including, the input map transformations + const valuesWithoutOutputMapConfig = cloneDeep( + values + ); + set( + valuesWithoutOutputMapConfig, + props.outputMapFieldPath, + [] + ); + set( + valuesWithoutOutputMapConfig, + fullResponsePathPath, + getIn( + formikProps.values, + 'full_response_path' + ) + ); + const curIngestPipeline = formikToPartialPipeline( + valuesWithoutOutputMapConfig, + props.uiConfig, + props.config.id, + true, + PROCESSOR_CONTEXT.INGEST + ) as IngestPipelineConfig; + const curDocs = prepareDocsForSimulate( + values.ingest.docs, + values.ingest.index.name + ); + await dispatch( + simulatePipeline({ + apiBody: { + pipeline: curIngestPipeline, + docs: [curDocs[0]], + }, + dataSourceId, + }) + ) + .unwrap() + .then( + ( + resp: SimulateIngestPipelineResponse + ) => { + try { + const docObjs = unwrapTransformedDocs( + resp + ); + if (docObjs.length > 0) { + const sampleModelResult = + docObjs[0]?.inference_results || + {}; + setSourceInput( + customStringify(sampleModelResult) + ); + } + } catch {} + } + ) + .catch((error: any) => { + getCore().notifications.toasts.addDanger( + `Failed to fetch input data` + ); + }) + .finally(() => { + setIsFetching(false); + }); + break; + } + case PROCESSOR_CONTEXT.SEARCH_RESPONSE: { + // get the current search pipeline up to, and including this processor. + // remove any currently-configured output map since we only want the transformation + // up to, and including, the input map transformations + const valuesWithoutOutputMapConfig = cloneDeep( + values + ); + set( + valuesWithoutOutputMapConfig, + props.outputMapFieldPath, + [] + ); + set( + valuesWithoutOutputMapConfig, + fullResponsePathPath, + getIn( + formikProps.values, + 'full_response_path' + ) + ); + const curSearchPipeline = formikToPartialPipeline( + valuesWithoutOutputMapConfig, + props.uiConfig, + props.config.id, + true, + PROCESSOR_CONTEXT.SEARCH_RESPONSE + ) as SearchPipelineConfig; + + // Execute search. Augment the existing query with + // the partial search pipeline (inline) to get the latest transformed + // version of the request. + dispatch( + searchIndex({ + apiBody: { + index: values.search.index.name, + body: JSON.stringify({ + ...JSON.parse( + values.search.request as string + ), + search_pipeline: + curSearchPipeline || {}, + }), + }, + dataSourceId, + }) + ) + .unwrap() + .then(async (resp) => { + const hits = resp?.hits?.hits?.map( + (hit: SearchHit) => hit._source + ) as any[]; + if (hits.length > 0) { + const sampleModelResult = + hits[0].inference_results || {}; + setSourceInput( + customStringify(sampleModelResult) + ); + } + }) + .catch((error: any) => { + getCore().notifications.toasts.addDanger( + `Failed to fetch source input data` + ); + }) + .finally(() => { + setIsFetching(false); + }); + break; + } + } + }} + > + Run preview + + + + + + Source data + + + + + + Extracted data + + + + + + + + + + + Cancel + + { + formikProps + .submitForm() + .then((value: any) => { + onUpdate(); + }) + .catch((err: any) => {}); + }} + isLoading={isUpdating} + isDisabled={tempErrors} // blocking update until valid input is given + fill={true} + color="primary" + data-testid="updateMultiExpressionButton" + > + Save + + + + ); + }} + + ); +} diff --git a/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs/modals/configure_prompt_modal.tsx b/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs/modals/configure_prompt_modal.tsx deleted file mode 100644 index 0a64c752..00000000 --- a/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs/modals/configure_prompt_modal.tsx +++ /dev/null @@ -1,305 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -import React, { useState, useEffect } from 'react'; -import { useFormikContext, getIn } from 'formik'; -import { isEmpty } from 'lodash'; -import { - EuiCodeEditor, - EuiFlexGroup, - EuiFlexItem, - EuiModal, - EuiModalBody, - EuiModalFooter, - EuiModalHeader, - EuiModalHeaderTitle, - EuiSmallButton, - EuiSpacer, - EuiText, - EuiPopover, - EuiSmallButtonEmpty, - EuiPopoverTitle, - EuiCodeBlock, - EuiCode, - EuiBasicTable, - EuiAccordion, - EuiCopy, - EuiButtonIcon, - EuiContextMenu, -} from '@elastic/eui'; -import { - IProcessorConfig, - ModelInputFormField, - ModelInterface, - PROMPT_FIELD, - PROMPT_PRESETS, - PromptPreset, - WorkflowFormValues, - customStringify, -} from '../../../../../../../common'; -import { - parseModelInputs, - parseModelInputsObj, -} from '../../../../../../utils/utils'; - -interface ConfigurePromptModalProps { - config: IProcessorConfig; - baseConfigPath: string; - modelInterface: ModelInterface | undefined; - onClose: () => void; -} - -/** - * A modal to configure a prompt template. Can manually configure, include placeholder values - * using other model inputs, and/or select from a presets library. - */ -export function ConfigurePromptModal(props: ConfigurePromptModalProps) { - const { values, setFieldValue, setFieldTouched } = useFormikContext< - WorkflowFormValues - >(); - - // get some current form values - const modelConfigPath = `${props.baseConfigPath}.${props.config.id}.model_config`; - const modelConfig = getIn(values, modelConfigPath) as string; - const modelInputs = parseModelInputs(props.modelInterface); - - // popover states - const [schemaPopoverOpen, setSchemaPopoverOpen] = useState(false); - const [presetsPopoverOpen, setPresetsPopoverOpen] = useState(false); - - // prompt str state. manipulated as users manually update, or - // from selecting a preset - const [promptStr, setPromptStr] = useState(''); - - // hook to set the prompt if found in the model config - useEffect(() => { - try { - const modelConfigObj = JSON.parse(getIn(values, modelConfigPath)); - const prompt = getIn(modelConfigObj, PROMPT_FIELD); - if (!isEmpty(prompt)) { - setPromptStr(prompt); - } else { - setPromptStr(''); - } - } catch {} - }, [getIn(values, modelConfigPath)]); - - return ( - - - -

{`Configure prompt`}

- - - - - Configure a custom prompt template for the model. Optionally inject - dynamic model inputs into the template. - - - - <> - - setPresetsPopoverOpen(!presetsPopoverOpen)} - iconSide="right" - iconType="arrowDown" - > - Choose from a preset - - } - isOpen={presetsPopoverOpen} - closePopover={() => setPresetsPopoverOpen(false)} - anchorPosition="downLeft" - > - ({ - name: preset.name, - onClick: () => { - try { - setFieldValue( - modelConfigPath, - customStringify({ - ...JSON.parse(modelConfig), - [PROMPT_FIELD]: preset.prompt, - }) - ); - } catch {} - setFieldTouched(modelConfigPath, true); - setPresetsPopoverOpen(false); - }, - })), - }, - ]} - /> - - - Prompt - - setPromptStr(value)} - onBlur={(e) => { - let updatedModelConfig = {} as any; - try { - updatedModelConfig = JSON.parse(modelConfig); - } catch {} - if (isEmpty(promptStr)) { - // if the input is blank, it is assumed the user - // does not want any prompt. hence, remove the "prompt" field - // from the config altogether. - delete updatedModelConfig[PROMPT_FIELD]; - } else { - updatedModelConfig[PROMPT_FIELD] = promptStr; - } - setFieldValue( - modelConfigPath, - customStringify(updatedModelConfig) - ); - setFieldTouched(modelConfigPath); - }} - /> - {modelInputs.length > 0 && ( - <> - - - <> - - - To use any model inputs in the prompt template, copy the - placeholder string directly. - - - - - setSchemaPopoverOpen(false)} - panelPaddingSize="s" - button={ - - setSchemaPopoverOpen(!schemaPopoverOpen) - } - > - View input schema - - } - > - - The JSON Schema defining the model's expected input - - - {customStringify( - parseModelInputsObj(props.modelInterface) - )} - - - - - - - )} - - - - - - - Close - - - - ); -} - -const columns = [ - { - name: 'Name', - field: 'label', - width: '25%', - }, - { - name: 'Type', - field: 'type', - width: '15%', - }, - { - name: 'Placeholder string', - field: 'label', - width: '50%', - render: (label: string, modelInput: ModelInputFormField) => ( - - {getPlaceholderString(modelInput.type, label)} - - ), - }, - { - name: 'Actions', - field: 'label', - width: '10%', - render: (label: string, modelInput: ModelInputFormField) => ( - - {(copy) => ( - - )} - - ), - }, -]; - -// small util fn to get the full placeholder string to be -// inserted into the template. String conversion is required -// if the input is an array, for example. Also, all values -// should be prepended with "parameters.", as all inputs -// will be nested under a base parameters obj. -function getPlaceholderString(type: string, label: string) { - return type === 'array' - ? `\$\{parameters.${label}.toString()\}` - : `\$\{parameters.${label}\}`; -} diff --git a/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs/modals/configure_template_modal.tsx b/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs/modals/configure_template_modal.tsx new file mode 100644 index 00000000..56b4feec --- /dev/null +++ b/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs/modals/configure_template_modal.tsx @@ -0,0 +1,777 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import React, { useEffect, useState } from 'react'; +import { useFormikContext, getIn, Formik } from 'formik'; +import { isEmpty } from 'lodash'; +import * as yup from 'yup'; +import { + EuiCodeEditor, + EuiFlexGroup, + EuiFlexItem, + EuiModal, + EuiModalBody, + EuiModalFooter, + EuiModalHeader, + EuiModalHeaderTitle, + EuiSmallButton, + EuiText, + EuiPopover, + EuiContextMenu, + EuiSmallButtonEmpty, + EuiSmallButtonIcon, + EuiSpacer, + EuiCopy, +} from '@elastic/eui'; +import { + customStringify, + IngestPipelineConfig, + InputMapEntry, + IProcessorConfig, + MAX_STRING_LENGTH, + MAX_TEMPLATE_STRING_LENGTH, + ModelInterface, + PROCESSOR_CONTEXT, + PROMPT_PRESETS, + PromptPreset, + SearchHit, + SimulateIngestPipelineResponse, + TemplateFormValues, + ExpressionVar, + TRANSFORM_CONTEXT, + WorkflowConfig, + WorkflowFormValues, +} from '../../../../../../../common'; +import { + formikToPartialPipeline, + generateArrayTransform, + generateTransform, + getDataSourceId, + getInitialValue, + prepareDocsForSimulate, + unwrapTransformedDocs, +} from '../../../../../../utils'; +import { TextField } from '../../../input_fields'; +import { + searchIndex, + simulatePipeline, + useAppDispatch, +} from '../../../../../../store'; +import { getCore } from '../../../../../../services'; + +interface ConfigureTemplateModalProps { + uiConfig: WorkflowConfig; + config: IProcessorConfig; + context: PROCESSOR_CONTEXT; + baseConfigPath: string; + fieldPath: string; + modelInterface: ModelInterface | undefined; + isDataFetchingAvailable: boolean; + onClose: () => void; +} + +// Spacing between the input field columns +const KEY_FLEX_RATIO = 4; +const VALUE_FLEX_RATIO = 6; + +// the max number of input docs we use to display & test transforms with (search response hits) +const MAX_INPUT_DOCS = 10; + +/** + * A modal to configure a prompt template. Can manually configure, include placeholder values + * using other model inputs, and/or select from a presets library. Used for configuring model + * input transforms. + */ +export function ConfigureTemplateModal(props: ConfigureTemplateModalProps) { + const dispatch = useAppDispatch(); + const dataSourceId = getDataSourceId(); + const { values, setFieldValue, setFieldTouched } = useFormikContext< + WorkflowFormValues + >(); + + // sub-form values/schema + const templateFormValues = { + value: getInitialValue('string'), + nestedVars: [], + } as TemplateFormValues; + const templateFormSchema = yup.object({ + value: yup + .string() + .trim() + .min(1, 'Too short') + .max(MAX_TEMPLATE_STRING_LENGTH, 'Too long') + .required('Required') as yup.Schema, + nestedVars: yup.array().of( + yup.object().shape({ + name: yup + .string() + .trim() + .min(1, 'Too short') + .max(MAX_STRING_LENGTH, 'Too long') + .required('Required') as yup.Schema, + transform: yup + .string() + .trim() + .min(1, 'Too short') + .max(MAX_STRING_LENGTH, 'Too long') + .required('Required') as yup.Schema, + }) + ), + }) as yup.Schema; + + // persist standalone values. update / initialize when it is first opened + const [tempTemplate, setTempTemplate] = useState(''); + const [tempNestedVars, setTempNestedVars] = useState([]); + const [tempErrors, setTempErrors] = useState(false); + + // get some current form values + const oneToOne = getIn( + values, + `${props.baseConfigPath}.${props.config.id}.one_to_one` + ); + const docs = getIn(values, 'ingest.docs'); + let docObjs = [] as {}[] | undefined; + try { + docObjs = JSON.parse(docs); + } catch {} + const query = getIn(values, 'search.request'); + let queryObj = {} as {} | undefined; + try { + queryObj = JSON.parse(query); + } catch {} + const onIngestAndNoDocs = + props.context === PROCESSOR_CONTEXT.INGEST && isEmpty(docObjs); + const onSearchAndNoQuery = + (props.context === PROCESSOR_CONTEXT.SEARCH_REQUEST || + props.context === PROCESSOR_CONTEXT.SEARCH_RESPONSE) && + isEmpty(queryObj); + + // transformed template state + const [transformedTemplate, setTransformedTemplate] = useState(''); + + // button updating state + const [isUpdating, setIsUpdating] = useState(false); + + // popover states + const [presetsPopoverOpen, setPresetsPopoverOpen] = useState(false); + + // source input / transformed input state + const [sourceInput, setSourceInput] = useState('{}'); + const [transformedInput, setTransformedInput] = useState('{}'); + + // fetching input data state + const [isFetching, setIsFetching] = useState(false); + + // hook to re-generate the transform when any inputs to the transform are updated + useEffect(() => { + const nestedVarsAsInputMap = tempNestedVars?.map((expressionVar) => { + return { + key: expressionVar.name, + value: { + value: expressionVar.transform, + }, + } as InputMapEntry; + }); + if (!isEmpty(nestedVarsAsInputMap) && !isEmpty(JSON.parse(sourceInput))) { + let sampleSourceInput = {} as {} | []; + try { + sampleSourceInput = JSON.parse(sourceInput); + const output = + // Edge case: users are collapsing input docs into a single input field when many-to-one is selected + // fo input transforms on search response processors. + oneToOne === false && + props.context === PROCESSOR_CONTEXT.SEARCH_RESPONSE && + Array.isArray(sampleSourceInput) + ? generateArrayTransform( + sampleSourceInput as [], + nestedVarsAsInputMap, + props.context, + TRANSFORM_CONTEXT.INPUT, + queryObj + ) + : generateTransform( + sampleSourceInput, + nestedVarsAsInputMap, + props.context, + TRANSFORM_CONTEXT.INPUT, + queryObj + ); + setTransformedInput(customStringify(output)); + } catch {} + } else { + setTransformedInput('{}'); + } + }, [tempNestedVars, sourceInput]); + + // hook to set the transformed template, when the template + // and/or its injected variables are updated + useEffect(() => { + if (!isEmpty(tempTemplate)) { + setTransformedTemplate( + injectValuesIntoTemplate(tempTemplate, JSON.parse(transformedInput)) + ); + } + }, [tempTemplate, transformedInput]); + + // if updating, take the temp vars and assign it to the parent form + function onUpdate() { + setIsUpdating(true); + setFieldValue(`${props.fieldPath}.value`, tempTemplate); + setFieldValue(`${props.fieldPath}.nestedVars`, tempNestedVars); + setFieldTouched(props.fieldPath, true); + props.onClose(); + } + + return ( + {}} + validate={(values) => {}} + > + {(formikProps) => { + // override to parent form values when changes detected + useEffect(() => { + formikProps.setFieldValue( + 'value', + getIn(values, `${props.fieldPath}.value`) + ); + formikProps.setFieldValue( + 'nestedVars', + getIn(values, `${props.fieldPath}.nestedVars`) + ); + }, [getIn(values, props.fieldPath)]); + + // update temp vars when form changes are detected + useEffect(() => { + setTempTemplate(getIn(formikProps.values, 'value')); + }, [getIn(formikProps.values, 'value')]); + useEffect(() => { + setTempNestedVars(getIn(formikProps.values, 'nestedVars')); + }, [getIn(formikProps.values, 'nestedVars')]); + + // update tempErrors if errors detected + useEffect(() => { + setTempErrors(!isEmpty(formikProps.errors)); + }, [formikProps.errors]); + + // Adding an input var to the end of the existing arr + function addInputVar(curInputVars: ExpressionVar[]): void { + const updatedInputVars = [ + ...curInputVars, + { name: '', transform: '' } as ExpressionVar, + ]; + formikProps.setFieldValue(`nestedVars`, updatedInputVars); + formikProps.setFieldTouched(`nestedVars`, true); + } + + // Deleting an input var + function deleteInputVar( + curInputVars: ExpressionVar[], + idxToDelete: number + ): void { + const updatedInputVars = [...curInputVars]; + updatedInputVars.splice(idxToDelete, 1); + formikProps.setFieldValue('nestedVars', updatedInputVars); + formikProps.setFieldTouched('nestedVars', true); + } + + return ( + + + +

{`Configure prompt`}

+ + + + + + + + + + Prompt + + + + setPresetsPopoverOpen(!presetsPopoverOpen) + } + iconSide="right" + iconType="arrowDown" + > + Choose from a preset + + } + isOpen={presetsPopoverOpen} + closePopover={() => setPresetsPopoverOpen(false)} + anchorPosition="downLeft" + > + ({ + name: preset.name, + onClick: () => { + try { + formikProps.setFieldValue( + 'value', + preset.prompt + ); + } catch {} + formikProps.setFieldTouched( + 'value', + true + ); + setPresetsPopoverOpen(false); + }, + }) + ), + }, + ]} + /> + + + + + + + + formikProps.setFieldValue('value', value) + } + onBlur={(e) => { + formikProps.setFieldTouched('value'); + }} + /> + + + + Input variables + + + + + + {`Name`} + + + + + {`Expression`} + + + + + {formikProps.values.nestedVars?.map( + (expressionVar, idx) => { + return ( +
+ + + + + + + + + + + + {(copy) => ( + + )} + + + + { + deleteInputVar( + formikProps.values.nestedVars || [], + idx + ); + }} + /> + + + + + +
+ ); + } + )} + { + addInputVar(formikProps.values.nestedVars || []); + }} + > + {`Add variable`} + + + + + + + + + + Prompt preview + + + { + setIsFetching(true); + switch (props.context) { + case PROCESSOR_CONTEXT.INGEST: { + // get the current ingest pipeline up to, but not including, this processor + const curIngestPipeline = formikToPartialPipeline( + values, + props.uiConfig, + props.config.id, + false, + PROCESSOR_CONTEXT.INGEST + ); + // if there are preceding processors, we need to simulate the partial ingest pipeline, + // in order to get the latest transformed version of the docs + if (curIngestPipeline !== undefined) { + const curDocs = prepareDocsForSimulate( + values.ingest.docs, + values.ingest.index.name + ); + await dispatch( + simulatePipeline({ + apiBody: { + pipeline: curIngestPipeline as IngestPipelineConfig, + docs: [curDocs[0]], + }, + dataSourceId, + }) + ) + .unwrap() + .then( + ( + resp: SimulateIngestPipelineResponse + ) => { + const docObjs = unwrapTransformedDocs( + resp + ); + if (docObjs.length > 0) { + setSourceInput( + customStringify(docObjs[0]) + ); + } + } + ) + .catch((error: any) => { + getCore().notifications.toasts.addDanger( + `Failed to fetch input data` + ); + }) + .finally(() => { + setIsFetching(false); + }); + } else { + try { + const docObjs = JSON.parse( + values.ingest.docs + ) as {}[]; + if (docObjs.length > 0) { + setSourceInput( + customStringify(docObjs[0]) + ); + } + } catch { + } finally { + setIsFetching(false); + } + } + break; + } + case PROCESSOR_CONTEXT.SEARCH_REQUEST: { + // get the current search pipeline up to, but not including, this processor + const curSearchPipeline = formikToPartialPipeline( + values, + props.uiConfig, + props.config.id, + false, + PROCESSOR_CONTEXT.SEARCH_REQUEST + ); + // if there are preceding processors, we cannot generate. The button to render + // this modal should be disabled if the search pipeline would be enabled. We add + // this if check as an extra layer of checking, and if mechanism for gating + // this is changed in the future. + if (curSearchPipeline === undefined) { + setSourceInput(values.search.request); + } + setIsFetching(false); + break; + } + case PROCESSOR_CONTEXT.SEARCH_RESPONSE: { + // get the current search pipeline up to, but not including, this processor + const curSearchPipeline = formikToPartialPipeline( + values, + props.uiConfig, + props.config.id, + false, + PROCESSOR_CONTEXT.SEARCH_RESPONSE + ); + // Execute search. If there are preceding processors, augment the existing query with + // the partial search pipeline (inline) to get the latest transformed version of the response. + dispatch( + searchIndex({ + apiBody: { + index: values.search.index.name, + body: JSON.stringify({ + ...JSON.parse( + values.search.request as string + ), + search_pipeline: + curSearchPipeline || {}, + }), + }, + dataSourceId, + }) + ) + .unwrap() + .then(async (resp) => { + const hits = resp?.hits?.hits + ?.map((hit: SearchHit) => hit._source) + .slice(0, MAX_INPUT_DOCS); + if (hits.length > 0) { + setSourceInput( + // if one-to-one, treat the source input as a single retrieved document + // else, treat it as all of the returned documents + customStringify( + oneToOne ? hits[0] : hits + ) + ); + } + }) + .catch((error: any) => { + getCore().notifications.toasts.addDanger( + `Failed to fetch source input data` + ); + }) + .finally(() => { + setIsFetching(false); + }); + break; + } + } + }} + > + Run preview + + + + + + Source data + + + + + + Prompt + + + + + + + + + + + Cancel + + { + formikProps + .submitForm() + .then((value: any) => { + onUpdate(); + }) + .catch((err: any) => {}); + }} + isLoading={isUpdating} + isDisabled={tempErrors} // blocking update until valid input is given + fill={true} + color="primary" + data-testid="updateTemplateButton" + > + Save + + + + ); + }} + + ); +} + +// small util fn to get the full placeholder string to be +// inserted into the template. String conversion is required +// if the input is an array, so default to toString() for +// all types. Also, all values +// should be prepended with "parameters.", as all inputs +// will be nested under a base parameters obj. +function getPlaceholderString(label: string) { + return `\$\{parameters.${label}.toString()\}`; +} + +function injectValuesIntoTemplate( + template: string, + parameters: { [key: string]: string } +): string { + let finalTemplate = template; + // replace any parameter placeholders in the prompt with any values found in the + // parameters obj. + // we do 2 checks - one for the regular prompt, and one with "toString()" appended. + // this is required for parameters that have values as a list, for example. + Object.keys(parameters).forEach((parameterKey) => { + const parameterValue = parameters[parameterKey]; + const regex = new RegExp(`\\$\\{parameters.${parameterKey}\\}`, 'g'); + const regexWithToString = new RegExp( + `\\$\\{parameters.${parameterKey}.toString\\(\\)\\}`, + 'g' + ); + finalTemplate = finalTemplate + .replace(regex, parameterValue) + .replace(regexWithToString, parameterValue); + }); + + return finalTemplate; +} diff --git a/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs/modals/index.ts b/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs/modals/index.ts index 6874447e..c6c153c2 100644 --- a/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs/modals/index.ts +++ b/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs/modals/index.ts @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -export * from './input_transform_modal'; -export * from './output_transform_modal'; -export * from './configure_prompt_modal'; export * from './override_query_modal'; +export * from './configure_template_modal'; +export * from './configure_expression_modal'; +export * from './configure_multi_expression_modal'; diff --git a/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs/modals/input_transform_modal.tsx b/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs/modals/input_transform_modal.tsx deleted file mode 100644 index a40015b3..00000000 --- a/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs/modals/input_transform_modal.tsx +++ /dev/null @@ -1,836 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -import React, { useState, useEffect } from 'react'; -import { useFormikContext, getIn, Formik } from 'formik'; -import { isEmpty } from 'lodash'; -import Ajv from 'ajv'; -import * as yup from 'yup'; -import { - EuiCodeEditor, - EuiFlexGroup, - EuiFlexItem, - EuiModal, - EuiModalBody, - EuiModalFooter, - EuiModalHeader, - EuiModalHeaderTitle, - EuiCompressedSelect, - EuiSelectOption, - EuiSmallButton, - EuiSpacer, - EuiText, - EuiPopover, - EuiSmallButtonEmpty, - EuiCodeBlock, - EuiPopoverTitle, - EuiIconTip, - EuiCompressedSwitch, - EuiCallOut, - EuiAccordion, -} from '@elastic/eui'; -import { - IConfigField, - IProcessorConfig, - IngestPipelineConfig, - InputTransformFormValues, - InputTransformSchema, - MapArrayFormValue, - ModelInterface, - PROCESSOR_CONTEXT, - SearchHit, - SimulateIngestPipelineResponse, - TRANSFORM_CONTEXT, - WorkflowConfig, - WorkflowFormValues, - customStringify, - REQUEST_PREFIX, - REQUEST_PREFIX_WITH_JSONPATH_ROOT_SELECTOR, -} from '../../../../../../../common'; -import { - formikToPartialPipeline, - generateTransform, - getFieldSchema, - getInitialValue, - prepareDocsForSimulate, - unwrapTransformedDocs, -} from '../../../../../../utils'; -import { - searchIndex, - simulatePipeline, - useAppDispatch, -} from '../../../../../../store'; -import { getCore } from '../../../../../../services'; -import { - generateArrayTransform, - getDataSourceId, - parseModelInputs, - parseModelInputsObj, -} from '../../../../../../utils/utils'; -import { BooleanField, MapArrayField } from '../../../input_fields'; - -interface InputTransformModalProps { - uiConfig: WorkflowConfig; - config: IProcessorConfig; - baseConfigPath: string; - context: PROCESSOR_CONTEXT; - inputMapFieldPath: string; - modelInterface: ModelInterface | undefined; - valueOptions: { label: string }[]; - onClose: () => void; -} - -// the max number of input docs we use to display & test transforms with (search response hits) -const MAX_INPUT_DOCS = 10; - -/** - * A modal to configure advanced JSON-to-JSON transforms into a model's expected input - */ -export function InputTransformModal(props: InputTransformModalProps) { - const dispatch = useAppDispatch(); - const dataSourceId = getDataSourceId(); - const { values, setFieldValue, setFieldTouched } = useFormikContext< - WorkflowFormValues - >(); - - // sub-form values/schema - const inputTransformFormValues = { - input_map: getInitialValue('mapArray'), - one_to_one: getInitialValue('boolean'), - } as InputTransformFormValues; - const inputTransformFormSchema = yup.object({ - input_map: getFieldSchema({ - type: 'mapArray', - } as IConfigField), - one_to_one: getFieldSchema( - { - type: 'boolean', - } as IConfigField, - true - ), - }) as InputTransformSchema; - - // persist standalone values. update / initialize when it is first opened - const [tempErrors, setTempErrors] = useState(false); - const [tempOneToOne, setTempOneToOne] = useState(false); - const [tempInputMap, setTempInputMap] = useState([]); - - // various prompt states - const [viewPromptDetails, setViewPromptDetails] = useState(false); - const [viewTransformedPrompt, setViewTransformedPrompt] = useState( - false - ); - const [originalPrompt, setOriginalPrompt] = useState(''); - const [transformedPrompt, setTransformedPrompt] = useState(''); - - // fetching input data state - const [isFetching, setIsFetching] = useState(false); - - // source input / transformed input state - const [sourceInput, setSourceInput] = useState('{}'); - const [transformedInput, setTransformedInput] = useState('{}'); - - // get some current form values - const oneToOnePath = `${props.baseConfigPath}.${props.config.id}.one_to_one`; - const docs = getIn(values, 'ingest.docs'); - let docObjs = [] as {}[] | undefined; - try { - docObjs = JSON.parse(docs); - } catch {} - const query = getIn(values, 'search.request'); - let queryObj = {} as {} | undefined; - try { - queryObj = JSON.parse(query); - } catch {} - const onIngestAndNoDocs = - props.context === PROCESSOR_CONTEXT.INGEST && isEmpty(docObjs); - const onSearchAndNoQuery = - (props.context === PROCESSOR_CONTEXT.SEARCH_REQUEST || - props.context === PROCESSOR_CONTEXT.SEARCH_RESPONSE) && - isEmpty(queryObj); - - // selected transform state - const transformOptions = tempInputMap.map((_, idx) => ({ - value: idx, - text: `Prediction ${idx + 1}`, - })) as EuiSelectOption[]; - const [selectedTransformOption, setSelectedTransformOption] = useState< - number - >((transformOptions[0]?.value as number) ?? 0); - - // popover state containing the model interface details, if applicable - const [popoverOpen, setPopoverOpen] = useState(false); - - // validation state utilizing the model interface, if applicable. undefined if - // there is no model interface and/or no source input - const [isValid, setIsValid] = useState(undefined); - - const description = - props.context === PROCESSOR_CONTEXT.SEARCH_REQUEST - ? 'Fetch an input query and see how it is transformed.' - : props.context === PROCESSOR_CONTEXT.INGEST - ? 'Fetch a sample document and see how it is transformed' - : `Fetch some returned documents (up to ${MAX_INPUT_DOCS}) and see how they are transformed.`; - - // hook to re-generate the transform when any inputs to the transform are updated - useEffect(() => { - if (!isEmpty(tempInputMap) && !isEmpty(JSON.parse(sourceInput))) { - let sampleSourceInput = {} as {} | []; - try { - sampleSourceInput = JSON.parse(sourceInput); - const output = - // Edge case: users are collapsing input docs into a single input field when many-to-one is selected - // fo input transforms on search response processors. - tempOneToOne === false && - props.context === PROCESSOR_CONTEXT.SEARCH_RESPONSE && - Array.isArray(sampleSourceInput) - ? generateArrayTransform( - sampleSourceInput as [], - tempInputMap[selectedTransformOption], - props.context, - TRANSFORM_CONTEXT.INPUT, - queryObj - ) - : generateTransform( - sampleSourceInput, - tempInputMap[selectedTransformOption], - props.context, - TRANSFORM_CONTEXT.INPUT, - queryObj - ); - - setTransformedInput(customStringify(output)); - } catch {} - } else { - setTransformedInput('{}'); - } - }, [tempInputMap, sourceInput, selectedTransformOption]); - - // hook to re-determine validity when the generated output changes - // utilize Ajv JSON schema validator library. For more info/examples, see - // https://www.npmjs.com/package/ajv - useEffect(() => { - if ( - !isEmpty(JSON.parse(sourceInput)) && - !isEmpty(props.modelInterface?.input?.properties?.parameters) - ) { - const validateFn = new Ajv().compile( - props.modelInterface?.input?.properties?.parameters || {} - ); - setIsValid(validateFn(JSON.parse(transformedInput))); - } else { - setIsValid(undefined); - } - }, [transformedInput]); - - // hook to set the prompt if found in the model config - useEffect(() => { - const modelConfigString = getIn( - values, - `${props.baseConfigPath}.${props.config.id}.model_config` - ); - try { - const prompt = JSON.parse(modelConfigString)?.prompt; - if (!isEmpty(prompt)) { - setOriginalPrompt(prompt); - } - } catch {} - }, [ - getIn(values, `${props.baseConfigPath}.${props.config.id}.model_config`), - ]); - - // hook to set the transformed prompt, if a valid prompt found, and - // valid parameters set - useEffect(() => { - const transformedInputObj = JSON.parse(transformedInput); - if (!isEmpty(originalPrompt) && !isEmpty(transformedInputObj)) { - setTransformedPrompt( - injectValuesIntoPrompt(originalPrompt, transformedInputObj) - ); - setViewPromptDetails(true); - setViewTransformedPrompt(true); - } else { - setViewPromptDetails(false); - setViewTransformedPrompt(false); - setTransformedPrompt(originalPrompt); - } - }, [originalPrompt, transformedInput]); - - // hook to clear the source input when one_to_one is toggled - useEffect(() => { - setSourceInput('{}'); - }, [tempOneToOne]); - - return ( - {}} - validate={(values) => {}} - > - {(formikProps) => { - // override to parent form values when changes detected - useEffect(() => { - formikProps.setFieldValue( - 'input_map', - getIn(values, props.inputMapFieldPath) - ); - }, [getIn(values, props.inputMapFieldPath)]); - useEffect(() => { - formikProps.setFieldValue('one_to_one', getIn(values, oneToOnePath)); - }, [getIn(values, oneToOnePath)]); - - // update temp vars when form changes are detected - useEffect(() => { - setTempInputMap(getIn(formikProps.values, 'input_map')); - }, [getIn(formikProps.values, 'input_map')]); - useEffect(() => { - setTempOneToOne(getIn(formikProps.values, 'one_to_one')); - }, [getIn(formikProps.values, 'one_to_one')]); - - // update tempErrors if errors detected - useEffect(() => { - setTempErrors(!isEmpty(formikProps.errors)); - }, [formikProps.errors]); - - const InputMap = ( - { - if (isEmpty(curArray)) { - setSelectedTransformOption(0); - } - }} - // If the map we are deleting is the one we last used to test, reset the state and - // default to the first map in the list. - onMapDelete={(idxToDelete) => { - if (selectedTransformOption === idxToDelete) { - setSelectedTransformOption(0); - setTransformedInput('{}'); - } - }} - addMapEntryButtonText="Add input" - addMapButtonText="Add input group (Advanced)" - mappingDirection="sortLeft" - /> - ); - - const OneToOneConfig = ( - - ); - - const FetchButton = ( - { - setIsFetching(true); - switch (props.context) { - case PROCESSOR_CONTEXT.INGEST: { - // get the current ingest pipeline up to, but not including, this processor - const curIngestPipeline = formikToPartialPipeline( - values, - props.uiConfig, - props.config.id, - false, - PROCESSOR_CONTEXT.INGEST - ); - // if there are preceding processors, we need to simulate the partial ingest pipeline, - // in order to get the latest transformed version of the docs - if (curIngestPipeline !== undefined) { - const curDocs = prepareDocsForSimulate( - values.ingest.docs, - values.ingest.index.name - ); - await dispatch( - simulatePipeline({ - apiBody: { - pipeline: curIngestPipeline as IngestPipelineConfig, - docs: [curDocs[0]], - }, - dataSourceId, - }) - ) - .unwrap() - .then((resp: SimulateIngestPipelineResponse) => { - const docObjs = unwrapTransformedDocs(resp); - if (docObjs.length > 0) { - setSourceInput(customStringify(docObjs[0])); - } - }) - .catch((error: any) => { - getCore().notifications.toasts.addDanger( - `Failed to fetch input data` - ); - }) - .finally(() => { - setIsFetching(false); - }); - } else { - try { - const docObjs = JSON.parse(values.ingest.docs) as {}[]; - if (docObjs.length > 0) { - setSourceInput(customStringify(docObjs[0])); - } - } catch { - } finally { - setIsFetching(false); - } - } - break; - } - case PROCESSOR_CONTEXT.SEARCH_REQUEST: { - // get the current search pipeline up to, but not including, this processor - const curSearchPipeline = formikToPartialPipeline( - values, - props.uiConfig, - props.config.id, - false, - PROCESSOR_CONTEXT.SEARCH_REQUEST - ); - // if there are preceding processors, we cannot generate. The button to render - // this modal should be disabled if the search pipeline would be enabled. We add - // this if check as an extra layer of checking, and if mechanism for gating - // this is changed in the future. - if (curSearchPipeline === undefined) { - setSourceInput(values.search.request); - } - setIsFetching(false); - break; - } - case PROCESSOR_CONTEXT.SEARCH_RESPONSE: { - // get the current search pipeline up to, but not including, this processor - const curSearchPipeline = formikToPartialPipeline( - values, - props.uiConfig, - props.config.id, - false, - PROCESSOR_CONTEXT.SEARCH_RESPONSE - ); - // Execute search. If there are preceding processors, augment the existing query with - // the partial search pipeline (inline) to get the latest transformed version of the response. - dispatch( - searchIndex({ - apiBody: { - index: values.search.index.name, - body: JSON.stringify({ - ...JSON.parse(values.search.request as string), - search_pipeline: curSearchPipeline || {}, - }), - }, - dataSourceId, - }) - ) - .unwrap() - .then(async (resp) => { - const hits = resp?.hits?.hits - ?.map((hit: SearchHit) => hit._source) - .slice(0, MAX_INPUT_DOCS); - if (hits.length > 0) { - setSourceInput( - // if one-to-one, treat the source input as a single retrieved document - // else, treat it as all of the returned documents - customStringify(tempOneToOne ? hits[0] : hits) - ); - } - }) - .catch((error: any) => { - getCore().notifications.toasts.addDanger( - `Failed to fetch source input data` - ); - }) - .finally(() => { - setIsFetching(false); - }); - break; - } - } - }} - > - Fetch data - - ); - - const SourceInput = ( - - ); - - const TransformedInput = ( - - ); - - return ( - - - -

{`Preview input transformation`}

- - - - - - <> - - - -

Define transform

- - - - - - - - {InputMap} - {props.context === PROCESSOR_CONTEXT.SEARCH_RESPONSE && ( - <> - - - - - {OneToOneConfig} - - - - )} - - - - - - -

Preview

- - - - - - - - <> - {(onIngestAndNoDocs || onSearchAndNoQuery) && ( - <> - - - - )} - {FetchButton} - - - - - - <> - - - - Data before transformation - - - - - {SourceInput} - - - - <> - - {isValid !== undefined && ( - - - - )} - - {transformOptions.length <= 1 ? ( - - Data after transformation - - ) : ( - - Data after transformation for - - } - options={transformOptions} - value={selectedTransformOption} - onChange={(e) => { - setSelectedTransformOption( - Number(e.target.value) - ); - }} - /> - )} - - {!isEmpty( - parseModelInputsObj(props.modelInterface) - ) && ( - - setPopoverOpen(false)} - panelPaddingSize="s" - button={ - setPopoverOpen(!popoverOpen)} - > - Input schema - - } - > - - The JSON Schema defining the model's expected - input - - - {customStringify( - parseModelInputsObj(props.modelInterface) - )} - - - - )} - - - {TransformedInput} - - - - - {!isEmpty(originalPrompt) && ( - - <> - - - Transformed prompt - - - - setViewPromptDetails(!viewPromptDetails) - } - disabled={isEmpty(JSON.parse(transformedInput))} - /> - - {isEmpty(JSON.parse(transformedInput)) && ( - - - Transformed input is empty - - - )} - - {viewPromptDetails && ( - <> - - - setViewTransformedPrompt(!viewTransformedPrompt) - } - /> - - - - )} - - - )} - - - - - Cancel - - { - // update the parent form values - if (props.context === PROCESSOR_CONTEXT.SEARCH_RESPONSE) { - setFieldValue( - oneToOnePath, - getIn(formikProps.values, 'one_to_one') - ); - setFieldTouched(oneToOnePath, true); - } - setFieldValue( - props.inputMapFieldPath, - getIn(formikProps.values, 'input_map') - ); - setFieldTouched(props.inputMapFieldPath, true); - props.onClose(); - }} - isDisabled={tempErrors} // blocking update until valid input is given - fill={true} - color="primary" - data-testid="updateInputTransformModalButton" - > - Save - - - - ); - }} - - ); -} - -function injectValuesIntoPrompt( - promptString: string, - parameters: { [key: string]: string } -): string { - let finalPromptString = promptString; - // replace any parameter placeholders in the prompt with any values found in the - // parameters obj. - // we do 2 checks - one for the regular prompt, and one with "toString()" appended. - // this is required for parameters that have values as a list, for example. - Object.keys(parameters).forEach((parameterKey) => { - const parameterValue = parameters[parameterKey]; - const regex = new RegExp(`\\$\\{parameters.${parameterKey}\\}`, 'g'); - const regexWithToString = new RegExp( - `\\$\\{parameters.${parameterKey}.toString\\(\\)\\}`, - 'g' - ); - finalPromptString = finalPromptString - .replace(regex, parameterValue) - .replace(regexWithToString, parameterValue); - }); - - return finalPromptString; -} diff --git a/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs/modals/output_transform_modal.tsx b/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs/modals/output_transform_modal.tsx deleted file mode 100644 index 9d1dc7a4..00000000 --- a/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs/modals/output_transform_modal.tsx +++ /dev/null @@ -1,656 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -import React, { useState, useEffect } from 'react'; -import { useFormikContext, getIn, Formik } from 'formik'; -import { cloneDeep, isEmpty, set } from 'lodash'; -import * as yup from 'yup'; -import { - EuiCodeEditor, - EuiFlexGroup, - EuiFlexItem, - EuiModal, - EuiModalBody, - EuiModalFooter, - EuiModalHeader, - EuiModalHeaderTitle, - EuiCompressedSelect, - EuiSelectOption, - EuiSmallButton, - EuiSpacer, - EuiText, - EuiPopover, - EuiSmallButtonEmpty, - EuiPopoverTitle, - EuiCodeBlock, - EuiCallOut, - EuiIconTip, - EuiAccordion, -} from '@elastic/eui'; -import { - IConfigField, - IProcessorConfig, - IngestPipelineConfig, - MapArrayFormValue, - MapFormValue, - ModelInterface, - OutputTransformFormValues, - OutputTransformSchema, - PROCESSOR_CONTEXT, - SearchHit, - SearchPipelineConfig, - SimulateIngestPipelineResponse, - TRANSFORM_CONTEXT, - WorkflowConfig, - WorkflowFormValues, - customStringify, -} from '../../../../../../../common'; -import { - formikToPartialPipeline, - generateTransform, - getFieldSchema, - getInitialValue, - prepareDocsForSimulate, - unwrapTransformedDocs, -} from '../../../../../../utils'; -import { - searchIndex, - simulatePipeline, - useAppDispatch, -} from '../../../../../../store'; -import { getCore } from '../../../../../../services'; -import { BooleanField, MapArrayField } from '../../../input_fields'; -import { - getDataSourceId, - parseModelOutputs, - parseModelOutputsObj, -} from '../../../../../../utils/utils'; - -interface OutputTransformModalProps { - uiConfig: WorkflowConfig; - config: IProcessorConfig; - baseConfigPath: string; - context: PROCESSOR_CONTEXT; - outputMapFieldPath: string; - modelInterface: ModelInterface | undefined; - onClose: () => void; -} - -/** - * A modal to configure advanced JSON-to-JSON transforms from a model's expected output - */ -export function OutputTransformModal(props: OutputTransformModalProps) { - const dispatch = useAppDispatch(); - const dataSourceId = getDataSourceId(); - const { values, setFieldValue, setFieldTouched } = useFormikContext< - WorkflowFormValues - >(); - - // sub-form values/schema - const outputTransformFormValues = { - output_map: getInitialValue('mapArray'), - full_response_path: getInitialValue('boolean'), - } as OutputTransformFormValues; - const outputTransformFormSchema = yup.object({ - output_map: getFieldSchema({ - type: 'mapArray', - } as IConfigField), - full_response_path: getFieldSchema( - { - type: 'boolean', - } as IConfigField, - true - ), - }) as OutputTransformSchema; - - // persist standalone values. update / initialize when it is first opened - const [tempErrors, setTempErrors] = useState(false); - const [tempFullResponsePath, setTempFullResponsePath] = useState( - false - ); - const [tempOutputMap, setTempOutputMap] = useState([]); - - // fetching input data state - const [isFetching, setIsFetching] = useState(false); - - // source output / transformed output state - const [sourceOutput, setSourceOutput] = useState('{}'); - const [transformedOutput, setTransformedOutput] = useState('{}'); - - // get some current form values - const fullResponsePathPath = `${props.baseConfigPath}.${props.config.id}.full_response_path`; - const docs = getIn(values, 'ingest.docs'); - let docObjs = [] as {}[] | undefined; - try { - docObjs = JSON.parse(docs); - } catch {} - const query = getIn(values, 'search.request'); - let queryObj = {} as {} | undefined; - try { - queryObj = JSON.parse(query); - } catch {} - const onIngestAndNoDocs = - props.context === PROCESSOR_CONTEXT.INGEST && isEmpty(docObjs); - const onSearchAndNoQuery = - (props.context === PROCESSOR_CONTEXT.SEARCH_REQUEST || - props.context === PROCESSOR_CONTEXT.SEARCH_RESPONSE) && - isEmpty(queryObj); - - // popover state containing the model interface details, if applicable - const [popoverOpen, setPopoverOpen] = useState(false); - - // selected transform state - const transformOptions = tempOutputMap.map((_, idx) => ({ - value: idx, - text: `Prediction ${idx + 1}`, - })) as EuiSelectOption[]; - const [selectedTransformOption, setSelectedTransformOption] = useState< - number - >((transformOptions[0]?.value as number) ?? 0); - - // hook to re-generate the transform when any inputs to the transform are updated - useEffect(() => { - if (!isEmpty(tempOutputMap) && !isEmpty(JSON.parse(sourceOutput))) { - let sampleSourceOutput = {}; - try { - sampleSourceOutput = JSON.parse(sourceOutput); - const output = generateTransform( - sampleSourceOutput, - reverseKeysAndValues(tempOutputMap[selectedTransformOption]), - props.context, - TRANSFORM_CONTEXT.OUTPUT - ); - setTransformedOutput(customStringify(output)); - } catch {} - } else { - setTransformedOutput('{}'); - } - }, [tempOutputMap, sourceOutput, selectedTransformOption]); - - // hook to clear the source output when full_response_path is toggled - useEffect(() => { - setSourceOutput('{}'); - }, [tempFullResponsePath]); - - return ( - {}} - validate={(values) => {}} - > - {(formikProps) => { - // override to parent form values when changes detected - useEffect(() => { - formikProps.setFieldValue( - 'output_map', - getIn(values, props.outputMapFieldPath) - ); - }, [getIn(values, props.outputMapFieldPath)]); - useEffect(() => { - formikProps.setFieldValue( - 'full_response_path', - getIn(values, fullResponsePathPath) - ); - }, [getIn(values, fullResponsePathPath)]); - - // update temp vars when form changes are detected - useEffect(() => { - setTempOutputMap(getIn(formikProps.values, 'output_map')); - }, [getIn(formikProps.values, 'output_map')]); - useEffect(() => { - setTempFullResponsePath( - getIn(formikProps.values, 'full_response_path') - ); - }, [getIn(formikProps.values, 'full_response_path')]); - - // update tempErrors if errors detected - useEffect(() => { - setTempErrors(!isEmpty(formikProps.errors)); - }, [formikProps.errors]); - - const OutputMap = ( - { - if (isEmpty(curArray)) { - setSelectedTransformOption(0); - } - }} - // If the map we are deleting is the one we last used to test, reset the state and - // default to the first map in the list. - onMapDelete={(idxToDelete) => { - if (selectedTransformOption === idxToDelete) { - setSelectedTransformOption(0); - setTransformedOutput('{}'); - } - }} - addMapEntryButtonText="Add output" - addMapButtonText="Add output group (Advanced)" - mappingDirection="sortRight" - /> - ); - - const FullResponsePathConfig = ( - - ); - - const FetchButton = ( - { - setIsFetching(true); - switch (props.context) { - // note we skip search request processor context. that is because empty output maps are not supported. - // for more details, see comment in ml_processor_inputs.tsx - case PROCESSOR_CONTEXT.INGEST: { - // get the current ingest pipeline up to, and including this processor. - // remove any currently-configured output map since we only want the transformation - // up to, and including, the input map transformations - const valuesWithoutOutputMapConfig = cloneDeep(values); - set( - valuesWithoutOutputMapConfig, - props.outputMapFieldPath, - [] - ); - set( - valuesWithoutOutputMapConfig, - fullResponsePathPath, - getIn(formikProps.values, 'full_response_path') - ); - const curIngestPipeline = formikToPartialPipeline( - valuesWithoutOutputMapConfig, - props.uiConfig, - props.config.id, - true, - PROCESSOR_CONTEXT.INGEST - ) as IngestPipelineConfig; - const curDocs = prepareDocsForSimulate( - values.ingest.docs, - values.ingest.index.name - ); - await dispatch( - simulatePipeline({ - apiBody: { - pipeline: curIngestPipeline, - docs: [curDocs[0]], - }, - dataSourceId, - }) - ) - .unwrap() - .then((resp: SimulateIngestPipelineResponse) => { - try { - const docObjs = unwrapTransformedDocs(resp); - if (docObjs.length > 0) { - const sampleModelResult = - docObjs[0]?.inference_results || {}; - setSourceOutput(customStringify(sampleModelResult)); - } - } catch {} - }) - .catch((error: any) => { - getCore().notifications.toasts.addDanger( - `Failed to fetch input data` - ); - }) - .finally(() => { - setIsFetching(false); - }); - break; - } - case PROCESSOR_CONTEXT.SEARCH_RESPONSE: { - // get the current search pipeline up to, and including this processor. - // remove any currently-configured output map since we only want the transformation - // up to, and including, the input map transformations - const valuesWithoutOutputMapConfig = cloneDeep(values); - set( - valuesWithoutOutputMapConfig, - props.outputMapFieldPath, - [] - ); - set( - valuesWithoutOutputMapConfig, - fullResponsePathPath, - getIn(formikProps.values, 'full_response_path') - ); - const curSearchPipeline = formikToPartialPipeline( - valuesWithoutOutputMapConfig, - props.uiConfig, - props.config.id, - true, - PROCESSOR_CONTEXT.SEARCH_RESPONSE - ) as SearchPipelineConfig; - - // Execute search. Augment the existing query with - // the partial search pipeline (inline) to get the latest transformed - // version of the request. - dispatch( - searchIndex({ - apiBody: { - index: values.ingest.index.name, - body: JSON.stringify({ - ...JSON.parse(values.search.request as string), - search_pipeline: curSearchPipeline || {}, - }), - }, - dataSourceId, - }) - ) - .unwrap() - .then(async (resp) => { - const hits = resp?.hits?.hits?.map( - (hit: SearchHit) => hit._source - ) as any[]; - if (hits.length > 0) { - const sampleModelResult = - hits[0].inference_results || {}; - setSourceOutput(customStringify(sampleModelResult)); - } - }) - .catch((error: any) => { - getCore().notifications.toasts.addDanger( - `Failed to fetch source output data` - ); - }) - .finally(() => { - setIsFetching(false); - }); - break; - } - } - }} - > - Fetch data - - ); - - const SourceOutput = ( - - ); - - const TransformedOutput = ( - - ); - - return ( - - - -

{`Preview output transformation`}

- - - - - - <> - - - -

Define transform

- - - - - - - - {OutputMap} - {(props.context === PROCESSOR_CONTEXT.INGEST || - props.context === PROCESSOR_CONTEXT.SEARCH_RESPONSE) && ( - <> - - - - - {FullResponsePathConfig} - - - - )} - - - - - - -

Preview

- - - - - - - - <> - {(onIngestAndNoDocs || onSearchAndNoQuery) && ( - <> - - - - )} - {FetchButton} - - - - - - <> - - - - - Data before transformation - - - {!isEmpty( - parseModelOutputsObj( - props.modelInterface, - tempFullResponsePath - ) - ) && ( - - setPopoverOpen(false)} - panelPaddingSize="s" - button={ - - setPopoverOpen(!popoverOpen) - } - > - Output schema - - } - > - - The JSON Schema defining the model's - expected output - - - {customStringify( - parseModelOutputsObj( - props.modelInterface, - tempFullResponsePath - ) - )} - - - - )} - - - {SourceOutput} - - - - - <> - {transformOptions.length <= 1 ? ( - Data after transformation - ) : ( - - Data after transformation for - - } - options={transformOptions} - value={selectedTransformOption} - onChange={(e) => { - setSelectedTransformOption( - Number(e.target.value) - ); - }} - /> - )} - - - - {TransformedOutput} - - - - - - - - - Cancel - - { - // update the parent form values - setFieldValue( - fullResponsePathPath, - getIn(formikProps.values, 'full_response_path') - ); - setFieldTouched(fullResponsePathPath, true); - - setFieldValue( - props.outputMapFieldPath, - getIn(formikProps.values, 'output_map') - ); - setFieldTouched(props.outputMapFieldPath, true); - props.onClose(); - }} - isDisabled={tempErrors} // blocking update until valid input is given - fill={true} - color="primary" - data-testid="updateOutputTransformModalButton" - > - Save - - - - ); - }} - - ); -} - -// since we persist the form keys/values reversed, we have a util fn to reverse back, so we can use -// single util fns for manipulation of the form values (generating transforms, etc.) -function reverseKeysAndValues(values: MapFormValue): MapFormValue { - return values.map((mapEntry) => { - return { - key: mapEntry.value, - value: mapEntry.key, - }; - }); -} diff --git a/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs/modals/override_query_modal.tsx b/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs/modals/override_query_modal.tsx index d4257556..430e8063 100644 --- a/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs/modals/override_query_modal.tsx +++ b/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs/modals/override_query_modal.tsx @@ -28,9 +28,9 @@ import { IMAGE_FIELD_PATTERN, IProcessorConfig, LABEL_FIELD_PATTERN, - MapEntry, MODEL_ID_PATTERN, ModelInterface, + OutputMapEntry, QUERY_IMAGE_PATTERN, QUERY_PRESETS, QUERY_TEXT_PATTERN, @@ -66,10 +66,8 @@ export function OverrideQueryModal(props: OverrideQueryModalProps) { values, `${props.baseConfigPath}.${props.config.id}.output_map` ); - // TODO: should handle edge case of multiple output maps configured. Currently - // defaulting to prediction 0 / assuming not multiple predictions to track. const outputMapValues = getIn(outputMap, '0', []).map( - (mapEntry: MapEntry) => mapEntry.value + (mapEntry: OutputMapEntry) => mapEntry.value.value ) as string[]; const finalModelOutputs = outputMapValues.length > 0 diff --git a/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs/model_inputs.tsx b/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs/model_inputs.tsx index 4d18aa44..85f1fee4 100644 --- a/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs/model_inputs.tsx +++ b/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs/model_inputs.tsx @@ -4,10 +4,21 @@ */ import React, { useState, useEffect } from 'react'; -import { useFormikContext, getIn } from 'formik'; +import { useFormikContext, getIn, Field, FieldProps } from 'formik'; import { isEmpty } from 'lodash'; import { useSelector } from 'react-redux'; import { flattie } from 'flattie'; +import { + EuiCompressedFormRow, + EuiFlexGroup, + EuiFlexItem, + EuiIcon, + EuiPanel, + EuiSmallButton, + EuiSmallButtonEmpty, + EuiSmallButtonIcon, + EuiText, +} from '@elastic/eui'; import { IProcessorConfig, IConfigField, @@ -15,23 +26,41 @@ import { WorkflowFormValues, ModelInterface, IndexMappings, - REQUEST_PREFIX, - REQUEST_PREFIX_WITH_JSONPATH_ROOT_SELECTOR, + InputMapEntry, + InputMapFormValue, + TRANSFORM_TYPE, + EMPTY_INPUT_MAP_ENTRY, + WorkflowConfig, + getCharacterLimitedString, } from '../../../../../../common'; -import { MapArrayField } from '../../input_fields'; +import { TextField, SelectWithCustomOptions } from '../../input_fields'; import { AppState, getMappings, useAppDispatch } from '../../../../../store'; import { getDataSourceId, parseModelInputs, sanitizeJSONPath, } from '../../../../../utils'; +import { ConfigureExpressionModal, ConfigureTemplateModal } from './modals/'; interface ModelInputsProps { config: IProcessorConfig; baseConfigPath: string; + uiConfig: WorkflowConfig; context: PROCESSOR_CONTEXT; + isDataFetchingAvailable: boolean; } +// Spacing between the input field columns +const KEY_FLEX_RATIO = 3; +const TYPE_FLEX_RATIO = 3; +const VALUE_FLEX_RATIO = 4; + +const TRANSFORM_OPTIONS = Object.values(TRANSFORM_TYPE).map((type) => { + return { + label: type, + }; +}); + /** * Base component to configure ML inputs. */ @@ -40,20 +69,34 @@ export function ModelInputs(props: ModelInputsProps) { const dataSourceId = getDataSourceId(); const { models } = useSelector((state: AppState) => state.ml); const indices = useSelector((state: AppState) => state.opensearch.indices); - const { values } = useFormikContext(); - + const { + setFieldValue, + setFieldTouched, + errors, + touched, + values, + } = useFormikContext(); // get some current form & config values const modelField = props.config.fields.find( (field) => field.type === 'model' ) as IConfigField; const modelFieldPath = `${props.baseConfigPath}.${props.config.id}.${modelField.id}`; - const inputMapFieldPath = `${props.baseConfigPath}.${props.config.id}.input_map`; + // Assuming no more than one set of input map entries. + const inputMapFieldPath = `${props.baseConfigPath}.${props.config.id}.input_map.0`; // model interface state const [modelInterface, setModelInterface] = useState< ModelInterface | undefined >(undefined); + // various modal states + const [templateModalIdx, setTemplateModalIdx] = useState( + undefined + ); + const [expressionModalIdx, setExpressionModalIdx] = useState< + number | undefined + >(undefined); + // on initial load of the models, update model interface states useEffect(() => { if (!isEmpty(models)) { @@ -90,6 +133,8 @@ export function ModelInputs(props: ModelInputsProps) { }; }) ); + } else { + setDocFields([]); } } catch {} }, [values?.ingest?.docs]); @@ -115,6 +160,7 @@ export function ModelInputs(props: ModelInputsProps) { } } catch {} }, [values?.search?.request]); + useEffect(() => { const indexName = values?.search?.index?.name as string | undefined; if (indexName !== undefined && indices[indexName] !== undefined) { @@ -141,45 +187,418 @@ export function ModelInputs(props: ModelInputsProps) { } }, [values?.search?.index?.name]); + // Adding a map entry to the end of the existing arr + function addMapEntry(curEntries: InputMapFormValue): void { + const updatedEntries = [ + ...curEntries, + { + key: '', + value: { + transformType: '' as TRANSFORM_TYPE, + value: '', + }, + } as InputMapEntry, + ]; + setFieldValue(inputMapFieldPath, updatedEntries); + setFieldTouched(inputMapFieldPath, true); + } + + // Deleting a map entry + function deleteMapEntry( + curEntries: InputMapFormValue, + entryIndexToDelete: number + ): void { + const updatedEntries = [...curEntries]; + updatedEntries.splice(entryIndexToDelete, 1); + setFieldValue(inputMapFieldPath, updatedEntries); + setFieldTouched(inputMapFieldPath, true); + } + + // Defining constants for the key/value text vars, typically dependent on the different processor contexts. + const keyOptions = parseModelInputs(modelInterface); + const valueOptions = + props.context === PROCESSOR_CONTEXT.INGEST + ? docFields + : props.context === PROCESSOR_CONTEXT.SEARCH_REQUEST + ? queryFields + : indexMappingFields; + return ( - + + {({ field, form }: FieldProps) => { + const populatedMap = field.value?.length !== 0; + return ( + <> + {populatedMap ? ( + + 0 + ? 'Invalid or missing mapping values' + : false + } + isInvalid={ + getIn(errors, field.name) !== undefined && + getIn(errors, field.name).length > 0 && + getIn(touched, field.name) !== undefined && + getIn(touched, field.name).length > 0 + } + > + + + + + + + + {`Name`} + + + + + + + + + {`Input type`} + + + + + + + + + Value + + + + + + + {field.value?.map( + (mapEntry: InputMapEntry, idx: number) => { + const transformType = getIn( + values, + `${inputMapFieldPath}.${idx}.value.transformType` + ); + return ( + + + + + <> + + <> + {/** + * We determine if there is an interface based on if there are key options or not, + * as the options would be derived from the underlying interface. + * And if so, these values should be static. + * So, we only display the static text with no mechanism to change it's value. + * Note we still allow more entries, if a user wants to override / add custom + * keys if there is some gaps in the model interface. + */} + {!isEmpty(keyOptions) && + !isEmpty( + getIn( + values, + `${inputMapFieldPath}.${idx}.key` + ) + ) ? ( + + {getIn( + values, + `${inputMapFieldPath}.${idx}.key` + )} + + ) : !isEmpty(keyOptions) ? ( + + ) : ( + + )} + + + + + + + + + + + { + // If the transform type changes, clear any set value and/or nested vars, + // as it will likely not make sense under other types/contexts. + setFieldValue( + `${inputMapFieldPath}.${idx}.value.value`, + '' + ); + setFieldValue( + `${inputMapFieldPath}.${idx}.value.nestedVars`, + [] + ); + }} + /> + + + + + <> + {/** + * Conditionally render the value form component based on the transform type. + * It may be a button, dropdown, or simply freeform text. + */} + {templateModalIdx === (idx as number) && ( + + setTemplateModalIdx(undefined) + } + /> + )} + {expressionModalIdx === (idx as number) && ( + + setExpressionModalIdx(undefined) + } + /> + )} + + <> + {transformType === + TRANSFORM_TYPE.TEMPLATE ? ( + <> + {isEmpty( + getIn( + values, + `${inputMapFieldPath}.${idx}.value.value` + ) + ) ? ( + + setTemplateModalIdx(idx) + } + data-testid="configureTemplateButton" + > + Configure + + ) : ( + + + + {getCharacterLimitedString( + getIn( + values, + `${inputMapFieldPath}.${idx}.value.value` + ), + 15 + )} + + + + + setTemplateModalIdx(idx) + } + /> + + + )} + + ) : transformType === + TRANSFORM_TYPE.EXPRESSION ? ( + <> + {isEmpty( + getIn( + values, + `${inputMapFieldPath}.${idx}.value.value` + ) + ) ? ( + + setExpressionModalIdx(idx) + } + data-testid="configureExpressionButton" + > + Configure + + ) : ( + + + + {getCharacterLimitedString( + getIn( + values, + `${inputMapFieldPath}.${idx}.value.value` + ), + 15 + )} + + + + + setExpressionModalIdx(idx) + } + /> + + + )} + + ) : isEmpty(transformType) || + transformType === + TRANSFORM_TYPE.STRING || + transformType === + TRANSFORM_TYPE.TEMPLATE || + isEmpty(valueOptions) ? ( + + ) : ( + + )} + + + + { + deleteMapEntry(field.value, idx); + }} + /> + + + + + + + ); + } + )} + +
+ { + addMapEntry(field.value); + }} + > + {`Add input`} + +
+ + + + + ) : ( + { + setFieldValue(field.name, [EMPTY_INPUT_MAP_ENTRY]); + }} + > + {'Configure'} + + )} + + ); + }} + ); } diff --git a/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs/model_outputs.tsx b/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs/model_outputs.tsx index 6d143dee..3e04286e 100644 --- a/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs/model_outputs.tsx +++ b/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs/model_outputs.tsx @@ -4,7 +4,7 @@ */ import React, { useState, useEffect } from 'react'; -import { getIn, useFormikContext } from 'formik'; +import { Field, FieldProps, getIn, useFormikContext } from 'formik'; import { isEmpty } from 'lodash'; import { useSelector } from 'react-redux'; import { @@ -13,36 +13,87 @@ import { PROCESSOR_CONTEXT, WorkflowFormValues, ModelInterface, + WorkflowConfig, + OutputMapEntry, + TRANSFORM_TYPE, + NO_TRANSFORMATION, + getCharacterLimitedString, + OutputMapFormValue, + EMPTY_OUTPUT_MAP_ENTRY, + ExpressionVar, } from '../../../../../../common'; -import { MapArrayField } from '../../input_fields'; +import { SelectWithCustomOptions, TextField } from '../../input_fields'; import { AppState } from '../../../../../store'; import { parseModelOutputs } from '../../../../../utils'; +import { + EuiCompressedFormRow, + EuiFlexGroup, + EuiFlexItem, + EuiIcon, + EuiPanel, + EuiSmallButton, + EuiSmallButtonEmpty, + EuiSmallButtonIcon, + EuiText, +} from '@elastic/eui'; +import { ConfigureMultiExpressionModal } from './modals'; interface ModelOutputsProps { config: IProcessorConfig; baseConfigPath: string; + uiConfig: WorkflowConfig; context: PROCESSOR_CONTEXT; + isDataFetchingAvailable: boolean; } +// Spacing between the output field columns +const KEY_FLEX_RATIO = 3; +const TYPE_FLEX_RATIO = 3; +const VALUE_FLEX_RATIO = 4; + +const TRANSFORM_OPTIONS = [ + { + label: TRANSFORM_TYPE.FIELD, + }, + { + label: TRANSFORM_TYPE.EXPRESSION, + }, + { + label: NO_TRANSFORMATION, + }, +]; + /** * Base component to configure ML outputs. */ export function ModelOutputs(props: ModelOutputsProps) { const { models } = useSelector((state: AppState) => state.ml); - const { values } = useFormikContext(); + const { + errors, + values, + touched, + setFieldValue, + setFieldTouched, + } = useFormikContext(); // get some current form & config values const modelField = props.config.fields.find( (field) => field.type === 'model' ) as IConfigField; const modelFieldPath = `${props.baseConfigPath}.${props.config.id}.${modelField.id}`; - const outputMapFieldPath = `${props.baseConfigPath}.${props.config.id}.output_map`; + // Assuming no more than one set of output map entries. + const outputMapFieldPath = `${props.baseConfigPath}.${props.config.id}.output_map.0`; const fullResponsePath = getIn( values, `${props.baseConfigPath}.${props.config.id}.full_response_path` ); + // various modal states + const [expressionsModalIdx, setExpressionsModalIdx] = useState< + number | undefined + >(undefined); + // model interface state const [modelInterface, setModelInterface] = useState< ModelInterface | undefined @@ -58,28 +109,353 @@ export function ModelOutputs(props: ModelOutputsProps) { } }, [models]); + // Adding a map entry to the end of the existing arr + function addMapEntry(curEntries: OutputMapFormValue): void { + const updatedEntries = [ + ...curEntries, + { + key: '', + value: { + transformType: '' as TRANSFORM_TYPE, + value: '', + }, + } as OutputMapEntry, + ]; + setFieldValue(outputMapFieldPath, updatedEntries); + setFieldTouched(outputMapFieldPath, true); + } + + // Deleting a map entry + function deleteMapEntry( + curEntries: OutputMapFormValue, + entryIndexToDelete: number + ): void { + const updatedEntries = [...curEntries]; + updatedEntries.splice(entryIndexToDelete, 1); + setFieldValue(outputMapFieldPath, updatedEntries); + setFieldTouched(outputMapFieldPath, true); + } + + const keyOptions = fullResponsePath + ? undefined + : parseModelOutputs(modelInterface, false); + return ( - + + {({ field, form }: FieldProps) => { + const populatedMap = field.value?.length !== 0; + return ( + <> + {populatedMap ? ( + + 0 + ? 'Invalid or missing mapping values' + : false + } + isInvalid={ + getIn(errors, field.name) !== undefined && + getIn(errors, field.name).length > 0 && + getIn(touched, field.name) !== undefined && + getIn(touched, field.name).length > 0 + } + > + + + + + + + + {`Name`} + + + + + + + + + {`Output transform`} + + + + + + + + + {props.context === + PROCESSOR_CONTEXT.SEARCH_REQUEST + ? 'Query field' + : 'New document field(s)'} + + + + + + + {field.value?.map( + (mapEntry: OutputMapEntry, idx: number) => { + // TODO: can I get this from mapEntry directly + const transformType = getIn( + values, + `${outputMapFieldPath}.${idx}.value.transformType` + ); + return ( + + + + + <> + + <> + {/** + * We determine if there is an interface based on if there are key options or not, + * as the options would be derived from the underlying interface. + * And if so, these values should be static. + * So, we only display the static text with no mechanism to change it's value. + * Note we still allow more entries, if a user wants to override / add custom + * keys if there is some gaps in the model interface. + */} + {!isEmpty(keyOptions) && + !isEmpty( + getIn( + values, + `${outputMapFieldPath}.${idx}.key` + ) + ) ? ( + + {getIn( + values, + `${outputMapFieldPath}.${idx}.key` + )} + + ) : !isEmpty(keyOptions) ? ( + + ) : ( + + )} + + + + + + + + + + + { + // If the transform type changes, clear any set value and/or nested vars, + // as it will likely not make sense under other types/contexts. + setFieldValue( + `${outputMapFieldPath}.${idx}.value.value`, + '' + ); + setFieldValue( + `${outputMapFieldPath}.${idx}.value.nestedVars`, + [] + ); + }} + /> + + + + + <> + {/** + * Conditionally render the value form component based on the transform type. + * It may be a button, dropdown, or simply freeform text. + */} + {expressionsModalIdx === + (idx as number) && ( + + setExpressionsModalIdx(undefined) + } + /> + )} + + <> + {transformType === + TRANSFORM_TYPE.EXPRESSION ? ( + <> + {isEmpty( + getIn( + values, + `${outputMapFieldPath}.${idx}.value.nestedVars` + ) + ) ? ( + + setExpressionsModalIdx(idx) + } + data-testid="configureExpressionsButton" + > + Configure + + ) : ( + + + + {(getIn( + values, + `${outputMapFieldPath}.${idx}.value.nestedVars` + ) as ExpressionVar[]).map( + ( + expression, + idx, + arr + ) => { + return idx < 2 + ? `${getCharacterLimitedString( + expression.transform || + '', + 15 + )}${ + idx === 0 && + arr.length > 1 + ? `,\n` + : '' + }` + : ''; + } + )} + + + + { + setExpressionsModalIdx( + idx + ); + }} + /> + + + )} + + ) : transformType === + TRANSFORM_TYPE.FIELD ? ( + + ) : undefined} + + + + { + deleteMapEntry(field.value, idx); + }} + /> + + + + + + + ); + } + )} + +
+ { + addMapEntry(field.value); + }} + > + {`Add output`} + +
+ + + + + ) : ( + { + setFieldValue(field.name, [EMPTY_OUTPUT_MAP_ENTRY]); + }} + > + {'Configure'} + + )} + + ); + }} + ); } diff --git a/public/pages/workflow_detail/workflow_inputs/search_inputs/edit_query_modal.tsx b/public/pages/workflow_detail/workflow_inputs/search_inputs/edit_query_modal.tsx index d6db7816..1f5bbc06 100644 --- a/public/pages/workflow_detail/workflow_inputs/search_inputs/edit_query_modal.tsx +++ b/public/pages/workflow_detail/workflow_inputs/search_inputs/edit_query_modal.tsx @@ -25,7 +25,6 @@ import { QUERY_PRESETS, QueryPreset, RequestFormValues, - RequestSchema, WorkflowFormValues, } from '../../../../../common'; import { getFieldSchema, getInitialValue } from '../../../../utils'; @@ -48,7 +47,7 @@ export function EditQueryModal(props: EditQueryModalProps) { request: getFieldSchema({ type: 'json', } as IConfigField), - }) as RequestSchema; + }) as yup.Schema; // persist standalone values. update / initialize when it is first opened const [tempRequest, setTempRequest] = useState('{}'); diff --git a/public/pages/workflows/new_workflow/quick_configure_modal.tsx b/public/pages/workflows/new_workflow/quick_configure_modal.tsx index 7ec123ca..daa9e58e 100644 --- a/public/pages/workflows/new_workflow/quick_configure_modal.tsx +++ b/public/pages/workflows/new_workflow/quick_configure_modal.tsx @@ -20,16 +20,21 @@ import { EuiCompressedFormRow, } from '@elastic/eui'; import { + EMPTY_INPUT_MAP_ENTRY, + EMPTY_OUTPUT_MAP_ENTRY, IMAGE_FIELD_PATTERN, IndexMappings, + InputMapArrayFormValue, + InputMapFormValue, LABEL_FIELD_PATTERN, MODEL_ID_PATTERN, - MapArrayFormValue, - MapFormValue, ModelInterface, + OutputMapArrayFormValue, + OutputMapFormValue, PROCESSOR_TYPE, QuickConfigureFields, TEXT_FIELD_PATTERN, + TRANSFORM_TYPE, VECTOR, VECTOR_FIELD_PATTERN, WORKFLOW_NAME_REGEXP, @@ -255,17 +260,23 @@ function updateIngestProcessors( field.value = { id: fields.modelId }; } if (field.id === 'input_map') { - const inputMap = generateMapFromModelInputs(modelInterface); + const inputMap = generateInputMapFromModelInputs(modelInterface); if (fields.textField) { if (inputMap.length > 0) { inputMap[0] = { ...inputMap[0], - value: fields.textField, + value: { + transformType: TRANSFORM_TYPE.FIELD, + value: fields.textField, + }, }; } else { inputMap.push({ key: '', - value: fields.textField, + value: { + transformType: TRANSFORM_TYPE.FIELD, + value: fields.textField, + }, }); } } @@ -273,19 +284,25 @@ function updateIngestProcessors( if (inputMap.length > 1) { inputMap[1] = { ...inputMap[1], - value: fields.imageField, + value: { + transformType: TRANSFORM_TYPE.FIELD, + value: fields.imageField, + }, }; } else { inputMap.push({ key: '', - value: fields.imageField, + value: { + transformType: TRANSFORM_TYPE.FIELD, + value: fields.imageField, + }, }); } } - field.value = [inputMap] as MapArrayFormValue; + field.value = [inputMap] as InputMapArrayFormValue; } if (field.id === 'output_map') { - const outputMap = generateMapFromModelOutputs(modelInterface); + const outputMap = generateOutputMapFromModelOutputs(modelInterface); const defaultField = isVectorSearchUseCase ? fields.vectorField : fields.labelField; @@ -293,13 +310,22 @@ function updateIngestProcessors( if (outputMap.length > 0) { outputMap[0] = { ...outputMap[0], - value: defaultField, + value: { + transformType: TRANSFORM_TYPE.FIELD, + value: defaultField, + }, }; } else { - outputMap.push({ key: '', value: defaultField }); + outputMap.push({ + key: '', + value: { + transformType: TRANSFORM_TYPE.FIELD, + value: defaultField, + }, + }); } } - field.value = [outputMap] as MapArrayFormValue; + field.value = [outputMap] as OutputMapArrayFormValue; } }); } @@ -328,37 +354,49 @@ function updateSearchRequestProcessors( field.value = { id: fields.modelId }; } if (field.id === 'input_map') { - const inputMap = generateMapFromModelInputs(modelInterface); + const inputMap = generateInputMapFromModelInputs(modelInterface); if (inputMap.length > 0) { inputMap[0] = { ...inputMap[0], - value: defaultQueryValue, + value: { + transformType: TRANSFORM_TYPE.FIELD, + value: defaultQueryValue, + }, }; } else { inputMap.push({ key: '', - value: defaultQueryValue, + value: { + transformType: TRANSFORM_TYPE.FIELD, + value: defaultQueryValue, + }, }); } - field.value = [inputMap] as MapArrayFormValue; + field.value = [inputMap] as InputMapArrayFormValue; } if (field.id === 'output_map') { - const outputMap = generateMapFromModelOutputs(modelInterface); + const outputMap = generateOutputMapFromModelOutputs(modelInterface); const defaultValue = isVectorSearchUseCase ? VECTOR : defaultQueryValue; if (outputMap.length > 0) { outputMap[0] = { ...outputMap[0], - value: defaultValue, + value: { + transformType: TRANSFORM_TYPE.FIELD, + value: defaultValue, + }, }; } else { outputMap.push({ key: '', - value: defaultValue, + value: { + transformType: TRANSFORM_TYPE.FIELD, + value: defaultValue, + }, }); } - field.value = [outputMap] as MapArrayFormValue; + field.value = [outputMap] as OutputMapArrayFormValue; } }); config.search.enrichRequest.processors[0].optionalFields = config.search.enrichRequest.processors[0].optionalFields?.map( @@ -392,35 +430,50 @@ function updateSearchResponseProcessors( field.value = { id: fields.modelId }; } if (field.id === 'input_map') { - const inputMap = generateMapFromModelInputs(modelInterface); + const inputMap = generateInputMapFromModelInputs(modelInterface); if (fields.textField) { if (inputMap.length > 0) { inputMap[0] = { ...inputMap[0], - value: fields.textField, + value: { + transformType: TRANSFORM_TYPE.FIELD, + value: fields.textField, + }, }; } else { inputMap.push({ key: '', - value: fields.textField, + value: { + transformType: TRANSFORM_TYPE.FIELD, + value: fields.textField, + }, }); } } - field.value = [inputMap] as MapArrayFormValue; + field.value = [inputMap] as InputMapArrayFormValue; } if (field.id === 'output_map') { - const outputMap = generateMapFromModelOutputs(modelInterface); + const outputMap = generateOutputMapFromModelOutputs(modelInterface); if (fields.llmResponseField) { if (outputMap.length > 0) { outputMap[0] = { ...outputMap[0], - value: fields.llmResponseField, + value: { + transformType: TRANSFORM_TYPE.FIELD, + value: fields.llmResponseField, + }, }; } else { - outputMap.push({ key: '', value: fields.llmResponseField }); + outputMap.push({ + key: '', + value: { + transformType: TRANSFORM_TYPE.FIELD, + value: fields.llmResponseField, + }, + }); } } - field.value = [outputMap] as MapArrayFormValue; + field.value = [outputMap] as OutputMapArrayFormValue; } }); } @@ -526,16 +579,16 @@ function injectPlaceholderValues( // generate a set of mappings s.t. each key is // a unique model input. -function generateMapFromModelInputs( +function generateInputMapFromModelInputs( modelInterface?: ModelInterface -): MapFormValue { - const inputMap = [] as MapFormValue; +): InputMapFormValue { + const inputMap = [] as InputMapFormValue; if (modelInterface) { const modelInputs = parseModelInputs(modelInterface); modelInputs.forEach((modelInput) => { inputMap.push({ + ...EMPTY_INPUT_MAP_ENTRY, key: modelInput.label, - value: '', }); }); } @@ -543,17 +596,17 @@ function generateMapFromModelInputs( } // generate a set of mappings s.t. each key is -// a unique model output -function generateMapFromModelOutputs( +// a unique model output. +function generateOutputMapFromModelOutputs( modelInterface?: ModelInterface -): MapFormValue { - const outputMap = [] as MapFormValue; +): OutputMapFormValue { + const outputMap = [] as OutputMapFormValue; if (modelInterface) { const modelOutputs = parseModelOutputs(modelInterface); modelOutputs.forEach((modelOutput) => { outputMap.push({ + ...EMPTY_OUTPUT_MAP_ENTRY, key: modelOutput.label, - value: '', }); }); } diff --git a/public/utils/config_to_form_utils.ts b/public/utils/config_to_form_utils.ts index c87b7c7d..62484fc5 100644 --- a/public/utils/config_to_form_utils.ts +++ b/public/utils/config_to_form_utils.ts @@ -146,7 +146,9 @@ export function getInitialValue(fieldType: ConfigFieldType): ConfigFieldValue { case 'jsonArray': { return '[]'; } - case 'mapArray': { + case 'mapArray': + case 'inputMapArray': + case 'outputMapArray': { return []; } case 'boolean': { diff --git a/public/utils/config_to_schema_utils.ts b/public/utils/config_to_schema_utils.ts index dd5a23d2..3741fbad 100644 --- a/public/utils/config_to_schema_utils.ts +++ b/public/utils/config_to_schema_utils.ts @@ -5,6 +5,7 @@ import { Schema, ObjectSchema } from 'yup'; import * as yup from 'yup'; +import { getIn } from 'formik'; import { WorkflowConfig, WorkflowSchema, @@ -18,6 +19,8 @@ import { MAX_DOCS, MAX_STRING_LENGTH, MAX_JSON_STRING_LENGTH, + MAX_TEMPLATE_STRING_LENGTH, + TRANSFORM_TYPE, } from '../../common'; /* @@ -171,7 +174,6 @@ export function getFieldSchema( } } ); - break; } case 'jsonString': { @@ -192,12 +194,102 @@ export function getFieldSchema( ); break; } + // an array of an array of transforms. + // this format comes from the ML inference processor input map. + case 'inputMapArray': { + baseSchema = yup.array().of( + yup.array().of( + yup.object().shape({ + key: defaultStringSchema.required(), + value: yup.object().shape({ + transformType: defaultStringSchema.required(), + value: yup + .string() + .when('transformType', (transformType, schema) => { + const finalType = getIn( + transformType, + '0', + TRANSFORM_TYPE.FIELD + ) as TRANSFORM_TYPE; + // accept longer string lengths if the input is a template + if (finalType === TRANSFORM_TYPE.TEMPLATE) { + return yup + .string() + .min(1, 'Too short') + .max(MAX_TEMPLATE_STRING_LENGTH, 'Too long') + .required(); + } else { + return defaultStringSchema.required(); + } + }), + nestedVars: yup.array().of( + yup.object().shape({ + name: defaultStringSchema.required(), + transform: defaultStringSchema.required(), + }) + ), + }), + }) + ) + ); + break; + } + case 'outputMapArray': { + baseSchema = yup.array().of( + yup.array().of( + yup.object().shape({ + key: defaultStringSchema.required(), + value: yup.object().shape({ + transformType: defaultStringSchema.required(), + // values are only required based on certain transform types + value: yup + .string() + .when('transformType', (transformType, schema) => { + const finalType = getIn( + transformType, + '0', + TRANSFORM_TYPE.FIELD + ) as TRANSFORM_TYPE; + if (finalType === TRANSFORM_TYPE.FIELD) { + return defaultStringSchema.required(); + } else { + return schema.optional(); + } + }), + nestedVars: yup + .array() + .when('transformType', (transformType, arraySchema) => { + const finalType = getIn( + transformType, + '0', + TRANSFORM_TYPE.FIELD + ) as TRANSFORM_TYPE; + const finalArraySchema = arraySchema.of( + yup.object().shape({ + name: defaultStringSchema.required(), + transform: defaultStringSchema.required(), + }) + ); + // the expression type must contain a list of final expressions + if (finalType === TRANSFORM_TYPE.EXPRESSION) { + return finalArraySchema.min(1, 'No transforms defined'); + } else { + return finalArraySchema; + } + }), + }), + }) + ) + ); + break; + } case 'boolean': { baseSchema = yup.boolean(); break; } case 'number': { baseSchema = yup.number(); + break; } } diff --git a/public/utils/config_to_template_utils.ts b/public/utils/config_to_template_utils.ts index 5b9908e4..92d75c28 100644 --- a/public/utils/config_to_template_utils.ts +++ b/public/utils/config_to_template_utils.ts @@ -25,7 +25,6 @@ import { SearchProcessor, IngestConfig, SearchConfig, - MapFormValue, MapEntry, TEXT_CHUNKING_ALGORITHM, SHARED_OPTIONAL_FIELDS, @@ -33,6 +32,11 @@ import { DELIMITER_OPTIONAL_FIELDS, IngestPipelineConfig, SearchPipelineConfig, + InputMapFormValue, + MapFormValue, + TRANSFORM_TYPE, + OutputMapFormValue, + NO_TRANSFORMATION, } from '../../common'; import { processorConfigToFormik } from './config_to_form_utils'; import { sanitizeJSONPath } from './utils'; @@ -181,17 +185,38 @@ export function processorConfigsToTemplateProcessors( }, } as MLInferenceProcessor; - // process input/output maps + // process model config. + // TODO: this special handling, plus the special handling on index settings/mappings + // could be improved if the 'json' obj returned {} during the conversion instead + // of "{}". We may have future JSON fields which right now are going to require + // this manual parsing before adding to the template. + let modelConfig = {}; + try { + // @ts-ignore + modelConfig = JSON.parse(model_config); + } catch (e) {} + + // process input/output maps. + // if static values found in the input map, add to the model config if (input_map?.length > 0) { processor.ml_inference.input_map = input_map.map( - (mapFormValue: MapFormValue) => mergeMapIntoSingleObj(mapFormValue) + (inputMapFormValue: InputMapFormValue) => { + const res = processModelInputs(inputMapFormValue); + if (!isEmpty(res.modelConfig)) { + modelConfig = { + ...modelConfig, + ...res.modelConfig, + }; + } + return res.inputMap; + } ); } if (output_map?.length > 0) { processor.ml_inference.output_map = output_map.map( - (mapFormValue: MapFormValue) => - mergeMapIntoSingleObj(mapFormValue, true) // we reverse the form inputs for the output map, so reverse back when converting back to the underlying template configuration + (outputMapFormValue: OutputMapFormValue) => + processModelOutputs(outputMapFormValue) ); } @@ -206,21 +231,10 @@ export function processorConfigsToTemplateProcessors( ); }); - // process model config. - // TODO: this special handling, plus the special handling on index settings/mappings - // could be improved if the 'json' obj returned {} during the conversion instead - // of "{}". We may have future JSON fields which right now are going to require - // this manual parsing before adding to the template. - let finalModelConfig = {}; - try { - // @ts-ignore - finalModelConfig = JSON.parse(model_config); - } catch (e) {} - processor.ml_inference = { ...processor.ml_inference, ...additionalFormValues, - model_config: finalModelConfig, + model_config: modelConfig, }; processorsList.push(processor); @@ -442,7 +456,7 @@ export function reduceToTemplate(workflow: Workflow): WorkflowTemplate { // Helper fn to merge the form map (an arr of objs) into a single obj, such that each key // is an obj property, and each value is a property value. Used to format into the -// expected inputs for input_maps and output_maps of the ML inference processors. +// expected inputs for processor configurations function mergeMapIntoSingleObj( mapFormValue: MapFormValue, reverse: boolean = false @@ -462,6 +476,99 @@ function mergeMapIntoSingleObj( return curMap; } +// Bucket the model inputs configured on the UI as input map entries containing dynamic data, +// or model config entries containing static data. +function processModelInputs( + mapFormValue: InputMapFormValue +): { inputMap: {}; modelConfig: {} } { + let inputMap = {}; + let modelConfig = {}; + mapFormValue.forEach((mapEntry) => { + // dynamic data + if ( + mapEntry.value.transformType === TRANSFORM_TYPE.FIELD || + mapEntry.value.transformType === TRANSFORM_TYPE.EXPRESSION + ) { + inputMap = { + ...inputMap, + [sanitizeJSONPath(mapEntry.key)]: sanitizeJSONPath( + mapEntry.value.value + ), + }; + // template with dynamic nested vars. Add the nested vars as input map entries, + // and add the static template itself to the model config. + } else if ( + mapEntry.value.transformType === TRANSFORM_TYPE.TEMPLATE && + !isEmpty(mapEntry.value.nestedVars) + ) { + mapEntry.value.nestedVars?.forEach((nestedVar) => { + inputMap = { + ...inputMap, + [sanitizeJSONPath(nestedVar.name)]: sanitizeJSONPath( + nestedVar.transform + ), + }; + }); + modelConfig = { + ...modelConfig, + [mapEntry.key]: mapEntry.value.value, + }; + // static data + } else { + modelConfig = { + ...modelConfig, + [mapEntry.key]: mapEntry.value.value, + }; + } + }); + return { + inputMap, + modelConfig, + }; +} + +// Parse out the model outputs and any sub-expressions into a single, final output map +function processModelOutputs(mapFormValue: OutputMapFormValue): {} { + let outputMap = {}; + mapFormValue.forEach((mapEntry) => { + // field transform: just a rename + if (mapEntry.value.transformType === TRANSFORM_TYPE.FIELD) { + outputMap = { + ...outputMap, + [sanitizeJSONPath(mapEntry.value.value)]: sanitizeJSONPath( + mapEntry.key + ), + }; + // expression transform: can have multiple nested expressions, since a user may want to parse + // out new sub-fields / sub-transforms based off of some model output field that contains nested + // data. Add the nested expressions as standalone output map entries + } else if ( + mapEntry.value.transformType === TRANSFORM_TYPE.EXPRESSION && + !isEmpty(mapEntry.value.nestedVars) + ) { + mapEntry.value.nestedVars?.forEach((nestedVar) => { + outputMap = { + ...outputMap, + [sanitizeJSONPath(nestedVar.name)]: sanitizeJSONPath( + nestedVar.transform + ), + }; + }); + // If there is no transformation selected, just map the same output + // field name to the new field name + // @ts-ignore + } else if (mapEntry.value.transformType === NO_TRANSFORMATION) { + outputMap = { + ...outputMap, + [sanitizeJSONPath(mapEntry.key)]: sanitizeJSONPath(mapEntry.key), + }; + // Placeholder logic for future transform types + } else { + } + }); + return outputMap; +} + // utility fn used to build the final set of processor config fields, filtering // by only adding if the field is valid function optionallyAddToFinalForm( diff --git a/public/utils/utils.ts b/public/utils/utils.ts index f750bddd..d456d0ae 100644 --- a/public/utils/utils.ts +++ b/public/utils/utils.ts @@ -10,7 +10,6 @@ import { JSONPATH_ROOT_SELECTOR, MODEL_OUTPUT_SCHEMA_FULL_PATH, MODEL_OUTPUT_SCHEMA_NESTED_PATH, - MapFormValue, ModelInputFormField, ModelInterface, ModelOutput, @@ -28,10 +27,11 @@ import { } from '../../common'; import { getCore, getDataSourceEnabled } from '../services'; import { + InputMapEntry, MDSQueryParams, - MapEntry, ModelInputMap, ModelOutputMap, + OutputMapEntry, } from '../../common/interfaces'; import queryString from 'query-string'; import { useLocation } from 'react-router-dom'; @@ -188,7 +188,7 @@ export function unwrapTransformedDocs( // We follow the same logic here to generate consistent results. export function generateTransform( input: {} | [], - map: MapFormValue, + map: (InputMapEntry | OutputMapEntry)[], context: PROCESSOR_CONTEXT, transformContext: TRANSFORM_CONTEXT, queryContext?: {} @@ -198,7 +198,7 @@ export function generateTransform( try { const transformedResult = getTransformedResult( input, - mapEntry.value, + mapEntry.value.value || '', context, transformContext, queryContext @@ -218,7 +218,7 @@ export function generateTransform( // and the input is an array. export function generateArrayTransform( input: [], - map: MapFormValue, + map: (InputMapEntry | OutputMapEntry)[], context: PROCESSOR_CONTEXT, transformContext: TRANSFORM_CONTEXT, queryContext?: {} @@ -230,8 +230,8 @@ export function generateArrayTransform( // prefix, parse the query context, instead of the other input. let transformedResult; if ( - (mapEntry.value.startsWith(REQUEST_PREFIX) || - mapEntry.value.startsWith( + (mapEntry.value.value?.startsWith(REQUEST_PREFIX) || + mapEntry.value.value?.startsWith( REQUEST_PREFIX_WITH_JSONPATH_ROOT_SELECTOR )) && queryContext !== undefined && @@ -239,7 +239,7 @@ export function generateArrayTransform( ) { transformedResult = getTransformedResult( {}, - mapEntry.value, + mapEntry.value.value, context, transformContext, queryContext @@ -248,7 +248,7 @@ export function generateArrayTransform( transformedResult = input.map((inputEntry) => getTransformedResult( inputEntry, - mapEntry.value, + mapEntry.value.value || '', context, transformContext, queryContext diff --git a/server/routes/helpers.ts b/server/routes/helpers.ts index 5ec5dd2c..87bd4e87 100644 --- a/server/routes/helpers.ts +++ b/server/routes/helpers.ts @@ -16,7 +16,6 @@ import { ModelInterface, ModelOutput, NO_MODIFICATIONS_FOUND_TEXT, - PROMPT_FIELD, SearchHit, WORKFLOW_RESOURCE_TYPE, WORKFLOW_STATE, @@ -161,7 +160,6 @@ export function getConnectorsFromResponses( parameters: { model: connectorHit._source?.parameters?.model, dimensions: connectorHit._source?.parameters.dimensions, - [PROMPT_FIELD]: connectorHit?._source?.parameters[PROMPT_FIELD], }, } as Connector; });