V4.8.17 feature (#3485)

* feat: add third party account config (#3443) * temp * editor workflow variable style * add team to dispatch * i18n * delete console * change openai account position * fix * fix * fix * fix * fix * 4.8.17 test (#3461) * perf: external provider config * perf: ui * feat: add template config (#3434) * change template position * template config * delete console * delete * fix * fix * perf: Mongo visutal field (#3464) * remve invalid code * perf: team member visutal code * perf: virtual search; perf: search test data * fix: ts * fix: image response headers * perf: template code * perf: auth layout；perf: auto save (#3472) * perf: auth layout * perf: auto save * perf: auto save * fix: template guide display & http input support external variables (#3475) * fix: template guide display * http editor support external workflow variables * perf: auto save;fix: ifelse checker line break; (#3478) * perf: auto save * perf: auto save * fix: ifelse checker line break * perf: doc * perf: doc * fix: update var type error * 4.8.17 test (#3479) * perf: auto save * perf: auto save * perf: template code * 4.8.17 test (#3480) * perf: auto save * perf: auto save * perf: model price model * feat: add react memo * perf: model provider filter * fix: ts (#3481) * perf: auto save * perf: auto save * fix: ts * simple app tool select (#3473) * workflow plugin userguide & simple tool ui * simple tool filter * reuse component * change component to hook * fix * perf: too selector modal (#3484) * perf: auto save * perf: auto save * perf: markdown render * perf: too selector * fix: app version require tmbId * perf: templates refresh * perf: templates refresh * hide auto save error tip * perf: toolkit guide --------- Co-authored-by: heheer <[email protected]>
labring · Dec 27, 2024 · b520988 · b520988
1 parent a209856
commit b520988
Show file tree

Hide file tree

Showing 207 changed files with 2,941 additions and 1,376 deletions.
diff --git a/README.md b/README.md
@@ -58,9 +58,9 @@ https://github.com/labring/FastGPT/assets/15308462/7d3a38df-eb0e-4388-9250-2409b
    - [x] 多库复用，混用
    - [x] chunk 记录修改和删除
    - [x] 支持手动输入，直接分段，QA 拆分导入
-   - [x] 支持 txt，md，html，pdf，docx，pptx，csv，xlsx (有需要更多可 PR file loader)
-   - [x] 支持 url 读取、CSV 批量导入
+   - [x] 支持 txt，md，html，pdf，docx，pptx，csv，xlsx (有需要更多可 PR file loader)，支持 url 读取、CSV 批量导入
    - [x] 混合检索 & 重排
+   - [x] API 知识库
    - [ ] 自定义文件读取服务
    - [ ] 自定义分块服务
 
@@ -69,7 +69,7 @@ https://github.com/labring/FastGPT/assets/15308462/7d3a38df-eb0e-4388-9250-2409b
    - [x] 对话时反馈引用并可修改与删除
    - [x] 完整上下文呈现
    - [x] 完整模块中间值呈现
-   - [x] 高级编排 DeBug 模式
+   - [ ] 高级编排 DeBug 模式
 
 `4` OpenAPI 接口
    - [x] completions 接口 (chat 模式对齐 GPT 接口)

diff --git a/docSite/content/zh-cn/docs/development/openapi/dataset.md b/docSite/content/zh-cn/docs/development/openapi/dataset.md
@@ -1424,7 +1424,11 @@ curl --location --request POST 'https://api.fastgpt.in/api/core/dataset/searchTe
     "limit": 5000,
     "similarity": 0,
     "searchMode": "embedding",
-    "usingReRank": false
+    "usingReRank": false,
+
+    "datasetSearchUsingExtensionQuery": true,
+    "datasetSearchExtensionModel": "gpt-4o-mini",
+    "datasetSearchExtensionBg": ""
 }'
 ```
 
@@ -1441,6 +1445,9 @@ curl --location --request POST 'https://api.fastgpt.in/api/core/dataset/searchTe
 - similarity - 最低相关度（0~1，可选）
 - searchMode - 搜索模式：embedding | fullTextRecall | mixedRecall
 - usingReRank - 使用重排
+- datasetSearchUsingExtensionQuery - 使用问题优化
+- datasetSearchExtensionModel - 问题优化模型
+- datasetSearchExtensionBg - 问题优化背景描述
 {{% /alert %}}
 
 {{< /markdownify >}}

diff --git a/docSite/content/zh-cn/docs/development/upgrading/4817.md b/docSite/content/zh-cn/docs/development/upgrading/4817.md
@@ -7,10 +7,30 @@ toc: true
 weight: 807
 ---
 
+## 运行升级脚本
+
+从任意终端，发起 1 个 HTTP 请求。其中 {{rootkey}} 替换成环境变量里的 `rootkey`；{{host}} 替换成**FastGPT 域名**。
+
+```bash
+curl --location --request POST 'https://{{host}}/api/admin/initv4817' \
+--header 'rootkey: {{rootkey}}' \
+--header 'Content-Type: application/json'
+```
+
+会将用户绑定的 OpenAI 账号移动到团队中。
 
 ## 完整更新内容
 
-1. 
-2. 新增 - LLM 模型参数支持关闭 max_tokens 和 temperature。
-3. 优化 - 知识库搜索参数，滑动条支持输入模式，可以更精准的控制。
-4. 优化 - 可用模型展示
+1. 新增 - 简易模式工具调用支持数组类型插件。
+2. 新增 - 工作流增加异常离开自动保存，避免工作流丢失。
+3. 新增 - LLM 模型参数支持关闭 max_tokens 和 temperature。
+4. 新增 - 商业版支持后台配置模板市场。
+5. 新增 - 商业版支持后台配置自定义工作流变量，用于与业务系统鉴权打通。
+6. 新增 - 搜索测试接口支持问题优化。
+7. 优化 - Markdown 大小测试，超出 20 万字符不使用 Markdown 组件，避免崩溃。
+8. 优化 - 知识库搜索参数，滑动条支持输入模式，可以更精准的控制。
+9. 优化 - 可用模型展示
+10. 优化 - Mongo 查询语句，增加 virtual 字段。
+11. 修复 - 文件返回接口缺少 Content-Length 头，导致通过非同源文件上传时，阿里 vision 模型无法识别图片。
+12. 修复 - 去除判断器两端字符串隐藏换行符，避免判断器失效。
+13. 修复 - 变量更新节点，手动输入更新内容时候，非字符串类型数据类型无法自动转化。
diff --git a/packages/global/common/system/types/index.d.ts b/packages/global/common/system/types/index.d.ts
@@ -18,6 +18,14 @@ export type NavbarItemType = {
   isActive: boolean;
 };
 
+export type ExternalProviderWorkflowVarType = {
+  name: string;
+  key: string;
+  intro: string;
+  isOpen: boolean;
+  url?: string;
+};
+
 /* fastgpt main */
 export type FastGPTConfigFileType = {
   feConfigs: FastGPTFeConfigsType;
@@ -84,6 +92,7 @@ export type FastGPTFeConfigsType = {
   uploadFileMaxSize?: number;
   lafEnv?: string;
   navbarItems?: NavbarItemType[];
+  externalProviderWorkflowVariables?: ExternalProviderWorkflowVarType[];
 };
 
 export type SystemEnvType = {

diff --git a/packages/global/core/app/type.d.ts b/packages/global/core/app/type.d.ts
@@ -13,6 +13,7 @@ import { StoreEdgeItemType } from '../workflow/type/edge';
 import { AppPermission } from '../../support/permission/app/controller';
 import { ParentIdType } from '../../common/parentFolder/type';
 import { FlowNodeInputTypeEnum } from 'core/workflow/node/constant';
+import { WorkflowTemplateBasicType } from '@fastgpt/global/core/workflow/type';
 
 export type AppSchema = {
   _id: string;
@@ -184,3 +185,28 @@ export type SystemPluginListItemType = {
   name: string;
   avatar: string;
 };
+
+export type AppTemplateSchemaType = {
+  templateId: string;
+  name: string;
+  intro: string;
+  avatar: string;
+  tags: string[];
+  type: string;
+  author?: string;
+  isActive?: boolean;
+  userGuide?: {
+    type: 'markdown' | 'link';
+    content?: string;
+    link?: string;
+  };
+  isQuickTemplate?: boolean;
+  order?: number;
+  workflow: WorkflowTemplateBasicType;
+};
+
+export type TemplateTypeSchemaType = {
+  typeName: string;
+  typeId: string;
+  typeOrder: number;
+};
diff --git a/packages/global/core/dataset/collection/utils.ts b/packages/global/core/dataset/collection/utils.ts
@@ -1,9 +1,7 @@
 import { DatasetCollectionTypeEnum, TrainingModeEnum, TrainingTypeMap } from '../constants';
-import { CollectionWithDatasetType, DatasetCollectionSchemaType } from '../type';
+import { DatasetCollectionSchemaType } from '../type';
 
-export const getCollectionSourceData = (
-  collection?: CollectionWithDatasetType | DatasetCollectionSchemaType
-) => {
+export const getCollectionSourceData = (collection?: DatasetCollectionSchemaType) => {
   return {
     sourceId:
       collection?.fileId ||

diff --git a/packages/global/core/dataset/type.d.ts b/packages/global/core/dataset/type.d.ts
@@ -133,11 +133,8 @@ export type DatasetTrainingSchemaType = {
   indexes: Omit<DatasetDataIndexItemType, 'dataId'>[];
 };
 
-export type CollectionWithDatasetType = Omit<DatasetCollectionSchemaType, 'datasetId'> & {
-  datasetId: DatasetSchemaType;
-};
-export type DatasetDataWithCollectionType = Omit<DatasetDataSchemaType, 'collectionId'> & {
-  collectionId: DatasetCollectionSchemaType;
+export type CollectionWithDatasetType = DatasetCollectionSchemaType & {
+  dataset: DatasetSchemaType;
 };
 
 /* ================= dataset ===================== */

diff --git a/packages/global/core/workflow/runtime/type.d.ts b/packages/global/core/workflow/runtime/type.d.ts
@@ -21,13 +21,20 @@ import { ReadFileNodeResponse } from '../template/system/readFiles/type';
 import { UserSelectOptionType } from '../template/system/userSelect/type';
 import { WorkflowResponseType } from '../../../../service/core/workflow/dispatch/type';
 import { AiChatQuoteRoleType } from '../template/system/aiChat/type';
+import { LafAccountType, OpenaiAccountType } from '../../../support/user/team/type';
+
+export type ExternalProviderType = {
+  openaiAccount?: OpenaiAccountType;
+  externalWorkflowVariables?: Record<string, string>;
+};
 
 /* workflow props */
 export type ChatDispatchProps = {
   res?: NextApiResponse;
   requestOrigin?: string;
   mode: 'test' | 'chat' | 'debug';
-  user: UserModelSchema;
+  timezone: string;
+  externalProvider: ExternalProviderType;
 
   runningAppInfo: {
     id: string; // May be the id of the system plug-in (cannot be used directly to look up the table)

diff --git a/packages/global/core/workflow/runtime/utils.ts b/packages/global/core/workflow/runtime/utils.ts
@@ -9,6 +9,7 @@ import { isValidReferenceValueFormat } from '../utils';
 import { FlowNodeOutputItemType, ReferenceValueType } from '../type/io';
 import { ChatItemType, NodeOutputItemType } from '../../../core/chat/type';
 import { ChatItemValueTypeEnum, ChatRoleEnum } from '../../../core/chat/constants';
+import { replaceVariable } from '../../../common/string/tools';
 
 export const getMaxHistoryLimitFromNodes = (nodes: StoreNodeItemType[]): number => {
   let limit = 10;
@@ -317,6 +318,8 @@ export function replaceEditorVariable({
 }) {
   if (typeof text !== 'string') return text;
 
+  text = replaceVariable(text, variables);
+
   const variablePattern = /\{\{\$([^.]+)\.([^$]+)\$\}\}/g;
   const matches = [...text.matchAll(variablePattern)];
   if (matches.length === 0) return text;

diff --git a/packages/global/core/workflow/template/system/workflowStart.ts b/packages/global/core/workflow/template/system/workflowStart.ts
@@ -30,7 +30,6 @@ export const WorkflowStart: FlowNodeTemplateType = {
   intro: '',
   forbidDelete: true,
   unique: true,
-  courseUrl: '/docs/guide/workbench/workflow/input/',
   version: '481',
   inputs: [{ ...Input_Template_UserChatInput, toolDescription: i18nT('workflow:user_question') }],
   outputs: [

diff --git a/packages/global/core/workflow/type/node.d.ts b/packages/global/core/workflow/type/node.d.ts
@@ -69,6 +69,7 @@ export type FlowNodeTemplateType = FlowNodeCommonType & {
 
   diagram?: string; // diagram url
   courseUrl?: string; // course url
+  userGuide?: string; // user guide
 };
 
 export type NodeTemplateListItemType = {
@@ -87,6 +88,7 @@ export type NodeTemplateListItemType = {
   currentCost?: number; // 当前积分消耗
   hasTokenFee?: boolean; // 是否配置积分
   instructions?: string; // 使用说明
+  courseUrl?: string; // 教程链接
 };
 
 export type NodeTemplateListType = {

diff --git a/packages/global/support/outLink/type.d.ts b/packages/global/support/outLink/type.d.ts
@@ -83,11 +83,6 @@ export type OutLinkSchema<T extends OutlinkAppType = undefined> = {
   app: T;
 };
 
-// to handle MongoDB querying
-export type OutLinkWithAppType = Omit<OutLinkSchema, 'appId'> & {
-  appId: AppSchema;
-};
-
 // Edit the Outlink
 export type OutLinkEditType<T = undefined> = {
   _id?: string;

diff --git a/packages/global/support/permission/type.d.ts b/packages/global/support/permission/type.d.ts
@@ -1,5 +1,6 @@
+import { UserModelSchema } from '../user/type';
 import { RequireOnlyOne } from '../../common/type/utils';
-import { TeamMemberWithUserSchema } from '../user/team/type';
+import { TeamMemberSchema } from '../user/team/type';
 import { AuthUserTypeEnum, PermissionKeyEnum, PerResourceTypeEnum } from './constant';
 import { MemberGroupSchemaType } from './memberGroup/type';
 
@@ -31,11 +32,7 @@ export type ResourcePermissionType = {
 }>;
 
 export type ResourcePerWithTmbWithUser = Omit<ResourcePermissionType, 'tmbId'> & {
-  tmbId: TeamMemberWithUserSchema;
-};
-
-export type ResourcePerWithGroup = Omit<ResourcePermissionType, 'groupId'> & {
-  groupId: MemberGroupSchemaType;
+  tmbId: TeamMemberSchema & { user: UserModelSchema };
 };
 
 export type PermissionSchemaType = {

diff --git a/packages/global/support/user/team/controller.d.ts b/packages/global/support/user/team/controller.d.ts
@@ -1,6 +1,6 @@
 import { PermissionValueType } from '../../permission/type';
 import { TeamMemberRoleEnum } from './constant';
-import { LafAccountType, TeamMemberSchema } from './type';
+import { LafAccountType, TeamMemberSchema, ThirdPartyAccountType } from './type';
 
 export type AuthTeamRoleProps = {
   teamId: string;
@@ -11,14 +11,13 @@ export type CreateTeamProps = {
   name: string;
   avatar?: string;
   defaultTeam?: boolean;
-  lafAccount?: LafAccountType;
   memberName?: string;
 };
-export type UpdateTeamProps = {
+export type UpdateTeamProps = Omit<ThirdPartyAccountType, 'externalWorkflowVariable'> & {
   name?: string;
   avatar?: string;
   teamDomain?: string;
-  lafAccount?: null | LafAccountType;
+  externalWorkflowVariable?: { key: string; value: string };
 };
 
 /* ------------- member ----------- */

diff --git a/packages/global/support/user/team/type.d.ts b/packages/global/support/user/team/type.d.ts
@@ -4,6 +4,12 @@ import { LafAccountType } from './type';
 import { PermissionValueType, ResourcePermissionType } from '../../permission/type';
 import { TeamPermission } from '../../permission/user/controller';
 
+export type ThirdPartyAccountType = {
+  lafAccount?: LafAccountType;
+  openaiAccount?: OpenaiAccountType;
+  externalWorkflowVariables?: Record<string, string>;
+};
+
 export type TeamSchema = {
   _id: string;
   name: string;
@@ -16,9 +22,8 @@ export type TeamSchema = {
     lastExportDatasetTime: Date;
     lastWebsiteSyncTime: Date;
   };
-  lafAccount: LafAccountType;
   notificationAccount?: string;
-};
+} & ThirdPartyAccountType;
 
 export type tagsType = {
   label: string;
@@ -42,16 +47,9 @@ export type TeamMemberSchema = {
   defaultTeam: boolean;
 };
 
-export type TeamMemberWithUserSchema = Omit<TeamMemberSchema, 'userId'> & {
-  userId: UserModelSchema;
-};
-
-export type TeamMemberWithTeamSchema = Omit<TeamMemberSchema, 'teamId'> & {
-  teamId: TeamSchema;
-};
-
-export type TeamMemberWithTeamAndUserSchema = Omit<TeamMemberWithTeamSchema, 'userId'> & {
-  userId: UserModelSchema;
+export type TeamMemberWithTeamAndUserSchema = TeamMemberSchema & {
+  team: TeamSchema;
+  user: UserModelSchema;
 };
 
 export type TeamTmbItemType = {
@@ -66,10 +64,9 @@ export type TeamTmbItemType = {
   defaultTeam: boolean;
   role: `${TeamMemberRoleEnum}`;
   status: `${TeamMemberStatusEnum}`;
-  lafAccount?: LafAccountType;
   notificationAccount?: string;
   permission: TeamPermission;
-};
+} & ThirdPartyAccountType;
 
 export type TeamMemberItemType = {
   userId: string;
@@ -88,11 +85,16 @@ export type TeamTagItemType = {
 };
 
 export type LafAccountType = {
-  token: string;
   appid: string;
+  token: string;
   pat: string;
 };
 
+export type OpenaiAccountType = {
+  key: string;
+  baseUrl: string;
+};
+
 export type TeamInvoiceHeaderType = {
   teamName: string;
   unifiedCreditCode: string;

diff --git a/packages/global/support/user/type.d.ts b/packages/global/support/user/type.d.ts
@@ -14,10 +14,6 @@ export type UserModelSchema = {
   timezone: string;
   status: `${UserStatusEnum}`;
   lastLoginTmbId?: string;
-  openaiAccount?: {
-    key: string;
-    baseUrl: string;
-  };
   fastgpt_sem?: {
     keyword: string;
   };
@@ -29,7 +25,6 @@ export type UserType = {
   avatar: string;
   timezone: string;
   promotionRate: UserModelSchema['promotionRate'];
-  openaiAccount: UserModelSchema['openaiAccount'];
   team: TeamTmbItemType;
   standardInfo?: standardInfoType;
   notificationAccount?: string;

diff --git a/packages/plugins/src/searchXNG/template.json b/packages/plugins/src/searchXNG/template.json
@@ -55,8 +55,7 @@
             "maxFiles": 5,
             "canSelectFile": true,
             "canSelectImg": true,
-            "required": true,
-            "toolDescription": "部署的searXNG服务的链接"
+            "required": true
           }
         ],
         "outputs": [