4.6.7 first pr (#726)

.gitignore

@@ -39,5 +39,4 @@ docSite/.vercel
 *.local.*
 
 
-# jetbrains
 .idea/

dev.md

@@ -0,0 +1,17 @@
+# 打包命令
+
+```sh
+# Build image, not proxy
+docker build -t registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.4.7 --build-arg name=app .
+
+# build image with proxy
+docker build -t registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.4.7 --build-arg name=app --build-arg proxy=taobao .
+```
+
+# Pg 常用索引
+
+```sql
+CREATE INDEX IF NOT EXISTS modelData_dataset_id_index ON modeldata (dataset_id);
+CREATE INDEX IF NOT EXISTS modelData_collection_id_index ON modeldata (collection_id);
+CREATE INDEX IF NOT EXISTS modelData_teamId_index ON modeldata (team_id);
+```

packages/global/common/file/api.d.ts

@@ -1,16 +1,23 @@
-export type UploadImgProps = {
- base64Img: string;
+import { MongoImageTypeEnum } from './image/constants';
+
+export type preUploadImgProps = {
+ type: `${MongoImageTypeEnum}`;
+
  expiredTime?: Date;
  metadata?: Record<string, any>;
  shareId?: string;
 };
+export type UploadImgProps = preUploadImgProps & {
+ base64Img: string;
+};
 
 export type UrlFetchParams = {
  urlList: string[];
  selector?: string;
 };
 export type UrlFetchResponse = {
  url: string;
+ title: string;
  content: string;
  selector?: string;
 }[];

packages/global/common/file/image/constants.ts

@@ -0,0 +1,52 @@
+export const imageBaseUrl = '/api/system/img/';
+
+export enum MongoImageTypeEnum {
+ systemAvatar = 'systemAvatar',
+ appAvatar = 'appAvatar',
+ pluginAvatar = 'pluginAvatar',
+ datasetAvatar = 'datasetAvatar',
+ userAvatar = 'userAvatar',
+ teamAvatar = 'teamAvatar',
+
+ chatImage = 'chatImage',
+ docImage = 'docImage'
+}
+export const mongoImageTypeMap = {
+ [MongoImageTypeEnum.systemAvatar]: {
+ label: 'common.file.type.appAvatar',
+ unique: true
+ },
+ [MongoImageTypeEnum.appAvatar]: {
+ label: 'common.file.type.appAvatar',
+ unique: true
+ },
+ [MongoImageTypeEnum.pluginAvatar]: {
+ label: 'common.file.type.pluginAvatar',
+ unique: true
+ },
+ [MongoImageTypeEnum.datasetAvatar]: {
+ label: 'common.file.type.datasetAvatar',
+ unique: true
+ },
+ [MongoImageTypeEnum.userAvatar]: {
+ label: 'common.file.type.userAvatar',
+ unique: true
+ },
+ [MongoImageTypeEnum.teamAvatar]: {
+ label: 'common.file.type.teamAvatar',
+ unique: true
+ },
+
+ [MongoImageTypeEnum.chatImage]: {
+ label: 'common.file.type.chatImage',
+ unique: false
+ },
+ [MongoImageTypeEnum.docImage]: {
+ label: 'common.file.type.docImage',
+ unique: false
+ }
+};
+
+export const uniqueImageTypeList = Object.entries(mongoImageTypeMap)
+ .filter(([key, value]) => value.unique)
+ .map(([key]) => key as `${MongoImageTypeEnum}`);

packages/global/common/file/image/type.d.ts

@@ -0,0 +1,11 @@
+import { MongoImageTypeEnum } from './constants';
+
+export type MongoImageSchemaType = {
+ teamId: string;
+ binary: Buffer;
+ createTime: Date;
+ expiredTime?: Date;
+ type: `${MongoImageTypeEnum}`;
+
+ metadata?: { fileId?: string };
+};

packages/global/common/math/date.ts

@@ -0,0 +1,10 @@
+// The number of days left in the month is calculated as 30 days per month, and less than 1 day is calculated as 1 day
+export const getMonthRemainingDays = () => {
+ const now = new Date();
+ const year = now.getFullYear();
+ const month = now.getMonth();
+ const date = now.getDate();
+ const days = new Date(year, month + 1, 0).getDate();
+ const remainingDays = days - date;
+ return remainingDays + 1;
+};

packages/global/common/string/markdown.ts

@@ -15,10 +15,10 @@ export const simpleMarkdownText = (rawText: string) => {
  return `[${cleanedLinkText}](${url})`;
  });
 
- // replace special \.* ……
- const reg1 = /\\([-.!`_(){}\[\]])/g;
+ // replace special #\.* ……
+ const reg1 = /\\([#`!*()+-_\[\]{}\\.])/g;
  if (reg1.test(rawText)) {
- rawText = rawText.replace(/\\([`!*()+-_\[\]{}\\.])/g, '$1');
+ rawText = rawText.replace(reg1, '$1');
  }
 
  // replace \\n
@@ -45,30 +45,47 @@ export const uploadMarkdownBase64 = async ({
  uploadImgController
 }: {
  rawText: string;
- uploadImgController: (base64: string) => Promise<string>;
+ uploadImgController?: (base64: string) => Promise<string>;
 }) => {
- // match base64, upload and replace it
- const base64Regex = /data:image\/.*;base64,([^\)]+)/g;
- const base64Arr = rawText.match(base64Regex) || [];
- // upload base64 and replace it
- await Promise.all(
- base64Arr.map(async (base64Img) => {
- try {
- const str = await uploadImgController(base64Img);
+ if (uploadImgController) {
+ // match base64, upload and replace it
+ const base64Regex = /data:image\/.*;base64,([^\)]+)/g;
+ const base64Arr = rawText.match(base64Regex) || [];
+ // upload base64 and replace it
+ await Promise.all(
+ base64Arr.map(async (base64Img) => {
+ try {
+ const str = await uploadImgController(base64Img);
 
- rawText = rawText.replace(base64Img, str);
- } catch (error) {
- rawText = rawText.replace(base64Img, '');
- rawText = rawText.replace(/!\[.*\]\(\)/g, '');
- }
- })
- );
+ rawText = rawText.replace(base64Img, str);
+ } catch (error) {
+ rawText = rawText.replace(base64Img, '');
+ rawText = rawText.replace(/!\[.*\]\(\)/g, '');
+ }
+ })
+ );
+ }
 
  // Remove white space on both sides of the picture
  const trimReg = /(!\[.*\]\(.*\))\s*/g;
  if (trimReg.test(rawText)) {
  rawText = rawText.replace(trimReg, '$1');
  }
 
- return simpleMarkdownText(rawText);
+ return rawText;
+};
+
+export const markdownProcess = async ({
+ rawText,
+ uploadImgController
+}: {
+ rawText: string;
+ uploadImgController?: (base64: string) => Promise<string>;
+}) => {
+ const imageProcess = await uploadMarkdownBase64({
+ rawText,
+ uploadImgController
+ });
+
+ return simpleMarkdownText(imageProcess);
 };

packages/global/common/string/tiktoken/index.ts

@@ -33,6 +33,12 @@ export function countPromptTokens(
 ) {
  const enc = getTikTokenEnc();
  const text = `${role}\n${prompt}`;
+
+ // too large a text will block the thread
+ if (text.length > 15000) {
+ return text.length * 1.7;
+ }
+
  try {
  const encodeText = enc.encode(text);
  return encodeText.length + role.length; // 补充 role 估算值

packages/global/common/string/time.ts

@@ -1,3 +1,4 @@
 import dayjs from 'dayjs';
 
-export const formatTime2YMDHM = (time: Date) => dayjs(time).format('YYYY-MM-DD HH:mm');
+export const formatTime2YMDHM = (time?: Date) =>
+ time ? dayjs(time).format('YYYY-MM-DD HH:mm') : '';

packages/global/common/string/tools.ts

@@ -1,4 +1,5 @@
 import crypto from 'crypto';
+import { customAlphabet } from 'nanoid';
 
 /* check string is a web link */
 export function strIsLink(str?: string) {
@@ -36,3 +37,7 @@ export function replaceVariable(text: string, obj: Record<string, string | numbe
  }
  return text || '';
 }
+
+export const getNanoid = (size = 12) => {
+ return customAlphabet('abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890', size)();
+};

packages/global/common/system/types/index.d.ts

@@ -51,6 +51,10 @@ export type FastGPTFeConfigsType = {
  favicon?: string;
  customApiDomain?: string;
  customSharePageDomain?: string;
+ subscription?: {
+ datasetStoreFreeSize?: number;
+ datasetStorePrice?: number;
+ };
 };
 
 export type SystemEnvType = {
@@ -63,4 +67,5 @@ export type SystemEnvType = {
 declare global {
  var feConfigs: FastGPTFeConfigsType;
  var systemEnv: SystemEnvType;
+ var systemInitd: boolean;
 }

packages/global/core/chat/constants.ts

@@ -31,16 +31,16 @@ export enum ChatSourceEnum {
 }
 export const ChatSourceMap = {
  [ChatSourceEnum.test]: {
- name: 'chat.logs.test'
+ name: 'core.chat.logs.test'
  },
  [ChatSourceEnum.online]: {
- name: 'chat.logs.online'
+ name: 'core.chat.logs.online'
  },
  [ChatSourceEnum.share]: {
- name: 'chat.logs.share'
+ name: 'core.chat.logs.share'
  },
  [ChatSourceEnum.api]: {
- name: 'chat.logs.api'
+ name: 'core.chat.logs.api'
  }
 };