Update docs and response tag in share page (#694)

* perf: chunk index show

* share response

* perf: vector query

* web printFinger

* remove log

* fix: bucket name

* perf: training schema

* perf: sort index

docSite/content/docs/commercial/intro.md

@@ -33,7 +33,7 @@ FastGPT 商业版是基于 FastGPT 开源版的增强版本，增加了一些独
 
 ## 商业版软件价格
 
-FastGPT 商业版软件根据不同的部署方式，分为 4 类收费模式。下面列举各种部署方式一些常规内容，如仍有问题，可[联系咨询](https://fael3z0zfze.feishu.cn/share/base/form/shrcnRxj3utrzjywsom96Px4sud)
+FastGPT 商业版软件根据不同的部署方式，分为 3 类收费模式。下面列举各种部署方式一些常规内容，如仍有问题，可[联系咨询](https://fael3z0zfze.feishu.cn/share/base/form/shrcnRxj3utrzjywsom96Px4sud)
 
 **共有服务**
 
@@ -46,8 +46,7 @@ FastGPT 商业版软件根据不同的部署方式，分为 4 类收费模式。
 {{< table "table-hover table-striped-columns" >}}
 | 部署方式 | 特有服务 | 上线时长 | 价格 |
 | ---- | ---- | ---- | ---- |
-| Sealos公有云部署 | 1. 6个版本的升级服务。<br>2. 赠送 8000 元 Sealos 云资源额度。 | 半天 | 10000元/年 | 
-| Sealos全托管 | 1. 有效期内免费升级。<br>2. 免运维服务&数据库。<br>3. 赠送 10000 元 Sealos 云资源额度。 | 半天 | 3000元起/月（3个月起）<br>或<br>30000元起/年 |
+| Sealos全托管 | 1. 有效期内免费升级。<br>2. 免运维服务&数据库。 | 半天 | 3000元起/月（3个月起）<br>或<br>30000元起/年 |
 | 自有服务器-单机版 | 1. 6个版本的升级服务。 | 14天内 | 60000元/套（不限时长） | 
 | 自有服务器-Sealos版 | 1. 6个版本的升级服务。 | 14天内 | 150000元/套（不限时长）|
 {{< /table >}}

docSite/content/docs/development/openapi/auth.md

@@ -25,7 +25,7 @@ FastGPT 的 API Key **有 2 类**，一类是全局通用的 key (无法直接
 
 | 通用key | 应用特定 key |
 | --------------------- | --------------------- |
-| ![](/imgs/fastgpt-api2.png) | ![](/imgs/fastgpt-api.png) |
+| ![](/imgs/fastgpt-api2.jpg) | ![](/imgs/fastgpt-api.jpg) |
 
 ## 基本配置
 

packages/global/common/file/constants.ts

@@ -1,5 +1,10 @@
 export enum BucketNameEnum {
  dataset = 'dataset'
 }
+export const bucketNameMap = {
+ [BucketNameEnum.dataset]: {
+ label: 'common.file.bucket.dataset'
+ }
+};
 
 export const FileBaseUrl = '/api/common/file/read';

packages/global/common/file/read/index.ts

@@ -20,13 +20,13 @@ export const readPdfFile = async ({ pdf }: { pdf: string | URL | ArrayBuffer })
 
  const viewport = page.getViewport({ scale: 1 });
  const pageHeight = viewport.height;
- const headerThreshold = pageHeight * 0.07; // 假设页头在页面顶部5%的区域内
- const footerThreshold = pageHeight * 0.93; // 假设页脚在页面底部5%的区域内
+ const headerThreshold = pageHeight * 0.95;
+ const footerThreshold = pageHeight * 0.05;
 
  const pageTexts: TokenType[] = tokenizedText.items.filter((token: TokenType) => {
  return (
  !token.transform ||
- (token.transform[5] > headerThreshold && token.transform[5] < footerThreshold)
+ (token.transform[5] < headerThreshold && token.transform[5] > footerThreshold)
  );
  });
 

packages/service/common/file/upload/multer.ts

@@ -2,7 +2,7 @@ import type { NextApiRequest, NextApiResponse } from 'next';
 import { customAlphabet } from 'nanoid';
 import multer from 'multer';
 import path from 'path';
-import { BucketNameEnum } from '@fastgpt/global/common/file/constants';
+import { BucketNameEnum, bucketNameMap } from '@fastgpt/global/common/file/constants';
 import fs from 'fs';
 
 const nanoid = customAlphabet('1234567890abcdef', 12);
@@ -45,6 +45,12 @@ export function getUploadModel({ maxSize = 500 }: { maxSize?: number }) {
  return reject(error);
  }
 
+ // check bucket name
+ const bucketName = req.body?.bucketName as `${BucketNameEnum}`;
+ if (bucketName && !bucketNameMap[bucketName]) {
+ return reject('BucketName is invalid');
+ }
+
  resolve({
  ...req.body,
  files:

packages/service/common/vectorStore/controller.ts

@@ -48,7 +48,7 @@ export const updateDatasetDataVector = async ({
  // get vector
  const { vectors, tokens } = await getVectorsByText({
  model,
- input: [query]
+ input: query
  });
 
  await getVectorObj().update({

packages/service/core/ai/embedding/index.ts

@@ -2,32 +2,21 @@ import { getAIApi } from '../config';
 
 export type GetVectorProps = {
  model: string;
- input: string | string[];
+ input: string;
 };
 
 // text to vector
 export async function getVectorsByText({
  model = 'text-embedding-ada-002',
  input
 }: GetVectorProps) {
- if (typeof input === 'string' && !input) {
+ if (!input) {
  return Promise.reject({
  code: 500,
  message: 'input is empty'
  });
- } else if (Array.isArray(input)) {
- for (let i = 0; i < input.length; i++) {
- if (!input[i]) {
- return Promise.reject({
- code: 500,
- message: 'input array is empty'
- });
- }
- }
- }
- if (typeof input === 'string') {
- input = [input];
  }
+
  try {
  // 获取 chatAPI
  const ai = getAIApi();
@@ -36,7 +25,7 @@ export async function getVectorsByText({
  const result = await ai.embeddings
  .create({
  model,
- input
+ input: [input]
  })
  .then(async (res) => {
  if (!res.data) {
@@ -47,6 +36,7 @@ export async function getVectorsByText({
  // @ts-ignore
  return Promise.reject(res.data?.err?.message || 'Embedding API Error');
  }
+
  return {
  tokens: res.usage.total_tokens || 0,
  vectors: await Promise.all(res.data.map((item) => unityDimensional(item.embedding)))

packages/service/core/dataset/training/schema.ts

@@ -102,6 +102,7 @@ const TrainingDataSchema = new Schema({
 });
 
 try {
+ TrainingDataSchema.index({ weight: -1 });
  TrainingDataSchema.index({ lockTime: 1 });
  TrainingDataSchema.index({ datasetId: 1 });
  TrainingDataSchema.index({ collectionId: 1 });

packages/web/common/system/utils.ts

@@ -0,0 +1,9 @@
+import FingerprintJS from '@fingerprintjs/fingerprintjs';
+
+const fpPromise = FingerprintJS.load();
+
+export const getUserFingerprint = async () => {
+ const fp = await fpPromise;
+ const result = await fp.get();
+ console.log(result.visitorId);
+};

packages/web/package.json

@@ -2,9 +2,6 @@
  "name": "@fastgpt/web",
  "version": "1.0.0",
  "dependencies": {
- "@fastgpt/global": "workspace:*",
- "joplin-turndown-plugin-gfm": "^1.0.12",
- "turndown": "^7.1.2",
  "@chakra-ui/anatomy": "^2.2.1",
  "@chakra-ui/icons": "^2.1.1",
  "@chakra-ui/next-js": "^2.1.5",
@@ -13,16 +10,20 @@
  "@chakra-ui/system": "^2.6.1",
  "@emotion/react": "^11.11.1",
  "@emotion/styled": "^11.11.0",
+ "@fastgpt/global": "workspace:*",
+ "@fingerprintjs/fingerprintjs": "^4.2.1",
  "@monaco-editor/react": "^4.6.0",
- "react": "18.2.0",
- "react-dom": "18.2.0",
  "i18next": "^22.5.1",
+ "joplin-turndown-plugin-gfm": "^1.0.12",
  "next-i18next": "^13.3.0",
- "react-i18next": "^12.3.1"
+ "react": "18.2.0",
+ "react-dom": "18.2.0",
+ "react-i18next": "^12.3.1",
+ "turndown": "^7.1.2"
  },
  "devDependencies": {
- "@types/turndown": "^5.0.4",
  "@types/react": "18.2.0",
- "@types/react-dom": "18.2.0"
+ "@types/react-dom": "18.2.0",
+ "@types/turndown": "^5.0.4"
  }
 }

projects/app/public/locales/en/common.json

@@ -522,9 +522,13 @@
  },
  "score": {
  "embedding": "Embedding",
+ "embedding desc": "",
  "fullText": "Full text",
+ "fullText desc": "",
  "reRank": "ReRank",
- "rrf": "RRF Merge"
+ "reRank desc": "",
+ "rrf": "RRF Merge",
+ "rrf desc": ""
  },
  "search mode": "Search Mode"
  },
@@ -874,8 +878,6 @@
  "QPM": "QPM",
  "QPM Tips": "The maximum number of queries per IP address per minute",
  "QPM is empty": "QPM is empty",
- "Response Detail": "Quote",
- "Response Detail tips": "Whether detailed data such as references to be returned",
  "token auth": "Token Auth",
  "token auth Tips": "Identity verification server address. If this value is set, the server will be specified to send a request for identity verification before each session",
  "token auth use cases": "Review the authentication instructions"
@@ -903,6 +905,12 @@
  "Update Your Plugin": "Update Plugin"
  },
  "support": {
+ "outlink": {
+ "share": {
+ "Response Quote": "Show Quote",
+ "Response Quote tips": "The referenced content is returned in the share link, but the user is not allowed to download the original document."
+ }
+ },
  "user": {
  "Price": "Price",
  "auth": {

projects/app/public/locales/zh/common.json

@@ -878,8 +878,6 @@
  "QPM": "",
  "QPM Tips": "每个 IP 每分钟最多提问多少次",
  "QPM is empty": "QPM 不能为空",
- "Response Detail": "返回详情",
- "Response Detail tips": "是否需要返回详情（引用内容，调用时间等，不会返回预设提示词和完整上下文）",
  "token auth": "身份验证",
  "token auth Tips": "身份校验服务器地址，如填写该值，每次对话前都会向指定服务器发送一个请求，进行身份校验",
  "token auth use cases": "查看身份验证使用说明"
@@ -907,6 +905,12 @@
  "Update Your Plugin": "更新插件"
  },
  "support": {
+ "outlink": {
+ "share": {
+ "Response Quote": "返回引用",
+ "Response Quote tips": "在分享链接中返回引用内容，但不会允许用户下载原文档"
+ }
+ },
  "user": {
  "Price": "计费标准",
  "auth": {