From 976c849135d8a1543ea7fd8c2cf90f31662f52e1 Mon Sep 17 00:00:00 2001 From: Service Account Date: Thu, 28 Mar 2024 03:58:03 +0000 Subject: [PATCH 01/67] janhq/jan: Update README.md with nightly build artifact URL --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 3ad55c542e..4ac8e9715b 100644 --- a/README.md +++ b/README.md @@ -76,31 +76,31 @@ Jan is an open-source ChatGPT alternative that runs 100% offline on your compute Experimental (Nightly Build) - + jan.exe - + Intel - + M1/M2 - + jan.deb - + jan.AppImage From 54ba41000a9e6fcb821cae04ad42e529e3c99d84 Mon Sep 17 00:00:00 2001 From: Service Account Date: Thu, 28 Mar 2024 05:04:50 +0000 Subject: [PATCH 02/67] janhq/jan: Update README.md with nightly build artifact URL --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 4ac8e9715b..312369a109 100644 --- a/README.md +++ b/README.md @@ -76,31 +76,31 @@ Jan is an open-source ChatGPT alternative that runs 100% offline on your compute Experimental (Nightly Build) - + jan.exe - + Intel - + M1/M2 - + jan.deb - + jan.AppImage From 1f8dc893bab3ef5b5cd636274b8555c371b2edd4 Mon Sep 17 00:00:00 2001 From: NamH Date: Thu, 28 Mar 2024 13:54:38 +0700 Subject: [PATCH 03/67] fix: remove files and memory when user clean thread (#2524) Signed-off-by: James Co-authored-by: James --- web/hooks/useDeleteThread.ts | 77 +++++++++++++++++++++++++----------- 1 file changed, 53 insertions(+), 24 deletions(-) diff --git a/web/hooks/useDeleteThread.ts b/web/hooks/useDeleteThread.ts index 87cee125d2..62f3a65b55 100644 --- a/web/hooks/useDeleteThread.ts +++ b/web/hooks/useDeleteThread.ts @@ -1,7 +1,11 @@ +import { useCallback } from 'react' + import { ChatCompletionRole, ExtensionTypeEnum, ConversationalExtension, + fs, + joinPath, } from '@janhq/core' import { useAtom, useAtomValue, useSetAtom } from 'jotai' @@ -12,10 +16,11 @@ import { toaster } from '@/containers/Toast' import { extensionManager } from '@/extension/ExtensionManager' +import { janDataFolderPathAtom } from '@/helpers/atoms/AppConfig.atom' import { + chatMessages, cleanChatMessageAtom as cleanChatMessagesAtom, deleteChatMessageAtom as deleteChatMessagesAtom, - getCurrentChatMessagesAtom, } from '@/helpers/atoms/ChatMessage.atom' import { threadsAtom, @@ -26,39 +31,63 @@ import { export default function useDeleteThread() { const [threads, setThreads] = useAtom(threadsAtom) - const setCurrentPrompt = useSetAtom(currentPromptAtom) - const messages = useAtomValue(getCurrentChatMessagesAtom) + const messages = useAtomValue(chatMessages) + const janDataFolderPath = useAtomValue(janDataFolderPathAtom) + const setCurrentPrompt = useSetAtom(currentPromptAtom) const setActiveThreadId = useSetAtom(setActiveThreadIdAtom) const deleteMessages = useSetAtom(deleteChatMessagesAtom) const cleanMessages = useSetAtom(cleanChatMessagesAtom) + const deleteThreadState = useSetAtom(deleteThreadStateAtom) const updateThreadLastMessage = useSetAtom(updateThreadStateLastMessageAtom) - const cleanThread = async (threadId: string) => { - if (threadId) { - const thread = threads.filter((c) => c.id === threadId)[0] + const cleanThread = useCallback( + async (threadId: string) => { cleanMessages(threadId) + const thread = threads.find((c) => c.id === threadId) + if (!thread) return - if (thread) { - await extensionManager - .get(ExtensionTypeEnum.Conversational) - ?.writeMessages( - threadId, - messages.filter((msg) => msg.role === ChatCompletionRole.System) - ) - - thread.metadata = { - ...thread.metadata, - lastMessage: undefined, - } - await extensionManager - .get(ExtensionTypeEnum.Conversational) - ?.saveThread(thread) - updateThreadLastMessage(threadId, undefined) + const updatedMessages = (messages[threadId] ?? []).filter( + (msg) => msg.role === ChatCompletionRole.System + ) + + // remove files + try { + const threadFolderPath = await joinPath([ + janDataFolderPath, + 'threads', + threadId, + ]) + const threadFilesPath = await joinPath([threadFolderPath, 'files']) + const threadMemoryPath = await joinPath([threadFolderPath, 'memory']) + await fs.rm(threadFilesPath) + await fs.rm(threadMemoryPath) + } catch (err) { + console.warn('Error deleting thread files', err) } - } - } + + await extensionManager + .get(ExtensionTypeEnum.Conversational) + ?.writeMessages(threadId, updatedMessages) + + thread.metadata = { + ...thread.metadata, + lastMessage: undefined, + } + await extensionManager + .get(ExtensionTypeEnum.Conversational) + ?.saveThread(thread) + updateThreadLastMessage(threadId, undefined) + }, + [ + janDataFolderPath, + threads, + messages, + cleanMessages, + updateThreadLastMessage, + ] + ) const deleteThread = async (threadId: string) => { if (!threadId) { From a03e37f5a6d507b60345b590177f581f398de5ea Mon Sep 17 00:00:00 2001 From: Service Account Date: Thu, 28 Mar 2024 09:52:29 +0000 Subject: [PATCH 04/67] janhq/jan: Update README.md with nightly build artifact URL --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 312369a109..4d02344f8a 100644 --- a/README.md +++ b/README.md @@ -76,31 +76,31 @@ Jan is an open-source ChatGPT alternative that runs 100% offline on your compute Experimental (Nightly Build) - + jan.exe - + Intel - + M1/M2 - + jan.deb - + jan.AppImage From 3ecdb81881ab1d0e17b983dd2f7ab27e802851b1 Mon Sep 17 00:00:00 2001 From: NamH Date: Fri, 29 Mar 2024 01:24:53 +0700 Subject: [PATCH 05/67] fix: file explore on windows show empty when importing model (#2484) Signed-off-by: James Co-authored-by: James --- core/src/types/api/index.ts | 2 +- core/src/types/miscellaneous/index.ts | 3 +- core/src/types/miscellaneous/selectFiles.ts | 30 ++++++ electron/handlers/native.ts | 52 +++++++---- web/containers/Layout/index.tsx | 2 + web/hooks/useImportModel.ts | 86 +++++++++++++++++- .../ChooseWhatToImportModal/index.tsx | 65 +++++++++++++ .../Settings/ImportingModelModal/index.tsx | 4 +- .../Settings/SelectingModelModal/index.tsx | 91 ++++--------------- 9 files changed, 236 insertions(+), 99 deletions(-) create mode 100644 core/src/types/miscellaneous/selectFiles.ts create mode 100644 web/screens/Settings/ChooseWhatToImportModal/index.tsx diff --git a/core/src/types/api/index.ts b/core/src/types/api/index.ts index 91d6ae755a..d95d0474e1 100644 --- a/core/src/types/api/index.ts +++ b/core/src/types/api/index.ts @@ -7,7 +7,7 @@ export enum NativeRoute { openAppDirectory = 'openAppDirectory', openFileExplore = 'openFileExplorer', selectDirectory = 'selectDirectory', - selectModelFiles = 'selectModelFiles', + selectFiles = 'selectFiles', relaunch = 'relaunch', hideQuickAskWindow = 'hideQuickAskWindow', diff --git a/core/src/types/miscellaneous/index.ts b/core/src/types/miscellaneous/index.ts index b4ef68ab61..2693ffd8b1 100644 --- a/core/src/types/miscellaneous/index.ts +++ b/core/src/types/miscellaneous/index.ts @@ -2,4 +2,5 @@ export * from './systemResourceInfo' export * from './promptTemplate' export * from './appUpdate' export * from './fileDownloadRequest' -export * from './networkConfig' \ No newline at end of file +export * from './networkConfig' +export * from './selectFiles' diff --git a/core/src/types/miscellaneous/selectFiles.ts b/core/src/types/miscellaneous/selectFiles.ts new file mode 100644 index 0000000000..3120be24e3 --- /dev/null +++ b/core/src/types/miscellaneous/selectFiles.ts @@ -0,0 +1,30 @@ +export type SelectFileOption = { + /** + * The title of the dialog. + */ + title?: string + /** + * Whether the dialog allows multiple selection. + */ + allowMultiple?: boolean + + buttonLabel?: string + + selectDirectory?: boolean + + props?: SelectFileProp[] +} + +export const SelectFilePropTuple = [ + 'openFile', + 'openDirectory', + 'multiSelections', + 'showHiddenFiles', + 'createDirectory', + 'promptToCreate', + 'noResolveAliases', + 'treatPackageAsDirectory', + 'dontAddToRecent', +] as const + +export type SelectFileProp = (typeof SelectFilePropTuple)[number] diff --git a/electron/handlers/native.ts b/electron/handlers/native.ts index 06d9d2a6ad..34bfeffa3d 100644 --- a/electron/handlers/native.ts +++ b/electron/handlers/native.ts @@ -6,8 +6,11 @@ import { getJanDataFolderPath, getJanExtensionsPath, init, - AppEvent, NativeRoute, + AppEvent, + NativeRoute, + SelectFileProp, } from '@janhq/core/node' +import { SelectFileOption } from '@janhq/core/.' export function handleAppIPCs() { /** @@ -84,23 +87,38 @@ export function handleAppIPCs() { } }) - ipcMain.handle(NativeRoute.selectModelFiles, async () => { - const mainWindow = windowManager.mainWindow - if (!mainWindow) { - console.error('No main window found') - return - } - const { canceled, filePaths } = await dialog.showOpenDialog(mainWindow, { - title: 'Select model files', - buttonLabel: 'Select', - properties: ['openFile', 'openDirectory', 'multiSelections'], - }) - if (canceled) { - return - } + ipcMain.handle( + NativeRoute.selectFiles, + async (_event, option?: SelectFileOption) => { + const mainWindow = windowManager.mainWindow + if (!mainWindow) { + console.error('No main window found') + return + } - return filePaths - }) + const title = option?.title ?? 'Select files' + const buttonLabel = option?.buttonLabel ?? 'Select' + const props: SelectFileProp[] = ['openFile'] + + if (option?.allowMultiple) { + props.push('multiSelections') + } + + if (option?.selectDirectory) { + props.push('openDirectory') + } + console.debug(`Select files with props: ${props}`) + const { canceled, filePaths } = await dialog.showOpenDialog(mainWindow, { + title, + buttonLabel, + properties: props, + }) + + if (canceled) return + + return filePaths + } + ) ipcMain.handle( NativeRoute.hideQuickAskWindow, diff --git a/web/containers/Layout/index.tsx b/web/containers/Layout/index.tsx index fb08bc6acd..c87b6cacc9 100644 --- a/web/containers/Layout/index.tsx +++ b/web/containers/Layout/index.tsx @@ -17,6 +17,7 @@ import { getImportModelStageAtom } from '@/hooks/useImportModel' import { SUCCESS_SET_NEW_DESTINATION } from '@/screens/Settings/Advanced/DataFolder' import CancelModelImportModal from '@/screens/Settings/CancelModelImportModal' +import ChooseWhatToImportModal from '@/screens/Settings/ChooseWhatToImportModal' import EditModelInfoModal from '@/screens/Settings/EditModelInfoModal' import ImportModelOptionModal from '@/screens/Settings/ImportModelOptionModal' import ImportingModelModal from '@/screens/Settings/ImportingModelModal' @@ -70,6 +71,7 @@ const BaseLayout = (props: PropsWithChildren) => { {importModelStage === 'IMPORTING_MODEL' && } {importModelStage === 'EDIT_MODEL_INFO' && } {importModelStage === 'CONFIRM_CANCEL' && } + ) diff --git a/web/hooks/useImportModel.ts b/web/hooks/useImportModel.ts index d4b6f2919b..170f03b5ea 100644 --- a/web/hooks/useImportModel.ts +++ b/web/hooks/useImportModel.ts @@ -6,15 +6,26 @@ import { Model, ModelExtension, OptionType, + baseName, + fs, + joinPath, } from '@janhq/core' -import { atom } from 'jotai' +import { atom, useSetAtom } from 'jotai' + +import { v4 as uuidv4 } from 'uuid' + +import { snackbar } from '@/containers/Toast' + +import { FilePathWithSize } from '@/utils/file' import { extensionManager } from '@/extension' +import { importingModelsAtom } from '@/helpers/atoms/Model.atom' export type ImportModelStage = | 'NONE' | 'SELECTING_MODEL' + | 'CHOOSE_WHAT_TO_IMPORT' | 'MODEL_SELECTED' | 'IMPORTING_MODEL' | 'EDIT_MODEL_INFO' @@ -38,6 +49,9 @@ export type ModelUpdate = { } const useImportModel = () => { + const setImportModelStage = useSetAtom(setImportModelStageAtom) + const setImportingModels = useSetAtom(importingModelsAtom) + const importModels = useCallback( (models: ImportingModel[], optionType: OptionType) => localImportModels(models, optionType), @@ -49,7 +63,75 @@ const useImportModel = () => { [] ) - return { importModels, updateModelInfo } + const sanitizeFilePaths = useCallback( + async (filePaths: string[]) => { + if (!filePaths || filePaths.length === 0) return + + const sanitizedFilePaths: FilePathWithSize[] = [] + for (const filePath of filePaths) { + const fileStats = await fs.fileStat(filePath, true) + if (!fileStats) continue + + if (!fileStats.isDirectory) { + const fileName = await baseName(filePath) + sanitizedFilePaths.push({ + path: filePath, + name: fileName, + size: fileStats.size, + }) + } else { + // allowing only one level of directory + const files = await fs.readdirSync(filePath) + + for (const file of files) { + const fullPath = await joinPath([filePath, file]) + const fileStats = await fs.fileStat(fullPath, true) + if (!fileStats || fileStats.isDirectory) continue + + sanitizedFilePaths.push({ + path: fullPath, + name: file, + size: fileStats.size, + }) + } + } + } + + const unsupportedFiles = sanitizedFilePaths.filter( + (file) => !file.path.endsWith('.gguf') + ) + const supportedFiles = sanitizedFilePaths.filter((file) => + file.path.endsWith('.gguf') + ) + + const importingModels: ImportingModel[] = supportedFiles.map( + ({ path, name, size }: FilePathWithSize) => ({ + importId: uuidv4(), + modelId: undefined, + name: name.replace('.gguf', ''), + description: '', + path: path, + tags: [], + size: size, + status: 'PREPARING', + format: 'gguf', + }) + ) + if (unsupportedFiles.length > 0) { + snackbar({ + description: `Only files with .gguf extension can be imported.`, + type: 'error', + }) + } + if (importingModels.length === 0) return + + setImportingModels(importingModels) + setImportModelStage('MODEL_SELECTED') + }, + [setImportModelStage, setImportingModels] + ) + + return { importModels, updateModelInfo, sanitizeFilePaths } } const localImportModels = async ( diff --git a/web/screens/Settings/ChooseWhatToImportModal/index.tsx b/web/screens/Settings/ChooseWhatToImportModal/index.tsx new file mode 100644 index 0000000000..8aa4169920 --- /dev/null +++ b/web/screens/Settings/ChooseWhatToImportModal/index.tsx @@ -0,0 +1,65 @@ +import { useCallback } from 'react' + +import { SelectFileOption } from '@janhq/core' +import { + Button, + Modal, + ModalContent, + ModalHeader, + ModalTitle, +} from '@janhq/uikit' +import { useSetAtom, useAtomValue } from 'jotai' + +import useImportModel, { + setImportModelStageAtom, + getImportModelStageAtom, +} from '@/hooks/useImportModel' + +const ChooseWhatToImportModal: React.FC = () => { + const setImportModelStage = useSetAtom(setImportModelStageAtom) + const importModelStage = useAtomValue(getImportModelStageAtom) + const { sanitizeFilePaths } = useImportModel() + + const onImportFileClick = useCallback(async () => { + const options: SelectFileOption = { + title: 'Select model files', + buttonLabel: 'Select', + allowMultiple: true, + } + const filePaths = await window.core?.api?.selectFiles(options) + if (!filePaths || filePaths.length === 0) return + sanitizeFilePaths(filePaths) + }, [sanitizeFilePaths]) + + const onImportFolderClick = useCallback(async () => { + const options: SelectFileOption = { + title: 'Select model folders', + buttonLabel: 'Select', + allowMultiple: true, + selectDirectory: true, + } + const filePaths = await window.core?.api?.selectFiles(options) + if (!filePaths || filePaths.length === 0) return + sanitizeFilePaths(filePaths) + }, [sanitizeFilePaths]) + + return ( + setImportModelStage('SELECTING_MODEL')} + > + + + Choose what to import + + +
+ + +
+ + + ) +} + +export default ChooseWhatToImportModal diff --git a/web/screens/Settings/ImportingModelModal/index.tsx b/web/screens/Settings/ImportingModelModal/index.tsx index f621c2fb74..3bf9e4de27 100644 --- a/web/screens/Settings/ImportingModelModal/index.tsx +++ b/web/screens/Settings/ImportingModelModal/index.tsx @@ -52,9 +52,7 @@ const ImportingModelModal: React.FC = () => { return ( { - setImportModelStage('NONE') - }} + onOpenChange={() => setImportModelStage('NONE')} > diff --git a/web/screens/Settings/SelectingModelModal/index.tsx b/web/screens/Settings/SelectingModelModal/index.tsx index 7579e0c3c1..6bf28cd008 100644 --- a/web/screens/Settings/SelectingModelModal/index.tsx +++ b/web/screens/Settings/SelectingModelModal/index.tsx @@ -1,99 +1,40 @@ import { useCallback } from 'react' import { useDropzone } from 'react-dropzone' -import { ImportingModel, baseName, fs, joinPath } from '@janhq/core' +import { SelectFileOption, systemInformation } from '@janhq/core' import { Modal, ModalContent, ModalHeader, ModalTitle } from '@janhq/uikit' import { useAtomValue, useSetAtom } from 'jotai' import { UploadCloudIcon } from 'lucide-react' -import { v4 as uuidv4 } from 'uuid' - -import { snackbar } from '@/containers/Toast' - import useDropModelBinaries from '@/hooks/useDropModelBinaries' -import { +import useImportModel, { getImportModelStageAtom, setImportModelStageAtom, } from '@/hooks/useImportModel' -import { FilePathWithSize } from '@/utils/file' - -import { importingModelsAtom } from '@/helpers/atoms/Model.atom' - const SelectingModelModal: React.FC = () => { const setImportModelStage = useSetAtom(setImportModelStageAtom) const importModelStage = useAtomValue(getImportModelStageAtom) - const setImportingModels = useSetAtom(importingModelsAtom) const { onDropModels } = useDropModelBinaries() + const { sanitizeFilePaths } = useImportModel() const onSelectFileClick = useCallback(async () => { - const filePaths = await window.core?.api?.selectModelFiles() - if (!filePaths || filePaths.length === 0) return - - const sanitizedFilePaths: FilePathWithSize[] = [] - for (const filePath of filePaths) { - const fileStats = await fs.fileStat(filePath, true) - if (!fileStats) continue - - if (!fileStats.isDirectory) { - const fileName = await baseName(filePath) - sanitizedFilePaths.push({ - path: filePath, - name: fileName, - size: fileStats.size, - }) - } else { - // allowing only one level of directory - const files = await fs.readdirSync(filePath) - - for (const file of files) { - const fullPath = await joinPath([filePath, file]) - const fileStats = await fs.fileStat(fullPath, true) - if (!fileStats || fileStats.isDirectory) continue - - sanitizedFilePaths.push({ - path: fullPath, - name: file, - size: fileStats.size, - }) - } - } + const platform = (await systemInformation()).osInfo?.platform + if (platform === 'win32') { + setImportModelStage('CHOOSE_WHAT_TO_IMPORT') + return } - - const unsupportedFiles = sanitizedFilePaths.filter( - (file) => !file.path.endsWith('.gguf') - ) - const supportedFiles = sanitizedFilePaths.filter((file) => - file.path.endsWith('.gguf') - ) - - const importingModels: ImportingModel[] = supportedFiles.map( - ({ path, name, size }: FilePathWithSize) => { - return { - importId: uuidv4(), - modelId: undefined, - name: name.replace('.gguf', ''), - description: '', - path: path, - tags: [], - size: size, - status: 'PREPARING', - format: 'gguf', - } - } - ) - if (unsupportedFiles.length > 0) { - snackbar({ - description: `Only files with .gguf extension can be imported.`, - type: 'error', - }) + const options: SelectFileOption = { + title: 'Select model folders', + buttonLabel: 'Select', + allowMultiple: true, + selectDirectory: true, } - if (importingModels.length === 0) return - - setImportingModels(importingModels) - setImportModelStage('MODEL_SELECTED') - }, [setImportingModels, setImportModelStage]) + const filePaths = await window.core?.api?.selectFiles(options) + if (!filePaths || filePaths.length === 0) return + sanitizeFilePaths(filePaths) + }, [sanitizeFilePaths, setImportModelStage]) const { isDragActive, getRootProps } = useDropzone({ noClick: true, From e60807f867c24a5b3c9587bf93740e82b8298dcc Mon Sep 17 00:00:00 2001 From: hiro <22463238+hiro-v@users.noreply.github.com> Date: Fri, 29 Mar 2024 09:50:35 +0700 Subject: [PATCH 06/67] chore: Refactor docker compose (#2531) * chore: docker compose dev with local build * chore: docker compose with prebuilt image on ghcr * fix: Update readme and by comments --- README.md | 1 + docker-compose-dev.yml | 171 +++++++++++++++++++++++++++++++++++++++++ docker-compose.yml | 69 +++++++---------- 3 files changed, 200 insertions(+), 41 deletions(-) create mode 100644 docker-compose-dev.yml diff --git a/README.md b/README.md index 4d02344f8a..46500c0a53 100644 --- a/README.md +++ b/README.md @@ -240,6 +240,7 @@ This will build the app MacOS m1/m2 for production (with code signing already do - If you intend to run Jan in GPU mode, you need to install `nvidia-driver` and `nvidia-docker2`. Follow the instruction [here](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html) for installation. - Run Jan in Docker mode + > User can choose between `docker-compose.yml` with latest prebuilt docker image or `docker-compose-dev.yml` with local docker build | Docker compose Profile | Description | | ---------------------- | -------------------------------------------- | diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml new file mode 100644 index 0000000000..2e09d641b3 --- /dev/null +++ b/docker-compose-dev.yml @@ -0,0 +1,171 @@ +# Docker Compose file for setting up Minio, createbuckets, app_cpu, and app_gpu services + +version: '3.7' + +services: + # Minio service for object storage + minio: + image: minio/minio + volumes: + - minio_data:/data + ports: + - '9000:9000' + - '9001:9001' + environment: + # Set the root user and password for Minio + MINIO_ROOT_USER: minioadmin # This acts as AWS_ACCESS_KEY + MINIO_ROOT_PASSWORD: minioadmin # This acts as AWS_SECRET_ACCESS_KEY + command: server --console-address ":9001" /data + restart: always + healthcheck: + test: ['CMD', 'curl', '-f', 'http://localhost:9000/minio/health/live'] + interval: 30s + timeout: 20s + retries: 3 + networks: + vpcbr: + ipv4_address: 10.5.0.2 + + # createbuckets service to create a bucket and set its policy + createbuckets: + image: minio/mc + depends_on: + - minio + entrypoint: > + /bin/sh -c " + /usr/bin/mc alias set myminio http://minio:9000 minioadmin minioadmin; + /usr/bin/mc mb myminio/mybucket; + /usr/bin/mc policy set public myminio/mybucket; + exit 0; + " + networks: + vpcbr: + + # app_cpu service for running the CPU version of the application + app_cpu_s3fs: + image: jan:latest + volumes: + - app_data_cpu_s3fs:/app/server/build/jan + build: + context: . + dockerfile: Dockerfile + environment: + # Set the AWS access key, secret access key, bucket name, endpoint, and region for app_cpu + AWS_ACCESS_KEY_ID: minioadmin + AWS_SECRET_ACCESS_KEY: minioadmin + S3_BUCKET_NAME: mybucket + AWS_ENDPOINT: http://10.5.0.2:9000 + AWS_REGION: us-east-1 + API_BASE_URL: http://localhost:1337 + restart: always + profiles: + - cpu-s3fs + ports: + - '3000:3000' + - '1337:1337' + - '3928:3928' + networks: + vpcbr: + ipv4_address: 10.5.0.3 + + # app_gpu service for running the GPU version of the application + app_gpu_s3fs: + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: all + capabilities: [gpu] + image: jan-gpu:latest + volumes: + - app_data_gpu_s3fs:/app/server/build/jan + build: + context: . + dockerfile: Dockerfile.gpu + restart: always + environment: + # Set the AWS access key, secret access key, bucket name, endpoint, and region for app_gpu + AWS_ACCESS_KEY_ID: minioadmin + AWS_SECRET_ACCESS_KEY: minioadmin + S3_BUCKET_NAME: mybucket + AWS_ENDPOINT: http://10.5.0.2:9000 + AWS_REGION: us-east-1 + API_BASE_URL: http://localhost:1337 + profiles: + - gpu-s3fs + ports: + - '3000:3000' + - '1337:1337' + - '3928:3928' + networks: + vpcbr: + ipv4_address: 10.5.0.4 + + app_cpu_fs: + image: jan:latest + volumes: + - app_data_cpu_fs:/app/server/build/jan + build: + context: . + dockerfile: Dockerfile + environment: + API_BASE_URL: http://localhost:1337 + restart: always + profiles: + - cpu-fs + ports: + - '3000:3000' + - '1337:1337' + - '3928:3928' + networks: + vpcbr: + ipv4_address: 10.5.0.5 + + # app_gpu service for running the GPU version of the application + app_gpu_fs: + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: all + capabilities: [gpu] + image: jan-gpu:latest + volumes: + - app_data_gpu_fs:/app/server/build/jan + build: + context: . + dockerfile: Dockerfile.gpu + restart: always + environment: + API_BASE_URL: http://localhost:1337 + profiles: + - gpu-fs + ports: + - '3000:3000' + - '1337:1337' + - '3928:3928' + networks: + vpcbr: + ipv4_address: 10.5.0.6 + +volumes: + minio_data: + app_data_cpu_s3fs: + app_data_gpu_s3fs: + app_data_cpu_fs: + app_data_gpu_fs: + +networks: + vpcbr: + driver: bridge + ipam: + config: + - subnet: 10.5.0.0/16 + gateway: 10.5.0.1 +# Usage: +# - Run 'docker compose -f docker-compose-dev.yml --profile cpu-s3fs up -d' to start the app_cpu service +# - Run 'docker compose -f docker-compose-dev.yml --profile gpu-s3fs up -d' to start the app_gpu service +# - Run 'docker compose -f docker-compose-dev.yml --profile cpu-fs up -d' to start the app_cpu service +# - Run 'docker compose -f docker-compose-dev.yml --profile gpu-fs up -d' to start the app_gpu service diff --git a/docker-compose.yml b/docker-compose.yml index 1691a841a0..1e5660c12b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,8 +9,8 @@ services: volumes: - minio_data:/data ports: - - "9000:9000" - - "9001:9001" + - '9000:9000' + - '9001:9001' environment: # Set the root user and password for Minio MINIO_ROOT_USER: minioadmin # This acts as AWS_ACCESS_KEY @@ -18,7 +18,7 @@ services: command: server --console-address ":9001" /data restart: always healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] + test: ['CMD', 'curl', '-f', 'http://localhost:9000/minio/health/live'] interval: 30s timeout: 20s retries: 3 @@ -43,12 +43,9 @@ services: # app_cpu service for running the CPU version of the application app_cpu_s3fs: - image: jan:latest volumes: - app_data_cpu_s3fs:/app/server/build/jan - build: - context: . - dockerfile: Dockerfile + image: ghcr.io/janhq/jan-server:dev-cpu-latest environment: # Set the AWS access key, secret access key, bucket name, endpoint, and region for app_cpu AWS_ACCESS_KEY_ID: minioadmin @@ -61,9 +58,9 @@ services: profiles: - cpu-s3fs ports: - - "3000:3000" - - "1337:1337" - - "3928:3928" + - '3000:3000' + - '1337:1337' + - '3928:3928' networks: vpcbr: ipv4_address: 10.5.0.3 @@ -74,15 +71,12 @@ services: resources: reservations: devices: - - driver: nvidia - count: all - capabilities: [gpu] - image: jan-gpu:latest + - driver: nvidia + count: all + capabilities: [gpu] + image: ghcr.io/janhq/jan-server:dev-cuda-12.2-latest volumes: - app_data_gpu_s3fs:/app/server/build/jan - build: - context: . - dockerfile: Dockerfile.gpu restart: always environment: # Set the AWS access key, secret access key, bucket name, endpoint, and region for app_gpu @@ -95,29 +89,26 @@ services: profiles: - gpu-s3fs ports: - - "3000:3000" - - "1337:1337" - - "3928:3928" + - '3000:3000' + - '1337:1337' + - '3928:3928' networks: vpcbr: ipv4_address: 10.5.0.4 app_cpu_fs: - image: jan:latest + image: ghcr.io/janhq/jan-server:dev-cpu-latest volumes: - app_data_cpu_fs:/app/server/build/jan - build: - context: . - dockerfile: Dockerfile environment: API_BASE_URL: http://localhost:1337 restart: always profiles: - cpu-fs ports: - - "3000:3000" - - "1337:1337" - - "3928:3928" + - '3000:3000' + - '1337:1337' + - '3928:3928' networks: vpcbr: ipv4_address: 10.5.0.5 @@ -128,24 +119,21 @@ services: resources: reservations: devices: - - driver: nvidia - count: all - capabilities: [gpu] - image: jan-gpu:latest + - driver: nvidia + count: all + capabilities: [gpu] + image: ghcr.io/janhq/jan-server:dev-cuda-12.2-latest volumes: - app_data_gpu_fs:/app/server/build/jan - build: - context: . - dockerfile: Dockerfile.gpu restart: always environment: API_BASE_URL: http://localhost:1337 profiles: - gpu-fs ports: - - "3000:3000" - - "1337:1337" - - "3928:3928" + - '3000:3000' + - '1337:1337' + - '3928:3928' networks: vpcbr: ipv4_address: 10.5.0.6 @@ -161,10 +149,9 @@ networks: vpcbr: driver: bridge ipam: - config: - - subnet: 10.5.0.0/16 - gateway: 10.5.0.1 - + config: + - subnet: 10.5.0.0/16 + gateway: 10.5.0.1 # Usage: # - Run 'docker compose --profile cpu-s3fs up -d' to start the app_cpu service # - Run 'docker compose --profile gpu-s3fs up -d' to start the app_gpu service From ec6bcf6357d7fa81b4d9628c1b3f46a3f52ccf9b Mon Sep 17 00:00:00 2001 From: Service Account Date: Fri, 29 Mar 2024 05:08:55 +0000 Subject: [PATCH 07/67] janhq/jan: Update README.md with nightly build artifact URL --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 46500c0a53..4ad8f3edb5 100644 --- a/README.md +++ b/README.md @@ -76,31 +76,31 @@ Jan is an open-source ChatGPT alternative that runs 100% offline on your compute Experimental (Nightly Build) - + jan.exe - + Intel - + M1/M2 - + jan.deb - + jan.AppImage From fa35aa6e148ed1526e07777141d3c5ee5104f841 Mon Sep 17 00:00:00 2001 From: NamH Date: Fri, 29 Mar 2024 15:44:46 +0700 Subject: [PATCH 08/67] feat: dynamically register extension settings (#2494) * feat: add extesion settings Signed-off-by: James --------- Signed-off-by: James Co-authored-by: James Co-authored-by: Louis --- core/src/browser/extension.ts | 129 +++++++- .../browser/extensions/engines/OAIEngine.ts | 8 +- .../extensions/engines/RemoteOAIEngine.ts | 11 +- core/src/node/api/common/handler.ts | 9 +- core/src/node/api/restful/helper/builder.ts | 3 +- core/src/node/helper/config.ts | 30 +- core/src/types/index.ts | 1 + core/src/types/setting/index.ts | 1 + core/src/types/setting/settingComponent.ts | 34 ++ .../assistant-extension/rollup.config.ts | 6 +- .../assistant-extension/src/node/engine.ts | 35 ++- .../resources/settings.json | 23 ++ .../inference-groq-extension/src/index.ts | 77 ++--- .../webpack.config.js | 3 +- .../resources/default_settings.json | 33 ++ .../rollup.config.ts | 7 +- .../src/@types/global.d.ts | 2 + .../inference-nitro-extension/src/index.ts | 2 - .../resources/settings.json | 23 ++ .../inference-openai-extension/src/index.ts | 82 ++--- .../webpack.config.js | 3 +- .../resources/settings.json | 23 ++ .../src/index.ts | 69 ++-- .../webpack.config.js | 2 + extensions/model-extension/rollup.config.ts | 5 +- .../tensorrt-llm-extension/src/index.ts | 2 +- web/containers/DropdownListSidebar/index.tsx | 295 +++++++++--------- web/containers/ModelConfigInput/index.tsx | 52 ++- web/containers/OpenAiKeyInput/index.tsx | 84 ----- web/containers/Providers/DataLoader.tsx | 11 + web/extension/ExtensionManager.ts | 8 +- web/helpers/atoms/Setting.atom.ts | 7 + web/helpers/atoms/Thread.atom.ts | 7 - web/hooks/useActiveModel.ts | 22 +- web/hooks/useCreateNewThread.ts | 17 +- web/hooks/useEngineSettings.ts | 78 ----- web/hooks/useSendChatMessage.ts | 1 - web/hooks/useUpdateModelParameters.ts | 90 +++--- web/screens/Chat/AssistantSetting/index.tsx | 149 +++++---- .../Chat/ChatBody/EmptyModel/index.tsx | 30 ++ .../Chat/ChatBody/EmptyThread/index.tsx | 46 +++ web/screens/Chat/ChatBody/index.tsx | 98 ++---- web/screens/Chat/EngineSetting/index.tsx | 68 ++-- web/screens/Chat/ErrorMessage/index.tsx | 31 +- .../Chat/ModelSetting/SettingComponent.tsx | 203 ++++-------- web/screens/Chat/ModelSetting/index.tsx | 66 ++-- .../Chat/ModelSetting/predefinedComponent.ts | 104 +++--- .../Chat/Sidebar/AssistantTool/index.tsx | 196 ++++++++++++ .../Sidebar/PromptTemplateSetting/index.tsx | 50 +++ web/screens/Chat/Sidebar/index.tsx | 287 ++++------------- web/screens/LocalServer/index.tsx | 27 +- .../Settings/ExtensionSetting/index.tsx | 62 ++++ .../Settings/SelectingModelModal/index.tsx | 2 +- .../SettingDetailTextInputItem/index.tsx | 54 ++++ .../SettingDetail/SettingDetailItem/index.tsx | 45 +++ web/screens/Settings/SettingDetail/index.tsx | 32 ++ .../SettingMenu/SettingItem/index.tsx | 44 +++ web/screens/Settings/SettingMenu/index.tsx | 88 +++--- web/screens/Settings/index.tsx | 49 ++- web/utils/componentSettings.ts | 33 +- web/utils/errorMessage.ts | 15 - 61 files changed, 1742 insertions(+), 1332 deletions(-) create mode 100644 core/src/types/setting/index.ts create mode 100644 core/src/types/setting/settingComponent.ts create mode 100644 extensions/inference-groq-extension/resources/settings.json create mode 100644 extensions/inference-nitro-extension/resources/default_settings.json create mode 100644 extensions/inference-openai-extension/resources/settings.json create mode 100644 extensions/inference-triton-trtllm-extension/resources/settings.json delete mode 100644 web/containers/OpenAiKeyInput/index.tsx create mode 100644 web/helpers/atoms/Setting.atom.ts delete mode 100644 web/hooks/useEngineSettings.ts create mode 100644 web/screens/Chat/ChatBody/EmptyModel/index.tsx create mode 100644 web/screens/Chat/ChatBody/EmptyThread/index.tsx create mode 100644 web/screens/Chat/Sidebar/AssistantTool/index.tsx create mode 100644 web/screens/Chat/Sidebar/PromptTemplateSetting/index.tsx create mode 100644 web/screens/Settings/ExtensionSetting/index.tsx create mode 100644 web/screens/Settings/SettingDetail/SettingDetailItem/SettingDetailTextInputItem/index.tsx create mode 100644 web/screens/Settings/SettingDetail/SettingDetailItem/index.tsx create mode 100644 web/screens/Settings/SettingDetail/index.tsx create mode 100644 web/screens/Settings/SettingMenu/SettingItem/index.tsx delete mode 100644 web/utils/errorMessage.ts diff --git a/core/src/browser/extension.ts b/core/src/browser/extension.ts index 973d4778a7..13f0e94f3d 100644 --- a/core/src/browser/extension.ts +++ b/core/src/browser/extension.ts @@ -1,3 +1,7 @@ +import { SettingComponentProps } from '../types' +import { getJanDataFolderPath, joinPath } from './core' +import { fs } from './fs' + export enum ExtensionTypeEnum { Assistant = 'assistant', Conversational = 'conversational', @@ -32,6 +36,38 @@ export type InstallationState = InstallationStateTuple[number] * This class should be extended by any class that represents an extension. */ export abstract class BaseExtension implements ExtensionType { + protected settingFolderName = 'settings' + protected settingFileName = 'settings.json' + + /** @type {string} Name of the extension. */ + name?: string + + /** @type {string} The URL of the extension to load. */ + url: string + + /** @type {boolean} Whether the extension is activated or not. */ + active + + /** @type {string} Extension's description. */ + description + + /** @type {string} Extension's version. */ + version + + constructor( + url: string, + name?: string, + active?: boolean, + description?: string, + version?: string + ) { + this.name = name + this.url = url + this.active = active + this.description = description + this.version = version + } + /** * Returns the type of the extension. * @returns {ExtensionType} The type of the extension @@ -40,11 +76,13 @@ export abstract class BaseExtension implements ExtensionType { type(): ExtensionTypeEnum | undefined { return undefined } + /** * Called when the extension is loaded. * Any initialization logic for the extension should be put here. */ abstract onLoad(): void + /** * Called when the extension is unloaded. * Any cleanup logic for the extension should be put here. @@ -67,6 +105,42 @@ export abstract class BaseExtension implements ExtensionType { return false } + async registerSettings(settings: SettingComponentProps[]): Promise { + if (!this.name) { + console.error('Extension name is not defined') + return + } + + const extensionSettingFolderPath = await joinPath([ + await getJanDataFolderPath(), + 'settings', + this.name, + ]) + settings.forEach((setting) => { + setting.extensionName = this.name + }) + try { + await fs.mkdir(extensionSettingFolderPath) + const settingFilePath = await joinPath([extensionSettingFolderPath, this.settingFileName]) + + if (await fs.existsSync(settingFilePath)) return + await fs.writeFileSync(settingFilePath, JSON.stringify(settings, null, 2)) + } catch (err) { + console.error(err) + } + } + + async getSetting(key: string, defaultValue: T) { + const keySetting = (await this.getSettings()).find((setting) => setting.key === key) + + const value = keySetting?.controllerProps.value + return (value as T) ?? defaultValue + } + + onSettingUpdate(key: string, value: T) { + return + } + /** * Determine if the prerequisites for the extension are installed. * @@ -81,8 +155,59 @@ export abstract class BaseExtension implements ExtensionType { * * @returns {Promise} */ - // @ts-ignore - async install(...args): Promise { + async install(): Promise { return } + + async getSettings(): Promise { + if (!this.name) return [] + + const settingPath = await joinPath([ + await getJanDataFolderPath(), + this.settingFolderName, + this.name, + this.settingFileName, + ]) + + try { + const content = await fs.readFileSync(settingPath, 'utf-8') + const settings: SettingComponentProps[] = JSON.parse(content) + return settings + } catch (err) { + console.warn(err) + return [] + } + } + + async updateSettings(componentProps: Partial[]): Promise { + if (!this.name) return + + const settings = await this.getSettings() + + const updatedSettings = settings.map((setting) => { + const updatedSetting = componentProps.find( + (componentProp) => componentProp.key === setting.key + ) + if (updatedSetting && updatedSetting.controllerProps) { + setting.controllerProps.value = updatedSetting.controllerProps.value + } + return setting + }) + + const settingPath = await joinPath([ + await getJanDataFolderPath(), + this.settingFolderName, + this.name, + this.settingFileName, + ]) + + await fs.writeFileSync(settingPath, JSON.stringify(updatedSettings, null, 2)) + + updatedSettings.forEach((setting) => { + this.onSettingUpdate( + setting.key, + setting.controllerProps.value + ) + }) + } } diff --git a/core/src/browser/extensions/engines/OAIEngine.ts b/core/src/browser/extensions/engines/OAIEngine.ts index 41b08f4598..12bf81d363 100644 --- a/core/src/browser/extensions/engines/OAIEngine.ts +++ b/core/src/browser/extensions/engines/OAIEngine.ts @@ -48,7 +48,7 @@ export abstract class OAIEngine extends AIEngine { /* * Inference request */ - override inference(data: MessageRequest) { + override async inference(data: MessageRequest) { if (data.model?.engine?.toString() !== this.provider) return const timestamp = Date.now() @@ -77,12 +77,14 @@ export abstract class OAIEngine extends AIEngine { ...data.model, } + const header = await this.headers() + requestInference( this.inferenceUrl, data.messages ?? [], model, this.controller, - this.headers() + header ).subscribe({ next: (content: any) => { const messageContent: ThreadContent = { @@ -123,7 +125,7 @@ export abstract class OAIEngine extends AIEngine { /** * Headers for the inference request */ - headers(): HeadersInit { + async headers(): Promise { return {} } } diff --git a/core/src/browser/extensions/engines/RemoteOAIEngine.ts b/core/src/browser/extensions/engines/RemoteOAIEngine.ts index 2d5126c6b9..b112353707 100644 --- a/core/src/browser/extensions/engines/RemoteOAIEngine.ts +++ b/core/src/browser/extensions/engines/RemoteOAIEngine.ts @@ -5,8 +5,7 @@ import { OAIEngine } from './OAIEngine' * Added the implementation of loading and unloading model (applicable to local inference providers) */ export abstract class RemoteOAIEngine extends OAIEngine { - // The inference engine - abstract apiKey: string + apiKey?: string /** * On extension load, subscribe to events. */ @@ -17,10 +16,12 @@ export abstract class RemoteOAIEngine extends OAIEngine { /** * Headers for the inference request */ - override headers(): HeadersInit { + override async headers(): Promise { return { - 'Authorization': `Bearer ${this.apiKey}`, - 'api-key': `${this.apiKey}`, + ...(this.apiKey && { + 'Authorization': `Bearer ${this.apiKey}`, + 'api-key': `${this.apiKey}`, + }), } } } diff --git a/core/src/node/api/common/handler.ts b/core/src/node/api/common/handler.ts index fb958dbd1b..5cf232d8a6 100644 --- a/core/src/node/api/common/handler.ts +++ b/core/src/node/api/common/handler.ts @@ -5,19 +5,16 @@ export type Handler = (route: string, args: any) => any export class RequestHandler { handler: Handler - adataper: RequestAdapter + adapter: RequestAdapter constructor(handler: Handler, observer?: Function) { this.handler = handler - this.adataper = new RequestAdapter(observer) + this.adapter = new RequestAdapter(observer) } handle() { CoreRoutes.map((route) => { - this.handler(route, async (...args: any[]) => { - const values = await this.adataper.process(route, ...args) - return values - }) + this.handler(route, async (...args: any[]) => this.adapter.process(route, ...args)) }) } } diff --git a/core/src/node/api/restful/helper/builder.ts b/core/src/node/api/restful/helper/builder.ts index e34fb606bc..01ab26394a 100644 --- a/core/src/node/api/restful/helper/builder.ts +++ b/core/src/node/api/restful/helper/builder.ts @@ -316,6 +316,7 @@ export const chatCompletions = async (request: any, reply: any) => { } const requestedModel = matchedModels[0] + const engineConfiguration = await getEngineConfiguration(requestedModel.engine) let apiKey: string | undefined = undefined @@ -323,7 +324,7 @@ export const chatCompletions = async (request: any, reply: any) => { if (engineConfiguration) { apiKey = engineConfiguration.api_key - apiUrl = engineConfiguration.full_url + apiUrl = engineConfiguration.full_url ?? DEFAULT_CHAT_COMPLETION_URL } const headers: Record = { diff --git a/core/src/node/helper/config.ts b/core/src/node/helper/config.ts index b5ec2e029a..2b828b5769 100644 --- a/core/src/node/helper/config.ts +++ b/core/src/node/helper/config.ts @@ -1,4 +1,4 @@ -import { AppConfiguration } from '../../types' +import { AppConfiguration, SettingComponentProps } from '../../types' import { join } from 'path' import fs from 'fs' import os from 'os' @@ -125,14 +125,30 @@ const exec = async (command: string): Promise => { }) } +// a hacky way to get the api key. we should comes up with a better +// way to handle this export const getEngineConfiguration = async (engineId: string) => { - if (engineId !== 'openai' && engineId !== 'groq') { - return undefined + if (engineId !== 'openai' && engineId !== 'groq') return undefined + + const settingDirectoryPath = join( + getJanDataFolderPath(), + 'settings', + engineId === 'openai' ? 'inference-openai-extension' : 'inference-groq-extension', + 'settings.json' + ) + + const content = fs.readFileSync(settingDirectoryPath, 'utf-8') + const settings: SettingComponentProps[] = JSON.parse(content) + const apiKeyId = engineId === 'openai' ? 'openai-api-key' : 'groq-api-key' + const keySetting = settings.find((setting) => setting.key === apiKeyId) + + let apiKey = keySetting?.controllerProps.value + if (typeof apiKey !== 'string') apiKey = '' + + return { + api_key: apiKey, + full_url: undefined, } - const directoryPath = join(getJanDataFolderPath(), 'engines') - const filePath = join(directoryPath, `${engineId}.json`) - const data = fs.readFileSync(filePath, 'utf-8') - return JSON.parse(data) } /** diff --git a/core/src/types/index.ts b/core/src/types/index.ts index 291c735246..6627ebff9b 100644 --- a/core/src/types/index.ts +++ b/core/src/types/index.ts @@ -9,3 +9,4 @@ export * from './config' export * from './huggingface' export * from './miscellaneous' export * from './api' +export * from './setting' diff --git a/core/src/types/setting/index.ts b/core/src/types/setting/index.ts new file mode 100644 index 0000000000..b3407460ce --- /dev/null +++ b/core/src/types/setting/index.ts @@ -0,0 +1 @@ +export * from './settingComponent' diff --git a/core/src/types/setting/settingComponent.ts b/core/src/types/setting/settingComponent.ts new file mode 100644 index 0000000000..4d9526505f --- /dev/null +++ b/core/src/types/setting/settingComponent.ts @@ -0,0 +1,34 @@ +export type SettingComponentProps = { + key: string + title: string + description: string + controllerType: ControllerType + controllerProps: SliderComponentProps | CheckboxComponentProps | InputComponentProps + + extensionName?: string + requireModelReload?: boolean + configType?: ConfigType +} + +export type ConfigType = 'runtime' | 'setting' + +export type ControllerType = 'slider' | 'checkbox' | 'input' + +export type InputType = 'password' | 'text' | 'email' | 'number' | 'tel' | 'url' + +export type InputComponentProps = { + placeholder: string + value: string + type?: InputType +} + +export type SliderComponentProps = { + min: number + max: number + step: number + value: number +} + +export type CheckboxComponentProps = { + value: boolean +} diff --git a/extensions/assistant-extension/rollup.config.ts b/extensions/assistant-extension/rollup.config.ts index 0d1e4832c1..744ef2b972 100644 --- a/extensions/assistant-extension/rollup.config.ts +++ b/extensions/assistant-extension/rollup.config.ts @@ -7,12 +7,10 @@ import replace from '@rollup/plugin-replace' const packageJson = require('./package.json') -const pkg = require('./package.json') - export default [ { input: `src/index.ts`, - output: [{ file: pkg.main, format: 'es', sourcemap: true }], + output: [{ file: packageJson.main, format: 'es', sourcemap: true }], // Indicate here external modules you don't wanna include in your bundle (i.e.: 'lodash') external: [], watch: { @@ -36,7 +34,7 @@ export default [ // https://github.com/rollup/rollup-plugin-node-resolve#usage resolve({ extensions: ['.js', '.ts', '.svelte'], - browser: true + browser: true, }), // Resolve source maps to the original source diff --git a/extensions/assistant-extension/src/node/engine.ts b/extensions/assistant-extension/src/node/engine.ts index 70d02af1f5..17094ffbc9 100644 --- a/extensions/assistant-extension/src/node/engine.ts +++ b/extensions/assistant-extension/src/node/engine.ts @@ -1,13 +1,36 @@ import fs from 'fs' import path from 'path' -import { getJanDataFolderPath } from '@janhq/core/node' +import { SettingComponentProps, getJanDataFolderPath } from '@janhq/core/node' // Sec: Do not send engine settings over requests // Read it manually instead export const readEmbeddingEngine = (engineName: string) => { - const engineSettings = fs.readFileSync( - path.join(getJanDataFolderPath(), 'engines', `${engineName}.json`), - 'utf-8' - ) - return JSON.parse(engineSettings) + if (engineName !== 'openai' && engineName !== 'groq') { + const engineSettings = fs.readFileSync( + path.join(getJanDataFolderPath(), 'engines', `${engineName}.json`), + 'utf-8' + ) + return JSON.parse(engineSettings) + } else { + const settingDirectoryPath = path.join( + getJanDataFolderPath(), + 'settings', + engineName === 'openai' + ? 'inference-openai-extension' + : 'inference-groq-extension', + 'settings.json' + ) + + const content = fs.readFileSync(settingDirectoryPath, 'utf-8') + const settings: SettingComponentProps[] = JSON.parse(content) + const apiKeyId = engineName === 'openai' ? 'openai-api-key' : 'groq-api-key' + const keySetting = settings.find((setting) => setting.key === apiKeyId) + + let apiKey = keySetting?.controllerProps.value + if (typeof apiKey !== 'string') apiKey = '' + + return { + api_key: apiKey, + } + } } diff --git a/extensions/inference-groq-extension/resources/settings.json b/extensions/inference-groq-extension/resources/settings.json new file mode 100644 index 0000000000..2e6ca413b6 --- /dev/null +++ b/extensions/inference-groq-extension/resources/settings.json @@ -0,0 +1,23 @@ +[ + { + "key": "chat-completions-endpoint", + "title": "Chat Completions Endpoint", + "description": "The endpoint to use for chat completions. See the [Groq Documentation](https://console.groq.com/docs/openai) for more information.", + "controllerType": "input", + "controllerProps": { + "placeholder": "Chat Completions Endpoint", + "value": "https://api.groq.com/openai/v1/chat/completions" + } + }, + { + "key": "groq-api-key", + "title": "API Key", + "description": "The Groq API uses API keys for authentication. Visit your [API Keys](https://console.groq.com/keys) page to retrieve the API key you'll use in your requests.", + "controllerType": "input", + "controllerProps": { + "placeholder": "API Key", + "value": "", + "type": "password" + } + } +] diff --git a/extensions/inference-groq-extension/src/index.ts b/extensions/inference-groq-extension/src/index.ts index cd22c62f97..ea62aa14ae 100644 --- a/extensions/inference-groq-extension/src/index.ts +++ b/extensions/inference-groq-extension/src/index.ts @@ -6,78 +6,41 @@ * @module inference-groq-extension/src/index */ -import { - events, - fs, - AppConfigurationEventName, - joinPath, - RemoteOAIEngine, -} from '@janhq/core' -import { join } from 'path' +import { RemoteOAIEngine } from '@janhq/core' -declare const COMPLETION_URL: string +declare const SETTINGS: Array +enum Settings { + apiKey = 'groq-api-key', + chatCompletionsEndPoint = 'chat-completions-endpoint', +} /** * A class that implements the InferenceExtension interface from the @janhq/core package. * The class provides methods for initializing and stopping a model, and for making inference requests. * It also subscribes to events emitted by the @janhq/core package and handles new message requests. */ export default class JanInferenceGroqExtension extends RemoteOAIEngine { - private readonly _engineDir = 'file://engines' - private readonly _engineMetadataFileName = 'groq.json' - - inferenceUrl: string = COMPLETION_URL + inferenceUrl: string = '' provider = 'groq' - apiKey = '' - - private _engineSettings = { - full_url: COMPLETION_URL, - api_key: 'gsk-', - } - /** - * Subscribes to events emitted by the @janhq/core package. - */ - async onLoad() { + override async onLoad(): Promise { super.onLoad() - if (!(await fs.existsSync(this._engineDir))) { - await fs.mkdir(this._engineDir) - } - - this.writeDefaultEngineSettings() - - const settingsFilePath = await joinPath([ - this._engineDir, - this._engineMetadataFileName, - ]) + // Register Settings + this.registerSettings(SETTINGS) - // Events subscription - events.on( - AppConfigurationEventName.OnConfigurationUpdate, - (settingsKey: string) => { - // Update settings on changes - if (settingsKey === settingsFilePath) this.writeDefaultEngineSettings() - } + // Retrieve API Key Setting + this.apiKey = await this.getSetting(Settings.apiKey, '') + this.inferenceUrl = await this.getSetting( + Settings.chatCompletionsEndPoint, + '' ) } - async writeDefaultEngineSettings() { - try { - const engineFile = join(this._engineDir, this._engineMetadataFileName) - if (await fs.existsSync(engineFile)) { - const engine = await fs.readFileSync(engineFile, 'utf-8') - this._engineSettings = - typeof engine === 'object' ? engine : JSON.parse(engine) - this.inferenceUrl = this._engineSettings.full_url - this.apiKey = this._engineSettings.api_key - } else { - await fs.writeFileSync( - engineFile, - JSON.stringify(this._engineSettings, null, 2) - ) - } - } catch (err) { - console.error(err) + onSettingUpdate(key: string, value: T): void { + if (key === Settings.apiKey) { + this.apiKey = value as string + } else if (key === Settings.chatCompletionsEndPoint) { + this.inferenceUrl = value as string } } } diff --git a/extensions/inference-groq-extension/webpack.config.js b/extensions/inference-groq-extension/webpack.config.js index 5352b56b72..13d32c52d3 100644 --- a/extensions/inference-groq-extension/webpack.config.js +++ b/extensions/inference-groq-extension/webpack.config.js @@ -1,6 +1,7 @@ const path = require('path') const webpack = require('webpack') const packageJson = require('./package.json') +const settingJson = require('./resources/settings.json') module.exports = { experiments: { outputModule: true }, @@ -17,8 +18,8 @@ module.exports = { }, plugins: [ new webpack.DefinePlugin({ + SETTINGS: JSON.stringify(settingJson), MODULE: JSON.stringify(`${packageJson.name}/${packageJson.module}`), - COMPLETION_URL: JSON.stringify('https://api.groq.com/openai/v1/chat/completions'), }), ], output: { diff --git a/extensions/inference-nitro-extension/resources/default_settings.json b/extensions/inference-nitro-extension/resources/default_settings.json new file mode 100644 index 0000000000..39f0880b0d --- /dev/null +++ b/extensions/inference-nitro-extension/resources/default_settings.json @@ -0,0 +1,33 @@ +[ + { + "key": "test", + "title": "Test", + "description": "Test", + "controllerType": "input", + "controllerProps": { + "placeholder": "Test", + "value": "" + } + }, + { + "key": "embedding", + "title": "Embedding", + "description": "Whether to enable embedding.", + "controllerType": "checkbox", + "controllerProps": { + "value": true + } + }, + { + "key": "ctx_len", + "title": "Context Length", + "description": "The context length for model operations varies; the maximum depends on the specific model used.", + "controllerType": "slider", + "controllerProps": { + "min": 0, + "max": 4096, + "step": 128, + "value": 4096 + } + } +] diff --git a/extensions/inference-nitro-extension/rollup.config.ts b/extensions/inference-nitro-extension/rollup.config.ts index 396c40d081..ea7f1b14d9 100644 --- a/extensions/inference-nitro-extension/rollup.config.ts +++ b/extensions/inference-nitro-extension/rollup.config.ts @@ -5,13 +5,12 @@ import typescript from 'rollup-plugin-typescript2' import json from '@rollup/plugin-json' import replace from '@rollup/plugin-replace' const packageJson = require('./package.json') - -const pkg = require('./package.json') +const defaultSettingJson = require('./resources/default_settings.json') export default [ { input: `src/index.ts`, - output: [{ file: pkg.main, format: 'es', sourcemap: true }], + output: [{ file: packageJson.main, format: 'es', sourcemap: true }], // Indicate here external modules you don't wanna include in your bundle (i.e.: 'lodash') external: [], watch: { @@ -19,7 +18,9 @@ export default [ }, plugins: [ replace({ + EXTENSION_NAME: JSON.stringify(packageJson.name), NODE: JSON.stringify(`${packageJson.name}/${packageJson.node}`), + DEFAULT_SETTINGS: JSON.stringify(defaultSettingJson), INFERENCE_URL: JSON.stringify( process.env.INFERENCE_URL || 'http://127.0.0.1:3928/inferences/llamacpp/chat_completion' diff --git a/extensions/inference-nitro-extension/src/@types/global.d.ts b/extensions/inference-nitro-extension/src/@types/global.d.ts index 3a3d2aa325..e16f77b310 100644 --- a/extensions/inference-nitro-extension/src/@types/global.d.ts +++ b/extensions/inference-nitro-extension/src/@types/global.d.ts @@ -2,6 +2,8 @@ declare const NODE: string declare const INFERENCE_URL: string declare const TROUBLESHOOTING_URL: string declare const JAN_SERVER_INFERENCE_URL: string +declare const EXTENSION_NAME: string +declare const DEFAULT_SETTINGS: Array /** * The response from the initModel function. diff --git a/extensions/inference-nitro-extension/src/index.ts b/extensions/inference-nitro-extension/src/index.ts index 313b67365b..3d08efdfa3 100644 --- a/extensions/inference-nitro-extension/src/index.ts +++ b/extensions/inference-nitro-extension/src/index.ts @@ -58,8 +58,6 @@ export default class JanInferenceNitroExtension extends LocalOAIEngine { this.inferenceUrl = `${window.core?.api?.baseApiUrl}/v1/chat/completions` } - console.debug('Inference url: ', this.inferenceUrl) - this.getNitroProcesHealthIntervalId = setInterval( () => this.periodicallyGetNitroHealth(), JanInferenceNitroExtension._intervalHealthCheck diff --git a/extensions/inference-openai-extension/resources/settings.json b/extensions/inference-openai-extension/resources/settings.json new file mode 100644 index 0000000000..1a19e30a75 --- /dev/null +++ b/extensions/inference-openai-extension/resources/settings.json @@ -0,0 +1,23 @@ +[ + { + "key": "chat-completions-endpoint", + "title": "Chat Completions Endpoint", + "description": "The endpoint to use for chat completions. See the [OpenAI API documentation](https://platform.openai.com/docs/api-reference/chat/create) for more information.", + "controllerType": "input", + "controllerProps": { + "placeholder": "Chat Completions Endpoint", + "value": "https://api.openai.com/v1/chat/completions" + } + }, + { + "key": "openai-api-key", + "title": "API Key", + "description": "The OpenAI API uses API keys for authentication. Visit your [API Keys](https://platform.openai.com/account/api-keys) page to retrieve the API key you'll use in your requests.", + "controllerType": "input", + "controllerProps": { + "placeholder": "API Key", + "value": "", + "type": "password" + } + } +] diff --git a/extensions/inference-openai-extension/src/index.ts b/extensions/inference-openai-extension/src/index.ts index eb60540fa8..0853fe0f69 100644 --- a/extensions/inference-openai-extension/src/index.ts +++ b/extensions/inference-openai-extension/src/index.ts @@ -5,85 +5,41 @@ * @version 1.0.0 * @module inference-openai-extension/src/index */ -declare const ENGINE: string -import { - events, - fs, - AppConfigurationEventName, - joinPath, - RemoteOAIEngine, -} from '@janhq/core' -import { join } from 'path' - -declare const COMPLETION_URL: string +import { RemoteOAIEngine } from '@janhq/core' +declare const SETTINGS: Array +enum Settings { + apiKey = 'openai-api-key', + chatCompletionsEndPoint = 'chat-completions-endpoint', +} /** * A class that implements the InferenceExtension interface from the @janhq/core package. * The class provides methods for initializing and stopping a model, and for making inference requests. * It also subscribes to events emitted by the @janhq/core package and handles new message requests. */ export default class JanInferenceOpenAIExtension extends RemoteOAIEngine { - private static readonly _engineDir = 'file://engines' - private static readonly _engineMetadataFileName = `${ENGINE}.json` - - private _engineSettings = { - full_url: COMPLETION_URL, - api_key: 'sk-', - } - - inferenceUrl: string = COMPLETION_URL + inferenceUrl: string = '' provider: string = 'openai' - apiKey: string = '' - // TODO: Just use registerSettings from BaseExtension - // Remove these methods - /** - * Subscribes to events emitted by the @janhq/core package. - */ - async onLoad() { + override async onLoad(): Promise { super.onLoad() - if (!(await fs.existsSync(JanInferenceOpenAIExtension._engineDir))) { - await fs.mkdir(JanInferenceOpenAIExtension._engineDir) - } - - this.writeDefaultEngineSettings() - - const settingsFilePath = await joinPath([ - JanInferenceOpenAIExtension._engineDir, - JanInferenceOpenAIExtension._engineMetadataFileName, - ]) + // Register Settings + this.registerSettings(SETTINGS) - events.on( - AppConfigurationEventName.OnConfigurationUpdate, - (settingsKey: string) => { - // Update settings on changes - if (settingsKey === settingsFilePath) this.writeDefaultEngineSettings() - } + this.apiKey = await this.getSetting(Settings.apiKey, '') + this.inferenceUrl = await this.getSetting( + Settings.chatCompletionsEndPoint, + '' ) } - async writeDefaultEngineSettings() { - try { - const engineFile = join( - JanInferenceOpenAIExtension._engineDir, - JanInferenceOpenAIExtension._engineMetadataFileName - ) - if (await fs.existsSync(engineFile)) { - const engine = await fs.readFileSync(engineFile, 'utf-8') - this._engineSettings = - typeof engine === 'object' ? engine : JSON.parse(engine) - this.inferenceUrl = this._engineSettings.full_url - this.apiKey = this._engineSettings.api_key - } else { - await fs.writeFileSync( - engineFile, - JSON.stringify(this._engineSettings, null, 2) - ) - } - } catch (err) { - console.error(err) + onSettingUpdate(key: string, value: T): void { + if (key === Settings.apiKey) { + this.apiKey = value as string + } else if (key === Settings.chatCompletionsEndPoint) { + this.inferenceUrl = value as string } } } diff --git a/extensions/inference-openai-extension/webpack.config.js b/extensions/inference-openai-extension/webpack.config.js index ee18035f29..3954d031d3 100644 --- a/extensions/inference-openai-extension/webpack.config.js +++ b/extensions/inference-openai-extension/webpack.config.js @@ -1,6 +1,7 @@ const path = require('path') const webpack = require('webpack') const packageJson = require('./package.json') +const settingJson = require('./resources/settings.json') module.exports = { experiments: { outputModule: true }, @@ -17,8 +18,8 @@ module.exports = { }, plugins: [ new webpack.DefinePlugin({ + SETTINGS: JSON.stringify(settingJson), ENGINE: JSON.stringify(packageJson.engine), - COMPLETION_URL: JSON.stringify('https://api.openai.com/v1/chat/completions'), }), ], output: { diff --git a/extensions/inference-triton-trtllm-extension/resources/settings.json b/extensions/inference-triton-trtllm-extension/resources/settings.json new file mode 100644 index 0000000000..bba69805e0 --- /dev/null +++ b/extensions/inference-triton-trtllm-extension/resources/settings.json @@ -0,0 +1,23 @@ +[ + { + "key": "chat-completions-endpoint", + "title": "Chat Completions Endpoint", + "description": "The endpoint to use for chat completions.", + "controllerType": "input", + "controllerProps": { + "placeholder": "Chat Completions Endpoint", + "value": "http://localhost:8000/v2/models/tensorrt_llm_bls/generate" + } + }, + { + "key": "tritonllm-api-key", + "title": "Triton LLM API Key", + "description": "The Triton LLM API uses API keys for authentication.", + "controllerType": "input", + "controllerProps": { + "placeholder": "API Key", + "value": "", + "type": "password" + } + } +] diff --git a/extensions/inference-triton-trtllm-extension/src/index.ts b/extensions/inference-triton-trtllm-extension/src/index.ts index 6f9a01d9b7..a3032f01d4 100644 --- a/extensions/inference-triton-trtllm-extension/src/index.ts +++ b/extensions/inference-triton-trtllm-extension/src/index.ts @@ -6,77 +6,44 @@ * @module inference-nvidia-triton-trt-llm-extension/src/index */ -import { - AppConfigurationEventName, - events, - fs, - joinPath, - Model, - RemoteOAIEngine, -} from '@janhq/core' -import { join } from 'path' +import { RemoteOAIEngine } from '@janhq/core' +declare const SETTINGS: Array +enum Settings { + apiKey = 'tritonllm-api-key', + chatCompletionsEndPoint = 'chat-completions-endpoint', +} /** * A class that implements the InferenceExtension interface from the @janhq/core package. * The class provides methods for initializing and stopping a model, and for making inference requests. * It also subscribes to events emitted by the @janhq/core package and handles new message requests. */ export default class JanInferenceTritonTrtLLMExtension extends RemoteOAIEngine { - private readonly _engineDir = 'file://engines' - private readonly _engineMetadataFileName = 'triton_trtllm.json' - inferenceUrl: string = '' provider: string = 'triton_trtllm' - apiKey: string = '' - - _engineSettings: { - base_url: '' - api_key: '' - } /** * Subscribes to events emitted by the @janhq/core package. */ async onLoad() { super.onLoad() - if (!(await fs.existsSync(this._engineDir))) { - await fs.mkdir(this._engineDir) - } - - this.writeDefaultEngineSettings() - const settingsFilePath = await joinPath([ - this._engineDir, - this._engineMetadataFileName, - ]) + // Register Settings + this.registerSettings(SETTINGS) - // Events subscription - events.on( - AppConfigurationEventName.OnConfigurationUpdate, - (settingsKey: string) => { - // Update settings on changes - if (settingsKey === settingsFilePath) this.writeDefaultEngineSettings() - } + // Retrieve API Key Setting + this.apiKey = await this.getSetting(Settings.apiKey, '') + this.inferenceUrl = await this.getSetting( + Settings.chatCompletionsEndPoint, + '' ) } - async writeDefaultEngineSettings() { - try { - const engine_json = join(this._engineDir, this._engineMetadataFileName) - if (await fs.existsSync(engine_json)) { - const engine = await fs.readFileSync(engine_json, 'utf-8') - this._engineSettings = - typeof engine === 'object' ? engine : JSON.parse(engine) - this.inferenceUrl = this._engineSettings.base_url - this.apiKey = this._engineSettings.api_key - } else { - await fs.writeFileSync( - engine_json, - JSON.stringify(this._engineSettings, null, 2) - ) - } - } catch (err) { - console.error(err) + onSettingUpdate(key: string, value: T): void { + if (key === Settings.apiKey) { + this.apiKey = value as string + } else if (key === Settings.chatCompletionsEndPoint) { + this.inferenceUrl = value as string } } } diff --git a/extensions/inference-triton-trtllm-extension/webpack.config.js b/extensions/inference-triton-trtllm-extension/webpack.config.js index e83370a1ac..13d32c52d3 100644 --- a/extensions/inference-triton-trtllm-extension/webpack.config.js +++ b/extensions/inference-triton-trtllm-extension/webpack.config.js @@ -1,6 +1,7 @@ const path = require('path') const webpack = require('webpack') const packageJson = require('./package.json') +const settingJson = require('./resources/settings.json') module.exports = { experiments: { outputModule: true }, @@ -17,6 +18,7 @@ module.exports = { }, plugins: [ new webpack.DefinePlugin({ + SETTINGS: JSON.stringify(settingJson), MODULE: JSON.stringify(`${packageJson.name}/${packageJson.module}`), }), ], diff --git a/extensions/model-extension/rollup.config.ts b/extensions/model-extension/rollup.config.ts index 722785aa38..0f5ad04dfe 100644 --- a/extensions/model-extension/rollup.config.ts +++ b/extensions/model-extension/rollup.config.ts @@ -1,5 +1,4 @@ import resolve from '@rollup/plugin-node-resolve' -import commonjs from '@rollup/plugin-commonjs' import sourceMaps from 'rollup-plugin-sourcemaps' import typescript from 'rollup-plugin-typescript2' import json from '@rollup/plugin-json' @@ -7,12 +6,10 @@ import replace from '@rollup/plugin-replace' const packageJson = require('./package.json') -const pkg = require('./package.json') - export default [ { input: `src/index.ts`, - output: [{ file: pkg.main, format: 'es', sourcemap: true }], + output: [{ file: packageJson.main, format: 'es', sourcemap: true }], // Indicate here external modules you don't wanna include in your bundle (i.e.: 'lodash') external: [], watch: { diff --git a/extensions/tensorrt-llm-extension/src/index.ts b/extensions/tensorrt-llm-extension/src/index.ts index ce89beca26..d099edfe99 100644 --- a/extensions/tensorrt-llm-extension/src/index.ts +++ b/extensions/tensorrt-llm-extension/src/index.ts @@ -251,7 +251,7 @@ export default class TensorRTLLMExtension extends LocalOAIEngine { return Promise.resolve() } - override inference(data: MessageRequest): void { + override async inference(data: MessageRequest) { if (!this.loadedModel) return // TensorRT LLM Extension supports streaming only if (data.model) data.model.parameters.stream = true diff --git a/web/containers/DropdownListSidebar/index.tsx b/web/containers/DropdownListSidebar/index.tsx index b0953cdea1..76cbd05dda 100644 --- a/web/containers/DropdownListSidebar/index.tsx +++ b/web/containers/DropdownListSidebar/index.tsx @@ -38,7 +38,6 @@ import useUpdateModelParameters from '@/hooks/useUpdateModelParameters' import { toGibibytes } from '@/utils/converter' import ModelLabel from '../ModelLabel' -import OpenAiKeyInput from '../OpenAiKeyInput' import { mainViewStateAtom } from '@/helpers/atoms/App.atom' import { serverEnabledAtom } from '@/helpers/atoms/LocalServer.atom' @@ -144,7 +143,7 @@ const DropdownListSidebar = ({ // Update model parameter to the thread file if (model) - updateModelParameter(activeThread.id, { + updateModelParameter(activeThread, { params: modelParams, modelId: model.id, engine: model.engine, @@ -170,172 +169,164 @@ const DropdownListSidebar = ({ stateModel.model === selectedModel?.id && stateModel.loading return ( - <> -
+ - - - {selectedModelLoading && ( -
- )} - - {selectedModel?.name} - - - - - + + {selectedModelLoading && ( +
+ )} + -
-
    - {engineOptions.map((name, i) => { - return ( -
  • + + + + +
    +
      + {engineOptions.map((name, i) => { + return ( +
    • setIsTabActive(i)} + > + {i === 0 ? ( + + ) : ( + + )} + setIsTabActive(i)} > - {i === 0 ? ( - - ) : ( - - )} - - {name} - -
      • - ) - })} -
    + {name} + +
    • + ) + })} +
+
+ +
+ {downloadedModels.length === 0 ? ( +
+

{`Oops, you don't have a model yet.`}

- -
- {downloadedModels.length === 0 ? ( -
-

{`Oops, you don't have a model yet.`}

-
- ) : ( - - <> - {modelOptions.map((x, i) => ( -
+ <> + {modelOptions.map((x, i) => ( +
+ - -
-
- - {x.name} - -
- - {toGibibytes(x.metadata.size)} - - {x.metadata.size && ( - - )} -
-
+
+ {x.name} +
+ + {toGibibytes(x.metadata.size)} + + {x.metadata.size && ( + + )}
- -
- {x.id} - {clipboard.copied && copyId === x.id ? ( - - ) : ( - { - clipboard.copy(x.id) - setCopyId(x.id) - }} - /> - )}
+ +
+ {x.id} + {clipboard.copied && copyId === x.id ? ( + + ) : ( + { + clipboard.copy(x.id) + setCopyId(x.id) + }} + /> + )}
- ))} - - - )} -
-
- - -
- - - -
- - - +
+ ))} + + + )} +
+
+ + +
+ + + +
) } diff --git a/web/containers/ModelConfigInput/index.tsx b/web/containers/ModelConfigInput/index.tsx index d573a0bf9f..0c16c916cf 100644 --- a/web/containers/ModelConfigInput/index.tsx +++ b/web/containers/ModelConfigInput/index.tsx @@ -26,33 +26,31 @@ const ModelConfigInput: React.FC = ({ description, placeholder, onValueChanged, -}) => { - return ( -
-
-

- {title} -

- - - - - - - {description} - - - - -
-