Sync release 0.4.9 to dev (#2407)

* fix: move tensorrt executable to engine (#2400) * fix: move tensorrt executable to engine Signed-off-by: James <[email protected]> * some update Signed-off-by: hiro <[email protected]> * chore: bump tensorrt version * fix: wrong destroy path * fix: install extensions in parallel * chore: update path for tensorrt engine (#2404) Signed-off-by: James <[email protected]> Co-authored-by: James <[email protected]> --------- Signed-off-by: James <[email protected]> Signed-off-by: hiro <[email protected]> Co-authored-by: James <[email protected]> Co-authored-by: hiro <[email protected]> Co-authored-by: Louis <[email protected]> * Release/v0.4.9 (#2421) * fix: turn off experimental settings should also turn off quick ask (#2411) * fix: app glitches 1s generating response before starting model (#2412) * fix: disable experimental feature should also disable vulkan (#2414) * fix: model load stuck on windows when can't get CPU core count (#2413) Signed-off-by: James <[email protected]> Co-authored-by: James <[email protected]> * feat: TensorRT-LLM engine update support (#2415) * fix: engine update * chore: add remove prepopulated models Signed-off-by: James <[email protected]> * update tinyjensen url Signed-off-by: James <[email protected]> * update llamacorn Signed-off-by: James <[email protected]> * update Mistral 7B Instruct v0.1 int4 Signed-off-by: James <[email protected]> * update tensorrt Signed-off-by: James <[email protected]> * update Signed-off-by: hiro <[email protected]> * update Signed-off-by: James <[email protected]> * prettier Signed-off-by: James <[email protected]> * update mistral config Signed-off-by: James <[email protected]> * fix some lint Signed-off-by: James <[email protected]> --------- Signed-off-by: James <[email protected]> Signed-off-by: hiro <[email protected]> Co-authored-by: James <[email protected]> Co-authored-by: hiro <[email protected]> * Tensorrt LLM disable turing support (#2418) Co-authored-by: Hien To <[email protected]> * chore: add prompt template tensorrtllm (#2375) * chore: add prompt template tensorrtllm * Add Prompt template for mistral and correct model metadata --------- Co-authored-by: Hien To <[email protected]> * fix: correct tensorrt mistral model.json (#2419) --------- Signed-off-by: James <[email protected]> Signed-off-by: hiro <[email protected]> Co-authored-by: Louis <[email protected]> Co-authored-by: James <[email protected]> Co-authored-by: hiro <[email protected]> Co-authored-by: hiento09 <[email protected]> Co-authored-by: Hien To <[email protected]> --------- Signed-off-by: James <[email protected]> Signed-off-by: hiro <[email protected]> Co-authored-by: NamH <[email protected]> Co-authored-by: James <[email protected]> Co-authored-by: hiro <[email protected]> Co-authored-by: hiento09 <[email protected]> Co-authored-by: Hien To <[email protected]>
janhq · Mar 19, 2024 · 489e8aa · 489e8aa
1 parent dd11ba2
commit 489e8aa
Show file tree

Hide file tree

Showing 29 changed files with 606 additions and 150 deletions.
diff --git a/.github/workflows/jan-electron-linter-and-test.yml b/.github/workflows/jan-electron-linter-and-test.yml
@@ -22,6 +22,7 @@ on:
  branches:
  - main
  - dev
+ - release/**
  paths:
  - "electron/**"
  - .github/workflows/jan-electron-linter-and-test.yml

diff --git a/core/package.json b/core/package.json
@@ -46,7 +46,7 @@
  },
  "devDependencies": {
  "@types/jest": "^29.5.12",
- "@types/node": "^12.0.2",
+ "@types/node": "^20.11.4",
  "eslint": "8.57.0",
  "eslint-plugin-jest": "^27.9.0",
  "jest": "^29.7.0",

diff --git a/core/src/api/index.ts b/core/src/api/index.ts
@@ -33,7 +33,7 @@ export enum AppRoute {
  stopServer = 'stopServer',
  log = 'log',
  logServer = 'logServer',
- systemInformations = 'systemInformations',
+ systemInformation = 'systemInformation',
  showToast = 'showToast',
 }
 
@@ -95,6 +95,8 @@ export enum FileManagerRoute {
  getUserHomePath = 'getUserHomePath',
  fileStat = 'fileStat',
  writeBlob = 'writeBlob',
+ mkdir = 'mkdir',
+ rm = 'rm',
 }
 
 export type ApiFunction = (...args: any[]) => any

diff --git a/core/src/core.ts b/core/src/core.ts
@@ -1,4 +1,4 @@
-import { DownloadRequest, FileStat, NetworkConfig } from './types'
+import { DownloadRequest, FileStat, NetworkConfig, SystemInformation } from './types'
 
 /**
  * Execute a extension module function in main process
@@ -110,7 +110,8 @@ const isSubdirectory: (from: string, to: string) => Promise<boolean> = (from: st
  * Get system information
  * @returns {Promise<any>} - A promise that resolves with the system information.
  */
-const systemInformations: () => Promise<any> = () => global.core.api?.systemInformations()
+const systemInformation: () => Promise<SystemInformation> = () =>
+ global.core.api?.systemInformation()
 
 /**
  * Show toast message from browser processes.
@@ -146,7 +147,7 @@ export {
  log,
  isSubdirectory,
  getUserHomePath,
- systemInformations,
+ systemInformation,
  showToast,
  FileStat,
 }
diff --git a/core/src/extension.ts b/core/src/extension.ts
@@ -19,6 +19,7 @@ export interface Compatibility {
 const ALL_INSTALLATION_STATE = [
  'NotRequired', // not required.
  'Installed', // require and installed. Good to go.
+ 'Updatable', // require and installed but need to be updated.
  'NotInstalled', // require to be installed.
  'Corrupted', // require but corrupted. Need to redownload.
 ] as const
@@ -59,6 +60,13 @@ export abstract class BaseExtension implements ExtensionType {
  return undefined
  }
 
+ /**
+ * Determine if the extension is updatable.
+ */
+ updatable(): boolean {
+ return false
+ }
+
  /**
  * Determine if the prerequisites for the extension are installed.
  *

diff --git a/core/src/extensions/ai-engines/LocalOAIEngine.ts b/core/src/extensions/ai-engines/LocalOAIEngine.ts
@@ -1,4 +1,4 @@
-import { executeOnMain, getJanDataFolderPath, joinPath } from '../../core'
+import { executeOnMain, getJanDataFolderPath, joinPath, systemInformation } from '../../core'
 import { events } from '../../events'
 import { Model, ModelEvent } from '../../types'
 import { OAIEngine } from './OAIEngine'
@@ -30,11 +30,11 @@ export abstract class LocalOAIEngine extends OAIEngine {
  if (model.engine.toString() !== this.provider) return
 
  const modelFolder = await joinPath([await getJanDataFolderPath(), this.modelFolder, model.id])
-
+ const systemInfo = await systemInformation()
  const res = await executeOnMain(this.nodeModule, this.loadModelFunctionName, {
  modelFolder,
  model,
- })
+ }, systemInfo)
 
  if (res?.error) {
  events.emit(ModelEvent.OnModelFail, {

diff --git a/core/src/extensions/monitoring.ts b/core/src/extensions/monitoring.ts
@@ -1,5 +1,5 @@
 import { BaseExtension, ExtensionTypeEnum } from '../extension'
-import { GpuSetting, MonitoringInterface } from '../index'
+import { GpuSetting, MonitoringInterface, OperatingSystemInfo } from '../index'
 
 /**
  * Monitoring extension for system monitoring.
@@ -16,4 +16,5 @@ export abstract class MonitoringExtension extends BaseExtension implements Monit
  abstract getGpuSetting(): Promise<GpuSetting>
  abstract getResourcesInfo(): Promise<any>
  abstract getCurrentLoad(): Promise<any>
+ abstract getOsInfo(): Promise<OperatingSystemInfo>
 }
diff --git a/core/src/fs.ts b/core/src/fs.ts
@@ -37,12 +37,17 @@ const readdirSync = (...args: any[]) => global.core.api?.readdirSync(...args)
  */
 const mkdirSync = (...args: any[]) => global.core.api?.mkdirSync(...args)
 
+const mkdir = (...args: any[]) => global.core.api?.mkdir(...args)
+
 /**
  * Removes a directory at the specified path.
  * @returns {Promise<any>} A Promise that resolves when the directory is removed successfully.
  */
 const rmdirSync = (...args: any[]) =>
  global.core.api?.rmdirSync(...args, { recursive: true, force: true })
+
+const rm = (path: string) => global.core.api?.rm(path)
+
 /**
  * Deletes a file from the local file system.
  * @param {string} path - The path of the file to delete.
@@ -92,7 +97,9 @@ export const fs = {
  existsSync,
  readdirSync,
  mkdirSync,
+ mkdir,
  rmdirSync,
+ rm,
  unlinkSync,
  appendFileSync,
  copyFileSync,

diff --git a/core/src/node/api/processors/fsExt.ts b/core/src/node/api/processors/fsExt.ts
@@ -88,4 +88,28 @@ export class FSExt implements Processor {
  })
  })
  }
+
+ mkdir(path: string): Promise<void> {
+ return new Promise((resolve, reject) => {
+ fs.mkdir(path, { recursive: true }, (err) => {
+ if (err) {
+ reject(err)
+ } else {
+ resolve()
+ }
+ })
+ })
+ }
+
+ rmdir(path: string): Promise<void> {
+ return new Promise((resolve, reject) => {
+ fs.rm(path, { recursive: true }, (err) => {
+ if (err) {
+ reject(err)
+ } else {
+ resolve()
+ }
+ })
+ })
+ }
 }
diff --git a/core/src/node/extension/store.ts b/core/src/node/extension/store.ts
@@ -93,24 +93,25 @@ export function persistExtensions() {
  */
 export async function installExtensions(extensions: any) {
  const installed: Extension[] = []
- for (const ext of extensions) {
- // Set install options and activation based on input type
+ const installations = extensions.map((ext: any): Promise<void> => {
  const isObject = typeof ext === 'object'
  const spec = isObject ? [ext.specifier, ext] : [ext]
  const activate = isObject ? ext.activate !== false : true
 
  // Install and possibly activate extension
  const extension = new Extension(...spec)
  if (!extension.origin) {
- continue
+ return Promise.resolve()
  }
- await extension._install()
- if (activate) extension.setActive(true)
+ return extension._install().then(() => {
+ if (activate) extension.setActive(true)
+ // Add extension to store if needed
+ addExtension(extension)
+ installed.push(extension)
+ })
+ })
 
- // Add extension to store if needed
- addExtension(extension)
- installed.push(extension)
- }
+ await Promise.all(installations)
 
  // Return list of all installed extensions
  return installed

diff --git a/core/src/node/helper/config.ts b/core/src/node/helper/config.ts
@@ -82,26 +82,34 @@ export const getJanExtensionsPath = (): string => {
  */
 export const physicalCpuCount = async (): Promise<number> => {
  const platform = os.platform()
- if (platform === 'linux') {
- const output = await exec('lscpu -p | egrep -v "^#" | sort -u -t, -k 2,4 | wc -l')
- return parseInt(output.trim(), 10)
- } else if (platform === 'darwin') {
- const output = await exec('sysctl -n hw.physicalcpu_max')
- return parseInt(output.trim(), 10)
- } else if (platform === 'win32') {
- const output = await exec('WMIC CPU Get NumberOfCores')
- return output
- .split(os.EOL)
- .map((line: string) => parseInt(line))
- .filter((value: number) => !isNaN(value))
- .reduce((sum: number, number: number) => sum + number, 1)
- } else {
- const cores = os.cpus().filter((cpu: any, index: number) => {
- const hasHyperthreading = cpu.model.includes('Intel')
- const isOdd = index % 2 === 1
- return !hasHyperthreading || isOdd
- })
- return cores.length
+ try {
+ if (platform === 'linux') {
+ const output = await exec('lscpu -p | egrep -v "^#" | sort -u -t, -k 2,4 | wc -l')
+ return parseInt(output.trim(), 10)
+ } else if (platform === 'darwin') {
+ const output = await exec('sysctl -n hw.physicalcpu_max')
+ return parseInt(output.trim(), 10)
+ } else if (platform === 'win32') {
+ const output = await exec('WMIC CPU Get NumberOfCores')
+ return output
+ .split(os.EOL)
+ .map((line: string) => parseInt(line))
+ .filter((value: number) => !isNaN(value))
+ .reduce((sum: number, number: number) => sum + number, 1)
+ } else {
+ const cores = os.cpus().filter((cpu: any, index: number) => {
+ const hasHyperthreading = cpu.model.includes('Intel')
+ const isOdd = index % 2 === 1
+ return !hasHyperthreading || isOdd
+ })
+ return cores.length
+ }
+ } catch (err) {
+ console.warn('Failed to get physical CPU count', err)
+ // Divide by 2 to get rid of hyper threading
+ const coreCount = Math.ceil(os.cpus().length / 2)
+ console.debug('Using node API to get physical CPU count:', coreCount)
+ return coreCount
  }
 }
 

diff --git a/core/src/node/helper/resource.ts b/core/src/node/helper/resource.ts
@@ -1,6 +1,6 @@
 import { SystemResourceInfo } from '../../types'
 import { physicalCpuCount } from './config'
-import { log, logServer } from './log'
+import { log } from './log'
 
 export const getSystemResourceInfo = async (): Promise<SystemResourceInfo> => {
  const cpu = await physicalCpuCount()

diff --git a/core/src/types/miscellaneous/systemResourceInfo.ts b/core/src/types/miscellaneous/systemResourceInfo.ts
@@ -30,3 +30,27 @@ export type GpuSettingInfo = {
  name: string
  arch?: string
 }
+
+export type SystemInformation = {
+ gpuSetting: GpuSetting
+ osInfo?: OperatingSystemInfo
+}
+
+export const SupportedPlatforms = ['win32', 'linux', 'darwin'] as const
+export type SupportedPlatformTuple = typeof SupportedPlatforms
+export type SupportedPlatform = SupportedPlatformTuple[number]
+
+export type OperatingSystemInfo = {
+ platform: SupportedPlatform | 'unknown'
+ arch: string
+ release: string
+ machine: string
+ version: string
+ totalMem: number
+ freeMem: number
+}
+
+export type CpuCoreInfo = {
+ model: string
+ speed: number
+}
diff --git a/extensions/model-extension/src/index.ts b/extensions/model-extension/src/index.ts
@@ -38,7 +38,7 @@ export default class JanModelExtension extends ModelExtension {
  private static readonly _tensorRtEngineFormat = '.engine'
  private static readonly _configDirName = 'config'
  private static readonly _defaultModelFileName = 'default-model.json'
- private static readonly _supportedGpuArch = ['turing', 'ampere', 'ada']
+ private static readonly _supportedGpuArch = ['ampere', 'ada']
 
  /**
  * Called when the extension is loaded.

diff --git a/extensions/monitoring-extension/src/index.ts b/extensions/monitoring-extension/src/index.ts
@@ -1,4 +1,9 @@
-import { GpuSetting, MonitoringExtension, executeOnMain } from '@janhq/core'
+import {
+ GpuSetting,
+ MonitoringExtension,
+ OperatingSystemInfo,
+ executeOnMain,
+} from '@janhq/core'
 
 /**
  * JanMonitoringExtension is a extension that provides system monitoring functionality.
@@ -41,4 +46,8 @@ export default class JanMonitoringExtension extends MonitoringExtension {
  getCurrentLoad(): Promise<any> {
  return executeOnMain(NODE, 'getCurrentLoad')
  }
+
+ getOsInfo(): Promise<OperatingSystemInfo> {
+ return executeOnMain(NODE, 'getOsInfo')
+ }
 }
diff --git a/extensions/monitoring-extension/src/node/index.ts b/extensions/monitoring-extension/src/node/index.ts
@@ -1,9 +1,16 @@
-import { GpuSetting, GpuSettingInfo, ResourceInfo } from '@janhq/core'
+import {
+ GpuSetting,
+ GpuSettingInfo,
+ OperatingSystemInfo,
+ ResourceInfo,
+ SupportedPlatforms,
+} from '@janhq/core'
 import { getJanDataFolderPath, log } from '@janhq/core/node'
 import { mem, cpu } from 'node-os-utils'
 import { exec } from 'child_process'
 import { writeFileSync, existsSync, readFileSync, mkdirSync } from 'fs'
 import path from 'path'
+import os from 'os'
 
 /**
  * Path to the settings directory
@@ -174,8 +181,7 @@ const updateNvidiaDriverInfo = async () =>
 const getGpuArch = (gpuName: string): string => {
  if (!gpuName.toLowerCase().includes('nvidia')) return 'unknown'
 
- if (gpuName.includes('20')) return 'turing'
- else if (gpuName.includes('30')) return 'ampere'
+ if (gpuName.includes('30')) return 'ampere'
  else if (gpuName.includes('40')) return 'ada'
  else return 'unknown'
 }
@@ -320,3 +326,20 @@ const updateCudaExistence = (
  data.is_initial = false
  return data
 }
+
+export const getOsInfo = (): OperatingSystemInfo => {
+ const platform =
+ SupportedPlatforms.find((p) => p === process.platform) || 'unknown'
+
+ const osInfo: OperatingSystemInfo = {
+ platform: platform,
+ arch: process.arch,
+ release: os.release(),
+ machine: os.machine(),
+ version: os.version(),
+ totalMem: os.totalmem(),
+ freeMem: os.freemem(),
+ }
+
+ return osInfo
+}