fix: move tensorrt executable to engine (#2400)

* fix: move tensorrt executable to engine Signed-off-by: James <[email protected]> * some update Signed-off-by: hiro <[email protected]> * chore: bump tensorrt version * fix: wrong destroy path * fix: install extensions in parallel * chore: update path for tensorrt engine (#2404) Signed-off-by: James <[email protected]> Co-authored-by: James <[email protected]> --------- Signed-off-by: James <[email protected]> Signed-off-by: hiro <[email protected]> Co-authored-by: James <[email protected]> Co-authored-by: hiro <[email protected]> Co-authored-by: Louis <[email protected]>
janhq · Mar 18, 2024 · c81a33f · c81a33f
1 parent 60cf8de
commit c81a33f
Show file tree

Hide file tree

Showing 17 changed files with 283 additions and 84 deletions.
diff --git a/core/src/api/index.ts b/core/src/api/index.ts
@@ -33,7 +33,7 @@ export enum AppRoute {
  stopServer = 'stopServer',
  log = 'log',
  logServer = 'logServer',
- systemInformations = 'systemInformations',
+ systemInformation = 'systemInformation',
  showToast = 'showToast',
 }
 
@@ -95,6 +95,7 @@ export enum FileManagerRoute {
  getUserHomePath = 'getUserHomePath',
  fileStat = 'fileStat',
  writeBlob = 'writeBlob',
+ mkdir = 'mkdir',
 }
 
 export type ApiFunction = (...args: any[]) => any

diff --git a/core/src/core.ts b/core/src/core.ts
@@ -1,4 +1,4 @@
-import { DownloadRequest, FileStat, NetworkConfig } from './types'
+import { DownloadRequest, FileStat, NetworkConfig, SystemInformation } from './types'
 
 /**
  * Execute a extension module function in main process
@@ -110,7 +110,8 @@ const isSubdirectory: (from: string, to: string) => Promise<boolean> = (from: st
  * Get system information
  * @returns {Promise<any>} - A promise that resolves with the system information.
  */
-const systemInformations: () => Promise<any> = () => global.core.api?.systemInformations()
+const systemInformation: () => Promise<SystemInformation> = () =>
+ global.core.api?.systemInformation()
 
 /**
  * Show toast message from browser processes.
@@ -146,7 +147,7 @@ export {
  log,
  isSubdirectory,
  getUserHomePath,
- systemInformations,
+ systemInformation,
  showToast,
  FileStat,
 }
diff --git a/core/src/extensions/ai-engines/LocalOAIEngine.ts b/core/src/extensions/ai-engines/LocalOAIEngine.ts
@@ -1,4 +1,4 @@
-import { executeOnMain, getJanDataFolderPath, joinPath } from '../../core'
+import { executeOnMain, getJanDataFolderPath, joinPath, systemInformation } from '../../core'
 import { events } from '../../events'
 import { Model, ModelEvent } from '../../types'
 import { OAIEngine } from './OAIEngine'
@@ -30,11 +30,11 @@ export abstract class LocalOAIEngine extends OAIEngine {
  if (model.engine.toString() !== this.provider) return
 
  const modelFolder = await joinPath([await getJanDataFolderPath(), this.modelFolder, model.id])
-
+ const systemInfo = await systemInformation()
  const res = await executeOnMain(this.nodeModule, this.loadModelFunctionName, {
  modelFolder,
  model,
- })
+ }, systemInfo)
 
  if (res?.error) {
  events.emit(ModelEvent.OnModelFail, {

diff --git a/core/src/extensions/monitoring.ts b/core/src/extensions/monitoring.ts
@@ -1,5 +1,5 @@
 import { BaseExtension, ExtensionTypeEnum } from '../extension'
-import { GpuSetting, MonitoringInterface } from '../index'
+import { GpuSetting, MonitoringInterface, OperatingSystemInfo } from '../index'
 
 /**
  * Monitoring extension for system monitoring.
@@ -16,4 +16,5 @@ export abstract class MonitoringExtension extends BaseExtension implements Monit
  abstract getGpuSetting(): Promise<GpuSetting>
  abstract getResourcesInfo(): Promise<any>
  abstract getCurrentLoad(): Promise<any>
+ abstract getOsInfo(): Promise<OperatingSystemInfo>
 }
diff --git a/core/src/fs.ts b/core/src/fs.ts
@@ -37,6 +37,8 @@ const readdirSync = (...args: any[]) => global.core.api?.readdirSync(...args)
  */
 const mkdirSync = (...args: any[]) => global.core.api?.mkdirSync(...args)
 
+const mkdir = (...args: any[]) => global.core.api?.mkdir(...args)
+
 /**
  * Removes a directory at the specified path.
  * @returns {Promise<any>} A Promise that resolves when the directory is removed successfully.
@@ -92,6 +94,7 @@ export const fs = {
  existsSync,
  readdirSync,
  mkdirSync,
+ mkdir,
  rmdirSync,
  unlinkSync,
  appendFileSync,

diff --git a/core/src/node/api/processors/fsExt.ts b/core/src/node/api/processors/fsExt.ts
@@ -88,4 +88,16 @@ export class FSExt implements Processor {
  })
  })
  }
+
+ mkdir(path: string): Promise<void> {
+ return new Promise((resolve, reject) => {
+ fs.mkdir(path, { recursive: true }, (err) => {
+ if (err) {
+ reject(err)
+ } else {
+ resolve()
+ }
+ })
+ })
+ }
 }
diff --git a/core/src/node/extension/store.ts b/core/src/node/extension/store.ts
@@ -93,24 +93,25 @@ export function persistExtensions() {
  */
 export async function installExtensions(extensions: any) {
  const installed: Extension[] = []
- for (const ext of extensions) {
- // Set install options and activation based on input type
+ const installations = extensions.map((ext: any): Promise<void> => {
  const isObject = typeof ext === 'object'
  const spec = isObject ? [ext.specifier, ext] : [ext]
  const activate = isObject ? ext.activate !== false : true
 
  // Install and possibly activate extension
  const extension = new Extension(...spec)
  if (!extension.origin) {
- continue
+ return Promise.resolve()
  }
- await extension._install()
- if (activate) extension.setActive(true)
+ return extension._install().then(() => {
+ if (activate) extension.setActive(true)
+ // Add extension to store if needed
+ addExtension(extension)
+ installed.push(extension)
+ })
+ })
 
- // Add extension to store if needed
- addExtension(extension)
- installed.push(extension)
- }
+ await Promise.all(installations)
 
  // Return list of all installed extensions
  return installed

diff --git a/core/src/types/miscellaneous/systemResourceInfo.ts b/core/src/types/miscellaneous/systemResourceInfo.ts
@@ -30,3 +30,27 @@ export type GpuSettingInfo = {
  name: string
  arch?: string
 }
+
+export type SystemInformation = {
+ gpuSetting: GpuSetting
+ osInfo?: OperatingSystemInfo
+}
+
+export const SupportedPlatforms = ['win32', 'linux', 'darwin'] as const
+export type SupportedPlatformTuple = typeof SupportedPlatforms
+export type SupportedPlatform = SupportedPlatformTuple[number]
+
+export type OperatingSystemInfo = {
+ platform: SupportedPlatform | 'unknown'
+ arch: string
+ release: string
+ machine: string
+ version: string
+ totalMem: number
+ freeMem: number
+}
+
+export type CpuCoreInfo = {
+ model: string
+ speed: number
+}
diff --git a/extensions/monitoring-extension/src/index.ts b/extensions/monitoring-extension/src/index.ts
@@ -1,4 +1,9 @@
-import { GpuSetting, MonitoringExtension, executeOnMain } from '@janhq/core'
+import {
+ GpuSetting,
+ MonitoringExtension,
+ OperatingSystemInfo,
+ executeOnMain,
+} from '@janhq/core'
 
 /**
  * JanMonitoringExtension is a extension that provides system monitoring functionality.
@@ -41,4 +46,8 @@ export default class JanMonitoringExtension extends MonitoringExtension {
  getCurrentLoad(): Promise<any> {
  return executeOnMain(NODE, 'getCurrentLoad')
  }
+
+ getOsInfo(): Promise<OperatingSystemInfo> {
+ return executeOnMain(NODE, 'getOsInfo')
+ }
 }
diff --git a/extensions/monitoring-extension/src/node/index.ts b/extensions/monitoring-extension/src/node/index.ts
@@ -1,9 +1,16 @@
-import { GpuSetting, GpuSettingInfo, ResourceInfo } from '@janhq/core'
+import {
+ GpuSetting,
+ GpuSettingInfo,
+ OperatingSystemInfo,
+ ResourceInfo,
+ SupportedPlatforms,
+} from '@janhq/core'
 import { getJanDataFolderPath, log } from '@janhq/core/node'
 import { mem, cpu } from 'node-os-utils'
 import { exec } from 'child_process'
 import { writeFileSync, existsSync, readFileSync, mkdirSync } from 'fs'
 import path from 'path'
+import os from 'os'
 
 /**
  * Path to the settings directory
@@ -320,3 +327,20 @@ const updateCudaExistence = (
  data.is_initial = false
  return data
 }
+
+export const getOsInfo = (): OperatingSystemInfo => {
+ const platform =
+ SupportedPlatforms.find((p) => p === process.platform) || 'unknown'
+
+ const osInfo: OperatingSystemInfo = {
+ platform: platform,
+ arch: process.arch,
+ release: os.release(),
+ machine: os.machine(),
+ version: os.version(),
+ totalMem: os.totalmem(),
+ freeMem: os.freemem(),
+ }
+
+ return osInfo
+}
diff --git a/extensions/tensorrt-llm-extension/package.json b/extensions/tensorrt-llm-extension/package.json
@@ -18,6 +18,8 @@
  "0.1.0"
  ]
  },
+ "tensorrtVersion": "0.1.6",
+ "provider": "nitro-tensorrt-llm",
  "scripts": {
  "build": "tsc --module commonjs && rollup -c rollup.config.ts",
  "build:publish:win32": "rimraf *.tgz --glob && npm run build && cpx \"bin/**\" \"dist/bin\" && npm pack && cpx *.tgz ../../pre-install",

diff --git a/extensions/tensorrt-llm-extension/rollup.config.ts b/extensions/tensorrt-llm-extension/rollup.config.ts
@@ -16,9 +16,10 @@ export default [
  plugins: [
  replace({
  EXTENSION_NAME: JSON.stringify(packageJson.name),
- TENSORRT_VERSION: JSON.stringify('0.1.5'),
+ TENSORRT_VERSION: JSON.stringify(packageJson.tensorrtVersion),
+ PROVIDER: JSON.stringify(packageJson.provider),
  DOWNLOAD_RUNNER_URL:
- process.platform === 'darwin' || process.platform === 'win32'
+ process.platform === 'win32'
  ? JSON.stringify(
  'https://github.com/janhq/nitro-tensorrt-llm/releases/download/windows-v<version>/nitro-windows-v<version>-amd64-tensorrt-llm-<gpuarch>.tar.gz'
  )
@@ -52,11 +53,14 @@ export default [
  },
  plugins: [
  replace({
+ EXTENSION_NAME: JSON.stringify(packageJson.name),
+ TENSORRT_VERSION: JSON.stringify(packageJson.tensorrtVersion),
+ PROVIDER: JSON.stringify(packageJson.provider),
  LOAD_MODEL_URL: JSON.stringify(
  `${packageJson.config?.protocol ?? 'http'}://${packageJson.config?.host}:${packageJson.config?.port}/inferences/tensorrtllm/loadmodel`
  ),
  TERMINATE_ENGINE_URL: JSON.stringify(
- `${packageJson.config?.protocol ?? 'http'}://${packageJson.config?.host}:${packageJson.config?.port}/inferences/processmanager/destroy`
+ `${packageJson.config?.protocol ?? 'http'}://${packageJson.config?.host}:${packageJson.config?.port}/processmanager/destroy`
  ),
  ENGINE_HOST: JSON.stringify(packageJson.config?.host ?? '127.0.0.1'),
  ENGINE_PORT: JSON.stringify(packageJson.config?.port ?? '3928'),

diff --git a/extensions/tensorrt-llm-extension/src/@types/global.d.ts b/extensions/tensorrt-llm-extension/src/@types/global.d.ts
@@ -8,3 +8,4 @@ declare const DOWNLOAD_RUNNER_URL: string
 declare const TENSORRT_VERSION: string
 declare const COMPATIBILITY: object
 declare const EXTENSION_NAME: string
+declare const PROVIDER: string