Add MLC Web-LLM Integration

docs/core_docs/docs/integrations/chat/web_llm.mdx

@@ -0,0 +1,42 @@
+---
+sidebar_class_name: web-only
+---
+
+# WebLLM
+
+:::tip Compatibility
+Only available in web environments.
+:::
+
+You can run LLMs directly in your web browser using LangChain's [WebLLM](https://webllm.mlc.ai) integration.
+
+## Setup
+
+You'll need to install the [WebLLM SDK](https://www.npmjs.com/package/@mlc-ai/web-llm) module to communicate with your local model.
+
+import IntegrationInstallTooltip from "@mdx_components/integration_install_tooltip.mdx";
+
+<IntegrationInstallTooltip></IntegrationInstallTooltip>
+
+```bash npm2yarn
+npm install -S @mlc-ai/web-llm @langchain/community
+```
+
+## Usage
+
+Note that the first time a model is called, WebLLM will download the full weights for that model. This can be multiple gigabytes, and may not be possible for all end-users of your application depending on their internet connection and computer specs.
+While the browser will cache future invocations of that model, we recommend using the smallest possible model you can.
+
+We also recommend using a [separate web worker](https://developer.mozilla.org/en-US/docs/Web/API/Web_Workers_API/Using_web_workers) when invoking and loading your models to
+not block execution.
+
+import CodeBlock from "@theme/CodeBlock";
+import Example from "@examples/models/chat/integration_webllm.ts";
+
+<CodeBlock language="typescript">{Example}</CodeBlock>
+
+Streaming is also supported.
+
+## Example
+
+For a full end-to-end example, check out [this project](https://github.com/jacoblee93/fully-local-pdf-chatbot).

docs/core_docs/src/css/custom.css

@@ -50,6 +50,7 @@
 }
 
 .node-only,
+.web-only,
 .beta {
  position: relative;
 }
@@ -60,7 +61,8 @@
 }
 
 .node-only::after,
-.beta::after {
+.beta::after,
+.web-only::after {
  position: absolute;
  right: 0.25rem;
  top: 5px;
@@ -91,6 +93,19 @@
  color: #fff;
 }
 
+/* Override `web-only` color */
+.web-only::after {
+ content: "Web-only";
+ color: #0a0072;
+ border: 1px solid #0a0072;
+}
+
+/* Override `web-only` color */
+[data-theme="dark"] .web-only::after {
+ background: #0a0072;
+ color: #fff;
+}
+
 .node-only-category,
 .beta-category {
  position: relative;

examples/src/models/chat/integration_webllm.ts

@@ -0,0 +1,33 @@
+// Must be run in a web environment, e.g. a web worker
+
+import { ChatWebLLM } from "@langchain/community/chat_models/webllm";
+import { HumanMessage } from "@langchain/core/messages";
+
+// Initialize the ChatWebLLM model with the model record and chat options.
+// Note that if the appConfig field is set, the list of model records
+// must include the selected model record for the engine.
+
+// You can import a list of models available by default here:
+// https://github.com/mlc-ai/web-llm/blob/main/src/config.ts
+//
+// Or by importing it via:
+// import { prebuiltAppConfig } from "@mlc-ai/web-llm";
+const model = new ChatWebLLM({
+ model: "Phi2-q4f32_1",
+ chatOptions: {
+ temperature: 0.5,
+ },
+});
+
+// Call the model with a message and await the response.
+const response = await model.invoke([
+ new HumanMessage({ content: "What is 1 + 1?" }),
+]);
+
+console.log(response);
+
+/*
+AIMessage {
+ content: ' 2\n',
+}
+*/

libs/langchain-community/.gitignore

@@ -538,6 +538,10 @@ chat_models/togetherai.cjs
 chat_models/togetherai.js
 chat_models/togetherai.d.ts
 chat_models/togetherai.d.cts
+chat_models/webllm.cjs
+chat_models/webllm.js
+chat_models/webllm.d.ts
+chat_models/webllm.d.cts
 chat_models/yandex.cjs
 chat_models/yandex.js
 chat_models/yandex.d.ts

libs/langchain-community/langchain.config.js

@@ -170,6 +170,7 @@ export const config = {
  "chat_models/portkey": "chat_models/portkey",
  "chat_models/premai": "chat_models/premai",
  "chat_models/togetherai": "chat_models/togetherai",
+ "chat_models/webllm": "chat_models/webllm",
  "chat_models/yandex": "chat_models/yandex",
  "chat_models/zhipuai": "chat_models/zhipuai",
  // callbacks
@@ -388,6 +389,7 @@ export const config = {
  "chat_models/premai",
  "chat_models/iflytek_xinghuo",
  "chat_models/iflytek_xinghuo/web",
+ "chat_models/webllm",
  "chat_models/zhipuai",
  "retrievers/amazon_kendra",
  "retrievers/amazon_knowledge_base",

libs/langchain-community/package.json

@@ -82,6 +82,7 @@
  "@jest/globals": "^29.5.0",
  "@langchain/scripts": "~0.0",
  "@mendable/firecrawl-js": "^0.0.13",
+ "@mlc-ai/web-llm": "^0.2.35",
  "@mozilla/readability": "^0.4.4",
  "@neondatabase/serverless": "^0.9.1",
  "@notionhq/client": "^2.2.10",
@@ -233,6 +234,7 @@
  "@gradientai/nodejs-sdk": "^1.2.0",
  "@huggingface/inference": "^2.6.4",
  "@mendable/firecrawl-js": "^0.0.13",
+ "@mlc-ai/web-llm": "^0.2.35",
  "@mozilla/readability": "*",
  "@neondatabase/serverless": "*",
  "@notionhq/client": "^2.2.10",
@@ -398,6 +400,9 @@
  "@mendable/firecrawl-js": {
  "optional": true
  },
+ "@mlc-ai/web-llm": {
+ "optional": true
+ },
  "@mozilla/readability": {
  "optional": true
  },
@@ -1882,6 +1887,15 @@
  "import": "./chat_models/togetherai.js",
  "require": "./chat_models/togetherai.cjs"
  },
+ "./chat_models/webllm": {
+ "types": {
+ "import": "./chat_models/webllm.d.ts",
+ "require": "./chat_models/webllm.d.cts",
+ "default": "./chat_models/webllm.d.ts"
+ },
+ "import": "./chat_models/webllm.js",
+ "require": "./chat_models/webllm.cjs"
+ },
  "./chat_models/yandex": {
  "types": {
  "import": "./chat_models/yandex.d.ts",
@@ -3362,6 +3376,10 @@
  "chat_models/togetherai.js",
  "chat_models/togetherai.d.ts",
  "chat_models/togetherai.d.cts",
+ "chat_models/webllm.cjs",
+ "chat_models/webllm.js",
+ "chat_models/webllm.d.ts",
+ "chat_models/webllm.d.cts",
  "chat_models/yandex.cjs",
  "chat_models/yandex.js",
  "chat_models/yandex.d.ts",

libs/langchain-community/src/chat_models/tests/webllm.int.test.ts

@@ -0,0 +1,69 @@
+// TODO: Fix for Node environments
+
+// import { ChatWebLLM, WebLLMInputs } from "../webllm.js";
+// import * as webllm from "@mlc-ai/web-llm";
+
+// jest.mock("@mlc-ai/web-llm", () => ({
+// Engine: jest.fn().mockImplementation(() => ({
+// reload: jest.fn().mockResolvedValue(undefined),
+// setInitProgressCallback: jest.fn(),
+// chat: {
+// completions: {
+// create: jest.fn().mockImplementation(() => {
+// const messages = [
+// {
+// choices: [
+// {
+// delta: { content: "Hello" },
+// logprobs: null,
+// finish_reason: "complete",
+// },
+// ],
+// },
+// {
+// choices: [
+// {
+// delta: { content: "How are you?" },
+// logprobs: null,
+// finish_reason: "complete",
+// },
+// ],
+// },
+// ];
+// return (async function* () {
+// for (let msg of messages) {
+// yield msg;
+// }
+// })();
+// }),
+// },
+// },
+// })),
+// }));
+
+describe("ChatWebLLM Integration Tests", () => {
+ // let chatWebLLM: ChatWebLLM;
+ // let modelRecord = { model_id: "test-model" };
+
+ // beforeEach(() => {
+ // const inputs: WebLLMInputs = {
+ // modelRecord: modelRecord,
+ // appConfig: {},
+ // chatOpts: {},
+ // };
+ // chatWebLLM = new ChatWebLLM(inputs);
+ // });
+
+ test("ChatWebLLM initializes and processes messages correctly", async () => {
+ // const options = {}; // Adjust options as necessary
+ // const response = await chatWebLLM.invoke("Hello", options);
+ // expect(response).toBe("Hello");
+ // expect(webllm.Engine).toHaveBeenCalled();
+ // expect(webllm.Engine().chat.completions.create).toHaveBeenCalledWith({
+ // stream: true,
+ // messages: [{ role: "user", content: "Hello" }],
+ // stop: null,
+ // logprobs: true,
+ // });
+ });
+});