diff --git a/CHANGELOG.md b/CHANGELOG.md index eca50a57ac..8aeccb292c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,27 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). Nothing unreleased! +## [1.1.0rc0] - 2024-05-06 + +### Added + +- `cl.on_audio_chunk` decorator to process incoming the user incoming audio stream +- `cl.on_audio_end` decorator to react to the end of the user audio stream +- The `cl.Audio` element now has an `auto_play` property +- `http_referer` is now available in `cl.user_session` + +### Changed + +- The UI has been revamped, especially the navigation +- The arrow up button has been removed from the input bar, however pressing the arrow up key still opens the last inputs menu +- **[breaking]** the `send()` method on `cl.Message` now returns the message instead of the message id +- **[breaking]** The `multi_modal` feature has been renamed `spontaneous_file_upload` in the config +- Element display property now defaults to `inline` instead of `side` + +### Fixed + +- Stopping a task should now work better (using asyncio task.cancel) + ## [1.0.506] - 2024-04-30 ### Added diff --git a/backend/chainlit/__init__.py b/backend/chainlit/__init__.py index 093245d31c..eca2420512 100644 --- a/backend/chainlit/__init__.py +++ b/backend/chainlit/__init__.py @@ -52,7 +52,7 @@ from chainlit.step import Step, step from chainlit.sync import make_async, run_sync from chainlit.telemetry import trace -from chainlit.types import ChatProfile, ThreadDict +from chainlit.types import AudioChunk, ChatProfile, ThreadDict from chainlit.user import PersistedUser, User from chainlit.user_session import user_session from chainlit.utils import make_module_getattr, wrap_user_function @@ -224,6 +224,38 @@ def on_chat_end(func: Callable) -> Callable: return func +@trace +def on_audio_chunk(func: Callable) -> Callable: + """ + Hook to react to the audio chunks being sent. + + Args: + chunk (AudioChunk): The audio chunk being sent. + + Returns: + Callable[], Any]: The decorated hook. + """ + + config.code.on_audio_chunk = wrap_user_function(func, with_task=False) + return func + + +@trace +def on_audio_end(func: Callable) -> Callable: + """ + Hook to react to the audio stream ending. This is called after the last audio chunk is sent. + + Args: + elements ([List[Element]): The files that were uploaded before starting the audio stream (if any). + + Returns: + Callable[], Any]: The decorated hook. + """ + + config.code.on_audio_end = wrap_user_function(func, with_task=True) + return func + + @trace def author_rename(func: Callable[[str], str]) -> Callable[[str], str]: """ @@ -318,6 +350,7 @@ def acall(self): __all__ = [ "user_session", "CopilotFunction", + "AudioChunk", "Action", "User", "PersistedUser", diff --git a/backend/chainlit/config.py b/backend/chainlit/config.py index d15080a45b..5faca03172 100644 --- a/backend/chainlit/config.py +++ b/backend/chainlit/config.py @@ -16,7 +16,9 @@ if TYPE_CHECKING: from chainlit.action import Action - from chainlit.types import ChatProfile, ThreadDict + from chainlit.element import ElementBased + from chainlit.message import Message + from chainlit.types import AudioChunk, ChatProfile, ThreadDict from chainlit.user import User from fastapi import Request, Response @@ -71,18 +73,26 @@ # Automatically tag threads with the current chat profile (if a chat profile is used) auto_tag_thread = true -# Authorize users to upload files with messages -[features.multi_modal] +# Authorize users to spontaneously upload files with messages +[features.spontaneous_file_upload] enabled = true accept = ["*/*"] max_files = 20 max_size_mb = 500 -# Allows user to use speech to text -[features.speech_to_text] - enabled = false - # See all languages here https://github.com/JamesBrill/react-speech-recognition/blob/HEAD/docs/API.md#language-string - # language = "en-US" +[features.audio] + # Threshold for audio recording + min_decibels = -45 + # Delay for the user to start speaking in MS + initial_silence_timeout = 3000 + # Delay for the user to continue speaking in MS. If the user stops speaking for this duration, the recording will stop. + silence_timeout = 1500 + # Above this duration (MS), the recording will forcefully stop. + max_duration = 15000 + # Duration of the audio chunks in MS + chunk_duration = 1000 + # Sample rate of the audio + sample_rate = 44100 [UI] # Name of the app and chatbot. @@ -189,26 +199,31 @@ class Theme(DataClassJsonMixin): @dataclass -class SpeechToTextFeature: - enabled: Optional[bool] = None - language: Optional[str] = None - - -@dataclass -class MultiModalFeature: +class SpontaneousFileUploadFeature(DataClassJsonMixin): enabled: Optional[bool] = None accept: Optional[Union[List[str], Dict[str, List[str]]]] = None max_files: Optional[int] = None max_size_mb: Optional[int] = None +@dataclass +class AudioFeature(DataClassJsonMixin): + min_decibels: int = -45 + initial_silence_timeout: int = 2000 + silence_timeout: int = 1500 + chunk_duration: int = 1000 + max_duration: int = 15000 + sample_rate: int = 44100 + enabled: bool = False + + @dataclass() class FeaturesSettings(DataClassJsonMixin): prompt_playground: bool = True - multi_modal: Optional[MultiModalFeature] = None + spontaneous_file_upload: Optional[SpontaneousFileUploadFeature] = None + audio: Optional[AudioFeature] = Field(default_factory=AudioFeature) latex: bool = False unsafe_allow_html: bool = False - speech_to_text: Optional[SpeechToTextFeature] = None auto_tag_thread: bool = True @@ -247,7 +262,10 @@ class CodeSettings: on_chat_start: Optional[Callable[[], Any]] = None on_chat_end: Optional[Callable[[], Any]] = None on_chat_resume: Optional[Callable[["ThreadDict"], Any]] = None - on_message: Optional[Callable[[str], Any]] = None + on_message: Optional[Callable[["Message"], Any]] = None + on_audio_chunk: Optional[Callable[["AudioChunk"], Any]] = None + on_audio_end: Optional[Callable[[List["ElementBased"]], Any]] = None + author_rename: Optional[Callable[[str], str]] = None on_settings_update: Optional[Callable[[Dict[str, Any]], Any]] = None set_chat_profiles: Optional[Callable[[Optional["User"]], List["ChatProfile"]]] = ( @@ -413,11 +431,13 @@ def load_settings(): ui_settings = UISettings(**ui_settings) + code_settings = CodeSettings(action_callbacks={}) + return { "features": features_settings, "ui": ui_settings, "project": project_settings, - "code": CodeSettings(action_callbacks={}), + "code": code_settings, } diff --git a/backend/chainlit/data/__init__.py b/backend/chainlit/data/__init__.py index b5a90e7ef4..186c358c8d 100644 --- a/backend/chainlit/data/__init__.py +++ b/backend/chainlit/data/__init__.py @@ -156,6 +156,7 @@ def attachment_to_element_dict(self, attachment: Attachment) -> "ElementDict": "chainlitKey": None, "display": metadata.get("display", "side"), "language": metadata.get("language"), + "autoPlay": metadata.get("autoPlay", None), "page": metadata.get("page"), "size": metadata.get("size"), "type": metadata.get("type", "file"), @@ -219,7 +220,7 @@ def step_to_step_dict(self, step: LiteralStep) -> "StepDict": "disableFeedback": metadata.get("disableFeedback", False), "indent": metadata.get("indent"), "language": metadata.get("language"), - "isError": metadata.get("isError", False), + "isError": bool(step.error), "waitForAnswer": metadata.get("waitForAnswer", False), } @@ -348,7 +349,6 @@ async def create_step(self, step_dict: "StepDict"): step_dict.get("metadata", {}), **{ "disableFeedback": step_dict.get("disableFeedback"), - "isError": step_dict.get("isError"), "waitForAnswer": step_dict.get("waitForAnswer"), "language": step_dict.get("language"), "showInput": step_dict.get("showInput"), @@ -372,6 +372,8 @@ async def create_step(self, step_dict: "StepDict"): step["input"] = {"content": step_dict.get("input")} if step_dict.get("output"): step["output"] = {"content": step_dict.get("output")} + if step_dict.get("isError"): + step["error"] = step_dict.get("output") await self.client.api.send_steps([step]) diff --git a/backend/chainlit/data/sql_alchemy.py b/backend/chainlit/data/sql_alchemy.py index b87e94fa5d..2902795821 100644 --- a/backend/chainlit/data/sql_alchemy.py +++ b/backend/chainlit/data/sql_alchemy.py @@ -170,12 +170,14 @@ async def update_thread( raise ValueError("User not found in session context") data = { "id": thread_id, - "createdAt": await self.get_current_timestamp() - if metadata is None - else None, - "name": name - if name is not None - else (metadata.get("name") if metadata and "name" in metadata else None), + "createdAt": ( + await self.get_current_timestamp() if metadata is None else None + ), + "name": ( + name + if name is not None + else (metadata.get("name") if metadata and "name" in metadata else None) + ), "userId": user_id, "userIdentifier": user_identifier, "tags": tags, @@ -552,13 +554,17 @@ async def get_all_user_threads( streaming=step_feedback.get("step_streaming", False), waitForAnswer=step_feedback.get("step_waitforanswer"), isError=step_feedback.get("step_iserror"), - metadata=step_feedback["step_metadata"] - if step_feedback.get("step_metadata") is not None - else {}, + metadata=( + step_feedback["step_metadata"] + if step_feedback.get("step_metadata") is not None + else {} + ), tags=step_feedback.get("step_tags"), - input=step_feedback.get("step_input", "") - if step_feedback["step_showinput"] - else "", + input=( + step_feedback.get("step_input", "") + if step_feedback["step_showinput"] + else "" + ), output=step_feedback.get("step_output", ""), createdAt=step_feedback.get("step_createdat"), start=step_feedback.get("step_start"), @@ -587,6 +593,7 @@ async def get_all_user_threads( display=element["element_display"], size=element.get("element_size"), language=element.get("element_language"), + autoPlay=element.get("element_autoPlay"), page=element.get("element_page"), forId=element.get("element_forid"), mime=element.get("element_mime"), diff --git a/backend/chainlit/element.py b/backend/chainlit/element.py index 73351549ac..bca8583450 100644 --- a/backend/chainlit/element.py +++ b/backend/chainlit/element.py @@ -38,6 +38,7 @@ class ElementDict(TypedDict): size: Optional[ElementSize] language: Optional[str] page: Optional[int] + autoPlay: Optional[bool] forId: Optional[str] mime: Optional[str] @@ -61,7 +62,7 @@ class Element: # The byte content of the element. content: Optional[Union[bytes, str]] = None # Controls how the image element should be displayed in the UI. Choices are “side” (default), “inline”, or “page”. - display: ElementDisplay = Field(default="side") + display: ElementDisplay = Field(default="inline") # Controls element size size: Optional[ElementSize] = None # The ID of the message this element is associated with. @@ -93,6 +94,7 @@ def to_dict(self) -> ElementDict: "objectKey": getattr(self, "object_key", None), "size": getattr(self, "size", None), "page": getattr(self, "page", None), + "autoPlay": getattr(self, "auto_play", None), "language": getattr(self, "language", None), "forId": getattr(self, "for_id", None), "mime": getattr(self, "mime", None), @@ -306,6 +308,7 @@ async def preprocess_content(self): @dataclass class Audio(Element): type: ClassVar[ElementType] = "audio" + auto_play: bool = False @dataclass diff --git a/backend/chainlit/llama_index/callbacks.py b/backend/chainlit/llama_index/callbacks.py index fc8ea56a07..7da19e8987 100644 --- a/backend/chainlit/llama_index/callbacks.py +++ b/backend/chainlit/llama_index/callbacks.py @@ -70,7 +70,7 @@ def on_event_start( ) -> str: """Run when an event starts and return id of event.""" self._restore_context() - + step_type: StepType = "undefined" if event_type == CBEventType.RETRIEVE: step_type = "retrieval" @@ -104,7 +104,6 @@ def on_event_end( """Run when an event ends.""" step = self.steps.get(event_id, None) - if payload is None or step is None: return @@ -117,11 +116,13 @@ def on_event_end( source_nodes = getattr(response, "source_nodes", None) if source_nodes: source_refs = ", ".join( - [f"Source {idx}" for idx, _ in enumerate(source_nodes)]) + [f"Source {idx}" for idx, _ in enumerate(source_nodes)] + ) step.elements = [ Text( name=f"Source {idx}", content=source.text or "Empty node", + display="side", ) for idx, source in enumerate(source_nodes) ] @@ -137,6 +138,7 @@ def on_event_end( step.elements = [ Text( name=f"Source {idx}", + display="side", content=source.node.get_text() or "Empty node", ) for idx, source in enumerate(sources) @@ -173,7 +175,7 @@ def on_event_end( token_count = self.total_llm_token_count or None raw_response = response.raw if response else None model = raw_response.get("model", None) if raw_response else None - + if messages and isinstance(response, ChatResponse): msg: ChatMessage = response.message step.generation = ChatGeneration( @@ -198,7 +200,7 @@ def on_event_end( else: step.output = payload self.context.loop.create_task(step.update()) - + self.steps.pop(event_id, None) def _noop(self, *args, **kwargs): @@ -206,4 +208,3 @@ def _noop(self, *args, **kwargs): start_trace = _noop end_trace = _noop - diff --git a/backend/chainlit/message.py b/backend/chainlit/message.py index 3abb7cb8cd..688a52f223 100644 --- a/backend/chainlit/message.py +++ b/backend/chainlit/message.py @@ -166,7 +166,7 @@ async def send(self): step_dict = await self._create() await context.emitter.send_step(step_dict) - return self.id + return self async def stream_token(self, token: str, is_sequence=False): """ @@ -251,7 +251,7 @@ def __init__( super().__post_init__() - async def send(self) -> str: + async def send(self): """ Send the message to the UI and persist it in the cloud if a project ID is configured. Return the ID of the message. @@ -268,7 +268,7 @@ async def send(self) -> str: # Run all tasks concurrently await asyncio.gather(*tasks) - return self.id + return self async def update(self): """ diff --git a/backend/chainlit/server.py b/backend/chainlit/server.py index 7fe842c956..83385060cf 100644 --- a/backend/chainlit/server.py +++ b/backend/chainlit/server.py @@ -536,6 +536,10 @@ async def project_settings( chat_profiles = await config.code.set_chat_profiles(current_user) if chat_profiles: profiles = [p.to_dict() for p in chat_profiles] + + if config.code.on_audio_chunk: + config.features.audio.enabled = True + return JSONResponse( content={ "ui": config.ui.to_dict(), diff --git a/backend/chainlit/session.py b/backend/chainlit/session.py index 5c1c766f15..05e2b4b967 100644 --- a/backend/chainlit/session.py +++ b/backend/chainlit/session.py @@ -1,3 +1,4 @@ +import asyncio import json import mimetypes import shutil @@ -45,6 +46,7 @@ class BaseSession: thread_id_to_resume: Optional[str] = None client_type: ClientType + current_task: Optional[asyncio.Task] = None def __init__( self, @@ -63,6 +65,8 @@ def __init__( root_message: Optional["Message"] = None, # Chat profile selected before the session was created chat_profile: Optional[str] = None, + # Origin of the request + http_referer: Optional[str] = None, ): if thread_id: self.thread_id_to_resume = thread_id @@ -74,6 +78,7 @@ def __init__( self.has_first_interaction = False self.user_env = user_env or {} self.chat_profile = chat_profile + self.http_referer = http_referer self.id = id @@ -115,7 +120,8 @@ def __init__( user_env: Optional[Dict[str, str]] = None, # Last message at the root of the chat root_message: Optional["Message"] = None, - # User specific environment variables. Empty if no user environment variables are required. + # Origin of the request + http_referer: Optional[str] = None, ): super().__init__( id=id, @@ -125,6 +131,7 @@ def __init__( client_type=client_type, user_env=user_env, root_message=root_message, + http_referer=http_referer, ) @@ -165,6 +172,8 @@ def __init__( chat_profile: Optional[str] = None, # Languages of the user's browser languages: Optional[str] = None, + # Origin of the request + http_referer: Optional[str] = None, ): super().__init__( id=id, @@ -175,13 +184,13 @@ def __init__( client_type=client_type, root_message=root_message, chat_profile=chat_profile, + http_referer=http_referer, ) self.socket_id = socket_id self.emit_call = emit_call self.emit = emit - self.should_stop = False self.restored = False self.thread_queues = {} # type: Dict[str, Deque[Callable]] @@ -217,6 +226,7 @@ async def persist_file( file_path = self.files_dir / file_id file_extension = mimetypes.guess_extension(mime) + if file_extension: file_path = file_path.with_suffix(file_extension) diff --git a/backend/chainlit/socket.py b/backend/chainlit/socket.py index 4e357228e3..34c6a892f4 100644 --- a/backend/chainlit/socket.py +++ b/backend/chainlit/socket.py @@ -9,12 +9,18 @@ from chainlit.config import config from chainlit.context import init_ws_context from chainlit.data import get_data_layer +from chainlit.element import Element from chainlit.logger import logger from chainlit.message import ErrorMessage, Message from chainlit.server import socket from chainlit.session import WebsocketSession from chainlit.telemetry import trace_event -from chainlit.types import UIMessagePayload +from chainlit.types import ( + AudioChunk, + AudioChunkPayload, + AudioEndPayload, + UIMessagePayload, +) from chainlit.user_session import user_sessions @@ -93,9 +99,13 @@ def build_anon_user_identifier(environ): @socket.on("connect") async def connect(sid, environ, auth): - if not config.code.on_chat_start and not config.code.on_message: + if ( + not config.code.on_chat_start + and not config.code.on_message + and not config.code.on_audio_chunk + ): logger.warning( - "You need to configure at least an on_chat_start or an on_message callback" + "You need to configure at least one of on_chat_start, on_message or on_audio_chunk callback" ) return False user = None @@ -113,18 +123,10 @@ async def connect(sid, environ, auth): # Session scoped function to emit to the client def emit_fn(event, data): - if session := WebsocketSession.get(sid): - if session.should_stop: - session.should_stop = False - raise InterruptedError("Task stopped by user") return socket.emit(event, data, to=sid) # Session scoped function to emit to the client and wait for a response def emit_call_fn(event: Literal["ask", "call_fn"], data, timeout): - if session := WebsocketSession.get(sid): - if session.should_stop: - session.should_stop = False - raise InterruptedError("Task stopped by user") return socket.call(event, data, timeout=timeout, to=sid) session_id = environ.get("HTTP_X_CHAINLIT_SESSION_ID") @@ -135,6 +137,7 @@ def emit_call_fn(event: Literal["ask", "call_fn"], data, timeout): user_env = load_user_env(user_env_string) client_type = environ.get("HTTP_X_CHAINLIT_CLIENT_TYPE") + http_referer = environ.get("HTTP_REFERER") ws_session = WebsocketSession( id=session_id, @@ -148,6 +151,7 @@ def emit_call_fn(event: Literal["ask", "call_fn"], data, timeout): chat_profile=environ.get("HTTP_X_CHAINLIT_CHAT_PROFILE"), thread_id=environ.get("HTTP_X_CHAINLIT_THREAD_ID"), languages=environ.get("HTTP_ACCEPT_LANGUAGE"), + http_referer=http_referer, ) trace_event("connection_successful") @@ -178,7 +182,8 @@ async def connection_successful(sid): return if config.code.on_chat_start: - await config.code.on_chat_start() + task = asyncio.create_task(config.code.on_chat_start()) + context.session.current_task = task @socket.on("clear_session") @@ -223,10 +228,11 @@ async def stop(sid): init_ws_context(session) await Message( - author="System", content="Task stopped by the user.", disable_feedback=True + author="System", content="Task manually stopped.", disable_feedback=True ).send() - session.should_stop = True + if session.current_task: + session.current_task.cancel() if config.code.on_stop: await config.code.on_stop() @@ -243,7 +249,7 @@ async def process_message(session: WebsocketSession, payload: UIMessagePayload): # Sleep 1ms to make sure any children step starts after the message step start time.sleep(0.001) await config.code.on_message(message) - except InterruptedError: + except asyncio.CancelledError: pass except Exception as e: logger.exception(e) @@ -258,9 +264,55 @@ async def process_message(session: WebsocketSession, payload: UIMessagePayload): async def message(sid, payload: UIMessagePayload): """Handle a message sent by the User.""" session = WebsocketSession.require(sid) - session.should_stop = False - await process_message(session, payload) + task = asyncio.create_task(process_message(session, payload)) + session.current_task = task + + +@socket.on("audio_chunk") +async def audio_chunk(sid, payload: AudioChunkPayload): + """Handle an audio chunk sent by the user.""" + session = WebsocketSession.require(sid) + + init_ws_context(session) + + if config.code.on_audio_chunk: + asyncio.create_task(config.code.on_audio_chunk(AudioChunk(**payload))) + + +@socket.on("audio_end") +async def audio_end(sid, payload: AudioEndPayload): + """Handle the end of the audio stream.""" + session = WebsocketSession.require(sid) + try: + context = init_ws_context(session) + await context.emitter.task_start() + + if not session.has_first_interaction: + session.has_first_interaction = True + asyncio.create_task(context.emitter.init_thread("audio")) + + file_elements = [] + if config.code.on_audio_end: + file_refs = payload.get("fileReferences") + if file_refs: + files = [ + session.files[file["id"]] + for file in file_refs + if file["id"] in session.files + ] + file_elements = [Element.from_dict(file) for file in files] + + await config.code.on_audio_end(file_elements) + except asyncio.CancelledError: + pass + except Exception as e: + logger.exception(e) + await ErrorMessage( + author="Error", content=str(e) or e.__class__.__name__ + ).send() + finally: + await context.emitter.task_end() async def process_action(action: Action): @@ -288,7 +340,7 @@ async def call_action(sid, action): id=action.id, status=True, response=res if isinstance(res, str) else None ) - except InterruptedError: + except asyncio.CancelledError: await context.emitter.send_action_response( id=action.id, status=False, response="Action interrupted by the user" ) diff --git a/backend/chainlit/types.py b/backend/chainlit/types.py index 27e16bd2a2..fbf7f79570 100644 --- a/backend/chainlit/types.py +++ b/backend/chainlit/types.py @@ -154,6 +154,25 @@ class UIMessagePayload(TypedDict): fileReferences: Optional[List[FileReference]] +class AudioChunkPayload(TypedDict): + isStart: bool + mimeType: str + elapsedTime: float + data: bytes + + +@dataclass +class AudioChunk: + isStart: bool + mimeType: str + elapsedTime: float + data: bytes + + +class AudioEndPayload(TypedDict): + fileReferences: Optional[List[FileReference]] + + @dataclass class AskFileResponse: id: str diff --git a/backend/chainlit/user_session.py b/backend/chainlit/user_session.py index 1348e04adf..e438ed95d6 100644 --- a/backend/chainlit/user_session.py +++ b/backend/chainlit/user_session.py @@ -28,6 +28,7 @@ def get(self, key, default=None): user_session["user"] = context.session.user user_session["chat_profile"] = context.session.chat_profile user_session["languages"] = context.session.languages + user_session["http_referer"] = context.session.http_referer if context.session.root_message: user_session["root_message"] = context.session.root_message diff --git a/backend/chainlit/utils.py b/backend/chainlit/utils.py index ff92881b30..922ddd4138 100644 --- a/backend/chainlit/utils.py +++ b/backend/chainlit/utils.py @@ -1,6 +1,7 @@ import functools import importlib import inspect +from asyncio import CancelledError from typing import Callable from chainlit.context import context @@ -39,7 +40,7 @@ async def wrapper(*args): return await user_function(**params_values) else: return user_function(**params_values) - except InterruptedError: + except CancelledError: pass except Exception as e: logger.exception(e) diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 8bc5f7b178..6440e28985 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "chainlit" -version = "1.0.506" +version = "1.1.0rc0" keywords = ['LLM', 'Agents', 'gen ai', 'chat ui', 'chatbot ui', 'openai', 'copilot', 'langchain', 'conversational ai'] description = "Build Conversational AI." authors = ["Chainlit"] diff --git a/cypress/e2e/chat_profiles/spec.cy.ts b/cypress/e2e/chat_profiles/spec.cy.ts index f5df8f7dab..95c7a476ac 100644 --- a/cypress/e2e/chat_profiles/spec.cy.ts +++ b/cypress/e2e/chat_profiles/spec.cy.ts @@ -10,13 +10,8 @@ describe('Chat profiles', () => { cy.get("input[name='email']").type('admin'); cy.get("input[name='password']").type('admin'); cy.get("button[type='submit']").click(); - cy.get('.MuiAlert-message').should('not.exist'); cy.get('#chat-input').should('exist'); - cy.get('[data-test="chat-profile:GPT-3.5"]').should('exist'); - cy.get('[data-test="chat-profile:GPT-4"]').should('exist'); - cy.get('[data-test="chat-profile:GPT-5"]').should('exist'); - cy.get('.step') .should('have.length', 1) .eq(0) @@ -25,9 +20,14 @@ describe('Chat profiles', () => { 'starting chat with admin using the GPT-3.5 chat profile' ); + cy.get('#chat-profile-selector').parent().click(); + cy.get('[data-test="select-item:GPT-3.5"]').should('exist'); + cy.get('[data-test="select-item:GPT-4"]').should('exist'); + cy.get('[data-test="select-item:GPT-5"]').should('exist'); + // Change chat profile - cy.get('[data-test="chat-profile:GPT-4"]').click(); + cy.get('[data-test="select-item:GPT-4"]').click(); cy.get('.step') .should('have.length', 1) @@ -37,7 +37,7 @@ describe('Chat profiles', () => { 'starting chat with admin using the GPT-4 chat profile' ); - cy.get('#new-chat-button').click(); + cy.get('#header').get('#new-chat-button').click({ force: true }); cy.get('#confirm').click(); cy.get('.step') @@ -50,7 +50,8 @@ describe('Chat profiles', () => { submitMessage('hello'); cy.get('.step').should('have.length', 2).eq(1).should('contain', 'hello'); - cy.get('[data-test="chat-profile:GPT-5"]').click(); + cy.get('#chat-profile-selector').parent().click(); + cy.get('[data-test="select-item:GPT-5"]').click(); cy.get('#confirm').click(); cy.get('.step') diff --git a/cypress/e2e/llama_index_cb/.chainlit/config.toml b/cypress/e2e/llama_index_cb/.chainlit/config.toml index e2a93af08f..a5eee5dfa4 100644 --- a/cypress/e2e/llama_index_cb/.chainlit/config.toml +++ b/cypress/e2e/llama_index_cb/.chainlit/config.toml @@ -28,18 +28,29 @@ unsafe_allow_html = false # Process and display mathematical expressions. This can clash with "$" characters in messages. latex = false -# Authorize users to upload files with messages -[features.multi_modal] +# Automatically tag threads with the current chat profile (if a chat profile is used) +auto_tag_thread = true + +# Authorize users to spontaneously upload files with messages +[features.spontaneous_file_upload] enabled = true accept = ["*/*"] max_files = 20 max_size_mb = 500 -# Allows user to use speech to text -[features.speech_to_text] - enabled = false - # See all languages here https://github.com/JamesBrill/react-speech-recognition/blob/HEAD/docs/API.md#language-string - # language = "en-US" +[features.audio] + # Threshold for audio recording + min_decibels = -45 + # Delay for the user to start speaking in MS + initial_silence_timeout = 3000 + # Delay for the user to continue speaking in MS. If the user stops speaking for this duration, the recording will stop. + silence_timeout = 1500 + # Above this duration (MS), the recording will forcefully stop. + max_duration = 15000 + # Duration of the audio chunks in MS + chunk_duration = 1000 + # Sample rate of the audio + sample_rate = 44100 [UI] # Name of the app and chatbot. @@ -74,6 +85,11 @@ hide_cot = false # Specify a custom font url. # custom_font = "https://fonts.googleapis.com/css2?family=Inter:wght@400;500;700&display=swap" +# Specify a custom build directory for the frontend. +# This can be used to customize the frontend code. +# Be careful: If this is a relative path, it should not start with a slash. +# custom_build = "./public/build" + # Override default MUI light theme. (Check theme.ts) [UI.theme] #font_family = "Inter, sans-serif" @@ -98,4 +114,4 @@ hide_cot = false [meta] -generated_by = "1.0.400" +generated_by = "1.0.504" diff --git a/cypress/e2e/password_auth/spec.cy.ts b/cypress/e2e/password_auth/spec.cy.ts index ebe7447b1b..ed2e78d2f0 100644 --- a/cypress/e2e/password_auth/spec.cy.ts +++ b/cypress/e2e/password_auth/spec.cy.ts @@ -17,7 +17,6 @@ describe('Password Auth', () => { cy.get("input[name='email']").type('admin'); cy.get("input[name='password']").type('admin'); cy.get("button[type='submit']").click(); - cy.get('.MuiAlert-message').should('not.exist'); cy.get('.step').eq(0).should('contain', 'Hello admin'); cy.reload(); diff --git a/cypress/e2e/stop_task/spec.cy.ts b/cypress/e2e/stop_task/spec.cy.ts index f8c3d51930..78144d55e1 100644 --- a/cypress/e2e/stop_task/spec.cy.ts +++ b/cypress/e2e/stop_task/spec.cy.ts @@ -18,7 +18,7 @@ describeSyncAsync('Stop task', (mode) => { cy.get('.step').should('have.length', 2); - cy.get('.step').last().should('contain.text', 'Task stopped by the user.'); + cy.get('.step').last().should('contain.text', 'Task manually stopped.'); cy.wait(5000); diff --git a/cypress/e2e/upload_attachments/.chainlit/config.toml b/cypress/e2e/upload_attachments/.chainlit/config.toml index 7670b17fec..a5eee5dfa4 100644 --- a/cypress/e2e/upload_attachments/.chainlit/config.toml +++ b/cypress/e2e/upload_attachments/.chainlit/config.toml @@ -2,6 +2,7 @@ # Whether to enable telemetry (default: true). No personal data is collected. enable_telemetry = true + # List of environment variables to be provided by each user to use the app. user_env = [] @@ -11,6 +12,9 @@ session_timeout = 3600 # Enable third parties caching (e.g LangChain cache) cache = false +# Authorized origins +allow_origins = ["*"] + # Follow symlink for asset mount (see https://github.com/Chainlit/chainlit/issues/317) # follow_symlink = false @@ -18,10 +22,43 @@ cache = false # Show the prompt playground prompt_playground = true +# Process and display HTML in messages. This can be a security risk (see https://stackoverflow.com/questions/19603097/why-is-it-dangerous-to-render-user-generated-html-or-javascript) +unsafe_allow_html = false + +# Process and display mathematical expressions. This can clash with "$" characters in messages. +latex = false + +# Automatically tag threads with the current chat profile (if a chat profile is used) +auto_tag_thread = true + +# Authorize users to spontaneously upload files with messages +[features.spontaneous_file_upload] + enabled = true + accept = ["*/*"] + max_files = 20 + max_size_mb = 500 + +[features.audio] + # Threshold for audio recording + min_decibels = -45 + # Delay for the user to start speaking in MS + initial_silence_timeout = 3000 + # Delay for the user to continue speaking in MS. If the user stops speaking for this duration, the recording will stop. + silence_timeout = 1500 + # Above this duration (MS), the recording will forcefully stop. + max_duration = 15000 + # Duration of the audio chunks in MS + chunk_duration = 1000 + # Sample rate of the audio + sample_rate = 44100 + [UI] # Name of the app and chatbot. name = "Chatbot" +# Show the readme while the thread is empty. +show_readme_as_default = true + # Description of the app and chatbot. This is used for HTML tags. # description = "" @@ -41,13 +78,21 @@ hide_cot = false # The CSS file can be served from the public directory or via an external link. # custom_css = "/public/test.css" -[features.multi_modal] - enabled = true - accept = ["*/*"] - max_files = 20 - max_size_mb = 500 +# Specify a Javascript file that can be used to customize the user interface. +# The Javascript file can be served from the public directory. +# custom_js = "/public/test.js" + +# Specify a custom font url. +# custom_font = "https://fonts.googleapis.com/css2?family=Inter:wght@400;500;700&display=swap" + +# Specify a custom build directory for the frontend. +# This can be used to customize the frontend code. +# Be careful: If this is a relative path, it should not start with a slash. +# custom_build = "./public/build" # Override default MUI light theme. (Check theme.ts) +[UI.theme] + #font_family = "Inter, sans-serif" [UI.theme.light] #background = "#FAFAFA" #paper = "#FFFFFF" @@ -69,4 +114,4 @@ hide_cot = false [meta] -generated_by = "0.7.2" +generated_by = "1.0.504" diff --git a/cypress/e2e/user_session/spec.cy.ts b/cypress/e2e/user_session/spec.cy.ts index e62cc86a9e..f441d6fe9f 100644 --- a/cypress/e2e/user_session/spec.cy.ts +++ b/cypress/e2e/user_session/spec.cy.ts @@ -1,7 +1,10 @@ import { runTestServer, submitMessage } from '../../support/testUtils'; function newSession() { - cy.get('#new-chat-button').should('exist').click(); + cy.get('#header') + .get('#new-chat-button') + .should('exist') + .click({ force: true }); cy.get('#new-chat-dialog').should('exist'); cy.get('#confirm').should('exist').click(); diff --git a/frontend/package.json b/frontend/package.json index f5d2c90b5f..743516d19f 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -38,7 +38,6 @@ "react-plotly.js": "^2.6.0", "react-resizable": "^3.0.5", "react-router-dom": "^6.15.0", - "react-speech-recognition": "^3.10.0", "recoil": "^0.7.7", "regenerator-runtime": "^0.14.0", "rehype-katex": "^7.0.0", @@ -62,7 +61,6 @@ "@types/react-file-icon": "^1.0.2", "@types/react-plotly.js": "^2.6.3", "@types/react-resizable": "^3.0.4", - "@types/react-speech-recognition": "^3.9.2", "@types/uuid": "^9.0.3", "@vitejs/plugin-react": "^4.0.4", "@vitejs/plugin-react-swc": "^3.3.2", diff --git a/frontend/src/assets/file.tsx b/frontend/src/assets/file.tsx deleted file mode 100644 index 92a38c5e6e..0000000000 --- a/frontend/src/assets/file.tsx +++ /dev/null @@ -1,59 +0,0 @@ -import SvgIcon, { SvgIconProps } from '@mui/material/SvgIcon'; - -const FileIcon = (props: SvgIconProps) => { - return ( - - - - - - - - - - - - - - - - - ); -}; - -export default FileIcon; diff --git a/frontend/src/components/atoms/buttons/githubButton.tsx b/frontend/src/components/atoms/buttons/githubButton.tsx index 2cc2296286..121b75a408 100644 --- a/frontend/src/components/atoms/buttons/githubButton.tsx +++ b/frontend/src/components/atoms/buttons/githubButton.tsx @@ -1,20 +1,21 @@ -import { IconButton, Tooltip } from '@mui/material'; +import { IconButton, IconButtonProps, Tooltip } from '@mui/material'; import GithubIcon from 'assets/github'; -interface Props { +interface Props extends IconButtonProps { href?: string; } -export default function GithubButton({ href }: Props) { +export default function GithubButton({ href, ...props }: Props) { if (!href) { return null; } return ( - - + {/* @ts-expect-error href breaks IconButton props */} + + ); diff --git a/frontend/src/components/atoms/buttons/userButton/avatar.tsx b/frontend/src/components/atoms/buttons/userButton/avatar.tsx index 2de275f29a..00964c1483 100644 --- a/frontend/src/components/atoms/buttons/userButton/avatar.tsx +++ b/frontend/src/components/atoms/buttons/userButton/avatar.tsx @@ -1,40 +1,40 @@ import { useAuth } from 'api/auth'; -import { Avatar, Box } from '@mui/material'; +import SettingsIcon from '@mui/icons-material/Settings'; +import { Avatar, Button, ButtonProps, Typography } from '@mui/material'; -import UserIcon from 'assets/user'; - -export default function UserAvatar() { +export default function UserAvatar(props: ButtonProps) { const { user } = useAuth(); - if (user) { - return ( - - {user.identifier?.[0]?.toUpperCase()} - - ); - } else { - return ( - - - - - - ); - } + return ( + + ); } diff --git a/frontend/src/components/atoms/buttons/userButton/index.tsx b/frontend/src/components/atoms/buttons/userButton/index.tsx index 07b72bc4ff..bd266ee7e8 100644 --- a/frontend/src/components/atoms/buttons/userButton/index.tsx +++ b/frontend/src/components/atoms/buttons/userButton/index.tsx @@ -1,7 +1,5 @@ import { useState } from 'react'; -import { IconButton } from '@mui/material'; - import UserAvatar from './avatar'; import UserMenu from './menu'; @@ -16,18 +14,15 @@ export default function UserButton() { }; return ( -
- + - - + /> -
+ ); } diff --git a/frontend/src/components/atoms/element/sideView.tsx b/frontend/src/components/atoms/element/sideView.tsx deleted file mode 100644 index 9d58db60d9..0000000000 --- a/frontend/src/components/atoms/element/sideView.tsx +++ /dev/null @@ -1,25 +0,0 @@ -import { useRecoilState } from 'recoil'; - -import { sideViewState } from '@chainlit/react-client'; - -import { ElementSideView } from 'components/atoms/elements/ElementSideView'; - -interface SideViewProps { - children: React.ReactNode; -} - -const SideView = ({ children }: SideViewProps) => { - const [sideViewElement, setSideViewElement] = useRecoilState(sideViewState); - - return ( - setSideViewElement(undefined)} - isOpen={!!sideViewElement} - element={sideViewElement} - > - {children} - - ); -}; - -export default SideView; diff --git a/frontend/src/components/atoms/elements/Audio.tsx b/frontend/src/components/atoms/elements/Audio.tsx index 54e4f0a403..6f7e95d06f 100644 --- a/frontend/src/components/atoms/elements/Audio.tsx +++ b/frontend/src/components/atoms/elements/Audio.tsx @@ -1,5 +1,6 @@ import { grey } from 'theme/palette'; +import { Typography } from '@mui/material'; import Box from '@mui/material/Box'; import useTheme from '@mui/material/styles/useTheme'; @@ -14,7 +15,7 @@ const AudioElement = ({ element }: { element: IAudioElement }) => { return ( - { }} > {element.name} - - + + ); }; diff --git a/frontend/src/components/atoms/elements/ElementSideView.tsx b/frontend/src/components/atoms/elements/ElementSideView.tsx index 9fd07cad6d..51d4b90749 100644 --- a/frontend/src/components/atoms/elements/ElementSideView.tsx +++ b/frontend/src/components/atoms/elements/ElementSideView.tsx @@ -1,5 +1,6 @@ import { SyntheticEvent, forwardRef, useState } from 'react'; import { Resizable } from 'react-resizable'; +import { useRecoilState } from 'recoil'; import { useWindowSize } from 'usehooks-ts'; import Close from '@mui/icons-material/Close'; @@ -12,7 +13,7 @@ import Typography from '@mui/material/Typography'; import styled from '@mui/material/styles/styled'; import useMediaQuery from '@mui/material/useMediaQuery'; -import type { IMessageElement } from 'client-types/'; +import { sideViewState } from '@chainlit/react-client'; import 'react-resizable/css/styles.css'; @@ -20,24 +21,12 @@ import { Element } from './Element'; const DRAWER_DEFAULT_WIDTH = 400; -interface MainDrawerProps { - open?: boolean; - width?: number; -} - interface DrawerProps extends BoxProps { open?: boolean; width?: number | string; isSmallScreen?: boolean; } -interface SideViewProps { - children: React.ReactNode; - element?: IMessageElement; - isOpen: boolean; - onClose: () => void; -} - const Handle = forwardRef(function Handle( { ...props }: { handleAxis?: string }, ref @@ -73,14 +62,10 @@ const Handle = forwardRef(function Handle( ); }); -const ElementSideView = ({ - children, - element, - isOpen, - onClose -}: SideViewProps) => { +const ElementSideView = () => { const [resizeInProgress, setResizeInProgress] = useState(false); const [drawerWidth, setDrawerWidth] = useState(DRAWER_DEFAULT_WIDTH); + const [sideViewElement, setSideViewElement] = useRecoilState(sideViewState); const handleResize = ( _event: SyntheticEvent, @@ -97,73 +82,58 @@ const ElementSideView = ({ } return ( - <> - - {children} - - setResizeInProgress(true)} + onResizeStop={() => setResizeInProgress(false)} + resizeHandles={['w']} + handle={!isSmallScreen ? : <>} + axis="x" + minConstraints={[100, 0]} // Minimum width of 100px and no limit on height. + maxConstraints={[width / 2, 0]} // Constraint the maximum width to the half of the screen without limit on height. + > + setResizeInProgress(true)} - onResizeStop={() => setResizeInProgress(false)} - resizeHandles={['w']} - handle={!isSmallScreen ? : <>} - axis="x" - minConstraints={[100, 0]} // Minimum width of 100px and no limit on height. - maxConstraints={[width / 2, 0]} // Constraint the maximum width to the half of the screen without limit on height. > - - - - {element?.name} - - - - - - - + + {sideViewElement?.name} + + setSideViewElement(undefined)} > - - - - - + + + + + + + + + ); }; -const MainDrawer = styled(Box, { - shouldForwardProp: (prop) => prop !== 'open' -})(({ theme, open }) => ({ - width: '100%', - display: 'flex', - flexDirection: 'column', - boxSizing: 'border-box', - transition: theme.transitions.create('margin', { - easing: theme.transitions.easing[open ? 'easeOut' : 'sharp'], - duration: - theme.transitions.duration[open ? 'enteringScreen' : 'leavingScreen'] - }) -})); - const Drawer = styled(MDrawer, { shouldForwardProp: (prop) => prop !== 'isSmallScreen' })(({ theme, open, width, isSmallScreen }) => ({ diff --git a/frontend/src/components/atoms/elements/ElementView.tsx b/frontend/src/components/atoms/elements/ElementView.tsx index a20859e27c..aa7a845525 100644 --- a/frontend/src/components/atoms/elements/ElementView.tsx +++ b/frontend/src/components/atoms/elements/ElementView.tsx @@ -25,7 +25,7 @@ const ElementView = ({ element, onGoBack }: ElementViewProps) => { mx="auto" sx={{ width: '100%', - maxWidth: '60rem', + maxWidth: '48rem', color: 'text.primary' }} id="element-view" diff --git a/frontend/src/components/atoms/inputs/InputStateHandler.tsx b/frontend/src/components/atoms/inputs/InputStateHandler.tsx index cd8af0d70f..c30cfe3bdb 100644 --- a/frontend/src/components/atoms/inputs/InputStateHandler.tsx +++ b/frontend/src/components/atoms/inputs/InputStateHandler.tsx @@ -25,7 +25,7 @@ const InputStateHandler = (props: InputStateHandlerProps): JSX.Element => { } = props; return ( - + {label ? ( void; placeholder?: string; renderLabel?: () => string; + onItemMouseEnter?: (e: MouseEvent, itemName: string) => void; + onItemMouseLeave?: (e: MouseEvent) => void; value?: string | number; iconSx?: SxProps; }; @@ -41,6 +44,8 @@ const SelectInput = ({ label, name, onChange, + onItemMouseEnter, + onItemMouseLeave, size = 'small', tooltip, value, @@ -136,6 +141,9 @@ const SelectInput = ({ items?.map((item) => ( onItemMouseEnter?.(e, item.label)} + onMouseLeave={(e) => onItemMouseLeave?.(e)} item={item} selected={item.value === value} key={item.value} diff --git a/frontend/src/components/i18n/Translator.tsx b/frontend/src/components/i18n/Translator.tsx index fe2c191610..66b26d7358 100644 --- a/frontend/src/components/i18n/Translator.tsx +++ b/frontend/src/components/i18n/Translator.tsx @@ -8,17 +8,23 @@ type options = TOptions<$Dictionary>; type TranslatorProps = { path: string | string[]; + suffix?: string; options?: options; }; -const Translator = ({ path, options }: TranslatorProps) => { +const Translator = ({ path, options, suffix }: TranslatorProps) => { const { t, i18n } = usei18nextTranslation(); if (!i18n.exists(path, options)) { return ; } - return {t(path, options)}; + return ( + + {t(path, options)} + {suffix} + + ); }; export const useTranslation = () => { diff --git a/frontend/src/components/molecules/auth/AuthTemplate.tsx b/frontend/src/components/molecules/auth/AuthTemplate.tsx index 198d12a408..3b9f0de1ac 100644 --- a/frontend/src/components/molecules/auth/AuthTemplate.tsx +++ b/frontend/src/components/molecules/auth/AuthTemplate.tsx @@ -29,8 +29,8 @@ const AuthTemplate = ({ border: (theme) => `1px solid ${theme.palette.divider}`, borderRadius: 1, padding: theme.spacing(5, 5), - maxWidth: '400px', + width: '90%', height: 'auto', maxHeight: '90%', diff --git a/frontend/src/components/molecules/chatProfiles.tsx b/frontend/src/components/molecules/chatProfiles.tsx index 4500083b91..de729a2cee 100644 --- a/frontend/src/components/molecules/chatProfiles.tsx +++ b/frontend/src/components/molecules/chatProfiles.tsx @@ -2,9 +2,8 @@ import size from 'lodash/size'; import { useState } from 'react'; import { useNavigate } from 'react-router-dom'; import { useRecoilValue } from 'recoil'; -import { grey } from 'theme'; -import { Box, Popover, Tab, Tabs } from '@mui/material'; +import { Box, Popover } from '@mui/material'; import { useChatInteract, @@ -12,11 +11,9 @@ import { useChatSession } from '@chainlit/react-client'; -import { InputStateHandler } from 'components/atoms/inputs'; +import { SelectInput } from 'components/atoms/inputs'; import { Markdown } from 'components/molecules/Markdown'; -import { useIsDarkMode } from 'hooks/useIsDarkMode'; - import { projectSettingsState } from 'state/project'; import NewChatDialog from './newChatDialog'; @@ -30,7 +27,6 @@ export default function ChatProfiles() { const { clear } = useChatInteract(); const [newChatProfile, setNewChatProfile] = useState(null); const [openDialog, setOpenDialog] = useState(false); - const isDarkMode = useIsDarkMode(); const navigate = useNavigate(); const handleClose = () => { @@ -64,97 +60,25 @@ export default function ChatProfiles() { const popoverOpen = Boolean(anchorEl); - return ( - - - `1px solid ${theme.palette.divider}`, - backgroundColor: (theme) => theme.palette.background.paper, - borderRadius: 1, - padding: 0.5 - }} - > - { - setNewChatProfile(newValue); - if (firstInteraction) { - setOpenDialog(true); - } else { - handleConfirm(newValue); - } - }} - variant="scrollable" - sx={{ - minHeight: '40px !important', + const items = pSettings.chatProfiles.map((item) => ({ + label: item.name, + value: item.name, + icon: item.icon ? ( + + ) : undefined + })); - '& .MuiButtonBase-root': { - textTransform: 'none', - zIndex: 1, - color: grey[isDarkMode ? 600 : 500], - fontSize: '14px', - fontWeight: 500, - padding: 0, - minHeight: '40px !important', - width: '125px' - }, - '& .Mui-selected': { - color: 'white !important' - }, - '& .MuiTabs-indicator': { - background: (theme) => - isDarkMode - ? theme.palette.divider - : theme.palette.primary.main, - height: '100%', - borderRadius: '5px' - } - }} - > - {pSettings.chatProfiles.map((item) => ( - - ) : undefined - } - iconPosition="start" - onMouseEnter={(event) => { - setChatProfileDescription(item.markdown_description); - setAnchorEl(event.currentTarget.parentElement); - }} - onMouseLeave={() => setAnchorEl(null)} - data-test={`chat-profile:${item.name}`} - /> - ))} - - - + return ( + <> setAnchorEl(null)} disableRestoreFocus @@ -194,11 +118,35 @@ export default function ChatProfiles() { + { + const item = pSettings.chatProfiles.find( + (item) => item.name === itemName + ); + if (!item) return; + setChatProfileDescription(item.markdown_description); + setAnchorEl(event.currentTarget); + }} + onItemMouseLeave={() => setAnchorEl(null)} + onChange={(e) => { + const newValue = e.target.value; + setNewChatProfile(newValue); + if (firstInteraction) { + setOpenDialog(true); + } else { + handleConfirm(newValue); + } + setAnchorEl(null); + }} + /> handleConfirm()} /> - + ); } diff --git a/frontend/src/components/molecules/messages/Message.tsx b/frontend/src/components/molecules/messages/Message.tsx index b8bc3ce153..8501daf692 100644 --- a/frontend/src/components/molecules/messages/Message.tsx +++ b/frontend/src/components/molecules/messages/Message.tsx @@ -58,19 +58,12 @@ const Message = memo( return null; } - const isUser = message.type === 'user_message'; const isAsk = message.waitForAnswer; return ( - isUser - ? 'transparent' - : theme.palette.mode === 'dark' - ? theme.palette.grey[800] - : theme.palette.grey[100] + color: 'text.primary' }} className="step" > @@ -78,7 +71,7 @@ const Message = memo( sx={{ boxSizing: 'border-box', mx: 'auto', - maxWidth: '60rem', + maxWidth: '48rem', px: 2, display: 'flex', flexDirection: 'column', diff --git a/frontend/src/components/molecules/newChatButton.tsx b/frontend/src/components/molecules/newChatButton.tsx index f06f0a03a8..0b1450d20c 100644 --- a/frontend/src/components/molecules/newChatButton.tsx +++ b/frontend/src/components/molecules/newChatButton.tsx @@ -1,18 +1,17 @@ import { useState } from 'react'; import { useNavigate } from 'react-router-dom'; -import { Box } from '@mui/material'; +import { Box, IconButton, IconButtonProps, Tooltip } from '@mui/material'; import { useChatInteract } from '@chainlit/react-client'; -import { AccentButton } from 'components/atoms/buttons'; import { Translator } from 'components/i18n'; import SquarePenIcon from 'assets/squarePen'; import NewChatDialog from './newChatDialog'; -export default function NewChatButton() { +export default function NewChatButton(props: IconButtonProps) { const [open, setOpen] = useState(false); const navigate = useNavigate(); const { clear } = useChatInteract(); @@ -33,14 +32,13 @@ export default function NewChatButton() { return ( - } + } > - - + + + + - + {/* */} diff --git a/frontend/src/components/organisms/chat/Messages/welcomeScreen.tsx b/frontend/src/components/organisms/chat/Messages/welcomeScreen.tsx index 2cfb68b5cc..ced196d382 100644 --- a/frontend/src/components/organisms/chat/Messages/welcomeScreen.tsx +++ b/frontend/src/components/organisms/chat/Messages/welcomeScreen.tsx @@ -23,9 +23,9 @@ const WelcomeScreen = memo( - - - + + + diff --git a/frontend/src/components/organisms/chat/history/index.tsx b/frontend/src/components/organisms/chat/history/index.tsx index c1e3c4a4e5..f5a3b395e0 100644 --- a/frontend/src/components/organisms/chat/history/index.tsx +++ b/frontend/src/components/organisms/chat/history/index.tsx @@ -5,21 +5,12 @@ import { useRecoilState } from 'recoil'; import { grey } from 'theme'; import AutoDelete from '@mui/icons-material/AutoDelete'; -import { - IconButton, - Menu, - MenuItem, - Stack, - Tooltip, - Typography -} from '@mui/material'; +import { IconButton, Menu, MenuItem, Stack, Typography } from '@mui/material'; import { UserInput } from '@chainlit/react-client'; import { Translator } from 'components/i18n'; -import ChevronUpIcon from 'assets/chevronUp'; - import { inputHistoryState } from 'state/userInputHistory'; interface Props { @@ -65,7 +56,7 @@ function buildInputHistory(userInputs: UserInput[]) { return inputHistory; } -export default function InputHistoryButton({ disabled, onClick }: Props) { +export default function InputHistoryButton({ onClick }: Props) { const [inputHistory, setInputHistory] = useRecoilState(inputHistoryState); const ref = useRef(); @@ -216,7 +207,7 @@ export default function InputHistoryButton({ disabled, onClick }: Props) { sx: { p: 1, backgroundImage: 'none', - mt: -2, + mt: -4, ml: -1, overflow: 'visible', maxHeight: '314px', @@ -236,28 +227,5 @@ export default function InputHistoryButton({ disabled, onClick }: Props) { ) : null; - return ( -
- {menu} - - } - > - { - // In MUI, a warning is triggered if we pass a disabled button. To avoid this warning, we should wrap the button in a element when it can be disabled. - } - - toggleChatHistoryMenu(!inputHistory.open)} - ref={ref} - > - - - - -
- ); + return
{menu}
; } diff --git a/frontend/src/components/organisms/chat/index.tsx b/frontend/src/components/organisms/chat/index.tsx index 15848d8a8d..0e901f6da7 100644 --- a/frontend/src/components/organisms/chat/index.tsx +++ b/frontend/src/components/organisms/chat/index.tsx @@ -5,7 +5,7 @@ import { useRecoilValue, useSetRecoilState } from 'recoil'; import { toast } from 'sonner'; import { v4 as uuidv4 } from 'uuid'; -import { Alert, Box } from '@mui/material'; +import { Alert, Box, Stack } from '@mui/material'; import { threadHistoryState, @@ -14,13 +14,10 @@ import { useChatMessages, useChatSession } from '@chainlit/react-client'; -import { sideViewState } from '@chainlit/react-client'; import { ErrorBoundary } from 'components/atoms/ErrorBoundary'; -import SideView from 'components/atoms/element/sideView'; import { Translator } from 'components/i18n'; import { useTranslation } from 'components/i18n/Translator'; -import ChatProfiles from 'components/molecules/chatProfiles'; import { TaskList } from 'components/molecules/tasklist/TaskList'; import { apiClientState } from 'state/apiClient'; @@ -37,7 +34,6 @@ const Chat = () => { const projectSettings = useRecoilValue(projectSettingsState); const setAttachments = useSetRecoilState(attachmentsState); const setThreads = useSetRecoilState(threadHistoryState); - const sideViewElement = useRecoilValue(sideViewState); const apiClient = useRecoilValue(apiClientState); const [autoScroll, setAutoScroll] = useState(true); @@ -48,9 +44,13 @@ const Chat = () => { const fileSpec = useMemo( () => ({ - max_size_mb: projectSettings?.features?.multi_modal?.max_size_mb || 500, - max_files: projectSettings?.features?.multi_modal?.max_files || 20, - accept: projectSettings?.features?.multi_modal?.accept || ['*/*'] + max_size_mb: + projectSettings?.features?.spontaneous_file_upload?.max_size_mb || 500, + max_files: + projectSettings?.features?.spontaneous_file_upload?.max_files || 20, + accept: projectSettings?.features?.spontaneous_file_upload?.accept || [ + '*/*' + ] }), [projectSettings] ); @@ -173,7 +173,7 @@ const Chat = () => { }, []); const enableMultiModalUpload = - !disabled && projectSettings?.features?.multi_modal?.enabled; + !disabled && projectSettings?.features?.spontaneous_file_upload?.enabled; return ( { {upload?.isDragActive ? : null} ) : null} - + {error ? ( { { ) : null} - { projectSettings={projectSettings} /> - - {sideViewElement ? null : } + ); }; diff --git a/frontend/src/components/organisms/chat/inputBox/MicButton/RecordScreen.tsx b/frontend/src/components/organisms/chat/inputBox/MicButton/RecordScreen.tsx new file mode 100644 index 0000000000..f457c349bd --- /dev/null +++ b/frontend/src/components/organisms/chat/inputBox/MicButton/RecordScreen.tsx @@ -0,0 +1,76 @@ +import { grey } from 'theme/palette'; + +import KeyboardVoiceIcon from '@mui/icons-material/KeyboardVoice'; +import { Box } from '@mui/material'; +import Backdrop from '@mui/material/Backdrop'; + +interface Props { + open?: boolean; + isSpeaking?: boolean; +} + +export default function RecordScreen({ open, isSpeaking }: Props) { + return ( + theme.zIndex.drawer + 1 }} + open={!!open} + > + + + + + + + + + + + + + ); +} diff --git a/frontend/src/components/organisms/chat/inputBox/MicButton/index.tsx b/frontend/src/components/organisms/chat/inputBox/MicButton/index.tsx new file mode 100644 index 0000000000..b245d1dc36 --- /dev/null +++ b/frontend/src/components/organisms/chat/inputBox/MicButton/index.tsx @@ -0,0 +1,214 @@ +import { useCallback, useState } from 'react'; +import { useHotkeys } from 'react-hotkeys-hook'; +import { useRecoilState, useRecoilValue } from 'recoil'; +import { toast } from 'sonner'; + +import KeyboardVoiceIcon from '@mui/icons-material/KeyboardVoice'; +import { IconButton, Theme, Tooltip, useMediaQuery } from '@mui/material'; + +import { Translator } from 'components/i18n'; + +import { attachmentsState } from 'state/chat'; +import { projectSettingsState } from 'state/project'; + +import { askUserState, useChatInteract } from 'client-types/*'; + +import RecordScreen from './RecordScreen'; + +interface Props { + disabled?: boolean; +} + +const MicButton = ({ disabled }: Props) => { + const askUser = useRecoilValue(askUserState); + const [isRecording, setIsRecording] = useState(false); + const [timer, setTimer] = useState(undefined); + const [isSpeaking, setIsSpeaking] = useState(false); + const { sendAudioChunk, endAudioStream } = useChatInteract(); + const pSettings = useRecoilValue(projectSettingsState); + const [attachments, setAttachments] = useRecoilState(attachmentsState); + + disabled = disabled || !!askUser; + + const startRecording = useCallback(() => { + if (isRecording || disabled) { + return; + } + clearTimeout(timer); + + if (!pSettings) { + return; + } + + const { + min_decibels, + silence_timeout, + initial_silence_timeout, + sample_rate, + chunk_duration, + max_duration + } = pSettings.features.audio; + + navigator.mediaDevices + .getUserMedia({ audio: { sampleRate: sample_rate } }) + .then((stream) => { + let spokeAtLeastOnce = false; + let isSpeaking = false; + let isFirstChunk = true; + let audioBuffer: Blob | null = null; + let startTime = Date.now(); + + const mediaRecorder = new MediaRecorder(stream); + + mediaRecorder.addEventListener('start', () => { + setIsRecording(true); + startTime = Date.now(); + }); + + mediaRecorder.addEventListener('dataavailable', async (event) => { + if (!spokeAtLeastOnce) { + if (!audioBuffer) { + audioBuffer = new Blob([event.data], { type: event.data.type }); + } else { + audioBuffer = new Blob([audioBuffer, event.data], { + type: event.data.type + }); + } + } + if (mediaRecorder.state === 'inactive') { + return; + } + const elapsedTime = Date.now() - startTime; + if (elapsedTime >= max_duration) { + mediaRecorder.stop(); + stream.getTracks().forEach((track) => track.stop()); + return; + } + + setIsSpeaking(isSpeaking); + const [mimeType, _] = mediaRecorder.mimeType.split(';'); + + if (audioBuffer) { + // If there is buffered data and the user has spoken, send the buffered data first + await sendAudioChunk( + isFirstChunk, + mimeType, + elapsedTime, + new Blob([audioBuffer, event.data]) + ); + audioBuffer = null; // Clear the buffer + } else { + await sendAudioChunk( + isFirstChunk, + mimeType, + elapsedTime, + event.data + ); + } + + if (isFirstChunk) { + isFirstChunk = false; + } + }); + + mediaRecorder.addEventListener('stop', async () => { + setIsRecording(false); + setIsSpeaking(false); + if (spokeAtLeastOnce) { + const fileReferences = attachments + ?.filter((a) => !!a.serverId) + .map((a) => ({ id: a.serverId! })); + await endAudioStream(fileReferences); + setAttachments([]); + } + }); + + const audioContext = new AudioContext(); + const audioStreamSource = audioContext.createMediaStreamSource(stream); + const analyser = audioContext.createAnalyser(); + analyser.minDecibels = min_decibels; + audioStreamSource.connect(analyser); + + const bufferLength = analyser.frequencyBinCount; + + const domainData = new Uint8Array(bufferLength); + + mediaRecorder.start(chunk_duration); + + const detectSound = () => { + if (mediaRecorder.state === 'inactive') { + return; + } + analyser.getByteFrequencyData(domainData); + const soundDetected = domainData.some((value) => value > 0); + + if (!isSpeaking) { + isSpeaking = soundDetected; + } + if (!spokeAtLeastOnce && soundDetected) { + setIsSpeaking(isSpeaking); + spokeAtLeastOnce = true; + } + requestAnimationFrame(detectSound); + }; + requestAnimationFrame(detectSound); + + setTimeout(() => { + if (!spokeAtLeastOnce) { + mediaRecorder.stop(); + stream.getTracks().forEach((track) => track.stop()); + } else { + setTimer( + setInterval(() => { + if (!isSpeaking) { + mediaRecorder.stop(); + stream.getTracks().forEach((track) => track.stop()); + } else { + isSpeaking = false; + } + }, silence_timeout) + ); + } + }, initial_silence_timeout); + }) + .catch((err) => { + toast.error('Failed to start recording: ' + err.message); + }); + }, [pSettings, timer, isRecording, disabled, attachments]); + + useHotkeys('p', startRecording); + + const size = useMediaQuery((theme) => theme.breakpoints.down('sm')) + ? 'small' + : 'medium'; + + if (!pSettings?.features.audio.enabled) { + return null; + } + + return ( + <> + + + } + > + + + + + + + + ); +}; +export default MicButton; diff --git a/frontend/src/components/organisms/chat/inputBox/UploadButton.tsx b/frontend/src/components/organisms/chat/inputBox/UploadButton.tsx index e59f83e389..c1b40d6425 100644 --- a/frontend/src/components/organisms/chat/inputBox/UploadButton.tsx +++ b/frontend/src/components/organisms/chat/inputBox/UploadButton.tsx @@ -36,7 +36,8 @@ const UploadButton = ({ ? 'small' : 'medium'; - if (!upload || !pSettings?.features?.multi_modal?.enabled) return null; + if (!upload || !pSettings?.features?.spontaneous_file_upload?.enabled) + return null; const { getRootProps, getInputProps } = upload; return ( diff --git a/frontend/src/components/organisms/chat/inputBox/index.tsx b/frontend/src/components/organisms/chat/inputBox/index.tsx index a73cf8029a..8b06292adf 100644 --- a/frontend/src/components/organisms/chat/inputBox/index.tsx +++ b/frontend/src/components/organisms/chat/inputBox/index.tsx @@ -105,7 +105,7 @@ const InputBox = memo( sx={{ boxSizing: 'border-box', width: '100%', - maxWidth: '60rem', + maxWidth: '48rem', m: 'auto', justifyContent: 'center' }} diff --git a/frontend/src/components/organisms/chat/inputBox/input.tsx b/frontend/src/components/organisms/chat/inputBox/input.tsx index 90fe215f5d..89b39c33a4 100644 --- a/frontend/src/components/organisms/chat/inputBox/input.tsx +++ b/frontend/src/components/organisms/chat/inputBox/input.tsx @@ -13,12 +13,12 @@ import { Attachments } from 'components/molecules/attachments'; import HistoryButton from 'components/organisms/chat/history'; import { IAttachment, attachmentsState } from 'state/chat'; -import { chatSettingsOpenState, projectSettingsState } from 'state/project'; +import { chatSettingsOpenState } from 'state/project'; import { inputHistoryState } from 'state/userInputHistory'; +import MicButton from './MicButton'; import { SubmitButton } from './SubmitButton'; import UploadButton from './UploadButton'; -import SpeechButton from './speechButton'; interface Props { fileSpec: FileSpec; @@ -40,7 +40,6 @@ function getLineCount(el: HTMLDivElement) { const Input = memo( ({ fileSpec, onFileUpload, onFileUploadError, onSubmit, onReply }: Props) => { const [attachments, setAttachments] = useRecoilState(attachmentsState); - const [pSettings] = useRecoilState(projectSettingsState); const setInputHistory = useSetRecoilState(inputHistoryState); const setChatSettingsOpen = useSetRecoilState(chatSettingsOpenState); @@ -57,8 +56,6 @@ const Input = memo( const [value, setValue] = useState(''); const [isComposing, setIsComposing] = useState(false); - const showTextToSpeech = pSettings?.features.speech_to_text?.enabled; - const { t } = useTranslation(); useEffect(() => { @@ -167,13 +164,7 @@ const Input = memo( )} - {showTextToSpeech ? ( - setValue((text) => text + transcript)} - language={pSettings.features?.speech_to_text?.language} - disabled={disabled} - /> - ) : null} + ); diff --git a/frontend/src/components/organisms/chat/inputBox/speechButton.tsx b/frontend/src/components/organisms/chat/inputBox/speechButton.tsx deleted file mode 100644 index d72a38e6ea..0000000000 --- a/frontend/src/components/organisms/chat/inputBox/speechButton.tsx +++ /dev/null @@ -1,99 +0,0 @@ -import { useEffect, useState } from 'react'; -import SpeechRecognition, { - useSpeechRecognition -} from 'react-speech-recognition'; - -import KeyboardVoiceIcon from '@mui/icons-material/KeyboardVoice'; -import StopCircleIcon from '@mui/icons-material/StopCircle'; -import { IconButton, Theme, Tooltip, useMediaQuery } from '@mui/material'; - -import { Translator } from 'components/i18n'; - -interface Props { - onSpeech: (text: string) => void; - language?: string; - disabled?: boolean; -} - -const SpeechButton = ({ onSpeech, language, disabled }: Props) => { - const { transcript, browserSupportsSpeechRecognition } = - useSpeechRecognition(); - const [isRecording, setIsRecording] = useState(false); - const [lastTranscript, setLastTranscript] = useState(''); - const [timer, setTimer] = useState(null); - - useEffect(() => { - if (lastTranscript.length < transcript.length) { - onSpeech(transcript.slice(lastTranscript.length)); - } - setLastTranscript(transcript); - }, [transcript]); - - useEffect(() => { - if (isRecording) { - if (timer) { - clearTimeout(timer); - } - setTimer( - setTimeout(() => { - setIsRecording(false); - SpeechRecognition.stopListening(); - }, 2000) // stop after 3 seconds of silence - ); - } - }, [transcript, isRecording]); - - const size = useMediaQuery((theme) => theme.breakpoints.down('sm')) - ? 'small' - : 'medium'; - - if (!browserSupportsSpeechRecognition) { - return null; - } - - return isRecording ? ( - - } - > - - { - setIsRecording(false); - SpeechRecognition.stopListening(); - }} - > - - - - - ) : ( - - } - > - - { - setIsRecording(true); - SpeechRecognition.startListening({ - continuous: true, - language: language - }); - }} - > - - - - - ); -}; -export default SpeechButton; diff --git a/frontend/src/components/organisms/header.tsx b/frontend/src/components/organisms/header.tsx index 2fc2007d90..4b34666a42 100644 --- a/frontend/src/components/organisms/header.tsx +++ b/frontend/src/components/organisms/header.tsx @@ -1,188 +1,46 @@ -import { useAuth } from 'api/auth'; -import React, { memo, useRef, useState } from 'react'; -import { Link, useLocation } from 'react-router-dom'; +import { memo } from 'react'; +import { useRecoilValue } from 'recoil'; -import MenuIcon from '@mui/icons-material/Menu'; -import { - AppBar, - Box, - Button, - IconButton, - Menu, - Stack, - Toolbar -} from '@mui/material'; +import { Box, Stack } from '@mui/material'; import useMediaQuery from '@mui/material/useMediaQuery'; -import { RegularButton } from 'components/atoms/buttons'; import GithubButton from 'components/atoms/buttons/githubButton'; -import UserButton from 'components/atoms/buttons/userButton'; -import { Logo } from 'components/atoms/logo'; -import { Translator } from 'components/i18n'; +import ChatProfiles from 'components/molecules/chatProfiles'; import NewChatButton from 'components/molecules/newChatButton'; -import { IProjectSettings } from 'state/project'; +import { settingsState } from 'state/settings'; -import { OpenThreadListButton } from './threadHistory/sidebar/OpenThreadListButton'; +import { OpenSideBarMobileButton } from './sidebar/OpenSideBarMobileButton'; -interface INavItem { - to: string; - label: React.ReactElement | string; -} +const Header = memo(() => { + const isMobile = useMediaQuery('(max-width: 66rem)'); + const { isChatHistoryOpen } = useRecoilValue(settingsState); -function ActiveNavItem({ to, label }: INavItem) { return ( - - {label} - - ); -} - -function NavItem({ to, label }: INavItem) { - return ( - - ); -} - -interface NavProps { - dataPersistence?: boolean; - hasReadme?: boolean; - matches?: boolean; -} - -const Nav = ({ dataPersistence, hasReadme, matches }: NavProps) => { - const location = useLocation(); - const { isAuthenticated } = useAuth(); - const [open, setOpen] = useState(false); - const ref = useRef(); - - let anchorEl; - - if (open && ref.current) { - anchorEl = ref.current; - } - - const tabs = [ - { to: '/', label: } - ]; - - if (hasReadme) { - tabs.push({ - to: '/readme', - label: - }); - } - - const nav = ( - - {tabs.map((t) => { - const active = location.pathname === t.to; - return ( -
- {active ? : } -
- ); - })} - + {isMobile ? : null} + {!isMobile && !isChatHistoryOpen ? : null} + + {isMobile ? ( + + + + + ) : null} + {!isMobile ? : null} + ); - - if (matches) { - return ( - <> - setOpen(true)} - > - - - {isAuthenticated && dataPersistence ? : null} - setOpen(false)} - PaperProps={{ - sx: { - p: 1, - backgroundImage: 'none', - mt: -2, - ml: -1, - overflow: 'visible', - overflowY: 'auto', - border: (theme) => `1px solid ${theme.palette.divider}`, - boxShadow: (theme) => - theme.palette.mode === 'light' - ? '0px 2px 4px 0px #0000000D' - : '0px 10px 10px 0px #0000000D' - } - }} - anchorOrigin={{ vertical: 'top', horizontal: 'left' }} - transformOrigin={{ vertical: 'bottom', horizontal: 'left' }} - > - {nav} - - - ); - } else { - return nav; - } -}; - -const Header = memo( - ({ projectSettings }: { projectSettings?: IProjectSettings }) => { - const matches = useMediaQuery('(max-width: 66rem)'); - - return ( - - `0 ${theme.spacing(2)} !important`, - minHeight: '60px !important', - borderBottomWidth: '1px', - borderBottomStyle: 'solid', - background: (theme) => theme.palette.background.paper, - borderBottomColor: (theme) => theme.palette.divider - }} - > - - {!matches ? : null} -