Wd/audio (#962)

* rework navigation

* wip

* fix buffering

* finalizing audio feature

* fix lint

* update changelog

CHANGELOG.md

@@ -8,6 +8,27 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 Nothing unreleased!
 
+## [1.1.0rc0] - 2024-05-06
+
+### Added
+
+- `cl.on_audio_chunk` decorator to process incoming the user incoming audio stream
+- `cl.on_audio_end` decorator to react to the end of the user audio stream
+- The `cl.Audio` element now has an `auto_play` property
+- `http_referer` is now available in `cl.user_session`
+
+### Changed
+
+- The UI has been revamped, especially the navigation
+- The arrow up button has been removed from the input bar, however pressing the arrow up key still opens the last inputs menu
+- **[breaking]** the `send()` method on `cl.Message` now returns the message instead of the message id
+- **[breaking]** The `multi_modal` feature has been renamed `spontaneous_file_upload` in the config
+- Element display property now defaults to `inline` instead of `side`
+
+### Fixed
+
+- Stopping a task should now work better (using asyncio task.cancel)
+
 ## [1.0.506] - 2024-04-30
 
 ### Added

backend/chainlit/__init__.py

@@ -52,7 +52,7 @@
 from chainlit.step import Step, step
 from chainlit.sync import make_async, run_sync
 from chainlit.telemetry import trace
-from chainlit.types import ChatProfile, ThreadDict
+from chainlit.types import AudioChunk, ChatProfile, ThreadDict
 from chainlit.user import PersistedUser, User
 from chainlit.user_session import user_session
 from chainlit.utils import make_module_getattr, wrap_user_function
@@ -224,6 +224,38 @@ def on_chat_end(func: Callable) -> Callable:
  return func
 
 
+@trace
+def on_audio_chunk(func: Callable) -> Callable:
+ """
+ Hook to react to the audio chunks being sent.
+
+ Args:
+ chunk (AudioChunk): The audio chunk being sent.
+
+ Returns:
+ Callable[], Any]: The decorated hook.
+ """
+
+ config.code.on_audio_chunk = wrap_user_function(func, with_task=False)
+ return func
+
+
+@trace
+def on_audio_end(func: Callable) -> Callable:
+ """
+ Hook to react to the audio stream ending. This is called after the last audio chunk is sent.
+
+ Args:
+ elements ([List[Element]): The files that were uploaded before starting the audio stream (if any).
+
+ Returns:
+ Callable[], Any]: The decorated hook.
+ """
+
+ config.code.on_audio_end = wrap_user_function(func, with_task=True)
+ return func
+
+
 @trace
 def author_rename(func: Callable[[str], str]) -> Callable[[str], str]:
  """
@@ -318,6 +350,7 @@ def acall(self):
 __all__ = [
  "user_session",
  "CopilotFunction",
+ "AudioChunk",
  "Action",
  "User",
  "PersistedUser",

backend/chainlit/config.py

@@ -16,7 +16,9 @@
 
 if TYPE_CHECKING:
  from chainlit.action import Action
- from chainlit.types import ChatProfile, ThreadDict
+ from chainlit.element import ElementBased
+ from chainlit.message import Message
+ from chainlit.types import AudioChunk, ChatProfile, ThreadDict
  from chainlit.user import User
  from fastapi import Request, Response
 
@@ -71,18 +73,26 @@
 # Automatically tag threads with the current chat profile (if a chat profile is used)
 auto_tag_thread = true
 
-# Authorize users to upload files with messages
-[features.multi_modal]
+# Authorize users to spontaneously upload files with messages
+[features.spontaneous_file_upload]
  enabled = true
  accept = ["*/*"]
  max_files = 20
  max_size_mb = 500
 
-# Allows user to use speech to text
-[features.speech_to_text]
- enabled = false
- # See all languages here https://github.com/JamesBrill/react-speech-recognition/blob/HEAD/docs/API.md#language-string
- # language = "en-US"
+[features.audio]
+ # Threshold for audio recording
+ min_decibels = -45
+ # Delay for the user to start speaking in MS
+ initial_silence_timeout = 3000
+ # Delay for the user to continue speaking in MS. If the user stops speaking for this duration, the recording will stop.
+ silence_timeout = 1500
+ # Above this duration (MS), the recording will forcefully stop.
+ max_duration = 15000
+ # Duration of the audio chunks in MS
+ chunk_duration = 1000
+ # Sample rate of the audio
+ sample_rate = 44100
 
 [UI]
 # Name of the app and chatbot.
@@ -189,26 +199,31 @@ class Theme(DataClassJsonMixin):
 
 
 @dataclass
-class SpeechToTextFeature:
- enabled: Optional[bool] = None
- language: Optional[str] = None
-
-
-@dataclass
-class MultiModalFeature:
+class SpontaneousFileUploadFeature(DataClassJsonMixin):
  enabled: Optional[bool] = None
  accept: Optional[Union[List[str], Dict[str, List[str]]]] = None
  max_files: Optional[int] = None
  max_size_mb: Optional[int] = None
 
 
+@dataclass
+class AudioFeature(DataClassJsonMixin):
+ min_decibels: int = -45
+ initial_silence_timeout: int = 2000
+ silence_timeout: int = 1500
+ chunk_duration: int = 1000
+ max_duration: int = 15000
+ sample_rate: int = 44100
+ enabled: bool = False
+
+
 @dataclass()
 class FeaturesSettings(DataClassJsonMixin):
  prompt_playground: bool = True
- multi_modal: Optional[MultiModalFeature] = None
+ spontaneous_file_upload: Optional[SpontaneousFileUploadFeature] = None
+ audio: Optional[AudioFeature] = Field(default_factory=AudioFeature)
  latex: bool = False
  unsafe_allow_html: bool = False
- speech_to_text: Optional[SpeechToTextFeature] = None
  auto_tag_thread: bool = True
 
 
@@ -247,7 +262,10 @@ class CodeSettings:
  on_chat_start: Optional[Callable[[], Any]] = None
  on_chat_end: Optional[Callable[[], Any]] = None
  on_chat_resume: Optional[Callable[["ThreadDict"], Any]] = None
- on_message: Optional[Callable[[str], Any]] = None
+ on_message: Optional[Callable[["Message"], Any]] = None
+ on_audio_chunk: Optional[Callable[["AudioChunk"], Any]] = None
+ on_audio_end: Optional[Callable[[List["ElementBased"]], Any]] = None
+
  author_rename: Optional[Callable[[str], str]] = None
  on_settings_update: Optional[Callable[[Dict[str, Any]], Any]] = None
  set_chat_profiles: Optional[Callable[[Optional["User"]], List["ChatProfile"]]] = (
@@ -413,11 +431,13 @@ def load_settings():
 
  ui_settings = UISettings(**ui_settings)
 
+ code_settings = CodeSettings(action_callbacks={})
+
  return {
  "features": features_settings,
  "ui": ui_settings,
  "project": project_settings,
- "code": CodeSettings(action_callbacks={}),
+ "code": code_settings,
  }
 
 

backend/chainlit/data/__init__.py

@@ -156,6 +156,7 @@ def attachment_to_element_dict(self, attachment: Attachment) -> "ElementDict":
  "chainlitKey": None,
  "display": metadata.get("display", "side"),
  "language": metadata.get("language"),
+ "autoPlay": metadata.get("autoPlay", None),
  "page": metadata.get("page"),
  "size": metadata.get("size"),
  "type": metadata.get("type", "file"),
@@ -219,7 +220,7 @@ def step_to_step_dict(self, step: LiteralStep) -> "StepDict":
  "disableFeedback": metadata.get("disableFeedback", False),
  "indent": metadata.get("indent"),
  "language": metadata.get("language"),
- "isError": metadata.get("isError", False),
+ "isError": bool(step.error),
  "waitForAnswer": metadata.get("waitForAnswer", False),
  }
 
@@ -348,7 +349,6 @@ async def create_step(self, step_dict: "StepDict"):
  step_dict.get("metadata", {}),
  **{
  "disableFeedback": step_dict.get("disableFeedback"),
- "isError": step_dict.get("isError"),
  "waitForAnswer": step_dict.get("waitForAnswer"),
  "language": step_dict.get("language"),
  "showInput": step_dict.get("showInput"),
@@ -372,6 +372,8 @@ async def create_step(self, step_dict: "StepDict"):
  step["input"] = {"content": step_dict.get("input")}
  if step_dict.get("output"):
  step["output"] = {"content": step_dict.get("output")}
+ if step_dict.get("isError"):
+ step["error"] = step_dict.get("output")
 
  await self.client.api.send_steps([step])
 

backend/chainlit/data/sql_alchemy.py

@@ -170,12 +170,14 @@ async def update_thread(
  raise ValueError("User not found in session context")
  data = {
  "id": thread_id,
- "createdAt": await self.get_current_timestamp()
- if metadata is None
- else None,
- "name": name
- if name is not None
- else (metadata.get("name") if metadata and "name" in metadata else None),
+ "createdAt": (
+ await self.get_current_timestamp() if metadata is None else None
+ ),
+ "name": (
+ name
+ if name is not None
+ else (metadata.get("name") if metadata and "name" in metadata else None)
+ ),
  "userId": user_id,
  "userIdentifier": user_identifier,
  "tags": tags,
@@ -552,13 +554,17 @@ async def get_all_user_threads(
  streaming=step_feedback.get("step_streaming", False),
  waitForAnswer=step_feedback.get("step_waitforanswer"),
  isError=step_feedback.get("step_iserror"),
- metadata=step_feedback["step_metadata"]
- if step_feedback.get("step_metadata") is not None
- else {},
+ metadata=(
+ step_feedback["step_metadata"]
+ if step_feedback.get("step_metadata") is not None
+ else {}
+ ),
  tags=step_feedback.get("step_tags"),
- input=step_feedback.get("step_input", "")
- if step_feedback["step_showinput"]
- else "",
+ input=(
+ step_feedback.get("step_input", "")
+ if step_feedback["step_showinput"]
+ else ""
+ ),
  output=step_feedback.get("step_output", ""),
  createdAt=step_feedback.get("step_createdat"),
  start=step_feedback.get("step_start"),
@@ -587,6 +593,7 @@ async def get_all_user_threads(
  display=element["element_display"],
  size=element.get("element_size"),
  language=element.get("element_language"),
+ autoPlay=element.get("element_autoPlay"),
  page=element.get("element_page"),
  forId=element.get("element_forid"),
  mime=element.get("element_mime"),

backend/chainlit/element.py

@@ -38,6 +38,7 @@ class ElementDict(TypedDict):
  size: Optional[ElementSize]
  language: Optional[str]
  page: Optional[int]
+ autoPlay: Optional[bool]
  forId: Optional[str]
  mime: Optional[str]
 
@@ -61,7 +62,7 @@ class Element:
  # The byte content of the element.
  content: Optional[Union[bytes, str]] = None
  # Controls how the image element should be displayed in the UI. Choices are “side” (default), “inline”, or “page”.
- display: ElementDisplay = Field(default="side")
+ display: ElementDisplay = Field(default="inline")
  # Controls element size
  size: Optional[ElementSize] = None
  # The ID of the message this element is associated with.
@@ -93,6 +94,7 @@ def to_dict(self) -> ElementDict:
  "objectKey": getattr(self, "object_key", None),
  "size": getattr(self, "size", None),
  "page": getattr(self, "page", None),
+ "autoPlay": getattr(self, "auto_play", None),
  "language": getattr(self, "language", None),
  "forId": getattr(self, "for_id", None),
  "mime": getattr(self, "mime", None),
@@ -306,6 +308,7 @@ async def preprocess_content(self):
 @dataclass
 class Audio(Element):
  type: ClassVar[ElementType] = "audio"
+ auto_play: bool = False
 
 
 @dataclass

backend/chainlit/llama_index/callbacks.py

@@ -70,7 +70,7 @@ def on_event_start(
  ) -> str:
  """Run when an event starts and return id of event."""
  self._restore_context()
- 
+
  step_type: StepType = "undefined"
  if event_type == CBEventType.RETRIEVE:
  step_type = "retrieval"
@@ -104,7 +104,6 @@ def on_event_end(
  """Run when an event ends."""
  step = self.steps.get(event_id, None)
 
-
  if payload is None or step is None:
  return
 
@@ -117,11 +116,13 @@ def on_event_end(
  source_nodes = getattr(response, "source_nodes", None)
  if source_nodes:
  source_refs = ", ".join(
- [f"Source {idx}" for idx, _ in enumerate(source_nodes)])
+ [f"Source {idx}" for idx, _ in enumerate(source_nodes)]
+ )
  step.elements = [
  Text(
  name=f"Source {idx}",
  content=source.text or "Empty node",
+ display="side",
  )
  for idx, source in enumerate(source_nodes)
  ]
@@ -137,6 +138,7 @@ def on_event_end(
  step.elements = [
  Text(
  name=f"Source {idx}",
+ display="side",
  content=source.node.get_text() or "Empty node",
  )
  for idx, source in enumerate(sources)
@@ -173,7 +175,7 @@ def on_event_end(
  token_count = self.total_llm_token_count or None
  raw_response = response.raw if response else None
  model = raw_response.get("model", None) if raw_response else None
- 
+
  if messages and isinstance(response, ChatResponse):
  msg: ChatMessage = response.message
  step.generation = ChatGeneration(
@@ -198,12 +200,11 @@ def on_event_end(
  else:
  step.output = payload
  self.context.loop.create_task(step.update())
- 
+
  self.steps.pop(event_id, None)
 
  def _noop(self, *args, **kwargs):
  pass
 
  start_trace = _noop
  end_trace = _noop
-