Samples cleanup (#43)

* Cleaned up example * Finished with first sample * Clean up microphone * Finish some more demos * Finished clean up for now * Minor changes to stream * Small stuff
Macoron · Jul 25, 2023 · 36526a3 · 36526a3
1 parent 6fc8727
commit 36526a3
Show file tree

Hide file tree

Showing 10 changed files with 944 additions and 1,097 deletions.
diff --git a/Assets/Samples/1 - Audio Clip/1 - Audio Clip.unity b/Assets/Samples/1 - Audio Clip/1 - Audio Clip.unity
diff --git a/Assets/Samples/1 - Audio Clip/AudioClipDemo.cs b/Assets/Samples/1 - Audio Clip/AudioClipDemo.cs
@@ -1,153 +1,97 @@
-using System;
-using System.Collections.Generic;
 using System.Diagnostics;
-using System.Linq;
-using System.Text;
 using UnityEngine;
 using UnityEngine.UI;
 using Whisper.Utils;
 
-// ReSharper disable ArrangeObjectCreationWhenTypeEvident - for Unity 2019/2020 support:
-
 namespace Whisper.Samples
 {
+ /// <summary>
+ /// Takes audio clip and make a transcription.
+ /// </summary>
  public class AudioClipDemo : MonoBehaviour
  {
- [Serializable]
- public class InitialPrompt
- {
- public string name;
- public string prompt;
- }
-
  public WhisperManager manager;
  public AudioClip clip;
- public bool echoSound = true;
-
- public List<InitialPrompt> initialPrompts = new List<InitialPrompt>
- {
- new InitialPrompt
- {
- name = "lowercase",
- prompt = "hello how is it going always use lowercase no punctuation goodbye one two three start stop i you me they",
- },
- new InitialPrompt
- {
- name = "Start of the clip",
- prompt = "And so my fellow Americans, ask not what your country can do for you",
- },
- new InitialPrompt
- {
- name = "UPPERCASE",
- prompt = "HELLO HOW IS IT GOING ALWAYS USE UPPERCASE NO PUNCTUATION GOODBYE ONE TWO THREE START STOP I YOU ME THEY",
- },
- new InitialPrompt
- {
- name = "Custom",
- prompt = "",
- },
- };
-
- [Header("Text Output")]
  public bool streamSegments = true;
+ public bool echoSound = true;
  public bool printLanguage = true;
- public bool showTimestamps;
 
  [Header("UI")]
  public Button button;
  public Text outputText;
  public Text timeText;
- public Dropdown initialPromptDropdown;
- public InputField selectedInitialPromptInput;
  public ScrollRect scroll;
-
+ public Dropdown languageDropdown;
+ public Toggle translateToggle;
+
  private string _buffer;
-
+ 
  private void Awake()
  {
- button.onClick.AddListener(ButtonPressed);
- if (streamSegments)
- manager.OnNewSegment += OnNewSegmentHandler;
+ manager.OnNewSegment += OnNewSegment;
  manager.OnProgress += OnProgressHandler;
 
- initialPromptDropdown.options = initialPrompts
- .Select(x => new Dropdown.OptionData(x.name))
- .ToList();
- initialPromptDropdown.onValueChanged.AddListener(OnInitialPromptChanged);
- initialPromptDropdown.value = 0;
- OnInitialPromptChanged(initialPromptDropdown.value);
- }
+ button.onClick.AddListener(ButtonPressed);
+ languageDropdown.value = languageDropdown.options
+ .FindIndex(op => op.text == manager.language);
+ languageDropdown.onValueChanged.AddListener(OnLanguageChanged);
 
- private void OnDestroy()
- {
- if (streamSegments)
- manager.OnNewSegment -= OnNewSegmentHandler;
+ translateToggle.isOn = manager.translateToEnglish;
+ translateToggle.onValueChanged.AddListener(OnTranslateChanged);
  }
 
- private void OnInitialPromptChanged(int ind) => selectedInitialPromptInput.text = initialPrompts[ind].prompt;
-
  public async void ButtonPressed()
  {
  _buffer = "";
  if (echoSound)
  AudioSource.PlayClipAtPoint(clip, Vector3.zero);
 
- // set initial prompt in manager
- manager.initialPrompt = selectedInitialPromptInput.text;
-
  var sw = new Stopwatch();
  sw.Start();
 
  var res = await manager.GetTextAsync(clip);
- if (res == null) 
+ if (res == null || !outputText) 
  return;
 
  var time = sw.ElapsedMilliseconds;
  var rate = clip.length / (time * 0.001f);
  timeText.text = $"Time: {time} ms\nRate: {rate:F1}x";
 
- var text = GetFinalText(res);
+ var text = res.Result;
  if (printLanguage)
  text += $"\n\nLanguage: {res.Language}";
 
  outputText.text = text;
+ UiUtils.ScrollDown(scroll);
  }
 
- private void OnNewSegmentHandler(WhisperSegment segment)
+ private void OnLanguageChanged(int ind)
  {
- if (!showTimestamps)
- {
- _buffer += segment.Text;
- outputText.text = _buffer + "...";
- }
- else
- {
- _buffer += $"<b>{segment.TimestampToString()}</b>{segment.Text}\n";
- outputText.text = _buffer;
- }
-
- UiUtils.ScrollDown(scroll);
+ var opt = languageDropdown.options[ind];
+ manager.language = opt.text;
  }
-
- private string GetFinalText(WhisperResult output)
+ 
+ private void OnTranslateChanged(bool translate)
  {
- if (!showTimestamps)
- return output.Result;
-
- var sb = new StringBuilder();
- foreach (var seg in output.Segments)
- {
- var segment = $"<b>{seg.TimestampToString()}</b>{seg.Text}\n";
- sb.Append(segment);
- }
-
- return sb.ToString();
+ manager.translateToEnglish = translate;
  }
- 
+
  private void OnProgressHandler(int progress)
  {
+ if (!timeText)
+ return;
  timeText.text = $"Progress: {progress}%";
  }
+
+ private void OnNewSegment(WhisperSegment segment)
+ {
+ if (!streamSegments || !outputText)
+ return;
+
+ _buffer += segment.Text;
+ outputText.text = _buffer + "...";
+ UiUtils.ScrollDown(scroll);
+ }
  }
 }
 

diff --git a/Assets/Samples/2 - Microphone/2 - Microphone.unity b/Assets/Samples/2 - Microphone/2 - Microphone.unity
@@ -1013,14 +1013,20 @@ MonoBehaviour:
  m_Name: 
  m_EditorClassIdentifier: 
  modelPath: Whisper/ggml-tiny.bin
+ isModelPathInStreamingAssets: 1
  initOnAwake: 1
  language: auto
  translateToEnglish: 0
  strategy: 0
  noContext: 1
- singleSegment: 1
+ singleSegment: 0
  enableTokens: 0
  initialPrompt: 
+ stepSec: 3
+ keepSec: 0.2
+ lengthSec: 10
+ updatePrompt: 1
+ dropOldBuffer: 0
  tokensTimestamps: 0
  speedUp: 0
  audioCtx: 0
@@ -1709,7 +1715,9 @@ MonoBehaviour:
  m_ItemImage: {fileID: 0}
  m_Value: 0
  m_Options:
- m_Options: []
+ m_Options:
+ - m_Text: Microphones
+ m_Image: {fileID: 0}
  m_OnValueChanged:
  m_PersistentCalls:
  m_Calls: []
@@ -2748,6 +2756,7 @@ MonoBehaviour:
  m_EditorClassIdentifier: 
  whisper: {fileID: 587639199}
  microphoneRecord: {fileID: 1337591123}
+ echoSound: 1
  streamSegments: 1
  printLanguage: 1
  button: {fileID: 2099288375}
@@ -2771,6 +2780,7 @@ MonoBehaviour:
  m_EditorClassIdentifier: 
  maxLengthSec: 30
  frequency: 16000
+ chunksLengthSec: 0.5
  echo: 1
  microphoneDropdown: {fileID: 948466958}
  microphoneDefaultLabel: Default mic
@@ -2972,7 +2982,7 @@ MonoBehaviour:
  m_HorizontalOverflow: 0
  m_VerticalOverflow: 0
  m_LineSpacing: 1
- m_Text: 
+ m_Text: Microphones
 --- !u!222 &1361419379
 CanvasRenderer:
  m_ObjectHideFlags: 0

diff --git a/Assets/Samples/2 - Microphone/MicrophoneDemo.cs b/Assets/Samples/2 - Microphone/MicrophoneDemo.cs
@@ -7,10 +7,14 @@
 
 namespace Whisper.Samples
 {
+ /// <summary>
+ /// Record audio clip from microphone and make a transcription.
+ /// </summary>
  public class MicrophoneDemo : MonoBehaviour
  {
  public WhisperManager whisper;
  public MicrophoneRecord microphoneRecord;
+ public bool echoSound = true;
  public bool streamSegments = true;
  public bool printLanguage = true;
 
@@ -22,80 +26,95 @@ public class MicrophoneDemo : MonoBehaviour
  public Dropdown languageDropdown;
  public Toggle translateToggle;
  public ScrollRect scroll;
-
+ 
  private string _buffer;
 
  private void Awake()
  {
+ whisper.OnNewSegment += OnNewSegment;
+ whisper.OnProgress += OnProgressHandler;
+
+ microphoneRecord.OnRecordStop += OnRecordStop;
+
  button.onClick.AddListener(OnButtonPressed);
-
  languageDropdown.value = languageDropdown.options
  .FindIndex(op => op.text == whisper.language);
  languageDropdown.onValueChanged.AddListener(OnLanguageChanged);
 
  translateToggle.isOn = whisper.translateToEnglish;
  translateToggle.onValueChanged.AddListener(OnTranslateChanged);
-
- microphoneRecord.OnRecordStop += Transcribe;
-
- if (streamSegments)
- whisper.OnNewSegment += WhisperOnOnNewSegment;
- whisper.OnProgress += OnProgressHandler;
  }
 
  private void OnButtonPressed()
  {
  if (!microphoneRecord.IsRecording)
+ {
  microphoneRecord.StartRecord();
+ buttonText.text = "Stop";
+ }
  else
+ {
  microphoneRecord.StopRecord();
-
- if (buttonText)
- buttonText.text = microphoneRecord.IsRecording ? "Stop" : "Record";
+ buttonText.text = "Record";
+ }
  }
 
- private void OnLanguageChanged(int ind)
- {
- var opt = languageDropdown.options[ind];
- whisper.language = opt.text;
- }
-
- private void OnTranslateChanged(bool translate)
- {
- whisper.translateToEnglish = translate;
- }
-
- private async void Transcribe(float[] data, int frequency, int channels, float length)
+ private async void OnRecordStop(float[] data, int frequency, int channels, float length)
  {
+ buttonText.text = "Record";
  _buffer = "";
 
+ if (echoSound)
+ {
+ var clip = AudioClip.Create("mic", data.Length, channels, frequency, false);
+ AudioSource.PlayClipAtPoint(clip, Vector3.zero);
+ }
+
  var sw = new Stopwatch();
  sw.Start();
 
  var res = await whisper.GetTextAsync(data, frequency, channels);
+ if (res == null || !outputText) 
+ return;
 
  var time = sw.ElapsedMilliseconds;
  var rate = length / (time * 0.001f);
  timeText.text = $"Time: {time} ms\nRate: {rate:F1}x";
- if (res == null)
- return;
 
  var text = res.Result;
  if (printLanguage)
  text += $"\n\nLanguage: {res.Language}";
+
  outputText.text = text;
+ UiUtils.ScrollDown(scroll);
  }
 
- private void WhisperOnOnNewSegment(WhisperSegment segment)
+ private void OnLanguageChanged(int ind)
  {
- _buffer += segment.Text;
- outputText.text = _buffer + "...";
- UiUtils.ScrollDown(scroll);
+ var opt = languageDropdown.options[ind];
+ whisper.language = opt.text;
  }
 
+ private void OnTranslateChanged(bool translate)
+ {
+ whisper.translateToEnglish = translate;
+ }
+
  private void OnProgressHandler(int progress)
  {
+ if (!timeText)
+ return;
  timeText.text = $"Progress: {progress}%";
  }
+
+ private void OnNewSegment(WhisperSegment segment)
+ {
+ if (!streamSegments || !outputText)
+ return;
+
+ _buffer += segment.Text;
+ outputText.text = _buffer + "...";
+ UiUtils.ScrollDown(scroll);
+ }
  }
 }
diff --git a/Assets/Samples/3 - Languages/LanguagesDemo.cs b/Assets/Samples/3 - Languages/LanguagesDemo.cs
@@ -4,6 +4,9 @@
 
 namespace Whisper.Samples
 {
+ /// <summary>
+ /// Check if whisper model is multilingual and print all languages.
+ /// </summary>
  public class LanguagesDemo : MonoBehaviour
  {
  public WhisperManager whisper;