smoother waveform (#589)

* smoothed waveform

* update comments

* do not reload audio on record end

* adjust waveform scale to match scratch

* add comments

spx-gui/src/components/editor/sound/SoundRecorder.vue

@@ -6,6 +6,7 @@
  v-if="recording || audioBlob"
  ref="wavesurferRef"
  v-model:range="audioRange"
+ recording
  :gain="gain"
  @init="handleWaveSurferInit"
  />
@@ -99,6 +100,7 @@ import { RecordPlugin } from '@/utils/wavesurfer-record'
 import VolumeSlider from './VolumeSlider.vue'
 import WavesurferWithRange from './WavesurferWithRange.vue'
 import type WaveSurfer from 'wavesurfer.js'
+import { toWav } from '@/utils/audio'
 
 const emit = defineEmits<{
  saved: [Sound]
@@ -162,10 +164,13 @@ const stopRecording = () => {
 }
 
 const saveRecording = async () => {
- if (!wavesurferRef.value) return
- const wav = await wavesurferRef.value.exportWav()
+ if (!wavesurferRef.value || !audioBlob.value) return
+ const wav = await toWav(await audioBlob.value.arrayBuffer())
 
- const file = fromBlob(`Recording_${dayjs().format('YYYY-MM-DD_HH:mm:ss')}.webm`, wav)
+ const file = fromBlob(
+ `Recording_${dayjs().format('YYYY-MM-DD_HH:mm:ss')}.wav`,
+ new Blob([wav], { type: 'audio/wav' })
+ )
  const sound = await Sound.create('recording', file)
  const action = { name: { en: 'Add recording', zh: '添加录音' } }
  await editorCtx.project.history.doAction(action, () => editorCtx.project.addSound(sound))

spx-gui/src/components/editor/sound/WavesurferWithRange.vue

@@ -16,6 +16,12 @@ const props = defineProps<{
  audioUrl?: string | null
  range: { left: number; right: number }
  gain: number
+ // When true, the component will render the waveform differently.
+ // It enables the cache and updates the waveform in real-time.
+ // TODO: Currently in the recording mode, the waveform will be rendered
+ // in different samples than in the normal mode, resulting different smoothness.
+ // We may fix this in the future.
+ recording?: boolean
 }>()
 
 const emit = defineEmits<{
@@ -75,7 +81,8 @@ const wavesurferDiv = ref<HTMLDivElement>()
 
 const wavesurferRef = useWavesurfer(
  () => wavesurferDiv.value,
- () => props.gain
+ () => props.gain,
+ props.recording
 )
 
 // we assume that wavesurferDiv.value will not change

spx-gui/src/components/editor/sound/wavesurfer.ts

@@ -7,7 +7,11 @@ import WaveSurfer from 'wavesurfer.js'
 import { useUIVariables } from '@/components/ui'
 import { getAudioContext } from '@/utils/audio'
 
-export function useWavesurfer(container: () => HTMLElement | undefined, gain: () => number) {
+export function useWavesurfer(
+ container: () => HTMLElement | undefined,
+ gain: () => number,
+ recording: boolean
+) {
  const uiVariables = useUIVariables()
 
  const wavesurfer = ref<WaveSurfer | null>(null)
@@ -35,6 +39,17 @@ export function useWavesurfer(container: () => HTMLElement | undefined, gain: ()
  gainNode_.gain.value = gain()
  gainNode = gainNode_
 
+ /**
+ * Cache for averaged data. Only used for recording.
+ * As we expect the `WaveSurfer` to be destroyed and recreated on every recording,
+ * we don't need to reset the cache when the recording stops.
+ */
+ let cache: {
+ averagedData: number[]
+ originalLength: number
+ blockSize: number
+ }
+
  wavesurfer.value = new WaveSurfer({
  interact: false,
  container: newContainer,
@@ -46,54 +61,124 @@ export function useWavesurfer(container: () => HTMLElement | undefined, gain: ()
  normalize: true,
  media: audioElement,
  renderFunction: (peaks: (Float32Array | number[])[], ctx: CanvasRenderingContext2D): void => {
- // TODO: Better drawing algorithm to reduce flashing?
- const smoothAndDrawChannel = (channel: Float32Array, vScale: number) => {
- const { width, height } = ctx.canvas
- const halfHeight = height / 2
- const numPoints = Math.floor(width / 5)
- const blockSize = Math.floor(channel.length / numPoints)
- const smoothedData = new Float32Array(numPoints)
-
- // Smooth the data by averaging blocks
- for (let i = 0; i < numPoints; i++) {
- let sum = 0
- for (let j = 0; j < blockSize; j++) {
- sum += Math.abs(channel[i * blockSize + j])
+ try {
+ const halfHeight = ctx.canvas.height / 2
+
+ const averageBlock = (data: number[] | Float32Array, blockSize: number): number[] => {
+ // Check if we can use the cached data
+ if (
+ recording &&
+ cache &&
+ cache.blockSize === blockSize &&
+ cache.originalLength <= data.length
+ ) {
+ const newBlocks =
+ Math.floor(data.length / blockSize) - Math.floor(cache.originalLength / blockSize)
+
+ // If there are new blocks to process
+ if (newBlocks > 0) {
+ const newAveragedData = cache.averagedData.slice()
+ const startIndex = cache.originalLength
+ for (let i = 0; i < newBlocks; i++) {
+ let sum = 0
+ for (let j = 0; j < blockSize; j++) {
+ const index = startIndex + i * blockSize + j
+ sum += Math.max(0, data[index])
+ }
+ newAveragedData.push(sum / blockSize)
+ }
+
+ cache = {
+ averagedData: newAveragedData,
+ originalLength: data.length,
+ blockSize
+ }
+ return newAveragedData
+ }
+
+ // If no new blocks to process, return cached data
+ return cache.averagedData
  }
- smoothedData[i] = sum / blockSize
- }
 
- // Draw with bezier curves
- ctx.beginPath()
- ctx.moveTo(0, halfHeight)
+ // Calculate new averaged data if cache is not valid or not present
+ const averagedData: number[] = new Array(Math.floor(data.length / blockSize))
+ for (let i = 0; i < averagedData.length; i++) {
+ let sum = 0
+ for (let j = 0; j < blockSize; j++) {
+ const index = i * blockSize + j
+ sum += Math.max(0, data[index])
+ }
+ averagedData[i] = sum / blockSize
+ }
 
- for (let i = 1; i < smoothedData.length; i++) {
- const prevX = (i - 1) * (width / numPoints)
- const currX = i * (width / numPoints)
- const midX = (prevX + currX) / 2
- const prevY = halfHeight + smoothedData[i - 1] * halfHeight * vScale
- const currY = halfHeight + smoothedData[i] * halfHeight * vScale
+  // Update cache with new averaged data
+ cache = {
+  averagedData,
+  originalLength: data.length,
+  blockSize
+ }
 
- // Use a quadratic bezier curve to the middle of the interval for a smoother line
- ctx.quadraticCurveTo(prevX, prevY, midX, (prevY + currY) / 2)
- ctx.quadraticCurveTo(midX, (prevY + currY) / 2, currX, currY)
+ return averagedData
  }
 
- ctx.lineTo(width, halfHeight)
- ctx.strokeStyle = uiVariables.color.sound[400]
- ctx.stroke()
- ctx.closePath()
- ctx.fillStyle = uiVariables.color.sound[400]
- ctx.fill()
- }
+ const drawSmoothCurve = (
+ ctx: CanvasRenderingContext2D,
+ points: number[],
+ getPoint: (index: number) => number
+ ) => {
+ const segmentLength = ctx.canvas.width / (points.length - 1)
 
- const channel = Array.isArray(peaks[0]) ? new Float32Array(peaks[0] as number[]) : peaks[0]
+ ctx.beginPath()
+ ctx.moveTo(0, halfHeight)
 
- const scale = gain() * 5
+ for (let i = 0; i < points.length - 2; i++) {
+ const xc = (i * segmentLength + (i + 1) * segmentLength) / 2
+ const yc = (getPoint(i) + getPoint(i + 1)) / 2
+ ctx.quadraticCurveTo(i * segmentLength, getPoint(i), xc, yc)
+ }
 
- // Only one channel is assumed, render it twice (mirrored)
- smoothAndDrawChannel(channel, scale) // Upper part
- smoothAndDrawChannel(channel, -scale) // Lower part (mirrored)
+ ctx.quadraticCurveTo(
+ (points.length - 2) * segmentLength,
+ getPoint(points.length - 2),
+ ctx.canvas.width,
+ getPoint(points.length - 1)
+ )
+
+ ctx.lineTo(ctx.canvas.width, halfHeight)
+
+ ctx.strokeStyle = uiVariables.color.sound[400]
+ ctx.lineWidth = 2
+ ctx.stroke()
+ ctx.closePath()
+ ctx.fillStyle = uiVariables.color.sound[400]
+ ctx.fill()
+ }
+
+ const channel = peaks[0]
+
+ const scale = gain() * 1200
+
+ // TODO: For now, blockSize is fixed to 2000 for longer recordings
+ // to address the issue of slow update. This should be optimized
+ // in the future.
+ const blockSize = channel.length > 200000 ? 2000 : channel.length > 100000 ? 1000 : 500
+
+ const smoothedChannel = averageBlock(channel, blockSize)
+
+ drawSmoothCurve(
+ ctx,
+ smoothedChannel,
+ (index: number) => smoothedChannel[index] * scale + halfHeight
+ )
+ drawSmoothCurve(
+ ctx,
+ smoothedChannel,
+ (index: number) => -smoothedChannel[index] * scale + halfHeight
+ )
+ } catch (e) {
+ // wavesurfer does not log errors so we do it ourselves
+ console.error(e)
+ }
  }
  })
 

spx-gui/src/utils/wavesurfer-record.ts

@@ -46,7 +46,6 @@ const findSupportedMimeType = () =>
 export class RecordPlugin extends BasePlugin<RecordPluginEvents, RecordPluginOptions> {
  private stream: MediaStream | null = null
  private mediaRecorder: MediaRecorder | null = null
- private dataWindow: Float32Array | null = null
  private isWaveformPaused = false
  private originalOptions: { cursorWidth: number; interact: boolean } | undefined
  private timer: Timer
@@ -82,10 +81,11 @@ export class RecordPlugin extends BasePlugin<RecordPluginEvents, RecordPluginOpt
  const audioContext = new AudioContext()
  const source = audioContext.createMediaStreamSource(stream)
  const analyser = audioContext.createAnalyser()
+ analyser.fftSize = 1024
  source.connect(analyser)
 
- const bufferLength = analyser.frequencyBinCount
- const dataArray = new Float32Array(bufferLength)
+ const dataArray = new Float32Array(analyser.frequencyBinCount)
+ const dataWindow: number[] = []
 
  let animationId: number
 
@@ -96,17 +96,7 @@ export class RecordPlugin extends BasePlugin<RecordPluginEvents, RecordPluginOpt
  }
 
  analyser.getFloatTimeDomainData(dataArray)
-
- const newLength = (this.dataWindow?.length || 0) + bufferLength
- const tempArray = new Float32Array(newLength)
- // Append the new data to the existing data
- if (this.dataWindow) {
- tempArray.set(this.dataWindow, 0)
- tempArray.set(dataArray, this.dataWindow.length)
- } else {
- tempArray.set(dataArray, 0)
- }
- this.dataWindow = tempArray
+ dataWindow.push(...dataArray)
 
  if (this.wavesurfer) {
  this.originalOptions ??= {
@@ -115,7 +105,7 @@ export class RecordPlugin extends BasePlugin<RecordPluginEvents, RecordPluginOpt
  }
  this.wavesurfer.options.cursorWidth = 0
  this.wavesurfer.options.interact = false
- this.wavesurfer.load('', [tempArray], this.duration)
+ this.wavesurfer.load('', [dataWindow], this.duration)
  }
 
  animationId = requestAnimationFrame(drawWaveform)
@@ -167,7 +157,6 @@ export class RecordPlugin extends BasePlugin<RecordPluginEvents, RecordPluginOpt
  /** Start recording audio from the microphone */
  public async startRecording(options?: RecordPluginDeviceOptions) {
  const stream = this.stream || (await this.startMic(options))
- this.dataWindow = null
  const mediaRecorder =
  this.mediaRecorder ||
  new MediaRecorder(stream, {
@@ -190,7 +179,6 @@ export class RecordPlugin extends BasePlugin<RecordPluginEvents, RecordPluginOpt
  this.emit(ev, blob)
  if (this.options.renderRecordedAudio) {
  this.applyOriginalOptionsIfNeeded()
- this.wavesurfer?.loadBlob(blob)
  }
  }