feat:add tts-streaming config and future (#5492)

web/app/components/base/audio-btn/audio.player.manager.ts

@@ -0,0 +1,53 @@
+import AudioPlayer from '@/app/components/base/audio-btn/audio'
+declare global {
+  // eslint-disable-next-line @typescript-eslint/consistent-type-definitions
+  interface AudioPlayerManager {
+    instance: AudioPlayerManager
+  }
+
+}
+
+export class AudioPlayerManager {
+  private static instance: AudioPlayerManager
+  private audioPlayers: AudioPlayer | null = null
+  private msgId: string | undefined
+
+  private constructor() {
+  }
+
+  public static getInstance(): AudioPlayerManager {
+    if (!AudioPlayerManager.instance) {
+      AudioPlayerManager.instance = new AudioPlayerManager()
+      this.instance = AudioPlayerManager.instance
+    }
+
+    return AudioPlayerManager.instance
+  }
+
+  public getAudioPlayer(url: string, isPublic: boolean, id: string | undefined, msgContent: string | null | undefined, voice: string | undefined, callback: ((event: string) => {}) | null): AudioPlayer {
+    if (this.msgId && this.msgId === id && this.audioPlayers) {
+      this.audioPlayers.setCallback(callback)
+      return this.audioPlayers
+    }
+    else {
+      if (this.audioPlayers) {
+        try {
+          this.audioPlayers.pauseAudio()
+          this.audioPlayers.cacheBuffers = []
+          this.audioPlayers.sourceBuffer?.abort()
+        }
+        catch (e) {
+        }
+      }
+
+      this.msgId = id
+      this.audioPlayers = new AudioPlayer(url, isPublic, id, msgContent, callback)
+      return this.audioPlayers
+    }
+  }
+
+  public resetMsgId(msgId: string) {
+    this.msgId = msgId
+    this.audioPlayers?.resetMsgId(msgId)
+  }
+}

web/app/components/base/audio-btn/audio.ts

@@ -0,0 +1,263 @@
+import Toast from '@/app/components/base/toast'
+import { textToAudioStream } from '@/service/share'
+
+declare global {
+  // eslint-disable-next-line @typescript-eslint/consistent-type-definitions
+  interface Window {
+    ManagedMediaSource: any
+  }
+}
+
+export default class AudioPlayer {
+  mediaSource: MediaSource | null
+  audio: HTMLAudioElement
+  audioContext: AudioContext
+  sourceBuffer?: SourceBuffer
+  cacheBuffers: ArrayBuffer[] = []
+  pauseTimer: number | null = null
+  msgId: string | undefined
+  msgContent: string | null | undefined = null
+  voice: string | undefined = undefined
+  isLoadData = false
+  url: string
+  isPublic: boolean
+  callback: ((event: string) => {}) | null
+
+  constructor(streamUrl: string, isPublic: boolean, msgId: string | undefined, msgContent: string | null | undefined, callback: ((event: string) => {}) | null) {
+    this.audioContext = new AudioContext()
+    this.msgId = msgId
+    this.msgContent = msgContent
+    this.url = streamUrl
+    this.isPublic = isPublic
+    this.callback = callback
+
+    // Compatible with iphone ios17 ManagedMediaSource
+    const MediaSource = window.MediaSource || window.ManagedMediaSource
+    if (!MediaSource) {
+      Toast.notify({
+        message: 'Your browser does not support audio streaming, if you are using an iPhone, please update to iOS 17.1 or later.',
+        type: 'error',
+      })
+    }
+    this.mediaSource = MediaSource ? new MediaSource() : null
+    this.audio = new Audio()
+    this.setCallback(callback)
+    this.audio.src = this.mediaSource ? URL.createObjectURL(this.mediaSource) : ''
+    this.audio.autoplay = true
+
+    const source = this.audioContext.createMediaElementSource(this.audio)
+    source.connect(this.audioContext.destination)
+    this.listenMediaSource('audio/mpeg')
+  }
+
+  public resetMsgId(msgId: string) {
+    this.msgId = msgId
+  }
+
+  private listenMediaSource(contentType: string) {
+    this.mediaSource?.addEventListener('sourceopen', () => {
+      if (this.sourceBuffer)
+        return
+
+      this.sourceBuffer = this.mediaSource?.addSourceBuffer(contentType)
+    //   this.sourceBuffer?.addEventListener('update', () => {
+    //     if (this.cacheBuffers.length && !this.sourceBuffer?.updating) {
+    //       const cacheBuffer = this.cacheBuffers.shift()!
+    //       this.sourceBuffer?.appendBuffer(cacheBuffer)
+    //     }
+    //     // this.pauseAudio()
+    //   })
+    //
+    //   this.sourceBuffer?.addEventListener('updateend', () => {
+    //     if (this.cacheBuffers.length && !this.sourceBuffer?.updating) {
+    //       const cacheBuffer = this.cacheBuffers.shift()!
+    //       this.sourceBuffer?.appendBuffer(cacheBuffer)
+    //     }
+    //     // this.pauseAudio()
+    //   })
+    })
+  }
+
+  public setCallback(callback: ((event: string) => {}) | null) {
+    this.callback = callback
+    if (callback) {
+      this.audio.addEventListener('ended', () => {
+        callback('ended')
+      }, false)
+      this.audio.addEventListener('paused', () => {
+        callback('paused')
+      }, true)
+      this.audio.addEventListener('loaded', () => {
+        callback('loaded')
+      }, true)
+      this.audio.addEventListener('play', () => {
+        callback('play')
+      }, true)
+      this.audio.addEventListener('timeupdate', () => {
+        callback('timeupdate')
+      }, true)
+      this.audio.addEventListener('loadeddate', () => {
+        callback('loadeddate')
+      }, true)
+      this.audio.addEventListener('canplay', () => {
+        callback('canplay')
+      }, true)
+      this.audio.addEventListener('error', () => {
+        callback('error')
+      }, true)
+    }
+  }
+
+  private async loadAudio() {
+    try {
+      const audioResponse: any = await textToAudioStream(this.url, this.isPublic, { content_type: 'audio/mpeg' }, {
+        message_id: this.msgId,
+        streaming: true,
+        voice: this.voice,
+        text: this.msgContent,
+      })
+
+      if (audioResponse.status !== 200) {
+        this.isLoadData = false
+        if (this.callback)
+          this.callback('error')
+      }
+
+      const reader = audioResponse.body.getReader()
+      while (true) {
+        const { value, done } = await reader.read()
+
+        if (done) {
+          this.receiveAudioData(value)
+          break
+        }
+
+        this.receiveAudioData(value)
+      }
+    }
+    catch (error) {
+      this.isLoadData = false
+      this.callback && this.callback('error')
+    }
+  }
+
+  // play audio
+  public playAudio() {
+    if (this.isLoadData) {
+      if (this.audioContext.state === 'suspended') {
+        this.audioContext.resume().then((_) => {
+          this.audio.play()
+          this.callback && this.callback('play')
+        })
+      }
+      else if (this.audio.ended) {
+        this.audio.play()
+        this.callback && this.callback('play')
+      }
+      if (this.callback)
+        this.callback('play')
+    }
+    else {
+      this.isLoadData = true
+      this.loadAudio()
+    }
+  }
+
+  private theEndOfStream() {
+    const endTimer = setInterval(() => {
+      if (!this.sourceBuffer?.updating) {
+        this.mediaSource?.endOfStream()
+        clearInterval(endTimer)
+      }
+      console.log('finishStream  endOfStream endTimer')
+    }, 10)
+  }
+
+  private finishStream() {
+    const timer = setInterval(() => {
+      if (!this.cacheBuffers.length) {
+        this.theEndOfStream()
+        clearInterval(timer)
+      }
+
+      if (this.cacheBuffers.length && !this.sourceBuffer?.updating) {
+        const arrayBuffer = this.cacheBuffers.shift()!
+        this.sourceBuffer?.appendBuffer(arrayBuffer)
+      }
+      console.log('finishStream  timer')
+    }, 10)
+  }
+
+  public async playAudioWithAudio(audio: string, play = true) {
+    if (!audio || !audio.length) {
+      this.finishStream()
+      return
+    }
+
+    const audioContent = Buffer.from(audio, 'base64')
+    this.receiveAudioData(new Uint8Array(audioContent))
+    if (play) {
+      this.isLoadData = true
+      if (this.audio.paused) {
+        this.audioContext.resume().then((_) => {
+          this.audio.play()
+          this.callback && this.callback('play')
+        })
+      }
+      else if (this.audio.ended) {
+        this.audio.play()
+        this.callback && this.callback('play')
+      }
+      else if (this.audio.played) { /* empty */ }
+
+      else {
+        this.audio.play()
+        this.callback && this.callback('play')
+      }
+    }
+  }
+
+  public pauseAudio() {
+    this.callback && this.callback('paused')
+    this.audio.pause()
+    this.audioContext.suspend()
+  }
+
+  private cancer() {
+
+  }
+
+  private receiveAudioData(unit8Array: Uint8Array) {
+    if (!unit8Array) {
+      this.finishStream()
+      return
+    }
+    const audioData = this.byteArrayToArrayBuffer(unit8Array)
+    if (!audioData.byteLength) {
+      if (this.mediaSource?.readyState === 'open')
+        this.finishStream()
+      return
+    }
+
+    if (this.sourceBuffer?.updating) {
+      this.cacheBuffers.push(audioData)
+    }
+    else {
+      if (this.cacheBuffers.length && !this.sourceBuffer?.updating) {
+        this.cacheBuffers.push(audioData)
+        const cacheBuffer = this.cacheBuffers.shift()!
+        this.sourceBuffer?.appendBuffer(cacheBuffer)
+      }
+      else {
+        this.sourceBuffer?.appendBuffer(audioData)
+      }
+    }
+  }
+
+  private byteArrayToArrayBuffer(byteArray: Uint8Array): ArrayBuffer {
+    const arrayBuffer = new ArrayBuffer(byteArray.length)
+    const uint8Array = new Uint8Array(arrayBuffer)
+    uint8Array.set(byteArray)
+    return arrayBuffer
+  }
+}

web/app/components/base/audio-btn/index.tsx

@@ -1,124 +1,78 @@
 'use client'
-import { useEffect, useRef, useState } from 'react'
+import { useRef, useState } from 'react'
 import { t } from 'i18next'
 import { useParams, usePathname } from 'next/navigation'
 import s from './style.module.css'
 import Tooltip from '@/app/components/base/tooltip'
 import { randomString } from '@/utils'
-import { textToAudio } from '@/service/share'
 import Loading from '@/app/components/base/loading'
+import { AudioPlayerManager } from '@/app/components/base/audio-btn/audio.player.manager'
 
 type AudioBtnProps = {
-  value: string
+  id?: string
   voice?: string
+  value?: string
   className?: string
   isAudition?: boolean
-  noCache: boolean
+  noCache?: boolean
 }
 
 type AudioState = 'initial' | 'loading' | 'playing' | 'paused' | 'ended'
 
 const AudioBtn = ({
-  value,
+  id,
   voice,
+  value,
   className,
   isAudition,
-  noCache,
 }: AudioBtnProps) => {
-  const audioRef = useRef<HTMLAudioElement | null>(null)
   const [audioState, setAudioState] = useState<AudioState>('initial')
 
   const selector = useRef(`play-tooltip-${randomString(4)}`)
   const params = useParams()
   const pathname = usePathname()
-  const removeCodeBlocks = (inputText: any) => {
-    const codeBlockRegex = /```[\s\S]*?```/g
-    if (inputText)
-      return inputText.replace(codeBlockRegex, '')
-    return ''
-  }
-
-  const loadAudio = async () => {
-    const formData = new FormData()
-    formData.append('text', removeCodeBlocks(value))
-    formData.append('voice', removeCodeBlocks(voice))
-
-    if (value !== '') {
-      setAudioState('loading')
-
-      let url = ''
-      let isPublic = false
-
-      if (params.token) {
-        url = '/text-to-audio'
-        isPublic = true
-      }
-      else if (params.appId) {
-        if (pathname.search('explore/installed') > -1)
-          url = `/installed-apps/${params.appId}/text-to-audio`
-        else
-          url = `/apps/${params.appId}/text-to-audio`
-      }
-
-      try {
-        const audioResponse = await textToAudio(url, isPublic, formData)
-        const blob_bytes = Buffer.from(audioResponse.data, 'latin1')
-        const blob = new Blob([blob_bytes], { type: 'audio/wav' })
-        const audioUrl = URL.createObjectURL(blob)
-        audioRef.current!.src = audioUrl
-      }
-      catch (error) {
-        setAudioState('initial')
-        console.error('Error playing audio:', error)
-      }
-    }
-  }
-
-  const handleToggle = async () => {
-    if (audioState === 'initial' || noCache) {
-      await loadAudio()
-    }
-    else if (audioRef.current) {
-      if (audioState === 'playing') {
-        audioRef.current.pause()
-        setAudioState('paused')
-      }
-      else {
-        audioRef.current.play()
+  const audio_finished_call = (event: string): any => {
+    switch (event) {
+      case 'ended':
+        setAudioState('ended')
+        break
+      case 'paused':
+        setAudioState('ended')
+        break
+      case 'loaded':
+        setAudioState('loading')
+        break
+      case 'play':
         setAudioState('playing')
-      }
+        break
+      case 'error':
+        setAudioState('ended')
+        break
     }
   }
+  let url = ''
+  let isPublic = false
 
-  useEffect(() => {
-    const currentAudio = audioRef.current
-
-    const handleLoading = () => {
+  if (params.token) {
+    url = '/text-to-audio'
+    isPublic = true
+  }
+  else if (params.appId) {
+    if (pathname.search('explore/installed') > -1)
+      url = `/installed-apps/${params.appId}/text-to-audio`
+    else
+      url = `/apps/${params.appId}/text-to-audio`
+  }
+  const handleToggle = async () => {
+    if (audioState === 'playing' || audioState === 'loading') {
+      setAudioState('paused')
+      AudioPlayerManager.getInstance().getAudioPlayer(url, isPublic, id, value, voice, audio_finished_call).pauseAudio()
+    }
+    else {
       setAudioState('loading')
+      AudioPlayerManager.getInstance().getAudioPlayer(url, isPublic, id, value, voice, audio_finished_call).playAudio()
     }
-
-    const handlePlay = () => {
-      currentAudio?.play()
-      setAudioState('playing')
-    }
-
-    const handleEnded = () => {
-      setAudioState('ended')
-    }
-
-    currentAudio?.addEventListener('progress', handleLoading)
-    currentAudio?.addEventListener('canplaythrough', handlePlay)
-    currentAudio?.addEventListener('ended', handleEnded)
-
-    return () => {
-      currentAudio?.removeEventListener('progress', handleLoading)
-      currentAudio?.removeEventListener('canplaythrough', handlePlay)
-      currentAudio?.removeEventListener('ended', handleEnded)
-      URL.revokeObjectURL(currentAudio?.src || '')
-      currentAudio?.pause()
-      currentAudio?.setAttribute('src', '')
-    }
-  }, [])
+  }
 
   const tooltipContent = {
     initial: t('appApi.play'),
@@ -151,7 +105,6 @@ const AudioBtn = ({
             )}
         </button>
       </Tooltip>
-      <audio ref={audioRef} src='' className='hidden' />
     </div>
   )
 }