youngkingdom/dify
1
0
Fork 0
mirror of https://github.com/langgenius/dify.git synced 2026-07-14 08:57:02 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(agent): add speech-to-text endpoint

Browse Source
This commit is contained in:
yyh
2026-07-10 15:37:57 +08:00
parent 3c8e0e2113
commit dfede8eef8
13 changed files with 676 additions and 79 deletions
Download Patch File Download Diff File Expand all files Collapse all files

176
api/controllers/console/app/audio.py
View File

@ -1,13 +1,17 @@
import logging
from uuid import UUID
from flask import request
from flask_restx import Resource
from pydantic import BaseModel, Field, RootModel
from werkzeug.exceptions import InternalServerError
from pydantic import BaseModel, ConfigDict, Field, RootModel
from sqlalchemy.orm import Session
from werkzeug.datastructures import FileStorage
from werkzeug.exceptions import HTTPException, InternalServerError
import services
from controllers.common.schema import query_params_from_model, register_response_schema_models, register_schema_models
from controllers.console import console_ns
from controllers.console.agent.app_helpers import resolve_agent_runtime_app_model
from controllers.console.app.error import (
AppUnavailableError,
AudioTooLargeError,
@ -19,13 +23,16 @@ from controllers.console.app.error import (
ProviderQuotaExceededError,
UnsupportedAudioTypeError,
)
from controllers.console.app.wraps import get_app_model
from controllers.console.app.wraps import get_app_model, with_session
from controllers.console.wraps import (
RBACPermission,
RBACResourceScope,
account_initialization_required,
edit_permission_required,
rbac_permission_required,
setup_required,
with_current_tenant_id,
with_current_user,
)
from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError
from extensions.ext_database import db
@ -33,7 +40,10 @@ from fields.base import ResponseModel
from graphon.model_runtime.errors.invoke import InvokeError
from libs.helper import dump_response
from libs.login import current_user, login_required
from models import App, AppMode
from models import Account, App, AppMode
from models.agent import AgentConfigDraftType
from models.agent_config_entities import AgentSoulConfig
from services.agent.composer_service import AgentComposerService
from services.app_ref_service import AppRefService
from services.audio_service import AudioService
from services.errors.audio import (
@ -61,6 +71,12 @@ class AudioTranscriptResponse(ResponseModel):
text: str = Field(description="Transcribed text from audio")
class AgentAudioTranscriptFormPayload(BaseModel):
model_config = ConfigDict(extra="forbid")
draft_type: AgentConfigDraftType = AgentConfigDraftType.DRAFT
class TextToSpeechVoiceResponse(ResponseModel):
# see api/core/plugin/impl/model.py
name: str = Field(description="Voice display name")
@ -71,7 +87,7 @@ class TextToSpeechVoiceListResponse(RootModel[list[TextToSpeechVoiceResponse]]):
root: list[TextToSpeechVoiceResponse] = Field(description="Available voices")
register_schema_models(console_ns, TextToSpeechPayload, TextToSpeechVoiceQuery)
register_schema_models(console_ns, AgentAudioTranscriptFormPayload, TextToSpeechPayload, TextToSpeechVoiceQuery)
register_response_schema_models(
console_ns,
AudioTranscriptResponse,
@ -79,12 +95,82 @@ register_response_schema_models(
TextToSpeechVoiceListResponse,
)
_AUDIO_TRANSCRIPT_FILE_PARAM = {
"description": "MP3 audio to transcribe",
"in": "formData",
"type": "file",
"required": True,
}
_AGENT_AUDIO_TRANSCRIPT_PARAMS = {
"file": _AUDIO_TRANSCRIPT_FILE_PARAM,
"draft_type": {
"description": "Agent debug config source",
"in": "formData",
"type": "string",
"enum": [draft_type.value for draft_type in AgentConfigDraftType],
"default": AgentConfigDraftType.DRAFT.value,
"required": False,
},
}
def _transcribe_audio_to_text(
*,
app_model: App,
file: FileStorage | None,
agent_soul: AgentSoulConfig | None = None,
) -> dict[str, str]:
try:
if agent_soul is None:
response = AudioService.transcript_asr(
app_model=app_model,
file=file,
end_user=None,
)
else:
response = AudioService.transcript_agent_asr(
app_model=app_model,
agent_soul=agent_soul,
file=file,
end_user=None,
)
return dump_response(AudioTranscriptResponse, response)
except services.errors.app_model_config.AppModelConfigBrokenError:
logger.exception("App model config broken.")
raise AppUnavailableError()
except NoAudioUploadedServiceError:
raise NoAudioUploadedError()
except AudioTooLargeServiceError as e:
raise AudioTooLargeError(str(e))
except UnsupportedAudioTypeServiceError:
raise UnsupportedAudioTypeError()
except ProviderNotSupportSpeechToTextServiceError:
raise ProviderNotSupportSpeechToTextError()
except ProviderTokenNotInitError as ex:
raise ProviderNotInitializeError(ex.description)
except QuotaExceededError:
raise ProviderQuotaExceededError()
except ModelCurrentlyNotSupportError:
raise ProviderModelCurrentlyNotSupportError()
except InvokeError as e:
raise CompletionRequestError(e.description)
except HTTPException:
raise
except ValueError:
raise
except Exception as e:
logger.exception("Failed to transcribe audio to text")
raise InternalServerError() from e
@console_ns.route("/apps/<uuid:app_id>/audio-to-text")
class ChatMessageAudioApi(Resource):
@console_ns.doc("chat_message_audio_transcript")
@console_ns.doc(description="Transcript audio to text for chat messages")
@console_ns.doc(params={"app_id": "App ID"})
@console_ns.doc(
consumes=["multipart/form-data"],
params={"app_id": "App ID", "file": _AUDIO_TRANSCRIPT_FILE_PARAM},
)
@console_ns.response(
200,
"Audio transcription successful",
@ -97,40 +183,54 @@ class ChatMessageAudioApi(Resource):
@account_initialization_required
@get_app_model(mode=[AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT])
def post(self, app_model: App):
file = request.files["file"]
return _transcribe_audio_to_text(app_model=app_model, file=request.files.get("file"))
try:
response = AudioService.transcript_asr(
app_model=app_model,
file=file,
end_user=None,
)
return dump_response(AudioTranscriptResponse, response)
except services.errors.app_model_config.AppModelConfigBrokenError:
logger.exception("App model config broken.")
raise AppUnavailableError()
except NoAudioUploadedServiceError:
raise NoAudioUploadedError()
except AudioTooLargeServiceError as e:
raise AudioTooLargeError(str(e))
except UnsupportedAudioTypeServiceError:
raise UnsupportedAudioTypeError()
except ProviderNotSupportSpeechToTextServiceError:
raise ProviderNotSupportSpeechToTextError()
except ProviderTokenNotInitError as ex:
raise ProviderNotInitializeError(ex.description)
except QuotaExceededError:
raise ProviderQuotaExceededError()
except ModelCurrentlyNotSupportError:
raise ProviderModelCurrentlyNotSupportError()
except InvokeError as e:
raise CompletionRequestError(e.description)
except ValueError as e:
raise e
except Exception as e:
logger.exception("Failed to handle post request to ChatMessageAudioApi")
raise InternalServerError()
@console_ns.route("/agent/<uuid:agent_id>/audio-to-text")
class AgentChatMessageAudioApi(Resource):
@console_ns.doc("agent_chat_message_audio_transcript")
@console_ns.doc(description="Transcribe audio using the current Agent debug configuration")
@console_ns.doc(
consumes=["multipart/form-data"],
params={"agent_id": "Agent ID", **_AGENT_AUDIO_TRANSCRIPT_PARAMS},
)
@console_ns.response(
200,
"Audio transcription successful",
console_ns.models[AudioTranscriptResponse.__name__],
)
@console_ns.response(400, "Bad request - Speech to text disabled or unsupported audio")
@console_ns.response(404, "Agent or build draft not found")
@console_ns.response(413, "Audio file too large")
@setup_required
@login_required
@account_initialization_required
@edit_permission_required
@rbac_permission_required(RBACResourceScope.APP, RBACPermission.APP_TEST_AND_RUN)
@with_current_user
@with_current_tenant_id
@with_session
def post(
self,
session: Session,
current_tenant_id: str,
current_user: Account,
agent_id: UUID,
):
payload = AgentAudioTranscriptFormPayload.model_validate(request.form.to_dict(flat=True))
app_model = resolve_agent_runtime_app_model(tenant_id=current_tenant_id, agent_id=agent_id)
agent_soul = AgentComposerService.load_agent_soul_for_debug(
tenant_id=current_tenant_id,
agent_id=str(agent_id),
account_id=current_user.id,
draft_type=payload.draft_type,
session=session,
)
return _transcribe_audio_to_text(
app_model=app_model,
agent_soul=agent_soul,
file=request.files.get("file"),
)
@console_ns.route("/apps/<uuid:app_id>/text-to-audio")

26
api/services/agent/composer_service.py
View File

@ -358,6 +358,32 @@ class AgentComposerService:
agent = cls._require_agent(tenant_id=tenant_id, agent_id=agent_id, session=session)
return cls._load_agent_composer_for_agent(tenant_id=tenant_id, agent=agent, session=session)
@classmethod
def load_agent_soul_for_debug(
cls,
*,
tenant_id: str,
agent_id: str,
account_id: str,
draft_type: AgentConfigDraftType,
session: Session,
) -> AgentSoulConfig:
"""Load the same normal or account-owned build draft used by Agent debug chat."""
if draft_type == AgentConfigDraftType.DEBUG_BUILD:
state = cls.load_agent_app_build_draft(
tenant_id=tenant_id,
agent_id=agent_id,
account_id=account_id,
session=session,
)
else:
state = cls.load_agent_composer(
tenant_id=tenant_id,
agent_id=agent_id,
session=session,
)
return AgentSoulConfig.model_validate(state["agent_soul"])
@classmethod
def _load_agent_composer_for_agent(cls, *, tenant_id: str, agent: Agent, session: Session) -> dict[str, Any]:
draft = cls._get_or_create_agent_draft(

12
api/services/agent/roster_service.py
View File

@ -798,6 +798,18 @@ class AgentRosterService:
)
)
def get_published_agent_soul_for_app(self, *, tenant_id: str, app_id: str) -> AgentSoulConfig | None:
"""Return the active Agent Soul used by a published Agent App runtime."""
agent = self.get_app_backing_agent(tenant_id=tenant_id, app_id=app_id)
if agent is None:
return None
version = self._get_version(
tenant_id=tenant_id,
agent_id=agent.id,
version_id=agent.active_config_snapshot_id,
)
return AgentSoulConfig.model_validate(version.config_snapshot_dict)
def get_agent_app_model(self, *, tenant_id: str, agent_id: str) -> App:
"""Resolve the Agent App hidden behind an app-backed Agent id.

43
api/services/audio_service.py
View File

@ -10,10 +10,14 @@ from sqlalchemy.orm import Session
from werkzeug.datastructures import FileStorage
from constants import AUDIO_EXTENSIONS
from core.app.apps.agent_app.app_feature_projection import merge_agent_app_features
from core.model_manager import ModelManager
from extensions.ext_database import db
from graphon.model_runtime.entities.model_entities import ModelType
from models.agent_config_entities import AgentSoulConfig
from models.enums import MessageStatus
from models.model import App, AppMode, Message
from services.agent.roster_service import AgentRosterService
from services.app_ref_service import MessageRef
from services.errors.audio import (
AudioTooLargeServiceError,
@ -41,7 +45,20 @@ class AudioService:
return session.scalar(stmt.limit(1))
@classmethod
def transcript_asr(cls, app_model: App, file: FileStorage | None, end_user: str | None = None):
def transcript_asr(cls, app_model: App, file: FileStorage | None, end_user: str | None = None) -> dict[str, str]:
if app_model.mode == AppMode.AGENT:
agent_soul = AgentRosterService(db.session).get_published_agent_soul_for_app(
tenant_id=app_model.tenant_id,
app_id=app_model.id,
)
if agent_soul is not None:
return cls.transcript_agent_asr(
app_model=app_model,
agent_soul=agent_soul,
file=file,
end_user=end_user,
)
if app_model.mode in {AppMode.ADVANCED_CHAT, AppMode.WORKFLOW}:
workflow = app_model.workflow
if workflow is None:
@ -58,6 +75,30 @@ class AudioService:
if not app_model_config.speech_to_text_dict["enabled"]:
raise ValueError("Speech to text is not enabled")
return cls._invoke_speech_to_text(app_model=app_model, file=file, end_user=end_user)
@classmethod
def transcript_agent_asr(
cls,
app_model: App,
agent_soul: AgentSoulConfig,
file: FileStorage | None,
end_user: str | None = None,
) -> dict[str, str]:
"""Transcribe Agent audio after applying the Agent runtime feature projection."""
features = merge_agent_app_features(
agent_soul=agent_soul,
app_model_config=app_model.app_model_config,
)
if not features.get("speech_to_text", {}).get("enabled"):
raise ValueError("Speech to text is not enabled")
return cls._invoke_speech_to_text(app_model=app_model, file=file, end_user=end_user)
@classmethod
def _invoke_speech_to_text(
cls, app_model: App, file: FileStorage | None, end_user: str | None = None
) -> dict[str, str]:
if file is None:
raise NoAudioUploadedServiceError()

134
api/tests/unit_tests/controllers/console/app/test_audio.py
View File

@ -4,13 +4,20 @@ import io
from inspect import unwrap
from types import SimpleNamespace
from unittest.mock import patch
from uuid import UUID
import pytest
from flask import Flask
from werkzeug.datastructures import FileStorage
from werkzeug.exceptions import InternalServerError
from controllers.console.app.audio import ChatMessageAudioApi, ChatMessageTextApi, TextModesApi
from controllers.console.app import audio as audio_module
from controllers.console.app.audio import (
AgentChatMessageAudioApi,
ChatMessageAudioApi,
ChatMessageTextApi,
TextModesApi,
)
from controllers.console.app.error import (
AppUnavailableError,
AudioTooLargeError,
@ -24,6 +31,10 @@ from controllers.console.app.error import (
)
from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError
from graphon.model_runtime.errors.invoke import InvokeError
from models.agent import AgentConfigDraftType
from models.agent_config_entities import AgentSoulConfig
from services.agent.composer_service import AgentComposerService
from services.agent.errors import AgentVersionNotFoundError
from services.app_ref_service import MessageRef
from services.audio_service import AudioService
from services.errors.app_model_config import AppModelConfigBrokenError
@ -52,6 +63,127 @@ def test_console_audio_api_success(app: Flask, monkeypatch: pytest.MonkeyPatch)
assert response == {"text": "ok"}
def test_agent_console_audio_api_uses_agent_draft(app: Flask, monkeypatch: pytest.MonkeyPatch) -> None:
agent_id = UUID("019ef3d2-b24c-7803-b428-18b5ee8fb853")
app_model = SimpleNamespace(id="backing-app-1")
agent_soul = AgentSoulConfig.model_validate({"app_features": {"speech_to_text": {"enabled": True}}})
calls: dict[str, object] = {}
def resolve_agent_runtime_app_model(**kwargs):
calls["resolver"] = kwargs
return app_model
def load_agent_soul_for_debug(**kwargs):
calls["draft"] = kwargs
return agent_soul
def transcript_agent_asr(**kwargs):
calls["asr"] = kwargs
return {"text": "agent transcript"}
monkeypatch.setattr(audio_module, "resolve_agent_runtime_app_model", resolve_agent_runtime_app_model)
monkeypatch.setattr(AgentComposerService, "load_agent_soul_for_debug", load_agent_soul_for_debug)
monkeypatch.setattr(AudioService, "transcript_agent_asr", transcript_agent_asr)
api = AgentChatMessageAudioApi()
handler = unwrap(api.post)
session = SimpleNamespace()
current_user = SimpleNamespace(id="account-1")
with app.test_request_context(
f"/console/api/agent/{agent_id}/audio-to-text",
method="POST",
data={"file": _file_data(), "draft_type": "debug_build"},
):
response = handler(
api,
session=session,
current_tenant_id="tenant-1",
current_user=current_user,
agent_id=agent_id,
)
assert response == {"text": "agent transcript"}
assert calls["resolver"] == {"tenant_id": "tenant-1", "agent_id": agent_id}
assert calls["draft"] == {
"tenant_id": "tenant-1",
"agent_id": str(agent_id),
"account_id": "account-1",
"draft_type": AgentConfigDraftType.DEBUG_BUILD,
"session": session,
}
assert calls["asr"] == {
"app_model": app_model,
"agent_soul": agent_soul,
"file": calls["asr"]["file"],
"end_user": None,
}
def test_agent_console_audio_api_defaults_to_normal_draft(app: Flask, monkeypatch: pytest.MonkeyPatch) -> None:
agent_id = UUID("019ef3d2-b24c-7803-b428-18b5ee8fb853")
captured: dict[str, object] = {}
monkeypatch.setattr(
audio_module,
"resolve_agent_runtime_app_model",
lambda **_kwargs: SimpleNamespace(id="backing-app-1"),
)
def load_agent_soul_for_debug(**kwargs):
captured.update(kwargs)
return AgentSoulConfig.model_validate({"app_features": {"speech_to_text": {"enabled": True}}})
monkeypatch.setattr(AgentComposerService, "load_agent_soul_for_debug", load_agent_soul_for_debug)
monkeypatch.setattr(AudioService, "transcript_agent_asr", lambda **_kwargs: {"text": "ok"})
api = AgentChatMessageAudioApi()
handler = unwrap(api.post)
with app.test_request_context(
f"/console/api/agent/{agent_id}/audio-to-text",
method="POST",
data={"file": _file_data()},
):
response = handler(
api,
session=SimpleNamespace(),
current_tenant_id="tenant-1",
current_user=SimpleNamespace(id="account-1"),
agent_id=agent_id,
)
assert response == {"text": "ok"}
assert captured["draft_type"] == AgentConfigDraftType.DRAFT
def test_agent_console_audio_api_preserves_missing_build_draft_404(app: Flask, monkeypatch: pytest.MonkeyPatch) -> None:
agent_id = UUID("019ef3d2-b24c-7803-b428-18b5ee8fb853")
monkeypatch.setattr(
audio_module,
"resolve_agent_runtime_app_model",
lambda **_kwargs: SimpleNamespace(id="backing-app-1"),
)
monkeypatch.setattr(
AgentComposerService,
"load_agent_soul_for_debug",
lambda **_kwargs: (_ for _ in ()).throw(AgentVersionNotFoundError()),
)
api = AgentChatMessageAudioApi()
handler = unwrap(api.post)
with app.test_request_context(
f"/console/api/agent/{agent_id}/audio-to-text",
method="POST",
data={"file": _file_data(), "draft_type": "debug_build"},
):
with pytest.raises(AgentVersionNotFoundError):
handler(
api,
session=SimpleNamespace(),
current_tenant_id="tenant-1",
current_user=SimpleNamespace(id="account-1"),
agent_id=agent_id,
)
@pytest.mark.parametrize(
("exc", "expected"),
[

89
api/tests/unit_tests/services/agent/test_agent_services.py
View File

@ -116,6 +116,25 @@ def test_agent_soul_has_model():
assert agent_soul_has_model(AgentSoulConfig()) is False
def test_get_published_agent_soul_for_app_uses_active_snapshot():
agent_soul = AgentSoulConfig.model_validate({"app_features": {"speech_to_text": {"enabled": True}}})
agent = SimpleNamespace(id="agent-1", active_config_snapshot_id="version-1")
version = SimpleNamespace(config_snapshot_dict=agent_soul.model_dump(mode="json"))
service = AgentRosterService(FakeSession(scalar=[agent, version]))
result = service.get_published_agent_soul_for_app(tenant_id="tenant-1", app_id="app-1")
assert result == agent_soul
def test_get_published_agent_soul_for_app_returns_none_without_backing_agent():
service = AgentRosterService(FakeSession(scalar=[None]))
result = service.get_published_agent_soul_for_app(tenant_id="tenant-1", app_id="legacy-app-1")
assert result is None
def test_load_workflow_composer_returns_empty_state(monkeypatch: pytest.MonkeyPatch):
monkeypatch.setattr(AgentComposerService, "_get_draft_workflow", lambda **kwargs: SimpleNamespace(id="workflow-1"))
monkeypatch.setattr(AgentComposerService, "_get_workflow_binding", lambda **kwargs: None)
@ -632,7 +651,7 @@ def test_publish_agent_app_draft_rejects_missing_model(monkeypatch: pytest.Monke
base_snapshot_id="version-1",
config_snapshot=AgentSoulConfig(),
)
fake_session = FakeSession(scalar=[agent, draft])
fake_session = FakeSession(scalar=[agent, draft, None])
def fail_create_config_version(**_kwargs):
raise AssertionError("config version must not be created when Agent Soul has no model")
@ -777,6 +796,74 @@ def test_agent_app_build_draft_checkout_and_apply_use_user_isolated_draft(monkey
assert fake_session.commits == 1
@pytest.mark.parametrize(
("draft_type", "account_id"),
[
(AgentConfigDraftType.DRAFT, None),
(AgentConfigDraftType.DEBUG_BUILD, "account-1"),
],
)
def test_load_agent_soul_for_debug_selects_requested_draft(
draft_type: AgentConfigDraftType,
account_id: str | None,
):
agent = Agent(
id="agent-1",
tenant_id="tenant-1",
name="Iris",
description="",
agent_kind=AgentKind.DIFY_AGENT,
scope=AgentScope.ROSTER,
source=AgentSource.AGENT_APP,
status=AgentStatus.ACTIVE,
)
agent_soul = AgentSoulConfig.model_validate({"app_features": {"speech_to_text": {"enabled": True}}})
draft = AgentConfigDraft(
tenant_id="tenant-1",
agent_id="agent-1",
draft_type=draft_type,
account_id=account_id,
draft_owner_key=account_id or "",
config_snapshot=agent_soul,
)
fake_session = FakeSession(
scalar=[draft] if draft_type == AgentConfigDraftType.DEBUG_BUILD else [agent, draft, None]
)
result = AgentComposerService.load_agent_soul_for_debug(
tenant_id="tenant-1",
agent_id="agent-1",
account_id="account-1",
draft_type=draft_type,
session=fake_session,
)
assert result == agent_soul
def test_load_agent_soul_for_debug_requires_existing_build_draft():
agent = Agent(
id="agent-1",
tenant_id="tenant-1",
name="Iris",
description="",
agent_kind=AgentKind.DIFY_AGENT,
scope=AgentScope.ROSTER,
source=AgentSource.AGENT_APP,
status=AgentStatus.ACTIVE,
)
fake_session = FakeSession(scalar=[None])
with pytest.raises(AgentVersionNotFoundError):
AgentComposerService.load_agent_soul_for_debug(
tenant_id="tenant-1",
agent_id="agent-1",
account_id="account-1",
draft_type=AgentConfigDraftType.DEBUG_BUILD,
session=fake_session,
)
def test_agent_app_build_draft_apply_marks_unpublished_when_build_draft_differs(monkeypatch: pytest.MonkeyPatch):
agent = Agent(
id="agent-1",

112
api/tests/unit_tests/services/test_audio_service.py
View File

@ -59,6 +59,7 @@ from unittest.mock import MagicMock, Mock, create_autospec, patch
import pytest
from werkzeug.datastructures import FileStorage
from models.agent_config_entities import AgentSoulConfig
from models.enums import MessageStatus
from models.model import App, AppMode, AppModelConfig, Message
from models.workflow import Workflow
@ -267,6 +268,117 @@ class TestAudioServiceASR:
# Assert
assert result == {"text": "Workflow transcribed text"}
@patch("services.audio_service.AgentRosterService", autospec=True)
@patch("services.audio_service.ModelManager.for_tenant", autospec=True)
def test_transcript_asr_success_published_agent_mode(
self,
mock_model_manager_class,
mock_roster_service_class,
factory: AudioServiceTestDataFactory,
):
app = factory.create_app_mock(mode=AppMode.AGENT)
file = factory.create_file_storage_mock()
agent_soul = AgentSoulConfig.model_validate({"app_features": {"speech_to_text": {"enabled": True}}})
mock_roster_service_class.return_value.get_published_agent_soul_for_app.return_value = agent_soul
mock_model_instance = MagicMock()
mock_model_instance.invoke_speech2text.return_value = "Published Agent transcript"
mock_model_manager_class.return_value.get_default_model_instance.return_value = mock_model_instance
result = AudioService.transcript_asr(app_model=app, file=file, end_user="end-user-1")
assert result == {"text": "Published Agent transcript"}
mock_roster_service_class.return_value.get_published_agent_soul_for_app.assert_called_once_with(
tenant_id=app.tenant_id,
app_id=app.id,
)
@patch("services.audio_service.AgentRosterService", autospec=True)
@patch("services.audio_service.ModelManager.for_tenant", autospec=True)
def test_transcript_asr_legacy_agent_falls_back_to_app_model_config(
self,
mock_model_manager_class,
mock_roster_service_class,
factory: AudioServiceTestDataFactory,
):
app_model_config = factory.create_app_model_config_mock(speech_to_text_dict={"enabled": True})
app = factory.create_app_mock(mode=AppMode.AGENT, app_model_config=app_model_config)
file = factory.create_file_storage_mock()
mock_roster_service_class.return_value.get_published_agent_soul_for_app.return_value = None
mock_model_instance = MagicMock()
mock_model_instance.invoke_speech2text.return_value = "Legacy Agent transcript"
mock_model_manager_class.return_value.get_default_model_instance.return_value = mock_model_instance
result = AudioService.transcript_asr(app_model=app, file=file)
assert result == {"text": "Legacy Agent transcript"}
@patch("services.audio_service.ModelManager.for_tenant", autospec=True)
def test_transcript_agent_asr_uses_agent_soul_feature(
self, mock_model_manager_class, factory: AudioServiceTestDataFactory
):
app = factory.create_app_mock(mode=AppMode.AGENT)
file = factory.create_file_storage_mock()
agent_soul = AgentSoulConfig.model_validate({"app_features": {"speech_to_text": {"enabled": True}}})
mock_model_instance = MagicMock()
mock_model_instance.invoke_speech2text.return_value = "Agent transcript"
mock_model_manager_class.return_value.get_default_model_instance.return_value = mock_model_instance
result = AudioService.transcript_agent_asr(
app_model=app,
agent_soul=agent_soul,
file=file,
end_user="account-1",
)
assert result == {"text": "Agent transcript"}
mock_model_manager_class.assert_called_once_with(tenant_id=app.tenant_id, user_id="account-1")
@pytest.mark.parametrize(
"agent_soul",
[
AgentSoulConfig(),
AgentSoulConfig.model_validate({"app_features": {"speech_to_text": {"enabled": False}}}),
],
)
def test_transcript_agent_asr_rejects_disabled_feature(
self, factory: AudioServiceTestDataFactory, agent_soul: AgentSoulConfig
):
app = factory.create_app_mock(mode=AppMode.AGENT)
file = factory.create_file_storage_mock()
with pytest.raises(ValueError, match="Speech to text is not enabled"):
AudioService.transcript_agent_asr(app_model=app, agent_soul=agent_soul, file=file)
@patch("services.audio_service.ModelManager.for_tenant", autospec=True)
def test_transcript_agent_asr_preserves_legacy_feature_fallback(
self, mock_model_manager_class, factory: AudioServiceTestDataFactory
):
app_model_config = factory.create_app_model_config_mock(speech_to_text_dict={"enabled": True})
app_model_config.to_dict.return_value = {"speech_to_text": {"enabled": True}}
app = factory.create_app_mock(mode=AppMode.AGENT, app_model_config=app_model_config)
file = factory.create_file_storage_mock()
mock_model_instance = MagicMock()
mock_model_instance.invoke_speech2text.return_value = "Legacy feature transcript"
mock_model_manager_class.return_value.get_default_model_instance.return_value = mock_model_instance
result = AudioService.transcript_agent_asr(
app_model=app,
agent_soul=AgentSoulConfig(),
file=file,
)
assert result == {"text": "Legacy feature transcript"}
def test_transcript_agent_asr_soul_disabled_overrides_legacy_feature(self, factory: AudioServiceTestDataFactory):
app_model_config = factory.create_app_model_config_mock(speech_to_text_dict={"enabled": True})
app_model_config.to_dict.return_value = {"speech_to_text": {"enabled": True}}
app = factory.create_app_mock(mode=AppMode.AGENT, app_model_config=app_model_config)
file = factory.create_file_storage_mock()
agent_soul = AgentSoulConfig.model_validate({"app_features": {"speech_to_text": {"enabled": False}}})
with pytest.raises(ValueError, match="Speech to text is not enabled"):
AudioService.transcript_agent_asr(app_model=app, agent_soul=agent_soul, file=file)
def test_transcript_asr_raises_error_when_feature_disabled_chat_mode(self, factory: AudioServiceTestDataFactory):
"""Test that ASR raises error when speech-to-text is disabled in CHAT mode."""
# Arrange

100
packages/contracts/generated/api/console/agent/orpc.gen.ts
View File

@ -118,6 +118,9 @@ import {
zPostAgentByAgentIdApiEnableResponse,
zPostAgentByAgentIdApiKeysPath,
zPostAgentByAgentIdApiKeysResponse,
zPostAgentByAgentIdAudioToTextBody,
zPostAgentByAgentIdAudioToTextPath,
zPostAgentByAgentIdAudioToTextResponse,
zPostAgentByAgentIdBuildChatFinalizePath,
zPostAgentByAgentIdBuildChatFinalizeResponse,
zPostAgentByAgentIdBuildDraftApplyPath,
@ -270,9 +273,33 @@ export const apiKeys = {
}
/**
* Run a build-draft Agent App turn that asks the agent to push config updates
* Transcribe audio using the current Agent debug configuration
*/
export const post3 = oc
.route({
description: 'Transcribe audio using the current Agent debug configuration',
inputStructure: 'detailed',
method: 'POST',
operationId: 'postAgentByAgentIdAudioToText',
path: '/agent/{agent_id}/audio-to-text',
tags: ['console'],
})
.input(
z.object({
body: zPostAgentByAgentIdAudioToTextBody,
params: zPostAgentByAgentIdAudioToTextPath,
}),
)
.output(zPostAgentByAgentIdAudioToTextResponse)
export const audioToText = {
post: post3,
}
/**
* Run a build-draft Agent App turn that asks the agent to push config updates
*/
export const post4 = oc
.route({
description: 'Run a build-draft Agent App turn that asks the agent to push config updates',
inputStructure: 'detailed',
@ -285,14 +312,14 @@ export const post3 = oc
.output(zPostAgentByAgentIdBuildChatFinalizeResponse)
export const finalize = {
post: post3,
post: post4,
}
export const buildChat = {
finalize,
}
export const post4 = oc
export const post5 = oc
.route({
inputStructure: 'detailed',
method: 'POST',
@ -304,10 +331,10 @@ export const post4 = oc
.output(zPostAgentByAgentIdBuildDraftApplyResponse)
export const apply = {
post: post4,
post: post5,
}
export const post5 = oc
export const post6 = oc
.route({
inputStructure: 'detailed',
method: 'POST',
@ -324,7 +351,7 @@ export const post5 = oc
.output(zPostAgentByAgentIdBuildDraftCheckoutResponse)
export const checkout = {
post: post5,
post: post6,
}
export const delete2 = oc
@ -396,7 +423,7 @@ export const byMessageId = {
/**
* Stop a running Agent App chat message generation
*/
export const post6 = oc
export const post7 = oc
.route({
description: 'Stop a running Agent App chat message generation',
inputStructure: 'detailed',
@ -409,7 +436,7 @@ export const post6 = oc
.output(zPostAgentByAgentIdChatMessagesByTaskIdStopResponse)
export const stop = {
post: post6,
post: post7,
}
export const byTaskId = {
@ -457,7 +484,7 @@ export const candidates = {
get: get7,
}
export const post7 = oc
export const post8 = oc
.route({
inputStructure: 'detailed',
method: 'POST',
@ -474,7 +501,7 @@ export const post7 = oc
.output(zPostAgentByAgentIdComposerValidateResponse)
export const validate = {
post: post7,
post: post8,
}
export const get8 = oc
@ -584,7 +611,7 @@ export const get11 = oc
)
.output(zGetAgentByAgentIdConfigFilesResponse)
export const post8 = oc
export const post9 = oc
.route({
inputStructure: 'detailed',
method: 'POST',
@ -604,7 +631,7 @@ export const post8 = oc
export const files = {
get: get11,
post: post8,
post: post9,
byName,
}
@ -628,7 +655,7 @@ export const manifest = {
get: get12,
}
export const post9 = oc
export const post10 = oc
.route({
inputStructure: 'detailed',
method: 'POST',
@ -647,7 +674,7 @@ export const post9 = oc
.output(zPostAgentByAgentIdConfigSkillsUploadResponse)
export const upload = {
post: post9,
post: post10,
}
export const get13 = oc
@ -802,7 +829,7 @@ export const config = {
skills,
}
export const post10 = oc
export const post11 = oc
.route({
inputStructure: 'detailed',
method: 'POST',
@ -815,10 +842,10 @@ export const post10 = oc
.output(zPostAgentByAgentIdCopyResponse)
export const copy = {
post: post10,
post: post11,
}
export const post11 = oc
export const post12 = oc
.route({
inputStructure: 'detailed',
method: 'POST',
@ -830,7 +857,7 @@ export const post11 = oc
.output(zPostAgentByAgentIdDebugConversationRefreshResponse)
export const refresh = {
post: post11,
post: post12,
}
export const debugConversation = {
@ -962,7 +989,7 @@ export const drive = {
/**
* Update an Agent App's presentation features (opener, follow-up, citations, ...)
*/
export const post12 = oc
export const post13 = oc
.route({
description: 'Update an Agent App\'s presentation features (opener, follow-up, citations, ...)',
inputStructure: 'detailed',
@ -977,13 +1004,13 @@ export const post12 = oc
.output(zPostAgentByAgentIdFeaturesResponse)
export const features = {
post: post12,
post: post13,
}
/**
* Create or update Agent App message feedback
*/
export const post13 = oc
export const post14 = oc
.route({
description: 'Create or update Agent App message feedback',
inputStructure: 'detailed',
@ -998,7 +1025,7 @@ export const post13 = oc
.output(zPostAgentByAgentIdFeedbacksResponse)
export const feedbacks = {
post: post13,
post: post14,
}
/**
@ -1021,7 +1048,7 @@ export const delete5 = oc
/**
* Commit an uploaded file into the Agent App drive under files/<name>
*/
export const post14 = oc
export const post15 = oc
.route({
description: 'Commit an uploaded file into the Agent App drive under files/<name>',
inputStructure: 'detailed',
@ -1036,7 +1063,7 @@ export const post14 = oc
export const files4 = {
delete: delete5,
post: post14,
post: post15,
}
export const get24 = oc
@ -1119,7 +1146,7 @@ export const messages2 = {
byMessageId: byMessageId2,
}
export const post15 = oc
export const post16 = oc
.route({
inputStructure: 'detailed',
method: 'POST',
@ -1131,7 +1158,7 @@ export const post15 = oc
.output(zPostAgentByAgentIdPublishResponse)
export const publish = {
post: post15,
post: post16,
}
/**
@ -1180,7 +1207,7 @@ export const read = {
/**
* Upload one Agent App sandbox file and return a signed download URL
*/
export const post16 = oc
export const post17 = oc
.route({
description: 'Upload one Agent App sandbox file and return a signed download URL',
inputStructure: 'detailed',
@ -1198,7 +1225,7 @@ export const post16 = oc
.output(zPostAgentByAgentIdSandboxFilesUploadResponse)
export const upload2 = {
post: post16,
post: post17,
}
/**
@ -1250,7 +1277,7 @@ export const sandbox = {
/**
* Upload + standardize a Skill into an Agent App drive
*/
export const post17 = oc
export const post18 = oc
.route({
description: 'Upload + standardize a Skill into an Agent App drive',
inputStructure: 'detailed',
@ -1269,13 +1296,13 @@ export const post17 = oc
.output(zPostAgentByAgentIdSkillsUploadResponse)
export const upload3 = {
post: post17,
post: post18,
}
/**
* Infer CLI tool + ENV suggestions from a standardized Agent App skill
*/
export const post18 = oc
export const post19 = oc
.route({
description: 'Infer CLI tool + ENV suggestions from a standardized Agent App skill',
inputStructure: 'detailed',
@ -1288,7 +1315,7 @@ export const post18 = oc
.output(zPostAgentByAgentIdSkillsBySlugInferToolsResponse)
export const inferTools = {
post: post18,
post: post19,
}
/**
@ -1340,7 +1367,7 @@ export const statistics = {
summary,
}
export const post19 = oc
export const post20 = oc
.route({
inputStructure: 'detailed',
method: 'POST',
@ -1352,7 +1379,7 @@ export const post19 = oc
.output(zPostAgentByAgentIdVersionsByVersionIdRestoreResponse)
export const restore = {
post: post19,
post: post20,
}
export const get33 = oc
@ -1428,6 +1455,7 @@ export const byAgentId = {
apiAccess,
apiEnable,
apiKeys,
audioToText,
buildChat,
buildDraft,
chatMessages,
@ -1461,7 +1489,7 @@ export const get36 = oc
.input(z.object({ query: zGetAgentQuery.optional() }))
.output(zGetAgentResponse)
export const post20 = oc
export const post21 = oc
.route({
inputStructure: 'detailed',
method: 'POST',
@ -1475,7 +1503,7 @@ export const post20 = oc
export const agent = {
get: get36,
post: post20,
post: post21,
inviteOptions,
byAgentId,
}

29
packages/contracts/generated/api/console/agent/types.gen.ts
View File

@ -111,6 +111,10 @@ export type ApiKeyItem = {
type: string
}
export type AudioTranscriptResponse = {
text: string
}
export type SimpleResultResponse = {
result: string
}
@ -2160,6 +2164,31 @@ export type DeleteAgentByAgentIdApiKeysByApiKeyIdResponses = {
export type DeleteAgentByAgentIdApiKeysByApiKeyIdResponse
= DeleteAgentByAgentIdApiKeysByApiKeyIdResponses[keyof DeleteAgentByAgentIdApiKeysByApiKeyIdResponses]
export type PostAgentByAgentIdAudioToTextData = {
body: {
draft_type?: 'debug_build' | 'draft'
file: Blob | File
}
path: {
agent_id: string
}
query?: never
url: '/agent/{agent_id}/audio-to-text'
}
export type PostAgentByAgentIdAudioToTextErrors = {
400: unknown
404: unknown
413: unknown
}
export type PostAgentByAgentIdAudioToTextResponses = {
200: AudioTranscriptResponse
}
export type PostAgentByAgentIdAudioToTextResponse
= PostAgentByAgentIdAudioToTextResponses[keyof PostAgentByAgentIdAudioToTextResponses]
export type PostAgentByAgentIdBuildChatFinalizeData = {
body?: never
path: {

21
packages/contracts/generated/api/console/agent/zod.gen.ts
View File

@ -47,6 +47,13 @@ export const zApiKeyList = z.object({
data: z.array(zApiKeyItem),
})
/**
* AudioTranscriptResponse
*/
export const zAudioTranscriptResponse = z.object({
text: z.string(),
})
/**
* SimpleResultResponse
*/
@ -2842,6 +2849,20 @@ export const zDeleteAgentByAgentIdApiKeysByApiKeyIdPath = z.object({
*/
export const zDeleteAgentByAgentIdApiKeysByApiKeyIdResponse = z.void()
export const zPostAgentByAgentIdAudioToTextBody = z.object({
draft_type: z.enum(['debug_build', 'draft']).optional().default('draft'),
file: z.custom<Blob | File>(),
})
export const zPostAgentByAgentIdAudioToTextPath = z.object({
agent_id: z.uuid(),
})
/**
* Audio transcription successful
*/
export const zPostAgentByAgentIdAudioToTextResponse = zAudioTranscriptResponse
export const zPostAgentByAgentIdBuildChatFinalizePath = z.object({
agent_id: z.uuid(),
})

5
packages/contracts/generated/api/console/apps/orpc.gen.ts
View File

@ -342,6 +342,7 @@ import {
zPostAppsByAppIdApiEnableBody,
zPostAppsByAppIdApiEnablePath,
zPostAppsByAppIdApiEnableResponse,
zPostAppsByAppIdAudioToTextBody,
zPostAppsByAppIdAudioToTextPath,
zPostAppsByAppIdAudioToTextResponse,
zPostAppsByAppIdChatMessagesByTaskIdStopPath,
@ -1769,7 +1770,9 @@ export const post20 = oc
path: '/apps/{app_id}/audio-to-text',
tags: ['console'],
})
.input(z.object({ params: zPostAppsByAppIdAudioToTextPath }))
.input(
z.object({ body: zPostAppsByAppIdAudioToTextBody, params: zPostAppsByAppIdAudioToTextPath }),
)
.output(zPostAppsByAppIdAudioToTextResponse)
export const audioToText = {

4
packages/contracts/generated/api/console/apps/types.gen.ts
View File

@ -4357,7 +4357,9 @@ export type PostAppsByAppIdApiEnableResponse
= PostAppsByAppIdApiEnableResponses[keyof PostAppsByAppIdApiEnableResponses]
export type PostAppsByAppIdAudioToTextData = {
body?: never
body: {
file: Blob | File
}
path: {
app_id: string
}

4
packages/contracts/generated/api/console/apps/zod.gen.ts
View File

@ -5094,6 +5094,10 @@ export const zPostAppsByAppIdApiEnablePath = z.object({
*/
export const zPostAppsByAppIdApiEnableResponse = zAppDetail
export const zPostAppsByAppIdAudioToTextBody = z.object({
file: z.custom<Blob | File>(),
})
export const zPostAppsByAppIdAudioToTextPath = z.object({
app_id: z.uuid(),
})