diff --git a/api/controllers/console/app/audio.py b/api/controllers/console/app/audio.py index 0c9ed786a1e..deb560d7114 100644 --- a/api/controllers/console/app/audio.py +++ b/api/controllers/console/app/audio.py @@ -1,13 +1,17 @@ import logging +from uuid import UUID from flask import request from flask_restx import Resource -from pydantic import BaseModel, Field, RootModel -from werkzeug.exceptions import InternalServerError +from pydantic import BaseModel, ConfigDict, Field, RootModel +from sqlalchemy.orm import Session +from werkzeug.datastructures import FileStorage +from werkzeug.exceptions import HTTPException, InternalServerError import services from controllers.common.schema import query_params_from_model, register_response_schema_models, register_schema_models from controllers.console import console_ns +from controllers.console.agent.app_helpers import resolve_agent_runtime_app_model from controllers.console.app.error import ( AppUnavailableError, AudioTooLargeError, @@ -19,13 +23,16 @@ from controllers.console.app.error import ( ProviderQuotaExceededError, UnsupportedAudioTypeError, ) -from controllers.console.app.wraps import get_app_model +from controllers.console.app.wraps import get_app_model, with_session from controllers.console.wraps import ( RBACPermission, RBACResourceScope, account_initialization_required, + edit_permission_required, rbac_permission_required, setup_required, + with_current_tenant_id, + with_current_user, ) from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError from extensions.ext_database import db @@ -33,7 +40,10 @@ from fields.base import ResponseModel from graphon.model_runtime.errors.invoke import InvokeError from libs.helper import dump_response from libs.login import current_user, login_required -from models import App, AppMode +from models import Account, App, AppMode +from models.agent import AgentConfigDraftType +from models.agent_config_entities import AgentSoulConfig +from services.agent.composer_service import AgentComposerService from services.app_ref_service import AppRefService from services.audio_service import AudioService from services.errors.audio import ( @@ -61,6 +71,12 @@ class AudioTranscriptResponse(ResponseModel): text: str = Field(description="Transcribed text from audio") +class AgentAudioTranscriptFormPayload(BaseModel): + model_config = ConfigDict(extra="forbid") + + draft_type: AgentConfigDraftType = AgentConfigDraftType.DRAFT + + class TextToSpeechVoiceResponse(ResponseModel): # see api/core/plugin/impl/model.py name: str = Field(description="Voice display name") @@ -71,7 +87,7 @@ class TextToSpeechVoiceListResponse(RootModel[list[TextToSpeechVoiceResponse]]): root: list[TextToSpeechVoiceResponse] = Field(description="Available voices") -register_schema_models(console_ns, TextToSpeechPayload, TextToSpeechVoiceQuery) +register_schema_models(console_ns, AgentAudioTranscriptFormPayload, TextToSpeechPayload, TextToSpeechVoiceQuery) register_response_schema_models( console_ns, AudioTranscriptResponse, @@ -79,12 +95,82 @@ register_response_schema_models( TextToSpeechVoiceListResponse, ) +_AUDIO_TRANSCRIPT_FILE_PARAM = { + "description": "MP3 audio to transcribe", + "in": "formData", + "type": "file", + "required": True, +} +_AGENT_AUDIO_TRANSCRIPT_PARAMS = { + "file": _AUDIO_TRANSCRIPT_FILE_PARAM, + "draft_type": { + "description": "Agent debug config source", + "in": "formData", + "type": "string", + "enum": [draft_type.value for draft_type in AgentConfigDraftType], + "default": AgentConfigDraftType.DRAFT.value, + "required": False, + }, +} + + +def _transcribe_audio_to_text( + *, + app_model: App, + file: FileStorage | None, + agent_soul: AgentSoulConfig | None = None, +) -> dict[str, str]: + try: + if agent_soul is None: + response = AudioService.transcript_asr( + app_model=app_model, + file=file, + end_user=None, + ) + else: + response = AudioService.transcript_agent_asr( + app_model=app_model, + agent_soul=agent_soul, + file=file, + end_user=None, + ) + return dump_response(AudioTranscriptResponse, response) + except services.errors.app_model_config.AppModelConfigBrokenError: + logger.exception("App model config broken.") + raise AppUnavailableError() + except NoAudioUploadedServiceError: + raise NoAudioUploadedError() + except AudioTooLargeServiceError as e: + raise AudioTooLargeError(str(e)) + except UnsupportedAudioTypeServiceError: + raise UnsupportedAudioTypeError() + except ProviderNotSupportSpeechToTextServiceError: + raise ProviderNotSupportSpeechToTextError() + except ProviderTokenNotInitError as ex: + raise ProviderNotInitializeError(ex.description) + except QuotaExceededError: + raise ProviderQuotaExceededError() + except ModelCurrentlyNotSupportError: + raise ProviderModelCurrentlyNotSupportError() + except InvokeError as e: + raise CompletionRequestError(e.description) + except HTTPException: + raise + except ValueError: + raise + except Exception as e: + logger.exception("Failed to transcribe audio to text") + raise InternalServerError() from e + @console_ns.route("/apps//audio-to-text") class ChatMessageAudioApi(Resource): @console_ns.doc("chat_message_audio_transcript") @console_ns.doc(description="Transcript audio to text for chat messages") - @console_ns.doc(params={"app_id": "App ID"}) + @console_ns.doc( + consumes=["multipart/form-data"], + params={"app_id": "App ID", "file": _AUDIO_TRANSCRIPT_FILE_PARAM}, + ) @console_ns.response( 200, "Audio transcription successful", @@ -97,40 +183,54 @@ class ChatMessageAudioApi(Resource): @account_initialization_required @get_app_model(mode=[AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT]) def post(self, app_model: App): - file = request.files["file"] + return _transcribe_audio_to_text(app_model=app_model, file=request.files.get("file")) - try: - response = AudioService.transcript_asr( - app_model=app_model, - file=file, - end_user=None, - ) - return dump_response(AudioTranscriptResponse, response) - except services.errors.app_model_config.AppModelConfigBrokenError: - logger.exception("App model config broken.") - raise AppUnavailableError() - except NoAudioUploadedServiceError: - raise NoAudioUploadedError() - except AudioTooLargeServiceError as e: - raise AudioTooLargeError(str(e)) - except UnsupportedAudioTypeServiceError: - raise UnsupportedAudioTypeError() - except ProviderNotSupportSpeechToTextServiceError: - raise ProviderNotSupportSpeechToTextError() - except ProviderTokenNotInitError as ex: - raise ProviderNotInitializeError(ex.description) - except QuotaExceededError: - raise ProviderQuotaExceededError() - except ModelCurrentlyNotSupportError: - raise ProviderModelCurrentlyNotSupportError() - except InvokeError as e: - raise CompletionRequestError(e.description) - except ValueError as e: - raise e - except Exception as e: - logger.exception("Failed to handle post request to ChatMessageAudioApi") - raise InternalServerError() +@console_ns.route("/agent//audio-to-text") +class AgentChatMessageAudioApi(Resource): + @console_ns.doc("agent_chat_message_audio_transcript") + @console_ns.doc(description="Transcribe audio using the current Agent debug configuration") + @console_ns.doc( + consumes=["multipart/form-data"], + params={"agent_id": "Agent ID", **_AGENT_AUDIO_TRANSCRIPT_PARAMS}, + ) + @console_ns.response( + 200, + "Audio transcription successful", + console_ns.models[AudioTranscriptResponse.__name__], + ) + @console_ns.response(400, "Bad request - Speech to text disabled or unsupported audio") + @console_ns.response(404, "Agent or build draft not found") + @console_ns.response(413, "Audio file too large") + @setup_required + @login_required + @account_initialization_required + @edit_permission_required + @rbac_permission_required(RBACResourceScope.APP, RBACPermission.APP_TEST_AND_RUN) + @with_current_user + @with_current_tenant_id + @with_session + def post( + self, + session: Session, + current_tenant_id: str, + current_user: Account, + agent_id: UUID, + ): + payload = AgentAudioTranscriptFormPayload.model_validate(request.form.to_dict(flat=True)) + app_model = resolve_agent_runtime_app_model(tenant_id=current_tenant_id, agent_id=agent_id) + agent_soul = AgentComposerService.load_agent_soul_for_debug( + tenant_id=current_tenant_id, + agent_id=str(agent_id), + account_id=current_user.id, + draft_type=payload.draft_type, + session=session, + ) + return _transcribe_audio_to_text( + app_model=app_model, + agent_soul=agent_soul, + file=request.files.get("file"), + ) @console_ns.route("/apps//text-to-audio") diff --git a/api/services/agent/composer_service.py b/api/services/agent/composer_service.py index 4839012dd3a..3f4c50a794d 100644 --- a/api/services/agent/composer_service.py +++ b/api/services/agent/composer_service.py @@ -358,6 +358,32 @@ class AgentComposerService: agent = cls._require_agent(tenant_id=tenant_id, agent_id=agent_id, session=session) return cls._load_agent_composer_for_agent(tenant_id=tenant_id, agent=agent, session=session) + @classmethod + def load_agent_soul_for_debug( + cls, + *, + tenant_id: str, + agent_id: str, + account_id: str, + draft_type: AgentConfigDraftType, + session: Session, + ) -> AgentSoulConfig: + """Load the same normal or account-owned build draft used by Agent debug chat.""" + if draft_type == AgentConfigDraftType.DEBUG_BUILD: + state = cls.load_agent_app_build_draft( + tenant_id=tenant_id, + agent_id=agent_id, + account_id=account_id, + session=session, + ) + else: + state = cls.load_agent_composer( + tenant_id=tenant_id, + agent_id=agent_id, + session=session, + ) + return AgentSoulConfig.model_validate(state["agent_soul"]) + @classmethod def _load_agent_composer_for_agent(cls, *, tenant_id: str, agent: Agent, session: Session) -> dict[str, Any]: draft = cls._get_or_create_agent_draft( diff --git a/api/services/agent/roster_service.py b/api/services/agent/roster_service.py index 33fd372398b..7758b81e59a 100644 --- a/api/services/agent/roster_service.py +++ b/api/services/agent/roster_service.py @@ -798,6 +798,18 @@ class AgentRosterService: ) ) + def get_published_agent_soul_for_app(self, *, tenant_id: str, app_id: str) -> AgentSoulConfig | None: + """Return the active Agent Soul used by a published Agent App runtime.""" + agent = self.get_app_backing_agent(tenant_id=tenant_id, app_id=app_id) + if agent is None: + return None + version = self._get_version( + tenant_id=tenant_id, + agent_id=agent.id, + version_id=agent.active_config_snapshot_id, + ) + return AgentSoulConfig.model_validate(version.config_snapshot_dict) + def get_agent_app_model(self, *, tenant_id: str, agent_id: str) -> App: """Resolve the Agent App hidden behind an app-backed Agent id. diff --git a/api/services/audio_service.py b/api/services/audio_service.py index 52c71edd576..c094e8d8f2d 100644 --- a/api/services/audio_service.py +++ b/api/services/audio_service.py @@ -10,10 +10,14 @@ from sqlalchemy.orm import Session from werkzeug.datastructures import FileStorage from constants import AUDIO_EXTENSIONS +from core.app.apps.agent_app.app_feature_projection import merge_agent_app_features from core.model_manager import ModelManager +from extensions.ext_database import db from graphon.model_runtime.entities.model_entities import ModelType +from models.agent_config_entities import AgentSoulConfig from models.enums import MessageStatus from models.model import App, AppMode, Message +from services.agent.roster_service import AgentRosterService from services.app_ref_service import MessageRef from services.errors.audio import ( AudioTooLargeServiceError, @@ -41,7 +45,20 @@ class AudioService: return session.scalar(stmt.limit(1)) @classmethod - def transcript_asr(cls, app_model: App, file: FileStorage | None, end_user: str | None = None): + def transcript_asr(cls, app_model: App, file: FileStorage | None, end_user: str | None = None) -> dict[str, str]: + if app_model.mode == AppMode.AGENT: + agent_soul = AgentRosterService(db.session).get_published_agent_soul_for_app( + tenant_id=app_model.tenant_id, + app_id=app_model.id, + ) + if agent_soul is not None: + return cls.transcript_agent_asr( + app_model=app_model, + agent_soul=agent_soul, + file=file, + end_user=end_user, + ) + if app_model.mode in {AppMode.ADVANCED_CHAT, AppMode.WORKFLOW}: workflow = app_model.workflow if workflow is None: @@ -58,6 +75,30 @@ class AudioService: if not app_model_config.speech_to_text_dict["enabled"]: raise ValueError("Speech to text is not enabled") + return cls._invoke_speech_to_text(app_model=app_model, file=file, end_user=end_user) + + @classmethod + def transcript_agent_asr( + cls, + app_model: App, + agent_soul: AgentSoulConfig, + file: FileStorage | None, + end_user: str | None = None, + ) -> dict[str, str]: + """Transcribe Agent audio after applying the Agent runtime feature projection.""" + features = merge_agent_app_features( + agent_soul=agent_soul, + app_model_config=app_model.app_model_config, + ) + if not features.get("speech_to_text", {}).get("enabled"): + raise ValueError("Speech to text is not enabled") + + return cls._invoke_speech_to_text(app_model=app_model, file=file, end_user=end_user) + + @classmethod + def _invoke_speech_to_text( + cls, app_model: App, file: FileStorage | None, end_user: str | None = None + ) -> dict[str, str]: if file is None: raise NoAudioUploadedServiceError() diff --git a/api/tests/unit_tests/controllers/console/app/test_audio.py b/api/tests/unit_tests/controllers/console/app/test_audio.py index 138683169c7..36a0e913612 100644 --- a/api/tests/unit_tests/controllers/console/app/test_audio.py +++ b/api/tests/unit_tests/controllers/console/app/test_audio.py @@ -4,13 +4,20 @@ import io from inspect import unwrap from types import SimpleNamespace from unittest.mock import patch +from uuid import UUID import pytest from flask import Flask from werkzeug.datastructures import FileStorage from werkzeug.exceptions import InternalServerError -from controllers.console.app.audio import ChatMessageAudioApi, ChatMessageTextApi, TextModesApi +from controllers.console.app import audio as audio_module +from controllers.console.app.audio import ( + AgentChatMessageAudioApi, + ChatMessageAudioApi, + ChatMessageTextApi, + TextModesApi, +) from controllers.console.app.error import ( AppUnavailableError, AudioTooLargeError, @@ -24,6 +31,10 @@ from controllers.console.app.error import ( ) from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError from graphon.model_runtime.errors.invoke import InvokeError +from models.agent import AgentConfigDraftType +from models.agent_config_entities import AgentSoulConfig +from services.agent.composer_service import AgentComposerService +from services.agent.errors import AgentVersionNotFoundError from services.app_ref_service import MessageRef from services.audio_service import AudioService from services.errors.app_model_config import AppModelConfigBrokenError @@ -52,6 +63,127 @@ def test_console_audio_api_success(app: Flask, monkeypatch: pytest.MonkeyPatch) assert response == {"text": "ok"} +def test_agent_console_audio_api_uses_agent_draft(app: Flask, monkeypatch: pytest.MonkeyPatch) -> None: + agent_id = UUID("019ef3d2-b24c-7803-b428-18b5ee8fb853") + app_model = SimpleNamespace(id="backing-app-1") + agent_soul = AgentSoulConfig.model_validate({"app_features": {"speech_to_text": {"enabled": True}}}) + calls: dict[str, object] = {} + + def resolve_agent_runtime_app_model(**kwargs): + calls["resolver"] = kwargs + return app_model + + def load_agent_soul_for_debug(**kwargs): + calls["draft"] = kwargs + return agent_soul + + def transcript_agent_asr(**kwargs): + calls["asr"] = kwargs + return {"text": "agent transcript"} + + monkeypatch.setattr(audio_module, "resolve_agent_runtime_app_model", resolve_agent_runtime_app_model) + monkeypatch.setattr(AgentComposerService, "load_agent_soul_for_debug", load_agent_soul_for_debug) + monkeypatch.setattr(AudioService, "transcript_agent_asr", transcript_agent_asr) + + api = AgentChatMessageAudioApi() + handler = unwrap(api.post) + session = SimpleNamespace() + current_user = SimpleNamespace(id="account-1") + with app.test_request_context( + f"/console/api/agent/{agent_id}/audio-to-text", + method="POST", + data={"file": _file_data(), "draft_type": "debug_build"}, + ): + response = handler( + api, + session=session, + current_tenant_id="tenant-1", + current_user=current_user, + agent_id=agent_id, + ) + + assert response == {"text": "agent transcript"} + assert calls["resolver"] == {"tenant_id": "tenant-1", "agent_id": agent_id} + assert calls["draft"] == { + "tenant_id": "tenant-1", + "agent_id": str(agent_id), + "account_id": "account-1", + "draft_type": AgentConfigDraftType.DEBUG_BUILD, + "session": session, + } + assert calls["asr"] == { + "app_model": app_model, + "agent_soul": agent_soul, + "file": calls["asr"]["file"], + "end_user": None, + } + + +def test_agent_console_audio_api_defaults_to_normal_draft(app: Flask, monkeypatch: pytest.MonkeyPatch) -> None: + agent_id = UUID("019ef3d2-b24c-7803-b428-18b5ee8fb853") + captured: dict[str, object] = {} + monkeypatch.setattr( + audio_module, + "resolve_agent_runtime_app_model", + lambda **_kwargs: SimpleNamespace(id="backing-app-1"), + ) + + def load_agent_soul_for_debug(**kwargs): + captured.update(kwargs) + return AgentSoulConfig.model_validate({"app_features": {"speech_to_text": {"enabled": True}}}) + + monkeypatch.setattr(AgentComposerService, "load_agent_soul_for_debug", load_agent_soul_for_debug) + monkeypatch.setattr(AudioService, "transcript_agent_asr", lambda **_kwargs: {"text": "ok"}) + + api = AgentChatMessageAudioApi() + handler = unwrap(api.post) + with app.test_request_context( + f"/console/api/agent/{agent_id}/audio-to-text", + method="POST", + data={"file": _file_data()}, + ): + response = handler( + api, + session=SimpleNamespace(), + current_tenant_id="tenant-1", + current_user=SimpleNamespace(id="account-1"), + agent_id=agent_id, + ) + + assert response == {"text": "ok"} + assert captured["draft_type"] == AgentConfigDraftType.DRAFT + + +def test_agent_console_audio_api_preserves_missing_build_draft_404(app: Flask, monkeypatch: pytest.MonkeyPatch) -> None: + agent_id = UUID("019ef3d2-b24c-7803-b428-18b5ee8fb853") + monkeypatch.setattr( + audio_module, + "resolve_agent_runtime_app_model", + lambda **_kwargs: SimpleNamespace(id="backing-app-1"), + ) + monkeypatch.setattr( + AgentComposerService, + "load_agent_soul_for_debug", + lambda **_kwargs: (_ for _ in ()).throw(AgentVersionNotFoundError()), + ) + + api = AgentChatMessageAudioApi() + handler = unwrap(api.post) + with app.test_request_context( + f"/console/api/agent/{agent_id}/audio-to-text", + method="POST", + data={"file": _file_data(), "draft_type": "debug_build"}, + ): + with pytest.raises(AgentVersionNotFoundError): + handler( + api, + session=SimpleNamespace(), + current_tenant_id="tenant-1", + current_user=SimpleNamespace(id="account-1"), + agent_id=agent_id, + ) + + @pytest.mark.parametrize( ("exc", "expected"), [ diff --git a/api/tests/unit_tests/services/agent/test_agent_services.py b/api/tests/unit_tests/services/agent/test_agent_services.py index 22c3f4c5500..2f80599689d 100644 --- a/api/tests/unit_tests/services/agent/test_agent_services.py +++ b/api/tests/unit_tests/services/agent/test_agent_services.py @@ -116,6 +116,25 @@ def test_agent_soul_has_model(): assert agent_soul_has_model(AgentSoulConfig()) is False +def test_get_published_agent_soul_for_app_uses_active_snapshot(): + agent_soul = AgentSoulConfig.model_validate({"app_features": {"speech_to_text": {"enabled": True}}}) + agent = SimpleNamespace(id="agent-1", active_config_snapshot_id="version-1") + version = SimpleNamespace(config_snapshot_dict=agent_soul.model_dump(mode="json")) + service = AgentRosterService(FakeSession(scalar=[agent, version])) + + result = service.get_published_agent_soul_for_app(tenant_id="tenant-1", app_id="app-1") + + assert result == agent_soul + + +def test_get_published_agent_soul_for_app_returns_none_without_backing_agent(): + service = AgentRosterService(FakeSession(scalar=[None])) + + result = service.get_published_agent_soul_for_app(tenant_id="tenant-1", app_id="legacy-app-1") + + assert result is None + + def test_load_workflow_composer_returns_empty_state(monkeypatch: pytest.MonkeyPatch): monkeypatch.setattr(AgentComposerService, "_get_draft_workflow", lambda **kwargs: SimpleNamespace(id="workflow-1")) monkeypatch.setattr(AgentComposerService, "_get_workflow_binding", lambda **kwargs: None) @@ -632,7 +651,7 @@ def test_publish_agent_app_draft_rejects_missing_model(monkeypatch: pytest.Monke base_snapshot_id="version-1", config_snapshot=AgentSoulConfig(), ) - fake_session = FakeSession(scalar=[agent, draft]) + fake_session = FakeSession(scalar=[agent, draft, None]) def fail_create_config_version(**_kwargs): raise AssertionError("config version must not be created when Agent Soul has no model") @@ -777,6 +796,74 @@ def test_agent_app_build_draft_checkout_and_apply_use_user_isolated_draft(monkey assert fake_session.commits == 1 +@pytest.mark.parametrize( + ("draft_type", "account_id"), + [ + (AgentConfigDraftType.DRAFT, None), + (AgentConfigDraftType.DEBUG_BUILD, "account-1"), + ], +) +def test_load_agent_soul_for_debug_selects_requested_draft( + draft_type: AgentConfigDraftType, + account_id: str | None, +): + agent = Agent( + id="agent-1", + tenant_id="tenant-1", + name="Iris", + description="", + agent_kind=AgentKind.DIFY_AGENT, + scope=AgentScope.ROSTER, + source=AgentSource.AGENT_APP, + status=AgentStatus.ACTIVE, + ) + agent_soul = AgentSoulConfig.model_validate({"app_features": {"speech_to_text": {"enabled": True}}}) + draft = AgentConfigDraft( + tenant_id="tenant-1", + agent_id="agent-1", + draft_type=draft_type, + account_id=account_id, + draft_owner_key=account_id or "", + config_snapshot=agent_soul, + ) + fake_session = FakeSession( + scalar=[draft] if draft_type == AgentConfigDraftType.DEBUG_BUILD else [agent, draft, None] + ) + + result = AgentComposerService.load_agent_soul_for_debug( + tenant_id="tenant-1", + agent_id="agent-1", + account_id="account-1", + draft_type=draft_type, + session=fake_session, + ) + + assert result == agent_soul + + +def test_load_agent_soul_for_debug_requires_existing_build_draft(): + agent = Agent( + id="agent-1", + tenant_id="tenant-1", + name="Iris", + description="", + agent_kind=AgentKind.DIFY_AGENT, + scope=AgentScope.ROSTER, + source=AgentSource.AGENT_APP, + status=AgentStatus.ACTIVE, + ) + fake_session = FakeSession(scalar=[None]) + + with pytest.raises(AgentVersionNotFoundError): + AgentComposerService.load_agent_soul_for_debug( + tenant_id="tenant-1", + agent_id="agent-1", + account_id="account-1", + draft_type=AgentConfigDraftType.DEBUG_BUILD, + session=fake_session, + ) + + def test_agent_app_build_draft_apply_marks_unpublished_when_build_draft_differs(monkeypatch: pytest.MonkeyPatch): agent = Agent( id="agent-1", diff --git a/api/tests/unit_tests/services/test_audio_service.py b/api/tests/unit_tests/services/test_audio_service.py index bd865dd5dd3..f30cd2b37e5 100644 --- a/api/tests/unit_tests/services/test_audio_service.py +++ b/api/tests/unit_tests/services/test_audio_service.py @@ -59,6 +59,7 @@ from unittest.mock import MagicMock, Mock, create_autospec, patch import pytest from werkzeug.datastructures import FileStorage +from models.agent_config_entities import AgentSoulConfig from models.enums import MessageStatus from models.model import App, AppMode, AppModelConfig, Message from models.workflow import Workflow @@ -267,6 +268,117 @@ class TestAudioServiceASR: # Assert assert result == {"text": "Workflow transcribed text"} + @patch("services.audio_service.AgentRosterService", autospec=True) + @patch("services.audio_service.ModelManager.for_tenant", autospec=True) + def test_transcript_asr_success_published_agent_mode( + self, + mock_model_manager_class, + mock_roster_service_class, + factory: AudioServiceTestDataFactory, + ): + app = factory.create_app_mock(mode=AppMode.AGENT) + file = factory.create_file_storage_mock() + agent_soul = AgentSoulConfig.model_validate({"app_features": {"speech_to_text": {"enabled": True}}}) + mock_roster_service_class.return_value.get_published_agent_soul_for_app.return_value = agent_soul + mock_model_instance = MagicMock() + mock_model_instance.invoke_speech2text.return_value = "Published Agent transcript" + mock_model_manager_class.return_value.get_default_model_instance.return_value = mock_model_instance + + result = AudioService.transcript_asr(app_model=app, file=file, end_user="end-user-1") + + assert result == {"text": "Published Agent transcript"} + mock_roster_service_class.return_value.get_published_agent_soul_for_app.assert_called_once_with( + tenant_id=app.tenant_id, + app_id=app.id, + ) + + @patch("services.audio_service.AgentRosterService", autospec=True) + @patch("services.audio_service.ModelManager.for_tenant", autospec=True) + def test_transcript_asr_legacy_agent_falls_back_to_app_model_config( + self, + mock_model_manager_class, + mock_roster_service_class, + factory: AudioServiceTestDataFactory, + ): + app_model_config = factory.create_app_model_config_mock(speech_to_text_dict={"enabled": True}) + app = factory.create_app_mock(mode=AppMode.AGENT, app_model_config=app_model_config) + file = factory.create_file_storage_mock() + mock_roster_service_class.return_value.get_published_agent_soul_for_app.return_value = None + mock_model_instance = MagicMock() + mock_model_instance.invoke_speech2text.return_value = "Legacy Agent transcript" + mock_model_manager_class.return_value.get_default_model_instance.return_value = mock_model_instance + + result = AudioService.transcript_asr(app_model=app, file=file) + + assert result == {"text": "Legacy Agent transcript"} + + @patch("services.audio_service.ModelManager.for_tenant", autospec=True) + def test_transcript_agent_asr_uses_agent_soul_feature( + self, mock_model_manager_class, factory: AudioServiceTestDataFactory + ): + app = factory.create_app_mock(mode=AppMode.AGENT) + file = factory.create_file_storage_mock() + agent_soul = AgentSoulConfig.model_validate({"app_features": {"speech_to_text": {"enabled": True}}}) + mock_model_instance = MagicMock() + mock_model_instance.invoke_speech2text.return_value = "Agent transcript" + mock_model_manager_class.return_value.get_default_model_instance.return_value = mock_model_instance + + result = AudioService.transcript_agent_asr( + app_model=app, + agent_soul=agent_soul, + file=file, + end_user="account-1", + ) + + assert result == {"text": "Agent transcript"} + mock_model_manager_class.assert_called_once_with(tenant_id=app.tenant_id, user_id="account-1") + + @pytest.mark.parametrize( + "agent_soul", + [ + AgentSoulConfig(), + AgentSoulConfig.model_validate({"app_features": {"speech_to_text": {"enabled": False}}}), + ], + ) + def test_transcript_agent_asr_rejects_disabled_feature( + self, factory: AudioServiceTestDataFactory, agent_soul: AgentSoulConfig + ): + app = factory.create_app_mock(mode=AppMode.AGENT) + file = factory.create_file_storage_mock() + + with pytest.raises(ValueError, match="Speech to text is not enabled"): + AudioService.transcript_agent_asr(app_model=app, agent_soul=agent_soul, file=file) + + @patch("services.audio_service.ModelManager.for_tenant", autospec=True) + def test_transcript_agent_asr_preserves_legacy_feature_fallback( + self, mock_model_manager_class, factory: AudioServiceTestDataFactory + ): + app_model_config = factory.create_app_model_config_mock(speech_to_text_dict={"enabled": True}) + app_model_config.to_dict.return_value = {"speech_to_text": {"enabled": True}} + app = factory.create_app_mock(mode=AppMode.AGENT, app_model_config=app_model_config) + file = factory.create_file_storage_mock() + mock_model_instance = MagicMock() + mock_model_instance.invoke_speech2text.return_value = "Legacy feature transcript" + mock_model_manager_class.return_value.get_default_model_instance.return_value = mock_model_instance + + result = AudioService.transcript_agent_asr( + app_model=app, + agent_soul=AgentSoulConfig(), + file=file, + ) + + assert result == {"text": "Legacy feature transcript"} + + def test_transcript_agent_asr_soul_disabled_overrides_legacy_feature(self, factory: AudioServiceTestDataFactory): + app_model_config = factory.create_app_model_config_mock(speech_to_text_dict={"enabled": True}) + app_model_config.to_dict.return_value = {"speech_to_text": {"enabled": True}} + app = factory.create_app_mock(mode=AppMode.AGENT, app_model_config=app_model_config) + file = factory.create_file_storage_mock() + agent_soul = AgentSoulConfig.model_validate({"app_features": {"speech_to_text": {"enabled": False}}}) + + with pytest.raises(ValueError, match="Speech to text is not enabled"): + AudioService.transcript_agent_asr(app_model=app, agent_soul=agent_soul, file=file) + def test_transcript_asr_raises_error_when_feature_disabled_chat_mode(self, factory: AudioServiceTestDataFactory): """Test that ASR raises error when speech-to-text is disabled in CHAT mode.""" # Arrange diff --git a/packages/contracts/generated/api/console/agent/orpc.gen.ts b/packages/contracts/generated/api/console/agent/orpc.gen.ts index e6249427ea0..f968aa64601 100644 --- a/packages/contracts/generated/api/console/agent/orpc.gen.ts +++ b/packages/contracts/generated/api/console/agent/orpc.gen.ts @@ -118,6 +118,9 @@ import { zPostAgentByAgentIdApiEnableResponse, zPostAgentByAgentIdApiKeysPath, zPostAgentByAgentIdApiKeysResponse, + zPostAgentByAgentIdAudioToTextBody, + zPostAgentByAgentIdAudioToTextPath, + zPostAgentByAgentIdAudioToTextResponse, zPostAgentByAgentIdBuildChatFinalizePath, zPostAgentByAgentIdBuildChatFinalizeResponse, zPostAgentByAgentIdBuildDraftApplyPath, @@ -270,9 +273,33 @@ export const apiKeys = { } /** - * Run a build-draft Agent App turn that asks the agent to push config updates + * Transcribe audio using the current Agent debug configuration */ export const post3 = oc + .route({ + description: 'Transcribe audio using the current Agent debug configuration', + inputStructure: 'detailed', + method: 'POST', + operationId: 'postAgentByAgentIdAudioToText', + path: '/agent/{agent_id}/audio-to-text', + tags: ['console'], + }) + .input( + z.object({ + body: zPostAgentByAgentIdAudioToTextBody, + params: zPostAgentByAgentIdAudioToTextPath, + }), + ) + .output(zPostAgentByAgentIdAudioToTextResponse) + +export const audioToText = { + post: post3, +} + +/** + * Run a build-draft Agent App turn that asks the agent to push config updates + */ +export const post4 = oc .route({ description: 'Run a build-draft Agent App turn that asks the agent to push config updates', inputStructure: 'detailed', @@ -285,14 +312,14 @@ export const post3 = oc .output(zPostAgentByAgentIdBuildChatFinalizeResponse) export const finalize = { - post: post3, + post: post4, } export const buildChat = { finalize, } -export const post4 = oc +export const post5 = oc .route({ inputStructure: 'detailed', method: 'POST', @@ -304,10 +331,10 @@ export const post4 = oc .output(zPostAgentByAgentIdBuildDraftApplyResponse) export const apply = { - post: post4, + post: post5, } -export const post5 = oc +export const post6 = oc .route({ inputStructure: 'detailed', method: 'POST', @@ -324,7 +351,7 @@ export const post5 = oc .output(zPostAgentByAgentIdBuildDraftCheckoutResponse) export const checkout = { - post: post5, + post: post6, } export const delete2 = oc @@ -396,7 +423,7 @@ export const byMessageId = { /** * Stop a running Agent App chat message generation */ -export const post6 = oc +export const post7 = oc .route({ description: 'Stop a running Agent App chat message generation', inputStructure: 'detailed', @@ -409,7 +436,7 @@ export const post6 = oc .output(zPostAgentByAgentIdChatMessagesByTaskIdStopResponse) export const stop = { - post: post6, + post: post7, } export const byTaskId = { @@ -457,7 +484,7 @@ export const candidates = { get: get7, } -export const post7 = oc +export const post8 = oc .route({ inputStructure: 'detailed', method: 'POST', @@ -474,7 +501,7 @@ export const post7 = oc .output(zPostAgentByAgentIdComposerValidateResponse) export const validate = { - post: post7, + post: post8, } export const get8 = oc @@ -584,7 +611,7 @@ export const get11 = oc ) .output(zGetAgentByAgentIdConfigFilesResponse) -export const post8 = oc +export const post9 = oc .route({ inputStructure: 'detailed', method: 'POST', @@ -604,7 +631,7 @@ export const post8 = oc export const files = { get: get11, - post: post8, + post: post9, byName, } @@ -628,7 +655,7 @@ export const manifest = { get: get12, } -export const post9 = oc +export const post10 = oc .route({ inputStructure: 'detailed', method: 'POST', @@ -647,7 +674,7 @@ export const post9 = oc .output(zPostAgentByAgentIdConfigSkillsUploadResponse) export const upload = { - post: post9, + post: post10, } export const get13 = oc @@ -802,7 +829,7 @@ export const config = { skills, } -export const post10 = oc +export const post11 = oc .route({ inputStructure: 'detailed', method: 'POST', @@ -815,10 +842,10 @@ export const post10 = oc .output(zPostAgentByAgentIdCopyResponse) export const copy = { - post: post10, + post: post11, } -export const post11 = oc +export const post12 = oc .route({ inputStructure: 'detailed', method: 'POST', @@ -830,7 +857,7 @@ export const post11 = oc .output(zPostAgentByAgentIdDebugConversationRefreshResponse) export const refresh = { - post: post11, + post: post12, } export const debugConversation = { @@ -962,7 +989,7 @@ export const drive = { /** * Update an Agent App's presentation features (opener, follow-up, citations, ...) */ -export const post12 = oc +export const post13 = oc .route({ description: 'Update an Agent App\'s presentation features (opener, follow-up, citations, ...)', inputStructure: 'detailed', @@ -977,13 +1004,13 @@ export const post12 = oc .output(zPostAgentByAgentIdFeaturesResponse) export const features = { - post: post12, + post: post13, } /** * Create or update Agent App message feedback */ -export const post13 = oc +export const post14 = oc .route({ description: 'Create or update Agent App message feedback', inputStructure: 'detailed', @@ -998,7 +1025,7 @@ export const post13 = oc .output(zPostAgentByAgentIdFeedbacksResponse) export const feedbacks = { - post: post13, + post: post14, } /** @@ -1021,7 +1048,7 @@ export const delete5 = oc /** * Commit an uploaded file into the Agent App drive under files/ */ -export const post14 = oc +export const post15 = oc .route({ description: 'Commit an uploaded file into the Agent App drive under files/', inputStructure: 'detailed', @@ -1036,7 +1063,7 @@ export const post14 = oc export const files4 = { delete: delete5, - post: post14, + post: post15, } export const get24 = oc @@ -1119,7 +1146,7 @@ export const messages2 = { byMessageId: byMessageId2, } -export const post15 = oc +export const post16 = oc .route({ inputStructure: 'detailed', method: 'POST', @@ -1131,7 +1158,7 @@ export const post15 = oc .output(zPostAgentByAgentIdPublishResponse) export const publish = { - post: post15, + post: post16, } /** @@ -1180,7 +1207,7 @@ export const read = { /** * Upload one Agent App sandbox file and return a signed download URL */ -export const post16 = oc +export const post17 = oc .route({ description: 'Upload one Agent App sandbox file and return a signed download URL', inputStructure: 'detailed', @@ -1198,7 +1225,7 @@ export const post16 = oc .output(zPostAgentByAgentIdSandboxFilesUploadResponse) export const upload2 = { - post: post16, + post: post17, } /** @@ -1250,7 +1277,7 @@ export const sandbox = { /** * Upload + standardize a Skill into an Agent App drive */ -export const post17 = oc +export const post18 = oc .route({ description: 'Upload + standardize a Skill into an Agent App drive', inputStructure: 'detailed', @@ -1269,13 +1296,13 @@ export const post17 = oc .output(zPostAgentByAgentIdSkillsUploadResponse) export const upload3 = { - post: post17, + post: post18, } /** * Infer CLI tool + ENV suggestions from a standardized Agent App skill */ -export const post18 = oc +export const post19 = oc .route({ description: 'Infer CLI tool + ENV suggestions from a standardized Agent App skill', inputStructure: 'detailed', @@ -1288,7 +1315,7 @@ export const post18 = oc .output(zPostAgentByAgentIdSkillsBySlugInferToolsResponse) export const inferTools = { - post: post18, + post: post19, } /** @@ -1340,7 +1367,7 @@ export const statistics = { summary, } -export const post19 = oc +export const post20 = oc .route({ inputStructure: 'detailed', method: 'POST', @@ -1352,7 +1379,7 @@ export const post19 = oc .output(zPostAgentByAgentIdVersionsByVersionIdRestoreResponse) export const restore = { - post: post19, + post: post20, } export const get33 = oc @@ -1428,6 +1455,7 @@ export const byAgentId = { apiAccess, apiEnable, apiKeys, + audioToText, buildChat, buildDraft, chatMessages, @@ -1461,7 +1489,7 @@ export const get36 = oc .input(z.object({ query: zGetAgentQuery.optional() })) .output(zGetAgentResponse) -export const post20 = oc +export const post21 = oc .route({ inputStructure: 'detailed', method: 'POST', @@ -1475,7 +1503,7 @@ export const post20 = oc export const agent = { get: get36, - post: post20, + post: post21, inviteOptions, byAgentId, } diff --git a/packages/contracts/generated/api/console/agent/types.gen.ts b/packages/contracts/generated/api/console/agent/types.gen.ts index 3f3bee9fdd2..4754c12a73d 100644 --- a/packages/contracts/generated/api/console/agent/types.gen.ts +++ b/packages/contracts/generated/api/console/agent/types.gen.ts @@ -111,6 +111,10 @@ export type ApiKeyItem = { type: string } +export type AudioTranscriptResponse = { + text: string +} + export type SimpleResultResponse = { result: string } @@ -2160,6 +2164,31 @@ export type DeleteAgentByAgentIdApiKeysByApiKeyIdResponses = { export type DeleteAgentByAgentIdApiKeysByApiKeyIdResponse = DeleteAgentByAgentIdApiKeysByApiKeyIdResponses[keyof DeleteAgentByAgentIdApiKeysByApiKeyIdResponses] +export type PostAgentByAgentIdAudioToTextData = { + body: { + draft_type?: 'debug_build' | 'draft' + file: Blob | File + } + path: { + agent_id: string + } + query?: never + url: '/agent/{agent_id}/audio-to-text' +} + +export type PostAgentByAgentIdAudioToTextErrors = { + 400: unknown + 404: unknown + 413: unknown +} + +export type PostAgentByAgentIdAudioToTextResponses = { + 200: AudioTranscriptResponse +} + +export type PostAgentByAgentIdAudioToTextResponse + = PostAgentByAgentIdAudioToTextResponses[keyof PostAgentByAgentIdAudioToTextResponses] + export type PostAgentByAgentIdBuildChatFinalizeData = { body?: never path: { diff --git a/packages/contracts/generated/api/console/agent/zod.gen.ts b/packages/contracts/generated/api/console/agent/zod.gen.ts index d62307a61d0..9e7c5457d99 100644 --- a/packages/contracts/generated/api/console/agent/zod.gen.ts +++ b/packages/contracts/generated/api/console/agent/zod.gen.ts @@ -47,6 +47,13 @@ export const zApiKeyList = z.object({ data: z.array(zApiKeyItem), }) +/** + * AudioTranscriptResponse + */ +export const zAudioTranscriptResponse = z.object({ + text: z.string(), +}) + /** * SimpleResultResponse */ @@ -2842,6 +2849,20 @@ export const zDeleteAgentByAgentIdApiKeysByApiKeyIdPath = z.object({ */ export const zDeleteAgentByAgentIdApiKeysByApiKeyIdResponse = z.void() +export const zPostAgentByAgentIdAudioToTextBody = z.object({ + draft_type: z.enum(['debug_build', 'draft']).optional().default('draft'), + file: z.custom(), +}) + +export const zPostAgentByAgentIdAudioToTextPath = z.object({ + agent_id: z.uuid(), +}) + +/** + * Audio transcription successful + */ +export const zPostAgentByAgentIdAudioToTextResponse = zAudioTranscriptResponse + export const zPostAgentByAgentIdBuildChatFinalizePath = z.object({ agent_id: z.uuid(), }) diff --git a/packages/contracts/generated/api/console/apps/orpc.gen.ts b/packages/contracts/generated/api/console/apps/orpc.gen.ts index 2b0d927e72d..a5a2de6e390 100644 --- a/packages/contracts/generated/api/console/apps/orpc.gen.ts +++ b/packages/contracts/generated/api/console/apps/orpc.gen.ts @@ -342,6 +342,7 @@ import { zPostAppsByAppIdApiEnableBody, zPostAppsByAppIdApiEnablePath, zPostAppsByAppIdApiEnableResponse, + zPostAppsByAppIdAudioToTextBody, zPostAppsByAppIdAudioToTextPath, zPostAppsByAppIdAudioToTextResponse, zPostAppsByAppIdChatMessagesByTaskIdStopPath, @@ -1769,7 +1770,9 @@ export const post20 = oc path: '/apps/{app_id}/audio-to-text', tags: ['console'], }) - .input(z.object({ params: zPostAppsByAppIdAudioToTextPath })) + .input( + z.object({ body: zPostAppsByAppIdAudioToTextBody, params: zPostAppsByAppIdAudioToTextPath }), + ) .output(zPostAppsByAppIdAudioToTextResponse) export const audioToText = { diff --git a/packages/contracts/generated/api/console/apps/types.gen.ts b/packages/contracts/generated/api/console/apps/types.gen.ts index 7bf3531d1ac..da184bf896c 100644 --- a/packages/contracts/generated/api/console/apps/types.gen.ts +++ b/packages/contracts/generated/api/console/apps/types.gen.ts @@ -4357,7 +4357,9 @@ export type PostAppsByAppIdApiEnableResponse = PostAppsByAppIdApiEnableResponses[keyof PostAppsByAppIdApiEnableResponses] export type PostAppsByAppIdAudioToTextData = { - body?: never + body: { + file: Blob | File + } path: { app_id: string } diff --git a/packages/contracts/generated/api/console/apps/zod.gen.ts b/packages/contracts/generated/api/console/apps/zod.gen.ts index 8e05a59f700..0fbf6d6198c 100644 --- a/packages/contracts/generated/api/console/apps/zod.gen.ts +++ b/packages/contracts/generated/api/console/apps/zod.gen.ts @@ -5094,6 +5094,10 @@ export const zPostAppsByAppIdApiEnablePath = z.object({ */ export const zPostAppsByAppIdApiEnableResponse = zAppDetail +export const zPostAppsByAppIdAudioToTextBody = z.object({ + file: z.custom(), +}) + export const zPostAppsByAppIdAudioToTextPath = z.object({ app_id: z.uuid(), })