youngkingdom/dify
1
0
Fork 0
mirror of https://github.com/langgenius/dify.git synced 2026-05-01 07:58:02 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(api): Enhance multi modal support.

Browse Source
This commit is contained in:
-LAN-
2024-09-02 00:46:59 +08:00
parent 7838f9f3a3
commit ea18dd1571
228 changed files with 5324 additions and 3062 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
api/core/tools/tool_engine.py
View File

@ -10,7 +10,8 @@ from yarl import URL
from core.app.entities.app_invoke_entities import InvokeFrom
from core.callback_handler.agent_tool_callback_handler import DifyAgentCallbackHandler
from core.callback_handler.workflow_tool_callback_handler import DifyWorkflowCallbackHandler
from core.file.file_obj import FileTransferMethod
from core.file import FileType
from core.file.models import FileTransferMethod
from core.ops.ops_trace_manager import TraceQueueManager
from core.tools.entities.tool_entities import ToolInvokeMessage, ToolInvokeMessageBinary, ToolInvokeMeta, ToolParameter
from core.tools.errors import (
@ -25,6 +26,7 @@ from core.tools.errors import (
from core.tools.tool.tool import Tool
from core.tools.tool.workflow_tool import WorkflowTool
from core.tools.utils.message_transformer import ToolFileMessageTransformer
from enums import CreatedByRole
from extensions.ext_database import db
from models.model import Message, MessageFile
@ -128,6 +130,7 @@ class ToolEngine:
"""
try:
# hit the callback handler
assert tool.identity is not None
workflow_tool_callback.on_tool_start(tool_name=tool.identity.name, tool_inputs=tool_parameters)
if isinstance(tool, WorkflowTool):
@ -258,7 +261,10 @@ class ToolEngine:
@staticmethod
def _create_message_files(
tool_messages: list[ToolInvokeMessageBinary], agent_message: Message, invoke_from: InvokeFrom, user_id: str
tool_messages: list[ToolInvokeMessageBinary],
agent_message: Message,
invoke_from: InvokeFrom,
user_id: str,
) -> list[tuple[Any, str]]:
"""
Create message file
@ -269,29 +275,31 @@ class ToolEngine:
result = []
for message in tool_messages:
file_type = "bin"
if "image" in message.mimetype:
file_type = "image"
file_type = FileType.IMAGE
elif "video" in message.mimetype:
file_type = "video"
file_type = FileType.VIDEO
elif "audio" in message.mimetype:
file_type = "audio"
elif "text" in message.mimetype:
file_type = "text"
elif "pdf" in message.mimetype:
file_type = "pdf"
elif "zip" in message.mimetype:
file_type = "archive"
# ...
file_type = FileType.AUDIO
elif "text" in message.mimetype or "pdf" in message.mimetype:
file_type = FileType.DOCUMENT
else:
file_type = FileType.CUSTOM
# extract tool file id from url
tool_file_id = message.url.split("/")[-1].split(".")[0]
message_file = MessageFile(
message_id=agent_message.id,
type=file_type,
transfer_method=FileTransferMethod.TOOL_FILE.value,
transfer_method=FileTransferMethod.TOOL_FILE,
belongs_to="assistant",
url=message.url,
upload_file_id=None,
created_by_role=("account" if invoke_from in {InvokeFrom.EXPLORE, InvokeFrom.DEBUGGER} else "end_user"),
upload_file_id=tool_file_id,
created_by_role=(
CreatedByRole.ACCOUNT
if invoke_from in {InvokeFrom.EXPLORE, InvokeFrom.DEBUGGER}
else CreatedByRole.END_USER
),
created_by=user_id,
)