mirror of
https://github.com/langgenius/dify.git
synced 2026-05-04 17:38:04 +08:00
refactor(workflow-file): phase1 migrate workflow file deps with compatibility bridge
Implement phase 1 of the file module migration by moving workflow-facing file primitives to core.workflow.file while keeping core.file as a temporary compatibility layer. What this commit changes - Add core.workflow.file package (constants/enums/models/helpers/file_manager/tool_file_parser). - Add protocol-based runtime binding in core.workflow.file.runtime and core.workflow.file.protocols. - Add application adapter core.app.workflow.file_runtime and bind runtime in extensions.ext_storage.init_app. - Bind runtime in tests via tests/conftest.py. - Migrate workflow-only imports from core.file.* to core.workflow.file.* across workflow runtime/nodes/entry/encoder and workflow node factory. - Update workflow unit tests to patch/import the new workflow file namespace. - Remove workflow-external-imports ignore_imports entries related to core.file from .importlinter. Compatibility guarantee for phase split - Keep core.file import path available in this phase by replacing core/file/*.py with forwarding bridge modules that re-export core.workflow.file symbols. - Preserve runtime behavior and isinstance(File) identity consistency while non-workflow modules are still on legacy import paths. Notes - This commit intentionally does not remove core.file. Full repository replacement and bridge removal are handled in phase 2.
This commit is contained in:
@ -115,18 +115,15 @@ ignore_imports =
|
|||||||
core.workflow.nodes.datasource.datasource_node -> models.tools
|
core.workflow.nodes.datasource.datasource_node -> models.tools
|
||||||
core.workflow.nodes.datasource.datasource_node -> services.datasource_provider_service
|
core.workflow.nodes.datasource.datasource_node -> services.datasource_provider_service
|
||||||
core.workflow.nodes.document_extractor.node -> configs
|
core.workflow.nodes.document_extractor.node -> configs
|
||||||
core.workflow.nodes.document_extractor.node -> core.file.file_manager
|
|
||||||
core.workflow.nodes.document_extractor.node -> core.helper.ssrf_proxy
|
core.workflow.nodes.document_extractor.node -> core.helper.ssrf_proxy
|
||||||
core.workflow.nodes.http_request.entities -> configs
|
core.workflow.nodes.http_request.entities -> configs
|
||||||
core.workflow.nodes.http_request.executor -> configs
|
core.workflow.nodes.http_request.executor -> configs
|
||||||
core.workflow.nodes.http_request.executor -> core.file.file_manager
|
|
||||||
core.workflow.nodes.http_request.node -> configs
|
core.workflow.nodes.http_request.node -> configs
|
||||||
core.workflow.nodes.http_request.node -> core.tools.tool_file_manager
|
core.workflow.nodes.http_request.node -> core.tools.tool_file_manager
|
||||||
core.workflow.nodes.iteration.iteration_node -> core.app.workflow.node_factory
|
core.workflow.nodes.iteration.iteration_node -> core.app.workflow.node_factory
|
||||||
core.workflow.nodes.knowledge_index.knowledge_index_node -> core.rag.index_processor.index_processor_factory
|
core.workflow.nodes.knowledge_index.knowledge_index_node -> core.rag.index_processor.index_processor_factory
|
||||||
core.workflow.nodes.llm.llm_utils -> configs
|
core.workflow.nodes.llm.llm_utils -> configs
|
||||||
core.workflow.nodes.llm.llm_utils -> core.app.entities.app_invoke_entities
|
core.workflow.nodes.llm.llm_utils -> core.app.entities.app_invoke_entities
|
||||||
core.workflow.nodes.llm.llm_utils -> core.file.models
|
|
||||||
core.workflow.nodes.llm.llm_utils -> core.model_manager
|
core.workflow.nodes.llm.llm_utils -> core.model_manager
|
||||||
core.workflow.nodes.llm.llm_utils -> core.model_runtime.model_providers.__base.large_language_model
|
core.workflow.nodes.llm.llm_utils -> core.model_runtime.model_providers.__base.large_language_model
|
||||||
core.workflow.nodes.llm.llm_utils -> models.model
|
core.workflow.nodes.llm.llm_utils -> models.model
|
||||||
@ -162,36 +159,10 @@ ignore_imports =
|
|||||||
core.workflow.nodes.llm.llm_utils -> core.entities.provider_entities
|
core.workflow.nodes.llm.llm_utils -> core.entities.provider_entities
|
||||||
core.workflow.nodes.parameter_extractor.parameter_extractor_node -> core.model_manager
|
core.workflow.nodes.parameter_extractor.parameter_extractor_node -> core.model_manager
|
||||||
core.workflow.nodes.question_classifier.question_classifier_node -> core.model_manager
|
core.workflow.nodes.question_classifier.question_classifier_node -> core.model_manager
|
||||||
core.workflow.node_events.node -> core.file
|
|
||||||
core.workflow.nodes.agent.agent_node -> core.file
|
|
||||||
core.workflow.nodes.datasource.datasource_node -> core.file
|
|
||||||
core.workflow.nodes.datasource.datasource_node -> core.file.enums
|
|
||||||
core.workflow.nodes.document_extractor.node -> core.file
|
|
||||||
core.workflow.nodes.http_request.executor -> core.file.enums
|
|
||||||
core.workflow.nodes.http_request.node -> core.file
|
|
||||||
core.workflow.nodes.http_request.node -> core.file.file_manager
|
|
||||||
core.workflow.nodes.knowledge_retrieval.knowledge_retrieval_node -> core.file.models
|
|
||||||
core.workflow.nodes.list_operator.node -> core.file
|
|
||||||
core.workflow.nodes.llm.file_saver -> core.file
|
|
||||||
core.workflow.nodes.llm.llm_utils -> core.variables.segments
|
core.workflow.nodes.llm.llm_utils -> core.variables.segments
|
||||||
core.workflow.nodes.llm.node -> core.file
|
|
||||||
core.workflow.nodes.llm.node -> core.file.file_manager
|
|
||||||
core.workflow.nodes.llm.node -> core.file.models
|
|
||||||
core.workflow.nodes.loop.entities -> core.variables.types
|
core.workflow.nodes.loop.entities -> core.variables.types
|
||||||
core.workflow.nodes.parameter_extractor.parameter_extractor_node -> core.file
|
|
||||||
core.workflow.nodes.protocols -> core.file
|
|
||||||
core.workflow.nodes.question_classifier.question_classifier_node -> core.file.models
|
|
||||||
core.workflow.nodes.tool.tool_node -> core.file
|
|
||||||
core.workflow.nodes.tool.tool_node -> core.tools.utils.message_transformer
|
core.workflow.nodes.tool.tool_node -> core.tools.utils.message_transformer
|
||||||
core.workflow.nodes.tool.tool_node -> models
|
core.workflow.nodes.tool.tool_node -> models
|
||||||
core.workflow.nodes.trigger_webhook.node -> core.file
|
|
||||||
core.workflow.runtime.variable_pool -> core.file
|
|
||||||
core.workflow.runtime.variable_pool -> core.file.file_manager
|
|
||||||
core.workflow.system_variable -> core.file.models
|
|
||||||
core.workflow.utils.condition.processor -> core.file
|
|
||||||
core.workflow.utils.condition.processor -> core.file.file_manager
|
|
||||||
core.workflow.workflow_entry -> core.file.models
|
|
||||||
core.workflow.workflow_type_encoder -> core.file.models
|
|
||||||
core.workflow.nodes.agent.agent_node -> models.model
|
core.workflow.nodes.agent.agent_node -> models.model
|
||||||
core.workflow.nodes.code.code_node -> core.helper.code_executor.code_node_provider
|
core.workflow.nodes.code.code_node -> core.helper.code_executor.code_node_provider
|
||||||
core.workflow.nodes.code.code_node -> core.helper.code_executor.javascript.javascript_code_provider
|
core.workflow.nodes.code.code_node -> core.helper.code_executor.javascript.javascript_code_provider
|
||||||
|
|||||||
45
api/core/app/workflow/file_runtime.py
Normal file
45
api/core/app/workflow/file_runtime.py
Normal file
@ -0,0 +1,45 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from configs import dify_config
|
||||||
|
from core.helper.ssrf_proxy import ssrf_proxy
|
||||||
|
from core.tools.signature import sign_tool_file
|
||||||
|
from core.workflow.file.protocols import HttpResponseProtocol, WorkflowFileRuntimeProtocol
|
||||||
|
from core.workflow.file.runtime import set_workflow_file_runtime
|
||||||
|
from extensions.ext_storage import storage
|
||||||
|
|
||||||
|
|
||||||
|
class DifyWorkflowFileRuntime(WorkflowFileRuntimeProtocol):
|
||||||
|
"""Production runtime wiring for ``core.workflow.file``."""
|
||||||
|
|
||||||
|
@property
|
||||||
|
def files_url(self) -> str:
|
||||||
|
return dify_config.FILES_URL
|
||||||
|
|
||||||
|
@property
|
||||||
|
def internal_files_url(self) -> str | None:
|
||||||
|
return dify_config.INTERNAL_FILES_URL
|
||||||
|
|
||||||
|
@property
|
||||||
|
def secret_key(self) -> str:
|
||||||
|
return dify_config.SECRET_KEY
|
||||||
|
|
||||||
|
@property
|
||||||
|
def files_access_timeout(self) -> int:
|
||||||
|
return dify_config.FILES_ACCESS_TIMEOUT
|
||||||
|
|
||||||
|
@property
|
||||||
|
def multimodal_send_format(self) -> str:
|
||||||
|
return dify_config.MULTIMODAL_SEND_FORMAT
|
||||||
|
|
||||||
|
def http_get(self, url: str, *, follow_redirects: bool = True) -> HttpResponseProtocol:
|
||||||
|
return ssrf_proxy.get(url, follow_redirects=follow_redirects)
|
||||||
|
|
||||||
|
def storage_load(self, path: str, *, stream: bool = False):
|
||||||
|
return storage.load(path, stream=stream)
|
||||||
|
|
||||||
|
def sign_tool_file(self, *, tool_file_id: str, extension: str, for_external: bool = True) -> str:
|
||||||
|
return sign_tool_file(tool_file_id=tool_file_id, extension=extension, for_external=for_external)
|
||||||
|
|
||||||
|
|
||||||
|
def bind_dify_workflow_file_runtime() -> None:
|
||||||
|
set_workflow_file_runtime(DifyWorkflowFileRuntime())
|
||||||
@ -4,7 +4,6 @@ from typing import TYPE_CHECKING, final
|
|||||||
from typing_extensions import override
|
from typing_extensions import override
|
||||||
|
|
||||||
from configs import dify_config
|
from configs import dify_config
|
||||||
from core.file.file_manager import file_manager
|
|
||||||
from core.helper.code_executor.code_executor import CodeExecutor
|
from core.helper.code_executor.code_executor import CodeExecutor
|
||||||
from core.helper.code_executor.code_node_provider import CodeNodeProvider
|
from core.helper.code_executor.code_node_provider import CodeNodeProvider
|
||||||
from core.helper.ssrf_proxy import ssrf_proxy
|
from core.helper.ssrf_proxy import ssrf_proxy
|
||||||
@ -12,6 +11,7 @@ from core.rag.retrieval.dataset_retrieval import DatasetRetrieval
|
|||||||
from core.tools.tool_file_manager import ToolFileManager
|
from core.tools.tool_file_manager import ToolFileManager
|
||||||
from core.workflow.entities.graph_config import NodeConfigDict
|
from core.workflow.entities.graph_config import NodeConfigDict
|
||||||
from core.workflow.enums import NodeType
|
from core.workflow.enums import NodeType
|
||||||
|
from core.workflow.file.file_manager import file_manager
|
||||||
from core.workflow.graph.graph import NodeFactory
|
from core.workflow.graph.graph import NodeFactory
|
||||||
from core.workflow.nodes.base.node import Node
|
from core.workflow.nodes.base.node import Node
|
||||||
from core.workflow.nodes.code.code_node import CodeNode
|
from core.workflow.nodes.code.code_node import CodeNode
|
||||||
|
|||||||
@ -1,7 +1,17 @@
|
|||||||
from .constants import FILE_MODEL_IDENTITY
|
"""Compatibility bridge for legacy ``core.file`` imports.
|
||||||
from .enums import ArrayFileAttribute, FileAttribute, FileBelongsTo, FileTransferMethod, FileType
|
|
||||||
from .models import (
|
Phase 1 keeps this package as a forwarding layer while canonical file models and
|
||||||
|
helpers live under ``core.workflow.file``.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from core.workflow.file import (
|
||||||
|
FILE_MODEL_IDENTITY,
|
||||||
|
ArrayFileAttribute,
|
||||||
File,
|
File,
|
||||||
|
FileAttribute,
|
||||||
|
FileBelongsTo,
|
||||||
|
FileTransferMethod,
|
||||||
|
FileType,
|
||||||
FileUploadConfig,
|
FileUploadConfig,
|
||||||
ImageConfig,
|
ImageConfig,
|
||||||
)
|
)
|
||||||
|
|||||||
@ -1,11 +1,8 @@
|
|||||||
from typing import Any
|
"""Compatibility bridge for legacy ``core.file.constants`` imports."""
|
||||||
|
|
||||||
# TODO(QuantumGhost): Refactor variable type identification. Instead of directly
|
from core.workflow.file.constants import FILE_MODEL_IDENTITY, maybe_file_object
|
||||||
# comparing `dify_model_identity` with constants throughout the codebase, extract
|
|
||||||
# this logic into a dedicated function. This would encapsulate the implementation
|
|
||||||
# details of how different variable types are identified.
|
|
||||||
FILE_MODEL_IDENTITY = "__dify__file__"
|
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
def maybe_file_object(o: Any) -> bool:
|
"FILE_MODEL_IDENTITY",
|
||||||
return isinstance(o, dict) and o.get("dify_model_identity") == FILE_MODEL_IDENTITY
|
"maybe_file_object",
|
||||||
|
]
|
||||||
|
|||||||
@ -1,57 +1,11 @@
|
|||||||
from enum import StrEnum
|
"""Compatibility bridge for legacy ``core.file.enums`` imports."""
|
||||||
|
|
||||||
|
from core.workflow.file.enums import ArrayFileAttribute, FileAttribute, FileBelongsTo, FileTransferMethod, FileType
|
||||||
|
|
||||||
class FileType(StrEnum):
|
__all__ = [
|
||||||
IMAGE = "image"
|
"FileType",
|
||||||
DOCUMENT = "document"
|
"FileTransferMethod",
|
||||||
AUDIO = "audio"
|
"FileBelongsTo",
|
||||||
VIDEO = "video"
|
"FileAttribute",
|
||||||
CUSTOM = "custom"
|
"ArrayFileAttribute",
|
||||||
|
]
|
||||||
@staticmethod
|
|
||||||
def value_of(value):
|
|
||||||
for member in FileType:
|
|
||||||
if member.value == value:
|
|
||||||
return member
|
|
||||||
raise ValueError(f"No matching enum found for value '{value}'")
|
|
||||||
|
|
||||||
|
|
||||||
class FileTransferMethod(StrEnum):
|
|
||||||
REMOTE_URL = "remote_url"
|
|
||||||
LOCAL_FILE = "local_file"
|
|
||||||
TOOL_FILE = "tool_file"
|
|
||||||
DATASOURCE_FILE = "datasource_file"
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def value_of(value):
|
|
||||||
for member in FileTransferMethod:
|
|
||||||
if member.value == value:
|
|
||||||
return member
|
|
||||||
raise ValueError(f"No matching enum found for value '{value}'")
|
|
||||||
|
|
||||||
|
|
||||||
class FileBelongsTo(StrEnum):
|
|
||||||
USER = "user"
|
|
||||||
ASSISTANT = "assistant"
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def value_of(value):
|
|
||||||
for member in FileBelongsTo:
|
|
||||||
if member.value == value:
|
|
||||||
return member
|
|
||||||
raise ValueError(f"No matching enum found for value '{value}'")
|
|
||||||
|
|
||||||
|
|
||||||
class FileAttribute(StrEnum):
|
|
||||||
TYPE = "type"
|
|
||||||
SIZE = "size"
|
|
||||||
NAME = "name"
|
|
||||||
MIME_TYPE = "mime_type"
|
|
||||||
TRANSFER_METHOD = "transfer_method"
|
|
||||||
URL = "url"
|
|
||||||
EXTENSION = "extension"
|
|
||||||
RELATED_ID = "related_id"
|
|
||||||
|
|
||||||
|
|
||||||
class ArrayFileAttribute(StrEnum):
|
|
||||||
LENGTH = "length"
|
|
||||||
|
|||||||
@ -1,185 +1,11 @@
|
|||||||
import base64
|
"""Compatibility bridge for legacy ``core.file.file_manager`` imports."""
|
||||||
from collections.abc import Mapping
|
|
||||||
|
|
||||||
from configs import dify_config
|
from core.workflow.file.file_manager import FileManager, download, file_manager, get_attr, to_prompt_message_content
|
||||||
from core.helper import ssrf_proxy
|
|
||||||
from core.model_runtime.entities import (
|
|
||||||
AudioPromptMessageContent,
|
|
||||||
DocumentPromptMessageContent,
|
|
||||||
ImagePromptMessageContent,
|
|
||||||
TextPromptMessageContent,
|
|
||||||
VideoPromptMessageContent,
|
|
||||||
)
|
|
||||||
from core.model_runtime.entities.message_entities import PromptMessageContentUnionTypes
|
|
||||||
from core.tools.signature import sign_tool_file
|
|
||||||
from extensions.ext_storage import storage
|
|
||||||
|
|
||||||
from . import helpers
|
__all__ = [
|
||||||
from .enums import FileAttribute
|
"FileManager",
|
||||||
from .models import File, FileTransferMethod, FileType
|
"download",
|
||||||
|
"file_manager",
|
||||||
|
"get_attr",
|
||||||
def get_attr(*, file: File, attr: FileAttribute):
|
"to_prompt_message_content",
|
||||||
match attr:
|
]
|
||||||
case FileAttribute.TYPE:
|
|
||||||
return file.type.value
|
|
||||||
case FileAttribute.SIZE:
|
|
||||||
return file.size
|
|
||||||
case FileAttribute.NAME:
|
|
||||||
return file.filename
|
|
||||||
case FileAttribute.MIME_TYPE:
|
|
||||||
return file.mime_type
|
|
||||||
case FileAttribute.TRANSFER_METHOD:
|
|
||||||
return file.transfer_method.value
|
|
||||||
case FileAttribute.URL:
|
|
||||||
return _to_url(file)
|
|
||||||
case FileAttribute.EXTENSION:
|
|
||||||
return file.extension
|
|
||||||
case FileAttribute.RELATED_ID:
|
|
||||||
return file.related_id
|
|
||||||
|
|
||||||
|
|
||||||
def to_prompt_message_content(
|
|
||||||
f: File,
|
|
||||||
/,
|
|
||||||
*,
|
|
||||||
image_detail_config: ImagePromptMessageContent.DETAIL | None = None,
|
|
||||||
) -> PromptMessageContentUnionTypes:
|
|
||||||
"""
|
|
||||||
Convert a file to prompt message content.
|
|
||||||
|
|
||||||
This function converts files to their appropriate prompt message content types.
|
|
||||||
For supported file types (IMAGE, AUDIO, VIDEO, DOCUMENT), it creates the
|
|
||||||
corresponding message content with proper encoding/URL.
|
|
||||||
|
|
||||||
For unsupported file types, instead of raising an error, it returns a
|
|
||||||
TextPromptMessageContent with a descriptive message about the file.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
f: The file to convert
|
|
||||||
image_detail_config: Optional detail configuration for image files
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
PromptMessageContentUnionTypes: The appropriate message content type
|
|
||||||
|
|
||||||
Raises:
|
|
||||||
ValueError: If file extension or mime_type is missing
|
|
||||||
"""
|
|
||||||
if f.extension is None:
|
|
||||||
raise ValueError("Missing file extension")
|
|
||||||
if f.mime_type is None:
|
|
||||||
raise ValueError("Missing file mime_type")
|
|
||||||
|
|
||||||
prompt_class_map: Mapping[FileType, type[PromptMessageContentUnionTypes]] = {
|
|
||||||
FileType.IMAGE: ImagePromptMessageContent,
|
|
||||||
FileType.AUDIO: AudioPromptMessageContent,
|
|
||||||
FileType.VIDEO: VideoPromptMessageContent,
|
|
||||||
FileType.DOCUMENT: DocumentPromptMessageContent,
|
|
||||||
}
|
|
||||||
|
|
||||||
# Check if file type is supported
|
|
||||||
if f.type not in prompt_class_map:
|
|
||||||
# For unsupported file types, return a text description
|
|
||||||
return TextPromptMessageContent(data=f"[Unsupported file type: {f.filename} ({f.type.value})]")
|
|
||||||
|
|
||||||
# Process supported file types
|
|
||||||
params = {
|
|
||||||
"base64_data": _get_encoded_string(f) if dify_config.MULTIMODAL_SEND_FORMAT == "base64" else "",
|
|
||||||
"url": _to_url(f) if dify_config.MULTIMODAL_SEND_FORMAT == "url" else "",
|
|
||||||
"format": f.extension.removeprefix("."),
|
|
||||||
"mime_type": f.mime_type,
|
|
||||||
"filename": f.filename or "",
|
|
||||||
}
|
|
||||||
if f.type == FileType.IMAGE:
|
|
||||||
params["detail"] = image_detail_config or ImagePromptMessageContent.DETAIL.LOW
|
|
||||||
|
|
||||||
return prompt_class_map[f.type].model_validate(params)
|
|
||||||
|
|
||||||
|
|
||||||
def download(f: File, /):
|
|
||||||
if f.transfer_method in (
|
|
||||||
FileTransferMethod.TOOL_FILE,
|
|
||||||
FileTransferMethod.LOCAL_FILE,
|
|
||||||
FileTransferMethod.DATASOURCE_FILE,
|
|
||||||
):
|
|
||||||
return _download_file_content(f.storage_key)
|
|
||||||
elif f.transfer_method == FileTransferMethod.REMOTE_URL:
|
|
||||||
if f.remote_url is None:
|
|
||||||
raise ValueError("Missing file remote_url")
|
|
||||||
response = ssrf_proxy.get(f.remote_url, follow_redirects=True)
|
|
||||||
response.raise_for_status()
|
|
||||||
return response.content
|
|
||||||
raise ValueError(f"unsupported transfer method: {f.transfer_method}")
|
|
||||||
|
|
||||||
|
|
||||||
def _download_file_content(path: str, /):
|
|
||||||
"""
|
|
||||||
Download and return the contents of a file as bytes.
|
|
||||||
|
|
||||||
This function loads the file from storage and ensures it's in bytes format.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
path (str): The path to the file in storage.
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
bytes: The contents of the file as a bytes object.
|
|
||||||
|
|
||||||
Raises:
|
|
||||||
ValueError: If the loaded file is not a bytes object.
|
|
||||||
"""
|
|
||||||
data = storage.load(path, stream=False)
|
|
||||||
if not isinstance(data, bytes):
|
|
||||||
raise ValueError(f"file {path} is not a bytes object")
|
|
||||||
return data
|
|
||||||
|
|
||||||
|
|
||||||
def _get_encoded_string(f: File, /):
|
|
||||||
match f.transfer_method:
|
|
||||||
case FileTransferMethod.REMOTE_URL:
|
|
||||||
if f.remote_url is None:
|
|
||||||
raise ValueError("Missing file remote_url")
|
|
||||||
response = ssrf_proxy.get(f.remote_url, follow_redirects=True)
|
|
||||||
response.raise_for_status()
|
|
||||||
data = response.content
|
|
||||||
case FileTransferMethod.LOCAL_FILE:
|
|
||||||
data = _download_file_content(f.storage_key)
|
|
||||||
case FileTransferMethod.TOOL_FILE:
|
|
||||||
data = _download_file_content(f.storage_key)
|
|
||||||
case FileTransferMethod.DATASOURCE_FILE:
|
|
||||||
data = _download_file_content(f.storage_key)
|
|
||||||
|
|
||||||
encoded_string = base64.b64encode(data).decode("utf-8")
|
|
||||||
return encoded_string
|
|
||||||
|
|
||||||
|
|
||||||
def _to_url(f: File, /):
|
|
||||||
if f.transfer_method == FileTransferMethod.REMOTE_URL:
|
|
||||||
if f.remote_url is None:
|
|
||||||
raise ValueError("Missing file remote_url")
|
|
||||||
return f.remote_url
|
|
||||||
elif f.transfer_method == FileTransferMethod.LOCAL_FILE:
|
|
||||||
if f.related_id is None:
|
|
||||||
raise ValueError("Missing file related_id")
|
|
||||||
return f.remote_url or helpers.get_signed_file_url(upload_file_id=f.related_id)
|
|
||||||
elif f.transfer_method == FileTransferMethod.TOOL_FILE:
|
|
||||||
# add sign url
|
|
||||||
if f.related_id is None or f.extension is None:
|
|
||||||
raise ValueError("Missing file related_id or extension")
|
|
||||||
return sign_tool_file(tool_file_id=f.related_id, extension=f.extension)
|
|
||||||
else:
|
|
||||||
raise ValueError(f"Unsupported transfer method: {f.transfer_method}")
|
|
||||||
|
|
||||||
|
|
||||||
class FileManager:
|
|
||||||
"""
|
|
||||||
Adapter exposing file manager helpers behind FileManagerProtocol.
|
|
||||||
|
|
||||||
This is intentionally a thin wrapper over the existing module-level functions so callers can inject it
|
|
||||||
where a protocol-typed file manager is expected.
|
|
||||||
"""
|
|
||||||
|
|
||||||
def download(self, f: File, /) -> bytes:
|
|
||||||
return download(f)
|
|
||||||
|
|
||||||
|
|
||||||
file_manager = FileManager()
|
|
||||||
|
|||||||
@ -1,83 +1,19 @@
|
|||||||
import base64
|
"""Compatibility bridge for legacy ``core.file.helpers`` imports."""
|
||||||
import hashlib
|
|
||||||
import hmac
|
|
||||||
import os
|
|
||||||
import time
|
|
||||||
import urllib.parse
|
|
||||||
|
|
||||||
from configs import dify_config
|
from core.workflow.file.helpers import (
|
||||||
|
get_signed_file_url,
|
||||||
|
get_signed_file_url_for_plugin,
|
||||||
|
get_signed_tool_file_url,
|
||||||
|
verify_file_signature,
|
||||||
|
verify_image_signature,
|
||||||
|
verify_plugin_file_signature,
|
||||||
|
)
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
def get_signed_file_url(upload_file_id: str, as_attachment=False, for_external: bool = True) -> str:
|
"get_signed_file_url",
|
||||||
base_url = dify_config.FILES_URL if for_external else (dify_config.INTERNAL_FILES_URL or dify_config.FILES_URL)
|
"get_signed_file_url_for_plugin",
|
||||||
url = f"{base_url}/files/{upload_file_id}/file-preview"
|
"get_signed_tool_file_url",
|
||||||
|
"verify_file_signature",
|
||||||
timestamp = str(int(time.time()))
|
"verify_image_signature",
|
||||||
nonce = os.urandom(16).hex()
|
"verify_plugin_file_signature",
|
||||||
key = dify_config.SECRET_KEY.encode()
|
]
|
||||||
msg = f"file-preview|{upload_file_id}|{timestamp}|{nonce}"
|
|
||||||
sign = hmac.new(key, msg.encode(), hashlib.sha256).digest()
|
|
||||||
encoded_sign = base64.urlsafe_b64encode(sign).decode()
|
|
||||||
query = {"timestamp": timestamp, "nonce": nonce, "sign": encoded_sign}
|
|
||||||
if as_attachment:
|
|
||||||
query["as_attachment"] = "true"
|
|
||||||
query_string = urllib.parse.urlencode(query)
|
|
||||||
|
|
||||||
return f"{url}?{query_string}"
|
|
||||||
|
|
||||||
|
|
||||||
def get_signed_file_url_for_plugin(filename: str, mimetype: str, tenant_id: str, user_id: str) -> str:
|
|
||||||
# Plugin access should use internal URL for Docker network communication
|
|
||||||
base_url = dify_config.INTERNAL_FILES_URL or dify_config.FILES_URL
|
|
||||||
url = f"{base_url}/files/upload/for-plugin"
|
|
||||||
timestamp = str(int(time.time()))
|
|
||||||
nonce = os.urandom(16).hex()
|
|
||||||
key = dify_config.SECRET_KEY.encode()
|
|
||||||
msg = f"upload|{filename}|{mimetype}|{tenant_id}|{user_id}|{timestamp}|{nonce}"
|
|
||||||
sign = hmac.new(key, msg.encode(), hashlib.sha256).digest()
|
|
||||||
encoded_sign = base64.urlsafe_b64encode(sign).decode()
|
|
||||||
return f"{url}?timestamp={timestamp}&nonce={nonce}&sign={encoded_sign}&user_id={user_id}&tenant_id={tenant_id}"
|
|
||||||
|
|
||||||
|
|
||||||
def verify_plugin_file_signature(
|
|
||||||
*, filename: str, mimetype: str, tenant_id: str, user_id: str, timestamp: str, nonce: str, sign: str
|
|
||||||
) -> bool:
|
|
||||||
data_to_sign = f"upload|{filename}|{mimetype}|{tenant_id}|{user_id}|{timestamp}|{nonce}"
|
|
||||||
secret_key = dify_config.SECRET_KEY.encode()
|
|
||||||
recalculated_sign = hmac.new(secret_key, data_to_sign.encode(), hashlib.sha256).digest()
|
|
||||||
recalculated_encoded_sign = base64.urlsafe_b64encode(recalculated_sign).decode()
|
|
||||||
|
|
||||||
# verify signature
|
|
||||||
if sign != recalculated_encoded_sign:
|
|
||||||
return False
|
|
||||||
|
|
||||||
current_time = int(time.time())
|
|
||||||
return current_time - int(timestamp) <= dify_config.FILES_ACCESS_TIMEOUT
|
|
||||||
|
|
||||||
|
|
||||||
def verify_image_signature(*, upload_file_id: str, timestamp: str, nonce: str, sign: str) -> bool:
|
|
||||||
data_to_sign = f"image-preview|{upload_file_id}|{timestamp}|{nonce}"
|
|
||||||
secret_key = dify_config.SECRET_KEY.encode()
|
|
||||||
recalculated_sign = hmac.new(secret_key, data_to_sign.encode(), hashlib.sha256).digest()
|
|
||||||
recalculated_encoded_sign = base64.urlsafe_b64encode(recalculated_sign).decode()
|
|
||||||
|
|
||||||
# verify signature
|
|
||||||
if sign != recalculated_encoded_sign:
|
|
||||||
return False
|
|
||||||
|
|
||||||
current_time = int(time.time())
|
|
||||||
return current_time - int(timestamp) <= dify_config.FILES_ACCESS_TIMEOUT
|
|
||||||
|
|
||||||
|
|
||||||
def verify_file_signature(*, upload_file_id: str, timestamp: str, nonce: str, sign: str) -> bool:
|
|
||||||
data_to_sign = f"file-preview|{upload_file_id}|{timestamp}|{nonce}"
|
|
||||||
secret_key = dify_config.SECRET_KEY.encode()
|
|
||||||
recalculated_sign = hmac.new(secret_key, data_to_sign.encode(), hashlib.sha256).digest()
|
|
||||||
recalculated_encoded_sign = base64.urlsafe_b64encode(recalculated_sign).decode()
|
|
||||||
|
|
||||||
# verify signature
|
|
||||||
if sign != recalculated_encoded_sign:
|
|
||||||
return False
|
|
||||||
|
|
||||||
current_time = int(time.time())
|
|
||||||
return current_time - int(timestamp) <= dify_config.FILES_ACCESS_TIMEOUT
|
|
||||||
|
|||||||
@ -1,164 +1,10 @@
|
|||||||
from collections.abc import Mapping, Sequence
|
"""Compatibility bridge for legacy ``core.file.models`` imports."""
|
||||||
from typing import Any
|
|
||||||
|
|
||||||
from pydantic import BaseModel, Field, model_validator
|
from core.workflow.file.models import File, FileUploadConfig, ImageConfig, sign_tool_file
|
||||||
|
|
||||||
from core.model_runtime.entities.message_entities import ImagePromptMessageContent
|
__all__ = [
|
||||||
from core.tools.signature import sign_tool_file
|
"File",
|
||||||
|
"FileUploadConfig",
|
||||||
from . import helpers
|
"ImageConfig",
|
||||||
from .constants import FILE_MODEL_IDENTITY
|
"sign_tool_file",
|
||||||
from .enums import FileTransferMethod, FileType
|
]
|
||||||
|
|
||||||
|
|
||||||
class ImageConfig(BaseModel):
|
|
||||||
"""
|
|
||||||
NOTE: This part of validation is deprecated, but still used in app features "Image Upload".
|
|
||||||
"""
|
|
||||||
|
|
||||||
number_limits: int = 0
|
|
||||||
transfer_methods: Sequence[FileTransferMethod] = Field(default_factory=list)
|
|
||||||
detail: ImagePromptMessageContent.DETAIL | None = None
|
|
||||||
|
|
||||||
|
|
||||||
class FileUploadConfig(BaseModel):
|
|
||||||
"""
|
|
||||||
File Upload Entity.
|
|
||||||
"""
|
|
||||||
|
|
||||||
image_config: ImageConfig | None = None
|
|
||||||
allowed_file_types: Sequence[FileType] = Field(default_factory=list)
|
|
||||||
allowed_file_extensions: Sequence[str] = Field(default_factory=list)
|
|
||||||
allowed_file_upload_methods: Sequence[FileTransferMethod] = Field(default_factory=list)
|
|
||||||
number_limits: int = 0
|
|
||||||
|
|
||||||
|
|
||||||
class File(BaseModel):
|
|
||||||
# NOTE: dify_model_identity is a special identifier used to distinguish between
|
|
||||||
# new and old data formats during serialization and deserialization.
|
|
||||||
dify_model_identity: str = FILE_MODEL_IDENTITY
|
|
||||||
|
|
||||||
id: str | None = None # message file id
|
|
||||||
tenant_id: str
|
|
||||||
type: FileType
|
|
||||||
transfer_method: FileTransferMethod
|
|
||||||
# If `transfer_method` is `FileTransferMethod.remote_url`, the
|
|
||||||
# `remote_url` attribute must not be `None`.
|
|
||||||
remote_url: str | None = None # remote url
|
|
||||||
# If `transfer_method` is `FileTransferMethod.local_file` or
|
|
||||||
# `FileTransferMethod.tool_file`, the `related_id` attribute must not be `None`.
|
|
||||||
#
|
|
||||||
# It should be set to `ToolFile.id` when `transfer_method` is `tool_file`.
|
|
||||||
related_id: str | None = None
|
|
||||||
filename: str | None = None
|
|
||||||
extension: str | None = Field(default=None, description="File extension, should contain dot")
|
|
||||||
mime_type: str | None = None
|
|
||||||
size: int = -1
|
|
||||||
|
|
||||||
# Those properties are private, should not be exposed to the outside.
|
|
||||||
_storage_key: str
|
|
||||||
|
|
||||||
def __init__(
|
|
||||||
self,
|
|
||||||
*,
|
|
||||||
id: str | None = None,
|
|
||||||
tenant_id: str,
|
|
||||||
type: FileType,
|
|
||||||
transfer_method: FileTransferMethod,
|
|
||||||
remote_url: str | None = None,
|
|
||||||
related_id: str | None = None,
|
|
||||||
filename: str | None = None,
|
|
||||||
extension: str | None = None,
|
|
||||||
mime_type: str | None = None,
|
|
||||||
size: int = -1,
|
|
||||||
storage_key: str | None = None,
|
|
||||||
dify_model_identity: str | None = FILE_MODEL_IDENTITY,
|
|
||||||
url: str | None = None,
|
|
||||||
# Legacy compatibility fields - explicitly handle known extra fields
|
|
||||||
tool_file_id: str | None = None,
|
|
||||||
upload_file_id: str | None = None,
|
|
||||||
datasource_file_id: str | None = None,
|
|
||||||
):
|
|
||||||
super().__init__(
|
|
||||||
id=id,
|
|
||||||
tenant_id=tenant_id,
|
|
||||||
type=type,
|
|
||||||
transfer_method=transfer_method,
|
|
||||||
remote_url=remote_url,
|
|
||||||
related_id=related_id,
|
|
||||||
filename=filename,
|
|
||||||
extension=extension,
|
|
||||||
mime_type=mime_type,
|
|
||||||
size=size,
|
|
||||||
dify_model_identity=dify_model_identity,
|
|
||||||
url=url,
|
|
||||||
)
|
|
||||||
self._storage_key = str(storage_key)
|
|
||||||
|
|
||||||
def to_dict(self) -> Mapping[str, str | int | None]:
|
|
||||||
data = self.model_dump(mode="json")
|
|
||||||
return {
|
|
||||||
**data,
|
|
||||||
"url": self.generate_url(),
|
|
||||||
}
|
|
||||||
|
|
||||||
@property
|
|
||||||
def markdown(self) -> str:
|
|
||||||
url = self.generate_url()
|
|
||||||
if self.type == FileType.IMAGE:
|
|
||||||
text = f""
|
|
||||||
else:
|
|
||||||
text = f"[{self.filename or url}]({url})"
|
|
||||||
|
|
||||||
return text
|
|
||||||
|
|
||||||
def generate_url(self, for_external: bool = True) -> str | None:
|
|
||||||
if self.transfer_method == FileTransferMethod.REMOTE_URL:
|
|
||||||
return self.remote_url
|
|
||||||
elif self.transfer_method == FileTransferMethod.LOCAL_FILE:
|
|
||||||
if self.related_id is None:
|
|
||||||
raise ValueError("Missing file related_id")
|
|
||||||
return helpers.get_signed_file_url(upload_file_id=self.related_id, for_external=for_external)
|
|
||||||
elif self.transfer_method in [FileTransferMethod.TOOL_FILE, FileTransferMethod.DATASOURCE_FILE]:
|
|
||||||
assert self.related_id is not None
|
|
||||||
assert self.extension is not None
|
|
||||||
return sign_tool_file(tool_file_id=self.related_id, extension=self.extension, for_external=for_external)
|
|
||||||
return None
|
|
||||||
|
|
||||||
def to_plugin_parameter(self) -> dict[str, Any]:
|
|
||||||
return {
|
|
||||||
"dify_model_identity": FILE_MODEL_IDENTITY,
|
|
||||||
"mime_type": self.mime_type,
|
|
||||||
"filename": self.filename,
|
|
||||||
"extension": self.extension,
|
|
||||||
"size": self.size,
|
|
||||||
"type": self.type,
|
|
||||||
"url": self.generate_url(for_external=False),
|
|
||||||
}
|
|
||||||
|
|
||||||
@model_validator(mode="after")
|
|
||||||
def validate_after(self):
|
|
||||||
match self.transfer_method:
|
|
||||||
case FileTransferMethod.REMOTE_URL:
|
|
||||||
if not self.remote_url:
|
|
||||||
raise ValueError("Missing file url")
|
|
||||||
if not isinstance(self.remote_url, str) or not self.remote_url.startswith("http"):
|
|
||||||
raise ValueError("Invalid file url")
|
|
||||||
case FileTransferMethod.LOCAL_FILE:
|
|
||||||
if not self.related_id:
|
|
||||||
raise ValueError("Missing file related_id")
|
|
||||||
case FileTransferMethod.TOOL_FILE:
|
|
||||||
if not self.related_id:
|
|
||||||
raise ValueError("Missing file related_id")
|
|
||||||
case FileTransferMethod.DATASOURCE_FILE:
|
|
||||||
if not self.related_id:
|
|
||||||
raise ValueError("Missing file related_id")
|
|
||||||
return self
|
|
||||||
|
|
||||||
@property
|
|
||||||
def storage_key(self) -> str:
|
|
||||||
return self._storage_key
|
|
||||||
|
|
||||||
@storage_key.setter
|
|
||||||
def storage_key(self, value: str):
|
|
||||||
self._storage_key = value
|
|
||||||
|
|||||||
@ -1,12 +1,20 @@
|
|||||||
|
"""Compatibility bridge for legacy ``core.file.tool_file_parser`` imports."""
|
||||||
|
|
||||||
from collections.abc import Callable
|
from collections.abc import Callable
|
||||||
from typing import TYPE_CHECKING
|
from typing import Any
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
from core.workflow.file import tool_file_parser as workflow_tool_file_parser
|
||||||
from core.tools.tool_file_manager import ToolFileManager
|
|
||||||
|
|
||||||
_tool_file_manager_factory: Callable[[], "ToolFileManager"] | None = None
|
_tool_file_manager_factory: Callable[[], Any] | None = None
|
||||||
|
|
||||||
|
|
||||||
def set_tool_file_manager_factory(factory: Callable[[], "ToolFileManager"]):
|
def set_tool_file_manager_factory(factory: Callable[[], Any]) -> None:
|
||||||
global _tool_file_manager_factory
|
global _tool_file_manager_factory
|
||||||
_tool_file_manager_factory = factory
|
_tool_file_manager_factory = factory
|
||||||
|
workflow_tool_file_parser.set_tool_file_manager_factory(factory)
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"_tool_file_manager_factory",
|
||||||
|
"set_tool_file_manager_factory",
|
||||||
|
]
|
||||||
|
|||||||
19
api/core/workflow/file/__init__.py
Normal file
19
api/core/workflow/file/__init__.py
Normal file
@ -0,0 +1,19 @@
|
|||||||
|
from .constants import FILE_MODEL_IDENTITY
|
||||||
|
from .enums import ArrayFileAttribute, FileAttribute, FileBelongsTo, FileTransferMethod, FileType
|
||||||
|
from .models import (
|
||||||
|
File,
|
||||||
|
FileUploadConfig,
|
||||||
|
ImageConfig,
|
||||||
|
)
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"FILE_MODEL_IDENTITY",
|
||||||
|
"ArrayFileAttribute",
|
||||||
|
"File",
|
||||||
|
"FileAttribute",
|
||||||
|
"FileBelongsTo",
|
||||||
|
"FileTransferMethod",
|
||||||
|
"FileType",
|
||||||
|
"FileUploadConfig",
|
||||||
|
"ImageConfig",
|
||||||
|
]
|
||||||
11
api/core/workflow/file/constants.py
Normal file
11
api/core/workflow/file/constants.py
Normal file
@ -0,0 +1,11 @@
|
|||||||
|
from typing import Any
|
||||||
|
|
||||||
|
# TODO(QuantumGhost): Refactor variable type identification. Instead of directly
|
||||||
|
# comparing `dify_model_identity` with constants throughout the codebase, extract
|
||||||
|
# this logic into a dedicated function. This would encapsulate the implementation
|
||||||
|
# details of how different variable types are identified.
|
||||||
|
FILE_MODEL_IDENTITY = "__dify__file__"
|
||||||
|
|
||||||
|
|
||||||
|
def maybe_file_object(o: Any) -> bool:
|
||||||
|
return isinstance(o, dict) and o.get("dify_model_identity") == FILE_MODEL_IDENTITY
|
||||||
57
api/core/workflow/file/enums.py
Normal file
57
api/core/workflow/file/enums.py
Normal file
@ -0,0 +1,57 @@
|
|||||||
|
from enum import StrEnum
|
||||||
|
|
||||||
|
|
||||||
|
class FileType(StrEnum):
|
||||||
|
IMAGE = "image"
|
||||||
|
DOCUMENT = "document"
|
||||||
|
AUDIO = "audio"
|
||||||
|
VIDEO = "video"
|
||||||
|
CUSTOM = "custom"
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def value_of(value):
|
||||||
|
for member in FileType:
|
||||||
|
if member.value == value:
|
||||||
|
return member
|
||||||
|
raise ValueError(f"No matching enum found for value '{value}'")
|
||||||
|
|
||||||
|
|
||||||
|
class FileTransferMethod(StrEnum):
|
||||||
|
REMOTE_URL = "remote_url"
|
||||||
|
LOCAL_FILE = "local_file"
|
||||||
|
TOOL_FILE = "tool_file"
|
||||||
|
DATASOURCE_FILE = "datasource_file"
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def value_of(value):
|
||||||
|
for member in FileTransferMethod:
|
||||||
|
if member.value == value:
|
||||||
|
return member
|
||||||
|
raise ValueError(f"No matching enum found for value '{value}'")
|
||||||
|
|
||||||
|
|
||||||
|
class FileBelongsTo(StrEnum):
|
||||||
|
USER = "user"
|
||||||
|
ASSISTANT = "assistant"
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def value_of(value):
|
||||||
|
for member in FileBelongsTo:
|
||||||
|
if member.value == value:
|
||||||
|
return member
|
||||||
|
raise ValueError(f"No matching enum found for value '{value}'")
|
||||||
|
|
||||||
|
|
||||||
|
class FileAttribute(StrEnum):
|
||||||
|
TYPE = "type"
|
||||||
|
SIZE = "size"
|
||||||
|
NAME = "name"
|
||||||
|
MIME_TYPE = "mime_type"
|
||||||
|
TRANSFER_METHOD = "transfer_method"
|
||||||
|
URL = "url"
|
||||||
|
EXTENSION = "extension"
|
||||||
|
RELATED_ID = "related_id"
|
||||||
|
|
||||||
|
|
||||||
|
class ArrayFileAttribute(StrEnum):
|
||||||
|
LENGTH = "length"
|
||||||
143
api/core/workflow/file/file_manager.py
Normal file
143
api/core/workflow/file/file_manager.py
Normal file
@ -0,0 +1,143 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import base64
|
||||||
|
from collections.abc import Mapping
|
||||||
|
|
||||||
|
from core.model_runtime.entities import (
|
||||||
|
AudioPromptMessageContent,
|
||||||
|
DocumentPromptMessageContent,
|
||||||
|
ImagePromptMessageContent,
|
||||||
|
TextPromptMessageContent,
|
||||||
|
VideoPromptMessageContent,
|
||||||
|
)
|
||||||
|
from core.model_runtime.entities.message_entities import PromptMessageContentUnionTypes
|
||||||
|
|
||||||
|
from . import helpers
|
||||||
|
from .enums import FileAttribute
|
||||||
|
from .models import File, FileTransferMethod, FileType
|
||||||
|
from .runtime import get_workflow_file_runtime
|
||||||
|
|
||||||
|
|
||||||
|
def get_attr(*, file: File, attr: FileAttribute):
|
||||||
|
match attr:
|
||||||
|
case FileAttribute.TYPE:
|
||||||
|
return file.type.value
|
||||||
|
case FileAttribute.SIZE:
|
||||||
|
return file.size
|
||||||
|
case FileAttribute.NAME:
|
||||||
|
return file.filename
|
||||||
|
case FileAttribute.MIME_TYPE:
|
||||||
|
return file.mime_type
|
||||||
|
case FileAttribute.TRANSFER_METHOD:
|
||||||
|
return file.transfer_method.value
|
||||||
|
case FileAttribute.URL:
|
||||||
|
return _to_url(file)
|
||||||
|
case FileAttribute.EXTENSION:
|
||||||
|
return file.extension
|
||||||
|
case FileAttribute.RELATED_ID:
|
||||||
|
return file.related_id
|
||||||
|
|
||||||
|
|
||||||
|
def to_prompt_message_content(
|
||||||
|
f: File,
|
||||||
|
/,
|
||||||
|
*,
|
||||||
|
image_detail_config: ImagePromptMessageContent.DETAIL | None = None,
|
||||||
|
) -> PromptMessageContentUnionTypes:
|
||||||
|
"""Convert a file to prompt message content."""
|
||||||
|
if f.extension is None:
|
||||||
|
raise ValueError("Missing file extension")
|
||||||
|
if f.mime_type is None:
|
||||||
|
raise ValueError("Missing file mime_type")
|
||||||
|
|
||||||
|
prompt_class_map: Mapping[FileType, type[PromptMessageContentUnionTypes]] = {
|
||||||
|
FileType.IMAGE: ImagePromptMessageContent,
|
||||||
|
FileType.AUDIO: AudioPromptMessageContent,
|
||||||
|
FileType.VIDEO: VideoPromptMessageContent,
|
||||||
|
FileType.DOCUMENT: DocumentPromptMessageContent,
|
||||||
|
}
|
||||||
|
|
||||||
|
if f.type not in prompt_class_map:
|
||||||
|
return TextPromptMessageContent(data=f"[Unsupported file type: {f.filename} ({f.type.value})]")
|
||||||
|
|
||||||
|
send_format = get_workflow_file_runtime().multimodal_send_format
|
||||||
|
params = {
|
||||||
|
"base64_data": _get_encoded_string(f) if send_format == "base64" else "",
|
||||||
|
"url": _to_url(f) if send_format == "url" else "",
|
||||||
|
"format": f.extension.removeprefix("."),
|
||||||
|
"mime_type": f.mime_type,
|
||||||
|
"filename": f.filename or "",
|
||||||
|
}
|
||||||
|
if f.type == FileType.IMAGE:
|
||||||
|
params["detail"] = image_detail_config or ImagePromptMessageContent.DETAIL.LOW
|
||||||
|
|
||||||
|
return prompt_class_map[f.type].model_validate(params)
|
||||||
|
|
||||||
|
|
||||||
|
def download(f: File, /) -> bytes:
|
||||||
|
if f.transfer_method in (
|
||||||
|
FileTransferMethod.TOOL_FILE,
|
||||||
|
FileTransferMethod.LOCAL_FILE,
|
||||||
|
FileTransferMethod.DATASOURCE_FILE,
|
||||||
|
):
|
||||||
|
return _download_file_content(f.storage_key)
|
||||||
|
elif f.transfer_method == FileTransferMethod.REMOTE_URL:
|
||||||
|
if f.remote_url is None:
|
||||||
|
raise ValueError("Missing file remote_url")
|
||||||
|
response = get_workflow_file_runtime().http_get(f.remote_url, follow_redirects=True)
|
||||||
|
response.raise_for_status()
|
||||||
|
return response.content
|
||||||
|
raise ValueError(f"unsupported transfer method: {f.transfer_method}")
|
||||||
|
|
||||||
|
|
||||||
|
def _download_file_content(path: str, /) -> bytes:
|
||||||
|
"""Download and return a file from storage as bytes."""
|
||||||
|
data = get_workflow_file_runtime().storage_load(path, stream=False)
|
||||||
|
if not isinstance(data, bytes):
|
||||||
|
raise ValueError(f"file {path} is not a bytes object")
|
||||||
|
return data
|
||||||
|
|
||||||
|
|
||||||
|
def _get_encoded_string(f: File, /) -> str:
|
||||||
|
match f.transfer_method:
|
||||||
|
case FileTransferMethod.REMOTE_URL:
|
||||||
|
if f.remote_url is None:
|
||||||
|
raise ValueError("Missing file remote_url")
|
||||||
|
response = get_workflow_file_runtime().http_get(f.remote_url, follow_redirects=True)
|
||||||
|
response.raise_for_status()
|
||||||
|
data = response.content
|
||||||
|
case FileTransferMethod.LOCAL_FILE:
|
||||||
|
data = _download_file_content(f.storage_key)
|
||||||
|
case FileTransferMethod.TOOL_FILE:
|
||||||
|
data = _download_file_content(f.storage_key)
|
||||||
|
case FileTransferMethod.DATASOURCE_FILE:
|
||||||
|
data = _download_file_content(f.storage_key)
|
||||||
|
|
||||||
|
return base64.b64encode(data).decode("utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
def _to_url(f: File, /):
|
||||||
|
if f.transfer_method == FileTransferMethod.REMOTE_URL:
|
||||||
|
if f.remote_url is None:
|
||||||
|
raise ValueError("Missing file remote_url")
|
||||||
|
return f.remote_url
|
||||||
|
elif f.transfer_method == FileTransferMethod.LOCAL_FILE:
|
||||||
|
if f.related_id is None:
|
||||||
|
raise ValueError("Missing file related_id")
|
||||||
|
return f.remote_url or helpers.get_signed_file_url(upload_file_id=f.related_id)
|
||||||
|
elif f.transfer_method == FileTransferMethod.TOOL_FILE:
|
||||||
|
if f.related_id is None or f.extension is None:
|
||||||
|
raise ValueError("Missing file related_id or extension")
|
||||||
|
return helpers.get_signed_tool_file_url(tool_file_id=f.related_id, extension=f.extension)
|
||||||
|
else:
|
||||||
|
raise ValueError(f"Unsupported transfer method: {f.transfer_method}")
|
||||||
|
|
||||||
|
|
||||||
|
class FileManager:
|
||||||
|
"""Adapter exposing file manager helpers behind FileManagerProtocol."""
|
||||||
|
|
||||||
|
def download(self, f: File, /) -> bytes:
|
||||||
|
return download(f)
|
||||||
|
|
||||||
|
|
||||||
|
file_manager = FileManager()
|
||||||
92
api/core/workflow/file/helpers.py
Normal file
92
api/core/workflow/file/helpers.py
Normal file
@ -0,0 +1,92 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import base64
|
||||||
|
import hashlib
|
||||||
|
import hmac
|
||||||
|
import os
|
||||||
|
import time
|
||||||
|
import urllib.parse
|
||||||
|
|
||||||
|
from .runtime import get_workflow_file_runtime
|
||||||
|
|
||||||
|
|
||||||
|
def get_signed_file_url(upload_file_id: str, as_attachment: bool = False, for_external: bool = True) -> str:
|
||||||
|
runtime = get_workflow_file_runtime()
|
||||||
|
base_url = runtime.files_url if for_external else (runtime.internal_files_url or runtime.files_url)
|
||||||
|
url = f"{base_url}/files/{upload_file_id}/file-preview"
|
||||||
|
|
||||||
|
timestamp = str(int(time.time()))
|
||||||
|
nonce = os.urandom(16).hex()
|
||||||
|
key = runtime.secret_key.encode()
|
||||||
|
msg = f"file-preview|{upload_file_id}|{timestamp}|{nonce}"
|
||||||
|
sign = hmac.new(key, msg.encode(), hashlib.sha256).digest()
|
||||||
|
encoded_sign = base64.urlsafe_b64encode(sign).decode()
|
||||||
|
query: dict[str, str] = {"timestamp": timestamp, "nonce": nonce, "sign": encoded_sign}
|
||||||
|
if as_attachment:
|
||||||
|
query["as_attachment"] = "true"
|
||||||
|
query_string = urllib.parse.urlencode(query)
|
||||||
|
|
||||||
|
return f"{url}?{query_string}"
|
||||||
|
|
||||||
|
|
||||||
|
def get_signed_file_url_for_plugin(filename: str, mimetype: str, tenant_id: str, user_id: str) -> str:
|
||||||
|
runtime = get_workflow_file_runtime()
|
||||||
|
# Plugin access should use internal URL for Docker network communication.
|
||||||
|
base_url = runtime.internal_files_url or runtime.files_url
|
||||||
|
url = f"{base_url}/files/upload/for-plugin"
|
||||||
|
timestamp = str(int(time.time()))
|
||||||
|
nonce = os.urandom(16).hex()
|
||||||
|
key = runtime.secret_key.encode()
|
||||||
|
msg = f"upload|{filename}|{mimetype}|{tenant_id}|{user_id}|{timestamp}|{nonce}"
|
||||||
|
sign = hmac.new(key, msg.encode(), hashlib.sha256).digest()
|
||||||
|
encoded_sign = base64.urlsafe_b64encode(sign).decode()
|
||||||
|
return f"{url}?timestamp={timestamp}&nonce={nonce}&sign={encoded_sign}&user_id={user_id}&tenant_id={tenant_id}"
|
||||||
|
|
||||||
|
|
||||||
|
def get_signed_tool_file_url(tool_file_id: str, extension: str, for_external: bool = True) -> str:
|
||||||
|
runtime = get_workflow_file_runtime()
|
||||||
|
return runtime.sign_tool_file(tool_file_id=tool_file_id, extension=extension, for_external=for_external)
|
||||||
|
|
||||||
|
|
||||||
|
def verify_plugin_file_signature(
|
||||||
|
*, filename: str, mimetype: str, tenant_id: str, user_id: str, timestamp: str, nonce: str, sign: str
|
||||||
|
) -> bool:
|
||||||
|
runtime = get_workflow_file_runtime()
|
||||||
|
data_to_sign = f"upload|{filename}|{mimetype}|{tenant_id}|{user_id}|{timestamp}|{nonce}"
|
||||||
|
secret_key = runtime.secret_key.encode()
|
||||||
|
recalculated_sign = hmac.new(secret_key, data_to_sign.encode(), hashlib.sha256).digest()
|
||||||
|
recalculated_encoded_sign = base64.urlsafe_b64encode(recalculated_sign).decode()
|
||||||
|
|
||||||
|
if sign != recalculated_encoded_sign:
|
||||||
|
return False
|
||||||
|
|
||||||
|
current_time = int(time.time())
|
||||||
|
return current_time - int(timestamp) <= runtime.files_access_timeout
|
||||||
|
|
||||||
|
|
||||||
|
def verify_image_signature(*, upload_file_id: str, timestamp: str, nonce: str, sign: str) -> bool:
|
||||||
|
runtime = get_workflow_file_runtime()
|
||||||
|
data_to_sign = f"image-preview|{upload_file_id}|{timestamp}|{nonce}"
|
||||||
|
secret_key = runtime.secret_key.encode()
|
||||||
|
recalculated_sign = hmac.new(secret_key, data_to_sign.encode(), hashlib.sha256).digest()
|
||||||
|
recalculated_encoded_sign = base64.urlsafe_b64encode(recalculated_sign).decode()
|
||||||
|
|
||||||
|
if sign != recalculated_encoded_sign:
|
||||||
|
return False
|
||||||
|
|
||||||
|
current_time = int(time.time())
|
||||||
|
return current_time - int(timestamp) <= runtime.files_access_timeout
|
||||||
|
|
||||||
|
|
||||||
|
def verify_file_signature(*, upload_file_id: str, timestamp: str, nonce: str, sign: str) -> bool:
|
||||||
|
runtime = get_workflow_file_runtime()
|
||||||
|
data_to_sign = f"file-preview|{upload_file_id}|{timestamp}|{nonce}"
|
||||||
|
secret_key = runtime.secret_key.encode()
|
||||||
|
recalculated_sign = hmac.new(secret_key, data_to_sign.encode(), hashlib.sha256).digest()
|
||||||
|
recalculated_encoded_sign = base64.urlsafe_b64encode(recalculated_sign).decode()
|
||||||
|
|
||||||
|
if sign != recalculated_encoded_sign:
|
||||||
|
return False
|
||||||
|
|
||||||
|
current_time = int(time.time())
|
||||||
|
return current_time - int(timestamp) <= runtime.files_access_timeout
|
||||||
178
api/core/workflow/file/models.py
Normal file
178
api/core/workflow/file/models.py
Normal file
@ -0,0 +1,178 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from collections.abc import Mapping, Sequence
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from pydantic import BaseModel, Field, model_validator
|
||||||
|
|
||||||
|
from core.model_runtime.entities.message_entities import ImagePromptMessageContent
|
||||||
|
|
||||||
|
from . import helpers
|
||||||
|
from .constants import FILE_MODEL_IDENTITY
|
||||||
|
from .enums import FileTransferMethod, FileType
|
||||||
|
|
||||||
|
|
||||||
|
def sign_tool_file(*, tool_file_id: str, extension: str, for_external: bool = True) -> str:
|
||||||
|
"""Compatibility shim for tests and legacy callers patching ``models.sign_tool_file``."""
|
||||||
|
return helpers.get_signed_tool_file_url(
|
||||||
|
tool_file_id=tool_file_id,
|
||||||
|
extension=extension,
|
||||||
|
for_external=for_external,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class ImageConfig(BaseModel):
|
||||||
|
"""
|
||||||
|
NOTE: This part of validation is deprecated, but still used in app features "Image Upload".
|
||||||
|
"""
|
||||||
|
|
||||||
|
number_limits: int = 0
|
||||||
|
transfer_methods: Sequence[FileTransferMethod] = Field(default_factory=list)
|
||||||
|
detail: ImagePromptMessageContent.DETAIL | None = None
|
||||||
|
|
||||||
|
|
||||||
|
class FileUploadConfig(BaseModel):
|
||||||
|
"""
|
||||||
|
File Upload Entity.
|
||||||
|
"""
|
||||||
|
|
||||||
|
image_config: ImageConfig | None = None
|
||||||
|
allowed_file_types: Sequence[FileType] = Field(default_factory=list)
|
||||||
|
allowed_file_extensions: Sequence[str] = Field(default_factory=list)
|
||||||
|
allowed_file_upload_methods: Sequence[FileTransferMethod] = Field(default_factory=list)
|
||||||
|
number_limits: int = 0
|
||||||
|
|
||||||
|
|
||||||
|
class File(BaseModel):
|
||||||
|
# NOTE: dify_model_identity is a special identifier used to distinguish between
|
||||||
|
# new and old data formats during serialization and deserialization.
|
||||||
|
dify_model_identity: str = FILE_MODEL_IDENTITY
|
||||||
|
|
||||||
|
id: str | None = None # message file id
|
||||||
|
tenant_id: str
|
||||||
|
type: FileType
|
||||||
|
transfer_method: FileTransferMethod
|
||||||
|
# If `transfer_method` is `FileTransferMethod.remote_url`, the
|
||||||
|
# `remote_url` attribute must not be `None`.
|
||||||
|
remote_url: str | None = None # remote url
|
||||||
|
# If `transfer_method` is `FileTransferMethod.local_file` or
|
||||||
|
# `FileTransferMethod.tool_file`, the `related_id` attribute must not be `None`.
|
||||||
|
#
|
||||||
|
# It should be set to `ToolFile.id` when `transfer_method` is `tool_file`.
|
||||||
|
related_id: str | None = None
|
||||||
|
filename: str | None = None
|
||||||
|
extension: str | None = Field(default=None, description="File extension, should contain dot")
|
||||||
|
mime_type: str | None = None
|
||||||
|
size: int = -1
|
||||||
|
|
||||||
|
# Those properties are private, should not be exposed to the outside.
|
||||||
|
_storage_key: str
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
id: str | None = None,
|
||||||
|
tenant_id: str,
|
||||||
|
type: FileType,
|
||||||
|
transfer_method: FileTransferMethod,
|
||||||
|
remote_url: str | None = None,
|
||||||
|
related_id: str | None = None,
|
||||||
|
filename: str | None = None,
|
||||||
|
extension: str | None = None,
|
||||||
|
mime_type: str | None = None,
|
||||||
|
size: int = -1,
|
||||||
|
storage_key: str | None = None,
|
||||||
|
dify_model_identity: str | None = FILE_MODEL_IDENTITY,
|
||||||
|
url: str | None = None,
|
||||||
|
# Legacy compatibility fields - explicitly handle known extra fields
|
||||||
|
tool_file_id: str | None = None,
|
||||||
|
upload_file_id: str | None = None,
|
||||||
|
datasource_file_id: str | None = None,
|
||||||
|
):
|
||||||
|
super().__init__(
|
||||||
|
id=id,
|
||||||
|
tenant_id=tenant_id,
|
||||||
|
type=type,
|
||||||
|
transfer_method=transfer_method,
|
||||||
|
remote_url=remote_url,
|
||||||
|
related_id=related_id,
|
||||||
|
filename=filename,
|
||||||
|
extension=extension,
|
||||||
|
mime_type=mime_type,
|
||||||
|
size=size,
|
||||||
|
dify_model_identity=dify_model_identity,
|
||||||
|
url=url,
|
||||||
|
)
|
||||||
|
self._storage_key = str(storage_key)
|
||||||
|
|
||||||
|
def to_dict(self) -> Mapping[str, str | int | None]:
|
||||||
|
data = self.model_dump(mode="json")
|
||||||
|
return {
|
||||||
|
**data,
|
||||||
|
"url": self.generate_url(),
|
||||||
|
}
|
||||||
|
|
||||||
|
@property
|
||||||
|
def markdown(self) -> str:
|
||||||
|
url = self.generate_url()
|
||||||
|
if self.type == FileType.IMAGE:
|
||||||
|
text = f""
|
||||||
|
else:
|
||||||
|
text = f"[{self.filename or url}]({url})"
|
||||||
|
|
||||||
|
return text
|
||||||
|
|
||||||
|
def generate_url(self, for_external: bool = True) -> str | None:
|
||||||
|
if self.transfer_method == FileTransferMethod.REMOTE_URL:
|
||||||
|
return self.remote_url
|
||||||
|
elif self.transfer_method == FileTransferMethod.LOCAL_FILE:
|
||||||
|
if self.related_id is None:
|
||||||
|
raise ValueError("Missing file related_id")
|
||||||
|
return helpers.get_signed_file_url(upload_file_id=self.related_id, for_external=for_external)
|
||||||
|
elif self.transfer_method in [FileTransferMethod.TOOL_FILE, FileTransferMethod.DATASOURCE_FILE]:
|
||||||
|
assert self.related_id is not None
|
||||||
|
assert self.extension is not None
|
||||||
|
return sign_tool_file(
|
||||||
|
tool_file_id=self.related_id,
|
||||||
|
extension=self.extension,
|
||||||
|
for_external=for_external,
|
||||||
|
)
|
||||||
|
return None
|
||||||
|
|
||||||
|
def to_plugin_parameter(self) -> dict[str, Any]:
|
||||||
|
return {
|
||||||
|
"dify_model_identity": FILE_MODEL_IDENTITY,
|
||||||
|
"mime_type": self.mime_type,
|
||||||
|
"filename": self.filename,
|
||||||
|
"extension": self.extension,
|
||||||
|
"size": self.size,
|
||||||
|
"type": self.type,
|
||||||
|
"url": self.generate_url(for_external=False),
|
||||||
|
}
|
||||||
|
|
||||||
|
@model_validator(mode="after")
|
||||||
|
def validate_after(self) -> File:
|
||||||
|
match self.transfer_method:
|
||||||
|
case FileTransferMethod.REMOTE_URL:
|
||||||
|
if not self.remote_url:
|
||||||
|
raise ValueError("Missing file url")
|
||||||
|
if not isinstance(self.remote_url, str) or not self.remote_url.startswith("http"):
|
||||||
|
raise ValueError("Invalid file url")
|
||||||
|
case FileTransferMethod.LOCAL_FILE:
|
||||||
|
if not self.related_id:
|
||||||
|
raise ValueError("Missing file related_id")
|
||||||
|
case FileTransferMethod.TOOL_FILE:
|
||||||
|
if not self.related_id:
|
||||||
|
raise ValueError("Missing file related_id")
|
||||||
|
case FileTransferMethod.DATASOURCE_FILE:
|
||||||
|
if not self.related_id:
|
||||||
|
raise ValueError("Missing file related_id")
|
||||||
|
return self
|
||||||
|
|
||||||
|
@property
|
||||||
|
def storage_key(self) -> str:
|
||||||
|
return self._storage_key
|
||||||
|
|
||||||
|
@storage_key.setter
|
||||||
|
def storage_key(self, value: str) -> None:
|
||||||
|
self._storage_key = value
|
||||||
42
api/core/workflow/file/protocols.py
Normal file
42
api/core/workflow/file/protocols.py
Normal file
@ -0,0 +1,42 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from collections.abc import Generator
|
||||||
|
from typing import Protocol
|
||||||
|
|
||||||
|
|
||||||
|
class HttpResponseProtocol(Protocol):
|
||||||
|
"""Subset of response behavior needed by workflow file helpers."""
|
||||||
|
|
||||||
|
content: bytes
|
||||||
|
|
||||||
|
def raise_for_status(self) -> None: ...
|
||||||
|
|
||||||
|
|
||||||
|
class WorkflowFileRuntimeProtocol(Protocol):
|
||||||
|
"""Runtime dependencies required by ``core.workflow.file``.
|
||||||
|
|
||||||
|
Implementations are expected to be provided by integration layers (for example,
|
||||||
|
``core.app.workflow.file_runtime``) so the workflow package avoids importing
|
||||||
|
application infrastructure modules directly.
|
||||||
|
"""
|
||||||
|
|
||||||
|
@property
|
||||||
|
def files_url(self) -> str: ...
|
||||||
|
|
||||||
|
@property
|
||||||
|
def internal_files_url(self) -> str | None: ...
|
||||||
|
|
||||||
|
@property
|
||||||
|
def secret_key(self) -> str: ...
|
||||||
|
|
||||||
|
@property
|
||||||
|
def files_access_timeout(self) -> int: ...
|
||||||
|
|
||||||
|
@property
|
||||||
|
def multimodal_send_format(self) -> str: ...
|
||||||
|
|
||||||
|
def http_get(self, url: str, *, follow_redirects: bool = True) -> HttpResponseProtocol: ...
|
||||||
|
|
||||||
|
def storage_load(self, path: str, *, stream: bool = False) -> bytes | Generator: ...
|
||||||
|
|
||||||
|
def sign_tool_file(self, *, tool_file_id: str, extension: str, for_external: bool = True) -> str: ...
|
||||||
57
api/core/workflow/file/runtime.py
Normal file
57
api/core/workflow/file/runtime.py
Normal file
@ -0,0 +1,57 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from collections.abc import Generator
|
||||||
|
|
||||||
|
from .protocols import HttpResponseProtocol, WorkflowFileRuntimeProtocol
|
||||||
|
|
||||||
|
|
||||||
|
class WorkflowFileRuntimeNotConfiguredError(RuntimeError):
|
||||||
|
"""Raised when workflow file runtime dependencies were not configured."""
|
||||||
|
|
||||||
|
|
||||||
|
class _UnconfiguredWorkflowFileRuntime(WorkflowFileRuntimeProtocol):
|
||||||
|
def _raise(self) -> None:
|
||||||
|
raise WorkflowFileRuntimeNotConfiguredError(
|
||||||
|
"workflow file runtime is not configured, call set_workflow_file_runtime(...) first"
|
||||||
|
)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def files_url(self) -> str:
|
||||||
|
self._raise()
|
||||||
|
|
||||||
|
@property
|
||||||
|
def internal_files_url(self) -> str | None:
|
||||||
|
self._raise()
|
||||||
|
|
||||||
|
@property
|
||||||
|
def secret_key(self) -> str:
|
||||||
|
self._raise()
|
||||||
|
|
||||||
|
@property
|
||||||
|
def files_access_timeout(self) -> int:
|
||||||
|
self._raise()
|
||||||
|
|
||||||
|
@property
|
||||||
|
def multimodal_send_format(self) -> str:
|
||||||
|
self._raise()
|
||||||
|
|
||||||
|
def http_get(self, url: str, *, follow_redirects: bool = True) -> HttpResponseProtocol:
|
||||||
|
self._raise()
|
||||||
|
|
||||||
|
def storage_load(self, path: str, *, stream: bool = False) -> bytes | Generator:
|
||||||
|
self._raise()
|
||||||
|
|
||||||
|
def sign_tool_file(self, *, tool_file_id: str, extension: str, for_external: bool = True) -> str:
|
||||||
|
self._raise()
|
||||||
|
|
||||||
|
|
||||||
|
_runtime: WorkflowFileRuntimeProtocol = _UnconfiguredWorkflowFileRuntime()
|
||||||
|
|
||||||
|
|
||||||
|
def set_workflow_file_runtime(runtime: WorkflowFileRuntimeProtocol) -> None:
|
||||||
|
global _runtime
|
||||||
|
_runtime = runtime
|
||||||
|
|
||||||
|
|
||||||
|
def get_workflow_file_runtime() -> WorkflowFileRuntimeProtocol:
|
||||||
|
return _runtime
|
||||||
9
api/core/workflow/file/tool_file_parser.py
Normal file
9
api/core/workflow/file/tool_file_parser.py
Normal file
@ -0,0 +1,9 @@
|
|||||||
|
from collections.abc import Callable
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
_tool_file_manager_factory: Callable[[], Any] | None = None
|
||||||
|
|
||||||
|
|
||||||
|
def set_tool_file_manager_factory(factory: Callable[[], Any]):
|
||||||
|
global _tool_file_manager_factory
|
||||||
|
_tool_file_manager_factory = factory
|
||||||
@ -3,10 +3,10 @@ from datetime import datetime
|
|||||||
|
|
||||||
from pydantic import Field
|
from pydantic import Field
|
||||||
|
|
||||||
from core.file import File
|
|
||||||
from core.model_runtime.entities.llm_entities import LLMUsage
|
from core.model_runtime.entities.llm_entities import LLMUsage
|
||||||
from core.rag.entities.citation_metadata import RetrievalSourceMetadata
|
from core.rag.entities.citation_metadata import RetrievalSourceMetadata
|
||||||
from core.workflow.entities.pause_reason import PauseReason
|
from core.workflow.entities.pause_reason import PauseReason
|
||||||
|
from core.workflow.file import File
|
||||||
from core.workflow.node_events import NodeRunResult
|
from core.workflow.node_events import NodeRunResult
|
||||||
|
|
||||||
from .base import NodeEventBase
|
from .base import NodeEventBase
|
||||||
|
|||||||
@ -11,7 +11,6 @@ from sqlalchemy.orm import Session
|
|||||||
|
|
||||||
from core.agent.entities import AgentToolEntity
|
from core.agent.entities import AgentToolEntity
|
||||||
from core.agent.plugin_entities import AgentStrategyParameter
|
from core.agent.plugin_entities import AgentStrategyParameter
|
||||||
from core.file import File, FileTransferMethod
|
|
||||||
from core.memory.token_buffer_memory import TokenBufferMemory
|
from core.memory.token_buffer_memory import TokenBufferMemory
|
||||||
from core.model_manager import ModelInstance, ModelManager
|
from core.model_manager import ModelInstance, ModelManager
|
||||||
from core.model_runtime.entities.llm_entities import LLMUsage, LLMUsageMetadata
|
from core.model_runtime.entities.llm_entities import LLMUsage, LLMUsageMetadata
|
||||||
@ -33,6 +32,7 @@ from core.workflow.enums import (
|
|||||||
WorkflowNodeExecutionMetadataKey,
|
WorkflowNodeExecutionMetadataKey,
|
||||||
WorkflowNodeExecutionStatus,
|
WorkflowNodeExecutionStatus,
|
||||||
)
|
)
|
||||||
|
from core.workflow.file import File, FileTransferMethod
|
||||||
from core.workflow.node_events import (
|
from core.workflow.node_events import (
|
||||||
AgentLogEvent,
|
AgentLogEvent,
|
||||||
NodeEventBase,
|
NodeEventBase,
|
||||||
|
|||||||
@ -14,13 +14,13 @@ from core.datasource.entities.datasource_entities import (
|
|||||||
from core.datasource.online_document.online_document_plugin import OnlineDocumentDatasourcePlugin
|
from core.datasource.online_document.online_document_plugin import OnlineDocumentDatasourcePlugin
|
||||||
from core.datasource.online_drive.online_drive_plugin import OnlineDriveDatasourcePlugin
|
from core.datasource.online_drive.online_drive_plugin import OnlineDriveDatasourcePlugin
|
||||||
from core.datasource.utils.message_transformer import DatasourceFileMessageTransformer
|
from core.datasource.utils.message_transformer import DatasourceFileMessageTransformer
|
||||||
from core.file import File
|
|
||||||
from core.file.enums import FileTransferMethod, FileType
|
|
||||||
from core.plugin.impl.exc import PluginDaemonClientSideError
|
from core.plugin.impl.exc import PluginDaemonClientSideError
|
||||||
from core.variables.segments import ArrayAnySegment
|
from core.variables.segments import ArrayAnySegment
|
||||||
from core.variables.variables import ArrayAnyVariable
|
from core.variables.variables import ArrayAnyVariable
|
||||||
from core.workflow.entities.workflow_node_execution import WorkflowNodeExecutionStatus
|
from core.workflow.entities.workflow_node_execution import WorkflowNodeExecutionStatus
|
||||||
from core.workflow.enums import NodeExecutionType, NodeType, SystemVariableKey
|
from core.workflow.enums import NodeExecutionType, NodeType, SystemVariableKey
|
||||||
|
from core.workflow.file import File
|
||||||
|
from core.workflow.file.enums import FileTransferMethod, FileType
|
||||||
from core.workflow.node_events import NodeRunResult, StreamChunkEvent, StreamCompletedEvent
|
from core.workflow.node_events import NodeRunResult, StreamChunkEvent, StreamCompletedEvent
|
||||||
from core.workflow.nodes.base.node import Node
|
from core.workflow.nodes.base.node import Node
|
||||||
from core.workflow.nodes.base.variable_template_parser import VariableTemplateParser
|
from core.workflow.nodes.base.variable_template_parser import VariableTemplateParser
|
||||||
|
|||||||
@ -21,11 +21,11 @@ from docx.table import Table
|
|||||||
from docx.text.paragraph import Paragraph
|
from docx.text.paragraph import Paragraph
|
||||||
|
|
||||||
from configs import dify_config
|
from configs import dify_config
|
||||||
from core.file import File, FileTransferMethod, file_manager
|
|
||||||
from core.helper import ssrf_proxy
|
from core.helper import ssrf_proxy
|
||||||
from core.variables import ArrayFileSegment
|
from core.variables import ArrayFileSegment
|
||||||
from core.variables.segments import ArrayStringSegment, FileSegment
|
from core.variables.segments import ArrayStringSegment, FileSegment
|
||||||
from core.workflow.enums import NodeType, WorkflowNodeExecutionStatus
|
from core.workflow.enums import NodeType, WorkflowNodeExecutionStatus
|
||||||
|
from core.workflow.file import File, FileTransferMethod, file_manager
|
||||||
from core.workflow.node_events import NodeRunResult
|
from core.workflow.node_events import NodeRunResult
|
||||||
from core.workflow.nodes.base.node import Node
|
from core.workflow.nodes.base.node import Node
|
||||||
|
|
||||||
|
|||||||
@ -11,10 +11,10 @@ import httpx
|
|||||||
from json_repair import repair_json
|
from json_repair import repair_json
|
||||||
|
|
||||||
from configs import dify_config
|
from configs import dify_config
|
||||||
from core.file.enums import FileTransferMethod
|
|
||||||
from core.file.file_manager import file_manager as default_file_manager
|
|
||||||
from core.helper.ssrf_proxy import ssrf_proxy
|
from core.helper.ssrf_proxy import ssrf_proxy
|
||||||
from core.variables.segments import ArrayFileSegment, FileSegment
|
from core.variables.segments import ArrayFileSegment, FileSegment
|
||||||
|
from core.workflow.file.enums import FileTransferMethod
|
||||||
|
from core.workflow.file.file_manager import file_manager as default_file_manager
|
||||||
from core.workflow.runtime import VariablePool
|
from core.workflow.runtime import VariablePool
|
||||||
|
|
||||||
from ..protocols import FileManagerProtocol, HttpClientProtocol
|
from ..protocols import FileManagerProtocol, HttpClientProtocol
|
||||||
|
|||||||
@ -4,12 +4,12 @@ from collections.abc import Callable, Mapping, Sequence
|
|||||||
from typing import TYPE_CHECKING, Any
|
from typing import TYPE_CHECKING, Any
|
||||||
|
|
||||||
from configs import dify_config
|
from configs import dify_config
|
||||||
from core.file import File, FileTransferMethod
|
|
||||||
from core.file.file_manager import file_manager as default_file_manager
|
|
||||||
from core.helper.ssrf_proxy import ssrf_proxy
|
from core.helper.ssrf_proxy import ssrf_proxy
|
||||||
from core.tools.tool_file_manager import ToolFileManager
|
from core.tools.tool_file_manager import ToolFileManager
|
||||||
from core.variables.segments import ArrayFileSegment
|
from core.variables.segments import ArrayFileSegment
|
||||||
from core.workflow.enums import NodeType, WorkflowNodeExecutionStatus
|
from core.workflow.enums import NodeType, WorkflowNodeExecutionStatus
|
||||||
|
from core.workflow.file import File, FileTransferMethod
|
||||||
|
from core.workflow.file.file_manager import file_manager as default_file_manager
|
||||||
from core.workflow.node_events import NodeRunResult
|
from core.workflow.node_events import NodeRunResult
|
||||||
from core.workflow.nodes.base import variable_template_parser
|
from core.workflow.nodes.base import variable_template_parser
|
||||||
from core.workflow.nodes.base.entities import VariableSelector
|
from core.workflow.nodes.base.entities import VariableSelector
|
||||||
|
|||||||
@ -30,7 +30,7 @@ from .exc import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
from core.file.models import File
|
from core.workflow.file.models import File
|
||||||
from core.workflow.runtime import GraphRuntimeState
|
from core.workflow.runtime import GraphRuntimeState
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|||||||
@ -1,10 +1,10 @@
|
|||||||
from collections.abc import Callable, Sequence
|
from collections.abc import Callable, Sequence
|
||||||
from typing import Any, TypeAlias, TypeVar
|
from typing import Any, TypeAlias, TypeVar
|
||||||
|
|
||||||
from core.file import File
|
|
||||||
from core.variables import ArrayFileSegment, ArrayNumberSegment, ArrayStringSegment
|
from core.variables import ArrayFileSegment, ArrayNumberSegment, ArrayStringSegment
|
||||||
from core.variables.segments import ArrayAnySegment, ArrayBooleanSegment, ArraySegment
|
from core.variables.segments import ArrayAnySegment, ArrayBooleanSegment, ArraySegment
|
||||||
from core.workflow.enums import NodeType, WorkflowNodeExecutionStatus
|
from core.workflow.enums import NodeType, WorkflowNodeExecutionStatus
|
||||||
|
from core.workflow.file import File
|
||||||
from core.workflow.node_events import NodeRunResult
|
from core.workflow.node_events import NodeRunResult
|
||||||
from core.workflow.nodes.base.node import Node
|
from core.workflow.nodes.base.node import Node
|
||||||
|
|
||||||
|
|||||||
@ -4,10 +4,10 @@ import typing as tp
|
|||||||
from sqlalchemy import Engine
|
from sqlalchemy import Engine
|
||||||
|
|
||||||
from constants.mimetypes import DEFAULT_EXTENSION, DEFAULT_MIME_TYPE
|
from constants.mimetypes import DEFAULT_EXTENSION, DEFAULT_MIME_TYPE
|
||||||
from core.file import File, FileTransferMethod, FileType
|
|
||||||
from core.helper import ssrf_proxy
|
from core.helper import ssrf_proxy
|
||||||
from core.tools.signature import sign_tool_file
|
from core.tools.signature import sign_tool_file
|
||||||
from core.tools.tool_file_manager import ToolFileManager
|
from core.tools.tool_file_manager import ToolFileManager
|
||||||
|
from core.workflow.file import File, FileTransferMethod, FileType
|
||||||
from extensions.ext_database import db as global_db
|
from extensions.ext_database import db as global_db
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@ -7,7 +7,6 @@ from sqlalchemy.orm import Session
|
|||||||
from configs import dify_config
|
from configs import dify_config
|
||||||
from core.app.entities.app_invoke_entities import ModelConfigWithCredentialsEntity
|
from core.app.entities.app_invoke_entities import ModelConfigWithCredentialsEntity
|
||||||
from core.entities.provider_entities import ProviderQuotaType, QuotaUnit
|
from core.entities.provider_entities import ProviderQuotaType, QuotaUnit
|
||||||
from core.file.models import File
|
|
||||||
from core.memory.token_buffer_memory import TokenBufferMemory
|
from core.memory.token_buffer_memory import TokenBufferMemory
|
||||||
from core.model_manager import ModelInstance, ModelManager
|
from core.model_manager import ModelInstance, ModelManager
|
||||||
from core.model_runtime.entities.llm_entities import LLMUsage
|
from core.model_runtime.entities.llm_entities import LLMUsage
|
||||||
@ -16,6 +15,7 @@ from core.model_runtime.model_providers.__base.large_language_model import Large
|
|||||||
from core.prompt.entities.advanced_prompt_entities import MemoryConfig
|
from core.prompt.entities.advanced_prompt_entities import MemoryConfig
|
||||||
from core.variables.segments import ArrayAnySegment, ArrayFileSegment, FileSegment, NoneSegment, StringSegment
|
from core.variables.segments import ArrayAnySegment, ArrayFileSegment, FileSegment, NoneSegment, StringSegment
|
||||||
from core.workflow.enums import SystemVariableKey
|
from core.workflow.enums import SystemVariableKey
|
||||||
|
from core.workflow.file.models import File
|
||||||
from core.workflow.nodes.llm.entities import ModelConfig
|
from core.workflow.nodes.llm.entities import ModelConfig
|
||||||
from core.workflow.runtime import VariablePool
|
from core.workflow.runtime import VariablePool
|
||||||
from extensions.ext_database import db
|
from extensions.ext_database import db
|
||||||
|
|||||||
@ -12,7 +12,6 @@ from typing import TYPE_CHECKING, Any, Literal
|
|||||||
from sqlalchemy import select
|
from sqlalchemy import select
|
||||||
|
|
||||||
from core.app.entities.app_invoke_entities import ModelConfigWithCredentialsEntity
|
from core.app.entities.app_invoke_entities import ModelConfigWithCredentialsEntity
|
||||||
from core.file import File, FileTransferMethod, FileType, file_manager
|
|
||||||
from core.helper.code_executor import CodeExecutor, CodeLanguage
|
from core.helper.code_executor import CodeExecutor, CodeLanguage
|
||||||
from core.llm_generator.output_parser.errors import OutputParserError
|
from core.llm_generator.output_parser.errors import OutputParserError
|
||||||
from core.llm_generator.output_parser.structured_output import invoke_llm_with_structured_output
|
from core.llm_generator.output_parser.structured_output import invoke_llm_with_structured_output
|
||||||
@ -65,6 +64,7 @@ from core.workflow.enums import (
|
|||||||
WorkflowNodeExecutionMetadataKey,
|
WorkflowNodeExecutionMetadataKey,
|
||||||
WorkflowNodeExecutionStatus,
|
WorkflowNodeExecutionStatus,
|
||||||
)
|
)
|
||||||
|
from core.workflow.file import File, FileTransferMethod, FileType, file_manager
|
||||||
from core.workflow.node_events import (
|
from core.workflow.node_events import (
|
||||||
ModelInvokeCompletedEvent,
|
ModelInvokeCompletedEvent,
|
||||||
NodeEventBase,
|
NodeEventBase,
|
||||||
@ -101,7 +101,7 @@ from .exc import (
|
|||||||
from .file_saver import FileSaverImpl, LLMFileSaver
|
from .file_saver import FileSaverImpl, LLMFileSaver
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
from core.file.models import File
|
from core.workflow.file.models import File
|
||||||
from core.workflow.runtime import GraphRuntimeState
|
from core.workflow.runtime import GraphRuntimeState
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|||||||
@ -71,9 +71,9 @@ class LoopNode(LLMUsageTrackingMixin, Node[LoopNodeData]):
|
|||||||
if self.node_data.loop_variables:
|
if self.node_data.loop_variables:
|
||||||
value_processor: dict[Literal["constant", "variable"], Callable[[LoopVariableData], Segment | None]] = {
|
value_processor: dict[Literal["constant", "variable"], Callable[[LoopVariableData], Segment | None]] = {
|
||||||
"constant": lambda var: self._get_segment_for_constant(var.var_type, var.value),
|
"constant": lambda var: self._get_segment_for_constant(var.var_type, var.value),
|
||||||
"variable": lambda var: self.graph_runtime_state.variable_pool.get(var.value)
|
"variable": lambda var: (
|
||||||
if isinstance(var.value, list)
|
self.graph_runtime_state.variable_pool.get(var.value) if isinstance(var.value, list) else None
|
||||||
else None,
|
),
|
||||||
}
|
}
|
||||||
for loop_variable in self.node_data.loop_variables:
|
for loop_variable in self.node_data.loop_variables:
|
||||||
if loop_variable.value_type not in value_processor:
|
if loop_variable.value_type not in value_processor:
|
||||||
|
|||||||
@ -6,7 +6,6 @@ from collections.abc import Mapping, Sequence
|
|||||||
from typing import Any, cast
|
from typing import Any, cast
|
||||||
|
|
||||||
from core.app.entities.app_invoke_entities import ModelConfigWithCredentialsEntity
|
from core.app.entities.app_invoke_entities import ModelConfigWithCredentialsEntity
|
||||||
from core.file import File
|
|
||||||
from core.memory.token_buffer_memory import TokenBufferMemory
|
from core.memory.token_buffer_memory import TokenBufferMemory
|
||||||
from core.model_manager import ModelInstance
|
from core.model_manager import ModelInstance
|
||||||
from core.model_runtime.entities import ImagePromptMessageContent
|
from core.model_runtime.entities import ImagePromptMessageContent
|
||||||
@ -28,6 +27,7 @@ from core.prompt.simple_prompt_transform import ModelMode
|
|||||||
from core.prompt.utils.prompt_message_util import PromptMessageUtil
|
from core.prompt.utils.prompt_message_util import PromptMessageUtil
|
||||||
from core.variables.types import ArrayValidation, SegmentType
|
from core.variables.types import ArrayValidation, SegmentType
|
||||||
from core.workflow.enums import NodeType, WorkflowNodeExecutionMetadataKey, WorkflowNodeExecutionStatus
|
from core.workflow.enums import NodeType, WorkflowNodeExecutionMetadataKey, WorkflowNodeExecutionStatus
|
||||||
|
from core.workflow.file import File
|
||||||
from core.workflow.node_events import NodeRunResult
|
from core.workflow.node_events import NodeRunResult
|
||||||
from core.workflow.nodes.base import variable_template_parser
|
from core.workflow.nodes.base import variable_template_parser
|
||||||
from core.workflow.nodes.base.node import Node
|
from core.workflow.nodes.base.node import Node
|
||||||
|
|||||||
@ -2,7 +2,7 @@ from typing import Any, Protocol
|
|||||||
|
|
||||||
import httpx
|
import httpx
|
||||||
|
|
||||||
from core.file import File
|
from core.workflow.file import File
|
||||||
|
|
||||||
|
|
||||||
class HttpClientProtocol(Protocol):
|
class HttpClientProtocol(Protocol):
|
||||||
|
|||||||
@ -39,7 +39,7 @@ from .template_prompts import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
from core.file.models import File
|
from core.workflow.file.models import File
|
||||||
from core.workflow.runtime import GraphRuntimeState
|
from core.workflow.runtime import GraphRuntimeState
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@ -5,7 +5,6 @@ from sqlalchemy import select
|
|||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
from core.callback_handler.workflow_tool_callback_handler import DifyWorkflowCallbackHandler
|
from core.callback_handler.workflow_tool_callback_handler import DifyWorkflowCallbackHandler
|
||||||
from core.file import File, FileTransferMethod
|
|
||||||
from core.model_runtime.entities.llm_entities import LLMUsage
|
from core.model_runtime.entities.llm_entities import LLMUsage
|
||||||
from core.tools.__base.tool import Tool
|
from core.tools.__base.tool import Tool
|
||||||
from core.tools.entities.tool_entities import ToolInvokeMessage, ToolParameter
|
from core.tools.entities.tool_entities import ToolInvokeMessage, ToolParameter
|
||||||
@ -20,6 +19,7 @@ from core.workflow.enums import (
|
|||||||
WorkflowNodeExecutionMetadataKey,
|
WorkflowNodeExecutionMetadataKey,
|
||||||
WorkflowNodeExecutionStatus,
|
WorkflowNodeExecutionStatus,
|
||||||
)
|
)
|
||||||
|
from core.workflow.file import File, FileTransferMethod
|
||||||
from core.workflow.node_events import NodeEventBase, NodeRunResult, StreamChunkEvent, StreamCompletedEvent
|
from core.workflow.node_events import NodeEventBase, NodeRunResult, StreamChunkEvent, StreamCompletedEvent
|
||||||
from core.workflow.nodes.base.node import Node
|
from core.workflow.nodes.base.node import Node
|
||||||
from core.workflow.nodes.base.variable_template_parser import VariableTemplateParser
|
from core.workflow.nodes.base.variable_template_parser import VariableTemplateParser
|
||||||
|
|||||||
@ -2,12 +2,12 @@ import logging
|
|||||||
from collections.abc import Mapping
|
from collections.abc import Mapping
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
from core.file import FileTransferMethod
|
|
||||||
from core.variables.types import SegmentType
|
from core.variables.types import SegmentType
|
||||||
from core.variables.variables import FileVariable
|
from core.variables.variables import FileVariable
|
||||||
from core.workflow.constants import SYSTEM_VARIABLE_NODE_ID
|
from core.workflow.constants import SYSTEM_VARIABLE_NODE_ID
|
||||||
from core.workflow.entities.workflow_node_execution import WorkflowNodeExecutionStatus
|
from core.workflow.entities.workflow_node_execution import WorkflowNodeExecutionStatus
|
||||||
from core.workflow.enums import NodeExecutionType, NodeType
|
from core.workflow.enums import NodeExecutionType, NodeType
|
||||||
|
from core.workflow.file import FileTransferMethod
|
||||||
from core.workflow.node_events import NodeRunResult
|
from core.workflow.node_events import NodeRunResult
|
||||||
from core.workflow.nodes.base.node import Node
|
from core.workflow.nodes.base.node import Node
|
||||||
from factories import file_factory
|
from factories import file_factory
|
||||||
|
|||||||
@ -8,7 +8,6 @@ from typing import Annotated, Any, Union, cast
|
|||||||
|
|
||||||
from pydantic import BaseModel, Field
|
from pydantic import BaseModel, Field
|
||||||
|
|
||||||
from core.file import File, FileAttribute, file_manager
|
|
||||||
from core.variables import Segment, SegmentGroup, VariableBase
|
from core.variables import Segment, SegmentGroup, VariableBase
|
||||||
from core.variables.consts import SELECTORS_LENGTH
|
from core.variables.consts import SELECTORS_LENGTH
|
||||||
from core.variables.segments import FileSegment, ObjectSegment
|
from core.variables.segments import FileSegment, ObjectSegment
|
||||||
@ -19,6 +18,7 @@ from core.workflow.constants import (
|
|||||||
RAG_PIPELINE_VARIABLE_NODE_ID,
|
RAG_PIPELINE_VARIABLE_NODE_ID,
|
||||||
SYSTEM_VARIABLE_NODE_ID,
|
SYSTEM_VARIABLE_NODE_ID,
|
||||||
)
|
)
|
||||||
|
from core.workflow.file import File, FileAttribute, file_manager
|
||||||
from core.workflow.system_variable import SystemVariable
|
from core.workflow.system_variable import SystemVariable
|
||||||
from factories import variable_factory
|
from factories import variable_factory
|
||||||
|
|
||||||
|
|||||||
@ -7,8 +7,8 @@ from uuid import uuid4
|
|||||||
|
|
||||||
from pydantic import AliasChoices, BaseModel, ConfigDict, Field, model_validator
|
from pydantic import AliasChoices, BaseModel, ConfigDict, Field, model_validator
|
||||||
|
|
||||||
from core.file.models import File
|
|
||||||
from core.workflow.enums import SystemVariableKey
|
from core.workflow.enums import SystemVariableKey
|
||||||
|
from core.workflow.file.models import File
|
||||||
|
|
||||||
|
|
||||||
class SystemVariable(BaseModel):
|
class SystemVariable(BaseModel):
|
||||||
|
|||||||
@ -2,9 +2,9 @@ import json
|
|||||||
from collections.abc import Mapping, Sequence
|
from collections.abc import Mapping, Sequence
|
||||||
from typing import Literal, NamedTuple
|
from typing import Literal, NamedTuple
|
||||||
|
|
||||||
from core.file import FileAttribute, file_manager
|
|
||||||
from core.variables import ArrayFileSegment
|
from core.variables import ArrayFileSegment
|
||||||
from core.variables.segments import ArrayBooleanSegment, BooleanSegment
|
from core.variables.segments import ArrayBooleanSegment, BooleanSegment
|
||||||
|
from core.workflow.file import FileAttribute, file_manager
|
||||||
from core.workflow.runtime import VariablePool
|
from core.workflow.runtime import VariablePool
|
||||||
|
|
||||||
from .entities import Condition, SubCondition, SupportedComparisonOperator
|
from .entities import Condition, SubCondition, SupportedComparisonOperator
|
||||||
|
|||||||
@ -9,10 +9,10 @@ from core.app.apps.exc import GenerateTaskStoppedError
|
|||||||
from core.app.entities.app_invoke_entities import InvokeFrom
|
from core.app.entities.app_invoke_entities import InvokeFrom
|
||||||
from core.app.workflow.layers.observability import ObservabilityLayer
|
from core.app.workflow.layers.observability import ObservabilityLayer
|
||||||
from core.app.workflow.node_factory import DifyNodeFactory
|
from core.app.workflow.node_factory import DifyNodeFactory
|
||||||
from core.file.models import File
|
|
||||||
from core.workflow.constants import ENVIRONMENT_VARIABLE_NODE_ID
|
from core.workflow.constants import ENVIRONMENT_VARIABLE_NODE_ID
|
||||||
from core.workflow.entities import GraphInitParams
|
from core.workflow.entities import GraphInitParams
|
||||||
from core.workflow.errors import WorkflowNodeRunFailedError
|
from core.workflow.errors import WorkflowNodeRunFailedError
|
||||||
|
from core.workflow.file.models import File
|
||||||
from core.workflow.graph import Graph
|
from core.workflow.graph import Graph
|
||||||
from core.workflow.graph_engine import GraphEngine, GraphEngineConfig
|
from core.workflow.graph_engine import GraphEngine, GraphEngineConfig
|
||||||
from core.workflow.graph_engine.command_channels import InMemoryChannel
|
from core.workflow.graph_engine.command_channels import InMemoryChannel
|
||||||
|
|||||||
@ -4,8 +4,8 @@ from typing import Any, overload
|
|||||||
|
|
||||||
from pydantic import BaseModel
|
from pydantic import BaseModel
|
||||||
|
|
||||||
from core.file.models import File
|
|
||||||
from core.variables import Segment
|
from core.variables import Segment
|
||||||
|
from core.workflow.file.models import File
|
||||||
|
|
||||||
|
|
||||||
class WorkflowRuntimeTypeConverter:
|
class WorkflowRuntimeTypeConverter:
|
||||||
|
|||||||
@ -124,3 +124,6 @@ storage = Storage()
|
|||||||
|
|
||||||
def init_app(app: DifyApp):
|
def init_app(app: DifyApp):
|
||||||
storage.init_app(app)
|
storage.init_app(app)
|
||||||
|
from core.app.workflow.file_runtime import bind_dify_workflow_file_runtime
|
||||||
|
|
||||||
|
bind_dify_workflow_file_runtime()
|
||||||
|
|||||||
8
api/tests/conftest.py
Normal file
8
api/tests/conftest.py
Normal file
@ -0,0 +1,8 @@
|
|||||||
|
import pytest
|
||||||
|
|
||||||
|
from core.app.workflow.file_runtime import bind_dify_workflow_file_runtime
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(autouse=True)
|
||||||
|
def _bind_workflow_file_runtime() -> None:
|
||||||
|
bind_dify_workflow_file_runtime()
|
||||||
@ -6,10 +6,10 @@ import httpx
|
|||||||
import pytest
|
import pytest
|
||||||
from sqlalchemy import Engine
|
from sqlalchemy import Engine
|
||||||
|
|
||||||
from core.file import FileTransferMethod, FileType, models
|
|
||||||
from core.helper import ssrf_proxy
|
from core.helper import ssrf_proxy
|
||||||
from core.tools import signature
|
from core.tools import signature
|
||||||
from core.tools.tool_file_manager import ToolFileManager
|
from core.tools.tool_file_manager import ToolFileManager
|
||||||
|
from core.workflow.file import FileTransferMethod, FileType, models
|
||||||
from core.workflow.nodes.llm.file_saver import (
|
from core.workflow.nodes.llm.file_saver import (
|
||||||
FileSaverImpl,
|
FileSaverImpl,
|
||||||
_extract_content_type_and_extension,
|
_extract_content_type_and_extension,
|
||||||
|
|||||||
@ -8,7 +8,6 @@ import pytest
|
|||||||
from core.app.entities.app_invoke_entities import InvokeFrom, ModelConfigWithCredentialsEntity
|
from core.app.entities.app_invoke_entities import InvokeFrom, ModelConfigWithCredentialsEntity
|
||||||
from core.entities.provider_configuration import ProviderConfiguration, ProviderModelBundle
|
from core.entities.provider_configuration import ProviderConfiguration, ProviderModelBundle
|
||||||
from core.entities.provider_entities import CustomConfiguration, SystemConfiguration
|
from core.entities.provider_entities import CustomConfiguration, SystemConfiguration
|
||||||
from core.file import File, FileTransferMethod, FileType
|
|
||||||
from core.model_runtime.entities.common_entities import I18nObject
|
from core.model_runtime.entities.common_entities import I18nObject
|
||||||
from core.model_runtime.entities.message_entities import (
|
from core.model_runtime.entities.message_entities import (
|
||||||
ImagePromptMessageContent,
|
ImagePromptMessageContent,
|
||||||
@ -21,6 +20,7 @@ from core.model_runtime.entities.model_entities import AIModelEntity, FetchFrom,
|
|||||||
from core.model_runtime.model_providers.model_provider_factory import ModelProviderFactory
|
from core.model_runtime.model_providers.model_provider_factory import ModelProviderFactory
|
||||||
from core.variables import ArrayAnySegment, ArrayFileSegment, NoneSegment
|
from core.variables import ArrayAnySegment, ArrayFileSegment, NoneSegment
|
||||||
from core.workflow.entities import GraphInitParams
|
from core.workflow.entities import GraphInitParams
|
||||||
|
from core.workflow.file import File, FileTransferMethod, FileType
|
||||||
from core.workflow.nodes.llm import llm_utils
|
from core.workflow.nodes.llm import llm_utils
|
||||||
from core.workflow.nodes.llm.entities import (
|
from core.workflow.nodes.llm.entities import (
|
||||||
ContextConfig,
|
ContextConfig,
|
||||||
|
|||||||
@ -2,9 +2,9 @@ from collections.abc import Mapping, Sequence
|
|||||||
|
|
||||||
from pydantic import BaseModel, Field
|
from pydantic import BaseModel, Field
|
||||||
|
|
||||||
from core.file import File
|
|
||||||
from core.model_runtime.entities.message_entities import PromptMessage
|
from core.model_runtime.entities.message_entities import PromptMessage
|
||||||
from core.model_runtime.entities.model_entities import ModelFeature
|
from core.model_runtime.entities.model_entities import ModelFeature
|
||||||
|
from core.workflow.file import File
|
||||||
from core.workflow.nodes.llm.entities import LLMNodeChatModelMessage
|
from core.workflow.nodes.llm.entities import LLMNodeChatModelMessage
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@ -6,12 +6,12 @@ import pytest
|
|||||||
from docx.oxml.text.paragraph import CT_P
|
from docx.oxml.text.paragraph import CT_P
|
||||||
|
|
||||||
from core.app.entities.app_invoke_entities import InvokeFrom
|
from core.app.entities.app_invoke_entities import InvokeFrom
|
||||||
from core.file import File, FileTransferMethod
|
|
||||||
from core.variables import ArrayFileSegment
|
from core.variables import ArrayFileSegment
|
||||||
from core.variables.segments import ArrayStringSegment
|
from core.variables.segments import ArrayStringSegment
|
||||||
from core.variables.variables import StringVariable
|
from core.variables.variables import StringVariable
|
||||||
from core.workflow.entities import GraphInitParams
|
from core.workflow.entities import GraphInitParams
|
||||||
from core.workflow.enums import NodeType, WorkflowNodeExecutionStatus
|
from core.workflow.enums import NodeType, WorkflowNodeExecutionStatus
|
||||||
|
from core.workflow.file import File, FileTransferMethod
|
||||||
from core.workflow.node_events import NodeRunResult
|
from core.workflow.node_events import NodeRunResult
|
||||||
from core.workflow.nodes.document_extractor import DocumentExtractorNode, DocumentExtractorNodeData
|
from core.workflow.nodes.document_extractor import DocumentExtractorNode, DocumentExtractorNodeData
|
||||||
from core.workflow.nodes.document_extractor.node import (
|
from core.workflow.nodes.document_extractor.node import (
|
||||||
@ -146,7 +146,7 @@ def test_run_extract_text(
|
|||||||
mock_ssrf_proxy_get.return_value.content = file_content
|
mock_ssrf_proxy_get.return_value.content = file_content
|
||||||
mock_ssrf_proxy_get.return_value.raise_for_status = Mock()
|
mock_ssrf_proxy_get.return_value.raise_for_status = Mock()
|
||||||
|
|
||||||
monkeypatch.setattr("core.file.file_manager.download", mock_download)
|
monkeypatch.setattr("core.workflow.file.file_manager.download", mock_download)
|
||||||
monkeypatch.setattr("core.helper.ssrf_proxy.get", mock_ssrf_proxy_get)
|
monkeypatch.setattr("core.helper.ssrf_proxy.get", mock_ssrf_proxy_get)
|
||||||
|
|
||||||
if mime_type == "application/pdf":
|
if mime_type == "application/pdf":
|
||||||
|
|||||||
@ -6,10 +6,10 @@ import pytest
|
|||||||
|
|
||||||
from core.app.entities.app_invoke_entities import InvokeFrom
|
from core.app.entities.app_invoke_entities import InvokeFrom
|
||||||
from core.app.workflow.node_factory import DifyNodeFactory
|
from core.app.workflow.node_factory import DifyNodeFactory
|
||||||
from core.file import File, FileTransferMethod, FileType
|
|
||||||
from core.variables import ArrayFileSegment
|
from core.variables import ArrayFileSegment
|
||||||
from core.workflow.entities import GraphInitParams
|
from core.workflow.entities import GraphInitParams
|
||||||
from core.workflow.enums import WorkflowNodeExecutionStatus
|
from core.workflow.enums import WorkflowNodeExecutionStatus
|
||||||
|
from core.workflow.file import File, FileTransferMethod, FileType
|
||||||
from core.workflow.graph import Graph
|
from core.workflow.graph import Graph
|
||||||
from core.workflow.nodes.if_else.entities import IfElseNodeData
|
from core.workflow.nodes.if_else.entities import IfElseNodeData
|
||||||
from core.workflow.nodes.if_else.if_else_node import IfElseNode
|
from core.workflow.nodes.if_else.if_else_node import IfElseNode
|
||||||
|
|||||||
@ -3,9 +3,9 @@ from unittest.mock import MagicMock
|
|||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from core.app.entities.app_invoke_entities import InvokeFrom
|
from core.app.entities.app_invoke_entities import InvokeFrom
|
||||||
from core.file import File, FileTransferMethod, FileType
|
|
||||||
from core.variables import ArrayFileSegment
|
from core.variables import ArrayFileSegment
|
||||||
from core.workflow.enums import WorkflowNodeExecutionStatus
|
from core.workflow.enums import WorkflowNodeExecutionStatus
|
||||||
|
from core.workflow.file import File, FileTransferMethod, FileType
|
||||||
from core.workflow.nodes.list_operator.entities import (
|
from core.workflow.nodes.list_operator.entities import (
|
||||||
ExtractConfig,
|
ExtractConfig,
|
||||||
FilterBy,
|
FilterBy,
|
||||||
|
|||||||
@ -8,12 +8,12 @@ from unittest.mock import MagicMock, patch
|
|||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from core.file import File, FileTransferMethod, FileType
|
|
||||||
from core.model_runtime.entities.llm_entities import LLMUsage
|
from core.model_runtime.entities.llm_entities import LLMUsage
|
||||||
from core.tools.entities.tool_entities import ToolInvokeMessage
|
from core.tools.entities.tool_entities import ToolInvokeMessage
|
||||||
from core.tools.utils.message_transformer import ToolFileMessageTransformer
|
from core.tools.utils.message_transformer import ToolFileMessageTransformer
|
||||||
from core.variables.segments import ArrayFileSegment
|
from core.variables.segments import ArrayFileSegment
|
||||||
from core.workflow.entities import GraphInitParams
|
from core.workflow.entities import GraphInitParams
|
||||||
|
from core.workflow.file import File, FileTransferMethod, FileType
|
||||||
from core.workflow.node_events import StreamChunkEvent, StreamCompletedEvent
|
from core.workflow.node_events import StreamChunkEvent, StreamCompletedEvent
|
||||||
from core.workflow.runtime import GraphRuntimeState, VariablePool
|
from core.workflow.runtime import GraphRuntimeState, VariablePool
|
||||||
from core.workflow.system_variable import SystemVariable
|
from core.workflow.system_variable import SystemVariable
|
||||||
|
|||||||
@ -3,10 +3,10 @@ from unittest.mock import patch
|
|||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from core.app.entities.app_invoke_entities import InvokeFrom
|
from core.app.entities.app_invoke_entities import InvokeFrom
|
||||||
from core.file import File, FileTransferMethod, FileType
|
|
||||||
from core.variables import FileVariable, StringVariable
|
from core.variables import FileVariable, StringVariable
|
||||||
from core.workflow.entities.graph_init_params import GraphInitParams
|
from core.workflow.entities.graph_init_params import GraphInitParams
|
||||||
from core.workflow.entities.workflow_node_execution import WorkflowNodeExecutionStatus
|
from core.workflow.entities.workflow_node_execution import WorkflowNodeExecutionStatus
|
||||||
|
from core.workflow.file import File, FileTransferMethod, FileType
|
||||||
from core.workflow.nodes.trigger_webhook.entities import (
|
from core.workflow.nodes.trigger_webhook.entities import (
|
||||||
ContentType,
|
ContentType,
|
||||||
Method,
|
Method,
|
||||||
|
|||||||
@ -4,8 +4,8 @@ from typing import Any
|
|||||||
import pytest
|
import pytest
|
||||||
from pydantic import ValidationError
|
from pydantic import ValidationError
|
||||||
|
|
||||||
from core.file.enums import FileTransferMethod, FileType
|
from core.workflow.file.enums import FileTransferMethod, FileType
|
||||||
from core.file.models import File
|
from core.workflow.file.models import File
|
||||||
from core.workflow.system_variable import SystemVariable
|
from core.workflow.system_variable import SystemVariable
|
||||||
|
|
||||||
# Test data constants for SystemVariable serialization tests
|
# Test data constants for SystemVariable serialization tests
|
||||||
|
|||||||
@ -2,7 +2,7 @@ from typing import cast
|
|||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from core.file.models import File, FileTransferMethod, FileType
|
from core.workflow.file.models import File, FileTransferMethod, FileType
|
||||||
from core.workflow.system_variable import SystemVariable, SystemVariableReadOnlyView
|
from core.workflow.system_variable import SystemVariable, SystemVariableReadOnlyView
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@ -3,7 +3,6 @@ from collections import defaultdict
|
|||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from core.file import File, FileTransferMethod, FileType
|
|
||||||
from core.variables import FileSegment, StringSegment
|
from core.variables import FileSegment, StringSegment
|
||||||
from core.variables.segments import (
|
from core.variables.segments import (
|
||||||
ArrayAnySegment,
|
ArrayAnySegment,
|
||||||
@ -27,6 +26,7 @@ from core.variables.variables import (
|
|||||||
Variable,
|
Variable,
|
||||||
)
|
)
|
||||||
from core.workflow.constants import CONVERSATION_VARIABLE_NODE_ID, ENVIRONMENT_VARIABLE_NODE_ID, SYSTEM_VARIABLE_NODE_ID
|
from core.workflow.constants import CONVERSATION_VARIABLE_NODE_ID, ENVIRONMENT_VARIABLE_NODE_ID, SYSTEM_VARIABLE_NODE_ID
|
||||||
|
from core.workflow.file import File, FileTransferMethod, FileType
|
||||||
from core.workflow.runtime import VariablePool
|
from core.workflow.runtime import VariablePool
|
||||||
from core.workflow.system_variable import SystemVariable
|
from core.workflow.system_variable import SystemVariable
|
||||||
from factories.variable_factory import build_segment, segment_to_variable
|
from factories.variable_factory import build_segment, segment_to_variable
|
||||||
|
|||||||
@ -3,14 +3,14 @@ from types import SimpleNamespace
|
|||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from configs import dify_config
|
from configs import dify_config
|
||||||
from core.file.enums import FileType
|
|
||||||
from core.file.models import File, FileTransferMethod
|
|
||||||
from core.helper.code_executor.code_executor import CodeLanguage
|
from core.helper.code_executor.code_executor import CodeLanguage
|
||||||
from core.variables.variables import StringVariable
|
from core.variables.variables import StringVariable
|
||||||
from core.workflow.constants import (
|
from core.workflow.constants import (
|
||||||
CONVERSATION_VARIABLE_NODE_ID,
|
CONVERSATION_VARIABLE_NODE_ID,
|
||||||
ENVIRONMENT_VARIABLE_NODE_ID,
|
ENVIRONMENT_VARIABLE_NODE_ID,
|
||||||
)
|
)
|
||||||
|
from core.workflow.file.enums import FileType
|
||||||
|
from core.workflow.file.models import File, FileTransferMethod
|
||||||
from core.workflow.nodes.code.code_node import CodeNode
|
from core.workflow.nodes.code.code_node import CodeNode
|
||||||
from core.workflow.nodes.code.limits import CodeNodeLimits
|
from core.workflow.nodes.code.limits import CodeNodeLimits
|
||||||
from core.workflow.runtime import VariablePool
|
from core.workflow.runtime import VariablePool
|
||||||
|
|||||||
Reference in New Issue
Block a user