refactor(api): continue decoupling dify_graph from API concerns (#33580)

Signed-off-by: -LAN- <laipz8200@outlook.com>
Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>
Co-authored-by: WH-2099 <wh2099@pm.me>

api/tests/integration_tests/core/workflow/nodes/datasource/test_datasource_node_integration.py

@@ -1,5 +1,5 @@
+from core.app.entities.app_invoke_entities import DIFY_RUN_CONTEXT_KEY
 from core.workflow.nodes.datasource.datasource_node import DatasourceNode
-from dify_graph.entities.graph_init_params import DIFY_RUN_CONTEXT_KEY
 from dify_graph.entities.workflow_node_execution import WorkflowNodeExecutionStatus
 from dify_graph.node_events import NodeRunResult, StreamCompletedEvent
 

api/tests/integration_tests/factories/test_storage_key_loader.py

@@ -6,6 +6,7 @@ from uuid import uuid4
 import pytest
 from sqlalchemy.orm import Session
 
+from core.app.file_access import DatabaseFileAccessController
 from dify_graph.file import File, FileTransferMethod, FileType
 from extensions.ext_database import db
 from extensions.storage.storage_type import StorageType
@@ -35,7 +36,11 @@ class TestStorageKeyLoader(unittest.TestCase):
         self.test_tool_files = []
 
         # Create StorageKeyLoader instance
-        self.loader = StorageKeyLoader(self.session, self.tenant_id)
+        self.loader = StorageKeyLoader(
+            self.session,
+            self.tenant_id,
+            access_controller=DatabaseFileAccessController(),
+        )
 
     def tearDown(self):
         """Clean up test data after each test method."""
@@ -192,19 +197,16 @@ class TestStorageKeyLoader(unittest.TestCase):
         # Should not raise any exceptions
         self.loader.load_storage_keys([])
 
-    def test_load_storage_keys_tenant_mismatch(self):
-        """Test tenant_id validation."""
-        # Create file with different tenant_id
+    def test_load_storage_keys_ignores_legacy_file_tenant_id(self):
+        """Legacy file tenant_id should not override the loader tenant scope."""
         upload_file = self._create_upload_file()
         file = self._create_file(
             related_id=upload_file.id, transfer_method=FileTransferMethod.LOCAL_FILE, tenant_id=str(uuid4())
         )
 
-        # Should raise ValueError for tenant mismatch
-        with pytest.raises(ValueError) as context:
-            self.loader.load_storage_keys([file])
+        self.loader.load_storage_keys([file])
 
-        assert "invalid file, expected tenant_id" in str(context.value)
+        assert file._storage_key == upload_file.key
 
     def test_load_storage_keys_missing_file_id(self):
         """Test with None file.related_id."""
@@ -313,7 +315,7 @@ class TestStorageKeyLoader(unittest.TestCase):
         with pytest.raises(ValueError) as context:
             self.loader.load_storage_keys([file_other])
 
-        assert "invalid file, expected tenant_id" in str(context.value)
+        assert "Upload file not found for id:" in str(context.value)
 
         # Current tenant's file should still work
         self.loader.load_storage_keys([file_current])
@@ -337,7 +339,7 @@ class TestStorageKeyLoader(unittest.TestCase):
         with pytest.raises(ValueError) as context:
             self.loader.load_storage_keys([file_current, file_other])
 
-        assert "invalid file, expected tenant_id" in str(context.value)
+        assert "Upload file not found for id:" in str(context.value)
 
     def test_load_storage_keys_duplicate_file_ids(self):
         """Test handling of duplicate file IDs in the batch."""
@@ -364,6 +366,10 @@ class TestStorageKeyLoader(unittest.TestCase):
         # Create loader with different session (same underlying connection)
 
         with Session(bind=db.engine) as other_session:
-            other_loader = StorageKeyLoader(other_session, self.tenant_id)
+            other_loader = StorageKeyLoader(
+                other_session,
+                self.tenant_id,
+                access_controller=DatabaseFileAccessController(),
+            )
             with pytest.raises(ValueError):
                 other_loader.load_storage_keys([file])

api/tests/integration_tests/services/test_workflow_draft_variable_service.py

@@ -6,7 +6,7 @@ import pytest
 from sqlalchemy import delete
 from sqlalchemy.orm import Session
 
-from dify_graph.constants import CONVERSATION_VARIABLE_NODE_ID, SYSTEM_VARIABLE_NODE_ID
+from core.workflow.variable_prefixes import CONVERSATION_VARIABLE_NODE_ID, SYSTEM_VARIABLE_NODE_ID
 from dify_graph.nodes import BuiltinNodeTypes
 from dify_graph.variables.segments import StringSegment
 from dify_graph.variables.types import SegmentType

api/tests/integration_tests/workflow/nodes/__mock/model.py

@@ -4,8 +4,8 @@ from core.app.entities.app_invoke_entities import ModelConfigWithCredentialsEnti
 from core.entities.provider_configuration import ProviderConfiguration, ProviderModelBundle
 from core.entities.provider_entities import CustomConfiguration, CustomProviderConfiguration, SystemConfiguration
 from core.model_manager import ModelInstance
+from core.plugin.impl.model_runtime_factory import create_plugin_model_provider_factory
 from dify_graph.model_runtime.entities.model_entities import ModelType
-from dify_graph.model_runtime.model_providers.model_provider_factory import ModelProviderFactory
 from models.provider import ProviderType
 
 
@@ -15,7 +15,7 @@ def get_mocked_fetch_model_config(
     mode: str,
     credentials: dict,
 ):
-    model_provider_factory = ModelProviderFactory(tenant_id="9d2074fc-6f86-45a9-b09d-6ecc63b9056b")
+    model_provider_factory = create_plugin_model_provider_factory(tenant_id="9d2074fc-6f86-45a9-b09d-6ecc63b9056b")
     model_type_instance = model_provider_factory.get_model_type_instance(provider, ModelType.LLM)
     provider_model_bundle = ProviderModelBundle(
         configuration=ProviderConfiguration(

api/tests/integration_tests/workflow/nodes/test_code.py

@@ -6,13 +6,13 @@ import pytest
 from configs import dify_config
 from core.app.entities.app_invoke_entities import InvokeFrom, UserFrom
 from core.workflow.node_factory import DifyNodeFactory
+from core.workflow.system_variables import build_system_variables
 from dify_graph.enums import WorkflowNodeExecutionStatus
 from dify_graph.graph import Graph
 from dify_graph.node_events import NodeRunResult
 from dify_graph.nodes.code.code_node import CodeNode
 from dify_graph.nodes.code.limits import CodeNodeLimits
 from dify_graph.runtime import GraphRuntimeState, VariablePool
-from dify_graph.system_variable import SystemVariable
 from tests.integration_tests.workflow.nodes.__mock.code_executor import setup_code_executor_mock
 from tests.workflow_test_utils import build_test_graph_init_params
 
@@ -44,7 +44,7 @@ def init_code_node(code_config: dict):
 
     # construct variable pool
     variable_pool = VariablePool(
-        system_variables=SystemVariable(user_id="aaa", files=[]),
+        system_variables=build_system_variables(user_id="aaa", files=[]),
         user_inputs={},
         environment_variables=[],
         conversation_variables=[],

api/tests/integration_tests/workflow/nodes/test_http.py

@@ -9,12 +9,13 @@ from core.app.entities.app_invoke_entities import InvokeFrom, UserFrom
 from core.helper.ssrf_proxy import ssrf_proxy
 from core.tools.tool_file_manager import ToolFileManager
 from core.workflow.node_factory import DifyNodeFactory
+from core.workflow.node_runtime import DifyFileReferenceFactory
+from core.workflow.system_variables import build_system_variables
 from dify_graph.enums import WorkflowNodeExecutionStatus
 from dify_graph.file.file_manager import file_manager
 from dify_graph.graph import Graph
 from dify_graph.nodes.http_request import HttpRequestNode, HttpRequestNodeConfig
 from dify_graph.runtime import GraphRuntimeState, VariablePool
-from dify_graph.system_variable import SystemVariable
 from tests.integration_tests.workflow.nodes.__mock.http import setup_http_mock
 from tests.workflow_test_utils import build_test_graph_init_params
 
@@ -54,7 +55,7 @@ def init_http_node(config: dict):
 
     # construct variable pool
     variable_pool = VariablePool(
-        system_variables=SystemVariable(user_id="aaa", files=[]),
+        system_variables=build_system_variables(user_id="aaa", files=[]),
         user_inputs={},
         environment_variables=[],
         conversation_variables=[],
@@ -81,6 +82,7 @@ def init_http_node(config: dict):
         http_client=ssrf_proxy,
         tool_file_manager_factory=ToolFileManager,
         file_manager=file_manager,
+        file_reference_factory=DifyFileReferenceFactory(init_params.run_context),
     )
 
     return node
@@ -189,6 +191,7 @@ def test_custom_authorization_header(setup_http_mock):
 @pytest.mark.parametrize("setup_http_mock", [["none"]], indirect=True)
 def test_custom_auth_with_empty_api_key_raises_error(setup_http_mock):
     """Test: In custom authentication mode, when the api_key is empty, AuthorizationConfigError should be raised."""
+    from core.workflow.system_variables import build_system_variables
     from dify_graph.enums import BuiltinNodeTypes
     from dify_graph.nodes.http_request.entities import (
         HttpRequestNodeAuthorization,
@@ -198,11 +201,10 @@ def test_custom_auth_with_empty_api_key_raises_error(setup_http_mock):
     from dify_graph.nodes.http_request.exc import AuthorizationConfigError
     from dify_graph.nodes.http_request.executor import Executor
     from dify_graph.runtime import VariablePool
-    from dify_graph.system_variable import SystemVariable
 
     # Create variable pool
     variable_pool = VariablePool(
-        system_variables=SystemVariable(user_id="test", files=[]),
+        system_variables=build_system_variables(user_id="test", files=[]),
         user_inputs={},
         environment_variables=[],
         conversation_variables=[],
@@ -700,7 +702,7 @@ def test_nested_object_variable_selector(setup_http_mock):
 
     # Create independent variable pool for this test only
     variable_pool = VariablePool(
-        system_variables=SystemVariable(user_id="aaa", files=[]),
+        system_variables=build_system_variables(user_id="aaa", files=[]),
         user_inputs={},
         environment_variables=[],
         conversation_variables=[],
@@ -728,6 +730,7 @@ def test_nested_object_variable_selector(setup_http_mock):
         http_client=ssrf_proxy,
         tool_file_manager_factory=ToolFileManager,
         file_manager=file_manager,
+        file_reference_factory=DifyFileReferenceFactory(init_params.run_context),
     )
 
     result = node._run()

api/tests/integration_tests/workflow/nodes/test_llm.py

@@ -7,13 +7,15 @@ from unittest.mock import MagicMock, patch
 from core.app.entities.app_invoke_entities import InvokeFrom, UserFrom
 from core.llm_generator.output_parser.structured_output import _parse_structured_output
 from core.model_manager import ModelInstance
+from core.workflow.system_variables import build_system_variables
 from dify_graph.enums import WorkflowNodeExecutionStatus
 from dify_graph.node_events import StreamCompletedEvent
+from dify_graph.nodes.llm.file_saver import LLMFileSaver
 from dify_graph.nodes.llm.node import LLMNode
-from dify_graph.nodes.llm.protocols import CredentialsProvider, ModelFactory, TemplateRenderer
+from dify_graph.nodes.llm.protocols import CredentialsProvider, ModelFactory
+from dify_graph.nodes.llm.runtime_protocols import PromptMessageSerializerProtocol
 from dify_graph.nodes.protocols import HttpClientProtocol
 from dify_graph.runtime import GraphRuntimeState, VariablePool
-from dify_graph.system_variable import SystemVariable
 from extensions.ext_database import db
 from tests.workflow_test_utils import build_test_graph_init_params
 
@@ -51,7 +53,7 @@ def init_llm_node(config: dict) -> LLMNode:
 
     # construct variable pool
     variable_pool = VariablePool(
-        system_variables=SystemVariable(
+        system_variables=build_system_variables(
             user_id="aaa",
             app_id=app_id,
             workflow_id=workflow_id,
@@ -66,6 +68,11 @@ def init_llm_node(config: dict) -> LLMNode:
     variable_pool.add(["abc", "output"], "sunny")
 
     graph_runtime_state = GraphRuntimeState(variable_pool=variable_pool, start_at=time.perf_counter())
+    prompt_message_serializer = MagicMock(spec=PromptMessageSerializerProtocol)
+    prompt_message_serializer.serialize.side_effect = lambda *, model_mode, prompt_messages: [
+        message.model_dump(mode="json") for message in prompt_messages
+    ]
+    llm_file_saver = MagicMock(spec=LLMFileSaver)
 
     node = LLMNode(
         id=str(uuid.uuid4()),
@@ -75,7 +82,8 @@ def init_llm_node(config: dict) -> LLMNode:
         credentials_provider=MagicMock(spec=CredentialsProvider),
         model_factory=MagicMock(spec=ModelFactory),
         model_instance=MagicMock(spec=ModelInstance),
-        template_renderer=MagicMock(spec=TemplateRenderer),
+        llm_file_saver=llm_file_saver,
+        prompt_message_serializer=prompt_message_serializer,
         http_client=MagicMock(spec=HttpClientProtocol),
     )
 
@@ -159,7 +167,7 @@ def test_execute_llm():
         return mock_model_instance
 
     # Mock fetch_prompt_messages to avoid database calls
-    def mock_fetch_prompt_messages_1(*_args, **_kwargs):
+    def mock_fetch_prompt_messages_1(**_kwargs):
         from dify_graph.model_runtime.entities.message_entities import SystemPromptMessage, UserPromptMessage
 
         return [

api/tests/integration_tests/workflow/nodes/test_parameter_extractor.py

@@ -5,12 +5,13 @@ from unittest.mock import MagicMock
 
 from core.app.entities.app_invoke_entities import InvokeFrom, UserFrom
 from core.model_manager import ModelInstance
+from core.workflow.node_runtime import DifyPromptMessageSerializer
+from core.workflow.system_variables import build_system_variables
 from dify_graph.enums import WorkflowNodeExecutionStatus
 from dify_graph.model_runtime.entities import AssistantPromptMessage, UserPromptMessage
 from dify_graph.nodes.llm.protocols import CredentialsProvider, ModelFactory
 from dify_graph.nodes.parameter_extractor.parameter_extractor_node import ParameterExtractorNode
 from dify_graph.runtime import GraphRuntimeState, VariablePool
-from dify_graph.system_variable import SystemVariable
 from extensions.ext_database import db
 from tests.integration_tests.workflow.nodes.__mock.model import get_mocked_fetch_model_instance
 from tests.workflow_test_utils import build_test_graph_init_params
@@ -56,7 +57,7 @@ def init_parameter_extractor_node(config: dict, memory=None):
 
     # construct variable pool
     variable_pool = VariablePool(
-        system_variables=SystemVariable(
+        system_variables=build_system_variables(
             user_id="aaa", files=[], query="what's the weather in SF", conversation_id="abababa"
         ),
         user_inputs={},
@@ -77,6 +78,7 @@ def init_parameter_extractor_node(config: dict, memory=None):
         model_factory=MagicMock(spec=ModelFactory),
         model_instance=MagicMock(spec=ModelInstance),
         memory=memory,
+        prompt_message_serializer=DifyPromptMessageSerializer(),
     )
     return node
 

api/tests/integration_tests/workflow/nodes/test_template_transform.py

@@ -3,12 +3,12 @@ import uuid
 
 from core.app.entities.app_invoke_entities import InvokeFrom, UserFrom
 from core.workflow.node_factory import DifyNodeFactory
+from core.workflow.system_variables import build_system_variables
 from dify_graph.enums import WorkflowNodeExecutionStatus
 from dify_graph.graph import Graph
-from dify_graph.nodes.template_transform.template_renderer import TemplateRenderError
 from dify_graph.nodes.template_transform.template_transform_node import TemplateTransformNode
 from dify_graph.runtime import GraphRuntimeState, VariablePool
-from dify_graph.system_variable import SystemVariable
+from dify_graph.template_rendering import TemplateRenderError
 from tests.workflow_test_utils import build_test_graph_init_params
 
 
@@ -66,7 +66,7 @@ def test_execute_template_transform():
 
     # construct variable pool
     variable_pool = VariablePool(
-        system_variables=SystemVariable(user_id="aaa", files=[]),
+        system_variables=build_system_variables(user_id="aaa", files=[]),
         user_inputs={},
         environment_variables=[],
         conversation_variables=[],
@@ -90,7 +90,7 @@ def test_execute_template_transform():
         config=config,
         graph_init_params=init_params,
         graph_runtime_state=graph_runtime_state,
-        template_renderer=_SimpleJinja2Renderer(),
+        jinja2_template_renderer=_SimpleJinja2Renderer(),
     )
 
     # execute node

api/tests/integration_tests/workflow/nodes/test_tool.py

@@ -5,13 +5,14 @@ from unittest.mock import MagicMock, patch
 from core.app.entities.app_invoke_entities import InvokeFrom, UserFrom
 from core.tools.utils.configuration import ToolParameterConfigurationManager
 from core.workflow.node_factory import DifyNodeFactory
+from core.workflow.node_runtime import DifyToolNodeRuntime
+from core.workflow.system_variables import build_system_variables
 from dify_graph.enums import WorkflowNodeExecutionStatus
 from dify_graph.graph import Graph
 from dify_graph.node_events import StreamCompletedEvent
 from dify_graph.nodes.protocols import ToolFileManagerProtocol
 from dify_graph.nodes.tool.tool_node import ToolNode
 from dify_graph.runtime import GraphRuntimeState, VariablePool
-from dify_graph.system_variable import SystemVariable
 from tests.workflow_test_utils import build_test_graph_init_params
 
 
@@ -40,7 +41,7 @@ def init_tool_node(config: dict):
 
     # construct variable pool
     variable_pool = VariablePool(
-        system_variables=SystemVariable(user_id="aaa", files=[]),
+        system_variables=build_system_variables(user_id="aaa", files=[]),
         user_inputs={},
         environment_variables=[],
         conversation_variables=[],
@@ -64,6 +65,7 @@ def init_tool_node(config: dict):
         graph_init_params=init_params,
         graph_runtime_state=graph_runtime_state,
         tool_file_manager_factory=tool_file_manager_factory,
+        runtime=DifyToolNodeRuntime(init_params.run_context),
     )
     return node