youngkingdom/dify
1
0
Fork 0
mirror of https://github.com/langgenius/dify.git synced 2026-05-04 09:28:04 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: queue-based graph engine

Browse Source 
Signed-off-by: -LAN- <laipz8200@outlook.com>
This commit is contained in:
-LAN-
2025-08-22 03:29:46 +08:00
parent f04844435f
commit 8c35663220
363 changed files with 20911 additions and 8927 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
api/tests/integration_tests/workflow/nodes/test_code.py
View File

@ -6,17 +6,15 @@ from typing import cast
import pytest
from core.app.entities.app_invoke_entities import InvokeFrom
from core.workflow.entities.node_entities import NodeRunResult
from core.workflow.entities.variable_pool import VariablePool
from core.workflow.entities.workflow_node_execution import WorkflowNodeExecutionStatus
from core.workflow.graph_engine.entities.graph import Graph
from core.workflow.graph_engine.entities.graph_init_params import GraphInitParams
from core.workflow.graph_engine.entities.graph_runtime_state import GraphRuntimeState
from core.workflow.entities import GraphInitParams, GraphRuntimeState, VariablePool
from core.workflow.enums import WorkflowNodeExecutionStatus
from core.workflow.graph import Graph
from core.workflow.node_events import NodeRunResult
from core.workflow.nodes.code.code_node import CodeNode
from core.workflow.nodes.code.entities import CodeNodeData
from core.workflow.nodes.node_factory import DifyNodeFactory
from core.workflow.system_variable import SystemVariable
from models.enums import UserFrom
from models.workflow import WorkflowType
from tests.integration_tests.workflow.nodes.__mock.code_executor import setup_code_executor_mock
CODE_MAX_STRING_LENGTH = int(getenv("CODE_MAX_STRING_LENGTH", "10000"))
@ -31,15 +29,12 @@ def init_code_node(code_config: dict):
"target": "code",
},
],
"nodes": [{"data": {"type": "start"}, "id": "start"}, code_config],
"nodes": [{"data": {"type": "start", "title": "Start"}, "id": "start"}, code_config],
}
graph = Graph.init(graph_config=graph_config)
init_params = GraphInitParams(
tenant_id="1",
app_id="1",
workflow_type=WorkflowType.WORKFLOW,
workflow_id="1",
graph_config=graph_config,
user_id="1",
@ -58,12 +53,21 @@ def init_code_node(code_config: dict):
variable_pool.add(["code", "args1"], 1)
variable_pool.add(["code", "args2"], 2)
graph_runtime_state = GraphRuntimeState(variable_pool=variable_pool, start_at=time.perf_counter())
# Create node factory
node_factory = DifyNodeFactory(
graph_init_params=init_params,
graph_runtime_state=graph_runtime_state,
)
graph = Graph.init(graph_config=graph_config, node_factory=node_factory)
node = CodeNode(
id=str(uuid.uuid4()),
graph_init_params=init_params,
graph=graph,
graph_runtime_state=GraphRuntimeState(variable_pool=variable_pool, start_at=time.perf_counter()),
config=code_config,
graph_init_params=init_params,
graph_runtime_state=graph_runtime_state,
)
# Initialize node data
@ -116,7 +120,7 @@ def test_execute_code(setup_code_executor_mock):
assert result.status == WorkflowNodeExecutionStatus.SUCCEEDED
assert result.outputs is not None
assert result.outputs["result"] == 3
assert result.error is None
assert result.error == ""
@pytest.mark.parametrize("setup_code_executor_mock", [["none"]], indirect=True)

48
api/tests/integration_tests/workflow/nodes/test_http.py
View File

@ -5,14 +5,12 @@ from urllib.parse import urlencode
import pytest
from core.app.entities.app_invoke_entities import InvokeFrom
from core.workflow.entities.variable_pool import VariablePool
from core.workflow.graph_engine.entities.graph import Graph
from core.workflow.graph_engine.entities.graph_init_params import GraphInitParams
from core.workflow.graph_engine.entities.graph_runtime_state import GraphRuntimeState
from core.workflow.entities import GraphInitParams, GraphRuntimeState, VariablePool
from core.workflow.graph import Graph
from core.workflow.nodes.http_request.node import HttpRequestNode
from core.workflow.nodes.node_factory import DifyNodeFactory
from core.workflow.system_variable import SystemVariable
from models.enums import UserFrom
from models.workflow import WorkflowType
from tests.integration_tests.workflow.nodes.__mock.http import setup_http_mock
@ -25,15 +23,12 @@ def init_http_node(config: dict):
"target": "1",
},
],
"nodes": [{"data": {"type": "start"}, "id": "start"}, config],
"nodes": [{"data": {"type": "start", "title": "Start"}, "id": "start"}, config],
}
graph = Graph.init(graph_config=graph_config)
init_params = GraphInitParams(
tenant_id="1",
app_id="1",
workflow_type=WorkflowType.WORKFLOW,
workflow_id="1",
graph_config=graph_config,
user_id="1",
@ -52,12 +47,21 @@ def init_http_node(config: dict):
variable_pool.add(["a", "args1"], 1)
variable_pool.add(["a", "args2"], 2)
graph_runtime_state = GraphRuntimeState(variable_pool=variable_pool, start_at=time.perf_counter())
# Create node factory
node_factory = DifyNodeFactory(
graph_init_params=init_params,
graph_runtime_state=graph_runtime_state,
)
graph = Graph.init(graph_config=graph_config, node_factory=node_factory)
node = HttpRequestNode(
id=str(uuid.uuid4()),
graph_init_params=init_params,
graph=graph,
graph_runtime_state=GraphRuntimeState(variable_pool=variable_pool, start_at=time.perf_counter()),
config=config,
graph_init_params=init_params,
graph_runtime_state=graph_runtime_state,
)
# Initialize node data
@ -627,7 +631,7 @@ def test_nested_object_variable_selector(setup_http_mock):
},
],
"nodes": [
{"data": {"type": "start"}, "id": "start"},
{"data": {"type": "start", "title": "Start"}, "id": "start"},
{
"id": "1",
"data": {
@ -651,12 +655,9 @@ def test_nested_object_variable_selector(setup_http_mock):
],
}
graph = Graph.init(graph_config=graph_config)
init_params = GraphInitParams(
tenant_id="1",
app_id="1",
workflow_type=WorkflowType.WORKFLOW,
workflow_id="1",
graph_config=graph_config,
user_id="1",
@ -676,12 +677,21 @@ def test_nested_object_variable_selector(setup_http_mock):
variable_pool.add(["a", "args2"], 2)
variable_pool.add(["a", "args3"], {"nested": "nested_value"}) # Only for this test
graph_runtime_state = GraphRuntimeState(variable_pool=variable_pool, start_at=time.perf_counter())
# Create node factory
node_factory = DifyNodeFactory(
graph_init_params=init_params,
graph_runtime_state=graph_runtime_state,
)
graph = Graph.init(graph_config=graph_config, node_factory=node_factory)
node = HttpRequestNode(
id=str(uuid.uuid4()),
graph_init_params=init_params,
graph=graph,
graph_runtime_state=GraphRuntimeState(variable_pool=variable_pool, start_at=time.perf_counter()),
config=graph_config["nodes"][1],
graph_init_params=init_params,
graph_runtime_state=graph_runtime_state,
)
# Initialize node data

60
api/tests/integration_tests/workflow/nodes/test_llm.py
View File

@ -6,17 +6,15 @@ from unittest.mock import MagicMock, patch
from core.app.entities.app_invoke_entities import InvokeFrom
from core.llm_generator.output_parser.structured_output import _parse_structured_output
from core.workflow.entities.variable_pool import VariablePool
from core.workflow.entities.workflow_node_execution import WorkflowNodeExecutionStatus
from core.workflow.graph_engine.entities.graph import Graph
from core.workflow.graph_engine.entities.graph_init_params import GraphInitParams
from core.workflow.graph_engine.entities.graph_runtime_state import GraphRuntimeState
from core.workflow.nodes.event import RunCompletedEvent
from core.workflow.entities import GraphInitParams, GraphRuntimeState, VariablePool
from core.workflow.enums import WorkflowNodeExecutionStatus
from core.workflow.graph import Graph
from core.workflow.node_events import StreamCompletedEvent
from core.workflow.nodes.llm.node import LLMNode
from core.workflow.nodes.node_factory import DifyNodeFactory
from core.workflow.system_variable import SystemVariable
from extensions.ext_database import db
from models.enums import UserFrom
from models.workflow import WorkflowType
"""FOR MOCK FIXTURES, DO NOT REMOVE"""
@ -30,11 +28,9 @@ def init_llm_node(config: dict) -> LLMNode:
"target": "llm",
},
],
"nodes": [{"data": {"type": "start"}, "id": "start"}, config],
"nodes": [{"data": {"type": "start", "title": "Start"}, "id": "start"}, config],
}
graph = Graph.init(graph_config=graph_config)
# Use proper UUIDs for database compatibility
tenant_id = "9d2074fc-6f86-45a9-b09d-6ecc63b9056b"
app_id = "9d2074fc-6f86-45a9-b09d-6ecc63b9056c"
@ -44,7 +40,6 @@ def init_llm_node(config: dict) -> LLMNode:
init_params = GraphInitParams(
tenant_id=tenant_id,
app_id=app_id,
workflow_type=WorkflowType.WORKFLOW,
workflow_id=workflow_id,
graph_config=graph_config,
user_id=user_id,
@ -69,12 +64,21 @@ def init_llm_node(config: dict) -> LLMNode:
)
variable_pool.add(["abc", "output"], "sunny")
graph_runtime_state = GraphRuntimeState(variable_pool=variable_pool, start_at=time.perf_counter())
# Create node factory
node_factory = DifyNodeFactory(
graph_init_params=init_params,
graph_runtime_state=graph_runtime_state,
)
graph = Graph.init(graph_config=graph_config, node_factory=node_factory)
node = LLMNode(
id=str(uuid.uuid4()),
graph_init_params=init_params,
graph=graph,
graph_runtime_state=GraphRuntimeState(variable_pool=variable_pool, start_at=time.perf_counter()),
config=config,
graph_init_params=init_params,
graph_runtime_state=graph_runtime_state,
)
# Initialize node data
@ -173,15 +177,15 @@ def test_execute_llm():
assert isinstance(result, Generator)
for item in result:
if isinstance(item, RunCompletedEvent):
if item.run_result.status != WorkflowNodeExecutionStatus.SUCCEEDED:
print(f"Error: {item.run_result.error}")
print(f"Error type: {item.run_result.error_type}")
assert item.run_result.status == WorkflowNodeExecutionStatus.SUCCEEDED
assert item.run_result.process_data is not None
assert item.run_result.outputs is not None
assert item.run_result.outputs.get("text") is not None
assert item.run_result.outputs.get("usage", {})["total_tokens"] > 0
if isinstance(item, StreamCompletedEvent):
if item.node_run_result.status != WorkflowNodeExecutionStatus.SUCCEEDED:
print(f"Error: {item.node_run_result.error}")
print(f"Error type: {item.node_run_result.error_type}")
assert item.node_run_result.status == WorkflowNodeExecutionStatus.SUCCEEDED
assert item.node_run_result.process_data is not None
assert item.node_run_result.outputs is not None
assert item.node_run_result.outputs.get("text") is not None
assert item.node_run_result.outputs.get("usage", {})["total_tokens"] > 0
def test_execute_llm_with_jinja2():
@ -284,11 +288,11 @@ def test_execute_llm_with_jinja2():
result = node._run()
for item in result:
if isinstance(item, RunCompletedEvent):
assert item.run_result.status == WorkflowNodeExecutionStatus.SUCCEEDED
assert item.run_result.process_data is not None
assert "sunny" in json.dumps(item.run_result.process_data)
assert "what's the weather today?" in json.dumps(item.run_result.process_data)
if isinstance(item, StreamCompletedEvent):
assert item.node_run_result.status == WorkflowNodeExecutionStatus.SUCCEEDED
assert item.node_run_result.process_data is not None
assert "sunny" in json.dumps(item.node_run_result.process_data)
assert "what's the weather today?" in json.dumps(item.node_run_result.process_data)
def test_extract_json():

30
api/tests/integration_tests/workflow/nodes/test_parameter_extractor.py
View File

@ -6,11 +6,10 @@ from unittest.mock import MagicMock
from core.app.entities.app_invoke_entities import InvokeFrom
from core.model_runtime.entities import AssistantPromptMessage
from core.workflow.entities.variable_pool import VariablePool
from core.workflow.entities.workflow_node_execution import WorkflowNodeExecutionStatus
from core.workflow.graph_engine.entities.graph import Graph
from core.workflow.graph_engine.entities.graph_init_params import GraphInitParams
from core.workflow.graph_engine.entities.graph_runtime_state import GraphRuntimeState
from core.workflow.entities import GraphInitParams, GraphRuntimeState, VariablePool
from core.workflow.enums import WorkflowNodeExecutionStatus
from core.workflow.graph import Graph
from core.workflow.nodes.node_factory import DifyNodeFactory
from core.workflow.nodes.parameter_extractor.parameter_extractor_node import ParameterExtractorNode
from core.workflow.system_variable import SystemVariable
from extensions.ext_database import db
@ -18,7 +17,6 @@ from models.enums import UserFrom
from tests.integration_tests.workflow.nodes.__mock.model import get_mocked_fetch_model_config
"""FOR MOCK FIXTURES, DO NOT REMOVE"""
from models.workflow import WorkflowType
from tests.integration_tests.model_runtime.__mock.plugin_daemon import setup_model_mock
@ -45,15 +43,12 @@ def init_parameter_extractor_node(config: dict):
"target": "llm",
},
],
"nodes": [{"data": {"type": "start"}, "id": "start"}, config],
"nodes": [{"data": {"type": "start", "title": "Start"}, "id": "start"}, config],
}
graph = Graph.init(graph_config=graph_config)
init_params = GraphInitParams(
tenant_id="1",
app_id="1",
workflow_type=WorkflowType.WORKFLOW,
workflow_id="1",
graph_config=graph_config,
user_id="1",
@ -74,12 +69,21 @@ def init_parameter_extractor_node(config: dict):
variable_pool.add(["a", "args1"], 1)
variable_pool.add(["a", "args2"], 2)
graph_runtime_state = GraphRuntimeState(variable_pool=variable_pool, start_at=time.perf_counter())
# Create node factory
node_factory = DifyNodeFactory(
graph_init_params=init_params,
graph_runtime_state=graph_runtime_state,
)
graph = Graph.init(graph_config=graph_config, node_factory=node_factory)
node = ParameterExtractorNode(
id=str(uuid.uuid4()),
graph_init_params=init_params,
graph=graph,
graph_runtime_state=GraphRuntimeState(variable_pool=variable_pool, start_at=time.perf_counter()),
config=config,
graph_init_params=init_params,
graph_runtime_state=graph_runtime_state,
)
node.init_node_data(config.get("data", {}))
return node

30
api/tests/integration_tests/workflow/nodes/test_template_transform.py
View File

@ -4,15 +4,13 @@ import uuid
import pytest
from core.app.entities.app_invoke_entities import InvokeFrom
from core.workflow.entities.variable_pool import VariablePool
from core.workflow.entities.workflow_node_execution import WorkflowNodeExecutionStatus
from core.workflow.graph_engine.entities.graph import Graph
from core.workflow.graph_engine.entities.graph_init_params import GraphInitParams
from core.workflow.graph_engine.entities.graph_runtime_state import GraphRuntimeState
from core.workflow.entities import GraphInitParams, GraphRuntimeState, VariablePool
from core.workflow.enums import WorkflowNodeExecutionStatus
from core.workflow.graph import Graph
from core.workflow.nodes.node_factory import DifyNodeFactory
from core.workflow.nodes.template_transform.template_transform_node import TemplateTransformNode
from core.workflow.system_variable import SystemVariable
from models.enums import UserFrom
from models.workflow import WorkflowType
from tests.integration_tests.workflow.nodes.__mock.code_executor import setup_code_executor_mock
@ -42,15 +40,12 @@ def test_execute_code(setup_code_executor_mock):
"target": "1",
},
],
"nodes": [{"data": {"type": "start"}, "id": "start"}, config],
"nodes": [{"data": {"type": "start", "title": "Start"}, "id": "start"}, config],
}
graph = Graph.init(graph_config=graph_config)
init_params = GraphInitParams(
tenant_id="1",
app_id="1",
workflow_type=WorkflowType.WORKFLOW,
workflow_id="1",
graph_config=graph_config,
user_id="1",
@ -69,12 +64,21 @@ def test_execute_code(setup_code_executor_mock):
variable_pool.add(["1", "args1"], 1)
variable_pool.add(["1", "args2"], 3)
graph_runtime_state = GraphRuntimeState(variable_pool=variable_pool, start_at=time.perf_counter())
# Create node factory
node_factory = DifyNodeFactory(
graph_init_params=init_params,
graph_runtime_state=graph_runtime_state,
)
graph = Graph.init(graph_config=graph_config, node_factory=node_factory)
node = TemplateTransformNode(
id=str(uuid.uuid4()),
graph_init_params=init_params,
graph=graph,
graph_runtime_state=GraphRuntimeState(variable_pool=variable_pool, start_at=time.perf_counter()),
config=config,
graph_init_params=init_params,
graph_runtime_state=graph_runtime_state,
)
node.init_node_data(config.get("data", {}))

48
api/tests/integration_tests/workflow/nodes/test_tool.py
View File

@ -4,16 +4,14 @@ from unittest.mock import MagicMock
from core.app.entities.app_invoke_entities import InvokeFrom
from core.tools.utils.configuration import ToolParameterConfigurationManager
from core.workflow.entities.variable_pool import VariablePool
from core.workflow.entities.workflow_node_execution import WorkflowNodeExecutionStatus
from core.workflow.graph_engine.entities.graph import Graph
from core.workflow.graph_engine.entities.graph_init_params import GraphInitParams
from core.workflow.graph_engine.entities.graph_runtime_state import GraphRuntimeState
from core.workflow.nodes.event.event import RunCompletedEvent
from core.workflow.entities import GraphInitParams, GraphRuntimeState, VariablePool
from core.workflow.enums import WorkflowNodeExecutionStatus
from core.workflow.graph import Graph
from core.workflow.node_events import StreamCompletedEvent
from core.workflow.nodes.node_factory import DifyNodeFactory
from core.workflow.nodes.tool.tool_node import ToolNode
from core.workflow.system_variable import SystemVariable
from models.enums import UserFrom
from models.workflow import WorkflowType
def init_tool_node(config: dict):
@ -25,15 +23,12 @@ def init_tool_node(config: dict):
"target": "1",
},
],
"nodes": [{"data": {"type": "start"}, "id": "start"}, config],
"nodes": [{"data": {"type": "start", "title": "Start"}, "id": "start"}, config],
}
graph = Graph.init(graph_config=graph_config)
init_params = GraphInitParams(
tenant_id="1",
app_id="1",
workflow_type=WorkflowType.WORKFLOW,
workflow_id="1",
graph_config=graph_config,
user_id="1",
@ -50,12 +45,21 @@ def init_tool_node(config: dict):
conversation_variables=[],
)
graph_runtime_state = GraphRuntimeState(variable_pool=variable_pool, start_at=time.perf_counter())
# Create node factory
node_factory = DifyNodeFactory(
graph_init_params=init_params,
graph_runtime_state=graph_runtime_state,
)
graph = Graph.init(graph_config=graph_config, node_factory=node_factory)
node = ToolNode(
id=str(uuid.uuid4()),
graph_init_params=init_params,
graph=graph,
graph_runtime_state=GraphRuntimeState(variable_pool=variable_pool, start_at=time.perf_counter()),
config=config,
graph_init_params=init_params,
graph_runtime_state=graph_runtime_state,
)
node.init_node_data(config.get("data", {}))
return node
@ -86,10 +90,10 @@ def test_tool_variable_invoke():
# execute node
result = node._run()
for item in result:
if isinstance(item, RunCompletedEvent):
assert item.run_result.status == WorkflowNodeExecutionStatus.SUCCEEDED
assert item.run_result.outputs is not None
assert item.run_result.outputs.get("text") is not None
if isinstance(item, StreamCompletedEvent):
assert item.node_run_result.status == WorkflowNodeExecutionStatus.SUCCEEDED
assert item.node_run_result.outputs is not None
assert item.node_run_result.outputs.get("text") is not None
def test_tool_mixed_invoke():
@ -117,7 +121,7 @@ def test_tool_mixed_invoke():
# execute node
result = node._run()
for item in result:
if isinstance(item, RunCompletedEvent):
assert item.run_result.status == WorkflowNodeExecutionStatus.SUCCEEDED
assert item.run_result.outputs is not None
assert item.run_result.outputs.get("text") is not None
if isinstance(item, StreamCompletedEvent):
assert item.node_run_result.status == WorkflowNodeExecutionStatus.SUCCEEDED
assert item.node_run_result.outputs is not None
assert item.node_run_result.outputs.get("text") is not None