Merge branch 'main' into feat/r2

api/commands.py

@@ -27,7 +27,7 @@ from models.dataset import Dataset, DatasetCollectionBinding, DatasetMetadata, D
 from models.dataset import Document as DatasetDocument
 from models.model import Account, App, AppAnnotationSetting, AppMode, Conversation, MessageAnnotation
 from models.provider import Provider, ProviderModel
-from services.account_service import RegisterService, TenantService
+from services.account_service import AccountService, RegisterService, TenantService
 from services.clear_free_plan_tenant_expired_logs import ClearFreePlanTenantExpiredLogs
 from services.plugin.data_migration import PluginDataMigration
 from services.plugin.plugin_migration import PluginMigration
@@ -68,6 +68,7 @@ def reset_password(email, new_password, password_confirm):
     account.password = base64_password_hashed
     account.password_salt = base64_salt
     db.session.commit()
+    AccountService.reset_login_error_rate_limit(email)
     click.echo(click.style("Password reset successfully.", fg="green"))
 
 

api/controllers/service_api/app/app.py

@@ -47,7 +47,13 @@ class AppInfoApi(Resource):
     def get(self, app_model: App):
         """Get app information"""
         tags = [tag.name for tag in app_model.tags]
-        return {"name": app_model.name, "description": app_model.description, "tags": tags, "mode": app_model.mode}
+        return {
+            "name": app_model.name,
+            "description": app_model.description,
+            "tags": tags,
+            "mode": app_model.mode,
+            "author_name": app_model.author_name,
+        }
 
 
 api.add_resource(AppParameterApi, "/parameters")

api/core/app/apps/base_app_runner.py

@@ -1,3 +1,4 @@
+import logging
 import time
 from collections.abc import Generator, Mapping, Sequence
 from typing import TYPE_CHECKING, Any, Optional, Union
@@ -33,6 +34,8 @@ from models.model import App, AppMode, Message, MessageAnnotation
 if TYPE_CHECKING:
     from core.file.models import File
 
+_logger = logging.getLogger(__name__)
+
 
 class AppRunner:
     def get_pre_calculate_rest_tokens(
@@ -298,7 +301,7 @@ class AppRunner:
         )
 
     def _handle_invoke_result_stream(
-        self, invoke_result: Generator, queue_manager: AppQueueManager, agent: bool
+        self, invoke_result: Generator[LLMResultChunk, None, None], queue_manager: AppQueueManager, agent: bool
     ) -> None:
         """
         Handle invoke result
@@ -317,18 +320,28 @@ class AppRunner:
             else:
                 queue_manager.publish(QueueAgentMessageEvent(chunk=result), PublishFrom.APPLICATION_MANAGER)
 
-            text += result.delta.message.content
+            message = result.delta.message
+            if isinstance(message.content, str):
+                text += message.content
+            elif isinstance(message.content, list):
+                for content in message.content:
+                    if not isinstance(content, str):
+                        # TODO(QuantumGhost): Add multimodal output support for easy ui.
+                        _logger.warning("received multimodal output, type=%s", type(content))
+                        text += content.data
+                    else:
+                        text += content  # failback to str
 
             if not model:
                 model = result.model
 
             if not prompt_messages:
-                prompt_messages = result.prompt_messages
+                prompt_messages = list(result.prompt_messages)
 
             if result.delta.usage:
                 usage = result.delta.usage
 
-        if not usage:
+        if usage is None:
             usage = LLMUsage.empty_usage()
 
         llm_result = LLMResult(

api/core/app/task_pipeline/easy_ui_based_generate_task_pipeline.py

@@ -48,6 +48,7 @@ from core.model_manager import ModelInstance
 from core.model_runtime.entities.llm_entities import LLMResult, LLMResultChunk, LLMResultChunkDelta, LLMUsage
 from core.model_runtime.entities.message_entities import (
     AssistantPromptMessage,
+    TextPromptMessageContent,
 )
 from core.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel
 from core.ops.entities.trace_entity import TraceTaskName
@@ -309,6 +310,23 @@ class EasyUIBasedGenerateTaskPipeline(BasedGenerateTaskPipeline):
                 delta_text = chunk.delta.message.content
                 if delta_text is None:
                     continue
+                if isinstance(chunk.delta.message.content, list):
+                    delta_text = ""
+                    for content in chunk.delta.message.content:
+                        logger.debug(
+                            "The content type %s in LLM chunk delta message content.: %r", type(content), content
+                        )
+                        if isinstance(content, TextPromptMessageContent):
+                            delta_text += content.data
+                        elif isinstance(content, str):
+                            delta_text += content  # failback to str
+                        else:
+                            logger.warning(
+                                "Unsupported content type %s in LLM chunk delta message content.: %r",
+                                type(content),
+                                content,
+                            )
+                            continue
 
                 if not self._task_state.llm_result.prompt_messages:
                     self._task_state.llm_result.prompt_messages = chunk.prompt_messages

api/core/rag/datasource/vdb/oceanbase/oceanbase_vector.py

@@ -80,6 +80,23 @@ class OceanBaseVector(BaseVector):
 
             self.delete()
 
+            vals = []
+            params = self._client.perform_raw_text_sql("SHOW PARAMETERS LIKE '%ob_vector_memory_limit_percentage%'")
+            for row in params:
+                val = int(row[6])
+                vals.append(val)
+            if len(vals) == 0:
+                raise ValueError("ob_vector_memory_limit_percentage not found in parameters.")
+            if any(val == 0 for val in vals):
+                try:
+                    self._client.perform_raw_text_sql("ALTER SYSTEM SET ob_vector_memory_limit_percentage = 30")
+                except Exception as e:
+                    raise Exception(
+                        "Failed to set ob_vector_memory_limit_percentage. "
+                        + "Maybe the database user has insufficient privilege.",
+                        e,
+                    )
+
             cols = [
                 Column("id", String(36), primary_key=True, autoincrement=False),
                 Column("vector", VECTOR(self._vec_dim)),
@@ -110,22 +127,6 @@ class OceanBaseVector(BaseVector):
                     + "to support fulltext index and vector index in the same table",
                     e,
                 )
-            vals = []
-            params = self._client.perform_raw_text_sql("SHOW PARAMETERS LIKE '%ob_vector_memory_limit_percentage%'")
-            for row in params:
-                val = int(row[6])
-                vals.append(val)
-            if len(vals) == 0:
-                raise ValueError("ob_vector_memory_limit_percentage not found in parameters.")
-            if any(val == 0 for val in vals):
-                try:
-                    self._client.perform_raw_text_sql("ALTER SYSTEM SET ob_vector_memory_limit_percentage = 30")
-                except Exception as e:
-                    raise Exception(
-                        "Failed to set ob_vector_memory_limit_percentage. "
-                        + "Maybe the database user has insufficient privilege.",
-                        e,
-                    )
             redis_client.set(collection_exist_cache_key, 1, ex=3600)
 
     def _check_hybrid_search_support(self) -> bool:

api/core/repositories/sqlalchemy_workflow_execution_repository.py

@@ -6,7 +6,7 @@ import json
 import logging
 from typing import Optional, Union
 
-from sqlalchemy import select
+from sqlalchemy import func, select
 from sqlalchemy.engine import Engine
 from sqlalchemy.orm import sessionmaker
 
@@ -151,11 +151,11 @@ class SQLAlchemyWorkflowExecutionRepository(WorkflowExecutionRepository):
             existing = session.scalar(select(WorkflowRun).where(WorkflowRun.id == domain_model.id_))
             if not existing:
                 # For new records, get the next sequence number
-                stmt = select(WorkflowRun.sequence_number).where(
+                stmt = select(func.max(WorkflowRun.sequence_number)).where(
                     WorkflowRun.app_id == self._app_id,
                     WorkflowRun.tenant_id == self._tenant_id,
                 )
-                max_sequence = session.scalar(stmt.order_by(WorkflowRun.sequence_number.desc()))
+                max_sequence = session.scalar(stmt)
                 db_model.sequence_number = (max_sequence or 0) + 1
             else:
                 # For updates, keep the existing sequence number

api/core/workflow/nodes/event/event.py

@@ -6,7 +6,6 @@ from pydantic import BaseModel, Field
 from core.model_runtime.entities.llm_entities import LLMUsage
 from core.rag.entities.citation_metadata import RetrievalSourceMetadata
 from core.workflow.entities.node_entities import NodeRunResult
-from core.workflow.entities.workflow_node_execution import WorkflowNodeExecutionStatus
 
 
 class RunCompletedEvent(BaseModel):
@@ -39,11 +38,3 @@ class RunRetryEvent(BaseModel):
     error: str = Field(..., description="error")
     retry_index: int = Field(..., description="Retry attempt number")
     start_at: datetime = Field(..., description="Retry start time")
-
-
-class SingleStepRetryEvent(NodeRunResult):
-    """Single step retry event"""
-
-    status: WorkflowNodeExecutionStatus = WorkflowNodeExecutionStatus.RETRY
-
-    elapsed_time: float = Field(..., description="elapsed time")

api/core/workflow/nodes/llm/node.py

@@ -525,6 +525,8 @@ class LLMNode(BaseNode[LLMNodeData]):
                 # Set appropriate response format based on model capabilities
                 self._set_response_format(completion_params, model_schema.parameter_rules)
         model_config_with_cred.parameters = completion_params
+        # NOTE(-LAN-): This line modify the `self.node_data.model`, which is used in `_invoke_llm()`.
+        node_data_model.completion_params = completion_params
         return model, model_config_with_cred
 
     def _fetch_prompt_messages(

api/factories/variable_factory.py

@@ -42,10 +42,6 @@ from core.workflow.constants import (
 )
 
 
-class InvalidSelectorError(ValueError):
-    pass
-
-
 class UnsupportedSegmentTypeError(Exception):
     pass
 

api/services/errors/__init__.py

@@ -4,7 +4,6 @@ from . import (
     app_model_config,
     audio,
     base,
-    completion,
     conversation,
     dataset,
     document,
@@ -19,7 +18,6 @@ __all__ = [
     "app_model_config",
     "audio",
     "base",
-    "completion",
     "conversation",
     "dataset",
     "document",

api/services/errors/account.py

@@ -55,7 +55,3 @@ class MemberNotInTenantError(BaseServiceError):
 
 class RoleAlreadyAssignedError(BaseServiceError):
     pass
-
-
-class RateLimitExceededError(BaseServiceError):
-    pass

api/services/errors/completion.py

@@ -1,5 +0,0 @@
-from services.errors.base import BaseServiceError
-
-
-class CompletionStoppedError(BaseServiceError):
-    pass

api/tasks/retry_document_indexing_task.py

@@ -30,11 +30,11 @@ def retry_document_indexing_task(dataset_id: str, document_ids: list[str]):
         logging.info(click.style("Dataset not found: {}".format(dataset_id), fg="red"))
         db.session.close()
         return
-
+    tenant_id = dataset.tenant_id
     for document_id in document_ids:
         retry_indexing_cache_key = "document_{}_is_retried".format(document_id)
         # check document limit
-        features = FeatureService.get_features(dataset.tenant_id)
+        features = FeatureService.get_features(tenant_id)
         try:
             if features.billing.enabled:
                 vector_space = features.vector_space

api/tests/integration_tests/vdb/oceanbase/check_oceanbase_ready.py

@@ -0,0 +1,49 @@
+import time
+
+import pymysql
+
+
+def check_oceanbase_ready() -> bool:
+    try:
+        connection = pymysql.connect(
+            host="localhost",
+            port=2881,
+            user="root",
+            password="difyai123456",
+        )
+        affected_rows = connection.query("SELECT 1")
+        return affected_rows == 1
+    except Exception as e:
+        print(f"Oceanbase is not ready. Exception: {e}")
+        return False
+    finally:
+        if connection:
+            connection.close()
+
+
+def main():
+    max_attempts = 50
+    retry_interval_seconds = 2
+    is_oceanbase_ready = False
+    for attempt in range(max_attempts):
+        try:
+            is_oceanbase_ready = check_oceanbase_ready()
+        except Exception as e:
+            print(f"Oceanbase is not ready. Exception: {e}")
+            is_oceanbase_ready = False
+
+        if is_oceanbase_ready:
+            break
+        else:
+            print(f"Attempt {attempt + 1} failed, retry in {retry_interval_seconds} seconds...")
+            time.sleep(retry_interval_seconds)
+
+    if is_oceanbase_ready:
+        print("Oceanbase is ready.")
+    else:
+        print(f"Oceanbase is not ready after {max_attempts} attempting checks.")
+        exit(1)
+
+
+if __name__ == "__main__":
+    main()

api/tests/integration_tests/vdb/oceanbase/test_oceanbase.py

@@ -1,15 +1,11 @@
-from unittest.mock import MagicMock, patch
-
 import pytest
 
 from core.rag.datasource.vdb.oceanbase.oceanbase_vector import (
     OceanBaseVector,
     OceanBaseVectorConfig,
 )
-from tests.integration_tests.vdb.__mock.tcvectordb import setup_tcvectordb_mock
 from tests.integration_tests.vdb.test_vector_store import (
     AbstractVectorTest,
-    get_example_text,
     setup_mock_redis,
 )
 
@@ -20,10 +16,11 @@ def oceanbase_vector():
         "dify_test_collection",
         config=OceanBaseVectorConfig(
             host="127.0.0.1",
-            port="2881",
-            user="root@test",
+            port=2881,
+            user="root",
             database="test",
-            password="test",
+            password="difyai123456",
+            enable_hybrid_search=True,
         ),
     )
 
@@ -33,39 +30,13 @@ class OceanBaseVectorTest(AbstractVectorTest):
         super().__init__()
         self.vector = vector
 
-    def search_by_vector(self):
-        hits_by_vector = self.vector.search_by_vector(query_vector=self.example_embedding)
-        assert len(hits_by_vector) == 0
-
-    def search_by_full_text(self):
-        hits_by_full_text = self.vector.search_by_full_text(query=get_example_text())
-        assert len(hits_by_full_text) == 0
-
-    def text_exists(self):
-        exist = self.vector.text_exists(self.example_doc_id)
-        assert exist == True
-
     def get_ids_by_metadata_field(self):
         ids = self.vector.get_ids_by_metadata_field(key="document_id", value=self.example_doc_id)
-        assert len(ids) == 0
-
-
-@pytest.fixture
-def setup_mock_oceanbase_client():
-    with patch("core.rag.datasource.vdb.oceanbase.oceanbase_vector.ObVecClient", new_callable=MagicMock) as mock_client:
-        yield mock_client
-
-
-@pytest.fixture
-def setup_mock_oceanbase_vector(oceanbase_vector):
-    with patch.object(oceanbase_vector, "_client"):
-        yield oceanbase_vector
+        assert len(ids) == 1
 
 
 def test_oceanbase_vector(
     setup_mock_redis,
-    setup_mock_oceanbase_client,
-    setup_mock_oceanbase_vector,
     oceanbase_vector,
 ):
     OceanBaseVectorTest(oceanbase_vector).run_all_tests()