feat: sync main branch (#31938)

Signed-off-by: majiayu000 <1835304752@qq.com>
Signed-off-by: dependabot[bot] <support@github.com>
Signed-off-by: NeatGuyCoding <15627489+NeatGuyCoding@users.noreply.github.com>
Signed-off-by: -LAN- <laipz8200@outlook.com>
Signed-off-by: yihong0618 <zouzou0208@gmail.com>
Co-authored-by: QuantumGhost <obelisk.reg+git@gmail.com>
Co-authored-by: 盐粒 Yanli <yanli@dify.ai>
Co-authored-by: wangxiaolei <fatelei@gmail.com>
Co-authored-by: Stephen Zhou <38493346+hyoban@users.noreply.github.com>
Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Co-authored-by: Cursx <33718736+Cursx@users.noreply.github.com>
Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>
Co-authored-by: lif <1835304752@qq.com>
Co-authored-by: 非法操作 <hjlarry@163.com>
Co-authored-by: Asuka Minato <i@asukaminato.eu.org>
Co-authored-by: fenglin <790872612@qq.com>
Co-authored-by: qiaofenglin <qiaofenglin@baidu.com>
Co-authored-by: -LAN- <laipz8200@outlook.com>
Co-authored-by: TomoOkuyama <49631611+TomoOkuyama@users.noreply.github.com>
Co-authored-by: Tomo Okuyama <tomo.okuyama@intersystems.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: zyssyz123 <916125788@qq.com>
Co-authored-by: hj24 <mambahj24@gmail.com>
Co-authored-by: Coding On Star <447357187@qq.com>
Co-authored-by: CodingOnStar <hanxujiang@dify.ai>
Co-authored-by: yyh <92089059+lyzno1@users.noreply.github.com>
Co-authored-by: Xiangxuan Qu <fghpdf@outlook.com>
Co-authored-by: fghpdf <fghpdf@users.noreply.github.com>
Co-authored-by: coopercoder <whitetiger0127@163.com>
Co-authored-by: zhaiguangpeng <zhaiguangpeng@didiglobal.com>
Co-authored-by: Junyan Qin (Chin) <rockchinq@gmail.com>
Co-authored-by: E.G <146701565+GlobalStar117@users.noreply.github.com>
Co-authored-by: GlobalStar117 <GlobalStar117@users.noreply.github.com>
Co-authored-by: Claude Haiku 4.5 <noreply@anthropic.com>
Co-authored-by: CodingOnStar <hanxujiang@dify.com>
Co-authored-by: crazywoola <100913391+crazywoola@users.noreply.github.com>
Co-authored-by: heyszt <270985384@qq.com>
Co-authored-by: NeatGuyCoding <15627489+NeatGuyCoding@users.noreply.github.com>
Co-authored-by: Yeuoly <45712896+Yeuoly@users.noreply.github.com>
Co-authored-by: zxhlyh <jasonapring2015@outlook.com>
Co-authored-by: moonpanda <chuanzegao@163.com>
Co-authored-by: warlocgao <warlocgao@tencent.com>
Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
Co-authored-by: claude[bot] <41898282+claude[bot]@users.noreply.github.com>
Co-authored-by: KVOJJJin <jzongcode@gmail.com>
Co-authored-by: eux <euxx@users.noreply.github.com>
Co-authored-by: bangjiehan <bangjiehan@gmail.com>
Co-authored-by: FFXN <31929997+FFXN@users.noreply.github.com>
Co-authored-by: Jyong <76649700+JohnJyong@users.noreply.github.com>
Co-authored-by: Nie Ronghua <nieronghua@sf-express.com>
Co-authored-by: JQSevenMiao <141806521+JQSevenMiao@users.noreply.github.com>
Co-authored-by: jiasiqi <jiasiqi3@tal.com>
Co-authored-by: Seokrin Taron Sung <sungsjade@gmail.com>
Co-authored-by: CrabSAMA <40541269+CrabSAMA@users.noreply.github.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: yihong <zouzou0208@gmail.com>
Co-authored-by: Joel <iamjoel007@gmail.com>
Co-authored-by: Wu Tianwei <30284043+WTW0313@users.noreply.github.com>
Co-authored-by: yessenia <yessenia.contact@gmail.com>
Co-authored-by: Jax <anobaka@qq.com>
Co-authored-by: niveshdandyan <155956228+niveshdandyan@users.noreply.github.com>
Co-authored-by: OSS Contributor <oss-contributor@example.com>
Co-authored-by: niveshdandyan <niveshdandyan@users.noreply.github.com>
Co-authored-by: Sean Kenneth Doherty <Smaster7772@gmail.com>

api/fields/annotation_fields.py

@@ -1,36 +1,69 @@
-from flask_restx import Namespace, fields
+from __future__ import annotations
 
-from libs.helper import TimestampField
+from datetime import datetime
 
-annotation_fields = {
-    "id": fields.String,
-    "question": fields.String,
-    "answer": fields.Raw(attribute="content"),
-    "hit_count": fields.Integer,
-    "created_at": TimestampField,
-    # 'account': fields.Nested(simple_account_fields, allow_null=True)
-}
+from pydantic import BaseModel, ConfigDict, Field, field_validator
 
 
-def build_annotation_model(api_or_ns: Namespace):
-    """Build the annotation model for the API or Namespace."""
-    return api_or_ns.model("Annotation", annotation_fields)
+def _to_timestamp(value: datetime | int | None) -> int | None:
+    if isinstance(value, datetime):
+        return int(value.timestamp())
+    return value
 
 
-annotation_list_fields = {
-    "data": fields.List(fields.Nested(annotation_fields)),
-}
+class ResponseModel(BaseModel):
+    model_config = ConfigDict(
+        from_attributes=True,
+        extra="ignore",
+        populate_by_name=True,
+        serialize_by_alias=True,
+        protected_namespaces=(),
+    )
 
-annotation_hit_history_fields = {
-    "id": fields.String,
-    "source": fields.String,
-    "score": fields.Float,
-    "question": fields.String,
-    "created_at": TimestampField,
-    "match": fields.String(attribute="annotation_question"),
-    "response": fields.String(attribute="annotation_content"),
-}
 
-annotation_hit_history_list_fields = {
-    "data": fields.List(fields.Nested(annotation_hit_history_fields)),
-}
+class Annotation(ResponseModel):
+    id: str
+    question: str | None = None
+    answer: str | None = Field(default=None, validation_alias="content")
+    hit_count: int | None = None
+    created_at: int | None = None
+
+    @field_validator("created_at", mode="before")
+    @classmethod
+    def _normalize_created_at(cls, value: datetime | int | None) -> int | None:
+        return _to_timestamp(value)
+
+
+class AnnotationList(ResponseModel):
+    data: list[Annotation]
+    has_more: bool
+    limit: int
+    total: int
+    page: int
+
+
+class AnnotationExportList(ResponseModel):
+    data: list[Annotation]
+
+
+class AnnotationHitHistory(ResponseModel):
+    id: str
+    source: str | None = None
+    score: float | None = None
+    question: str | None = None
+    created_at: int | None = None
+    match: str | None = Field(default=None, validation_alias="annotation_question")
+    response: str | None = Field(default=None, validation_alias="annotation_content")
+
+    @field_validator("created_at", mode="before")
+    @classmethod
+    def _normalize_created_at(cls, value: datetime | int | None) -> int | None:
+        return _to_timestamp(value)
+
+
+class AnnotationHitHistoryList(ResponseModel):
+    data: list[AnnotationHitHistory]
+    has_more: bool
+    limit: int
+    total: int
+    page: int

api/fields/dataset_fields.py

@@ -39,6 +39,14 @@ dataset_retrieval_model_fields = {
     "score_threshold_enabled": fields.Boolean,
     "score_threshold": fields.Float,
 }
+
+dataset_summary_index_fields = {
+    "enable": fields.Boolean,
+    "model_name": fields.String,
+    "model_provider_name": fields.String,
+    "summary_prompt": fields.String,
+}
+
 external_retrieval_model_fields = {
     "top_k": fields.Integer,
     "score_threshold": fields.Float,
@@ -83,6 +91,7 @@ dataset_detail_fields = {
     "embedding_model_provider": fields.String,
     "embedding_available": fields.Boolean,
     "retrieval_model_dict": fields.Nested(dataset_retrieval_model_fields),
+    "summary_index_setting": fields.Nested(dataset_summary_index_fields),
     "tags": fields.List(fields.Nested(tag_fields)),
     "doc_form": fields.String,
     "external_knowledge_info": fields.Nested(external_knowledge_info_fields),

api/fields/document_fields.py

@@ -33,6 +33,11 @@ document_fields = {
     "hit_count": fields.Integer,
     "doc_form": fields.String,
     "doc_metadata": fields.List(fields.Nested(document_metadata_fields), attribute="doc_metadata_details"),
+    # Summary index generation status:
+    # "SUMMARIZING" (when task is queued and generating)
+    "summary_index_status": fields.String,
+    # Whether this document needs summary index generation
+    "need_summary": fields.Boolean,
 }
 
 document_with_segments_fields = {
@@ -60,6 +65,10 @@ document_with_segments_fields = {
     "completed_segments": fields.Integer,
     "total_segments": fields.Integer,
     "doc_metadata": fields.List(fields.Nested(document_metadata_fields), attribute="doc_metadata_details"),
+    # Summary index generation status:
+    # "SUMMARIZING" (when task is queued and generating)
+    "summary_index_status": fields.String,
+    "need_summary": fields.Boolean,  # Whether this document needs summary index generation
 }
 
 dataset_and_document_fields = {

api/fields/end_user_fields.py

@@ -1,4 +1,7 @@
-from flask_restx import Namespace, fields
+from __future__ import annotations
+
+from flask_restx import fields
+from pydantic import BaseModel, ConfigDict
 
 simple_end_user_fields = {
     "id": fields.String,
@@ -8,5 +11,18 @@ simple_end_user_fields = {
 }
 
 
-def build_simple_end_user_model(api_or_ns: Namespace):
-    return api_or_ns.model("SimpleEndUser", simple_end_user_fields)
+class ResponseModel(BaseModel):
+    model_config = ConfigDict(
+        from_attributes=True,
+        extra="ignore",
+        populate_by_name=True,
+        serialize_by_alias=True,
+        protected_namespaces=(),
+    )
+
+
+class SimpleEndUser(ResponseModel):
+    id: str
+    type: str
+    is_anonymous: bool
+    session_id: str | None = None

api/fields/hit_testing_fields.py

@@ -58,4 +58,5 @@ hit_testing_record_fields = {
     "score": fields.Float,
     "tsne_position": fields.Raw,
     "files": fields.List(fields.Nested(files_fields)),
+    "summary": fields.String,  # Summary content if retrieved via summary index
 }

api/fields/member_fields.py

@@ -1,6 +1,11 @@
-from flask_restx import Namespace, fields
+from __future__ import annotations
 
-from libs.helper import AvatarUrlField, TimestampField
+from datetime import datetime
+
+from flask_restx import fields
+from pydantic import BaseModel, ConfigDict, computed_field, field_validator
+
+from core.file import helpers as file_helpers
 
 simple_account_fields = {
     "id": fields.String,
@@ -9,36 +14,78 @@ simple_account_fields = {
 }
 
 
-def build_simple_account_model(api_or_ns: Namespace):
-    return api_or_ns.model("SimpleAccount", simple_account_fields)
+def _to_timestamp(value: datetime | int | None) -> int | None:
+    if isinstance(value, datetime):
+        return int(value.timestamp())
+    return value
 
 
-account_fields = {
-    "id": fields.String,
-    "name": fields.String,
-    "avatar": fields.String,
-    "avatar_url": AvatarUrlField,
-    "email": fields.String,
-    "is_password_set": fields.Boolean,
-    "interface_language": fields.String,
-    "interface_theme": fields.String,
-    "timezone": fields.String,
-    "last_login_at": TimestampField,
-    "last_login_ip": fields.String,
-    "created_at": TimestampField,
-}
+def _build_avatar_url(avatar: str | None) -> str | None:
+    if avatar is None:
+        return None
+    if avatar.startswith(("http://", "https://")):
+        return avatar
+    return file_helpers.get_signed_file_url(avatar)
 
-account_with_role_fields = {
-    "id": fields.String,
-    "name": fields.String,
-    "avatar": fields.String,
-    "avatar_url": AvatarUrlField,
-    "email": fields.String,
-    "last_login_at": TimestampField,
-    "last_active_at": TimestampField,
-    "created_at": TimestampField,
-    "role": fields.String,
-    "status": fields.String,
-}
 
-account_with_role_list_fields = {"accounts": fields.List(fields.Nested(account_with_role_fields))}
+class ResponseModel(BaseModel):
+    model_config = ConfigDict(
+        from_attributes=True,
+        extra="ignore",
+        populate_by_name=True,
+        serialize_by_alias=True,
+        protected_namespaces=(),
+    )
+
+
+class SimpleAccount(ResponseModel):
+    id: str
+    name: str
+    email: str
+
+
+class _AccountAvatar(ResponseModel):
+    avatar: str | None = None
+
+    @computed_field(return_type=str | None)  # type: ignore[prop-decorator]
+    @property
+    def avatar_url(self) -> str | None:
+        return _build_avatar_url(self.avatar)
+
+
+class Account(_AccountAvatar):
+    id: str
+    name: str
+    email: str
+    is_password_set: bool
+    interface_language: str | None = None
+    interface_theme: str | None = None
+    timezone: str | None = None
+    last_login_at: int | None = None
+    last_login_ip: str | None = None
+    created_at: int | None = None
+
+    @field_validator("last_login_at", "created_at", mode="before")
+    @classmethod
+    def _normalize_timestamp(cls, value: datetime | int | None) -> int | None:
+        return _to_timestamp(value)
+
+
+class AccountWithRole(_AccountAvatar):
+    id: str
+    name: str
+    email: str
+    last_login_at: int | None = None
+    last_active_at: int | None = None
+    created_at: int | None = None
+    role: str
+    status: str
+
+    @field_validator("last_login_at", "last_active_at", "created_at", mode="before")
+    @classmethod
+    def _normalize_timestamp(cls, value: datetime | int | None) -> int | None:
+        return _to_timestamp(value)
+
+
+class AccountWithRoleList(ResponseModel):
+    accounts: list[AccountWithRole]

api/fields/message_fields.py

@@ -36,6 +36,7 @@ class RetrieverResource(ResponseModel):
     segment_position: int | None = None
     index_node_hash: str | None = None
     content: str | None = None
+    summary: str | None = None
     created_at: int | None = None
 
     @field_validator("created_at", mode="before")

api/fields/segment_fields.py

@@ -49,4 +49,5 @@ segment_fields = {
     "stopped_at": TimestampField,
     "child_chunks": fields.List(fields.Nested(child_chunk_fields)),
     "attachments": fields.List(fields.Nested(attachment_fields)),
+    "summary": fields.String,  # Summary content for the segment
 }

api/fields/tag_fields.py

@@ -1,12 +1,20 @@
-from flask_restx import Namespace, fields
+from __future__ import annotations
 
-dataset_tag_fields = {
-    "id": fields.String,
-    "name": fields.String,
-    "type": fields.String,
-    "binding_count": fields.String,
-}
+from pydantic import BaseModel, ConfigDict
 
 
-def build_dataset_tag_fields(api_or_ns: Namespace):
-    return api_or_ns.model("DataSetTag", dataset_tag_fields)
+class ResponseModel(BaseModel):
+    model_config = ConfigDict(
+        from_attributes=True,
+        extra="ignore",
+        populate_by_name=True,
+        serialize_by_alias=True,
+        protected_namespaces=(),
+    )
+
+
+class DataSetTag(ResponseModel):
+    id: str
+    name: str
+    type: str
+    binding_count: str | None = None

api/fields/workflow_app_log_fields.py

@@ -1,8 +1,13 @@
 from flask_restx import Namespace, fields
 
-from fields.end_user_fields import build_simple_end_user_model, simple_end_user_fields
-from fields.member_fields import build_simple_account_model, simple_account_fields
-from fields.workflow_run_fields import build_workflow_run_for_log_model, workflow_run_for_log_fields
+from fields.end_user_fields import simple_end_user_fields
+from fields.member_fields import simple_account_fields
+from fields.workflow_run_fields import (
+    build_workflow_run_for_archived_log_model,
+    build_workflow_run_for_log_model,
+    workflow_run_for_archived_log_fields,
+    workflow_run_for_log_fields,
+)
 from libs.helper import TimestampField
 
 workflow_app_log_partial_fields = {
@@ -20,20 +25,31 @@ workflow_app_log_partial_fields = {
 def build_workflow_app_log_partial_model(api_or_ns: Namespace):
     """Build the workflow app log partial model for the API or Namespace."""
     workflow_run_model = build_workflow_run_for_log_model(api_or_ns)
-    simple_account_model = build_simple_account_model(api_or_ns)
-    simple_end_user_model = build_simple_end_user_model(api_or_ns)
 
     copied_fields = workflow_app_log_partial_fields.copy()
     copied_fields["workflow_run"] = fields.Nested(workflow_run_model, attribute="workflow_run", allow_null=True)
-    copied_fields["created_by_account"] = fields.Nested(
-        simple_account_model, attribute="created_by_account", allow_null=True
-    )
-    copied_fields["created_by_end_user"] = fields.Nested(
-        simple_end_user_model, attribute="created_by_end_user", allow_null=True
-    )
     return api_or_ns.model("WorkflowAppLogPartial", copied_fields)
 
 
+workflow_archived_log_partial_fields = {
+    "id": fields.String,
+    "workflow_run": fields.Nested(workflow_run_for_archived_log_fields, allow_null=True),
+    "trigger_metadata": fields.Raw,
+    "created_by_account": fields.Nested(simple_account_fields, attribute="created_by_account", allow_null=True),
+    "created_by_end_user": fields.Nested(simple_end_user_fields, attribute="created_by_end_user", allow_null=True),
+    "created_at": TimestampField,
+}
+
+
+def build_workflow_archived_log_partial_model(api_or_ns: Namespace):
+    """Build the workflow archived log partial model for the API or Namespace."""
+    workflow_run_model = build_workflow_run_for_archived_log_model(api_or_ns)
+
+    copied_fields = workflow_archived_log_partial_fields.copy()
+    copied_fields["workflow_run"] = fields.Nested(workflow_run_model, allow_null=True)
+    return api_or_ns.model("WorkflowArchivedLogPartial", copied_fields)
+
+
 workflow_app_log_pagination_fields = {
     "page": fields.Integer,
     "limit": fields.Integer,
@@ -51,3 +67,21 @@ def build_workflow_app_log_pagination_model(api_or_ns: Namespace):
     copied_fields = workflow_app_log_pagination_fields.copy()
     copied_fields["data"] = fields.List(fields.Nested(workflow_app_log_partial_model))
     return api_or_ns.model("WorkflowAppLogPagination", copied_fields)
+
+
+workflow_archived_log_pagination_fields = {
+    "page": fields.Integer,
+    "limit": fields.Integer,
+    "total": fields.Integer,
+    "has_more": fields.Boolean,
+    "data": fields.List(fields.Nested(workflow_archived_log_partial_fields)),
+}
+
+
+def build_workflow_archived_log_pagination_model(api_or_ns: Namespace):
+    """Build the workflow archived log pagination model for the API or Namespace."""
+    workflow_archived_log_partial_model = build_workflow_archived_log_partial_model(api_or_ns)
+
+    copied_fields = workflow_archived_log_pagination_fields.copy()
+    copied_fields["data"] = fields.List(fields.Nested(workflow_archived_log_partial_model))
+    return api_or_ns.model("WorkflowArchivedLogPagination", copied_fields)

api/fields/workflow_run_fields.py

@@ -23,6 +23,19 @@ def build_workflow_run_for_log_model(api_or_ns: Namespace):
     return api_or_ns.model("WorkflowRunForLog", workflow_run_for_log_fields)
 
 
+workflow_run_for_archived_log_fields = {
+    "id": fields.String,
+    "status": fields.String,
+    "triggered_from": fields.String,
+    "elapsed_time": fields.Float,
+    "total_tokens": fields.Integer,
+}
+
+
+def build_workflow_run_for_archived_log_model(api_or_ns: Namespace):
+    return api_or_ns.model("WorkflowRunForArchivedLog", workflow_run_for_archived_log_fields)
+
+
 workflow_run_for_list_fields = {
     "id": fields.String,
     "version": fields.String,