fix: margin in rerank switch

# Conflicts:
#	api/poetry.lock

.github/workflows/web-tests.yml

@@ -0,0 +1,46 @@
+name: Web Tests
+
+on:
+  pull_request:
+    branches:
+      - main
+    paths:
+      - web/**
+
+concurrency:
+  group: web-tests-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+jobs:
+  test:
+    name: Web Tests
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: ./web
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Check changed files
+        id: changed-files
+        uses: tj-actions/changed-files@v45
+        with:
+          files: web/**
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        if: steps.changed-files.outputs.any_changed == 'true'
+        with:
+          node-version: 20
+          cache: yarn
+          cache-dependency-path: ./web/package.json
+
+      - name: Install dependencies
+        if: steps.changed-files.outputs.any_changed == 'true'
+        run: yarn install --frozen-lockfile
+
+      - name: Run tests
+        if: steps.changed-files.outputs.any_changed == 'true'
+        run: yarn test

.gitignore

@@ -153,6 +153,9 @@ docker-legacy/volumes/etcd/*
 docker-legacy/volumes/minio/*
 docker-legacy/volumes/milvus/*
 docker-legacy/volumes/chroma/*
+docker-legacy/volumes/opensearch/data/*
+docker-legacy/volumes/pgvectors/data/*
+docker-legacy/volumes/pgvector/data/*
 
 docker/volumes/app/storage/*
 docker/volumes/certbot/*
@@ -164,6 +167,12 @@ docker/volumes/etcd/*
 docker/volumes/minio/*
 docker/volumes/milvus/*
 docker/volumes/chroma/*
+docker/volumes/opensearch/data/*
+docker/volumes/myscale/data/*
+docker/volumes/myscale/log/*
+docker/volumes/unstructured/*
+docker/volumes/pgvector/data/*
+docker/volumes/pgvecto_rs/data/*
 
 docker/nginx/conf.d/default.conf
 docker/middleware.env

CONTRIBUTING_CN.md

@@ -36,7 +36,7 @@
   | 被团队成员标记为高优先级的功能    | 高优先级   |
   | 在 [community feedback board](https://github.com/langgenius/dify/discussions/categories/feedbacks) 内反馈的常见功能请求 | 中等优先级 |
   | 非核心功能和小幅改进                     | 低优先级    |
-  | 有价值当不紧急                                   | 未来功能  |
+  | 有价值但不紧急                                   | 未来功能  |
 
 ### 其他任何事情（例如 bug 报告、性能优化、拼写错误更正）：
 * 立即开始编码。
@@ -138,7 +138,7 @@ Dify 的后端使用 Python 编写，使用 [Flask](https://flask.palletsproject
 ├── models                // 描述数据模型和 API 响应的形状
 ├── public                // 如 favicon 等元资源
 ├── service               // 定义 API 操作的形状
-├── test                  
+├── test
 ├── types                 // 函数参数和返回值的描述
 └── utils                 // 共享的实用函数
 ```

api/.env.example

@@ -162,6 +162,8 @@ PGVECTOR_PORT=5433
 PGVECTOR_USER=postgres
 PGVECTOR_PASSWORD=postgres
 PGVECTOR_DATABASE=postgres
+PGVECTOR_MIN_CONNECTION=1
+PGVECTOR_MAX_CONNECTION=5
 
 # Tidb Vector configuration
 TIDB_VECTOR_HOST=xxx.eu-central-1.xxx.aws.tidbcloud.com

api/README.md

@@ -65,14 +65,12 @@
 
 8. Start Dify [web](../web) service.
 9. Setup your application by visiting `http://localhost:3000`...
-10. If you need to debug local async processing, please start the worker service.
+10. If you need to handle and debug the async tasks (e.g. dataset importing and documents indexing), please start the worker service.
 
    ```bash
    poetry run python -m celery -A app.celery worker -P gevent -c 1 --loglevel INFO -Q dataset,generation,mail,ops_trace,app_deletion
    ```
 
-   The started celery app handles the async tasks, e.g. dataset importing and documents indexing.
-
 ## Testing
 
 1. Install dependencies for both the backend and the test environment

api/app.py

@@ -53,11 +53,9 @@ from services.account_service import AccountService
 
 warnings.simplefilter("ignore", ResourceWarning)
 
-# fix windows platform
+os.environ["TZ"] = "UTC"
-if os.name == "nt":
+# windows platform not support tzset
-    os.system('tzutil /s "UTC"')
+if hasattr(time, "tzset"):
-else:
-    os.environ["TZ"] = "UTC"
     time.tzset()
 
 
@@ -164,7 +162,7 @@ def initialize_extensions(app):
 @login_manager.request_loader
 def load_user_from_request(request_from_flask_login):
     """Load user based on the request."""
-    if request.blueprint not in ["console", "inner_api"]:
+    if request.blueprint not in {"console", "inner_api"}:
         return None
     # Check if the user_id contains a dot, indicating the old format
     auth_header = request.headers.get("Authorization", "")

api/commands.py

@@ -28,28 +28,28 @@ from services.account_service import RegisterService, TenantService
 
 
 @click.command("reset-password", help="Reset the account password.")
-@click.option("--email", prompt=True, help="The email address of the account whose password you need to reset")
+@click.option("--email", prompt=True, help="Account email to reset password for")
-@click.option("--new-password", prompt=True, help="the new password.")
+@click.option("--new-password", prompt=True, help="New password")
-@click.option("--password-confirm", prompt=True, help="the new password confirm.")
+@click.option("--password-confirm", prompt=True, help="Confirm new password")
 def reset_password(email, new_password, password_confirm):
     """
     Reset password of owner account
     Only available in SELF_HOSTED mode
     """
     if str(new_password).strip() != str(password_confirm).strip():
-        click.echo(click.style("sorry. The two passwords do not match.", fg="red"))
+        click.echo(click.style("Passwords do not match.", fg="red"))
         return
 
     account = db.session.query(Account).filter(Account.email == email).one_or_none()
 
     if not account:
-        click.echo(click.style("sorry. the account: [{}] not exist .".format(email), fg="red"))
+        click.echo(click.style("Account not found for email: {}".format(email), fg="red"))
         return
 
     try:
         valid_password(new_password)
     except:
-        click.echo(click.style("sorry. The passwords must match {} ".format(password_pattern), fg="red"))
+        click.echo(click.style("Invalid password. Must match {}".format(password_pattern), fg="red"))
         return
 
     # generate password salt
@@ -62,37 +62,37 @@ def reset_password(email, new_password, password_confirm):
     account.password = base64_password_hashed
     account.password_salt = base64_salt
     db.session.commit()
-    click.echo(click.style("Congratulations! Password has been reset.", fg="green"))
+    click.echo(click.style("Password reset successfully.", fg="green"))
 
 
 @click.command("reset-email", help="Reset the account email.")
-@click.option("--email", prompt=True, help="The old email address of the account whose email you need to reset")
+@click.option("--email", prompt=True, help="Current account email")
-@click.option("--new-email", prompt=True, help="the new email.")
+@click.option("--new-email", prompt=True, help="New email")
-@click.option("--email-confirm", prompt=True, help="the new email confirm.")
+@click.option("--email-confirm", prompt=True, help="Confirm new email")
 def reset_email(email, new_email, email_confirm):
     """
     Replace account email
     :return:
     """
     if str(new_email).strip() != str(email_confirm).strip():
-        click.echo(click.style("Sorry, new email and confirm email do not match.", fg="red"))
+        click.echo(click.style("New emails do not match.", fg="red"))
         return
 
     account = db.session.query(Account).filter(Account.email == email).one_or_none()
 
     if not account:
-        click.echo(click.style("sorry. the account: [{}] not exist .".format(email), fg="red"))
+        click.echo(click.style("Account not found for email: {}".format(email), fg="red"))
         return
 
     try:
         email_validate(new_email)
     except:
-        click.echo(click.style("sorry. {} is not a valid email. ".format(email), fg="red"))
+        click.echo(click.style("Invalid email: {}".format(new_email), fg="red"))
         return
 
     account.email = new_email
     db.session.commit()
-    click.echo(click.style("Congratulations!, email has been reset.", fg="green"))
+    click.echo(click.style("Email updated successfully.", fg="green"))
 
 
 @click.command(
@@ -104,7 +104,7 @@ def reset_email(email, new_email, email_confirm):
 )
 @click.confirmation_option(
     prompt=click.style(
-        "Are you sure you want to reset encrypt key pair?" " this operation cannot be rolled back!", fg="red"
+        "Are you sure you want to reset encrypt key pair? This operation cannot be rolled back!", fg="red"
     )
 )
 def reset_encrypt_key_pair():
@@ -114,13 +114,13 @@ def reset_encrypt_key_pair():
     Only support SELF_HOSTED mode.
     """
     if dify_config.EDITION != "SELF_HOSTED":
-        click.echo(click.style("Sorry, only support SELF_HOSTED mode.", fg="red"))
+        click.echo(click.style("This command is only for SELF_HOSTED installations.", fg="red"))
         return
 
     tenants = db.session.query(Tenant).all()
     for tenant in tenants:
         if not tenant:
-            click.echo(click.style("Sorry, no workspace found. Please enter /install to initialize.", fg="red"))
+            click.echo(click.style("No workspaces found. Run /install first.", fg="red"))
             return
 
         tenant.encrypt_public_key = generate_key_pair(tenant.id)
@@ -131,18 +131,18 @@ def reset_encrypt_key_pair():
 
         click.echo(
             click.style(
-                "Congratulations! " "the asymmetric key pair of workspace {} has been reset.".format(tenant.id),
+                "Congratulations! The asymmetric key pair of workspace {} has been reset.".format(tenant.id),
                 fg="green",
             )
         )
 
 
-@click.command("vdb-migrate", help="migrate vector db.")
+@click.command("vdb-migrate", help="Migrate vector db.")
 @click.option("--scope", default="all", prompt=False, help="The scope of vector database to migrate, Default is All.")
 def vdb_migrate(scope: str):
-    if scope in ["knowledge", "all"]:
+    if scope in {"knowledge", "all"}:
         migrate_knowledge_vector_database()
-    if scope in ["annotation", "all"]:
+    if scope in {"annotation", "all"}:
         migrate_annotation_vector_database()
 
 
@@ -150,7 +150,7 @@ def migrate_annotation_vector_database():
     """
     Migrate annotation datas to target vector database .
     """
-    click.echo(click.style("Start migrate annotation data.", fg="green"))
+    click.echo(click.style("Starting annotation data migration.", fg="green"))
     create_count = 0
     skipped_count = 0
     total_count = 0
@@ -174,14 +174,14 @@ def migrate_annotation_vector_database():
                 f"Processing the {total_count} app {app.id}. " + f"{create_count} created, {skipped_count} skipped."
             )
             try:
-                click.echo("Create app annotation index: {}".format(app.id))
+                click.echo("Creating app annotation index: {}".format(app.id))
                 app_annotation_setting = (
                     db.session.query(AppAnnotationSetting).filter(AppAnnotationSetting.app_id == app.id).first()
                 )
 
                 if not app_annotation_setting:
                     skipped_count = skipped_count + 1
-                    click.echo("App annotation setting is disabled: {}".format(app.id))
+                    click.echo("App annotation setting disabled: {}".format(app.id))
                     continue
                 # get dataset_collection_binding info
                 dataset_collection_binding = (
@@ -190,7 +190,7 @@ def migrate_annotation_vector_database():
                     .first()
                 )
                 if not dataset_collection_binding:
-                    click.echo("App annotation collection binding is not exist: {}".format(app.id))
+                    click.echo("App annotation collection binding not found: {}".format(app.id))
                     continue
                 annotations = db.session.query(MessageAnnotation).filter(MessageAnnotation.app_id == app.id).all()
                 dataset = Dataset(
@@ -211,11 +211,11 @@ def migrate_annotation_vector_database():
                         documents.append(document)
 
                 vector = Vector(dataset, attributes=["doc_id", "annotation_id", "app_id"])
-                click.echo(f"Start to migrate annotation, app_id: {app.id}.")
+                click.echo(f"Migrating annotations for app: {app.id}.")
 
                 try:
                     vector.delete()
-                    click.echo(click.style(f"Successfully delete vector index for app: {app.id}.", fg="green"))
+                    click.echo(click.style(f"Deleted vector index for app {app.id}.", fg="green"))
                 except Exception as e:
                     click.echo(click.style(f"Failed to delete vector index for app {app.id}.", fg="red"))
                     raise e
@@ -223,12 +223,12 @@ def migrate_annotation_vector_database():
                     try:
                         click.echo(
                             click.style(
-                                f"Start to created vector index with {len(documents)} annotations for app {app.id}.",
+                                f"Creating vector index with {len(documents)} annotations for app {app.id}.",
                                 fg="green",
                             )
                         )
                         vector.create(documents)
-                        click.echo(click.style(f"Successfully created vector index for app {app.id}.", fg="green"))
+                        click.echo(click.style(f"Created vector index for app {app.id}.", fg="green"))
                     except Exception as e:
                         click.echo(click.style(f"Failed to created vector index for app {app.id}.", fg="red"))
                         raise e
@@ -237,14 +237,14 @@ def migrate_annotation_vector_database():
             except Exception as e:
                 click.echo(
                     click.style(
-                        "Create app annotation index error: {} {}".format(e.__class__.__name__, str(e)), fg="red"
+                        "Error creating app annotation index: {} {}".format(e.__class__.__name__, str(e)), fg="red"
                     )
                 )
                 continue
 
     click.echo(
         click.style(
-            f"Congratulations! Create {create_count} app annotation indexes, and skipped {skipped_count} apps.",
+            f"Migration complete. Created {create_count} app annotation indexes. Skipped {skipped_count} apps.",
             fg="green",
         )
     )
@@ -254,7 +254,7 @@ def migrate_knowledge_vector_database():
     """
     Migrate vector database datas to target vector database .
     """
-    click.echo(click.style("Start migrate vector db.", fg="green"))
+    click.echo(click.style("Starting vector database migration.", fg="green"))
     create_count = 0
     skipped_count = 0
     total_count = 0
@@ -275,11 +275,10 @@ def migrate_knowledge_vector_database():
         for dataset in datasets:
             total_count = total_count + 1
             click.echo(
-                f"Processing the {total_count} dataset {dataset.id}. "
+                f"Processing the {total_count} dataset {dataset.id}. {create_count} created, {skipped_count} skipped."
-                + f"{create_count} created, {skipped_count} skipped."
             )
             try:
-                click.echo("Create dataset vdb index: {}".format(dataset.id))
+                click.echo("Creating dataset vector database index: {}".format(dataset.id))
                 if dataset.index_struct_dict:
                     if dataset.index_struct_dict["type"] == vector_type:
                         skipped_count = skipped_count + 1
@@ -300,7 +299,7 @@ def migrate_knowledge_vector_database():
                         if dataset_collection_binding:
                             collection_name = dataset_collection_binding.collection_name
                         else:
-                            raise ValueError("Dataset Collection Bindings is not exist!")
+                            raise ValueError("Dataset Collection Binding not found")
                     else:
                         dataset_id = dataset.id
                         collection_name = Dataset.gen_collection_name_by_id(dataset_id)
@@ -352,14 +351,12 @@ def migrate_knowledge_vector_database():
                     raise ValueError(f"Vector store {vector_type} is not supported.")
 
                 vector = Vector(dataset)
-                click.echo(f"Start to migrate dataset {dataset.id}.")
+                click.echo(f"Migrating dataset {dataset.id}.")
 
                 try:
                     vector.delete()
                     click.echo(
-                        click.style(
+                        click.style(f"Deleted vector index {collection_name} for dataset {dataset.id}.", fg="green")
-                            f"Successfully delete vector index {collection_name} for dataset {dataset.id}.", fg="green"
-                        )
                     )
                 except Exception as e:
                     click.echo(
@@ -411,15 +408,13 @@ def migrate_knowledge_vector_database():
                     try:
                         click.echo(
                             click.style(
-                                f"Start to created vector index with {len(documents)} documents of {segments_count}"
+                                f"Creating vector index with {len(documents)} documents of {segments_count}"
                                 f" segments for dataset {dataset.id}.",
                                 fg="green",
                             )
                         )
                         vector.create(documents)
-                        click.echo(
+                        click.echo(click.style(f"Created vector index for dataset {dataset.id}.", fg="green"))
-                            click.style(f"Successfully created vector index for dataset {dataset.id}.", fg="green")
-                        )
                     except Exception as e:
                         click.echo(click.style(f"Failed to created vector index for dataset {dataset.id}.", fg="red"))
                         raise e
@@ -430,13 +425,13 @@ def migrate_knowledge_vector_database():
             except Exception as e:
                 db.session.rollback()
                 click.echo(
-                    click.style("Create dataset index error: {} {}".format(e.__class__.__name__, str(e)), fg="red")
+                    click.style("Error creating dataset index: {} {}".format(e.__class__.__name__, str(e)), fg="red")
                 )
                 continue
 
     click.echo(
         click.style(
-            f"Congratulations! Create {create_count} dataset indexes, and skipped {skipped_count} datasets.", fg="green"
+            f"Migration complete. Created {create_count} dataset indexes. Skipped {skipped_count} datasets.", fg="green"
         )
     )
 
@@ -446,7 +441,7 @@ def convert_to_agent_apps():
     """
     Convert Agent Assistant to Agent App.
     """
-    click.echo(click.style("Start convert to agent apps.", fg="green"))
+    click.echo(click.style("Starting convert to agent apps.", fg="green"))
 
     proceeded_app_ids = []
 
@@ -497,23 +492,23 @@ def convert_to_agent_apps():
             except Exception as e:
                 click.echo(click.style("Convert app error: {} {}".format(e.__class__.__name__, str(e)), fg="red"))
 
-    click.echo(click.style("Congratulations! Converted {} agent apps.".format(len(proceeded_app_ids)), fg="green"))
+    click.echo(click.style("Conversion complete. Converted {} agent apps.".format(len(proceeded_app_ids)), fg="green"))
 
 
-@click.command("add-qdrant-doc-id-index", help="add qdrant doc_id index.")
+@click.command("add-qdrant-doc-id-index", help="Add Qdrant doc_id index.")
-@click.option("--field", default="metadata.doc_id", prompt=False, help="index field , default is metadata.doc_id.")
+@click.option("--field", default="metadata.doc_id", prompt=False, help="Index field , default is metadata.doc_id.")
 def add_qdrant_doc_id_index(field: str):
-    click.echo(click.style("Start add qdrant doc_id index.", fg="green"))
+    click.echo(click.style("Starting Qdrant doc_id index creation.", fg="green"))
     vector_type = dify_config.VECTOR_STORE
     if vector_type != "qdrant":
-        click.echo(click.style("Sorry, only support qdrant vector store.", fg="red"))
+        click.echo(click.style("This command only supports Qdrant vector store.", fg="red"))
         return
     create_count = 0
 
     try:
         bindings = db.session.query(DatasetCollectionBinding).all()
         if not bindings:
-            click.echo(click.style("Sorry, no dataset collection bindings found.", fg="red"))
+            click.echo(click.style("No dataset collection bindings found.", fg="red"))
             return
         import qdrant_client
         from qdrant_client.http.exceptions import UnexpectedResponse
@@ -523,7 +518,7 @@ def add_qdrant_doc_id_index(field: str):
 
         for binding in bindings:
             if dify_config.QDRANT_URL is None:
-                raise ValueError("Qdrant url is required.")
+                raise ValueError("Qdrant URL is required.")
             qdrant_config = QdrantConfig(
                 endpoint=dify_config.QDRANT_URL,
                 api_key=dify_config.QDRANT_API_KEY,
@@ -540,41 +535,39 @@ def add_qdrant_doc_id_index(field: str):
             except UnexpectedResponse as e:
                 # Collection does not exist, so return
                 if e.status_code == 404:
-                    click.echo(
+                    click.echo(click.style(f"Collection not found: {binding.collection_name}.", fg="red"))
-                        click.style(f"Collection not found, collection_name:{binding.collection_name}.", fg="red")
-                    )
                     continue
                 # Some other error occurred, so re-raise the exception
                 else:
                     click.echo(
                         click.style(
-                            f"Failed to create qdrant index, collection_name:{binding.collection_name}.", fg="red"
+                            f"Failed to create Qdrant index for collection: {binding.collection_name}.", fg="red"
                         )
                     )
 
     except Exception as e:
-        click.echo(click.style("Failed to create qdrant client.", fg="red"))
+        click.echo(click.style("Failed to create Qdrant client.", fg="red"))
 
-    click.echo(click.style(f"Congratulations! Create {create_count} collection indexes.", fg="green"))
+    click.echo(click.style(f"Index creation complete. Created {create_count} collection indexes.", fg="green"))
 
 
 @click.command("create-tenant", help="Create account and tenant.")
-@click.option("--email", prompt=True, help="The email address of the tenant account.")
+@click.option("--email", prompt=True, help="Tenant account email.")
-@click.option("--name", prompt=True, help="The workspace name of the tenant account.")
+@click.option("--name", prompt=True, help="Workspace name.")
 @click.option("--language", prompt=True, help="Account language, default: en-US.")
 def create_tenant(email: str, language: Optional[str] = None, name: Optional[str] = None):
     """
     Create tenant account
     """
     if not email:
-        click.echo(click.style("Sorry, email is required.", fg="red"))
+        click.echo(click.style("Email is required.", fg="red"))
         return
 
     # Create account
     email = email.strip()
 
     if "@" not in email:
-        click.echo(click.style("Sorry, invalid email address.", fg="red"))
+        click.echo(click.style("Invalid email address.", fg="red"))
         return
 
     account_name = email.split("@")[0]
@@ -594,19 +587,19 @@ def create_tenant(email: str, language: Optional[str] = None, name: Optional[str
 
     click.echo(
         click.style(
-            "Congratulations! Account and tenant created.\n" "Account: {}\nPassword: {}".format(email, new_password),
+            "Account and tenant created.\nAccount: {}\nPassword: {}".format(email, new_password),
             fg="green",
         )
     )
 
 
-@click.command("upgrade-db", help="upgrade the database")
+@click.command("upgrade-db", help="Upgrade the database")
 def upgrade_db():
     click.echo("Preparing database migration...")
     lock = redis_client.lock(name="db_upgrade_lock", timeout=60)
     if lock.acquire(blocking=False):
         try:
-            click.echo(click.style("Start database migration.", fg="green"))
+            click.echo(click.style("Starting database migration.", fg="green"))
 
             # run db migration
             import flask_migrate
@@ -616,7 +609,7 @@ def upgrade_db():
             click.echo(click.style("Database migration successful!", fg="green"))
 
         except Exception as e:
-            logging.exception(f"Database migration failed, error: {e}")
+            logging.exception(f"Database migration failed: {e}")
         finally:
             lock.release()
     else:
@@ -628,7 +621,7 @@ def fix_app_site_missing():
     """
     Fix app related site missing issue.
     """
-    click.echo(click.style("Start fix app related site missing issue.", fg="green"))
+    click.echo(click.style("Starting fix for missing app-related sites.", fg="green"))
 
     failed_app_ids = []
     while True:
@@ -651,22 +644,22 @@ where sites.id is null limit 1000"""
                     if tenant:
                         accounts = tenant.get_accounts()
                         if not accounts:
-                            print("Fix app {} failed.".format(app.id))
+                            print("Fix failed for app {}".format(app.id))
                             continue
 
                         account = accounts[0]
-                        print("Fix app {} related site missing issue.".format(app.id))
+                        print("Fixing missing site for app {}".format(app.id))
                         app_was_created.send(app, account=account)
                 except Exception as e:
                     failed_app_ids.append(app_id)
-                    click.echo(click.style("Fix app {} related site missing issue failed!".format(app_id), fg="red"))
+                    click.echo(click.style("Failed to fix missing site for app {}".format(app_id), fg="red"))
                     logging.exception(f"Fix app related site missing issue failed, error: {e}")
                     continue
 
             if not processed_count:
                 break
 
-    click.echo(click.style("Congratulations! Fix app related site missing issue successful!", fg="green"))
+    click.echo(click.style("Fix for missing app-related sites completed successfully!", fg="green"))
 
 
 def register_commands(app):

api/configs/deploy/__init__.py

@@ -4,30 +4,30 @@ from pydantic_settings import BaseSettings
 
 class DeploymentConfig(BaseSettings):
     """
-    Deployment configs
+    Configuration settings for application deployment
     """
 
     APPLICATION_NAME: str = Field(
-        description="application name",
+        description="Name of the application, used for identification and logging purposes",
         default="langgenius/dify",
     )
 
     DEBUG: bool = Field(
-        description="whether to enable debug mode.",
+        description="Enable debug mode for additional logging and development features",
         default=False,
     )
 
     TESTING: bool = Field(
-        description="",
+        description="Enable testing mode for running automated tests",
         default=False,
     )
 
     EDITION: str = Field(
-        description="deployment edition",
+        description="Deployment edition of the application (e.g., 'SELF_HOSTED', 'CLOUD')",
         default="SELF_HOSTED",
     )
 
     DEPLOY_ENV: str = Field(
-        description="deployment environment, default to PRODUCTION.",
+        description="Deployment environment (e.g., 'PRODUCTION', 'DEVELOPMENT'), default to PRODUCTION",
         default="PRODUCTION",
     )

api/configs/enterprise/__init__.py

@@ -4,17 +4,17 @@ from pydantic_settings import BaseSettings
 
 class EnterpriseFeatureConfig(BaseSettings):
     """
-    Enterprise feature configs.
+    Configuration for enterprise-level features.
     **Before using, please contact business@dify.ai by email to inquire about licensing matters.**
     """
 
     ENTERPRISE_ENABLED: bool = Field(
-        description="whether to enable enterprise features."
+        description="Enable or disable enterprise-level features."
         "Before using, please contact business@dify.ai by email to inquire about licensing matters.",
         default=False,
     )
 
     CAN_REPLACE_LOGO: bool = Field(
-        description="whether to allow replacing enterprise logo.",
+        description="Allow customization of the enterprise logo.",
         default=False,
     )

api/configs/extra/notion_config.py

@@ -6,30 +6,31 @@ from pydantic_settings import BaseSettings
 
 class NotionConfig(BaseSettings):
     """
-    Notion integration configs
+    Configuration settings for Notion integration
     """
 
     NOTION_CLIENT_ID: Optional[str] = Field(
-        description="Notion client ID",
+        description="Client ID for Notion API authentication. Required for OAuth 2.0 flow.",
         default=None,
     )
 
     NOTION_CLIENT_SECRET: Optional[str] = Field(
-        description="Notion client secret key",
+        description="Client secret for Notion API authentication. Required for OAuth 2.0 flow.",
         default=None,
     )
 
     NOTION_INTEGRATION_TYPE: Optional[str] = Field(
-        description="Notion integration type, default to None, available values: internal.",
+        description="Type of Notion integration."
+        " Set to 'internal' for internal integrations, or None for public integrations.",
         default=None,
     )
 
     NOTION_INTERNAL_SECRET: Optional[str] = Field(
-        description="Notion internal secret key",
+        description="Secret key for internal Notion integrations. Required when NOTION_INTEGRATION_TYPE is 'internal'.",
         default=None,
     )
 
     NOTION_INTEGRATION_TOKEN: Optional[str] = Field(
-        description="Notion integration token",
+        description="Integration token for Notion API access. Used for direct API calls without OAuth flow.",
         default=None,
     )

api/configs/extra/sentry_config.py

@@ -6,20 +6,23 @@ from pydantic_settings import BaseSettings
 
 class SentryConfig(BaseSettings):
     """
-    Sentry configs
+    Configuration settings for Sentry error tracking and performance monitoring
     """
 
     SENTRY_DSN: Optional[str] = Field(
-        description="Sentry DSN",
+        description="Sentry Data Source Name (DSN)."
+        " This is the unique identifier of your Sentry project, used to send events to the correct project.",
         default=None,
     )
 
     SENTRY_TRACES_SAMPLE_RATE: NonNegativeFloat = Field(
-        description="Sentry trace sample rate",
+        description="Sample rate for Sentry performance monitoring traces."
+        " Value between 0.0 and 1.0, where 1.0 means 100% of traces are sent to Sentry.",
         default=1.0,
     )
 
     SENTRY_PROFILES_SAMPLE_RATE: NonNegativeFloat = Field(
-        description="Sentry profiles sample rate",
+        description="Sample rate for Sentry profiling."
+        " Value between 0.0 and 1.0, where 1.0 means 100% of profiles are sent to Sentry.",
         default=1.0,
     )

api/configs/feature/__init__.py

@@ -8,145 +8,143 @@ from configs.feature.hosted_service import HostedServiceConfig
 
 class SecurityConfig(BaseSettings):
     """
-    Secret Key configs
+    Security-related configurations for the application
     """
 
     SECRET_KEY: Optional[str] = Field(
-        description="Your App secret key will be used for securely signing the session cookie"
+        description="Secret key for secure session cookie signing."
         "Make sure you are changing this key for your deployment with a strong key."
-        "You can generate a strong key using `openssl rand -base64 42`."
+        "Generate a strong key using `openssl rand -base64 42` or set via the `SECRET_KEY` environment variable.",
-        "Alternatively you can set it with `SECRET_KEY` environment variable.",
         default=None,
     )
 
     RESET_PASSWORD_TOKEN_EXPIRY_HOURS: PositiveInt = Field(
-        description="Expiry time in hours for reset token",
+        description="Duration in hours for which a password reset token remains valid",
         default=24,
     )
 
 
 class AppExecutionConfig(BaseSettings):
     """
-    App Execution configs
+    Configuration parameters for application execution
     """
 
     APP_MAX_EXECUTION_TIME: PositiveInt = Field(
-        description="execution timeout in seconds for app execution",
+        description="Maximum allowed execution time for the application in seconds",
         default=1200,
     )
     APP_MAX_ACTIVE_REQUESTS: NonNegativeInt = Field(
-        description="max active request per app, 0 means unlimited",
+        description="Maximum number of concurrent active requests per app (0 for unlimited)",
         default=0,
     )
 
 
 class CodeExecutionSandboxConfig(BaseSettings):
     """
-    Code Execution Sandbox configs
+    Configuration for the code execution sandbox environment
     """
 
     CODE_EXECUTION_ENDPOINT: HttpUrl = Field(
-        description="endpoint URL of code execution service",
+        description="URL endpoint for the code execution service",
         default="http://sandbox:8194",
     )
 
     CODE_EXECUTION_API_KEY: str = Field(
-        description="API key for code execution service",
+        description="API key for accessing the code execution service",
         default="dify-sandbox",
     )
 
     CODE_EXECUTION_CONNECT_TIMEOUT: Optional[float] = Field(
-        description="connect timeout in seconds for code execution request",
+        description="Connection timeout in seconds for code execution requests",
         default=10.0,
     )
 
     CODE_EXECUTION_READ_TIMEOUT: Optional[float] = Field(
-        description="read timeout in seconds for code execution request",
+        description="Read timeout in seconds for code execution requests",
         default=60.0,
     )
 
     CODE_EXECUTION_WRITE_TIMEOUT: Optional[float] = Field(
-        description="write timeout in seconds for code execution request",
+        description="Write timeout in seconds for code execution request",
         default=10.0,
     )
 
     CODE_MAX_NUMBER: PositiveInt = Field(
-        description="max depth for code execution",
+        description="Maximum allowed numeric value in code execution",
         default=9223372036854775807,
     )
 
     CODE_MIN_NUMBER: NegativeInt = Field(
-        description="",
+        description="Minimum allowed numeric value in code execution",
         default=-9223372036854775807,
     )
 
     CODE_MAX_DEPTH: PositiveInt = Field(
-        description="max depth for code execution",
+        description="Maximum allowed depth for nested structures in code execution",
         default=5,
     )
 
     CODE_MAX_PRECISION: PositiveInt = Field(
-        description="max precision digits for float type in code execution",
+        description="mMaximum number of decimal places for floating-point numbers in code execution",
         default=20,
     )
 
     CODE_MAX_STRING_LENGTH: PositiveInt = Field(
-        description="max string length for code execution",
+        description="Maximum allowed length for strings in code execution",
         default=80000,
     )
 
     CODE_MAX_STRING_ARRAY_LENGTH: PositiveInt = Field(
-        description="",
+        description="Maximum allowed length for string arrays in code execution",
         default=30,
     )
 
     CODE_MAX_OBJECT_ARRAY_LENGTH: PositiveInt = Field(
-        description="",
+        description="Maximum allowed length for object arrays in code execution",
         default=30,
     )
 
     CODE_MAX_NUMBER_ARRAY_LENGTH: PositiveInt = Field(
-        description="",
+        description="Maximum allowed length for numeric arrays in code execution",
         default=1000,
     )
 
 
 class EndpointConfig(BaseSettings):
     """
-    Module URL configs
+    Configuration for various application endpoints and URLs
     """
 
     CONSOLE_API_URL: str = Field(
-        description="The backend URL prefix of the console API."
+        description="Base URL for the console API,"
-        "used to concatenate the login authorization callback or notion integration callback.",
+        "used for login authentication callback or notion integration callbacks",
         default="",
     )
 
     CONSOLE_WEB_URL: str = Field(
-        description="The front-end URL prefix of the console web."
+        description="Base URL for the console web interface," "used for frontend references and CORS configuration",
-        "used to concatenate some front-end addresses and for CORS configuration use.",
         default="",
     )
 
     SERVICE_API_URL: str = Field(
-        description="Service API Url prefix." "used to display Service API Base Url to the front-end.",
+        description="Base URL for the service API, displayed to users for API access",
         default="",
     )
 
     APP_WEB_URL: str = Field(
-        description="WebApp Url prefix." "used to display WebAPP API Base Url to the front-end.",
+        description="Base URL for the web application, used for frontend references",
         default="",
     )
 
 
 class FileAccessConfig(BaseSettings):
     """
-    File Access configs
+    Configuration for file access and handling
     """
 
     FILES_URL: str = Field(
-        description="File preview or download Url prefix."
+        description="Base URL for file preview or download,"
-        " used to display File preview or download Url to the front-end or as Multi-model inputs;"
+        " used for frontend display and multi-model inputs"
         "Url is signed and has expiration time.",
         validation_alias=AliasChoices("FILES_URL", "CONSOLE_API_URL"),
         alias_priority=1,
@@ -154,49 +152,49 @@ class FileAccessConfig(BaseSettings):
     )
 
     FILES_ACCESS_TIMEOUT: int = Field(
-        description="timeout in seconds for file accessing",
+        description="Expiration time in seconds for file access URLs",
         default=300,
     )
 
 
 class FileUploadConfig(BaseSettings):
     """
-    File Uploading configs
+    Configuration for file upload limitations
     """
 
     UPLOAD_FILE_SIZE_LIMIT: NonNegativeInt = Field(
-        description="size limit in Megabytes for uploading files",
+        description="Maximum allowed file size for uploads in megabytes",
         default=15,
     )
 
     UPLOAD_FILE_BATCH_LIMIT: NonNegativeInt = Field(
-        description="batch size limit for uploading files",
+        description="Maximum number of files allowed in a single upload batch",
         default=5,
     )
 
     UPLOAD_IMAGE_FILE_SIZE_LIMIT: NonNegativeInt = Field(
-        description="image file size limit in Megabytes for uploading files",
+        description="Maximum allowed image file size for uploads in megabytes",
         default=10,
     )
 
     BATCH_UPLOAD_LIMIT: NonNegativeInt = Field(
-        description="",  # todo: to be clarified
+        description="Maximum number of files allowed in a batch upload operation",
         default=20,
     )
 
 
 class HttpConfig(BaseSettings):
     """
-    HTTP configs
+    HTTP-related configurations for the application
     """
 
     API_COMPRESSION_ENABLED: bool = Field(
-        description="whether to enable HTTP response compression of gzip",
+        description="Enable or disable gzip compression for HTTP responses",
         default=False,
     )
 
     inner_CONSOLE_CORS_ALLOW_ORIGINS: str = Field(
-        description="",
+        description="Comma-separated list of allowed origins for CORS in the console",
         validation_alias=AliasChoices("CONSOLE_CORS_ALLOW_ORIGINS", "CONSOLE_WEB_URL"),
         default="",
     )
@@ -218,359 +216,360 @@ class HttpConfig(BaseSettings):
         return self.inner_WEB_API_CORS_ALLOW_ORIGINS.split(",")
 
     HTTP_REQUEST_MAX_CONNECT_TIMEOUT: Annotated[
-        PositiveInt, Field(ge=10, description="connect timeout in seconds for HTTP request")
+        PositiveInt, Field(ge=10, description="Maximum connection timeout in seconds for HTTP requests")
     ] = 10
 
     HTTP_REQUEST_MAX_READ_TIMEOUT: Annotated[
-        PositiveInt, Field(ge=60, description="read timeout in seconds for HTTP request")
+        PositiveInt, Field(ge=60, description="Maximum read timeout in seconds for HTTP requests")
     ] = 60
 
     HTTP_REQUEST_MAX_WRITE_TIMEOUT: Annotated[
-        PositiveInt, Field(ge=10, description="read timeout in seconds for HTTP request")
+        PositiveInt, Field(ge=10, description="Maximum write timeout in seconds for HTTP requests")
     ] = 20
 
     HTTP_REQUEST_NODE_MAX_BINARY_SIZE: PositiveInt = Field(
-        description="",
+        description="Maximum allowed size in bytes for binary data in HTTP requests",
         default=10 * 1024 * 1024,
     )
 
     HTTP_REQUEST_NODE_MAX_TEXT_SIZE: PositiveInt = Field(
-        description="",
+        description="Maximum allowed size in bytes for text data in HTTP requests",
         default=1 * 1024 * 1024,
     )
 
     SSRF_PROXY_HTTP_URL: Optional[str] = Field(
-        description="HTTP URL for SSRF proxy",
+        description="Proxy URL for HTTP requests to prevent Server-Side Request Forgery (SSRF)",
         default=None,
     )
 
     SSRF_PROXY_HTTPS_URL: Optional[str] = Field(
-        description="HTTPS URL for SSRF proxy",
+        description="Proxy URL for HTTPS requests to prevent Server-Side Request Forgery (SSRF)",
         default=None,
     )
 
 
 class InnerAPIConfig(BaseSettings):
     """
-    Inner API configs
+    Configuration for internal API functionality
     """
 
     INNER_API: bool = Field(
-        description="whether to enable the inner API",
+        description="Enable or disable the internal API",
         default=False,
     )
 
     INNER_API_KEY: Optional[str] = Field(
-        description="The inner API key is used to authenticate the inner API",
+        description="API key for accessing the internal API",
         default=None,
     )
 
 
 class LoggingConfig(BaseSettings):
     """
-    Logging configs
+    Configuration for application logging
     """
 
     LOG_LEVEL: str = Field(
-        description="Log output level, default to INFO." "It is recommended to set it to ERROR for production.",
+        description="Logging level, default to INFO. Set to ERROR for production environments.",
         default="INFO",
     )
 
     LOG_FILE: Optional[str] = Field(
-        description="logging output file path",
+        description="File path for log output.",
         default=None,
     )
 
     LOG_FORMAT: str = Field(
-        description="log format",
+        description="Format string for log messages",
         default="%(asctime)s.%(msecs)03d %(levelname)s [%(threadName)s] [%(filename)s:%(lineno)d] - %(message)s",
     )
 
     LOG_DATEFORMAT: Optional[str] = Field(
-        description="log date format",
+        description="Date format string for log timestamps",
         default=None,
     )
 
     LOG_TZ: Optional[str] = Field(
-        description="specify log timezone, eg: America/New_York",
+        description="Timezone for log timestamps (e.g., 'America/New_York')",
         default=None,
     )
 
 
 class ModelLoadBalanceConfig(BaseSettings):
     """
-    Model load balance configs
+    Configuration for model load balancing
     """
 
     MODEL_LB_ENABLED: bool = Field(
-        description="whether to enable model load balancing",
+        description="Enable or disable load balancing for models",
         default=False,
     )
 
 
 class BillingConfig(BaseSettings):
     """
-    Platform Billing Configurations
+    Configuration for platform billing features
     """
 
     BILLING_ENABLED: bool = Field(
-        description="whether to enable billing",
+        description="Enable or disable billing functionality",
         default=False,
     )
 
 
 class UpdateConfig(BaseSettings):
     """
-    Update configs
+    Configuration for application update checks
     """
 
     CHECK_UPDATE_URL: str = Field(
-        description="url for checking updates",
+        description="URL to check for application updates",
         default="https://updates.dify.ai",
     )
 
 
 class WorkflowConfig(BaseSettings):
     """
-    Workflow feature configs
+    Configuration for workflow execution
     """
 
     WORKFLOW_MAX_EXECUTION_STEPS: PositiveInt = Field(
-        description="max execution steps in single workflow execution",
+        description="Maximum number of steps allowed in a single workflow execution",
         default=500,
     )
 
     WORKFLOW_MAX_EXECUTION_TIME: PositiveInt = Field(
-        description="max execution time in seconds in single workflow execution",
+        description="Maximum execution time in seconds for a single workflow",
         default=1200,
     )
 
     WORKFLOW_CALL_MAX_DEPTH: PositiveInt = Field(
-        description="max depth of calling in single workflow execution",
+        description="Maximum allowed depth for nested workflow calls",
         default=5,
     )
 
     MAX_VARIABLE_SIZE: PositiveInt = Field(
-        description="The maximum size in bytes of a variable. default to 5KB.",
+        description="Maximum size in bytes for a single variable in workflows. Default to 5KB.",
         default=5 * 1024,
     )
 
 
 class OAuthConfig(BaseSettings):
     """
-    oauth configs
+    Configuration for OAuth authentication
     """
 
     OAUTH_REDIRECT_PATH: str = Field(
-        description="redirect path for OAuth",
+        description="Redirect path for OAuth authentication callbacks",
         default="/console/api/oauth/authorize",
     )
 
     GITHUB_CLIENT_ID: Optional[str] = Field(
-        description="GitHub client id for OAuth",
+        description="GitHub OAuth client secret",
         default=None,
     )
 
     GITHUB_CLIENT_SECRET: Optional[str] = Field(
-        description="GitHub client secret key for OAuth",
+        description="GitHub OAuth client secret",
         default=None,
     )
 
     GOOGLE_CLIENT_ID: Optional[str] = Field(
-        description="Google client id for OAuth",
+        description="Google OAuth client ID",
         default=None,
     )
 
     GOOGLE_CLIENT_SECRET: Optional[str] = Field(
-        description="Google client secret key for OAuth",
+        description="Google OAuth client secret",
         default=None,
     )
 
 
 class ModerationConfig(BaseSettings):
     """
-    Moderation in app configs.
+    Configuration for content moderation
     """
 
     MODERATION_BUFFER_SIZE: PositiveInt = Field(
-        description="buffer size for moderation",
+        description="Size of the buffer for content moderation processing",
         default=300,
     )
 
 
 class ToolConfig(BaseSettings):
     """
-    Tool configs
+    Configuration for tool management
     """
 
     TOOL_ICON_CACHE_MAX_AGE: PositiveInt = Field(
-        description="max age in seconds for tool icon caching",
+        description="Maximum age in seconds for caching tool icons",
         default=3600,
     )
 
 
 class MailConfig(BaseSettings):
     """
-    Mail Configurations
+    Configuration for email services
     """
 
     MAIL_TYPE: Optional[str] = Field(
-        description="Mail provider type name, default to None, available values are `smtp` and `resend`.",
+        description="Email service provider type ('smtp' or 'resend'), default to None.",
         default=None,
     )
 
     MAIL_DEFAULT_SEND_FROM: Optional[str] = Field(
-        description="default email address for sending from ",
+        description="Default email address to use as the sender",
         default=None,
     )
 
     RESEND_API_KEY: Optional[str] = Field(
-        description="API key for Resend",
+        description="API key for Resend email service",
         default=None,
     )
 
     RESEND_API_URL: Optional[str] = Field(
-        description="API URL for Resend",
+        description="API URL for Resend email service",
         default=None,
     )
 
     SMTP_SERVER: Optional[str] = Field(
-        description="smtp server host",
+        description="SMTP server hostname",
         default=None,
     )
 
     SMTP_PORT: Optional[int] = Field(
-        description="smtp server port",
+        description="SMTP server port number",
         default=465,
     )
 
     SMTP_USERNAME: Optional[str] = Field(
-        description="smtp server username",
+        description="Username for SMTP authentication",
         default=None,
     )
 
     SMTP_PASSWORD: Optional[str] = Field(
-        description="smtp server password",
+        description="Password for SMTP authentication",
         default=None,
     )
 
     SMTP_USE_TLS: bool = Field(
-        description="whether to use TLS connection to smtp server",
+        description="Enable TLS encryption for SMTP connections",
         default=False,
     )
 
     SMTP_OPPORTUNISTIC_TLS: bool = Field(
-        description="whether to use opportunistic TLS connection to smtp server",
+        description="Enable opportunistic TLS for SMTP connections",
         default=False,
     )
 
 
 class RagEtlConfig(BaseSettings):
     """
-    RAG ETL Configurations.
+    Configuration for RAG ETL processes
     """
 
     ETL_TYPE: str = Field(
-        description="RAG ETL type name, default to `dify`, available values are `dify` and `Unstructured`. ",
+        description="RAG ETL type ('dify' or 'Unstructured'), default to 'dify'",
         default="dify",
     )
 
     KEYWORD_DATA_SOURCE_TYPE: str = Field(
-        description="source type for keyword data, default to `database`, available values are `database` .",
+        description="Data source type for keyword extraction"
+        " ('database' or other supported types), default to 'database'",
         default="database",
     )
 
     UNSTRUCTURED_API_URL: Optional[str] = Field(
-        description="API URL for Unstructured",
+        description="API URL for Unstructured.io service",
         default=None,
     )
 
     UNSTRUCTURED_API_KEY: Optional[str] = Field(
-        description="API key for Unstructured",
+        description="API key for Unstructured.io service",
         default=None,
     )
 
 
 class DataSetConfig(BaseSettings):
     """
-    Dataset configs
+    Configuration for dataset management
     """
 
     CLEAN_DAY_SETTING: PositiveInt = Field(
-        description="interval in days for cleaning up dataset",
+        description="Interval in days for dataset cleanup operations",
         default=30,
     )
 
     DATASET_OPERATOR_ENABLED: bool = Field(
-        description="whether to enable dataset operator",
+        description="Enable or disable dataset operator functionality",
         default=False,
     )
 
 
 class WorkspaceConfig(BaseSettings):
     """
-    Workspace configs
+    Configuration for workspace management
     """
 
     INVITE_EXPIRY_HOURS: PositiveInt = Field(
-        description="workspaces invitation expiration in hours",
+        description="Expiration time in hours for workspace invitation links",
         default=72,
     )
 
 
 class IndexingConfig(BaseSettings):
     """
-    Indexing configs.
+    Configuration for indexing operations
     """
 
     INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH: PositiveInt = Field(
-        description="max segmentation token length for indexing",
+        description="Maximum token length for text segmentation during indexing",
         default=1000,
     )
 
 
 class ImageFormatConfig(BaseSettings):
     MULTIMODAL_SEND_IMAGE_FORMAT: str = Field(
-        description="multi model send image format, support base64, url, default is base64",
+        description="Format for sending images in multimodal contexts ('base64' or 'url'), default is base64",
         default="base64",
     )
 
 
 class CeleryBeatConfig(BaseSettings):
     CELERY_BEAT_SCHEDULER_TIME: int = Field(
-        description="the time of the celery scheduler, default to 1 day",
+        description="Interval in days for Celery Beat scheduler execution, default to 1 day",
         default=1,
     )
 
 
 class PositionConfig(BaseSettings):
     POSITION_PROVIDER_PINS: str = Field(
-        description="The heads of model providers",
+        description="Comma-separated list of pinned model providers",
         default="",
     )
 
     POSITION_PROVIDER_INCLUDES: str = Field(
-        description="The included model providers",
+        description="Comma-separated list of included model providers",
         default="",
     )
 
     POSITION_PROVIDER_EXCLUDES: str = Field(
-        description="The excluded model providers",
+        description="Comma-separated list of excluded model providers",
         default="",
     )
 
     POSITION_TOOL_PINS: str = Field(
-        description="The heads of tools",
+        description="Comma-separated list of pinned tools",
         default="",
     )
 
     POSITION_TOOL_INCLUDES: str = Field(
-        description="The included tools",
+        description="Comma-separated list of included tools",
         default="",
     )
 
     POSITION_TOOL_EXCLUDES: str = Field(
-        description="The excluded tools",
+        description="Comma-separated list of excluded tools",
         default="",
     )
 

api/configs/feature/hosted_service/__init__.py

@@ -6,31 +6,31 @@ from pydantic_settings import BaseSettings
 
 class HostedOpenAiConfig(BaseSettings):
     """
-    Hosted OpenAI service config
+    Configuration for hosted OpenAI service
     """
 
     HOSTED_OPENAI_API_KEY: Optional[str] = Field(
-        description="",
+        description="API key for hosted OpenAI service",
         default=None,
     )
 
     HOSTED_OPENAI_API_BASE: Optional[str] = Field(
-        description="",
+        description="Base URL for hosted OpenAI API",
         default=None,
     )
 
     HOSTED_OPENAI_API_ORGANIZATION: Optional[str] = Field(
-        description="",
+        description="Organization ID for hosted OpenAI service",
         default=None,
     )
 
     HOSTED_OPENAI_TRIAL_ENABLED: bool = Field(
-        description="",
+        description="Enable trial access to hosted OpenAI service",
         default=False,
     )
 
     HOSTED_OPENAI_TRIAL_MODELS: str = Field(
-        description="",
+        description="Comma-separated list of available models for trial access",
         default="gpt-3.5-turbo,"
         "gpt-3.5-turbo-1106,"
         "gpt-3.5-turbo-instruct,"
@@ -42,17 +42,17 @@ class HostedOpenAiConfig(BaseSettings):
     )
 
     HOSTED_OPENAI_QUOTA_LIMIT: NonNegativeInt = Field(
-        description="",
+        description="Quota limit for hosted OpenAI service usage",
         default=200,
     )
 
     HOSTED_OPENAI_PAID_ENABLED: bool = Field(
-        description="",
+        description="Enable paid access to hosted OpenAI service",
         default=False,
     )
 
     HOSTED_OPENAI_PAID_MODELS: str = Field(
-        description="",
+        description="Comma-separated list of available models for paid access",
         default="gpt-4,"
         "gpt-4-turbo-preview,"
         "gpt-4-turbo-2024-04-09,"
@@ -71,124 +71,122 @@ class HostedOpenAiConfig(BaseSettings):
 
 class HostedAzureOpenAiConfig(BaseSettings):
     """
-    Hosted OpenAI service config
+    Configuration for hosted Azure OpenAI service
     """
 
     HOSTED_AZURE_OPENAI_ENABLED: bool = Field(
-        description="",
+        description="Enable hosted Azure OpenAI service",
         default=False,
     )
 
     HOSTED_AZURE_OPENAI_API_KEY: Optional[str] = Field(
-        description="",
+        description="API key for hosted Azure OpenAI service",
         default=None,
     )
 
     HOSTED_AZURE_OPENAI_API_BASE: Optional[str] = Field(
-        description="",
+        description="Base URL for hosted Azure OpenAI API",
         default=None,
     )
 
     HOSTED_AZURE_OPENAI_QUOTA_LIMIT: NonNegativeInt = Field(
-        description="",
+        description="Quota limit for hosted Azure OpenAI service usage",
         default=200,
     )
 
 
 class HostedAnthropicConfig(BaseSettings):
     """
-    Hosted Azure OpenAI service config
+    Configuration for hosted Anthropic service
     """
 
     HOSTED_ANTHROPIC_API_BASE: Optional[str] = Field(
-        description="",
+        description="Base URL for hosted Anthropic API",
         default=None,
     )
 
     HOSTED_ANTHROPIC_API_KEY: Optional[str] = Field(
-        description="",
+        description="API key for hosted Anthropic service",
         default=None,
     )
 
     HOSTED_ANTHROPIC_TRIAL_ENABLED: bool = Field(
-        description="",
+        description="Enable trial access to hosted Anthropic service",
         default=False,
     )
 
     HOSTED_ANTHROPIC_QUOTA_LIMIT: NonNegativeInt = Field(
-        description="",
+        description="Quota limit for hosted Anthropic service usage",
         default=600000,
     )
 
     HOSTED_ANTHROPIC_PAID_ENABLED: bool = Field(
-        description="",
+        description="Enable paid access to hosted Anthropic service",
         default=False,
     )
 
 
 class HostedMinmaxConfig(BaseSettings):
     """
-    Hosted Minmax service config
+    Configuration for hosted Minmax service
     """
 
     HOSTED_MINIMAX_ENABLED: bool = Field(
-        description="",
+        description="Enable hosted Minmax service",
         default=False,
     )
 
 
 class HostedSparkConfig(BaseSettings):
     """
-    Hosted Spark service config
+    Configuration for hosted Spark service
     """
 
     HOSTED_SPARK_ENABLED: bool = Field(
-        description="",
+        description="Enable hosted Spark service",
         default=False,
     )
 
 
 class HostedZhipuAIConfig(BaseSettings):
     """
-    Hosted Minmax service config
+    Configuration for hosted ZhipuAI service
     """
 
     HOSTED_ZHIPUAI_ENABLED: bool = Field(
-        description="",
+        description="Enable hosted ZhipuAI service",
         default=False,
     )
 
 
 class HostedModerationConfig(BaseSettings):
     """
-    Hosted Moderation service config
+    Configuration for hosted Moderation service
     """
 
     HOSTED_MODERATION_ENABLED: bool = Field(
-        description="",
+        description="Enable hosted Moderation service",
         default=False,
     )
 
     HOSTED_MODERATION_PROVIDERS: str = Field(
-        description="",
+        description="Comma-separated list of moderation providers",
         default="",
     )
 
 
 class HostedFetchAppTemplateConfig(BaseSettings):
     """
-    Hosted Moderation service config
+    Configuration for fetching app templates
     """
 
     HOSTED_FETCH_APP_TEMPLATES_MODE: str = Field(
-        description="the mode for fetching app templates,"
+        description="Mode for fetching app templates: remote, db, or builtin" " default to remote,",
-        " default to remote,"
-        " available values: remote, db, builtin",
         default="remote",
     )
 
     HOSTED_FETCH_APP_TEMPLATES_REMOTE_DOMAIN: str = Field(
-        description="the domain for fetching remote app templates",
+        description="Domain for fetching remote app templates",
         default="https://tmpl.dify.ai",
     )
 

api/configs/middleware/__init__.py

@@ -5,6 +5,7 @@ from pydantic import Field, NonNegativeInt, PositiveFloat, PositiveInt, computed
 from pydantic_settings import BaseSettings
 
 from configs.middleware.cache.redis_config import RedisConfig
+from configs.middleware.external.bedrock_config import BedrockConfig
 from configs.middleware.storage.aliyun_oss_storage_config import AliyunOSSStorageConfig
 from configs.middleware.storage.amazon_s3_storage_config import S3StorageConfig
 from configs.middleware.storage.azure_blob_storage_config import AzureBlobStorageConfig
@@ -31,70 +32,71 @@ from configs.middleware.vdb.weaviate_config import WeaviateConfig
 
 class StorageConfig(BaseSettings):
     STORAGE_TYPE: str = Field(
-        description="storage type,"
+        description="Type of storage to use."
-        " default to `local`,"
+        " Options: 'local', 's3', 'azure-blob', 'aliyun-oss', 'google-storage'. Default is 'local'.",
-        " available values are `local`, `s3`, `azure-blob`, `aliyun-oss`, `google-storage`.",
         default="local",
     )
 
     STORAGE_LOCAL_PATH: str = Field(
-        description="local storage path",
+        description="Path for local storage when STORAGE_TYPE is set to 'local'.",
         default="storage",
     )
 
 
 class VectorStoreConfig(BaseSettings):
     VECTOR_STORE: Optional[str] = Field(
-        description="vector store type",
+        description="Type of vector store to use for efficient similarity search."
+        " Set to None if not using a vector store.",
         default=None,
     )
 
 
 class KeywordStoreConfig(BaseSettings):
     KEYWORD_STORE: str = Field(
-        description="keyword store type",
+        description="Method for keyword extraction and storage."
+        " Default is 'jieba', a Chinese text segmentation library.",
         default="jieba",
     )
 
 
 class DatabaseConfig:
     DB_HOST: str = Field(
-        description="db host",
+        description="Hostname or IP address of the database server.",
         default="localhost",
     )
 
     DB_PORT: PositiveInt = Field(
-        description="db port",
+        description="Port number for database connection.",
         default=5432,
     )
 
     DB_USERNAME: str = Field(
-        description="db username",
+        description="Username for database authentication.",
         default="postgres",
     )
 
     DB_PASSWORD: str = Field(
-        description="db password",
+        description="Password for database authentication.",
         default="",
     )
 
     DB_DATABASE: str = Field(
-        description="db database",
+        description="Name of the database to connect to.",
         default="dify",
     )
 
     DB_CHARSET: str = Field(
-        description="db charset",
+        description="Character set for database connection.",
         default="",
     )
 
     DB_EXTRAS: str = Field(
-        description="db extras options. Example: keepalives_idle=60&keepalives=1",
+        description="Additional database connection parameters. Example: 'keepalives_idle=60&keepalives=1'",
         default="",
     )
 
     SQLALCHEMY_DATABASE_URI_SCHEME: str = Field(
-        description="db uri scheme",
+        description="Database URI scheme for SQLAlchemy connection.",
         default="postgresql",
     )
 
@@ -112,27 +114,27 @@ class DatabaseConfig:
         )
 
     SQLALCHEMY_POOL_SIZE: NonNegativeInt = Field(
-        description="pool size of SqlAlchemy",
+        description="Maximum number of database connections in the pool.",
         default=30,
     )
 
     SQLALCHEMY_MAX_OVERFLOW: NonNegativeInt = Field(
-        description="max overflows for SqlAlchemy",
+        description="Maximum number of connections that can be created beyond the pool_size.",
         default=10,
     )
 
     SQLALCHEMY_POOL_RECYCLE: NonNegativeInt = Field(
-        description="SqlAlchemy pool recycle",
+        description="Number of seconds after which a connection is automatically recycled.",
         default=3600,
     )
 
     SQLALCHEMY_POOL_PRE_PING: bool = Field(
-        description="whether to enable pool pre-ping in SqlAlchemy",
+        description="If True, enables connection pool pre-ping feature to check connections.",
         default=False,
     )
 
     SQLALCHEMY_ECHO: bool | str = Field(
-        description="whether to enable SqlAlchemy echo",
+        description="If True, SQLAlchemy will log all SQL statements.",
         default=False,
     )
 
@@ -150,27 +152,27 @@ class DatabaseConfig:
 
 class CeleryConfig(DatabaseConfig):
     CELERY_BACKEND: str = Field(
-        description="Celery backend, available values are `database`, `redis`",
+        description="Backend for Celery task results. Options: 'database', 'redis'.",
         default="database",
     )
 
     CELERY_BROKER_URL: Optional[str] = Field(
-        description="CELERY_BROKER_URL",
+        description="URL of the message broker for Celery tasks.",
         default=None,
     )
 
     CELERY_USE_SENTINEL: Optional[bool] = Field(
-        description="Whether to use Redis Sentinel mode",
+        description="Whether to use Redis Sentinel for high availability.",
         default=False,
     )
 
     CELERY_SENTINEL_MASTER_NAME: Optional[str] = Field(
-        description="Redis Sentinel master name",
+        description="Name of the Redis Sentinel master.",
         default=None,
     )
 
     CELERY_SENTINEL_SOCKET_TIMEOUT: Optional[PositiveFloat] = Field(
-        description="Redis Sentinel socket timeout",
+        description="Timeout for Redis Sentinel socket operations in seconds.",
         default=0.1,
     )
 
@@ -221,5 +223,6 @@ class MiddlewareConfig(
     TiDBVectorConfig,
     WeaviateConfig,
     ElasticsearchConfig,
+    BedrockConfig,
 ):
     pass

api/configs/middleware/cache/redis_config.py

@@ -6,65 +6,65 @@ from pydantic_settings import BaseSettings
 
 class RedisConfig(BaseSettings):
     """
-    Redis configs
+    Configuration settings for Redis connection
     """
 
     REDIS_HOST: str = Field(
-        description="Redis host",
+        description="Hostname or IP address of the Redis server",
         default="localhost",
     )
 
     REDIS_PORT: PositiveInt = Field(
-        description="Redis port",
+        description="Port number on which the Redis server is listening",
         default=6379,
     )
 
     REDIS_USERNAME: Optional[str] = Field(
-        description="Redis username",
+        description="Username for Redis authentication (if required)",
         default=None,
     )
 
     REDIS_PASSWORD: Optional[str] = Field(
-        description="Redis password",
+        description="Password for Redis authentication (if required)",
         default=None,
     )
 
     REDIS_DB: NonNegativeInt = Field(
-        description="Redis database id, default to 0",
+        description="Redis database number to use (0-15)",
         default=0,
     )
 
     REDIS_USE_SSL: bool = Field(
-        description="whether to use SSL for Redis connection",
+        description="Enable SSL/TLS for the Redis connection",
         default=False,
     )
 
     REDIS_USE_SENTINEL: Optional[bool] = Field(
-        description="Whether to use Redis Sentinel mode",
+        description="Enable Redis Sentinel mode for high availability",
         default=False,
     )
 
     REDIS_SENTINELS: Optional[str] = Field(
-        description="Redis Sentinel nodes",
+        description="Comma-separated list of Redis Sentinel nodes (host:port)",
         default=None,
     )
 
     REDIS_SENTINEL_SERVICE_NAME: Optional[str] = Field(
-        description="Redis Sentinel service name",
+        description="Name of the Redis Sentinel service to monitor",
         default=None,
     )
 
     REDIS_SENTINEL_USERNAME: Optional[str] = Field(
-        description="Redis Sentinel username",
+        description="Username for Redis Sentinel authentication (if required)",
         default=None,
     )
 
     REDIS_SENTINEL_PASSWORD: Optional[str] = Field(
-        description="Redis Sentinel password",
+        description="Password for Redis Sentinel authentication (if required)",
         default=None,
     )
 
     REDIS_SENTINEL_SOCKET_TIMEOUT: Optional[PositiveFloat] = Field(
-        description="Redis Sentinel socket timeout",
+        description="Socket timeout in seconds for Redis Sentinel connections",
         default=0.1,
     )

api/configs/middleware/external/bedrock_config.py

@@ -0,0 +1,20 @@
+from typing import Optional
+
+from pydantic import Field
+from pydantic_settings import BaseSettings
+
+
+class BedrockConfig(BaseSettings):
+    """
+    bedrock configs
+    """
+
+    AWS_SECRET_ACCESS_KEY: Optional[str] = Field(
+        description="AWS secret access key",
+        default=None,
+    )
+
+    AWS_ACCESS_KEY_ID: Optional[str] = Field(
+        description="AWS secret access id",
+        default=None,
+    )

api/configs/middleware/storage/aliyun_oss_storage_config.py

@@ -6,40 +6,40 @@ from pydantic_settings import BaseSettings
 
 class AliyunOSSStorageConfig(BaseSettings):
     """
-    Aliyun storage configs
+    Configuration settings for Aliyun Object Storage Service (OSS)
     """
 
     ALIYUN_OSS_BUCKET_NAME: Optional[str] = Field(
-        description="Aliyun OSS bucket name",
+        description="Name of the Aliyun OSS bucket to store and retrieve objects",
         default=None,
     )
 
     ALIYUN_OSS_ACCESS_KEY: Optional[str] = Field(
-        description="Aliyun OSS access key",
+        description="Access key ID for authenticating with Aliyun OSS",
         default=None,
     )
 
     ALIYUN_OSS_SECRET_KEY: Optional[str] = Field(
-        description="Aliyun OSS secret key",
+        description="Secret access key for authenticating with Aliyun OSS",
         default=None,
     )
 
     ALIYUN_OSS_ENDPOINT: Optional[str] = Field(
-        description="Aliyun OSS endpoint URL",
+        description="URL of the Aliyun OSS endpoint for your chosen region",
         default=None,
     )
 
     ALIYUN_OSS_REGION: Optional[str] = Field(
-        description="Aliyun OSS region",
+        description="Aliyun OSS region where your bucket is located (e.g., 'oss-cn-hangzhou')",
         default=None,
     )
 
     ALIYUN_OSS_AUTH_VERSION: Optional[str] = Field(
-        description="Aliyun OSS authentication version",
+        description="Version of the authentication protocol to use with Aliyun OSS (e.g., 'v4')",
         default=None,
     )
 
     ALIYUN_OSS_PATH: Optional[str] = Field(
-        description="Aliyun OSS path",
+        description="Base path within the bucket to store objects (e.g., 'my-app-data/')",
         default=None,
     )

api/configs/middleware/storage/amazon_s3_storage_config.py

@@ -6,40 +6,40 @@ from pydantic_settings import BaseSettings
 
 class S3StorageConfig(BaseSettings):
     """
-    S3 storage configs
+    Configuration settings for S3-compatible object storage
     """
 
     S3_ENDPOINT: Optional[str] = Field(
-        description="S3 storage endpoint",
+        description="URL of the S3-compatible storage endpoint (e.g., 'https://s3.amazonaws.com')",
         default=None,
     )
 
     S3_REGION: Optional[str] = Field(
-        description="S3 storage region",
+        description="Region where the S3 bucket is located (e.g., 'us-east-1')",
         default=None,
     )
 
     S3_BUCKET_NAME: Optional[str] = Field(
-        description="S3 storage bucket name",
+        description="Name of the S3 bucket to store and retrieve objects",
         default=None,
     )
 
     S3_ACCESS_KEY: Optional[str] = Field(
-        description="S3 storage access key",
+        description="Access key ID for authenticating with the S3 service",
         default=None,
     )
 
     S3_SECRET_KEY: Optional[str] = Field(
-        description="S3 storage secret key",
+        description="Secret access key for authenticating with the S3 service",
         default=None,
     )
 
     S3_ADDRESS_STYLE: str = Field(
-        description="S3 storage address style",
+        description="S3 addressing style: 'auto', 'path', or 'virtual'",
         default="auto",
     )
 
     S3_USE_AWS_MANAGED_IAM: bool = Field(
-        description="whether to use aws managed IAM for S3",
+        description="Use AWS managed IAM roles for authentication instead of access/secret keys",
         default=False,
     )

api/configs/middleware/storage/azure_blob_storage_config.py

@@ -6,25 +6,25 @@ from pydantic_settings import BaseSettings
 
 class AzureBlobStorageConfig(BaseSettings):
     """
-    Azure Blob storage configs
+    Configuration settings for Azure Blob Storage
     """
 
     AZURE_BLOB_ACCOUNT_NAME: Optional[str] = Field(
-        description="Azure Blob account name",
+        description="Name of the Azure Storage account (e.g., 'mystorageaccount')",
         default=None,
     )
 
     AZURE_BLOB_ACCOUNT_KEY: Optional[str] = Field(
-        description="Azure Blob account key",
+        description="Access key for authenticating with the Azure Storage account",
         default=None,
     )
 
     AZURE_BLOB_CONTAINER_NAME: Optional[str] = Field(
-        description="Azure Blob container name",
+        description="Name of the Azure Blob container to store and retrieve objects",
         default=None,
     )
 
     AZURE_BLOB_ACCOUNT_URL: Optional[str] = Field(
-        description="Azure Blob account URL",
+        description="URL of the Azure Blob storage endpoint (e.g., 'https://mystorageaccount.blob.core.windows.net')",
         default=None,
     )

api/configs/middleware/storage/google_cloud_storage_config.py

@@ -6,15 +6,15 @@ from pydantic_settings import BaseSettings
 
 class GoogleCloudStorageConfig(BaseSettings):
     """
-    Google Cloud storage configs
+    Configuration settings for Google Cloud Storage
     """
 
     GOOGLE_STORAGE_BUCKET_NAME: Optional[str] = Field(
-        description="Google Cloud storage bucket name",
+        description="Name of the Google Cloud Storage bucket to store and retrieve objects (e.g., 'my-gcs-bucket')",
         default=None,
     )
 
     GOOGLE_STORAGE_SERVICE_ACCOUNT_JSON_BASE64: Optional[str] = Field(
-        description="Google Cloud storage service account json base64",
+        description="Base64-encoded JSON key file for Google Cloud service account authentication",
         default=None,
     )

api/configs/middleware/storage/huawei_obs_storage_config.py

@@ -5,25 +5,25 @@ from pydantic import BaseModel, Field
 
 class HuaweiCloudOBSStorageConfig(BaseModel):
     """
-    Huawei Cloud OBS storage configs
+    Configuration settings for Huawei Cloud Object Storage Service (OBS)
     """
 
     HUAWEI_OBS_BUCKET_NAME: Optional[str] = Field(
-        description="Huawei Cloud OBS bucket name",
+        description="Name of the Huawei Cloud OBS bucket to store and retrieve objects (e.g., 'my-obs-bucket')",
         default=None,
     )
 
     HUAWEI_OBS_ACCESS_KEY: Optional[str] = Field(
-        description="Huawei Cloud OBS Access key",
+        description="Access Key ID for authenticating with Huawei Cloud OBS",
         default=None,
     )
 
     HUAWEI_OBS_SECRET_KEY: Optional[str] = Field(
-        description="Huawei Cloud OBS Secret key",
+        description="Secret Access Key for authenticating with Huawei Cloud OBS",
         default=None,
     )
 
     HUAWEI_OBS_SERVER: Optional[str] = Field(
-        description="Huawei Cloud OBS server URL",
+        description="Endpoint URL for Huawei Cloud OBS (e.g., 'https://obs.cn-north-4.myhuaweicloud.com')",
         default=None,
     )

api/configs/middleware/storage/oci_storage_config.py

@@ -6,30 +6,30 @@ from pydantic_settings import BaseSettings
 
 class OCIStorageConfig(BaseSettings):
     """
-    OCI storage configs
+    Configuration settings for Oracle Cloud Infrastructure (OCI) Object Storage
     """
 
     OCI_ENDPOINT: Optional[str] = Field(
-        description="OCI storage endpoint",
+        description="URL of the OCI Object Storage endpoint (e.g., 'https://objectstorage.us-phoenix-1.oraclecloud.com')",
         default=None,
     )
 
     OCI_REGION: Optional[str] = Field(
-        description="OCI storage region",
+        description="OCI region where the bucket is located (e.g., 'us-phoenix-1')",
         default=None,
     )
 
     OCI_BUCKET_NAME: Optional[str] = Field(
-        description="OCI storage bucket name",
+        description="Name of the OCI Object Storage bucket to store and retrieve objects (e.g., 'my-oci-bucket')",
         default=None,
     )
 
     OCI_ACCESS_KEY: Optional[str] = Field(
-        description="OCI storage access key",
+        description="Access key (also known as API key) for authenticating with OCI Object Storage",
         default=None,
     )
 
     OCI_SECRET_KEY: Optional[str] = Field(
-        description="OCI storage secret key",
+        description="Secret key associated with the access key for authenticating with OCI Object Storage",
         default=None,
     )

api/configs/middleware/storage/tencent_cos_storage_config.py

@@ -6,30 +6,30 @@ from pydantic_settings import BaseSettings
 
 class TencentCloudCOSStorageConfig(BaseSettings):
     """
-    Tencent Cloud COS storage configs
+    Configuration settings for Tencent Cloud Object Storage (COS)
     """
 
     TENCENT_COS_BUCKET_NAME: Optional[str] = Field(
-        description="Tencent Cloud COS bucket name",
+        description="Name of the Tencent Cloud COS bucket to store and retrieve objects",
         default=None,
     )
 
     TENCENT_COS_REGION: Optional[str] = Field(
-        description="Tencent Cloud COS region",
+        description="Tencent Cloud region where the COS bucket is located (e.g., 'ap-guangzhou')",
         default=None,
     )
 
     TENCENT_COS_SECRET_ID: Optional[str] = Field(
-        description="Tencent Cloud COS secret id",
+        description="SecretId for authenticating with Tencent Cloud COS (part of API credentials)",
         default=None,
     )
 
     TENCENT_COS_SECRET_KEY: Optional[str] = Field(
-        description="Tencent Cloud COS secret key",
+        description="SecretKey for authenticating with Tencent Cloud COS (part of API credentials)",
         default=None,
     )
 
     TENCENT_COS_SCHEME: Optional[str] = Field(
-        description="Tencent Cloud COS scheme",
+        description="Protocol scheme for COS requests: 'https' (recommended) or 'http'",
         default=None,
     )

api/configs/middleware/storage/volcengine_tos_storage_config.py

@@ -5,30 +5,30 @@ from pydantic import BaseModel, Field
 
 class VolcengineTOSStorageConfig(BaseModel):
     """
-    Volcengine tos storage configs
+    Configuration settings for Volcengine Tinder Object Storage (TOS)
     """
 
     VOLCENGINE_TOS_BUCKET_NAME: Optional[str] = Field(
-        description="Volcengine TOS Bucket Name",
+        description="Name of the Volcengine TOS bucket to store and retrieve objects (e.g., 'my-tos-bucket')",
         default=None,
     )
 
     VOLCENGINE_TOS_ACCESS_KEY: Optional[str] = Field(
-        description="Volcengine TOS Access Key",
+        description="Access Key ID for authenticating with Volcengine TOS",
         default=None,
     )
 
     VOLCENGINE_TOS_SECRET_KEY: Optional[str] = Field(
-        description="Volcengine TOS Secret Key",
+        description="Secret Access Key for authenticating with Volcengine TOS",
         default=None,
     )
 
     VOLCENGINE_TOS_ENDPOINT: Optional[str] = Field(
-        description="Volcengine TOS Endpoint URL",
+        description="URL of the Volcengine TOS endpoint (e.g., 'https://tos-cn-beijing.volces.com')",
         default=None,
     )
 
     VOLCENGINE_TOS_REGION: Optional[str] = Field(
-        description="Volcengine TOS Region",
+        description="Volcengine region where the TOS bucket is located (e.g., 'cn-beijing')",
         default=None,
     )

api/configs/middleware/vdb/analyticdb_config.py

@@ -5,33 +5,38 @@ from pydantic import BaseModel, Field
 
 class AnalyticdbConfig(BaseModel):
     """
-    Configuration for connecting to AnalyticDB.
+    Configuration for connecting to Alibaba Cloud AnalyticDB for PostgreSQL.
     Refer to the following documentation for details on obtaining credentials:
     https://www.alibabacloud.com/help/en/analyticdb-for-postgresql/getting-started/create-an-instance-instances-with-vector-engine-optimization-enabled
     """
 
     ANALYTICDB_KEY_ID: Optional[str] = Field(
-        default=None, description="The Access Key ID provided by Alibaba Cloud for authentication."
+        default=None, description="The Access Key ID provided by Alibaba Cloud for API authentication."
     )
     ANALYTICDB_KEY_SECRET: Optional[str] = Field(
-        default=None, description="The Secret Access Key corresponding to the Access Key ID for secure access."
+        default=None, description="The Secret Access Key corresponding to the Access Key ID for secure API access."
     )
     ANALYTICDB_REGION_ID: Optional[str] = Field(
-        default=None, description="The region where the AnalyticDB instance is deployed (e.g., 'cn-hangzhou')."
+        default=None,
+        description="The region where the AnalyticDB instance is deployed (e.g., 'cn-hangzhou', 'ap-southeast-1').",
     )
     ANALYTICDB_INSTANCE_ID: Optional[str] = Field(
         default=None,
-        description="The unique identifier of the AnalyticDB instance you want to connect to (e.g., 'gp-ab123456')..",
+        description="The unique identifier of the AnalyticDB instance you want to connect to.",
     )
     ANALYTICDB_ACCOUNT: Optional[str] = Field(
-        default=None, description="The account name used to log in to the AnalyticDB instance."
+        default=None,
+        description="The account name used to log in to the AnalyticDB instance"
+        " (usually the initial account created with the instance).",
     )
     ANALYTICDB_PASSWORD: Optional[str] = Field(
-        default=None, description="The password associated with the AnalyticDB account for authentication."
+        default=None, description="The password associated with the AnalyticDB account for database authentication."
     )
     ANALYTICDB_NAMESPACE: Optional[str] = Field(
-        default=None, description="The namespace within AnalyticDB for schema isolation."
+        default=None, description="The namespace within AnalyticDB for schema isolation (if using namespace feature)."
     )
     ANALYTICDB_NAMESPACE_PASSWORD: Optional[str] = Field(
-        default=None, description="The password for accessing the specified namespace within the AnalyticDB instance."
+        default=None,
+        description="The password for accessing the specified namespace within the AnalyticDB instance"
+        " (if namespace feature is enabled).",
     )

api/configs/middleware/vdb/chroma_config.py

@@ -6,35 +6,35 @@ from pydantic_settings import BaseSettings
 
 class ChromaConfig(BaseSettings):
     """
-    Chroma configs
+    Configuration settings for Chroma vector database
     """
 
     CHROMA_HOST: Optional[str] = Field(
-        description="Chroma host",
+        description="Hostname or IP address of the Chroma server (e.g., 'localhost' or '192.168.1.100')",
         default=None,
     )
 
     CHROMA_PORT: PositiveInt = Field(
-        description="Chroma port",
+        description="Port number on which the Chroma server is listening (default is 8000)",
         default=8000,
     )
 
     CHROMA_TENANT: Optional[str] = Field(
-        description="Chroma database",
+        description="Tenant identifier for multi-tenancy support in Chroma",
         default=None,
     )
 
     CHROMA_DATABASE: Optional[str] = Field(
-        description="Chroma database",
+        description="Name of the Chroma database to connect to",
         default=None,
     )
 
     CHROMA_AUTH_PROVIDER: Optional[str] = Field(
-        description="Chroma authentication provider",
+        description="Authentication provider for Chroma (e.g., 'basic', 'token', or a custom provider)",
         default=None,
     )
 
     CHROMA_AUTH_CREDENTIALS: Optional[str] = Field(
-        description="Chroma authentication credentials",
+        description="Authentication credentials for Chroma (format depends on the auth provider)",
         default=None,
     )

api/configs/middleware/vdb/elasticsearch_config.py

@@ -6,25 +6,25 @@ from pydantic_settings import BaseSettings
 
 class ElasticsearchConfig(BaseSettings):
     """
-    Elasticsearch configs
+    Configuration settings for Elasticsearch
     """
 
     ELASTICSEARCH_HOST: Optional[str] = Field(
-        description="Elasticsearch host",
+        description="Hostname or IP address of the Elasticsearch server (e.g., 'localhost' or '192.168.1.100')",
         default="127.0.0.1",
     )
 
     ELASTICSEARCH_PORT: PositiveInt = Field(
-        description="Elasticsearch port",
+        description="Port number on which the Elasticsearch server is listening (default is 9200)",
         default=9200,
     )
 
     ELASTICSEARCH_USERNAME: Optional[str] = Field(
-        description="Elasticsearch username",
+        description="Username for authenticating with Elasticsearch (default is 'elastic')",
         default="elastic",
     )
 
     ELASTICSEARCH_PASSWORD: Optional[str] = Field(
-        description="Elasticsearch password",
+        description="Password for authenticating with Elasticsearch (default is 'elastic')",
         default="elastic",
     )

api/configs/middleware/vdb/milvus_config.py

@@ -6,30 +6,30 @@ from pydantic_settings import BaseSettings
 
 class MilvusConfig(BaseSettings):
     """
-    Milvus configs
+    Configuration settings for Milvus vector database
     """
 
     MILVUS_URI: Optional[str] = Field(
-        description="Milvus uri",
+        description="URI for connecting to the Milvus server (e.g., 'http://localhost:19530' or 'https://milvus-instance.example.com:19530')",
         default="http://127.0.0.1:19530",
     )
 
     MILVUS_TOKEN: Optional[str] = Field(
-        description="Milvus token",
+        description="Authentication token for Milvus, if token-based authentication is enabled",
         default=None,
     )
 
     MILVUS_USER: Optional[str] = Field(
-        description="Milvus user",
+        description="Username for authenticating with Milvus, if username/password authentication is enabled",
         default=None,
     )
 
     MILVUS_PASSWORD: Optional[str] = Field(
-        description="Milvus password",
+        description="Password for authenticating with Milvus, if username/password authentication is enabled",
         default=None,
     )
 
     MILVUS_DATABASE: str = Field(
-        description="Milvus database, default to `default`",
+        description="Name of the Milvus database to connect to (default is 'default')",
         default="default",
     )

api/configs/middleware/vdb/myscale_config.py

@@ -3,35 +3,35 @@ from pydantic import BaseModel, Field, PositiveInt
 
 class MyScaleConfig(BaseModel):
     """
-    MyScale configs
+    Configuration settings for MyScale vector database
     """
 
     MYSCALE_HOST: str = Field(
-        description="MyScale host",
+        description="Hostname or IP address of the MyScale server (e.g., 'localhost' or 'myscale.example.com')",
         default="localhost",
     )
 
     MYSCALE_PORT: PositiveInt = Field(
-        description="MyScale port",
+        description="Port number on which the MyScale server is listening (default is 8123)",
         default=8123,
     )
 
     MYSCALE_USER: str = Field(
-        description="MyScale user",
+        description="Username for authenticating with MyScale (default is 'default')",
         default="default",
     )
 
     MYSCALE_PASSWORD: str = Field(
-        description="MyScale password",
+        description="Password for authenticating with MyScale (default is an empty string)",
         default="",
     )
 
     MYSCALE_DATABASE: str = Field(
-        description="MyScale database name",
+        description="Name of the MyScale database to connect to (default is 'default')",
         default="default",
     )
 
     MYSCALE_FTS_PARAMS: str = Field(
-        description="MyScale fts index parameters",
+        description="Additional parameters for MyScale Full Text Search index)",
         default="",
     )

api/configs/middleware/vdb/opensearch_config.py

@@ -6,30 +6,30 @@ from pydantic_settings import BaseSettings
 
 class OpenSearchConfig(BaseSettings):
     """
-    OpenSearch configs
+    Configuration settings for OpenSearch
     """
 
     OPENSEARCH_HOST: Optional[str] = Field(
-        description="OpenSearch host",
+        description="Hostname or IP address of the OpenSearch server (e.g., 'localhost' or 'opensearch.example.com')",
         default=None,
     )
 
     OPENSEARCH_PORT: PositiveInt = Field(
-        description="OpenSearch port",
+        description="Port number on which the OpenSearch server is listening (default is 9200)",
         default=9200,
     )
 
     OPENSEARCH_USER: Optional[str] = Field(
-        description="OpenSearch user",
+        description="Username for authenticating with OpenSearch",
         default=None,
     )
 
     OPENSEARCH_PASSWORD: Optional[str] = Field(
-        description="OpenSearch password",
+        description="Password for authenticating with OpenSearch",
         default=None,
     )
 
     OPENSEARCH_SECURE: bool = Field(
-        description="whether to use SSL connection for OpenSearch",
+        description="Whether to use SSL/TLS encrypted connection for OpenSearch (True for HTTPS, False for HTTP)",
         default=False,
     )

api/configs/middleware/vdb/oracle_config.py

@@ -6,30 +6,30 @@ from pydantic_settings import BaseSettings
 
 class OracleConfig(BaseSettings):
     """
-    ORACLE configs
+    Configuration settings for Oracle database
     """
 
     ORACLE_HOST: Optional[str] = Field(
-        description="ORACLE host",
+        description="Hostname or IP address of the Oracle database server (e.g., 'localhost' or 'oracle.example.com')",
         default=None,
     )
 
     ORACLE_PORT: Optional[PositiveInt] = Field(
-        description="ORACLE port",
+        description="Port number on which the Oracle database server is listening (default is 1521)",
         default=1521,
     )
 
     ORACLE_USER: Optional[str] = Field(
-        description="ORACLE user",
+        description="Username for authenticating with the Oracle database",
         default=None,
     )
 
     ORACLE_PASSWORD: Optional[str] = Field(
-        description="ORACLE password",
+        description="Password for authenticating with the Oracle database",
         default=None,
     )
 
     ORACLE_DATABASE: Optional[str] = Field(
-        description="ORACLE database",
+        description="Name of the Oracle database or service to connect to (e.g., 'ORCL' or 'pdborcl')",
         default=None,
     )

api/configs/middleware/vdb/pgvector_config.py

@@ -6,30 +6,40 @@ from pydantic_settings import BaseSettings
 
 class PGVectorConfig(BaseSettings):
     """
-    PGVector configs
+    Configuration settings for PGVector (PostgreSQL with vector extension)
     """
 
     PGVECTOR_HOST: Optional[str] = Field(
-        description="PGVector host",
+        description="Hostname or IP address of the PostgreSQL server with PGVector extension (e.g., 'localhost')",
         default=None,
     )
 
     PGVECTOR_PORT: Optional[PositiveInt] = Field(
-        description="PGVector port",
+        description="Port number on which the PostgreSQL server is listening (default is 5433)",
         default=5433,
     )
 
     PGVECTOR_USER: Optional[str] = Field(
-        description="PGVector user",
+        description="Username for authenticating with the PostgreSQL database",
         default=None,
     )
 
     PGVECTOR_PASSWORD: Optional[str] = Field(
-        description="PGVector password",
+        description="Password for authenticating with the PostgreSQL database",
         default=None,
     )
 
     PGVECTOR_DATABASE: Optional[str] = Field(
-        description="PGVector database",
+        description="Name of the PostgreSQL database to connect to",
         default=None,
     )
+
+    PGVECTOR_MIN_CONNECTION: PositiveInt = Field(
+        description="Min connection of the PostgreSQL database",
+        default=1,
+    )
+
+    PGVECTOR_MAX_CONNECTION: PositiveInt = Field(
+        description="Max connection of the PostgreSQL database",
+        default=5,
+    )

api/configs/middleware/vdb/pgvectors_config.py

@@ -6,30 +6,30 @@ from pydantic_settings import BaseSettings
 
 class PGVectoRSConfig(BaseSettings):
     """
-    PGVectoRS configs
+    Configuration settings for PGVecto.RS (Rust-based vector extension for PostgreSQL)
     """
 
     PGVECTO_RS_HOST: Optional[str] = Field(
-        description="PGVectoRS host",
+        description="Hostname or IP address of the PostgreSQL server with PGVecto.RS extension (e.g., 'localhost')",
         default=None,
     )
 
     PGVECTO_RS_PORT: Optional[PositiveInt] = Field(
-        description="PGVectoRS port",
+        description="Port number on which the PostgreSQL server with PGVecto.RS is listening (default is 5431)",
         default=5431,
     )
 
     PGVECTO_RS_USER: Optional[str] = Field(
-        description="PGVectoRS user",
+        description="Username for authenticating with the PostgreSQL database using PGVecto.RS",
         default=None,
     )
 
     PGVECTO_RS_PASSWORD: Optional[str] = Field(
-        description="PGVectoRS password",
+        description="Password for authenticating with the PostgreSQL database using PGVecto.RS",
         default=None,
     )
 
     PGVECTO_RS_DATABASE: Optional[str] = Field(
-        description="PGVectoRS database",
+        description="Name of the PostgreSQL database with PGVecto.RS extension to connect to",
         default=None,
     )

api/configs/middleware/vdb/qdrant_config.py

@@ -6,30 +6,30 @@ from pydantic_settings import BaseSettings
 
 class QdrantConfig(BaseSettings):
     """
-    Qdrant configs
+    Configuration settings for Qdrant vector database
     """
 
     QDRANT_URL: Optional[str] = Field(
-        description="Qdrant url",
+        description="URL of the Qdrant server (e.g., 'http://localhost:6333' or 'https://qdrant.example.com')",
         default=None,
     )
 
     QDRANT_API_KEY: Optional[str] = Field(
-        description="Qdrant api key",
+        description="API key for authenticating with the Qdrant server",
         default=None,
     )
 
     QDRANT_CLIENT_TIMEOUT: NonNegativeInt = Field(
-        description="Qdrant client timeout in seconds",
+        description="Timeout in seconds for Qdrant client operations (default is 20 seconds)",
         default=20,
     )
 
     QDRANT_GRPC_ENABLED: bool = Field(
-        description="whether enable grpc support for Qdrant connection",
+        description="Whether to enable gRPC support for Qdrant connection (True for gRPC, False for HTTP)",
         default=False,
     )
 
     QDRANT_GRPC_PORT: PositiveInt = Field(
-        description="Qdrant grpc port",
+        description="Port number for gRPC connection to Qdrant server (default is 6334)",
         default=6334,
     )

api/configs/middleware/vdb/relyt_config.py

@@ -6,30 +6,30 @@ from pydantic_settings import BaseSettings
 
 class RelytConfig(BaseSettings):
     """
-    Relyt configs
+    Configuration settings for Relyt database
     """
 
     RELYT_HOST: Optional[str] = Field(
-        description="Relyt host",
+        description="Hostname or IP address of the Relyt server (e.g., 'localhost' or 'relyt.example.com')",
         default=None,
     )
 
     RELYT_PORT: PositiveInt = Field(
-        description="Relyt port",
+        description="Port number on which the Relyt server is listening (default is 9200)",
         default=9200,
     )
 
     RELYT_USER: Optional[str] = Field(
-        description="Relyt user",
+        description="Username for authenticating with the Relyt database",
         default=None,
     )
 
     RELYT_PASSWORD: Optional[str] = Field(
-        description="Relyt password",
+        description="Password for authenticating with the Relyt database",
         default=None,
     )
 
     RELYT_DATABASE: Optional[str] = Field(
-        description="Relyt database",
+        description="Name of the Relyt database to connect to (default is 'default')",
         default="default",
     )

api/configs/middleware/vdb/tencent_vector_config.py

@@ -6,45 +6,45 @@ from pydantic_settings import BaseSettings
 
 class TencentVectorDBConfig(BaseSettings):
     """
-    Tencent Vector configs
+    Configuration settings for Tencent Vector Database
     """
 
     TENCENT_VECTOR_DB_URL: Optional[str] = Field(
-        description="Tencent Vector URL",
+        description="URL of the Tencent Vector Database service (e.g., 'https://vectordb.tencentcloudapi.com')",
         default=None,
     )
 
     TENCENT_VECTOR_DB_API_KEY: Optional[str] = Field(
-        description="Tencent Vector API key",
+        description="API key for authenticating with the Tencent Vector Database service",
         default=None,
     )
 
     TENCENT_VECTOR_DB_TIMEOUT: PositiveInt = Field(
-        description="Tencent Vector timeout in seconds",
+        description="Timeout in seconds for Tencent Vector Database operations (default is 30 seconds)",
         default=30,
     )
 
     TENCENT_VECTOR_DB_USERNAME: Optional[str] = Field(
-        description="Tencent Vector username",
+        description="Username for authenticating with the Tencent Vector Database (if required)",
         default=None,
     )
 
     TENCENT_VECTOR_DB_PASSWORD: Optional[str] = Field(
-        description="Tencent Vector password",
+        description="Password for authenticating with the Tencent Vector Database (if required)",
         default=None,
     )
 
     TENCENT_VECTOR_DB_SHARD: PositiveInt = Field(
-        description="Tencent Vector sharding number",
+        description="Number of shards for the Tencent Vector Database (default is 1)",
         default=1,
     )
 
     TENCENT_VECTOR_DB_REPLICAS: NonNegativeInt = Field(
-        description="Tencent Vector replicas",
+        description="Number of replicas for the Tencent Vector Database (default is 2)",
         default=2,
     )
 
     TENCENT_VECTOR_DB_DATABASE: Optional[str] = Field(
-        description="Tencent Vector Database",
+        description="Name of the specific Tencent Vector Database to connect to",
         default=None,
     )

api/configs/middleware/vdb/tidb_vector_config.py

@@ -6,30 +6,30 @@ from pydantic_settings import BaseSettings
 
 class TiDBVectorConfig(BaseSettings):
     """
-    TiDB Vector configs
+    Configuration settings for TiDB Vector database
     """
 
     TIDB_VECTOR_HOST: Optional[str] = Field(
-        description="TiDB Vector host",
+        description="Hostname or IP address of the TiDB Vector server (e.g., 'localhost' or 'tidb.example.com')",
         default=None,
     )
 
     TIDB_VECTOR_PORT: Optional[PositiveInt] = Field(
-        description="TiDB Vector port",
+        description="Port number on which the TiDB Vector server is listening (default is 4000)",
         default=4000,
     )
 
     TIDB_VECTOR_USER: Optional[str] = Field(
-        description="TiDB Vector user",
+        description="Username for authenticating with the TiDB Vector database",
         default=None,
     )
 
     TIDB_VECTOR_PASSWORD: Optional[str] = Field(
-        description="TiDB Vector password",
+        description="Password for authenticating with the TiDB Vector database",
         default=None,
     )
 
     TIDB_VECTOR_DATABASE: Optional[str] = Field(
-        description="TiDB Vector database",
+        description="Name of the TiDB Vector database to connect to",
         default=None,
     )

api/configs/middleware/vdb/weaviate_config.py

@@ -6,25 +6,25 @@ from pydantic_settings import BaseSettings
 
 class WeaviateConfig(BaseSettings):
     """
-    Weaviate configs
+    Configuration settings for Weaviate vector database
     """
 
     WEAVIATE_ENDPOINT: Optional[str] = Field(
-        description="Weaviate endpoint URL",
+        description="URL of the Weaviate server (e.g., 'http://localhost:8080' or 'https://weaviate.example.com')",
         default=None,
     )
 
     WEAVIATE_API_KEY: Optional[str] = Field(
-        description="Weaviate API key",
+        description="API key for authenticating with the Weaviate server",
         default=None,
     )
 
     WEAVIATE_GRPC_ENABLED: bool = Field(
-        description="whether to enable gRPC for Weaviate connection",
+        description="Whether to enable gRPC for Weaviate connection (True for gRPC, False for HTTP)",
         default=True,
     )
 
     WEAVIATE_BATCH_SIZE: PositiveInt = Field(
-        description="Weaviate batch size",
+        description="Number of objects to be processed in a single batch operation (default is 100)",
         default=100,
     )

api/configs/packaging/__init__.py

@@ -9,7 +9,7 @@ class PackagingInfo(BaseSettings):
 
     CURRENT_VERSION: str = Field(
         description="Dify version",
-        default="0.8.2",
+        default="0.8.3",
     )
 
     COMMIT_SHA: str = Field(

api/constants/__init__.py

@@ -1 +1,2 @@
 HIDDEN_VALUE = "[__HIDDEN__]"
+UUID_NIL = "00000000-0000-0000-0000-000000000000"

api/controllers/console/__init__.py

@@ -37,7 +37,17 @@ from .auth import activate, data_source_bearer_auth, data_source_oauth, forgot_p
 from .billing import billing
 
 # Import datasets controllers
-from .datasets import data_source, datasets, datasets_document, datasets_segments, file, hit_testing, website
+from .datasets import (
+    data_source,
+    datasets,
+    datasets_document,
+    datasets_segments,
+    external,
+    file,
+    hit_testing,
+    test_external,
+    website,
+)
 
 # Import explore controllers
 from .explore import (

api/controllers/console/app/audio.py

@@ -94,7 +94,7 @@ class ChatMessageTextApi(Resource):
             message_id = args.get("message_id", None)
             text = args.get("text", None)
             if (
-                app_model.mode in [AppMode.ADVANCED_CHAT.value, AppMode.WORKFLOW.value]
+                app_model.mode in {AppMode.ADVANCED_CHAT.value, AppMode.WORKFLOW.value}
                 and app_model.workflow
                 and app_model.workflow.features_dict
             ):

api/controllers/console/app/completion.py

@@ -109,6 +109,7 @@ class ChatMessageApi(Resource):
         parser.add_argument("files", type=list, required=False, location="json")
         parser.add_argument("model_config", type=dict, required=True, location="json")
         parser.add_argument("conversation_id", type=uuid_value, location="json")
+        parser.add_argument("parent_message_id", type=uuid_value, required=False, location="json")
         parser.add_argument("response_mode", type=str, choices=["blocking", "streaming"], location="json")
         parser.add_argument("retriever_from", type=str, required=False, default="dev", location="json")
         args = parser.parse_args()

api/controllers/console/app/message.py

@@ -105,8 +105,6 @@ class ChatMessageListApi(Resource):
             if rest_count > 0:
                 has_more = True
 
-        history_messages = list(reversed(history_messages))
-
         return InfiniteScrollPagination(data=history_messages, limit=args["limit"], has_more=has_more)
 
 

api/controllers/console/app/workflow.py

@@ -166,6 +166,8 @@ class AdvancedChatDraftWorkflowRunApi(Resource):
         parser.add_argument("query", type=str, required=True, location="json", default="")
         parser.add_argument("files", type=list, location="json")
         parser.add_argument("conversation_id", type=uuid_value, location="json")
+        parser.add_argument("parent_message_id", type=uuid_value, required=False, location="json")
+
         args = parser.parse_args()
 
         try:
@@ -465,6 +467,6 @@ api.add_resource(
 api.add_resource(PublishedWorkflowApi, "/apps/<uuid:app_id>/workflows/publish")
 api.add_resource(DefaultBlockConfigsApi, "/apps/<uuid:app_id>/workflows/default-workflow-block-configs")
 api.add_resource(
-    DefaultBlockConfigApi, "/apps/<uuid:app_id>/workflows/default-workflow-block-configs" "/<string:block_type>"
+    DefaultBlockConfigApi, "/apps/<uuid:app_id>/workflows/default-workflow-block-configs/<string:block_type>"
 )
 api.add_resource(ConvertToWorkflowApi, "/apps/<uuid:app_id>/convert-to-workflow")

api/controllers/console/auth/oauth.py

@@ -71,7 +71,7 @@ class OAuthCallback(Resource):
 
         account = _generate_account(provider, user_info)
         # Check account status
-        if account.status == AccountStatus.BANNED.value or account.status == AccountStatus.CLOSED.value:
+        if account.status in {AccountStatus.BANNED.value, AccountStatus.CLOSED.value}:
             return {"error": "Account is banned or closed."}, 403
 
         if account.status == AccountStatus.PENDING.value:

api/controllers/console/datasets/datasets.py

@@ -49,7 +49,7 @@ class DatasetListApi(Resource):
         page = request.args.get("page", default=1, type=int)
         limit = request.args.get("limit", default=20, type=int)
         ids = request.args.getlist("ids")
-        provider = request.args.get("provider", default="vendor")
+        # provider = request.args.get("provider", default="vendor")
         search = request.args.get("keyword", default=None, type=str)
         tag_ids = request.args.getlist("tag_ids")
 
@@ -57,7 +57,7 @@ class DatasetListApi(Resource):
             datasets, total = DatasetService.get_datasets_by_ids(ids, current_user.current_tenant_id)
         else:
             datasets, total = DatasetService.get_datasets(
-                page, limit, provider, current_user.current_tenant_id, current_user, search, tag_ids
+                page, limit, current_user.current_tenant_id, current_user, search, tag_ids
             )
 
         # check embedding setting
@@ -110,6 +110,26 @@ class DatasetListApi(Resource):
             nullable=True,
             help="Invalid indexing technique.",
         )
+        parser.add_argument(
+            "external_knowledge_api_id",
+            type=str,
+            nullable=True,
+            required=False,
+        )
+        parser.add_argument(
+            "provider",
+            type=str,
+            nullable=True,
+            choices=Dataset.PROVIDER_LIST,
+            required=False,
+            default="vendor",
+        )
+        parser.add_argument(
+            "external_knowledge_id",
+            type=str,
+            nullable=True,
+            required=False,
+        )
         args = parser.parse_args()
 
         # The role of the current user in the ta table must be admin, owner, or editor, or dataset_operator
@@ -123,6 +143,9 @@ class DatasetListApi(Resource):
                 indexing_technique=args["indexing_technique"],
                 account=current_user,
                 permission=DatasetPermissionEnum.ONLY_ME,
+                provider=args["provider"],
+                external_knowledge_api_id=args["external_knowledge_api_id"],
+                external_knowledge_id=args["external_knowledge_id"],
             )
         except services.errors.dataset.DatasetNameDuplicateError:
             raise DatasetNameDuplicateError()
@@ -211,6 +234,33 @@ class DatasetApi(Resource):
         )
         parser.add_argument("retrieval_model", type=dict, location="json", help="Invalid retrieval model.")
         parser.add_argument("partial_member_list", type=list, location="json", help="Invalid parent user list.")
+
+        parser.add_argument(
+            "external_retrieval_model",
+            type=dict,
+            required=False,
+            nullable=True,
+            location="json",
+            help="Invalid external retrieval model.",
+        )
+
+        parser.add_argument(
+            "external_knowledge_id",
+            type=str,
+            required=False,
+            nullable=True,
+            location="json",
+            help="Invalid external knowledge id.",
+        )
+
+        parser.add_argument(
+            "external_knowledge_api_id",
+            type=str,
+            required=False,
+            nullable=True,
+            location="json",
+            help="Invalid external knowledge api id.",
+        )
         args = parser.parse_args()
         data = request.get_json()
 
@@ -399,7 +449,7 @@ class DatasetIndexingEstimateApi(Resource):
             )
         except LLMBadRequestError:
             raise ProviderNotInitializeError(
-                "No Embedding Model available. Please configure a valid provider " "in the Settings -> Model Provider."
+                "No Embedding Model available. Please configure a valid provider in the Settings -> Model Provider."
             )
         except ProviderTokenNotInitError as ex:
             raise ProviderNotInitializeError(ex.description)

api/controllers/console/datasets/datasets_document.py

@@ -354,7 +354,7 @@ class DocumentIndexingEstimateApi(DocumentResource):
         document_id = str(document_id)
         document = self.get_document(dataset_id, document_id)
 
-        if document.indexing_status in ["completed", "error"]:
+        if document.indexing_status in {"completed", "error"}:
             raise DocumentAlreadyFinishedError()
 
         data_process_rule = document.dataset_process_rule
@@ -421,7 +421,7 @@ class DocumentBatchIndexingEstimateApi(DocumentResource):
         info_list = []
         extract_settings = []
         for document in documents:
-            if document.indexing_status in ["completed", "error"]:
+            if document.indexing_status in {"completed", "error"}:
                 raise DocumentAlreadyFinishedError()
             data_source_info = document.data_source_info_dict
             # format document files info
@@ -665,7 +665,7 @@ class DocumentProcessingApi(DocumentResource):
             db.session.commit()
 
         elif action == "resume":
-            if document.indexing_status not in ["paused", "error"]:
+            if document.indexing_status not in {"paused", "error"}:
                 raise InvalidActionError("Document not in paused or error state.")
 
             document.paused_by = None

api/controllers/console/datasets/external.py

@@ -0,0 +1,282 @@
+from flask import request
+from flask_login import current_user
+from flask_restful import Resource, marshal, reqparse
+from werkzeug.exceptions import Forbidden, InternalServerError, NotFound
+
+import services
+from controllers.console import api
+from controllers.console.app.error import ProviderNotInitializeError
+from controllers.console.datasets.error import DatasetNameDuplicateError
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required
+from fields.dataset_fields import dataset_detail_fields
+from libs.login import login_required
+from services.dataset_service import DatasetService
+from services.external_knowledge_service import ExternalDatasetService
+from services.hit_testing_service import HitTestingService
+
+
+def _validate_name(name):
+    if not name or len(name) < 1 or len(name) > 100:
+        raise ValueError("Name must be between 1 to 100 characters.")
+    return name
+
+
+def _validate_description_length(description):
+    if description and len(description) > 400:
+        raise ValueError("Description cannot exceed 400 characters.")
+    return description
+
+
+class ExternalApiTemplateListApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def get(self):
+        page = request.args.get("page", default=1, type=int)
+        limit = request.args.get("limit", default=20, type=int)
+        search = request.args.get("keyword", default=None, type=str)
+
+        external_knowledge_apis, total = ExternalDatasetService.get_external_knowledge_apis(
+            page, limit, current_user.current_tenant_id, search
+        )
+        response = {
+            "data": [item.to_dict() for item in external_knowledge_apis],
+            "has_more": len(external_knowledge_apis) == limit,
+            "limit": limit,
+            "total": total,
+            "page": page,
+        }
+        return response, 200
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def post(self):
+        parser = reqparse.RequestParser()
+        parser.add_argument(
+            "name",
+            nullable=False,
+            required=True,
+            help="Name is required. Name must be between 1 to 100 characters.",
+            type=_validate_name,
+        )
+        parser.add_argument(
+            "settings",
+            type=dict,
+            location="json",
+            nullable=False,
+            required=True,
+        )
+        args = parser.parse_args()
+
+        ExternalDatasetService.validate_api_list(args["settings"])
+
+        # The role of the current user in the ta table must be admin, owner, or editor, or dataset_operator
+        if not current_user.is_dataset_editor:
+            raise Forbidden()
+
+        try:
+            external_knowledge_api = ExternalDatasetService.create_external_knowledge_api(
+                tenant_id=current_user.current_tenant_id, user_id=current_user.id, args=args
+            )
+        except services.errors.dataset.DatasetNameDuplicateError:
+            raise DatasetNameDuplicateError()
+
+        return external_knowledge_api.to_dict(), 201
+
+
+class ExternalApiTemplateApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def get(self, external_knowledge_api_id):
+        external_knowledge_api_id = str(external_knowledge_api_id)
+        external_knowledge_api = ExternalDatasetService.get_external_knowledge_api(external_knowledge_api_id)
+        if external_knowledge_api is None:
+            raise NotFound("API template not found.")
+
+        return external_knowledge_api.to_dict(), 200
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def patch(self, external_knowledge_api_id):
+        external_knowledge_api_id = str(external_knowledge_api_id)
+
+        parser = reqparse.RequestParser()
+        parser.add_argument(
+            "name",
+            nullable=False,
+            required=True,
+            help="type is required. Name must be between 1 to 100 characters.",
+            type=_validate_name,
+        )
+        parser.add_argument(
+            "settings",
+            type=dict,
+            location="json",
+            nullable=False,
+            required=True,
+        )
+        args = parser.parse_args()
+        ExternalDatasetService.validate_api_list(args["settings"])
+
+        external_knowledge_api = ExternalDatasetService.update_external_knowledge_api(
+            tenant_id=current_user.current_tenant_id,
+            user_id=current_user.id,
+            external_knowledge_api_id=external_knowledge_api_id,
+            args=args,
+        )
+
+        return external_knowledge_api.to_dict(), 200
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def delete(self, external_knowledge_api_id):
+        external_knowledge_api_id = str(external_knowledge_api_id)
+
+        # The role of the current user in the ta table must be admin, owner, or editor
+        if not current_user.is_editor or current_user.is_dataset_operator:
+            raise Forbidden()
+
+        ExternalDatasetService.delete_external_knowledge_api(current_user.current_tenant_id, external_knowledge_api_id)
+        return {"result": "success"}, 200
+
+
+class ExternalApiUseCheckApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def get(self, external_knowledge_api_id):
+        external_knowledge_api_id = str(external_knowledge_api_id)
+
+        external_knowledge_api_is_using, count = ExternalDatasetService.external_knowledge_api_use_check(
+            external_knowledge_api_id
+        )
+        return {"is_using": external_knowledge_api_is_using, "count": count}, 200
+
+
+class ExternalDatasetInitApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def post(self):
+        # The role of the current user in the ta table must be admin, owner, or editor
+        if not current_user.is_editor:
+            raise Forbidden()
+
+        parser = reqparse.RequestParser()
+        parser.add_argument("external_knowledge_api_id", type=str, required=True, nullable=True, location="json")
+        # parser.add_argument('name', nullable=False, required=True,
+        #                     help='name is required. Name must be between 1 to 100 characters.',
+        #                     type=_validate_name)
+        # parser.add_argument('description', type=str, required=True, nullable=True, location='json')
+        parser.add_argument("data_source", type=dict, required=True, nullable=True, location="json")
+        parser.add_argument("process_parameter", type=dict, required=True, nullable=True, location="json")
+
+        args = parser.parse_args()
+
+        # The role of the current user in the ta table must be admin, owner, or editor, or dataset_operator
+        if not current_user.is_dataset_editor:
+            raise Forbidden()
+
+        # validate args
+        ExternalDatasetService.document_create_args_validate(
+            current_user.current_tenant_id, args["external_knowledge_api_id"], args["process_parameter"]
+        )
+
+        try:
+            dataset, documents, batch = ExternalDatasetService.init_external_dataset(
+                tenant_id=current_user.current_tenant_id,
+                user_id=current_user.id,
+                args=args,
+            )
+        except Exception as ex:
+            raise ProviderNotInitializeError(ex.description)
+        response = {"dataset": dataset, "documents": documents, "batch": batch}
+
+        return response
+
+
+class ExternalDatasetCreateApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def post(self):
+        # The role of the current user in the ta table must be admin, owner, or editor
+        if not current_user.is_editor:
+            raise Forbidden()
+
+        parser = reqparse.RequestParser()
+        parser.add_argument("external_knowledge_api_id", type=str, required=True, nullable=False, location="json")
+        parser.add_argument("external_knowledge_id", type=str, required=True, nullable=False, location="json")
+        parser.add_argument(
+            "name",
+            nullable=False,
+            required=True,
+            help="name is required. Name must be between 1 to 100 characters.",
+            type=_validate_name,
+        )
+        parser.add_argument("description", type=str, required=False, nullable=True, location="json")
+        parser.add_argument("external_retrieval_model", type=dict, required=False, location="json")
+
+        args = parser.parse_args()
+
+        # The role of the current user in the ta table must be admin, owner, or editor, or dataset_operator
+        if not current_user.is_dataset_editor:
+            raise Forbidden()
+
+        try:
+            dataset = ExternalDatasetService.create_external_dataset(
+                tenant_id=current_user.current_tenant_id,
+                user_id=current_user.id,
+                args=args,
+            )
+        except services.errors.dataset.DatasetNameDuplicateError:
+            raise DatasetNameDuplicateError()
+
+        return marshal(dataset, dataset_detail_fields), 201
+
+
+class ExternalKnowledgeHitTestingApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def post(self, dataset_id):
+        dataset_id_str = str(dataset_id)
+        dataset = DatasetService.get_dataset(dataset_id_str)
+        if dataset is None:
+            raise NotFound("Dataset not found.")
+
+        try:
+            DatasetService.check_dataset_permission(dataset, current_user)
+        except services.errors.account.NoPermissionError as e:
+            raise Forbidden(str(e))
+
+        parser = reqparse.RequestParser()
+        parser.add_argument("query", type=str, location="json")
+        parser.add_argument("external_retrieval_model", type=dict, required=False, location="json")
+        args = parser.parse_args()
+
+        HitTestingService.hit_testing_args_check(args)
+
+        try:
+            response = HitTestingService.external_retrieve(
+                dataset=dataset,
+                query=args["query"],
+                account=current_user,
+                external_retrieval_model=args["external_retrieval_model"],
+            )
+
+            return response
+        except Exception as e:
+            raise InternalServerError(str(e))
+
+
+api.add_resource(ExternalKnowledgeHitTestingApi, "/datasets/<uuid:dataset_id>/external-hit-testing")
+api.add_resource(ExternalDatasetCreateApi, "/datasets/external")
+api.add_resource(ExternalApiTemplateListApi, "/datasets/external-knowledge-api")
+api.add_resource(ExternalApiTemplateApi, "/datasets/external-knowledge-api/<uuid:external_knowledge_api_id>")
+api.add_resource(ExternalApiUseCheckApi, "/datasets/external-knowledge-api/<uuid:external_knowledge_api_id>/use-check")

api/controllers/console/datasets/hit_testing.py

@@ -47,6 +47,7 @@ class HitTestingApi(Resource):
         parser = reqparse.RequestParser()
         parser.add_argument("query", type=str, location="json")
         parser.add_argument("retrieval_model", type=dict, required=False, location="json")
+        parser.add_argument("external_retrieval_model", type=dict, required=False, location="json")
         args = parser.parse_args()
 
         HitTestingService.hit_testing_args_check(args)
@@ -57,6 +58,7 @@ class HitTestingApi(Resource):
                 query=args["query"],
                 account=current_user,
                 retrieval_model=args["retrieval_model"],
+                external_retrieval_model=args["external_retrieval_model"],
                 limit=10,
             )
 

api/controllers/console/datasets/test_external.py

@@ -0,0 +1,33 @@
+from flask_restful import Resource, reqparse
+
+from controllers.console import api
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required
+from libs.login import login_required
+from services.external_knowledge_service import ExternalDatasetService
+
+
+class TestExternalApi(Resource):
+    def post(self):
+        parser = reqparse.RequestParser()
+        parser.add_argument("retrieval_setting", nullable=False, required=True, type=dict, location="json")
+        parser.add_argument(
+            "query",
+            nullable=False,
+            required=True,
+            type=str,
+        )
+        parser.add_argument(
+            "knowledge_id",
+            nullable=False,
+            required=True,
+            type=str,
+        )
+        args = parser.parse_args()
+        result = ExternalDatasetService.test_external_knowledge_retrieval(
+            args["retrieval_setting"], args["query"], args["knowledge_id"]
+        )
+        return result, 200
+
+
+api.add_resource(TestExternalApi, "/retrieval")

api/controllers/console/error.py

@@ -18,9 +18,7 @@ class NotSetupError(BaseHTTPException):
 
 class NotInitValidateError(BaseHTTPException):
     error_code = "not_init_validated"
-    description = (
+    description = "Init validation has not been completed yet. Please proceed with the init validation process first."
-        "Init validation has not been completed yet. " "Please proceed with the init validation process first."
-    )
     code = 401
 
 

api/controllers/console/explore/audio.py

@@ -81,7 +81,7 @@ class ChatTextApi(InstalledAppResource):
             message_id = args.get("message_id", None)
             text = args.get("text", None)
             if (
-                app_model.mode in [AppMode.ADVANCED_CHAT.value, AppMode.WORKFLOW.value]
+                app_model.mode in {AppMode.ADVANCED_CHAT.value, AppMode.WORKFLOW.value}
                 and app_model.workflow
                 and app_model.workflow.features_dict
             ):

api/controllers/console/explore/completion.py

@@ -92,7 +92,7 @@ class ChatApi(InstalledAppResource):
     def post(self, installed_app):
         app_model = installed_app.app
         app_mode = AppMode.value_of(app_model.mode)
-        if app_mode not in [AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT]:
+        if app_mode not in {AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT}:
             raise NotChatAppError()
 
         parser = reqparse.RequestParser()
@@ -100,6 +100,7 @@ class ChatApi(InstalledAppResource):
         parser.add_argument("query", type=str, required=True, location="json")
         parser.add_argument("files", type=list, required=False, location="json")
         parser.add_argument("conversation_id", type=uuid_value, location="json")
+        parser.add_argument("parent_message_id", type=uuid_value, required=False, location="json")
         parser.add_argument("retriever_from", type=str, required=False, default="explore_app", location="json")
         args = parser.parse_args()
 
@@ -140,7 +141,7 @@ class ChatStopApi(InstalledAppResource):
     def post(self, installed_app, task_id):
         app_model = installed_app.app
         app_mode = AppMode.value_of(app_model.mode)
-        if app_mode not in [AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT]:
+        if app_mode not in {AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT}:
             raise NotChatAppError()
 
         AppQueueManager.set_stop_flag(task_id, InvokeFrom.EXPLORE, current_user.id)

api/controllers/console/explore/conversation.py

@@ -20,7 +20,7 @@ class ConversationListApi(InstalledAppResource):
     def get(self, installed_app):
         app_model = installed_app.app
         app_mode = AppMode.value_of(app_model.mode)
-        if app_mode not in [AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT]:
+        if app_mode not in {AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT}:
             raise NotChatAppError()
 
         parser = reqparse.RequestParser()
@@ -50,7 +50,7 @@ class ConversationApi(InstalledAppResource):
     def delete(self, installed_app, c_id):
         app_model = installed_app.app
         app_mode = AppMode.value_of(app_model.mode)
-        if app_mode not in [AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT]:
+        if app_mode not in {AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT}:
             raise NotChatAppError()
 
         conversation_id = str(c_id)
@@ -68,7 +68,7 @@ class ConversationRenameApi(InstalledAppResource):
     def post(self, installed_app, c_id):
         app_model = installed_app.app
         app_mode = AppMode.value_of(app_model.mode)
-        if app_mode not in [AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT]:
+        if app_mode not in {AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT}:
             raise NotChatAppError()
 
         conversation_id = str(c_id)
@@ -90,7 +90,7 @@ class ConversationPinApi(InstalledAppResource):
     def patch(self, installed_app, c_id):
         app_model = installed_app.app
         app_mode = AppMode.value_of(app_model.mode)
-        if app_mode not in [AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT]:
+        if app_mode not in {AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT}:
             raise NotChatAppError()
 
         conversation_id = str(c_id)
@@ -107,7 +107,7 @@ class ConversationUnPinApi(InstalledAppResource):
     def patch(self, installed_app, c_id):
         app_model = installed_app.app
         app_mode = AppMode.value_of(app_model.mode)
-        if app_mode not in [AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT]:
+        if app_mode not in {AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT}:
             raise NotChatAppError()
 
         conversation_id = str(c_id)

api/controllers/console/explore/installed_app.py

@@ -31,7 +31,7 @@ class InstalledAppsListApi(Resource):
                 "app_owner_tenant_id": installed_app.app_owner_tenant_id,
                 "is_pinned": installed_app.is_pinned,
                 "last_used_at": installed_app.last_used_at,
-                "editable": current_user.role in ["owner", "admin"],
+                "editable": current_user.role in {"owner", "admin"},
                 "uninstallable": current_tenant_id == installed_app.app_owner_tenant_id,
             }
             for installed_app in installed_apps

api/controllers/console/explore/message.py

@@ -40,7 +40,7 @@ class MessageListApi(InstalledAppResource):
         app_model = installed_app.app
 
         app_mode = AppMode.value_of(app_model.mode)
-        if app_mode not in [AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT]:
+        if app_mode not in {AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT}:
             raise NotChatAppError()
 
         parser = reqparse.RequestParser()
@@ -51,7 +51,7 @@ class MessageListApi(InstalledAppResource):
 
         try:
             return MessageService.pagination_by_first_id(
-                app_model, current_user, args["conversation_id"], args["first_id"], args["limit"]
+                app_model, current_user, args["conversation_id"], args["first_id"], args["limit"], "desc"
             )
         except services.errors.conversation.ConversationNotExistsError:
             raise NotFound("Conversation Not Exists.")
@@ -125,7 +125,7 @@ class MessageSuggestedQuestionApi(InstalledAppResource):
     def get(self, installed_app, message_id):
         app_model = installed_app.app
         app_mode = AppMode.value_of(app_model.mode)
-        if app_mode not in [AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT]:
+        if app_mode not in {AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT}:
             raise NotChatAppError()
 
         message_id = str(message_id)

api/controllers/console/explore/parameter.py

@@ -43,7 +43,7 @@ class AppParameterApi(InstalledAppResource):
         """Retrieve app parameters."""
         app_model = installed_app.app
 
-        if app_model.mode in [AppMode.ADVANCED_CHAT.value, AppMode.WORKFLOW.value]:
+        if app_model.mode in {AppMode.ADVANCED_CHAT.value, AppMode.WORKFLOW.value}:
             workflow = app_model.workflow
             if workflow is None:
                 raise AppUnavailableError()

api/controllers/console/workspace/model_providers.py

@@ -218,7 +218,7 @@ api.add_resource(ModelProviderCredentialApi, "/workspaces/current/model-provider
 api.add_resource(ModelProviderValidateApi, "/workspaces/current/model-providers/<string:provider>/credentials/validate")
 api.add_resource(ModelProviderApi, "/workspaces/current/model-providers/<string:provider>")
 api.add_resource(
-    ModelProviderIconApi, "/workspaces/current/model-providers/<string:provider>/" "<string:icon_type>/<string:lang>"
+    ModelProviderIconApi, "/workspaces/current/model-providers/<string:provider>/<string:icon_type>/<string:lang>"
 )
 
 api.add_resource(

api/controllers/console/workspace/workspace.py

@@ -194,7 +194,7 @@ class WebappLogoWorkspaceApi(Resource):
             raise TooManyFilesError()
 
         extension = file.filename.split(".")[-1]
-        if extension.lower() not in ["svg", "png"]:
+        if extension.lower() not in {"svg", "png"}:
             raise UnsupportedFileTypeError()
 
         try:

api/controllers/service_api/app/app.py

@@ -42,7 +42,7 @@ class AppParameterApi(Resource):
     @marshal_with(parameters_fields)
     def get(self, app_model: App):
         """Retrieve app parameters."""
-        if app_model.mode in [AppMode.ADVANCED_CHAT.value, AppMode.WORKFLOW.value]:
+        if app_model.mode in {AppMode.ADVANCED_CHAT.value, AppMode.WORKFLOW.value}:
             workflow = app_model.workflow
             if workflow is None:
                 raise AppUnavailableError()

api/controllers/service_api/app/audio.py

@@ -79,7 +79,7 @@ class TextApi(Resource):
             message_id = args.get("message_id", None)
             text = args.get("text", None)
             if (
-                app_model.mode in [AppMode.ADVANCED_CHAT.value, AppMode.WORKFLOW.value]
+                app_model.mode in {AppMode.ADVANCED_CHAT.value, AppMode.WORKFLOW.value}
                 and app_model.workflow
                 and app_model.workflow.features_dict
             ):

api/controllers/service_api/app/completion.py

@@ -96,7 +96,7 @@ class ChatApi(Resource):
     @validate_app_token(fetch_user_arg=FetchUserArg(fetch_from=WhereisUserArg.JSON, required=True))
     def post(self, app_model: App, end_user: EndUser):
         app_mode = AppMode.value_of(app_model.mode)
-        if app_mode not in [AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT]:
+        if app_mode not in {AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT}:
             raise NotChatAppError()
 
         parser = reqparse.RequestParser()
@@ -144,7 +144,7 @@ class ChatStopApi(Resource):
     @validate_app_token(fetch_user_arg=FetchUserArg(fetch_from=WhereisUserArg.JSON, required=True))
     def post(self, app_model: App, end_user: EndUser, task_id):
         app_mode = AppMode.value_of(app_model.mode)
-        if app_mode not in [AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT]:
+        if app_mode not in {AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT}:
             raise NotChatAppError()
 
         AppQueueManager.set_stop_flag(task_id, InvokeFrom.SERVICE_API, end_user.id)

api/controllers/service_api/app/conversation.py

@@ -18,7 +18,7 @@ class ConversationApi(Resource):
     @marshal_with(conversation_infinite_scroll_pagination_fields)
     def get(self, app_model: App, end_user: EndUser):
         app_mode = AppMode.value_of(app_model.mode)
-        if app_mode not in [AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT]:
+        if app_mode not in {AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT}:
             raise NotChatAppError()
 
         parser = reqparse.RequestParser()
@@ -52,7 +52,7 @@ class ConversationDetailApi(Resource):
     @marshal_with(simple_conversation_fields)
     def delete(self, app_model: App, end_user: EndUser, c_id):
         app_mode = AppMode.value_of(app_model.mode)
-        if app_mode not in [AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT]:
+        if app_mode not in {AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT}:
             raise NotChatAppError()
 
         conversation_id = str(c_id)
@@ -69,7 +69,7 @@ class ConversationRenameApi(Resource):
     @marshal_with(simple_conversation_fields)
     def post(self, app_model: App, end_user: EndUser, c_id):
         app_mode = AppMode.value_of(app_model.mode)
-        if app_mode not in [AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT]:
+        if app_mode not in {AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT}:
             raise NotChatAppError()
 
         conversation_id = str(c_id)

api/controllers/service_api/app/message.py

@@ -54,6 +54,7 @@ class MessageListApi(Resource):
     message_fields = {
         "id": fields.String,
         "conversation_id": fields.String,
+        "parent_message_id": fields.String,
         "inputs": fields.Raw,
         "query": fields.String,
         "answer": fields.String(attribute="re_sign_file_url_answer"),
@@ -76,7 +77,7 @@ class MessageListApi(Resource):
     @marshal_with(message_infinite_scroll_pagination_fields)
     def get(self, app_model: App, end_user: EndUser):
         app_mode = AppMode.value_of(app_model.mode)
-        if app_mode not in [AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT]:
+        if app_mode not in {AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT}:
             raise NotChatAppError()
 
         parser = reqparse.RequestParser()
@@ -117,7 +118,7 @@ class MessageSuggestedApi(Resource):
     def get(self, app_model: App, end_user: EndUser, message_id):
         message_id = str(message_id)
         app_mode = AppMode.value_of(app_model.mode)
-        if app_mode not in [AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT]:
+        if app_mode not in {AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT}:
             raise NotChatAppError()
 
         try:

api/controllers/service_api/dataset/dataset.py

@@ -28,11 +28,11 @@ class DatasetListApi(DatasetApiResource):
 
         page = request.args.get("page", default=1, type=int)
         limit = request.args.get("limit", default=20, type=int)
-        provider = request.args.get("provider", default="vendor")
+        # provider = request.args.get("provider", default="vendor")
         search = request.args.get("keyword", default=None, type=str)
         tag_ids = request.args.getlist("tag_ids")
 
-        datasets, total = DatasetService.get_datasets(page, limit, provider, tenant_id, current_user, search, tag_ids)
+        datasets, total = DatasetService.get_datasets(page, limit, tenant_id, current_user, search, tag_ids)
         # check embedding setting
         provider_manager = ProviderManager()
         configurations = provider_manager.get_configurations(tenant_id=current_user.current_tenant_id)
@@ -82,6 +82,26 @@ class DatasetListApi(DatasetApiResource):
             required=False,
             nullable=False,
         )
+        parser.add_argument(
+            "external_knowledge_api_id",
+            type=str,
+            nullable=True,
+            required=False,
+            default="_validate_name",
+        )
+        parser.add_argument(
+            "provider",
+            type=str,
+            nullable=True,
+            required=False,
+            default="vendor",
+        )
+        parser.add_argument(
+            "external_knowledge_id",
+            type=str,
+            nullable=True,
+            required=False,
+        )
         args = parser.parse_args()
 
         try:
@@ -91,6 +111,9 @@ class DatasetListApi(DatasetApiResource):
                 indexing_technique=args["indexing_technique"],
                 account=current_user,
                 permission=args["permission"],
+                provider=args["provider"],
+                external_knowledge_api_id=args["external_knowledge_api_id"],
+                external_knowledge_id=args["external_knowledge_id"],
             )
         except services.errors.dataset.DatasetNameDuplicateError:
             raise DatasetNameDuplicateError()

api/controllers/web/app.py

@@ -41,7 +41,7 @@ class AppParameterApi(WebApiResource):
     @marshal_with(parameters_fields)
     def get(self, app_model: App, end_user):
         """Retrieve app parameters."""
-        if app_model.mode in [AppMode.ADVANCED_CHAT.value, AppMode.WORKFLOW.value]:
+        if app_model.mode in {AppMode.ADVANCED_CHAT.value, AppMode.WORKFLOW.value}:
             workflow = app_model.workflow
             if workflow is None:
                 raise AppUnavailableError()

api/controllers/web/audio.py

@@ -78,7 +78,7 @@ class TextApi(WebApiResource):
             message_id = args.get("message_id", None)
             text = args.get("text", None)
             if (
-                app_model.mode in [AppMode.ADVANCED_CHAT.value, AppMode.WORKFLOW.value]
+                app_model.mode in {AppMode.ADVANCED_CHAT.value, AppMode.WORKFLOW.value}
                 and app_model.workflow
                 and app_model.workflow.features_dict
             ):

api/controllers/web/completion.py

@@ -87,7 +87,7 @@ class CompletionStopApi(WebApiResource):
 class ChatApi(WebApiResource):
     def post(self, app_model, end_user):
         app_mode = AppMode.value_of(app_model.mode)
-        if app_mode not in [AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT]:
+        if app_mode not in {AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT}:
             raise NotChatAppError()
 
         parser = reqparse.RequestParser()
@@ -96,6 +96,7 @@ class ChatApi(WebApiResource):
         parser.add_argument("files", type=list, required=False, location="json")
         parser.add_argument("response_mode", type=str, choices=["blocking", "streaming"], location="json")
         parser.add_argument("conversation_id", type=uuid_value, location="json")
+        parser.add_argument("parent_message_id", type=uuid_value, required=False, location="json")
         parser.add_argument("retriever_from", type=str, required=False, default="web_app", location="json")
 
         args = parser.parse_args()
@@ -136,7 +137,7 @@ class ChatApi(WebApiResource):
 class ChatStopApi(WebApiResource):
     def post(self, app_model, end_user, task_id):
         app_mode = AppMode.value_of(app_model.mode)
-        if app_mode not in [AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT]:
+        if app_mode not in {AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT}:
             raise NotChatAppError()
 
         AppQueueManager.set_stop_flag(task_id, InvokeFrom.WEB_APP, end_user.id)

api/controllers/web/conversation.py

@@ -18,7 +18,7 @@ class ConversationListApi(WebApiResource):
     @marshal_with(conversation_infinite_scroll_pagination_fields)
     def get(self, app_model, end_user):
         app_mode = AppMode.value_of(app_model.mode)
-        if app_mode not in [AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT]:
+        if app_mode not in {AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT}:
             raise NotChatAppError()
 
         parser = reqparse.RequestParser()
@@ -56,7 +56,7 @@ class ConversationListApi(WebApiResource):
 class ConversationApi(WebApiResource):
     def delete(self, app_model, end_user, c_id):
         app_mode = AppMode.value_of(app_model.mode)
-        if app_mode not in [AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT]:
+        if app_mode not in {AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT}:
             raise NotChatAppError()
 
         conversation_id = str(c_id)
@@ -73,7 +73,7 @@ class ConversationRenameApi(WebApiResource):
     @marshal_with(simple_conversation_fields)
     def post(self, app_model, end_user, c_id):
         app_mode = AppMode.value_of(app_model.mode)
-        if app_mode not in [AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT]:
+        if app_mode not in {AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT}:
             raise NotChatAppError()
 
         conversation_id = str(c_id)
@@ -92,7 +92,7 @@ class ConversationRenameApi(WebApiResource):
 class ConversationPinApi(WebApiResource):
     def patch(self, app_model, end_user, c_id):
         app_mode = AppMode.value_of(app_model.mode)
-        if app_mode not in [AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT]:
+        if app_mode not in {AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT}:
             raise NotChatAppError()
 
         conversation_id = str(c_id)
@@ -108,7 +108,7 @@ class ConversationPinApi(WebApiResource):
 class ConversationUnPinApi(WebApiResource):
     def patch(self, app_model, end_user, c_id):
         app_mode = AppMode.value_of(app_model.mode)
-        if app_mode not in [AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT]:
+        if app_mode not in {AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT}:
             raise NotChatAppError()
 
         conversation_id = str(c_id)

api/controllers/web/message.py

@@ -57,6 +57,7 @@ class MessageListApi(WebApiResource):
     message_fields = {
         "id": fields.String,
         "conversation_id": fields.String,
+        "parent_message_id": fields.String,
         "inputs": fields.Raw,
         "query": fields.String,
         "answer": fields.String(attribute="re_sign_file_url_answer"),
@@ -78,7 +79,7 @@ class MessageListApi(WebApiResource):
     @marshal_with(message_infinite_scroll_pagination_fields)
     def get(self, app_model, end_user):
         app_mode = AppMode.value_of(app_model.mode)
-        if app_mode not in [AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT]:
+        if app_mode not in {AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT}:
             raise NotChatAppError()
 
         parser = reqparse.RequestParser()
@@ -89,7 +90,7 @@ class MessageListApi(WebApiResource):
 
         try:
             return MessageService.pagination_by_first_id(
-                app_model, end_user, args["conversation_id"], args["first_id"], args["limit"]
+                app_model, end_user, args["conversation_id"], args["first_id"], args["limit"], "desc"
             )
         except services.errors.conversation.ConversationNotExistsError:
             raise NotFound("Conversation Not Exists.")
@@ -160,7 +161,7 @@ class MessageMoreLikeThisApi(WebApiResource):
 class MessageSuggestedQuestionApi(WebApiResource):
     def get(self, app_model, end_user, message_id):
         app_mode = AppMode.value_of(app_model.mode)
-        if app_mode not in [AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT]:
+        if app_mode not in {AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT}:
             raise NotCompletionAppError()
 
         message_id = str(message_id)

api/core/agent/base_agent_runner.py

@@ -32,6 +32,7 @@ from core.model_runtime.entities.message_entities import (
 from core.model_runtime.entities.model_entities import ModelFeature
 from core.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel
 from core.model_runtime.utils.encoders import jsonable_encoder
+from core.prompt.utils.extract_thread_messages import extract_thread_messages
 from core.tools.entities.tool_entities import (
     ToolParameter,
     ToolRuntimeVariablePool,
@@ -441,10 +442,12 @@ class BaseAgentRunner(AppRunner):
             .filter(
                 Message.conversation_id == self.message.conversation_id,
             )
-            .order_by(Message.created_at.asc())
+            .order_by(Message.created_at.desc())
             .all()
         )
 
+        messages = list(reversed(extract_thread_messages(messages)))
+
         for message in messages:
             if message.id == self.message.id:
                 continue

api/core/agent/output_parser/cot_output_parser.py

@@ -90,7 +90,7 @@ class CotAgentOutputParser:
 
                 if not in_code_block and not in_json:
                     if delta.lower() == action_str[action_idx] and action_idx == 0:
-                        if last_character not in ["\n", " ", ""]:
+                        if last_character not in {"\n", " ", ""}:
                             index += steps
                             yield delta
                             continue
@@ -117,7 +117,7 @@ class CotAgentOutputParser:
                             action_idx = 0
 
                     if delta.lower() == thought_str[thought_idx] and thought_idx == 0:
-                        if last_character not in ["\n", " ", ""]:
+                        if last_character not in {"\n", " ", ""}:
                             index += steps
                             yield delta
                             continue

api/core/app/app_config/base_app_config_manager.py

@@ -29,7 +29,7 @@ class BaseAppConfigManager:
         additional_features.show_retrieve_source = RetrievalResourceConfigManager.convert(config=config_dict)
 
         additional_features.file_upload = FileUploadConfigManager.convert(
-            config=config_dict, is_vision=app_mode in [AppMode.CHAT, AppMode.COMPLETION, AppMode.AGENT_CHAT]
+            config=config_dict, is_vision=app_mode in {AppMode.CHAT, AppMode.COMPLETION, AppMode.AGENT_CHAT}
         )
 
         additional_features.opening_statement, additional_features.suggested_questions = (

api/core/app/app_config/easy_ui_based_app/agent/manager.py

@@ -18,7 +18,7 @@ class AgentConfigManager:
 
             if agent_strategy == "function_call":
                 strategy = AgentEntity.Strategy.FUNCTION_CALLING
-            elif agent_strategy == "cot" or agent_strategy == "react":
+            elif agent_strategy in {"cot", "react"}:
                 strategy = AgentEntity.Strategy.CHAIN_OF_THOUGHT
             else:
                 # old configs, try to detect default strategy
@@ -43,10 +43,10 @@ class AgentConfigManager:
 
                     agent_tools.append(AgentToolEntity(**agent_tool_properties))
 
-            if "strategy" in config["agent_mode"] and config["agent_mode"]["strategy"] not in [
+            if "strategy" in config["agent_mode"] and config["agent_mode"]["strategy"] not in {
                 "react_router",
                 "router",
-            ]:
+            }:
                 agent_prompt = agent_dict.get("prompt", None) or {}
                 # check model mode
                 model_mode = config.get("model", {}).get("mode", "completion")

api/core/app/app_config/easy_ui_based_app/dataset/manager.py

@@ -167,7 +167,7 @@ class DatasetConfigManager:
             config["agent_mode"]["strategy"] = PlanningStrategy.ROUTER.value
 
         has_datasets = False
-        if config["agent_mode"]["strategy"] in [PlanningStrategy.ROUTER.value, PlanningStrategy.REACT_ROUTER.value]:
+        if config["agent_mode"]["strategy"] in {PlanningStrategy.ROUTER.value, PlanningStrategy.REACT_ROUTER.value}:
             for tool in config["agent_mode"]["tools"]:
                 key = list(tool.keys())[0]
                 if key == "dataset":

api/core/app/app_config/easy_ui_based_app/prompt_template/manager.py

@@ -86,7 +86,7 @@ class PromptTemplateConfigManager:
         if config["prompt_type"] == PromptTemplateEntity.PromptType.ADVANCED.value:
             if not config["chat_prompt_config"] and not config["completion_prompt_config"]:
                 raise ValueError(
-                    "chat_prompt_config or completion_prompt_config is required " "when prompt_type is advanced"
+                    "chat_prompt_config or completion_prompt_config is required when prompt_type is advanced"
                 )
 
             model_mode_vals = [mode.value for mode in ModelMode]

api/core/app/app_config/easy_ui_based_app/variables/manager.py

@@ -42,12 +42,12 @@ class BasicVariablesConfigManager:
                         variable=variable["variable"], type=variable["type"], config=variable["config"]
                     )
                 )
-            elif variable_type in [
+            elif variable_type in {
                 VariableEntityType.TEXT_INPUT,
                 VariableEntityType.PARAGRAPH,
                 VariableEntityType.NUMBER,
                 VariableEntityType.SELECT,
-            ]:
+            }:
                 variable = variables[variable_type]
                 variable_entities.append(
                     VariableEntity(
@@ -97,7 +97,7 @@ class BasicVariablesConfigManager:
         variables = []
         for item in config["user_input_form"]:
             key = list(item.keys())[0]
-            if key not in ["text-input", "select", "paragraph", "number", "external_data_tool"]:
+            if key not in {"text-input", "select", "paragraph", "number", "external_data_tool"}:
                 raise ValueError("Keys in user_input_form list can only be 'text-input', 'paragraph'  or 'select'")
 
             form_item = item[key]
@@ -115,7 +115,7 @@ class BasicVariablesConfigManager:
 
             pattern = re.compile(r"^(?!\d)[\u4e00-\u9fa5A-Za-z0-9_\U0001F300-\U0001F64F\U0001F680-\U0001F6FF]{1,100}$")
             if pattern.match(form_item["variable"]) is None:
-                raise ValueError("variable in user_input_form must be a string, " "and cannot start with a number")
+                raise ValueError("variable in user_input_form must be a string, and cannot start with a number")
 
             variables.append(form_item["variable"])
 

api/core/app/app_config/features/file_upload/manager.py

@@ -54,14 +54,14 @@ class FileUploadConfigManager:
 
             if is_vision:
                 detail = config["file_upload"]["image"]["detail"]
-                if detail not in ["high", "low"]:
+                if detail not in {"high", "low"}:
                     raise ValueError("detail must be in ['high', 'low']")
 
             transfer_methods = config["file_upload"]["image"]["transfer_methods"]
             if not isinstance(transfer_methods, list):
                 raise ValueError("transfer_methods must be of list type")
             for method in transfer_methods:
-                if method not in ["remote_url", "local_file"]:
+                if method not in {"remote_url", "local_file"}:
                     raise ValueError("transfer_methods must be in ['remote_url', 'local_file']")
 
         return config, ["file_upload"]

api/core/app/apps/advanced_chat/app_generator.py

@@ -121,6 +121,7 @@ class AdvancedChatAppGenerator(MessageBasedAppGenerator):
             inputs=conversation.inputs if conversation else self._get_cleaned_inputs(inputs, app_config),
             query=query,
             files=file_objs,
+            parent_message_id=args.get("parent_message_id"),
             user_id=user.id,
             stream=stream,
             invoke_from=invoke_from,

api/core/app/apps/advanced_chat/app_runner.py

@@ -73,7 +73,7 @@ class AdvancedChatAppRunner(WorkflowBasedAppRunner):
             raise ValueError("Workflow not initialized")
 
         user_id = None
-        if self.application_generate_entity.invoke_from in [InvokeFrom.WEB_APP, InvokeFrom.SERVICE_API]:
+        if self.application_generate_entity.invoke_from in {InvokeFrom.WEB_APP, InvokeFrom.SERVICE_API}:
             end_user = db.session.query(EndUser).filter(EndUser.id == self.application_generate_entity.user_id).first()
             if end_user:
                 user_id = end_user.session_id
@@ -175,7 +175,7 @@ class AdvancedChatAppRunner(WorkflowBasedAppRunner):
             user_id=self.application_generate_entity.user_id,
             user_from=(
                 UserFrom.ACCOUNT
-                if self.application_generate_entity.invoke_from in [InvokeFrom.EXPLORE, InvokeFrom.DEBUGGER]
+                if self.application_generate_entity.invoke_from in {InvokeFrom.EXPLORE, InvokeFrom.DEBUGGER}
                 else UserFrom.END_USER
             ),
             invoke_from=self.application_generate_entity.invoke_from,

api/core/app/apps/agent_chat/app_generator.py

@@ -127,6 +127,7 @@ class AgentChatAppGenerator(MessageBasedAppGenerator):
             inputs=conversation.inputs if conversation else self._get_cleaned_inputs(inputs, app_config),
             query=query,
             files=file_objs,
+            parent_message_id=args.get("parent_message_id"),
             user_id=user.id,
             stream=stream,
             invoke_from=invoke_from,

api/core/app/apps/base_app_generate_response_converter.py

@@ -16,7 +16,7 @@ class AppGenerateResponseConverter(ABC):
     def convert(
         cls, response: Union[AppBlockingResponse, Generator[AppStreamResponse, Any, None]], invoke_from: InvokeFrom
     ) -> dict[str, Any] | Generator[str, Any, None]:
-        if invoke_from in [InvokeFrom.DEBUGGER, InvokeFrom.SERVICE_API]:
+        if invoke_from in {InvokeFrom.DEBUGGER, InvokeFrom.SERVICE_API}:
             if isinstance(response, AppBlockingResponse):
                 return cls.convert_blocking_full_response(response)
             else:
@@ -75,10 +75,10 @@ class AppGenerateResponseConverter(ABC):
         :return:
         """
         # show_retrieve_source
+        updated_resources = []
         if "retriever_resources" in metadata:
-            metadata["retriever_resources"] = []
             for resource in metadata["retriever_resources"]:
-                metadata["retriever_resources"].append(
+                updated_resources.append(
                     {
                         "segment_id": resource["segment_id"],
                         "position": resource["position"],
@@ -87,6 +87,7 @@ class AppGenerateResponseConverter(ABC):
                         "content": resource["content"],
                     }
                 )
+            metadata["retriever_resources"] = updated_resources
 
         # show annotation reply
         if "annotation_reply" in metadata:

api/core/app/apps/base_app_generator.py

@@ -22,11 +22,11 @@ class BaseAppGenerator:
             return var.default or ""
         if (
             var.type
-            in (
+            in {
                 VariableEntityType.TEXT_INPUT,
                 VariableEntityType.SELECT,
                 VariableEntityType.PARAGRAPH,
-            )
+            }
             and user_input_value
             and not isinstance(user_input_value, str)
         ):
@@ -44,7 +44,7 @@ class BaseAppGenerator:
             options = var.options or []
             if user_input_value not in options:
                 raise ValueError(f"{var.variable} in input form must be one of the following: {options}")
-        elif var.type in (VariableEntityType.TEXT_INPUT, VariableEntityType.PARAGRAPH):
+        elif var.type in {VariableEntityType.TEXT_INPUT, VariableEntityType.PARAGRAPH}:
             if var.max_length and user_input_value and len(user_input_value) > var.max_length:
                 raise ValueError(f"{var.variable} in input form must be less than {var.max_length} characters")
 

api/core/app/apps/base_app_queue_manager.py

@@ -32,7 +32,7 @@ class AppQueueManager:
         self._user_id = user_id
         self._invoke_from = invoke_from
 
-        user_prefix = "account" if self._invoke_from in [InvokeFrom.EXPLORE, InvokeFrom.DEBUGGER] else "end-user"
+        user_prefix = "account" if self._invoke_from in {InvokeFrom.EXPLORE, InvokeFrom.DEBUGGER} else "end-user"
         redis_client.setex(
             AppQueueManager._generate_task_belong_cache_key(self._task_id), 1800, f"{user_prefix}-{self._user_id}"
         )
@@ -118,7 +118,7 @@ class AppQueueManager:
         if result is None:
             return
 
-        user_prefix = "account" if invoke_from in [InvokeFrom.EXPLORE, InvokeFrom.DEBUGGER] else "end-user"
+        user_prefix = "account" if invoke_from in {InvokeFrom.EXPLORE, InvokeFrom.DEBUGGER} else "end-user"
         if result.decode("utf-8") != f"{user_prefix}-{user_id}":
             return
 

api/core/app/apps/base_app_runner.py

@@ -309,7 +309,7 @@ class AppRunner:
             if not prompt_messages:
                 prompt_messages = result.prompt_messages
 
-            if not usage and result.delta.usage:
+            if result.delta.usage:
                 usage = result.delta.usage
 
         if not usage:
@@ -379,7 +379,7 @@ class AppRunner:
                 queue_manager=queue_manager,
                 app_generate_entity=application_generate_entity,
                 prompt_messages=prompt_messages,
-                text="I apologize for any confusion, " "but I'm an AI assistant to be helpful, harmless, and honest.",
+                text="I apologize for any confusion, but I'm an AI assistant to be helpful, harmless, and honest.",
                 stream=application_generate_entity.stream,
             )
 

api/core/app/apps/chat/app_generator.py

@@ -128,6 +128,7 @@ class ChatAppGenerator(MessageBasedAppGenerator):
             inputs=conversation.inputs if conversation else self._get_cleaned_inputs(inputs, app_config),
             query=query,
             files=file_objs,
+            parent_message_id=args.get("parent_message_id"),
             user_id=user.id,
             stream=stream,
             invoke_from=invoke_from,

api/core/app/apps/message_based_app_generator.py

@@ -148,7 +148,7 @@ class MessageBasedAppGenerator(BaseAppGenerator):
         # get from source
         end_user_id = None
         account_id = None
-        if application_generate_entity.invoke_from in [InvokeFrom.WEB_APP, InvokeFrom.SERVICE_API]:
+        if application_generate_entity.invoke_from in {InvokeFrom.WEB_APP, InvokeFrom.SERVICE_API}:
             from_source = "api"
             end_user_id = application_generate_entity.user_id
         else:
@@ -165,11 +165,11 @@ class MessageBasedAppGenerator(BaseAppGenerator):
             model_provider = application_generate_entity.model_conf.provider
             model_id = application_generate_entity.model_conf.model
             override_model_configs = None
-            if app_config.app_model_config_from == EasyUIBasedAppModelConfigFrom.ARGS and app_config.app_mode in [
+            if app_config.app_model_config_from == EasyUIBasedAppModelConfigFrom.ARGS and app_config.app_mode in {
                 AppMode.AGENT_CHAT,
                 AppMode.CHAT,
                 AppMode.COMPLETION,
-            ]:
+            }:
                 override_model_configs = app_config.app_model_config_dict
 
         # get conversation introduction
@@ -218,6 +218,7 @@ class MessageBasedAppGenerator(BaseAppGenerator):
             answer_tokens=0,
             answer_unit_price=0,
             answer_price_unit=0,
+            parent_message_id=getattr(application_generate_entity, "parent_message_id", None),
             provider_response_latency=0,
             total_price=0,
             currency="USD",

api/core/app/apps/workflow/app_runner.py

@@ -53,7 +53,7 @@ class WorkflowAppRunner(WorkflowBasedAppRunner):
         app_config = cast(WorkflowAppConfig, app_config)
 
         user_id = None
-        if self.application_generate_entity.invoke_from in [InvokeFrom.WEB_APP, InvokeFrom.SERVICE_API]:
+        if self.application_generate_entity.invoke_from in {InvokeFrom.WEB_APP, InvokeFrom.SERVICE_API}:
             end_user = db.session.query(EndUser).filter(EndUser.id == self.application_generate_entity.user_id).first()
             if end_user:
                 user_id = end_user.session_id
@@ -113,7 +113,7 @@ class WorkflowAppRunner(WorkflowBasedAppRunner):
             user_id=self.application_generate_entity.user_id,
             user_from=(
                 UserFrom.ACCOUNT
-                if self.application_generate_entity.invoke_from in [InvokeFrom.EXPLORE, InvokeFrom.DEBUGGER]
+                if self.application_generate_entity.invoke_from in {InvokeFrom.EXPLORE, InvokeFrom.DEBUGGER}
                 else UserFrom.END_USER
             ),
             invoke_from=self.application_generate_entity.invoke_from,

api/core/app/apps/workflow_logging_callback.py

@@ -84,7 +84,7 @@ class WorkflowLoggingCallback(WorkflowCallback):
         if route_node_state.node_run_result:
             node_run_result = route_node_state.node_run_result
             self.print_text(
-                f"Inputs: " f"{jsonable_encoder(node_run_result.inputs) if node_run_result.inputs else ''}",
+                f"Inputs: {jsonable_encoder(node_run_result.inputs) if node_run_result.inputs else ''}",
                 color="green",
             )
             self.print_text(
@@ -116,7 +116,7 @@ class WorkflowLoggingCallback(WorkflowCallback):
             node_run_result = route_node_state.node_run_result
             self.print_text(f"Error: {node_run_result.error}", color="red")
             self.print_text(
-                f"Inputs: " f"" f"{jsonable_encoder(node_run_result.inputs) if node_run_result.inputs else ''}",
+                f"Inputs: {jsonable_encoder(node_run_result.inputs) if node_run_result.inputs else ''}",
                 color="red",
             )
             self.print_text(
@@ -125,7 +125,7 @@ class WorkflowLoggingCallback(WorkflowCallback):
                 color="red",
             )
             self.print_text(
-                f"Outputs: " f"{jsonable_encoder(node_run_result.outputs) if node_run_result.outputs else ''}",
+                f"Outputs: {jsonable_encoder(node_run_result.outputs) if node_run_result.outputs else ''}",
                 color="red",
             )
 

api/core/app/entities/app_invoke_entities.py

@@ -122,6 +122,7 @@ class ChatAppGenerateEntity(EasyUIBasedAppGenerateEntity):
     """
 
     conversation_id: Optional[str] = None
+    parent_message_id: Optional[str] = None
 
 
 class CompletionAppGenerateEntity(EasyUIBasedAppGenerateEntity):
@@ -138,6 +139,7 @@ class AgentChatAppGenerateEntity(EasyUIBasedAppGenerateEntity):
     """
 
     conversation_id: Optional[str] = None
+    parent_message_id: Optional[str] = None
 
 
 class AdvancedChatAppGenerateEntity(AppGenerateEntity):
@@ -149,6 +151,7 @@ class AdvancedChatAppGenerateEntity(AppGenerateEntity):
     app_config: WorkflowUIBasedAppConfig
 
     conversation_id: Optional[str] = None
+    parent_message_id: Optional[str] = None
     query: str
 
     class SingleIterationRunEntity(BaseModel):

api/core/app/features/annotation_reply/annotation_reply.py

@@ -63,7 +63,7 @@ class AnnotationReplyFeature:
                 score = documents[0].metadata["score"]
                 annotation = AppAnnotationService.get_annotation_by_id(annotation_id)
                 if annotation:
-                    if invoke_from in [InvokeFrom.SERVICE_API, InvokeFrom.WEB_APP]:
+                    if invoke_from in {InvokeFrom.SERVICE_API, InvokeFrom.WEB_APP}:
                         from_source = "api"
                     else:
                         from_source = "console"

api/core/app/task_pipeline/easy_ui_based_generate_task_pipeline.py

@@ -372,7 +372,7 @@ class EasyUIBasedGenerateTaskPipeline(BasedGenerateTaskPipeline, MessageCycleMan
             self._message,
             application_generate_entity=self._application_generate_entity,
             conversation=self._conversation,
-            is_first_message=self._application_generate_entity.app_config.app_mode in [AppMode.AGENT_CHAT, AppMode.CHAT]
+            is_first_message=self._application_generate_entity.app_config.app_mode in {AppMode.AGENT_CHAT, AppMode.CHAT}
             and self._application_generate_entity.conversation_id is None,
             extras=self._application_generate_entity.extras,
         )

api/core/app/task_pipeline/workflow_cycle_manage.py

@@ -383,7 +383,7 @@ class WorkflowCycleManage:
         :param workflow_node_execution: workflow node execution
         :return:
         """
-        if workflow_node_execution.node_type in [NodeType.ITERATION.value, NodeType.LOOP.value]:
+        if workflow_node_execution.node_type in {NodeType.ITERATION.value, NodeType.LOOP.value}:
             return None
 
         response = NodeStartStreamResponse(
@@ -430,7 +430,7 @@ class WorkflowCycleManage:
         :param workflow_node_execution: workflow node execution
         :return:
         """
-        if workflow_node_execution.node_type in [NodeType.ITERATION.value, NodeType.LOOP.value]:
+        if workflow_node_execution.node_type in {NodeType.ITERATION.value, NodeType.LOOP.value}:
             return None
 
         return NodeFinishStreamResponse(

api/core/callback_handler/index_tool_callback_handler.py

@@ -29,7 +29,7 @@ class DatasetIndexToolCallbackHandler:
             source="app",
             source_app_id=self._app_id,
             created_by_role=(
-                "account" if self._invoke_from in [InvokeFrom.EXPLORE, InvokeFrom.DEBUGGER] else "end_user"
+                "account" if self._invoke_from in {InvokeFrom.EXPLORE, InvokeFrom.DEBUGGER} else "end_user"
             ),
             created_by=self._user_id,
         )
@@ -59,7 +59,7 @@ class DatasetIndexToolCallbackHandler:
             for item in resource:
                 dataset_retriever_resource = DatasetRetrieverResource(
                     message_id=self._message_id,
-                    position=item.get("position"),
+                    position=item.get("position") or 0,
                     dataset_id=item.get("dataset_id"),
                     dataset_name=item.get("dataset_name"),
                     document_id=item.get("document_id"),

api/core/embedding/cached_embedding.py

@@ -5,6 +5,7 @@ from typing import Optional, cast
 import numpy as np
 from sqlalchemy.exc import IntegrityError
 
+from core.embedding.embedding_constant import EmbeddingInputType
 from core.model_manager import ModelInstance
 from core.model_runtime.entities.model_entities import ModelPropertyKey
 from core.model_runtime.model_providers.__base.text_embedding_model import TextEmbeddingModel
@@ -56,7 +57,9 @@ class CacheEmbedding(Embeddings):
                 for i in range(0, len(embedding_queue_texts), max_chunks):
                     batch_texts = embedding_queue_texts[i : i + max_chunks]
 
-                    embedding_result = self._model_instance.invoke_text_embedding(texts=batch_texts, user=self._user)
+                    embedding_result = self._model_instance.invoke_text_embedding(
+                        texts=batch_texts, user=self._user, input_type=EmbeddingInputType.DOCUMENT
+                    )
 
                     for vector in embedding_result.embeddings:
                         try:
@@ -65,7 +68,7 @@ class CacheEmbedding(Embeddings):
                         except IntegrityError:
                             db.session.rollback()
                         except Exception as e:
-                            logging.exception("Failed transform embedding: ", e)
+                            logging.exception("Failed transform embedding: %s", e)
                 cache_embeddings = []
                 try:
                     for i, embedding in zip(embedding_queue_indices, embedding_queue_embeddings):
@@ -85,7 +88,7 @@ class CacheEmbedding(Embeddings):
                     db.session.rollback()
             except Exception as ex:
                 db.session.rollback()
-                logger.error("Failed to embed documents: ", ex)
+                logger.error("Failed to embed documents: %s", ex)
                 raise ex
 
         return text_embeddings
@@ -100,7 +103,9 @@ class CacheEmbedding(Embeddings):
             redis_client.expire(embedding_cache_key, 600)
             return list(np.frombuffer(base64.b64decode(embedding), dtype="float"))
         try:
-            embedding_result = self._model_instance.invoke_text_embedding(texts=[text], user=self._user)
+            embedding_result = self._model_instance.invoke_text_embedding(
+                texts=[text], user=self._user, input_type=EmbeddingInputType.QUERY
+            )
 
             embedding_results = embedding_result.embeddings[0]
             embedding_results = (embedding_results / np.linalg.norm(embedding_results)).tolist()
@@ -116,10 +121,7 @@ class CacheEmbedding(Embeddings):
             # Transform to string
             encoded_str = encoded_vector.decode("utf-8")
             redis_client.setex(embedding_cache_key, 600, encoded_str)
-
+        except Exception as ex:
-        except IntegrityError:
+            logging.exception("Failed to add embedding to redis %s", ex)
-            db.session.rollback()
-        except:
-            logging.exception("Failed to add embedding to redis")
 
         return embedding_results

api/core/embedding/embedding_constant.py

@@ -0,0 +1,10 @@
+from enum import Enum
+
+
+class EmbeddingInputType(Enum):
+    """
+    Enum for embedding input type.
+    """
+
+    DOCUMENT = "document"
+    QUERY = "query"

api/core/indexing_runner.py

@@ -292,7 +292,7 @@ class IndexingRunner:
         self, index_processor: BaseIndexProcessor, dataset_document: DatasetDocument, process_rule: dict
     ) -> list[Document]:
         # load file
-        if dataset_document.data_source_type not in ["upload_file", "notion_import", "website_crawl"]:
+        if dataset_document.data_source_type not in {"upload_file", "notion_import", "website_crawl"}:
             return []
 
         data_source_info = dataset_document.data_source_info_dict

api/core/llm_generator/llm_generator.py

@@ -47,6 +47,8 @@ class LLMGenerator:
             )
         answer = response.message.content
         cleaned_answer = re.sub(r"^.*(\{.*\}).*$", r"\1", answer, flags=re.DOTALL)
+        if cleaned_answer is None:
+            return ""
         result_dict = json.loads(cleaned_answer)
         answer = result_dict["Your Output"]
         name = answer.strip()

api/core/llm_generator/prompts.py

@@ -65,7 +65,6 @@ SUGGESTED_QUESTIONS_AFTER_ANSWER_INSTRUCTION_PROMPT = (
     "Please help me predict the three most likely questions that human would ask, "
     "and keeping each question under 20 characters.\n"
     "MAKE SURE your output is the SAME language as the Assistant's latest response"
-    "(if the main response is written in Chinese, then the language of your output must be using Chinese.)!\n"
     "The output must be an array in JSON format following the specified schema:\n"
     '["question1","question2","question3"]\n'
 )