if session unauthorized, rejoin

Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>

.claude/settings.json.example

@@ -0,0 +1,19 @@
+{
+    "permissions": {
+      "allow": [],
+      "deny": []
+    },
+    "env": {
+      "__comment": "Environment variables for MCP servers. Override in .claude/settings.local.json with actual values.",
+      "GITHUB_PERSONAL_ACCESS_TOKEN": "ghp_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
+    },
+    "enabledMcpjsonServers": [
+      "context7",
+      "sequential-thinking",
+      "github",
+      "fetch",
+      "playwright",
+      "ide"
+    ],
+    "enableAllProjectMcpServers": true
+  }

.devcontainer/Dockerfile

@@ -1,4 +1,4 @@
-FROM mcr.microsoft.com/devcontainers/python:3.12
+FROM mcr.microsoft.com/devcontainers/python:3.12-bookworm
 
 RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
      && apt-get -y install libgmp-dev libmpfr-dev libmpc-dev

.devcontainer/README.md

@@ -1,23 +1,26 @@
 # Development with devcontainer
+
 This project includes a devcontainer configuration that allows you to open the project in a container with a fully configured development environment.
 Both frontend and backend environments are initialized when the container is started.
+
 ## GitHub Codespaces
+
 [![Open in GitHub Codespaces](https://github.com/codespaces/badge.svg)](https://codespaces.new/langgenius/dify)
 
 you can simply click the button above to open this project in GitHub Codespaces.
 
 For more info, check out the [GitHub documentation](https://docs.github.com/en/free-pro-team@latest/github/developing-online-with-codespaces/creating-a-codespace#creating-a-codespace).
 
-
 ## VS Code Dev Containers
+
 [![Open in Dev Containers](https://img.shields.io/static/v1?label=Dev%20Containers&message=Open&color=blue&logo=visualstudiocode)](https://vscode.dev/redirect?url=vscode://ms-vscode-remote.remote-containers/cloneInVolume?url=https://github.com/langgenius/dify)
 
 if you have VS Code installed, you can click the button above to open this project in VS Code Dev Containers.
 
 You can learn more in the [Dev Containers documentation](https://code.visualstudio.com/docs/devcontainers/containers).
 
-
 ## Pros of Devcontainer
+
 Unified Development Environment: By using devcontainers, you can ensure that all developers are developing in the same environment, reducing the occurrence of "it works on my machine" type of issues.
 
 Quick Start: New developers can set up their development environment in a few simple steps, without spending a lot of time on environment configuration.
@@ -25,11 +28,13 @@ Quick Start: New developers can set up their development environment in a few si
 Isolation: Devcontainers isolate your project from your host operating system, reducing the chance of OS updates or other application installations impacting the development environment.
 
 ## Cons of Devcontainer
+
 Learning Curve: For developers unfamiliar with Docker and VS Code, using devcontainers may be somewhat complex.
 
 Performance Impact: While usually minimal, programs running inside a devcontainer may be slightly slower than those running directly on the host.
 
 ## Troubleshooting
+
 if you see such error message when you open this project in codespaces:
 ![Alt text](troubleshooting.png)
 

.devcontainer/post_create_command.sh

@@ -1,15 +1,16 @@
 #!/bin/bash
+WORKSPACE_ROOT=$(pwd)
 
-npm add -g pnpm@10.13.1
+corepack enable
 cd web && pnpm install
 pipx install uv
 
-echo 'alias start-api="cd /workspaces/dify/api && uv run python -m flask run --host 0.0.0.0 --port=5001 --debug"' >> ~/.bashrc
-echo 'alias start-worker="cd /workspaces/dify/api && uv run python -m celery -A app.celery worker -P gevent -c 1 --loglevel INFO -Q dataset,generation,mail,ops_trace,app_deletion"' >> ~/.bashrc
-echo 'alias start-web="cd /workspaces/dify/web && pnpm dev"' >> ~/.bashrc
-echo 'alias start-web-prod="cd /workspaces/dify/web && pnpm build && pnpm start"' >> ~/.bashrc
-echo 'alias start-containers="cd /workspaces/dify/docker && docker-compose -f docker-compose.middleware.yaml -p dify --env-file middleware.env up -d"' >> ~/.bashrc
-echo 'alias stop-containers="cd /workspaces/dify/docker && docker-compose -f docker-compose.middleware.yaml -p dify --env-file middleware.env down"' >> ~/.bashrc
+echo "alias start-api=\"cd $WORKSPACE_ROOT/api && uv run python -m flask run --host 0.0.0.0 --port=5001 --debug\"" >> ~/.bashrc
+echo "alias start-worker=\"cd $WORKSPACE_ROOT/api && uv run python -m celery -A app.celery worker -P gevent -c 1 --loglevel INFO -Q dataset,generation,mail,ops_trace,app_deletion,plugin,workflow_storage\"" >> ~/.bashrc
+echo "alias start-web=\"cd $WORKSPACE_ROOT/web && pnpm dev\"" >> ~/.bashrc
+echo "alias start-web-prod=\"cd $WORKSPACE_ROOT/web && pnpm build && pnpm start\"" >> ~/.bashrc
+echo "alias start-containers=\"cd $WORKSPACE_ROOT/docker && docker-compose -f docker-compose.middleware.yaml -p dify --env-file middleware.env up -d\"" >> ~/.bashrc
+echo "alias stop-containers=\"cd $WORKSPACE_ROOT/docker && docker-compose -f docker-compose.middleware.yaml -p dify --env-file middleware.env down\"" >> ~/.bashrc
 
 source /home/vscode/.bashrc
 

.github/CODE_OF_CONDUCT.md

@@ -17,27 +17,25 @@ diverse, inclusive, and healthy community.
 Examples of behavior that contributes to a positive environment for our
 community include:
 
-* Demonstrating empathy and kindness toward other people
-* Being respectful of differing opinions, viewpoints, and experiences
-* Giving and gracefully accepting constructive feedback
-* Accepting responsibility and apologizing to those affected by our mistakes,
+- Demonstrating empathy and kindness toward other people
+- Being respectful of differing opinions, viewpoints, and experiences
+- Giving and gracefully accepting constructive feedback
+- Accepting responsibility and apologizing to those affected by our mistakes,
   and learning from the experience
-* Focusing on what is best not just for us as individuals, but for the
+- Focusing on what is best not just for us as individuals, but for the
   overall community
 
 Examples of unacceptable behavior include:
 
-* The use of sexualized language or imagery, and sexual attention or
+- The use of sexualized language or imagery, and sexual attention or
   advances of any kind
-* Trolling, insulting or derogatory comments, and personal or political attacks
-* Public or private harassment
-* Publishing others' private information, such as a physical or email
+- Trolling, insulting or derogatory comments, and personal or political attacks
+- Public or private harassment
+- Publishing others' private information, such as a physical or email
   address, without their explicit permission
-* Other conduct which could reasonably be considered inappropriate in a
+- Other conduct which could reasonably be considered inappropriate in a
   professional setting
 
 ## Language Policy
 
 To facilitate clear and effective communication, all discussions, comments, documentation, and pull requests in this project should be conducted in English. This ensures that all contributors can participate and collaborate effectively.
-
-

.github/ISSUE_TEMPLATE/config.yml

@@ -1,5 +1,8 @@
 blank_issues_enabled: false
 contact_links:
+  - name: "\U0001F510 Security Vulnerabilities"
+    url: "https://github.com/langgenius/dify/security/advisories/new"
+    about: Report security vulnerabilities through GitHub Security Advisories to ensure responsible disclosure. 💡 Please do not report security vulnerabilities in public issues.
   - name: "\U0001F4A1 Model Providers & Plugins"
     url: "https://github.com/langgenius/dify-official-plugins/issues/new/choose"
     about: Report issues with official plugins or model providers, you will need to provide the plugin version and other relevant details.

.github/actions/setup-uv/action.yml

@@ -1,34 +0,0 @@
-name: Setup UV and Python
-
-inputs:
-  python-version:
-    description: Python version to use and the UV installed with
-    required: true
-    default: '3.12'
-  uv-version:
-    description: UV version to set up
-    required: true
-    default: '~=0.7.11'
-  uv-lockfile:
-    description: Path to the UV lockfile to restore cache from
-    required: true
-    default: ''
-  enable-cache:
-    required: true
-    default: true
-
-runs:
-  using: composite
-  steps:
-    - name: Set up Python ${{ inputs.python-version }}
-      uses: actions/setup-python@v5
-      with:
-        python-version: ${{ inputs.python-version }}
-
-    - name: Install uv
-      uses: astral-sh/setup-uv@v5
-      with:
-        version: ${{ inputs.uv-version }}
-        python-version: ${{ inputs.python-version }}
-        enable-cache: ${{ inputs.enable-cache }}
-        cache-dependency-glob: ${{ inputs.uv-lockfile }}

.github/dependabot.yml

@@ -0,0 +1,12 @@
+version: 2
+updates:
+  - package-ecosystem: "npm"
+    directory: "/web"
+    schedule:
+      interval: "weekly"
+    open-pull-requests-limit: 2
+  - package-ecosystem: "uv"
+    directory: "/api"
+    schedule:
+      interval: "weekly"
+    open-pull-requests-limit: 2

.github/pull_request_template.md

@@ -1,8 +1,8 @@
 > [!IMPORTANT]
 >
 > 1. Make sure you have read our [contribution guidelines](https://github.com/langgenius/dify/blob/main/CONTRIBUTING.md)
-> 2. Ensure there is an associated issue and you have been assigned to it
-> 3. Use the correct syntax to link this PR: `Fixes #<issue number>`.
+> 1. Ensure there is an associated issue and you have been assigned to it
+> 1. Use the correct syntax to link this PR: `Fixes #<issue number>`.
 
 ## Summary
 
@@ -12,7 +12,7 @@
 
 | Before | After |
 |--------|-------|
-| ...    | ...   |
+| ... | ... |
 
 ## Checklist
 

.github/workflows/api-tests.yml

@@ -1,13 +1,7 @@
 name: Run Pytest
 
 on:
-  pull_request:
-    branches:
-      - main
-    paths:
-      - api/**
-      - docker/**
-      - .github/workflows/api-tests.yml
+  workflow_call:
 
 concurrency:
   group: api-tests-${{ github.head_ref || github.run_id }}
@@ -33,10 +27,11 @@ jobs:
           persist-credentials: false
 
       - name: Setup UV and Python
-        uses: ./.github/actions/setup-uv
+        uses: astral-sh/setup-uv@v6
         with:
+          enable-cache: true
           python-version: ${{ matrix.python-version }}
-          uv-lockfile: api/uv.lock
+          cache-dependency-glob: api/uv.lock
 
       - name: Check UV lockfile
         run: uv lock --project api --check
@@ -44,33 +39,15 @@ jobs:
       - name: Install dependencies
         run: uv sync --project api --dev
 
-      - name: Run Unit tests
+      - name: Run pyrefly check
         run: |
-          uv run --project api bash dev/pytest/pytest_unit_tests.sh
-
-      - name: Coverage Summary
-        run: |
-          set -x
-          # Extract coverage percentage and create a summary
-          TOTAL_COVERAGE=$(python -c 'import json; print(json.load(open("coverage.json"))["totals"]["percent_covered_display"])')
-
-          # Create a detailed coverage summary
-          echo "### Test Coverage Summary :test_tube:" >> $GITHUB_STEP_SUMMARY
-          echo "Total Coverage: ${TOTAL_COVERAGE}%" >> $GITHUB_STEP_SUMMARY
-          uv run --project api coverage report --format=markdown >> $GITHUB_STEP_SUMMARY
+          cd api
+          uv add --dev pyrefly
+          uv run pyrefly check || true
 
       - name: Run dify config tests
         run: uv run --project api dev/pytest/pytest_config_tests.py
 
-      - name: MyPy Cache
-        uses: actions/cache@v4
-        with:
-          path: api/.mypy_cache
-          key: mypy-${{ matrix.python-version }}-${{ runner.os }}-${{ hashFiles('api/uv.lock') }}
-
-      - name: Run MyPy Checks
-        run: dev/mypy-check
-
       - name: Set up dotenvs
         run: |
           cp docker/.env.example docker/.env
@@ -102,3 +79,19 @@ jobs:
 
       - name: Run TestContainers
         run: uv run --project api bash dev/pytest/pytest_testcontainers.sh
+
+      - name: Run Unit tests
+        run: |
+          uv run --project api bash dev/pytest/pytest_unit_tests.sh
+
+      - name: Coverage Summary
+        run: |
+          set -x
+          # Extract coverage percentage and create a summary
+          TOTAL_COVERAGE=$(python -c 'import json; print(json.load(open("coverage.json"))["totals"]["percent_covered_display"])')
+
+          # Create a detailed coverage summary
+          echo "### Test Coverage Summary :test_tube:" >> $GITHUB_STEP_SUMMARY
+          echo "Total Coverage: ${TOTAL_COVERAGE}%" >> $GITHUB_STEP_SUMMARY
+          uv run --project api coverage report --format=markdown >> $GITHUB_STEP_SUMMARY
+

.github/workflows/autofix.yml

@@ -1,9 +1,7 @@
 name: autofix.ci
 on:
-  workflow_call:
   pull_request:
-  push:
-    branches: [ "main" ]
+    branches: ["main"]
 permissions:
   contents: read
 
@@ -15,14 +13,71 @@ jobs:
       - uses: actions/checkout@v4
 
       # Use uv to ensure we have the same ruff version in CI and locally.
-      - uses: astral-sh/setup-uv@7edac99f961f18b581bbd960d59d049f04c0002f 
+      - uses: astral-sh/setup-uv@v6
+        with:
+          python-version: "3.11"
       - run: |
           cd api
           uv sync --dev
+          # fmt first to avoid line too long
+          uv run ruff format ..
           # Fix lint errors
-          uv run ruff check --fix-only .
+          uv run ruff check --fix .
           # Format code
-          uv run ruff format .
+          uv run ruff format ..
+
+      - name: ast-grep
+        run: |
+          uvx --from ast-grep-cli sg --pattern 'db.session.query($WHATEVER).filter($HERE)' --rewrite 'db.session.query($WHATEVER).where($HERE)' -l py --update-all
+          uvx --from ast-grep-cli sg --pattern 'session.query($WHATEVER).filter($HERE)' --rewrite 'session.query($WHATEVER).where($HERE)' -l py --update-all
+          uvx --from ast-grep-cli sg -p '$A = db.Column($$$B)' -r '$A = mapped_column($$$B)' -l py --update-all
+          uvx --from ast-grep-cli sg -p '$A : $T = db.Column($$$B)' -r '$A : $T = mapped_column($$$B)' -l py --update-all
+          # Convert Optional[T] to T | None (ignoring quoted types)
+          cat > /tmp/optional-rule.yml << 'EOF'
+          id: convert-optional-to-union
+          language: python
+          rule:
+            kind: generic_type
+            all:
+              - has:
+                  kind: identifier
+                  pattern: Optional
+              - has:
+                  kind: type_parameter
+                  has:
+                    kind: type
+                    pattern: $T
+          fix: $T | None
+          EOF
+          uvx --from ast-grep-cli sg scan --inline-rules "$(cat /tmp/optional-rule.yml)" --update-all
+          # Fix forward references that were incorrectly converted (Python doesn't support "Type" | None syntax)
+          find . -name "*.py" -type f -exec sed -i.bak -E 's/"([^"]+)" \| None/Optional["\1"]/g; s/'"'"'([^'"'"']+)'"'"' \| None/Optional['"'"'\1'"'"']/g' {} \;
+          find . -name "*.py.bak" -type f -delete
+
+      - name: mdformat
+        run: |
+          uvx mdformat .
+
+      - name: Install pnpm
+        uses: pnpm/action-setup@v4
+        with:
+          package_json_file: web/package.json
+          run_install: false
+
+      - name: Setup NodeJS
+        uses: actions/setup-node@v4
+        with:
+          node-version: 22
+          cache: pnpm
+          cache-dependency-path: ./web/package.json
+
+      - name: Web dependencies
+        working-directory: ./web
+        run: pnpm install --frozen-lockfile
+
+      - name: oxlint
+        working-directory: ./web
+        run: |
+          pnpx oxlint --fix
 
       - uses: autofix-ci/action@635ffb0c9798bd160680f18fd73371e355b85f27
-

.github/workflows/build-push.yml

@@ -4,10 +4,10 @@ on:
   push:
     branches:
       - "main"
-      - "deploy/dev"
-      - "deploy/enterprise"
+      - "deploy/**"
       - "build/**"
       - "release/e-*"
+      - "hotfix/**"
     tags:
       - "*"
 

.github/workflows/db-migration-test.yml

@@ -1,13 +1,7 @@
 name: DB Migration Test
 
 on:
-  pull_request:
-    branches:
-      - main
-      - plugins/beta
-    paths:
-      - api/migrations/**
-      - .github/workflows/db-migration-test.yml
+  workflow_call:
 
 concurrency:
   group: db-migration-test-${{ github.ref }}
@@ -25,12 +19,20 @@ jobs:
           persist-credentials: false
 
       - name: Setup UV and Python
-        uses: ./.github/actions/setup-uv
+        uses: astral-sh/setup-uv@v6
         with:
-          uv-lockfile: api/uv.lock
+          enable-cache: true
+          python-version: "3.12"
+          cache-dependency-glob: api/uv.lock
 
       - name: Install dependencies
         run: uv sync --project api
+      - name: Ensure Offline migration are supported
+        run: |
+          # upgrade
+          uv run --directory api flask db upgrade 'base:head' --sql
+          # downgrade
+          uv run --directory api flask db downgrade 'head:base' --sql
 
       - name: Prepare middleware env
         run: |

.github/workflows/deploy-dev.yml

@@ -12,7 +12,8 @@ jobs:
   deploy:
     runs-on: ubuntu-latest
     if: |
-      github.event.workflow_run.conclusion == 'success'
+      github.event.workflow_run.conclusion == 'success' &&
+      github.event.workflow_run.head_branch == 'deploy/dev'
     steps:
       - name: Deploy to server
         uses: appleboy/ssh-action@v0.1.8

.github/workflows/deploy-enterprise.yml

@@ -19,11 +19,23 @@ jobs:
       github.event.workflow_run.head_branch == 'deploy/enterprise'
 
     steps:
-      - name: Deploy to server
-        uses: appleboy/ssh-action@v0.1.8
-        with:
-          host: ${{ secrets.ENTERPRISE_SSH_HOST }}
-          username: ${{ secrets.ENTERPRISE_SSH_USER }}
-          password: ${{ secrets.ENTERPRISE_SSH_PASSWORD }}
-          script: |
-            ${{ vars.ENTERPRISE_SSH_SCRIPT || secrets.ENTERPRISE_SSH_SCRIPT }}
+      - name: trigger deployments
+        env:
+          DEV_ENV_ADDRS: ${{ vars.DEV_ENV_ADDRS }}
+          DEPLOY_SECRET: ${{ secrets.DEPLOY_SECRET }}
+        run: |
+          IFS=',' read -ra ENDPOINTS <<< "${DEV_ENV_ADDRS:-}"
+          BODY='{"project":"dify-api","tag":"deploy-enterprise"}'
+
+          for ENDPOINT in "${ENDPOINTS[@]}"; do
+            ENDPOINT="$(echo "$ENDPOINT" | xargs)"
+            [ -z "$ENDPOINT" ] && continue
+
+            API_SIGNATURE=$(printf '%s' "$BODY" | openssl dgst -sha256 -hmac "$DEPLOY_SECRET" | awk '{print "sha256="$2}')
+
+            curl -sSf -X POST \
+              -H "Content-Type: application/json" \
+              -H "X-Hub-Signature-256: $API_SIGNATURE" \
+              -d "$BODY" \
+              "$ENDPOINT"
+          done

.github/workflows/deploy-trigger-dev.yml

@@ -1,4 +1,4 @@
-name: Deploy RAG Dev
+name: Deploy Trigger Dev
 
 permissions:
   contents: read
@@ -7,7 +7,7 @@ on:
   workflow_run:
     workflows: ["Build and Push API & Web"]
     branches:
-      - "deploy/rag-dev"
+      - "deploy/trigger-dev"
     types:
       - completed
 
@@ -16,12 +16,12 @@ jobs:
     runs-on: ubuntu-latest
     if: |
       github.event.workflow_run.conclusion == 'success' &&
-      github.event.workflow_run.head_branch == 'deploy/rag-dev'
+      github.event.workflow_run.head_branch == 'deploy/trigger-dev'
     steps:
       - name: Deploy to server
         uses: appleboy/ssh-action@v0.1.8
         with:
-          host: ${{ secrets.RAG_SSH_HOST }}
+          host: ${{ secrets.TRIGGER_SSH_HOST }}
           username: ${{ secrets.SSH_USER }}
           key: ${{ secrets.SSH_PRIVATE_KEY }}
           script: |

.github/workflows/expose_service_ports.sh

@@ -1,6 +1,7 @@
 #!/bin/bash
 
 yq eval '.services.weaviate.ports += ["8080:8080"]' -i docker/docker-compose.yaml
+yq eval '.services.weaviate.ports += ["50051:50051"]' -i docker/docker-compose.yaml
 yq eval '.services.qdrant.ports += ["6333:6333"]' -i docker/docker-compose.yaml
 yq eval '.services.chroma.ports += ["8000:8000"]' -i docker/docker-compose.yaml
 yq eval '.services["milvus-standalone"].ports += ["19530:19530"]' -i docker/docker-compose.yaml
@@ -13,4 +14,4 @@ yq eval '.services.tidb.ports += ["4000:4000"]' -i docker/tidb/docker-compose.ya
 yq eval '.services.oceanbase.ports += ["2881:2881"]' -i docker/docker-compose.yaml
 yq eval '.services.opengauss.ports += ["6600:6600"]' -i docker/docker-compose.yaml
 
-echo "Ports exposed for sandbox, weaviate, tidb, qdrant, chroma, milvus, pgvector, pgvecto-rs, elasticsearch, couchbase, opengauss"
+echo "Ports exposed for sandbox, weaviate (HTTP 8080, gRPC 50051), tidb, qdrant, chroma, milvus, pgvector, pgvecto-rs, elasticsearch, couchbase, opengauss"

.github/workflows/main-ci.yml

@@ -0,0 +1,78 @@
+name: Main CI Pipeline
+
+on:
+  pull_request:
+    branches: ["main"]
+  push:
+    branches: ["main"]
+
+permissions:
+  contents: write
+  pull-requests: write
+  checks: write
+  statuses: write
+
+concurrency:
+  group: main-ci-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+jobs:
+  # Check which paths were changed to determine which tests to run
+  check-changes:
+    name: Check Changed Files
+    runs-on: ubuntu-latest
+    outputs:
+      api-changed: ${{ steps.changes.outputs.api }}
+      web-changed: ${{ steps.changes.outputs.web }}
+      vdb-changed: ${{ steps.changes.outputs.vdb }}
+      migration-changed: ${{ steps.changes.outputs.migration }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dorny/paths-filter@v3
+        id: changes
+        with:
+          filters: |
+            api:
+              - 'api/**'
+              - 'docker/**'
+              - '.github/workflows/api-tests.yml'
+            web:
+              - 'web/**'
+            vdb:
+              - 'api/core/rag/datasource/**'
+              - 'docker/**'
+              - '.github/workflows/vdb-tests.yml'
+              - 'api/uv.lock'
+              - 'api/pyproject.toml'
+            migration:
+              - 'api/migrations/**'
+              - '.github/workflows/db-migration-test.yml'
+
+  # Run tests in parallel
+  api-tests:
+    name: API Tests
+    needs: check-changes
+    if: needs.check-changes.outputs.api-changed == 'true'
+    uses: ./.github/workflows/api-tests.yml
+
+  web-tests:
+    name: Web Tests
+    needs: check-changes
+    if: needs.check-changes.outputs.web-changed == 'true'
+    uses: ./.github/workflows/web-tests.yml
+
+  style-check:
+    name: Style Check
+    uses: ./.github/workflows/style.yml
+
+  vdb-tests:
+    name: VDB Tests
+    needs: check-changes
+    if: needs.check-changes.outputs.vdb-changed == 'true'
+    uses: ./.github/workflows/vdb-tests.yml
+
+  db-migration-test:
+    name: DB Migration Test
+    needs: check-changes
+    if: needs.check-changes.outputs.migration-changed == 'true'
+    uses: ./.github/workflows/db-migration-test.yml

.github/workflows/style.yml

@@ -1,9 +1,7 @@
 name: Style check
 
 on:
-  pull_request:
-    branches:
-      - main
+  workflow_call:
 
 concurrency:
   group: style-${{ github.head_ref || github.run_id }}
@@ -14,7 +12,6 @@ permissions:
   statuses: write
   contents: read
 
-
 jobs:
   python-style:
     name: Python Style
@@ -36,30 +33,32 @@ jobs:
 
       - name: Setup UV and Python
         if: steps.changed-files.outputs.any_changed == 'true'
-        uses: ./.github/actions/setup-uv
+        uses: astral-sh/setup-uv@v6
         with:
-          uv-lockfile: api/uv.lock
           enable-cache: false
+          python-version: "3.12"
+          cache-dependency-glob: api/uv.lock
 
       - name: Install dependencies
         if: steps.changed-files.outputs.any_changed == 'true'
         run: uv sync --project api --dev
 
-      - name: Ruff check
+      - name: Run Import Linter
         if: steps.changed-files.outputs.any_changed == 'true'
-        run: |
-          uv run --directory api ruff --version
-          uv run --directory api ruff check ./
-          uv run --directory api ruff format --check ./
+        run: uv run --directory api --dev lint-imports
+
+      - name: Run Basedpyright Checks
+        if: steps.changed-files.outputs.any_changed == 'true'
+        run: dev/basedpyright-check
+
+      - name: Run Mypy Type Checks
+        if: steps.changed-files.outputs.any_changed == 'true'
+        run: uv --directory api run mypy --exclude-gitignore --exclude 'tests/' --exclude 'migrations/' --check-untyped-defs --disable-error-code=import-untyped .
 
       - name: Dotenv check
         if: steps.changed-files.outputs.any_changed == 'true'
         run: uv run --project api dotenv-linter ./api/.env.example ./web/.env.example
 
-      - name: Lint hints
-        if: failure()
-        run: echo "Please run 'dev/reformat' to fix the fixable linting errors."
-
   web-style:
     name: Web Style
     runs-on: ubuntu-latest
@@ -82,7 +81,7 @@ jobs:
       - name: Install pnpm
         uses: pnpm/action-setup@v4
         with:
-          version: 10
+          package_json_file: web/package.json
           run_install: false
 
       - name: Setup NodeJS
@@ -95,11 +94,14 @@ jobs:
 
       - name: Web dependencies
         if: steps.changed-files.outputs.any_changed == 'true'
+        working-directory: ./web
         run: pnpm install --frozen-lockfile
 
       - name: Web style check
         if: steps.changed-files.outputs.any_changed == 'true'
-        run: pnpm run lint
+        working-directory: ./web
+        run: |
+          pnpm run lint
 
   docker-compose-template:
     name: Docker Compose Template

.github/workflows/translate-i18n-base-on-english.yml

@@ -46,7 +46,7 @@ jobs:
       - name: Install pnpm
         uses: pnpm/action-setup@v4
         with:
-          version: 10
+          package_json_file: web/package.json
           run_install: false
 
       - name: Set up Node.js
@@ -59,18 +59,30 @@ jobs:
 
       - name: Install dependencies
         if: env.FILES_CHANGED == 'true'
+        working-directory: ./web
         run: pnpm install --frozen-lockfile
 
       - name: Generate i18n translations
         if: env.FILES_CHANGED == 'true'
+        working-directory: ./web
         run: pnpm run auto-gen-i18n ${{ env.FILE_ARGS }}
 
+      - name: Generate i18n type definitions
+        if: env.FILES_CHANGED == 'true'
+        working-directory: ./web
+        run: pnpm run gen:i18n-types
+
       - name: Create Pull Request
         if: env.FILES_CHANGED == 'true'
         uses: peter-evans/create-pull-request@v6
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
-          commit-message: Update i18n files based on en-US changes
-          title: 'chore: translate i18n files'
-          body: This PR was automatically created to update i18n files based on changes in en-US locale.
+          commit-message: Update i18n files and type definitions based on en-US changes
+          title: 'chore: translate i18n files and update type definitions'
+          body: |
+            This PR was automatically created to update i18n files and TypeScript type definitions based on changes in en-US locale.
+            
+            **Changes included:**
+            - Updated translation files for all locales
+            - Regenerated TypeScript type definitions for type safety
           branch: chore/automated-i18n-updates

.github/workflows/vdb-tests.yml

@@ -1,15 +1,7 @@
 name: Run VDB Tests
 
 on:
-  pull_request:
-    branches:
-      - main
-    paths:
-      - api/core/rag/datasource/**
-      - docker/**
-      - .github/workflows/vdb-tests.yml
-      - api/uv.lock
-      - api/pyproject.toml
+  workflow_call:
 
 concurrency:
   group: vdb-tests-${{ github.head_ref || github.run_id }}
@@ -39,10 +31,11 @@ jobs:
           remove_tool_cache: true
 
       - name: Setup UV and Python
-        uses: ./.github/actions/setup-uv
+        uses: astral-sh/setup-uv@v6
         with:
+          enable-cache: true
           python-version: ${{ matrix.python-version }}
-          uv-lockfile: api/uv.lock
+          cache-dependency-glob: api/uv.lock
 
       - name: Check UV lockfile
         run: uv lock --project api --check

.github/workflows/web-tests.yml

@@ -1,11 +1,7 @@
 name: Web Tests
 
 on:
-  pull_request:
-    branches:
-      - main
-    paths:
-      - web/**
+  workflow_call:
 
 concurrency:
   group: web-tests-${{ github.head_ref || github.run_id }}
@@ -35,7 +31,7 @@ jobs:
         if: steps.changed-files.outputs.any_changed == 'true'
         uses: pnpm/action-setup@v4
         with:
-          version: 10
+          package_json_file: web/package.json
           run_install: false
 
       - name: Setup Node.js
@@ -48,8 +44,15 @@ jobs:
 
       - name: Install dependencies
         if: steps.changed-files.outputs.any_changed == 'true'
+        working-directory: ./web
         run: pnpm install --frozen-lockfile
 
+      - name: Check i18n types synchronization
+        if: steps.changed-files.outputs.any_changed == 'true'
+        working-directory: ./web
+        run: pnpm run check:i18n-types
+
       - name: Run tests
         if: steps.changed-files.outputs.any_changed == 'true'
+        working-directory: ./web
         run: pnpm test

.gitignore

@@ -123,10 +123,12 @@ venv.bak/
 # mkdocs documentation
 /site
 
-# mypy
+# type checking
 .mypy_cache/
 .dmypy.json
 dmypy.json
+pyrightconfig.json
+!api/pyrightconfig.json
 
 # Pyre type checker
 .pyre/
@@ -195,8 +197,10 @@ sdks/python-client/dify_client.egg-info
 .vscode/*
 !.vscode/launch.json.template
 !.vscode/README.md
-pyrightconfig.json
 api/.vscode
+web/.vscode
+# vscode Code History Extension
+.history
 
 .idea/
 
@@ -212,7 +216,22 @@ mise.toml
 # Next.js build output
 .next/
 
+# PWA generated files
+web/public/sw.js
+web/public/sw.js.map
+web/public/workbox-*.js
+web/public/workbox-*.js.map
+web/public/fallback-*.js
+
 # AI Assistant
 .roo/
 api/.env.backup
 /clickzetta
+
+# Benchmark
+scripts/stress-test/setup/config/
+scripts/stress-test/reports/
+
+# mcp
+.playwright-mcp/
+.serena/

.mcp.json

@@ -0,0 +1,34 @@
+{
+    "mcpServers": {
+      "context7": {
+        "type": "http",
+        "url": "https://mcp.context7.com/mcp"
+      },
+      "sequential-thinking": {
+        "type": "stdio",
+        "command": "npx",
+        "args": ["-y", "@modelcontextprotocol/server-sequential-thinking"],
+        "env": {}
+      },
+      "github": {
+        "type": "stdio",
+        "command": "npx",
+        "args": ["-y", "@modelcontextprotocol/server-github"],
+        "env": {
+          "GITHUB_PERSONAL_ACCESS_TOKEN": "${GITHUB_PERSONAL_ACCESS_TOKEN}"
+        }
+      },
+      "fetch": {
+        "type": "stdio",
+        "command": "uvx",
+        "args": ["mcp-server-fetch"],
+        "env": {}
+      },
+      "playwright": {
+        "type": "stdio",
+        "command": "npx",
+        "args": ["-y", "@playwright/mcp@latest"],
+        "env": {}
+      }
+    }
+  }

.vscode/README.md

@@ -4,10 +4,10 @@ This `launch.json.template` file provides various debug configurations for the D
 
 ## How to Use
 
-1.  **Create `launch.json`**: If you don't have one, create a file named `launch.json` inside the `.vscode` directory.
-2.  **Copy Content**: Copy the entire content from `launch.json.template` into your newly created `launch.json` file.
-3.  **Select Debug Configuration**: Go to the Run and Debug view in VS Code / Cursor (Ctrl+Shift+D or Cmd+Shift+D).
-4.  **Start Debugging**: Select the desired configuration from the dropdown menu and click the green play button.
+1. **Create `launch.json`**: If you don't have one, create a file named `launch.json` inside the `.vscode` directory.
+1. **Copy Content**: Copy the entire content from `launch.json.template` into your newly created `launch.json` file.
+1. **Select Debug Configuration**: Go to the Run and Debug view in VS Code / Cursor (Ctrl+Shift+D or Cmd+Shift+D).
+1. **Start Debugging**: Select the desired configuration from the dropdown menu and click the green play button.
 
 ## Tips
 

AGENTS.md

@@ -0,0 +1,54 @@
+# AGENTS.md
+
+## Project Overview
+
+Dify is an open-source platform for developing LLM applications with an intuitive interface combining agentic AI workflows, RAG pipelines, agent capabilities, and model management.
+
+The codebase is split into:
+
+- **Backend API** (`/api`): Python Flask application organized with Domain-Driven Design
+- **Frontend Web** (`/web`): Next.js 15 application using TypeScript and React 19
+- **Docker deployment** (`/docker`): Containerized deployment configurations
+
+## Backend Workflow
+
+- Run backend CLI commands through `uv run --project api <command>`.
+
+- Before submission, all backend modifications must pass local checks: `make lint`, `make type-check`, and `uv run --project api --dev dev/pytest/pytest_unit_tests.sh`.
+
+- Use Makefile targets for linting and formatting; `make lint` and `make type-check` cover the required checks.
+
+- Integration tests are CI-only and are not expected to run in the local environment.
+
+## Frontend Workflow
+
+```bash
+cd web
+pnpm lint
+pnpm lint:fix
+pnpm test
+```
+
+## Testing & Quality Practices
+
+- Follow TDD: red → green → refactor.
+- Use `pytest` for backend tests with Arrange-Act-Assert structure.
+- Enforce strong typing; avoid `Any` and prefer explicit type annotations.
+- Write self-documenting code; only add comments that explain intent.
+
+## Language Style
+
+- **Python**: Keep type hints on functions and attributes, and implement relevant special methods (e.g., `__repr__`, `__str__`).
+- **TypeScript**: Use the strict config, lean on ESLint + Prettier workflows, and avoid `any` types.
+
+## General Practices
+
+- Prefer editing existing files; add new documentation only when requested.
+- Inject dependencies through constructors and preserve clean architecture boundaries.
+- Handle errors with domain-specific exceptions at the correct layer.
+
+## Project Conventions
+
+- Backend architecture adheres to DDD and Clean Architecture principles.
+- Async work runs through Celery with Redis as the broker.
+- Frontend user-facing strings must use `web/i18n/en-US/`; avoid hardcoded text.

CLAUDE.md

@@ -0,0 +1 @@
+AGENTS.md

CONTRIBUTING.md

@@ -34,11 +34,11 @@ Don't forget to link an existing issue or open a new issue in the PR's descripti
 
 How we prioritize:
 
-  | Issue Type                                                   | Priority        |
-  | ------------------------------------------------------------ | --------------- |
-  | Bugs in core functions (cloud service, cannot login, applications not working, security loopholes) | Critical        |
-  | Non-critical bugs, performance boosts                        | Medium Priority |
-  | Minor fixes (typos, confusing but working UI)                | Low Priority    |
+| Issue Type | Priority |
+| ------------------------------------------------------------ | --------------- |
+| Bugs in core functions (cloud service, cannot login, applications not working, security loopholes) | Critical |
+| Non-critical bugs, performance boosts | Medium Priority |
+| Minor fixes (typos, confusing but working UI) | Low Priority |
 
 ### Feature requests
 
@@ -52,23 +52,25 @@ How we prioritize:
 
 How we prioritize:
 
-  | Feature Type                                                 | Priority        |
-  | ------------------------------------------------------------ | --------------- |
-  | High-Priority Features as being labeled by a team member     | High Priority   |
-  | Popular feature requests from our [community feedback board](https://github.com/langgenius/dify/discussions/categories/feedbacks) | Medium Priority |
-  | Non-core features and minor enhancements                     | Low Priority    |
-  | Valuable but not immediate                                   | Future-Feature  |
+| Feature Type | Priority |
+| ------------------------------------------------------------ | --------------- |
+| High-Priority Features as being labeled by a team member | High Priority |
+| Popular feature requests from our [community feedback board](https://github.com/langgenius/dify/discussions/categories/feedbacks) | Medium Priority |
+| Non-core features and minor enhancements | Low Priority |
+| Valuable but not immediate | Future-Feature |
+
 ## Submitting your PR
 
 ### Pull Request Process
 
 1. Fork the repository
-2. Before you draft a PR, please create an issue to discuss the changes you want to make
-3. Create a new branch for your changes
-4. Please add tests for your changes accordingly
-5. Ensure your code passes the existing tests
-6. Please link the issue in the PR description, `fixes #<issue_number>`
-7. Get merged! 
+1. Before you draft a PR, please create an issue to discuss the changes you want to make
+1. Create a new branch for your changes
+1. Please add tests for your changes accordingly
+1. Ensure your code passes the existing tests
+1. Please link the issue in the PR description, `fixes #<issue_number>`
+1. Get merged!
+
 ### Setup the project
 
 #### Frontend
@@ -82,12 +84,14 @@ For setting up the backend service, kindly refer to our detailed [instructions](
 #### Other things to note
 
 We recommend reviewing this document carefully before proceeding with the setup, as it contains essential information about:
+
 - Prerequisites and dependencies
 - Installation steps
 - Configuration details
 - Common troubleshooting tips
 
 Feel free to reach out if you encounter any issues during the setup process.
+
 ## Getting Help
 
 If you ever get stuck or get a burning question while contributing, simply shoot your queries our way via the related GitHub issue, or hop onto our [Discord](https://discord.gg/8Tpq4AcN9c) for a quick chat.

Makefile

@@ -4,6 +4,72 @@ WEB_IMAGE=$(DOCKER_REGISTRY)/dify-web
 API_IMAGE=$(DOCKER_REGISTRY)/dify-api
 VERSION=latest
 
+# Default target - show help
+.DEFAULT_GOAL := help
+
+# Backend Development Environment Setup
+.PHONY: dev-setup prepare-docker prepare-web prepare-api
+
+# Dev setup target
+dev-setup: prepare-docker prepare-web prepare-api
+	@echo "✅ Backend development environment setup complete!"
+
+# Step 1: Prepare Docker middleware
+prepare-docker:
+	@echo "🐳 Setting up Docker middleware..."
+	@cp -n docker/middleware.env.example docker/middleware.env 2>/dev/null || echo "Docker middleware.env already exists"
+	@cd docker && docker compose -f docker-compose.middleware.yaml --env-file middleware.env -p dify-middlewares-dev up -d
+	@echo "✅ Docker middleware started"
+
+# Step 2: Prepare web environment
+prepare-web:
+	@echo "🌐 Setting up web environment..."
+	@cp -n web/.env.example web/.env 2>/dev/null || echo "Web .env already exists"
+	@cd web && pnpm install
+	@echo "✅ Web environment prepared (not started)"
+
+# Step 3: Prepare API environment
+prepare-api:
+	@echo "🔧 Setting up API environment..."
+	@cp -n api/.env.example api/.env 2>/dev/null || echo "API .env already exists"
+	@cd api && uv sync --dev
+	@cd api && uv run flask db upgrade
+	@echo "✅ API environment prepared (not started)"
+
+# Clean dev environment
+dev-clean:
+	@echo "⚠️  Stopping Docker containers..."
+	@cd docker && docker compose -f docker-compose.middleware.yaml --env-file middleware.env -p dify-middlewares-dev down
+	@echo "🗑️  Removing volumes..."
+	@rm -rf docker/volumes/db
+	@rm -rf docker/volumes/redis
+	@rm -rf docker/volumes/plugin_daemon
+	@rm -rf docker/volumes/weaviate
+	@rm -rf api/storage
+	@echo "✅ Cleanup complete"
+
+# Backend Code Quality Commands
+format:
+	@echo "🎨 Running ruff format..."
+	@uv run --project api --dev ruff format ./api
+	@echo "✅ Code formatting complete"
+
+check:
+	@echo "🔍 Running ruff check..."
+	@uv run --project api --dev ruff check ./api
+	@echo "✅ Code check complete"
+
+lint:
+	@echo "🔧 Running ruff format, check with fixes, and import linter..."
+	@uv run --project api --dev sh -c 'ruff format ./api && ruff check --fix ./api'
+	@uv run --directory api --dev lint-imports
+	@echo "✅ Linting complete"
+
+type-check:
+	@echo "📝 Running type check with basedpyright..."
+	@uv run --directory api --dev basedpyright
+	@echo "✅ Type check complete"
+
 # Build Docker images
 build-web:
 	@echo "Building web Docker image: $(WEB_IMAGE):$(VERSION)..."
@@ -39,5 +105,27 @@ build-push-web: build-web push-web
 build-push-all: build-all push-all
 	@echo "All Docker images have been built and pushed."
 
+# Help target
+help:
+	@echo "Development Setup Targets:"
+	@echo "  make dev-setup      - Run all setup steps for backend dev environment"
+	@echo "  make prepare-docker - Set up Docker middleware"
+	@echo "  make prepare-web    - Set up web environment"
+	@echo "  make prepare-api    - Set up API environment"
+	@echo "  make dev-clean      - Stop Docker middleware containers"
+	@echo ""
+	@echo "Backend Code Quality:"
+	@echo "  make format         - Format code with ruff"
+	@echo "  make check          - Check code with ruff"
+	@echo "  make lint           - Format and fix code with ruff"
+	@echo "  make type-check     - Run type checking with basedpyright"
+	@echo ""
+	@echo "Docker Build Targets:"
+	@echo "  make build-web      - Build web Docker image"
+	@echo "  make build-api      - Build API Docker image"
+	@echo "  make build-all      - Build all Docker images"
+	@echo "  make push-all       - Push all Docker images"
+	@echo "  make build-push-all - Build and push all Docker images"
+
 # Phony targets
-.PHONY: build-web build-api push-web push-api build-all push-all build-push-all
+.PHONY: build-web build-api push-web push-api build-all push-all build-push-all dev-setup prepare-docker prepare-web prepare-api dev-clean help format check lint type-check

README.md

@@ -40,18 +40,18 @@
 
 <p align="center">
   <a href="./README.md"><img alt="README in English" src="https://img.shields.io/badge/English-d9d9d9"></a>
-  <a href="./README_TW.md"><img alt="繁體中文文件" src="https://img.shields.io/badge/繁體中文-d9d9d9"></a>
-  <a href="./README_CN.md"><img alt="简体中文版自述文件" src="https://img.shields.io/badge/简体中文-d9d9d9"></a>
-  <a href="./README_JA.md"><img alt="日本語のREADME" src="https://img.shields.io/badge/日本語-d9d9d9"></a>
-  <a href="./README_ES.md"><img alt="README en Español" src="https://img.shields.io/badge/Español-d9d9d9"></a>
-  <a href="./README_FR.md"><img alt="README en Français" src="https://img.shields.io/badge/Français-d9d9d9"></a>
-  <a href="./README_KL.md"><img alt="README tlhIngan Hol" src="https://img.shields.io/badge/Klingon-d9d9d9"></a>
-  <a href="./README_KR.md"><img alt="README in Korean" src="https://img.shields.io/badge/한국어-d9d9d9"></a>
-  <a href="./README_AR.md"><img alt="README بالعربية" src="https://img.shields.io/badge/العربية-d9d9d9"></a>
-  <a href="./README_TR.md"><img alt="Türkçe README" src="https://img.shields.io/badge/Türkçe-d9d9d9"></a>
-  <a href="./README_VI.md"><img alt="README Tiếng Việt" src="https://img.shields.io/badge/Ti%E1%BA%BFng%20Vi%E1%BB%87t-d9d9d9"></a>
-  <a href="./README_DE.md"><img alt="README in Deutsch" src="https://img.shields.io/badge/German-d9d9d9"></a>
-  <a href="./README_BN.md"><img alt="README in বাংলা" src="https://img.shields.io/badge/বাংলা-d9d9d9"></a>
+  <a href="./docs/zh-TW/README.md"><img alt="繁體中文文件" src="https://img.shields.io/badge/繁體中文-d9d9d9"></a>
+  <a href="./docs/zh-CN/README.md"><img alt="简体中文文件" src="https://img.shields.io/badge/简体中文-d9d9d9"></a>
+  <a href="./docs/ja-JP/README.md"><img alt="日本語のREADME" src="https://img.shields.io/badge/日本語-d9d9d9"></a>
+  <a href="./docs/es-ES/README.md"><img alt="README en Español" src="https://img.shields.io/badge/Español-d9d9d9"></a>
+  <a href="./docs/fr-FR/README.md"><img alt="README en Français" src="https://img.shields.io/badge/Français-d9d9d9"></a>
+  <a href="./docs/tlh/README.md"><img alt="README tlhIngan Hol" src="https://img.shields.io/badge/Klingon-d9d9d9"></a>
+  <a href="./docs/ko-KR/README.md"><img alt="README in Korean" src="https://img.shields.io/badge/한국어-d9d9d9"></a>
+  <a href="./docs/ar-SA/README.md"><img alt="README بالعربية" src="https://img.shields.io/badge/العربية-d9d9d9"></a>
+  <a href="./docs/tr-TR/README.md"><img alt="Türkçe README" src="https://img.shields.io/badge/Türkçe-d9d9d9"></a>
+  <a href="./docs/vi-VN/README.md"><img alt="README Tiếng Việt" src="https://img.shields.io/badge/Ti%E1%BA%BFng%20Vi%E1%BB%87t-d9d9d9"></a>
+  <a href="./docs/de-DE/README.md"><img alt="README in Deutsch" src="https://img.shields.io/badge/German-d9d9d9"></a>
+  <a href="./docs/bn-BD/README.md"><img alt="README in বাংলা" src="https://img.shields.io/badge/বাংলা-d9d9d9"></a>
 </p>
 
 Dify is an open-source platform for developing LLM applications. Its intuitive interface combines agentic AI workflows, RAG pipelines, agent capabilities, model management, observability features, and more—allowing you to quickly move from prototype to production.
@@ -107,74 +107,6 @@ Monitor and analyze application logs and performance over time. You could contin
 **7. Backend-as-a-Service**:
 All of Dify's offerings come with corresponding APIs, so you could effortlessly integrate Dify into your own business logic.
 
-## Feature Comparison
-
-<table style="width: 100%;">
-  <tr>
-    <th align="center">Feature</th>
-    <th align="center">Dify.AI</th>
-    <th align="center">LangChain</th>
-    <th align="center">Flowise</th>
-    <th align="center">OpenAI Assistants API</th>
-  </tr>
-  <tr>
-    <td align="center">Programming Approach</td>
-    <td align="center">API + App-oriented</td>
-    <td align="center">Python Code</td>
-    <td align="center">App-oriented</td>
-    <td align="center">API-oriented</td>
-  </tr>
-  <tr>
-    <td align="center">Supported LLMs</td>
-    <td align="center">Rich Variety</td>
-    <td align="center">Rich Variety</td>
-    <td align="center">Rich Variety</td>
-    <td align="center">OpenAI-only</td>
-  </tr>
-  <tr>
-    <td align="center">RAG Engine</td>
-    <td align="center">✅</td>
-    <td align="center">✅</td>
-    <td align="center">✅</td>
-    <td align="center">✅</td>
-  </tr>
-  <tr>
-    <td align="center">Agent</td>
-    <td align="center">✅</td>
-    <td align="center">✅</td>
-    <td align="center">❌</td>
-    <td align="center">✅</td>
-  </tr>
-  <tr>
-    <td align="center">Workflow</td>
-    <td align="center">✅</td>
-    <td align="center">❌</td>
-    <td align="center">✅</td>
-    <td align="center">❌</td>
-  </tr>
-  <tr>
-    <td align="center">Observability</td>
-    <td align="center">✅</td>
-    <td align="center">✅</td>
-    <td align="center">❌</td>
-    <td align="center">❌</td>
-  </tr>
-  <tr>
-    <td align="center">Enterprise Feature (SSO/Access control)</td>
-    <td align="center">✅</td>
-    <td align="center">❌</td>
-    <td align="center">❌</td>
-    <td align="center">❌</td>
-  </tr>
-  <tr>
-    <td align="center">Local Deployment</td>
-    <td align="center">✅</td>
-    <td align="center">✅</td>
-    <td align="center">✅</td>
-    <td align="center">❌</td>
-  </tr>
-</table>
-
 ## Using Dify
 
 - **Cloud </br>**
@@ -185,7 +117,8 @@ All of Dify's offerings come with corresponding APIs, so you could effortlessly
   Use our [documentation](https://docs.dify.ai) for further references and more in-depth instructions.
 
 - **Dify for enterprise / organizations</br>**
-  We provide additional enterprise-centric features. [Log your questions for us through this chatbot](https://udify.app/chat/22L1zSxg6yW1cWQg) or [send us an email](mailto:business@dify.ai?subject=[GitHub]Business%20License%20Inquiry) to discuss enterprise needs. </br>
+  We provide additional enterprise-centric features. [Log your questions for us through this chatbot](https://udify.app/chat/22L1zSxg6yW1cWQg) or [send us an email](mailto:business@dify.ai?subject=%5BGitHub%5DBusiness%20License%20Inquiry) to discuss enterprise needs. </br>
+
   > For startups and small businesses using AWS, check out [Dify Premium on AWS Marketplace](https://aws.amazon.com/marketplace/pp/prodview-t22mebxzwjhu6) and deploy it to your own AWS VPC with one click. It's an affordable AMI offering with the option to create apps with custom logo and branding.
 
 ## Staying ahead
@@ -196,8 +129,18 @@ Star Dify on GitHub and be instantly notified of new releases.
 
 ## Advanced Setup
 
+### Custom configurations
+
 If you need to customize the configuration, please refer to the comments in our [.env.example](docker/.env.example) file and update the corresponding values in your `.env` file. Additionally, you might need to make adjustments to the `docker-compose.yaml` file itself, such as changing image versions, port mappings, or volume mounts, based on your specific deployment environment and requirements. After making any changes, please re-run `docker-compose up -d`. You can find the full list of available environment variables [here](https://docs.dify.ai/getting-started/install-self-hosted/environments).
 
+### Metrics Monitoring with Grafana
+
+Import the dashboard to Grafana, using Dify's PostgreSQL database as data source, to monitor metrics in granularity of apps, tenants, messages, and more.
+
+- [Grafana Dashboard by @bowenliang123](https://github.com/bowenliang123/dify-grafana-dashboard)
+
+### Deployment with Kubernetes
+
 If you'd like to configure a highly-available setup, there are community-contributed [Helm Charts](https://helm.sh/) and YAML files which allow Dify to be deployed on Kubernetes.
 
 - [Helm Chart by @LeoQuote](https://github.com/douban/charts/tree/master/charts/dify)
@@ -225,20 +168,20 @@ Deploy Dify to AWS with [CDK](https://aws.amazon.com/cdk/)
 
 ##### AWS
 
-- [AWS CDK by @KevinZhao](https://github.com/aws-samples/solution-for-deploying-dify-on-aws)
+- [AWS CDK by @KevinZhao (EKS based)](https://github.com/aws-samples/solution-for-deploying-dify-on-aws)
+- [AWS CDK by @tmokmss (ECS based)](https://github.com/aws-samples/dify-self-hosted-on-aws)
 
 #### Using Alibaba Cloud Computing Nest
 
-Quickly deploy Dify to Alibaba cloud with [Alibaba Cloud Computing Nest](https://computenest.console.aliyun.com/service/instance/create/default?type=user&ServiceName=Dify%E7%A4%BE%E5%8C%BA%E7%89%88) 
+Quickly deploy Dify to Alibaba cloud with [Alibaba Cloud Computing Nest](https://computenest.console.aliyun.com/service/instance/create/default?type=user&ServiceName=Dify%E7%A4%BE%E5%8C%BA%E7%89%88)
 
 #### Using Alibaba Cloud Data Management
 
-One-Click deploy Dify to Alibaba Cloud with [Alibaba Cloud Data Management](https://www.alibabacloud.com/help/en/dms/dify-in-invitational-preview/) 
+One-Click deploy Dify to Alibaba Cloud with [Alibaba Cloud Data Management](https://www.alibabacloud.com/help/en/dms/dify-in-invitational-preview/)
 
 #### Deploy to AKS with Azure Devops Pipeline
 
-One-Click deploy Dify to AKS with [Azure Devops Pipeline Helm Chart by @LeoZhang](https://github.com/Ruiruiz30/Dify-helm-chart-AKS) 
-
+One-Click deploy Dify to AKS with [Azure Devops Pipeline Helm Chart by @LeoZhang](https://github.com/Ruiruiz30/Dify-helm-chart-AKS)
 
 ## Contributing
 

api/.env.example

@@ -30,6 +30,9 @@ INTERNAL_FILES_URL=http://127.0.0.1:5001
 # The time in seconds after the signature is rejected
 FILES_ACCESS_TIMEOUT=300
 
+# Collaboration mode toggle
+ENABLE_COLLABORATION_MODE=false
+
 # Access token expiration time in minutes
 ACCESS_TOKEN_EXPIRE_MINUTES=60
 
@@ -42,6 +45,15 @@ REDIS_PORT=6379
 REDIS_USERNAME=
 REDIS_PASSWORD=difyai123456
 REDIS_USE_SSL=false
+# SSL configuration for Redis (when REDIS_USE_SSL=true)
+REDIS_SSL_CERT_REQS=CERT_NONE
+# Options: CERT_NONE, CERT_OPTIONAL, CERT_REQUIRED
+REDIS_SSL_CA_CERTS=
+# Path to CA certificate file for SSL verification
+REDIS_SSL_CERTFILE=
+# Path to client certificate file for SSL authentication
+REDIS_SSL_KEYFILE=
+# Path to client private key file for SSL authentication
 REDIS_DB=0
 
 # redis Sentinel configuration.
@@ -66,6 +78,8 @@ DB_PASSWORD=difyai123456
 DB_HOST=localhost
 DB_PORT=5432
 DB_DATABASE=dify
+SQLALCHEMY_POOL_PRE_PING=true
+SQLALCHEMY_POOL_TIMEOUT=30
 
 # Storage configuration
 # use for store upload files, private keys...
@@ -293,6 +307,8 @@ BAIDU_VECTOR_DB_API_KEY=dify
 BAIDU_VECTOR_DB_DATABASE=dify
 BAIDU_VECTOR_DB_SHARD=1
 BAIDU_VECTOR_DB_REPLICAS=3
+BAIDU_VECTOR_DB_INVERTED_INDEX_ANALYZER=DEFAULT_ANALYZER
+BAIDU_VECTOR_DB_INVERTED_INDEX_PARSER_MODE=COARSE_MODE
 
 # Upstash configuration
 UPSTASH_VECTOR_URL=your-server-url
@@ -318,7 +334,7 @@ MATRIXONE_DATABASE=dify
 LINDORM_URL=http://ld-*******************-proxy-search-pub.lindorm.aliyuncs.com:30070
 LINDORM_USERNAME=admin
 LINDORM_PASSWORD=admin
-USING_UGC_INDEX=False
+LINDORM_USING_UGC=True
 LINDORM_QUERY_TIMEOUT=1
 
 # OceanBase Vector configuration
@@ -330,6 +346,15 @@ OCEANBASE_VECTOR_DATABASE=test
 OCEANBASE_MEMORY_LIMIT=6G
 OCEANBASE_ENABLE_HYBRID_SEARCH=false
 
+# AlibabaCloud MySQL Vector configuration
+ALIBABACLOUD_MYSQL_HOST=127.0.0.1
+ALIBABACLOUD_MYSQL_PORT=3306
+ALIBABACLOUD_MYSQL_USER=root
+ALIBABACLOUD_MYSQL_PASSWORD=root
+ALIBABACLOUD_MYSQL_DATABASE=dify
+ALIBABACLOUD_MYSQL_MAX_CONNECTION=5
+ALIBABACLOUD_MYSQL_HNSW_M=6
+
 # openGauss configuration
 OPENGAUSS_HOST=127.0.0.1
 OPENGAUSS_PORT=6600
@@ -395,6 +420,9 @@ SSRF_DEFAULT_TIME_OUT=5
 SSRF_DEFAULT_CONNECT_TIME_OUT=5
 SSRF_DEFAULT_READ_TIME_OUT=5
 SSRF_DEFAULT_WRITE_TIME_OUT=5
+SSRF_POOL_MAX_CONNECTIONS=100
+SSRF_POOL_MAX_KEEPALIVE_CONNECTIONS=20
+SSRF_POOL_KEEPALIVE_EXPIRY=5.0
 
 BATCH_UPLOAD_LIMIT=10
 KEYWORD_DATA_SOURCE_TYPE=database
@@ -405,10 +433,14 @@ WORKFLOW_FILE_UPLOAD_LIMIT=10
 # CODE EXECUTION CONFIGURATION
 CODE_EXECUTION_ENDPOINT=http://127.0.0.1:8194
 CODE_EXECUTION_API_KEY=dify-sandbox
+CODE_EXECUTION_SSL_VERIFY=True
+CODE_EXECUTION_POOL_MAX_CONNECTIONS=100
+CODE_EXECUTION_POOL_MAX_KEEPALIVE_CONNECTIONS=20
+CODE_EXECUTION_POOL_KEEPALIVE_EXPIRY=5.0
 CODE_MAX_NUMBER=9223372036854775807
 CODE_MIN_NUMBER=-9223372036854775808
-CODE_MAX_STRING_LENGTH=80000
-TEMPLATE_TRANSFORM_MAX_LENGTH=80000
+CODE_MAX_STRING_LENGTH=400000
+TEMPLATE_TRANSFORM_MAX_LENGTH=400000
 CODE_MAX_STRING_ARRAY_LENGTH=30
 CODE_MAX_OBJECT_ARRAY_LENGTH=30
 CODE_MAX_NUMBER_ARRAY_LENGTH=1000
@@ -448,9 +480,18 @@ INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH=4000
 WORKFLOW_MAX_EXECUTION_STEPS=500
 WORKFLOW_MAX_EXECUTION_TIME=1200
 WORKFLOW_CALL_MAX_DEPTH=5
-WORKFLOW_PARALLEL_DEPTH_LIMIT=3
 MAX_VARIABLE_SIZE=204800
 
+# GraphEngine Worker Pool Configuration
+# Minimum number of workers per GraphEngine instance (default: 1)
+GRAPH_ENGINE_MIN_WORKERS=1
+# Maximum number of workers per GraphEngine instance (default: 10)
+GRAPH_ENGINE_MAX_WORKERS=10
+# Queue depth threshold that triggers worker scale up (default: 3)
+GRAPH_ENGINE_SCALE_UP_THRESHOLD=3
+# Seconds of idle time before scaling down workers (default: 5.0)
+GRAPH_ENGINE_SCALE_DOWN_IDLE_TIME=5.0
+
 # Workflow storage configuration
 # Options: rdbms, hybrid
 # rdbms: Use only the relational database (default)
@@ -469,6 +510,13 @@ API_WORKFLOW_NODE_EXECUTION_REPOSITORY=repositories.sqlalchemy_api_workflow_node
 
 # API workflow run repository implementation
 API_WORKFLOW_RUN_REPOSITORY=repositories.sqlalchemy_api_workflow_run_repository.DifyAPISQLAlchemyWorkflowRunRepository
+# Workflow log cleanup configuration
+# Enable automatic cleanup of workflow run logs to manage database size
+WORKFLOW_LOG_CLEANUP_ENABLED=true
+# Number of days to retain workflow run logs (default: 30 days)
+WORKFLOW_LOG_RETENTION_DAYS=30
+# Batch size for workflow log cleanup operations (default: 100)
+WORKFLOW_LOG_CLEANUP_BATCH_SIZE=100
 
 # App configuration
 APP_MAX_EXECUTION_TIME=1200
@@ -513,6 +561,7 @@ ENDPOINT_URL_TEMPLATE=http://localhost:5002/e/{hook_id}
 
 # Reset password token expiry minutes
 RESET_PASSWORD_TOKEN_EXPIRY_MINUTES=5
+EMAIL_REGISTER_TOKEN_EXPIRY_MINUTES=5
 CHANGE_EMAIL_TOKEN_EXPIRY_MINUTES=5
 OWNER_TRANSFER_TOKEN_EXPIRY_MINUTES=5
 
@@ -548,3 +597,11 @@ QUEUE_MONITOR_THRESHOLD=200
 QUEUE_MONITOR_ALERT_EMAILS=
 # Monitor interval in minutes, default is 30 minutes
 QUEUE_MONITOR_INTERVAL=30
+
+# Swagger UI configuration
+SWAGGER_UI_ENABLED=true
+SWAGGER_UI_PATH=/swagger-ui.html
+
+# Whether to encrypt dataset IDs when exporting DSL files (default: true)
+# Set to false to export dataset IDs as plain text for easier cross-environment import
+DSL_EXPORT_ENCRYPT_DATASET_ID=true

api/.importlinter

@@ -0,0 +1,105 @@
+[importlinter]
+root_packages =
+    core
+    configs
+    controllers
+    models
+    tasks
+    services
+
+[importlinter:contract:workflow]
+name = Workflow
+type=layers
+layers =
+    graph_engine
+    graph_events
+    graph
+    nodes
+    node_events
+    entities
+containers =
+    core.workflow
+ignore_imports =
+    core.workflow.nodes.base.node -> core.workflow.graph_events
+    core.workflow.nodes.iteration.iteration_node -> core.workflow.graph_events
+    core.workflow.nodes.loop.loop_node -> core.workflow.graph_events
+
+    core.workflow.nodes.node_factory -> core.workflow.graph
+    core.workflow.nodes.iteration.iteration_node -> core.workflow.graph_engine
+    core.workflow.nodes.iteration.iteration_node -> core.workflow.graph
+    core.workflow.nodes.iteration.iteration_node -> core.workflow.graph_engine.command_channels
+    core.workflow.nodes.loop.loop_node -> core.workflow.graph_engine
+    core.workflow.nodes.loop.loop_node -> core.workflow.graph
+    core.workflow.nodes.loop.loop_node -> core.workflow.graph_engine.command_channels
+
+[importlinter:contract:rsc]
+name = RSC
+type = layers
+layers =
+    graph_engine
+    response_coordinator
+containers =
+    core.workflow.graph_engine
+
+[importlinter:contract:worker]
+name = Worker
+type = layers
+layers =
+    graph_engine
+    worker
+containers =
+    core.workflow.graph_engine
+
+[importlinter:contract:graph-engine-architecture]
+name = Graph Engine Architecture
+type = layers
+layers =
+    graph_engine
+    orchestration
+    command_processing
+    event_management
+    error_handler
+    graph_traversal
+    graph_state_manager
+    worker_management
+    domain
+containers =
+    core.workflow.graph_engine
+
+[importlinter:contract:domain-isolation]
+name = Domain Model Isolation
+type = forbidden
+source_modules =
+    core.workflow.graph_engine.domain
+forbidden_modules =
+    core.workflow.graph_engine.worker_management
+    core.workflow.graph_engine.command_channels
+    core.workflow.graph_engine.layers
+    core.workflow.graph_engine.protocols
+
+[importlinter:contract:worker-management]
+name = Worker Management
+type = forbidden
+source_modules =
+    core.workflow.graph_engine.worker_management
+forbidden_modules =
+    core.workflow.graph_engine.orchestration
+    core.workflow.graph_engine.command_processing
+    core.workflow.graph_engine.event_management
+
+
+[importlinter:contract:graph-traversal-components]
+name = Graph Traversal Components
+type = layers
+layers =
+    edge_processor
+    skip_propagator
+containers =
+    core.workflow.graph_engine.graph_traversal
+
+[importlinter:contract:command-channels]
+name = Command Channels Independence
+type = independence
+modules =
+    core.workflow.graph_engine.command_channels.in_memory_channel
+    core.workflow.graph_engine.command_channels.redis_channel

api/.ruff.toml

@@ -5,7 +5,7 @@ line-length = 120
 quote-style = "double"
 
 [lint]
-preview = false
+preview = true
 select = [
     "B",       # flake8-bugbear rules
     "C4",      # flake8-comprehensions
@@ -30,6 +30,7 @@ select = [
     "RUF022",  # unsorted-dunder-all
     "S506",    # unsafe-yaml-load
     "SIM",     # flake8-simplify rules
+    "T201",    # print-found
     "TRY400",  # error-instead-of-exception
     "TRY401",  # verbose-log-message
     "UP",      # pyupgrade rules
@@ -43,7 +44,9 @@ select = [
     "S302", # suspicious-marshal-usage, disallow use of `marshal` module
     "S311", # suspicious-non-cryptographic-random-usage
     "G001", # don't use str format to logging messages
+    "G003", # don't use + in logging messages
     "G004", # don't use f-strings to format logging messages
+    "UP042", # use StrEnum
 ]
 
 ignore = [
@@ -63,6 +66,7 @@ ignore = [
     "B006",    # mutable-argument-default
     "B007",    # unused-loop-control-variable
     "B026",    # star-arg-unpacking-after-keyword-arg
+    "B901",    # allow return in yield
     "B903",    # class-as-data-structure
     "B904",    # raise-without-from-inside-except
     "B905",    # zip-without-explicit-strict
@@ -77,7 +81,6 @@ ignore = [
     "SIM113",  # enumerate-for-loop
     "SIM117",  # multiple-with-statements
     "SIM210",  # if-expr-with-true-false
-    "UP038",   # deprecated and not recommended by Ruff, https://docs.astral.sh/ruff/rules/non-pep604-isinstance/
 ]
 
 [lint.per-file-ignores]
@@ -88,11 +91,18 @@ ignore = [
 "configs/*" = [
     "N802", # invalid-function-name
 ]
+"core/model_runtime/callbacks/base_callback.py" = [
+    "T201",
+]
+"core/workflow/callbacks/workflow_logging_callback.py" = [
+    "T201",
+]
 "libs/gmpy2_pkcs10aep_cipher.py" = [
     "N803", # invalid-argument-name
 ]
 "tests/*" = [
     "F811", # redefined-while-unused
+    "T201", # allow print in tests
 ]
 
 [lint.pyflakes]

api/Dockerfile

@@ -4,7 +4,7 @@ FROM python:3.12-slim-bookworm AS base
 WORKDIR /app/api
 
 # Install uv
-ENV UV_VERSION=0.7.11
+ENV UV_VERSION=0.8.9
 
 RUN pip install --no-cache-dir uv==${UV_VERSION}
 

api/README.md

@@ -3,7 +3,7 @@
 ## Usage
 
 > [!IMPORTANT]
-> 
+>
 > In the v1.3.0 release, `poetry` has been replaced with
 > [`uv`](https://docs.astral.sh/uv/) as the package manager
 > for Dify API backend service.
@@ -20,25 +20,29 @@
    cd ../api
    ```
 
-2. Copy `.env.example` to `.env`
+1. Copy `.env.example` to `.env`
 
    ```cli
-   cp .env.example .env 
+   cp .env.example .env
    ```
-3. Generate a `SECRET_KEY` in the `.env` file.
+
+1. Generate a `SECRET_KEY` in the `.env` file.
 
    bash for Linux
+
    ```bash for Linux
    sed -i "/^SECRET_KEY=/c\SECRET_KEY=$(openssl rand -base64 42)" .env
    ```
+
    bash for Mac
+
    ```bash for Mac
    secret_key=$(openssl rand -base64 42)
    sed -i '' "/^SECRET_KEY=/c\\
    SECRET_KEY=${secret_key}" .env
    ```
 
-4. Create environment.
+1. Create environment.
 
    Dify API service uses [UV](https://docs.astral.sh/uv/) to manage dependencies.
    First, you need to add the uv package manager, if you don't have it already.
@@ -49,13 +53,13 @@
    brew install uv
    ```
 
-5. Install dependencies
+1. Install dependencies
 
    ```bash
    uv sync --dev
    ```
 
-6. Run migrate
+1. Run migrate
 
    Before the first launch, migrate the database to the latest version.
 
@@ -63,24 +67,27 @@
    uv run flask db upgrade
    ```
 
-7. Start backend
+1. Start backend
 
    ```bash
    uv run flask run --host 0.0.0.0 --port=5001 --debug
    ```
 
-8. Start Dify [web](../web) service.
-9. Setup your application by visiting `http://localhost:3000`.
-10. If you need to handle and debug the async tasks (e.g. dataset importing and documents indexing), please start the worker service.
+1. Start Dify [web](../web) service.
 
-   ```bash
-   uv run celery -A app.celery worker -P gevent -c 1 --loglevel INFO -Q dataset,generation,mail,ops_trace,app_deletion,plugin
-   ```
+1. Setup your application by visiting `http://localhost:3000`.
 
-   Addition, if you want to debug the celery scheduled tasks, you can use the following command in another terminal:
-   ```bash
-   uv run celery -A app.celery beat 
-   ```
+1. If you need to handle and debug the async tasks (e.g. dataset importing and documents indexing), please start the worker service.
+
+```bash
+uv run celery -A app.celery worker -P gevent -c 2 --loglevel INFO -Q dataset,generation,mail,ops_trace,app_deletion,plugin,workflow_storage,conversation
+```
+
+Additionally, if you want to debug the celery scheduled tasks, you can run the following command in another terminal to start the beat service:
+
+```bash
+uv run celery -A app.celery beat
+```
 
 ## Testing
 
@@ -90,9 +97,16 @@
    uv sync --dev
    ```
 
-2. Run the tests locally with mocked system environment variables in `tool.pytest_env` section in `pyproject.toml`
+1. Run the tests locally with mocked system environment variables in `tool.pytest_env` section in `pyproject.toml`, more can check [Claude.md](../CLAUDE.md)
 
    ```bash
-   uv run -P api bash dev/pytest/pytest_all_tests.sh
-   ```
+   uv run pytest                           # Run all tests
+   uv run pytest tests/unit_tests/         # Unit tests only
+   uv run pytest tests/integration_tests/  # Integration tests
 
+   # Code quality
+   ../dev/reformat               # Run all formatters and linters
+   uv run ruff check --fix ./    # Fix linting issues
+   uv run ruff format ./         # Format code
+   uv run basedpyright .         # Type checking
+   ```

api/app.py

@@ -9,33 +9,46 @@ def is_db_command():
 
 
 # create app
+celery = None
+flask_app = None
+socketio_app = None
+
 if is_db_command():
     from app_factory import create_migrations_app
 
     app = create_migrations_app()
+    socketio_app = app
+    flask_app = app
 else:
     # It seems that JetBrains Python debugger does not work well with gevent,
     # so we need to disable gevent in debug mode.
     # If you are using debugpy and set GEVENT_SUPPORT=True, you can debug with gevent.
-    if (flask_debug := os.environ.get("FLASK_DEBUG", "0")) and flask_debug.lower() in {"false", "0", "no"}:
-        from gevent import monkey
+    # if (flask_debug := os.environ.get("FLASK_DEBUG", "0")) and flask_debug.lower() in {"false", "0", "no"}:
+    # from gevent import monkey
+    #
+    # # gevent
+    # monkey.patch_all()
+    #
+    # from grpc.experimental import gevent as grpc_gevent  # type: ignore
+    #
+    # # grpc gevent
+    # grpc_gevent.init_gevent()
 
-        # gevent
-        monkey.patch_all()
-
-        from grpc.experimental import gevent as grpc_gevent  # type: ignore
-
-        # grpc gevent
-        grpc_gevent.init_gevent()
-
-        import psycogreen.gevent  # type: ignore
-
-        psycogreen.gevent.patch_psycopg()
+    # import psycogreen.gevent  # type: ignore
+    #
+    # psycogreen.gevent.patch_psycopg()
 
     from app_factory import create_app
 
-    app = create_app()
-    celery = app.extensions["celery"]
+    socketio_app, flask_app = create_app()
+    app = flask_app
+    celery = flask_app.extensions["celery"]
 
 if __name__ == "__main__":
-    app.run(host="0.0.0.0", port=5001)
+    from gevent import pywsgi
+    from geventwebsocket.handler import WebSocketHandler
+
+    host = os.environ.get("HOST", "0.0.0.0")
+    port = int(os.environ.get("PORT", 5001))
+    server = pywsgi.WSGIServer((host, port), socketio_app, handler_class=WebSocketHandler)
+    server.serve_forever()

api/app_factory.py

@@ -5,6 +5,8 @@ from configs import dify_config
 from contexts.wrapper import RecyclableContextVar
 from dify_app import DifyApp
 
+logger = logging.getLogger(__name__)
+
 
 # ----------------------------
 # Application Factory Function
@@ -23,17 +25,28 @@ def create_flask_app_with_configs() -> DifyApp:
         # add an unique identifier to each request
         RecyclableContextVar.increment_thread_recycles()
 
+    # Capture the decorator's return value to avoid pyright reportUnusedFunction
+    _ = before_request
+
     return dify_app
 
 
-def create_app() -> DifyApp:
+def create_app() -> tuple[any, DifyApp]:
     start_time = time.perf_counter()
     app = create_flask_app_with_configs()
     initialize_extensions(app)
+
+    import socketio
+
+    from extensions.ext_socketio import sio
+
+    sio.app = app
+    socketio_app = socketio.WSGIApp(sio, app)
+
     end_time = time.perf_counter()
     if dify_config.DEBUG:
-        logging.info("Finished create_app (%s ms)", round((end_time - start_time) * 1000, 2))
-    return app
+        logger.info("Finished create_app (%s ms)", round((end_time - start_time) * 1000, 2))
+    return socketio_app, app
 
 
 def initialize_extensions(app: DifyApp):
@@ -51,6 +64,7 @@ def initialize_extensions(app: DifyApp):
         ext_login,
         ext_mail,
         ext_migrate,
+        ext_orjson,
         ext_otel,
         ext_proxy_fix,
         ext_redis,
@@ -67,6 +81,7 @@ def initialize_extensions(app: DifyApp):
         ext_logging,
         ext_warnings,
         ext_import_modules,
+        ext_orjson,
         ext_set_secretkey,
         ext_compress,
         ext_code_based_extension,
@@ -91,14 +106,14 @@ def initialize_extensions(app: DifyApp):
         is_enabled = ext.is_enabled() if hasattr(ext, "is_enabled") else True
         if not is_enabled:
             if dify_config.DEBUG:
-                logging.info("Skipped %s", short_name)
+                logger.info("Skipped %s", short_name)
             continue
 
         start_time = time.perf_counter()
         ext.init_app(app)
         end_time = time.perf_counter()
         if dify_config.DEBUG:
-            logging.info("Loaded %s (%s ms)", short_name, round((end_time - start_time) * 1000, 2))
+            logger.info("Loaded %s (%s ms)", short_name, round((end_time - start_time) * 1000, 2))
 
 
 def create_migrations_app():

api/celery_entrypoint.py

@@ -0,0 +1,13 @@
+import psycogreen.gevent as pscycogreen_gevent  # type: ignore
+from grpc.experimental import gevent as grpc_gevent  # type: ignore
+
+# grpc gevent
+grpc_gevent.init_gevent()
+print("gRPC patched with gevent.", flush=True)  # noqa: T201
+pscycogreen_gevent.patch_psycopg()
+print("psycopg2 patched with gevent.", flush=True)  # noqa: T201
+
+
+from app import app, celery
+
+__all__ = ["app", "celery"]

api/configs/__init__.py

@@ -1,3 +1,3 @@
 from .app_config import DifyConfig
 
-dify_config = DifyConfig()
+dify_config = DifyConfig()  # type: ignore

api/configs/extra/notion_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field
 from pydantic_settings import BaseSettings
 
@@ -9,28 +7,28 @@ class NotionConfig(BaseSettings):
     Configuration settings for Notion integration
     """
 
-    NOTION_CLIENT_ID: Optional[str] = Field(
+    NOTION_CLIENT_ID: str | None = Field(
         description="Client ID for Notion API authentication. Required for OAuth 2.0 flow.",
         default=None,
     )
 
-    NOTION_CLIENT_SECRET: Optional[str] = Field(
+    NOTION_CLIENT_SECRET: str | None = Field(
         description="Client secret for Notion API authentication. Required for OAuth 2.0 flow.",
         default=None,
     )
 
-    NOTION_INTEGRATION_TYPE: Optional[str] = Field(
+    NOTION_INTEGRATION_TYPE: str | None = Field(
         description="Type of Notion integration."
         " Set to 'internal' for internal integrations, or None for public integrations.",
         default=None,
     )
 
-    NOTION_INTERNAL_SECRET: Optional[str] = Field(
+    NOTION_INTERNAL_SECRET: str | None = Field(
         description="Secret key for internal Notion integrations. Required when NOTION_INTEGRATION_TYPE is 'internal'.",
         default=None,
     )
 
-    NOTION_INTEGRATION_TOKEN: Optional[str] = Field(
+    NOTION_INTEGRATION_TOKEN: str | None = Field(
         description="Integration token for Notion API access. Used for direct API calls without OAuth flow.",
         default=None,
     )

api/configs/extra/sentry_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field, NonNegativeFloat
 from pydantic_settings import BaseSettings
 
@@ -9,7 +7,7 @@ class SentryConfig(BaseSettings):
     Configuration settings for Sentry error tracking and performance monitoring
     """
 
-    SENTRY_DSN: Optional[str] = Field(
+    SENTRY_DSN: str | None = Field(
         description="Sentry Data Source Name (DSN)."
         " This is the unique identifier of your Sentry project, used to send events to the correct project.",
         default=None,

api/configs/feature/__init__.py

@@ -1,4 +1,5 @@
-from typing import Annotated, Literal, Optional
+from enum import StrEnum
+from typing import Literal
 
 from pydantic import (
     AliasChoices,
@@ -31,6 +32,12 @@ class SecurityConfig(BaseSettings):
         description="Duration in minutes for which a password reset token remains valid",
         default=5,
     )
+
+    EMAIL_REGISTER_TOKEN_EXPIRY_MINUTES: PositiveInt = Field(
+        description="Duration in minutes for which a email register token remains valid",
+        default=5,
+    )
+
     CHANGE_EMAIL_TOKEN_EXPIRY_MINUTES: PositiveInt = Field(
         description="Duration in minutes for which a change email token remains valid",
         default=5,
@@ -51,7 +58,7 @@ class SecurityConfig(BaseSettings):
         default=False,
     )
 
-    ADMIN_API_KEY: Optional[str] = Field(
+    ADMIN_API_KEY: str | None = Field(
         description="admin api key for authentication",
         default=None,
     )
@@ -91,21 +98,36 @@ class CodeExecutionSandboxConfig(BaseSettings):
         default="dify-sandbox",
     )
 
-    CODE_EXECUTION_CONNECT_TIMEOUT: Optional[float] = Field(
+    CODE_EXECUTION_CONNECT_TIMEOUT: float | None = Field(
         description="Connection timeout in seconds for code execution requests",
         default=10.0,
     )
 
-    CODE_EXECUTION_READ_TIMEOUT: Optional[float] = Field(
+    CODE_EXECUTION_READ_TIMEOUT: float | None = Field(
         description="Read timeout in seconds for code execution requests",
         default=60.0,
     )
 
-    CODE_EXECUTION_WRITE_TIMEOUT: Optional[float] = Field(
+    CODE_EXECUTION_WRITE_TIMEOUT: float | None = Field(
         description="Write timeout in seconds for code execution request",
         default=10.0,
     )
 
+    CODE_EXECUTION_POOL_MAX_CONNECTIONS: PositiveInt = Field(
+        description="Maximum number of concurrent connections for the code execution HTTP client",
+        default=100,
+    )
+
+    CODE_EXECUTION_POOL_MAX_KEEPALIVE_CONNECTIONS: PositiveInt = Field(
+        description="Maximum number of persistent keep-alive connections for the code execution HTTP client",
+        default=20,
+    )
+
+    CODE_EXECUTION_POOL_KEEPALIVE_EXPIRY: PositiveFloat | None = Field(
+        description="Keep-alive expiry in seconds for idle connections (set to None to disable)",
+        default=5.0,
+    )
+
     CODE_MAX_NUMBER: PositiveInt = Field(
         description="Maximum allowed numeric value in code execution",
         default=9223372036854775807,
@@ -128,7 +150,7 @@ class CodeExecutionSandboxConfig(BaseSettings):
 
     CODE_MAX_STRING_LENGTH: PositiveInt = Field(
         description="Maximum allowed length for strings in code execution",
-        default=80000,
+        default=400_000,
     )
 
     CODE_MAX_STRING_ARRAY_LENGTH: PositiveInt = Field(
@@ -146,6 +168,11 @@ class CodeExecutionSandboxConfig(BaseSettings):
         default=1000,
     )
 
+    CODE_EXECUTION_SSL_VERIFY: bool = Field(
+        description="Enable or disable SSL verification for code execution requests",
+        default=True,
+    )
+
 
 class PluginConfig(BaseSettings):
     """
@@ -162,6 +189,11 @@ class PluginConfig(BaseSettings):
         default="plugin-api-key",
     )
 
+    PLUGIN_DAEMON_TIMEOUT: PositiveFloat | None = Field(
+        description="Timeout in seconds for requests to the plugin daemon (set to None to disable)",
+        default=300.0,
+    )
+
     INNER_API_KEY_FOR_PLUGIN: str = Field(description="Inner api key for plugin", default="inner-api-key")
 
     PLUGIN_REMOTE_INSTALL_HOST: str = Field(
@@ -335,11 +367,11 @@ class HttpConfig(BaseSettings):
     )
 
     HTTP_REQUEST_MAX_READ_TIMEOUT: int = Field(
-        ge=1, description="Maximum read timeout in seconds for HTTP requests", default=60
+        ge=1, description="Maximum read timeout in seconds for HTTP requests", default=600
     )
 
     HTTP_REQUEST_MAX_WRITE_TIMEOUT: int = Field(
-        ge=1, description="Maximum write timeout in seconds for HTTP requests", default=20
+        ge=1, description="Maximum write timeout in seconds for HTTP requests", default=600
     )
 
     HTTP_REQUEST_NODE_MAX_BINARY_SIZE: PositiveInt = Field(
@@ -362,17 +394,17 @@ class HttpConfig(BaseSettings):
         default=3,
     )
 
-    SSRF_PROXY_ALL_URL: Optional[str] = Field(
+    SSRF_PROXY_ALL_URL: str | None = Field(
         description="Proxy URL for HTTP or HTTPS requests to prevent Server-Side Request Forgery (SSRF)",
         default=None,
     )
 
-    SSRF_PROXY_HTTP_URL: Optional[str] = Field(
+    SSRF_PROXY_HTTP_URL: str | None = Field(
         description="Proxy URL for HTTP requests to prevent Server-Side Request Forgery (SSRF)",
         default=None,
     )
 
-    SSRF_PROXY_HTTPS_URL: Optional[str] = Field(
+    SSRF_PROXY_HTTPS_URL: str | None = Field(
         description="Proxy URL for HTTPS requests to prevent Server-Side Request Forgery (SSRF)",
         default=None,
     )
@@ -397,6 +429,21 @@ class HttpConfig(BaseSettings):
         default=5,
     )
 
+    SSRF_POOL_MAX_CONNECTIONS: PositiveInt = Field(
+        description="Maximum number of concurrent connections for the SSRF HTTP client",
+        default=100,
+    )
+
+    SSRF_POOL_MAX_KEEPALIVE_CONNECTIONS: PositiveInt = Field(
+        description="Maximum number of persistent keep-alive connections for the SSRF HTTP client",
+        default=20,
+    )
+
+    SSRF_POOL_KEEPALIVE_EXPIRY: PositiveFloat | None = Field(
+        description="Keep-alive expiry in seconds for idle SSRF connections (set to None to disable)",
+        default=5.0,
+    )
+
     RESPECT_XFORWARD_HEADERS_ENABLED: bool = Field(
         description="Enable handling of X-Forwarded-For, X-Forwarded-Proto, and X-Forwarded-Port headers"
         " when the app is behind a single trusted reverse proxy.",
@@ -414,7 +461,7 @@ class InnerAPIConfig(BaseSettings):
         default=False,
     )
 
-    INNER_API_KEY: Optional[str] = Field(
+    INNER_API_KEY: str | None = Field(
         description="API key for accessing the internal API",
         default=None,
     )
@@ -430,7 +477,7 @@ class LoggingConfig(BaseSettings):
         default="INFO",
     )
 
-    LOG_FILE: Optional[str] = Field(
+    LOG_FILE: str | None = Field(
         description="File path for log output.",
         default=None,
     )
@@ -450,12 +497,12 @@ class LoggingConfig(BaseSettings):
         default="%(asctime)s.%(msecs)03d %(levelname)s [%(threadName)s] [%(filename)s:%(lineno)d] - %(message)s",
     )
 
-    LOG_DATEFORMAT: Optional[str] = Field(
+    LOG_DATEFORMAT: str | None = Field(
         description="Date format string for log timestamps",
         default=None,
     )
 
-    LOG_TZ: Optional[str] = Field(
+    LOG_TZ: str | None = Field(
         description="Timezone for log timestamps (e.g., 'America/New_York')",
         default="UTC",
     )
@@ -499,6 +546,22 @@ class UpdateConfig(BaseSettings):
     )
 
 
+class WorkflowVariableTruncationConfig(BaseSettings):
+    WORKFLOW_VARIABLE_TRUNCATION_MAX_SIZE: PositiveInt = Field(
+        # 1000 KiB
+        1024_000,
+        description="Maximum size for variable to trigger final truncation.",
+    )
+    WORKFLOW_VARIABLE_TRUNCATION_STRING_LENGTH: PositiveInt = Field(
+        100000,
+        description="maximum length for string to trigger tuncation, measure in number of characters",
+    )
+    WORKFLOW_VARIABLE_TRUNCATION_ARRAY_LENGTH: PositiveInt = Field(
+        1000,
+        description="maximum length for array to trigger truncation.",
+    )
+
+
 class WorkflowConfig(BaseSettings):
     """
     Configuration for workflow execution
@@ -519,16 +582,38 @@ class WorkflowConfig(BaseSettings):
         default=5,
     )
 
-    WORKFLOW_PARALLEL_DEPTH_LIMIT: PositiveInt = Field(
-        description="Maximum allowed depth for nested parallel executions",
-        default=3,
-    )
-
     MAX_VARIABLE_SIZE: PositiveInt = Field(
         description="Maximum size in bytes for a single variable in workflows. Default to 200 KB.",
         default=200 * 1024,
     )
 
+    TEMPLATE_TRANSFORM_MAX_LENGTH: PositiveInt = Field(
+        description="Maximum number of characters allowed in Template Transform node output",
+        default=400_000,
+    )
+
+    # GraphEngine Worker Pool Configuration
+    GRAPH_ENGINE_MIN_WORKERS: PositiveInt = Field(
+        description="Minimum number of workers per GraphEngine instance",
+        default=1,
+    )
+
+    GRAPH_ENGINE_MAX_WORKERS: PositiveInt = Field(
+        description="Maximum number of workers per GraphEngine instance",
+        default=10,
+    )
+
+    GRAPH_ENGINE_SCALE_UP_THRESHOLD: PositiveInt = Field(
+        description="Queue depth threshold that triggers worker scale up",
+        default=3,
+    )
+
+    GRAPH_ENGINE_SCALE_DOWN_IDLE_TIME: float = Field(
+        description="Seconds of idle time before scaling down workers",
+        default=5.0,
+        ge=0.1,
+    )
+
 
 class WorkflowNodeExecutionConfig(BaseSettings):
     """
@@ -552,12 +637,18 @@ class RepositoryConfig(BaseSettings):
     """
 
     CORE_WORKFLOW_EXECUTION_REPOSITORY: str = Field(
-        description="Repository implementation for WorkflowExecution. Specify as a module path",
+        description="Repository implementation for WorkflowExecution. Options: "
+        "'core.repositories.sqlalchemy_workflow_execution_repository.SQLAlchemyWorkflowExecutionRepository' (default), "
+        "'core.repositories.celery_workflow_execution_repository.CeleryWorkflowExecutionRepository'",
         default="core.repositories.sqlalchemy_workflow_execution_repository.SQLAlchemyWorkflowExecutionRepository",
     )
 
     CORE_WORKFLOW_NODE_EXECUTION_REPOSITORY: str = Field(
-        description="Repository implementation for WorkflowNodeExecution. Specify as a module path",
+        description="Repository implementation for WorkflowNodeExecution. Options: "
+        "'core.repositories.sqlalchemy_workflow_node_execution_repository."
+        "SQLAlchemyWorkflowNodeExecutionRepository' (default), "
+        "'core.repositories.celery_workflow_node_execution_repository."
+        "CeleryWorkflowNodeExecutionRepository'",
         default="core.repositories.sqlalchemy_workflow_node_execution_repository.SQLAlchemyWorkflowNodeExecutionRepository",
     )
 
@@ -583,22 +674,22 @@ class AuthConfig(BaseSettings):
         default="/console/api/oauth/authorize",
     )
 
-    GITHUB_CLIENT_ID: Optional[str] = Field(
+    GITHUB_CLIENT_ID: str | None = Field(
         description="GitHub OAuth client ID",
         default=None,
     )
 
-    GITHUB_CLIENT_SECRET: Optional[str] = Field(
+    GITHUB_CLIENT_SECRET: str | None = Field(
         description="GitHub OAuth client secret",
         default=None,
     )
 
-    GOOGLE_CLIENT_ID: Optional[str] = Field(
+    GOOGLE_CLIENT_ID: str | None = Field(
         description="Google OAuth client ID",
         default=None,
     )
 
-    GOOGLE_CLIENT_SECRET: Optional[str] = Field(
+    GOOGLE_CLIENT_SECRET: str | None = Field(
         description="Google OAuth client secret",
         default=None,
     )
@@ -633,6 +724,11 @@ class AuthConfig(BaseSettings):
         default=86400,
     )
 
+    EMAIL_REGISTER_LOCKOUT_DURATION: PositiveInt = Field(
+        description="Time (in seconds) a user must wait before retrying email register after exceeding the rate limit.",
+        default=86400,
+    )
+
 
 class ModerationConfig(BaseSettings):
     """
@@ -656,47 +752,71 @@ class ToolConfig(BaseSettings):
     )
 
 
+class TemplateMode(StrEnum):
+    # unsafe mode allows flexible operations in templates, but may cause security vulnerabilities
+    UNSAFE = "unsafe"
+
+    # sandbox mode restricts some unsafe operations like accessing __class__.
+    # however, it is still not 100% safe, for example, cpu exploitation can happen.
+    SANDBOX = "sandbox"
+
+    # templating is disabled
+    DISABLED = "disabled"
+
+
 class MailConfig(BaseSettings):
     """
     Configuration for email services
     """
 
-    MAIL_TYPE: Optional[str] = Field(
+    MAIL_TEMPLATING_MODE: TemplateMode = Field(
+        description="Template mode for email services",
+        default=TemplateMode.SANDBOX,
+    )
+
+    MAIL_TEMPLATING_TIMEOUT: int = Field(
+        description="""
+        Timeout for email templating in seconds. Used to prevent infinite loops in malicious templates.
+        Only available in sandbox mode.""",
+        default=3,
+    )
+
+    MAIL_TYPE: str | None = Field(
         description="Email service provider type ('smtp' or 'resend' or 'sendGrid), default to None.",
         default=None,
     )
 
-    MAIL_DEFAULT_SEND_FROM: Optional[str] = Field(
+    MAIL_DEFAULT_SEND_FROM: str | None = Field(
         description="Default email address to use as the sender",
         default=None,
     )
 
-    RESEND_API_KEY: Optional[str] = Field(
+    RESEND_API_KEY: str | None = Field(
         description="API key for Resend email service",
         default=None,
     )
 
-    RESEND_API_URL: Optional[str] = Field(
+    RESEND_API_URL: str | None = Field(
         description="API URL for Resend email service",
         default=None,
     )
 
-    SMTP_SERVER: Optional[str] = Field(
+    SMTP_SERVER: str | None = Field(
         description="SMTP server hostname",
         default=None,
     )
 
-    SMTP_PORT: Optional[int] = Field(
+    SMTP_PORT: int | None = Field(
         description="SMTP server port number",
         default=465,
     )
 
-    SMTP_USERNAME: Optional[str] = Field(
+    SMTP_USERNAME: str | None = Field(
         description="Username for SMTP authentication",
         default=None,
     )
 
-    SMTP_PASSWORD: Optional[str] = Field(
+    SMTP_PASSWORD: str | None = Field(
         description="Password for SMTP authentication",
         default=None,
     )
@@ -716,7 +836,7 @@ class MailConfig(BaseSettings):
         default=50,
     )
 
-    SENDGRID_API_KEY: Optional[str] = Field(
+    SENDGRID_API_KEY: str | None = Field(
         description="API key for SendGrid service",
         default=None,
     )
@@ -739,17 +859,17 @@ class RagEtlConfig(BaseSettings):
         default="database",
     )
 
-    UNSTRUCTURED_API_URL: Optional[str] = Field(
+    UNSTRUCTURED_API_URL: str | None = Field(
         description="API URL for Unstructured.io service",
         default=None,
     )
 
-    UNSTRUCTURED_API_KEY: Optional[str] = Field(
+    UNSTRUCTURED_API_KEY: str | None = Field(
         description="API key for Unstructured.io service",
         default="",
     )
 
-    SCARF_NO_ANALYTICS: Optional[str] = Field(
+    SCARF_NO_ANALYTICS: str | None = Field(
         description="This is about whether to disable Scarf analytics in Unstructured library.",
         default="false",
     )
@@ -790,6 +910,11 @@ class DataSetConfig(BaseSettings):
         default=30,
     )
 
+    DSL_EXPORT_ENCRYPT_DATASET_ID: bool = Field(
+        description="Enable or disable dataset ID encryption when exporting DSL files",
+        default=True,
+    )
+
 
 class WorkspaceConfig(BaseSettings):
     """
@@ -923,6 +1048,13 @@ class PositionConfig(BaseSettings):
         return {item.strip() for item in self.POSITION_TOOL_EXCLUDES.split(",") if item.strip() != ""}
 
 
+class CollaborationConfig(BaseSettings):
+    ENABLE_COLLABORATION_MODE: bool = Field(
+        description="Whether to enable collaboration mode features across the workspace",
+        default=False,
+    )
+
+
 class LoginConfig(BaseSettings):
     ENABLE_EMAIL_CODE_LOGIN: bool = Field(
         description="whether to enable email code login",
@@ -962,6 +1094,26 @@ class AccountConfig(BaseSettings):
     )
 
 
+class WorkflowLogConfig(BaseSettings):
+    WORKFLOW_LOG_CLEANUP_ENABLED: bool = Field(default=True, description="Enable workflow run log cleanup")
+    WORKFLOW_LOG_RETENTION_DAYS: int = Field(default=30, description="Retention days for workflow run logs")
+    WORKFLOW_LOG_CLEANUP_BATCH_SIZE: int = Field(
+        default=100, description="Batch size for workflow run log cleanup operations"
+    )
+
+
+class SwaggerUIConfig(BaseSettings):
+    SWAGGER_UI_ENABLED: bool = Field(
+        description="Whether to enable Swagger UI in api module",
+        default=True,
+    )
+
+    SWAGGER_UI_PATH: str = Field(
+        description="Swagger UI page path in api module",
+        default="/swagger-ui.html",
+    )
+
+
 class FeatureConfig(
     # place the configs in alphabet order
     AppExecutionConfig,
@@ -991,11 +1143,15 @@ class FeatureConfig(
     WorkflowConfig,
     WorkflowNodeExecutionConfig,
     WorkspaceConfig,
+    CollaborationConfig,
     LoginConfig,
     AccountConfig,
+    SwaggerUIConfig,
     # hosted services config
     HostedServiceConfig,
     CeleryBeatConfig,
     CeleryScheduleTasksConfig,
+    WorkflowLogConfig,
+    WorkflowVariableTruncationConfig,
 ):
     pass

api/configs/feature/hosted_service/__init__.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field, NonNegativeInt
 from pydantic_settings import BaseSettings
 
@@ -40,17 +38,17 @@ class HostedOpenAiConfig(BaseSettings):
     Configuration for hosted OpenAI service
     """
 
-    HOSTED_OPENAI_API_KEY: Optional[str] = Field(
+    HOSTED_OPENAI_API_KEY: str | None = Field(
         description="API key for hosted OpenAI service",
         default=None,
     )
 
-    HOSTED_OPENAI_API_BASE: Optional[str] = Field(
+    HOSTED_OPENAI_API_BASE: str | None = Field(
         description="Base URL for hosted OpenAI API",
         default=None,
     )
 
-    HOSTED_OPENAI_API_ORGANIZATION: Optional[str] = Field(
+    HOSTED_OPENAI_API_ORGANIZATION: str | None = Field(
         description="Organization ID for hosted OpenAI service",
         default=None,
     )
@@ -110,12 +108,12 @@ class HostedAzureOpenAiConfig(BaseSettings):
         default=False,
     )
 
-    HOSTED_AZURE_OPENAI_API_KEY: Optional[str] = Field(
+    HOSTED_AZURE_OPENAI_API_KEY: str | None = Field(
         description="API key for hosted Azure OpenAI service",
         default=None,
     )
 
-    HOSTED_AZURE_OPENAI_API_BASE: Optional[str] = Field(
+    HOSTED_AZURE_OPENAI_API_BASE: str | None = Field(
         description="Base URL for hosted Azure OpenAI API",
         default=None,
     )
@@ -131,12 +129,12 @@ class HostedAnthropicConfig(BaseSettings):
     Configuration for hosted Anthropic service
     """
 
-    HOSTED_ANTHROPIC_API_BASE: Optional[str] = Field(
+    HOSTED_ANTHROPIC_API_BASE: str | None = Field(
         description="Base URL for hosted Anthropic API",
         default=None,
     )
 
-    HOSTED_ANTHROPIC_API_KEY: Optional[str] = Field(
+    HOSTED_ANTHROPIC_API_KEY: str | None = Field(
         description="API key for hosted Anthropic service",
         default=None,
     )
@@ -222,11 +220,28 @@ class HostedFetchAppTemplateConfig(BaseSettings):
     )
 
 
+class HostedFetchPipelineTemplateConfig(BaseSettings):
+    """
+    Configuration for fetching pipeline templates
+    """
+
+    HOSTED_FETCH_PIPELINE_TEMPLATES_MODE: str = Field(
+        description="Mode for fetching pipeline templates: remote, db, or builtin default to remote,",
+        default="remote",
+    )
+
+    HOSTED_FETCH_PIPELINE_TEMPLATES_REMOTE_DOMAIN: str = Field(
+        description="Domain for fetching remote pipeline templates",
+        default="https://tmpl.dify.ai",
+    )
+
+
 class HostedServiceConfig(
     # place the configs in alphabet order
     HostedAnthropicConfig,
     HostedAzureOpenAiConfig,
     HostedFetchAppTemplateConfig,
+    HostedFetchPipelineTemplateConfig,
     HostedMinmaxConfig,
     HostedOpenAiConfig,
     HostedSparkConfig,

api/configs/middleware/__init__.py

@@ -1,5 +1,5 @@
 import os
-from typing import Any, Literal, Optional
+from typing import Any, Literal
 from urllib.parse import parse_qsl, quote_plus
 
 from pydantic import Field, NonNegativeFloat, NonNegativeInt, PositiveFloat, PositiveInt, computed_field
@@ -18,6 +18,7 @@ from .storage.opendal_storage_config import OpenDALStorageConfig
 from .storage.supabase_storage_config import SupabaseStorageConfig
 from .storage.tencent_cos_storage_config import TencentCloudCOSStorageConfig
 from .storage.volcengine_tos_storage_config import VolcengineTOSStorageConfig
+from .vdb.alibabacloud_mysql_config import AlibabaCloudMySQLConfig
 from .vdb.analyticdb_config import AnalyticdbConfig
 from .vdb.baidu_vector_config import BaiduVectorDBConfig
 from .vdb.chroma_config import ChromaConfig
@@ -78,18 +79,18 @@ class StorageConfig(BaseSettings):
 
 
 class VectorStoreConfig(BaseSettings):
-    VECTOR_STORE: Optional[str] = Field(
+    VECTOR_STORE: str | None = Field(
         description="Type of vector store to use for efficient similarity search."
         " Set to None if not using a vector store.",
         default=None,
     )
 
-    VECTOR_STORE_WHITELIST_ENABLE: Optional[bool] = Field(
+    VECTOR_STORE_WHITELIST_ENABLE: bool | None = Field(
         description="Enable whitelist for vector store.",
         default=False,
     )
 
-    VECTOR_INDEX_NAME_PREFIX: Optional[str] = Field(
+    VECTOR_INDEX_NAME_PREFIX: str | None = Field(
         description="Prefix used to create collection name in vector database",
         default="Vector_index",
     )
@@ -187,6 +188,11 @@ class DatabaseConfig(BaseSettings):
         default=False,
     )
 
+    SQLALCHEMY_POOL_TIMEOUT: NonNegativeInt = Field(
+        description="Number of seconds to wait for a connection from the pool before raising a timeout error.",
+        default=30,
+    )
+
     RETRIEVAL_SERVICE_EXECUTORS: NonNegativeInt = Field(
         description="Number of processes for the retrieval service, default to CPU cores.",
         default=os.cpu_count() or 1,
@@ -215,6 +221,8 @@ class DatabaseConfig(BaseSettings):
             "pool_pre_ping": self.SQLALCHEMY_POOL_PRE_PING,
             "connect_args": connect_args,
             "pool_use_lifo": self.SQLALCHEMY_POOL_USE_LIFO,
+            "pool_reset_on_return": None,
+            "pool_timeout": self.SQLALCHEMY_POOL_TIMEOUT,
         }
 
 
@@ -224,26 +232,26 @@ class CeleryConfig(DatabaseConfig):
         default="redis",
     )
 
-    CELERY_BROKER_URL: Optional[str] = Field(
+    CELERY_BROKER_URL: str | None = Field(
         description="URL of the message broker for Celery tasks.",
         default=None,
     )
 
-    CELERY_USE_SENTINEL: Optional[bool] = Field(
+    CELERY_USE_SENTINEL: bool | None = Field(
         description="Whether to use Redis Sentinel for high availability.",
         default=False,
     )
 
-    CELERY_SENTINEL_MASTER_NAME: Optional[str] = Field(
+    CELERY_SENTINEL_MASTER_NAME: str | None = Field(
         description="Name of the Redis Sentinel master.",
         default=None,
     )
 
-    CELERY_SENTINEL_PASSWORD: Optional[str] = Field(
+    CELERY_SENTINEL_PASSWORD: str | None = Field(
         description="Password of the Redis Sentinel master.",
         default=None,
     )
-    CELERY_SENTINEL_SOCKET_TIMEOUT: Optional[PositiveFloat] = Field(
+    CELERY_SENTINEL_SOCKET_TIMEOUT: PositiveFloat | None = Field(
         description="Timeout for Redis Sentinel socket operations in seconds.",
         default=0.1,
     )
@@ -267,12 +275,12 @@ class InternalTestConfig(BaseSettings):
     Configuration settings for Internal Test
     """
 
-    AWS_SECRET_ACCESS_KEY: Optional[str] = Field(
+    AWS_SECRET_ACCESS_KEY: str | None = Field(
         description="Internal test AWS secret access key",
         default=None,
     )
 
-    AWS_ACCESS_KEY_ID: Optional[str] = Field(
+    AWS_ACCESS_KEY_ID: str | None = Field(
         description="Internal test AWS access key ID",
         default=None,
     )
@@ -283,15 +291,15 @@ class DatasetQueueMonitorConfig(BaseSettings):
     Configuration settings for Dataset Queue Monitor
     """
 
-    QUEUE_MONITOR_THRESHOLD: Optional[NonNegativeInt] = Field(
+    QUEUE_MONITOR_THRESHOLD: NonNegativeInt | None = Field(
         description="Threshold for dataset queue monitor",
         default=200,
     )
-    QUEUE_MONITOR_ALERT_EMAILS: Optional[str] = Field(
+    QUEUE_MONITOR_ALERT_EMAILS: str | None = Field(
         description="Emails for dataset queue monitor alert, separated by commas",
         default=None,
     )
-    QUEUE_MONITOR_INTERVAL: Optional[NonNegativeFloat] = Field(
+    QUEUE_MONITOR_INTERVAL: NonNegativeFloat | None = Field(
         description="Interval for dataset queue monitor in minutes",
         default=30,
     )
@@ -299,8 +307,7 @@ class DatasetQueueMonitorConfig(BaseSettings):
 
 class MiddlewareConfig(
     # place the configs in alphabet order
-    CeleryConfig,
-    DatabaseConfig,
+    CeleryConfig,  # Note: CeleryConfig already inherits from DatabaseConfig
     KeywordStoreConfig,
     RedisConfig,
     # configs of storage and storage providers
@@ -324,6 +331,7 @@ class MiddlewareConfig(
     ClickzettaConfig,
     HuaweiCloudConfig,
     MilvusConfig,
+    AlibabaCloudMySQLConfig,
     MyScaleConfig,
     OpenSearchConfig,
     OracleConfig,

api/configs/middleware/cache/redis_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field, NonNegativeInt, PositiveFloat, PositiveInt
 from pydantic_settings import BaseSettings
 
@@ -19,12 +17,12 @@ class RedisConfig(BaseSettings):
         default=6379,
     )
 
-    REDIS_USERNAME: Optional[str] = Field(
+    REDIS_USERNAME: str | None = Field(
         description="Username for Redis authentication (if required)",
         default=None,
     )
 
-    REDIS_PASSWORD: Optional[str] = Field(
+    REDIS_PASSWORD: str | None = Field(
         description="Password for Redis authentication (if required)",
         default=None,
     )
@@ -39,32 +37,52 @@ class RedisConfig(BaseSettings):
         default=False,
     )
 
-    REDIS_USE_SENTINEL: Optional[bool] = Field(
+    REDIS_SSL_CERT_REQS: str = Field(
+        description="SSL certificate requirements (CERT_NONE, CERT_OPTIONAL, CERT_REQUIRED)",
+        default="CERT_NONE",
+    )
+
+    REDIS_SSL_CA_CERTS: str | None = Field(
+        description="Path to the CA certificate file for SSL verification",
+        default=None,
+    )
+
+    REDIS_SSL_CERTFILE: str | None = Field(
+        description="Path to the client certificate file for SSL authentication",
+        default=None,
+    )
+
+    REDIS_SSL_KEYFILE: str | None = Field(
+        description="Path to the client private key file for SSL authentication",
+        default=None,
+    )
+
+    REDIS_USE_SENTINEL: bool | None = Field(
         description="Enable Redis Sentinel mode for high availability",
         default=False,
     )
 
-    REDIS_SENTINELS: Optional[str] = Field(
+    REDIS_SENTINELS: str | None = Field(
         description="Comma-separated list of Redis Sentinel nodes (host:port)",
         default=None,
     )
 
-    REDIS_SENTINEL_SERVICE_NAME: Optional[str] = Field(
+    REDIS_SENTINEL_SERVICE_NAME: str | None = Field(
         description="Name of the Redis Sentinel service to monitor",
         default=None,
     )
 
-    REDIS_SENTINEL_USERNAME: Optional[str] = Field(
+    REDIS_SENTINEL_USERNAME: str | None = Field(
         description="Username for Redis Sentinel authentication (if required)",
         default=None,
     )
 
-    REDIS_SENTINEL_PASSWORD: Optional[str] = Field(
+    REDIS_SENTINEL_PASSWORD: str | None = Field(
         description="Password for Redis Sentinel authentication (if required)",
         default=None,
     )
 
-    REDIS_SENTINEL_SOCKET_TIMEOUT: Optional[PositiveFloat] = Field(
+    REDIS_SENTINEL_SOCKET_TIMEOUT: PositiveFloat | None = Field(
         description="Socket timeout in seconds for Redis Sentinel connections",
         default=0.1,
     )
@@ -74,12 +92,12 @@ class RedisConfig(BaseSettings):
         default=False,
     )
 
-    REDIS_CLUSTERS: Optional[str] = Field(
+    REDIS_CLUSTERS: str | None = Field(
         description="Comma-separated list of Redis Clusters nodes (host:port)",
         default=None,
     )
 
-    REDIS_CLUSTERS_PASSWORD: Optional[str] = Field(
+    REDIS_CLUSTERS_PASSWORD: str | None = Field(
         description="Password for Redis Clusters authentication (if required)",
         default=None,
     )

api/configs/middleware/storage/aliyun_oss_storage_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field
 from pydantic_settings import BaseSettings
 
@@ -9,37 +7,37 @@ class AliyunOSSStorageConfig(BaseSettings):
     Configuration settings for Aliyun Object Storage Service (OSS)
     """
 
-    ALIYUN_OSS_BUCKET_NAME: Optional[str] = Field(
+    ALIYUN_OSS_BUCKET_NAME: str | None = Field(
         description="Name of the Aliyun OSS bucket to store and retrieve objects",
         default=None,
     )
 
-    ALIYUN_OSS_ACCESS_KEY: Optional[str] = Field(
+    ALIYUN_OSS_ACCESS_KEY: str | None = Field(
         description="Access key ID for authenticating with Aliyun OSS",
         default=None,
     )
 
-    ALIYUN_OSS_SECRET_KEY: Optional[str] = Field(
+    ALIYUN_OSS_SECRET_KEY: str | None = Field(
         description="Secret access key for authenticating with Aliyun OSS",
         default=None,
     )
 
-    ALIYUN_OSS_ENDPOINT: Optional[str] = Field(
+    ALIYUN_OSS_ENDPOINT: str | None = Field(
         description="URL of the Aliyun OSS endpoint for your chosen region",
         default=None,
     )
 
-    ALIYUN_OSS_REGION: Optional[str] = Field(
+    ALIYUN_OSS_REGION: str | None = Field(
         description="Aliyun OSS region where your bucket is located (e.g., 'oss-cn-hangzhou')",
         default=None,
     )
 
-    ALIYUN_OSS_AUTH_VERSION: Optional[str] = Field(
+    ALIYUN_OSS_AUTH_VERSION: str | None = Field(
         description="Version of the authentication protocol to use with Aliyun OSS (e.g., 'v4')",
         default=None,
     )
 
-    ALIYUN_OSS_PATH: Optional[str] = Field(
+    ALIYUN_OSS_PATH: str | None = Field(
         description="Base path within the bucket to store objects (e.g., 'my-app-data/')",
         default=None,
     )

api/configs/middleware/storage/amazon_s3_storage_config.py

@@ -1,4 +1,4 @@
-from typing import Literal, Optional
+from typing import Literal
 
 from pydantic import Field
 from pydantic_settings import BaseSettings
@@ -9,27 +9,27 @@ class S3StorageConfig(BaseSettings):
     Configuration settings for S3-compatible object storage
     """
 
-    S3_ENDPOINT: Optional[str] = Field(
+    S3_ENDPOINT: str | None = Field(
         description="URL of the S3-compatible storage endpoint (e.g., 'https://s3.amazonaws.com')",
         default=None,
     )
 
-    S3_REGION: Optional[str] = Field(
+    S3_REGION: str | None = Field(
         description="Region where the S3 bucket is located (e.g., 'us-east-1')",
         default=None,
     )
 
-    S3_BUCKET_NAME: Optional[str] = Field(
+    S3_BUCKET_NAME: str | None = Field(
         description="Name of the S3 bucket to store and retrieve objects",
         default=None,
     )
 
-    S3_ACCESS_KEY: Optional[str] = Field(
+    S3_ACCESS_KEY: str | None = Field(
         description="Access key ID for authenticating with the S3 service",
         default=None,
     )
 
-    S3_SECRET_KEY: Optional[str] = Field(
+    S3_SECRET_KEY: str | None = Field(
         description="Secret access key for authenticating with the S3 service",
         default=None,
     )

api/configs/middleware/storage/azure_blob_storage_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field
 from pydantic_settings import BaseSettings
 
@@ -9,22 +7,22 @@ class AzureBlobStorageConfig(BaseSettings):
     Configuration settings for Azure Blob Storage
     """
 
-    AZURE_BLOB_ACCOUNT_NAME: Optional[str] = Field(
+    AZURE_BLOB_ACCOUNT_NAME: str | None = Field(
         description="Name of the Azure Storage account (e.g., 'mystorageaccount')",
         default=None,
     )
 
-    AZURE_BLOB_ACCOUNT_KEY: Optional[str] = Field(
+    AZURE_BLOB_ACCOUNT_KEY: str | None = Field(
         description="Access key for authenticating with the Azure Storage account",
         default=None,
     )
 
-    AZURE_BLOB_CONTAINER_NAME: Optional[str] = Field(
+    AZURE_BLOB_CONTAINER_NAME: str | None = Field(
         description="Name of the Azure Blob container to store and retrieve objects",
         default=None,
     )
 
-    AZURE_BLOB_ACCOUNT_URL: Optional[str] = Field(
+    AZURE_BLOB_ACCOUNT_URL: str | None = Field(
         description="URL of the Azure Blob storage endpoint (e.g., 'https://mystorageaccount.blob.core.windows.net')",
         default=None,
     )

api/configs/middleware/storage/baidu_obs_storage_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field
 from pydantic_settings import BaseSettings
 
@@ -9,22 +7,22 @@ class BaiduOBSStorageConfig(BaseSettings):
     Configuration settings for Baidu Object Storage Service (OBS)
     """
 
-    BAIDU_OBS_BUCKET_NAME: Optional[str] = Field(
+    BAIDU_OBS_BUCKET_NAME: str | None = Field(
         description="Name of the Baidu OBS bucket to store and retrieve objects (e.g., 'my-obs-bucket')",
         default=None,
     )
 
-    BAIDU_OBS_ACCESS_KEY: Optional[str] = Field(
+    BAIDU_OBS_ACCESS_KEY: str | None = Field(
         description="Access Key ID for authenticating with Baidu OBS",
         default=None,
     )
 
-    BAIDU_OBS_SECRET_KEY: Optional[str] = Field(
+    BAIDU_OBS_SECRET_KEY: str | None = Field(
         description="Secret Access Key for authenticating with Baidu OBS",
         default=None,
     )
 
-    BAIDU_OBS_ENDPOINT: Optional[str] = Field(
+    BAIDU_OBS_ENDPOINT: str | None = Field(
         description="URL of the Baidu OSS endpoint for your chosen region (e.g., 'https://.bj.bcebos.com')",
         default=None,
     )

api/configs/middleware/storage/clickzetta_volume_storage_config.py

@@ -1,7 +1,5 @@
 """ClickZetta Volume Storage Configuration"""
 
-from typing import Optional
-
 from pydantic import Field
 from pydantic_settings import BaseSettings
 
@@ -9,17 +7,17 @@ from pydantic_settings import BaseSettings
 class ClickZettaVolumeStorageConfig(BaseSettings):
     """Configuration for ClickZetta Volume storage."""
 
-    CLICKZETTA_VOLUME_USERNAME: Optional[str] = Field(
+    CLICKZETTA_VOLUME_USERNAME: str | None = Field(
         description="Username for ClickZetta Volume authentication",
         default=None,
     )
 
-    CLICKZETTA_VOLUME_PASSWORD: Optional[str] = Field(
+    CLICKZETTA_VOLUME_PASSWORD: str | None = Field(
         description="Password for ClickZetta Volume authentication",
         default=None,
     )
 
-    CLICKZETTA_VOLUME_INSTANCE: Optional[str] = Field(
+    CLICKZETTA_VOLUME_INSTANCE: str | None = Field(
         description="ClickZetta instance identifier",
         default=None,
     )
@@ -49,7 +47,7 @@ class ClickZettaVolumeStorageConfig(BaseSettings):
         default="user",
     )
 
-    CLICKZETTA_VOLUME_NAME: Optional[str] = Field(
+    CLICKZETTA_VOLUME_NAME: str | None = Field(
         description="ClickZetta volume name for external volumes",
         default=None,
     )

api/configs/middleware/storage/google_cloud_storage_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field
 from pydantic_settings import BaseSettings
 
@@ -9,12 +7,12 @@ class GoogleCloudStorageConfig(BaseSettings):
     Configuration settings for Google Cloud Storage
     """
 
-    GOOGLE_STORAGE_BUCKET_NAME: Optional[str] = Field(
+    GOOGLE_STORAGE_BUCKET_NAME: str | None = Field(
         description="Name of the Google Cloud Storage bucket to store and retrieve objects (e.g., 'my-gcs-bucket')",
         default=None,
     )
 
-    GOOGLE_STORAGE_SERVICE_ACCOUNT_JSON_BASE64: Optional[str] = Field(
+    GOOGLE_STORAGE_SERVICE_ACCOUNT_JSON_BASE64: str | None = Field(
         description="Base64-encoded JSON key file for Google Cloud service account authentication",
         default=None,
     )

api/configs/middleware/storage/huawei_obs_storage_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field
 from pydantic_settings import BaseSettings
 
@@ -9,22 +7,22 @@ class HuaweiCloudOBSStorageConfig(BaseSettings):
     Configuration settings for Huawei Cloud Object Storage Service (OBS)
     """
 
-    HUAWEI_OBS_BUCKET_NAME: Optional[str] = Field(
+    HUAWEI_OBS_BUCKET_NAME: str | None = Field(
         description="Name of the Huawei Cloud OBS bucket to store and retrieve objects (e.g., 'my-obs-bucket')",
         default=None,
     )
 
-    HUAWEI_OBS_ACCESS_KEY: Optional[str] = Field(
+    HUAWEI_OBS_ACCESS_KEY: str | None = Field(
         description="Access Key ID for authenticating with Huawei Cloud OBS",
         default=None,
     )
 
-    HUAWEI_OBS_SECRET_KEY: Optional[str] = Field(
+    HUAWEI_OBS_SECRET_KEY: str | None = Field(
         description="Secret Access Key for authenticating with Huawei Cloud OBS",
         default=None,
     )
 
-    HUAWEI_OBS_SERVER: Optional[str] = Field(
+    HUAWEI_OBS_SERVER: str | None = Field(
         description="Endpoint URL for Huawei Cloud OBS (e.g., 'https://obs.cn-north-4.myhuaweicloud.com')",
         default=None,
     )

api/configs/middleware/storage/oci_storage_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field
 from pydantic_settings import BaseSettings
 
@@ -9,27 +7,27 @@ class OCIStorageConfig(BaseSettings):
     Configuration settings for Oracle Cloud Infrastructure (OCI) Object Storage
     """
 
-    OCI_ENDPOINT: Optional[str] = Field(
+    OCI_ENDPOINT: str | None = Field(
         description="URL of the OCI Object Storage endpoint (e.g., 'https://objectstorage.us-phoenix-1.oraclecloud.com')",
         default=None,
     )
 
-    OCI_REGION: Optional[str] = Field(
+    OCI_REGION: str | None = Field(
         description="OCI region where the bucket is located (e.g., 'us-phoenix-1')",
         default=None,
     )
 
-    OCI_BUCKET_NAME: Optional[str] = Field(
+    OCI_BUCKET_NAME: str | None = Field(
         description="Name of the OCI Object Storage bucket to store and retrieve objects (e.g., 'my-oci-bucket')",
         default=None,
     )
 
-    OCI_ACCESS_KEY: Optional[str] = Field(
+    OCI_ACCESS_KEY: str | None = Field(
         description="Access key (also known as API key) for authenticating with OCI Object Storage",
         default=None,
     )
 
-    OCI_SECRET_KEY: Optional[str] = Field(
+    OCI_SECRET_KEY: str | None = Field(
         description="Secret key associated with the access key for authenticating with OCI Object Storage",
         default=None,
     )

api/configs/middleware/storage/supabase_storage_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field
 from pydantic_settings import BaseSettings
 
@@ -9,17 +7,17 @@ class SupabaseStorageConfig(BaseSettings):
     Configuration settings for Supabase Object Storage Service
     """
 
-    SUPABASE_BUCKET_NAME: Optional[str] = Field(
+    SUPABASE_BUCKET_NAME: str | None = Field(
         description="Name of the Supabase bucket to store and retrieve objects (e.g., 'dify-bucket')",
         default=None,
     )
 
-    SUPABASE_API_KEY: Optional[str] = Field(
+    SUPABASE_API_KEY: str | None = Field(
         description="API KEY for authenticating with Supabase",
         default=None,
     )
 
-    SUPABASE_URL: Optional[str] = Field(
+    SUPABASE_URL: str | None = Field(
         description="URL of the Supabase",
         default=None,
     )

api/configs/middleware/storage/tencent_cos_storage_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field
 from pydantic_settings import BaseSettings
 
@@ -9,27 +7,27 @@ class TencentCloudCOSStorageConfig(BaseSettings):
     Configuration settings for Tencent Cloud Object Storage (COS)
     """
 
-    TENCENT_COS_BUCKET_NAME: Optional[str] = Field(
+    TENCENT_COS_BUCKET_NAME: str | None = Field(
         description="Name of the Tencent Cloud COS bucket to store and retrieve objects",
         default=None,
     )
 
-    TENCENT_COS_REGION: Optional[str] = Field(
+    TENCENT_COS_REGION: str | None = Field(
         description="Tencent Cloud region where the COS bucket is located (e.g., 'ap-guangzhou')",
         default=None,
     )
 
-    TENCENT_COS_SECRET_ID: Optional[str] = Field(
+    TENCENT_COS_SECRET_ID: str | None = Field(
         description="SecretId for authenticating with Tencent Cloud COS (part of API credentials)",
         default=None,
     )
 
-    TENCENT_COS_SECRET_KEY: Optional[str] = Field(
+    TENCENT_COS_SECRET_KEY: str | None = Field(
         description="SecretKey for authenticating with Tencent Cloud COS (part of API credentials)",
         default=None,
     )
 
-    TENCENT_COS_SCHEME: Optional[str] = Field(
+    TENCENT_COS_SCHEME: str | None = Field(
         description="Protocol scheme for COS requests: 'https' (recommended) or 'http'",
         default=None,
     )

api/configs/middleware/storage/volcengine_tos_storage_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field
 from pydantic_settings import BaseSettings
 
@@ -9,27 +7,27 @@ class VolcengineTOSStorageConfig(BaseSettings):
     Configuration settings for Volcengine Tinder Object Storage (TOS)
     """
 
-    VOLCENGINE_TOS_BUCKET_NAME: Optional[str] = Field(
+    VOLCENGINE_TOS_BUCKET_NAME: str | None = Field(
         description="Name of the Volcengine TOS bucket to store and retrieve objects (e.g., 'my-tos-bucket')",
         default=None,
     )
 
-    VOLCENGINE_TOS_ACCESS_KEY: Optional[str] = Field(
+    VOLCENGINE_TOS_ACCESS_KEY: str | None = Field(
         description="Access Key ID for authenticating with Volcengine TOS",
         default=None,
     )
 
-    VOLCENGINE_TOS_SECRET_KEY: Optional[str] = Field(
+    VOLCENGINE_TOS_SECRET_KEY: str | None = Field(
         description="Secret Access Key for authenticating with Volcengine TOS",
         default=None,
     )
 
-    VOLCENGINE_TOS_ENDPOINT: Optional[str] = Field(
+    VOLCENGINE_TOS_ENDPOINT: str | None = Field(
         description="URL of the Volcengine TOS endpoint (e.g., 'https://tos-cn-beijing.volces.com')",
         default=None,
     )
 
-    VOLCENGINE_TOS_REGION: Optional[str] = Field(
+    VOLCENGINE_TOS_REGION: str | None = Field(
         description="Volcengine region where the TOS bucket is located (e.g., 'cn-beijing')",
         default=None,
     )

api/configs/middleware/vdb/alibabacloud_mysql_config.py

@@ -0,0 +1,54 @@
+from pydantic import Field, PositiveInt
+from pydantic_settings import BaseSettings
+
+
+class AlibabaCloudMySQLConfig(BaseSettings):
+    """
+    Configuration settings for AlibabaCloud MySQL vector database
+    """
+
+    ALIBABACLOUD_MYSQL_HOST: str = Field(
+        description="Hostname or IP address of the AlibabaCloud MySQL server (e.g., 'localhost' or 'mysql.aliyun.com')",
+        default="localhost",
+    )
+
+    ALIBABACLOUD_MYSQL_PORT: PositiveInt = Field(
+        description="Port number on which the AlibabaCloud MySQL server is listening (default is 3306)",
+        default=3306,
+    )
+
+    ALIBABACLOUD_MYSQL_USER: str = Field(
+        description="Username for authenticating with AlibabaCloud MySQL (default is 'root')",
+        default="root",
+    )
+
+    ALIBABACLOUD_MYSQL_PASSWORD: str = Field(
+        description="Password for authenticating with AlibabaCloud MySQL (default is an empty string)",
+        default="",
+    )
+
+    ALIBABACLOUD_MYSQL_DATABASE: str = Field(
+        description="Name of the AlibabaCloud MySQL database to connect to (default is 'dify')",
+        default="dify",
+    )
+
+    ALIBABACLOUD_MYSQL_MAX_CONNECTION: PositiveInt = Field(
+        description="Maximum number of connections in the connection pool",
+        default=5,
+    )
+
+    ALIBABACLOUD_MYSQL_CHARSET: str = Field(
+        description="Character set for AlibabaCloud MySQL connection (default is 'utf8mb4')",
+        default="utf8mb4",
+    )
+
+    ALIBABACLOUD_MYSQL_DISTANCE_FUNCTION: str = Field(
+        description="Distance function used for vector similarity search in AlibabaCloud MySQL "
+        "(e.g., 'cosine', 'euclidean')",
+        default="cosine",
+    )
+
+    ALIBABACLOUD_MYSQL_HNSW_M: PositiveInt = Field(
+        description="Maximum number of connections per layer for HNSW vector index (default is 6, range: 3-200)",
+        default=6,
+    )

api/configs/middleware/vdb/analyticdb_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field, PositiveInt
 from pydantic_settings import BaseSettings
 
@@ -11,37 +9,37 @@ class AnalyticdbConfig(BaseSettings):
     https://www.alibabacloud.com/help/en/analyticdb-for-postgresql/getting-started/create-an-instance-instances-with-vector-engine-optimization-enabled
     """
 
-    ANALYTICDB_KEY_ID: Optional[str] = Field(
+    ANALYTICDB_KEY_ID: str | None = Field(
         default=None, description="The Access Key ID provided by Alibaba Cloud for API authentication."
     )
-    ANALYTICDB_KEY_SECRET: Optional[str] = Field(
+    ANALYTICDB_KEY_SECRET: str | None = Field(
         default=None, description="The Secret Access Key corresponding to the Access Key ID for secure API access."
     )
-    ANALYTICDB_REGION_ID: Optional[str] = Field(
+    ANALYTICDB_REGION_ID: str | None = Field(
         default=None,
         description="The region where the AnalyticDB instance is deployed (e.g., 'cn-hangzhou', 'ap-southeast-1').",
     )
-    ANALYTICDB_INSTANCE_ID: Optional[str] = Field(
+    ANALYTICDB_INSTANCE_ID: str | None = Field(
         default=None,
         description="The unique identifier of the AnalyticDB instance you want to connect to.",
     )
-    ANALYTICDB_ACCOUNT: Optional[str] = Field(
+    ANALYTICDB_ACCOUNT: str | None = Field(
         default=None,
         description="The account name used to log in to the AnalyticDB instance"
         " (usually the initial account created with the instance).",
     )
-    ANALYTICDB_PASSWORD: Optional[str] = Field(
+    ANALYTICDB_PASSWORD: str | None = Field(
         default=None, description="The password associated with the AnalyticDB account for database authentication."
     )
-    ANALYTICDB_NAMESPACE: Optional[str] = Field(
+    ANALYTICDB_NAMESPACE: str | None = Field(
         default=None, description="The namespace within AnalyticDB for schema isolation (if using namespace feature)."
     )
-    ANALYTICDB_NAMESPACE_PASSWORD: Optional[str] = Field(
+    ANALYTICDB_NAMESPACE_PASSWORD: str | None = Field(
         default=None,
         description="The password for accessing the specified namespace within the AnalyticDB instance"
         " (if namespace feature is enabled).",
     )
-    ANALYTICDB_HOST: Optional[str] = Field(
+    ANALYTICDB_HOST: str | None = Field(
         default=None, description="The host of the AnalyticDB instance you want to connect to."
     )
     ANALYTICDB_PORT: PositiveInt = Field(

api/configs/middleware/vdb/baidu_vector_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field, NonNegativeInt, PositiveInt
 from pydantic_settings import BaseSettings
 
@@ -9,7 +7,7 @@ class BaiduVectorDBConfig(BaseSettings):
     Configuration settings for Baidu Vector Database
     """
 
-    BAIDU_VECTOR_DB_ENDPOINT: Optional[str] = Field(
+    BAIDU_VECTOR_DB_ENDPOINT: str | None = Field(
         description="URL of the Baidu Vector Database service (e.g., 'http://vdb.bj.baidubce.com')",
         default=None,
     )
@@ -19,17 +17,17 @@ class BaiduVectorDBConfig(BaseSettings):
         default=30000,
     )
 
-    BAIDU_VECTOR_DB_ACCOUNT: Optional[str] = Field(
+    BAIDU_VECTOR_DB_ACCOUNT: str | None = Field(
         description="Account for authenticating with the Baidu Vector Database",
         default=None,
     )
 
-    BAIDU_VECTOR_DB_API_KEY: Optional[str] = Field(
+    BAIDU_VECTOR_DB_API_KEY: str | None = Field(
         description="API key for authenticating with the Baidu Vector Database service",
         default=None,
     )
 
-    BAIDU_VECTOR_DB_DATABASE: Optional[str] = Field(
+    BAIDU_VECTOR_DB_DATABASE: str | None = Field(
         description="Name of the specific Baidu Vector Database to connect to",
         default=None,
     )
@@ -43,3 +41,13 @@ class BaiduVectorDBConfig(BaseSettings):
         description="Number of replicas for the Baidu Vector Database (default is 3)",
         default=3,
     )
+
+    BAIDU_VECTOR_DB_INVERTED_INDEX_ANALYZER: str = Field(
+        description="Analyzer type for inverted index in Baidu Vector Database (default is DEFAULT_ANALYZER)",
+        default="DEFAULT_ANALYZER",
+    )
+
+    BAIDU_VECTOR_DB_INVERTED_INDEX_PARSER_MODE: str = Field(
+        description="Parser mode for inverted index in Baidu Vector Database (default is COARSE_MODE)",
+        default="COARSE_MODE",
+    )

api/configs/middleware/vdb/chroma_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field, PositiveInt
 from pydantic_settings import BaseSettings
 
@@ -9,7 +7,7 @@ class ChromaConfig(BaseSettings):
     Configuration settings for Chroma vector database
     """
 
-    CHROMA_HOST: Optional[str] = Field(
+    CHROMA_HOST: str | None = Field(
         description="Hostname or IP address of the Chroma server (e.g., 'localhost' or '192.168.1.100')",
         default=None,
     )
@@ -19,22 +17,22 @@ class ChromaConfig(BaseSettings):
         default=8000,
     )
 
-    CHROMA_TENANT: Optional[str] = Field(
+    CHROMA_TENANT: str | None = Field(
         description="Tenant identifier for multi-tenancy support in Chroma",
         default=None,
     )
 
-    CHROMA_DATABASE: Optional[str] = Field(
+    CHROMA_DATABASE: str | None = Field(
         description="Name of the Chroma database to connect to",
         default=None,
     )
 
-    CHROMA_AUTH_PROVIDER: Optional[str] = Field(
+    CHROMA_AUTH_PROVIDER: str | None = Field(
         description="Authentication provider for Chroma (e.g., 'basic', 'token', or a custom provider)",
         default=None,
     )
 
-    CHROMA_AUTH_CREDENTIALS: Optional[str] = Field(
+    CHROMA_AUTH_CREDENTIALS: str | None = Field(
         description="Authentication credentials for Chroma (format depends on the auth provider)",
         default=None,
     )

api/configs/middleware/vdb/clickzetta_config.py

@@ -1,69 +1,68 @@
-from typing import Optional
-
-from pydantic import BaseModel, Field
+from pydantic import Field
+from pydantic_settings import BaseSettings
 
 
-class ClickzettaConfig(BaseModel):
+class ClickzettaConfig(BaseSettings):
     """
     Clickzetta Lakehouse vector database configuration
     """
 
-    CLICKZETTA_USERNAME: Optional[str] = Field(
+    CLICKZETTA_USERNAME: str | None = Field(
         description="Username for authenticating with Clickzetta Lakehouse",
         default=None,
     )
 
-    CLICKZETTA_PASSWORD: Optional[str] = Field(
+    CLICKZETTA_PASSWORD: str | None = Field(
         description="Password for authenticating with Clickzetta Lakehouse",
         default=None,
     )
 
-    CLICKZETTA_INSTANCE: Optional[str] = Field(
+    CLICKZETTA_INSTANCE: str | None = Field(
         description="Clickzetta Lakehouse instance ID",
         default=None,
     )
 
-    CLICKZETTA_SERVICE: Optional[str] = Field(
+    CLICKZETTA_SERVICE: str | None = Field(
         description="Clickzetta API service endpoint (e.g., 'api.clickzetta.com')",
         default="api.clickzetta.com",
     )
 
-    CLICKZETTA_WORKSPACE: Optional[str] = Field(
+    CLICKZETTA_WORKSPACE: str | None = Field(
         description="Clickzetta workspace name",
         default="default",
     )
 
-    CLICKZETTA_VCLUSTER: Optional[str] = Field(
+    CLICKZETTA_VCLUSTER: str | None = Field(
         description="Clickzetta virtual cluster name",
         default="default_ap",
     )
 
-    CLICKZETTA_SCHEMA: Optional[str] = Field(
+    CLICKZETTA_SCHEMA: str | None = Field(
         description="Database schema name in Clickzetta",
         default="public",
     )
 
-    CLICKZETTA_BATCH_SIZE: Optional[int] = Field(
+    CLICKZETTA_BATCH_SIZE: int | None = Field(
         description="Batch size for bulk insert operations",
         default=100,
     )
 
-    CLICKZETTA_ENABLE_INVERTED_INDEX: Optional[bool] = Field(
+    CLICKZETTA_ENABLE_INVERTED_INDEX: bool | None = Field(
         description="Enable inverted index for full-text search capabilities",
         default=True,
     )
 
-    CLICKZETTA_ANALYZER_TYPE: Optional[str] = Field(
+    CLICKZETTA_ANALYZER_TYPE: str | None = Field(
         description="Analyzer type for full-text search: keyword, english, chinese, unicode",
         default="chinese",
     )
 
-    CLICKZETTA_ANALYZER_MODE: Optional[str] = Field(
+    CLICKZETTA_ANALYZER_MODE: str | None = Field(
         description="Analyzer mode for tokenization: max_word (fine-grained) or smart (intelligent)",
         default="smart",
     )
 
-    CLICKZETTA_VECTOR_DISTANCE_FUNCTION: Optional[str] = Field(
+    CLICKZETTA_VECTOR_DISTANCE_FUNCTION: str | None = Field(
         description="Distance function for vector similarity: l2_distance or cosine_distance",
         default="cosine_distance",
     )

api/configs/middleware/vdb/couchbase_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field
 from pydantic_settings import BaseSettings
 
@@ -9,27 +7,27 @@ class CouchbaseConfig(BaseSettings):
     Couchbase configs
     """
 
-    COUCHBASE_CONNECTION_STRING: Optional[str] = Field(
+    COUCHBASE_CONNECTION_STRING: str | None = Field(
         description="COUCHBASE connection string",
         default=None,
     )
 
-    COUCHBASE_USER: Optional[str] = Field(
+    COUCHBASE_USER: str | None = Field(
         description="COUCHBASE user",
         default=None,
     )
 
-    COUCHBASE_PASSWORD: Optional[str] = Field(
+    COUCHBASE_PASSWORD: str | None = Field(
         description="COUCHBASE password",
         default=None,
     )
 
-    COUCHBASE_BUCKET_NAME: Optional[str] = Field(
+    COUCHBASE_BUCKET_NAME: str | None = Field(
         description="COUCHBASE bucket name",
         default=None,
     )
 
-    COUCHBASE_SCOPE_NAME: Optional[str] = Field(
+    COUCHBASE_SCOPE_NAME: str | None = Field(
         description="COUCHBASE scope name",
         default=None,
     )

api/configs/middleware/vdb/elasticsearch_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field, PositiveInt, model_validator
 from pydantic_settings import BaseSettings
 
@@ -10,7 +8,7 @@ class ElasticsearchConfig(BaseSettings):
     Can load from environment variables or .env files.
     """
 
-    ELASTICSEARCH_HOST: Optional[str] = Field(
+    ELASTICSEARCH_HOST: str | None = Field(
         description="Hostname or IP address of the Elasticsearch server (e.g., 'localhost' or '192.168.1.100')",
         default="127.0.0.1",
     )
@@ -20,30 +18,28 @@ class ElasticsearchConfig(BaseSettings):
         default=9200,
     )
 
-    ELASTICSEARCH_USERNAME: Optional[str] = Field(
+    ELASTICSEARCH_USERNAME: str | None = Field(
         description="Username for authenticating with Elasticsearch (default is 'elastic')",
         default="elastic",
     )
 
-    ELASTICSEARCH_PASSWORD: Optional[str] = Field(
+    ELASTICSEARCH_PASSWORD: str | None = Field(
         description="Password for authenticating with Elasticsearch (default is 'elastic')",
         default="elastic",
     )
 
     # Elastic Cloud (optional)
-    ELASTICSEARCH_USE_CLOUD: Optional[bool] = Field(
+    ELASTICSEARCH_USE_CLOUD: bool | None = Field(
         description="Set to True to use Elastic Cloud instead of self-hosted Elasticsearch", default=False
     )
-    ELASTICSEARCH_CLOUD_URL: Optional[str] = Field(
+    ELASTICSEARCH_CLOUD_URL: str | None = Field(
         description="Full URL for Elastic Cloud deployment (e.g., 'https://example.es.region.aws.found.io:443')",
         default=None,
     )
-    ELASTICSEARCH_API_KEY: Optional[str] = Field(
-        description="API key for authenticating with Elastic Cloud", default=None
-    )
+    ELASTICSEARCH_API_KEY: str | None = Field(description="API key for authenticating with Elastic Cloud", default=None)
 
     # Common options
-    ELASTICSEARCH_CA_CERTS: Optional[str] = Field(
+    ELASTICSEARCH_CA_CERTS: str | None = Field(
         description="Path to CA certificate file for SSL verification", default=None
     )
     ELASTICSEARCH_VERIFY_CERTS: bool = Field(

api/configs/middleware/vdb/huawei_cloud_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field
 from pydantic_settings import BaseSettings
 
@@ -9,17 +7,17 @@ class HuaweiCloudConfig(BaseSettings):
     Configuration settings for Huawei cloud search service
     """
 
-    HUAWEI_CLOUD_HOSTS: Optional[str] = Field(
+    HUAWEI_CLOUD_HOSTS: str | None = Field(
         description="Hostname or IP address of the Huawei cloud search service instance",
         default=None,
     )
 
-    HUAWEI_CLOUD_USER: Optional[str] = Field(
+    HUAWEI_CLOUD_USER: str | None = Field(
         description="Username for authenticating with Huawei cloud search service",
         default=None,
     )
 
-    HUAWEI_CLOUD_PASSWORD: Optional[str] = Field(
+    HUAWEI_CLOUD_PASSWORD: str | None = Field(
         description="Password for authenticating with Huawei cloud search service",
         default=None,
     )

api/configs/middleware/vdb/lindorm_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field
 from pydantic_settings import BaseSettings
 
@@ -9,27 +7,27 @@ class LindormConfig(BaseSettings):
     Lindorm configs
     """
 
-    LINDORM_URL: Optional[str] = Field(
+    LINDORM_URL: str | None = Field(
         description="Lindorm url",
         default=None,
     )
-    LINDORM_USERNAME: Optional[str] = Field(
+    LINDORM_USERNAME: str | None = Field(
         description="Lindorm user",
         default=None,
     )
-    LINDORM_PASSWORD: Optional[str] = Field(
+    LINDORM_PASSWORD: str | None = Field(
         description="Lindorm password",
         default=None,
     )
-    DEFAULT_INDEX_TYPE: Optional[str] = Field(
+    LINDORM_INDEX_TYPE: str | None = Field(
         description="Lindorm Vector Index Type, hnsw or flat is available in dify",
         default="hnsw",
     )
-    DEFAULT_DISTANCE_TYPE: Optional[str] = Field(
+    LINDORM_DISTANCE_TYPE: str | None = Field(
         description="Vector Distance Type, support l2, cosinesimil, innerproduct", default="l2"
     )
-    USING_UGC_INDEX: Optional[bool] = Field(
-        description="Using UGC index will store the same type of Index in a single index but can retrieve separately.",
-        default=False,
+    LINDORM_USING_UGC: bool | None = Field(
+        description="Using UGC index will store indexes with the same IndexType/Dimension in a single big index.",
+        default=True,
     )
-    LINDORM_QUERY_TIMEOUT: Optional[float] = Field(description="The lindorm search request timeout (s)", default=2.0)
+    LINDORM_QUERY_TIMEOUT: float | None = Field(description="The lindorm search request timeout (s)", default=2.0)

api/configs/middleware/vdb/matrixone_config.py

@@ -1,7 +1,8 @@
-from pydantic import BaseModel, Field
+from pydantic import Field
+from pydantic_settings import BaseSettings
 
 
-class MatrixoneConfig(BaseModel):
+class MatrixoneConfig(BaseSettings):
     """Matrixone vector database configuration."""
 
     MATRIXONE_HOST: str = Field(default="localhost", description="Host address of the Matrixone server")

api/configs/middleware/vdb/milvus_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field
 from pydantic_settings import BaseSettings
 
@@ -9,22 +7,22 @@ class MilvusConfig(BaseSettings):
     Configuration settings for Milvus vector database
     """
 
-    MILVUS_URI: Optional[str] = Field(
+    MILVUS_URI: str | None = Field(
         description="URI for connecting to the Milvus server (e.g., 'http://localhost:19530' or 'https://milvus-instance.example.com:19530')",
         default="http://127.0.0.1:19530",
     )
 
-    MILVUS_TOKEN: Optional[str] = Field(
+    MILVUS_TOKEN: str | None = Field(
         description="Authentication token for Milvus, if token-based authentication is enabled",
         default=None,
     )
 
-    MILVUS_USER: Optional[str] = Field(
+    MILVUS_USER: str | None = Field(
         description="Username for authenticating with Milvus, if username/password authentication is enabled",
         default=None,
     )
 
-    MILVUS_PASSWORD: Optional[str] = Field(
+    MILVUS_PASSWORD: str | None = Field(
         description="Password for authenticating with Milvus, if username/password authentication is enabled",
         default=None,
     )
@@ -40,7 +38,7 @@ class MilvusConfig(BaseSettings):
         default=True,
     )
 
-    MILVUS_ANALYZER_PARAMS: Optional[str] = Field(
+    MILVUS_ANALYZER_PARAMS: str | None = Field(
         description='Milvus text analyzer parameters, e.g., {"type": "chinese"} for Chinese segmentation support.',
         default=None,
     )

api/configs/middleware/vdb/oceanbase_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field, PositiveInt
 from pydantic_settings import BaseSettings
 
@@ -9,27 +7,27 @@ class OceanBaseVectorConfig(BaseSettings):
     Configuration settings for OceanBase Vector database
     """
 
-    OCEANBASE_VECTOR_HOST: Optional[str] = Field(
+    OCEANBASE_VECTOR_HOST: str | None = Field(
         description="Hostname or IP address of the OceanBase Vector server (e.g. 'localhost')",
         default=None,
     )
 
-    OCEANBASE_VECTOR_PORT: Optional[PositiveInt] = Field(
+    OCEANBASE_VECTOR_PORT: PositiveInt | None = Field(
         description="Port number on which the OceanBase Vector server is listening (default is 2881)",
         default=2881,
     )
 
-    OCEANBASE_VECTOR_USER: Optional[str] = Field(
+    OCEANBASE_VECTOR_USER: str | None = Field(
         description="Username for authenticating with the OceanBase Vector database",
         default=None,
     )
 
-    OCEANBASE_VECTOR_PASSWORD: Optional[str] = Field(
+    OCEANBASE_VECTOR_PASSWORD: str | None = Field(
         description="Password for authenticating with the OceanBase Vector database",
         default=None,
     )
 
-    OCEANBASE_VECTOR_DATABASE: Optional[str] = Field(
+    OCEANBASE_VECTOR_DATABASE: str | None = Field(
         description="Name of the OceanBase Vector database to connect to",
         default=None,
     )
@@ -39,3 +37,15 @@ class OceanBaseVectorConfig(BaseSettings):
         "with older versions",
         default=False,
     )
+
+    OCEANBASE_FULLTEXT_PARSER: str | None = Field(
+        description=(
+            "Fulltext parser to use for text indexing. "
+            "Built-in options: 'ngram' (N-gram tokenizer for English/numbers), "
+            "'beng' (Basic English tokenizer), 'space' (Space-based tokenizer), "
+            "'ngram2' (Improved N-gram tokenizer), 'ik' (Chinese tokenizer). "
+            "External plugins (require installation): 'japanese_ftparser' (Japanese tokenizer), "
+            "'thai_ftparser' (Thai tokenizer). Default is 'ik'"
+        ),
+        default="ik",
+    )

api/configs/middleware/vdb/opengauss_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field, PositiveInt
 from pydantic_settings import BaseSettings
 
@@ -9,7 +7,7 @@ class OpenGaussConfig(BaseSettings):
     Configuration settings for OpenGauss
     """
 
-    OPENGAUSS_HOST: Optional[str] = Field(
+    OPENGAUSS_HOST: str | None = Field(
         description="Hostname or IP address of the OpenGauss server(e.g., 'localhost')",
         default=None,
     )
@@ -19,17 +17,17 @@ class OpenGaussConfig(BaseSettings):
         default=6600,
     )
 
-    OPENGAUSS_USER: Optional[str] = Field(
+    OPENGAUSS_USER: str | None = Field(
         description="Username for authenticating with the OpenGauss database",
         default=None,
     )
 
-    OPENGAUSS_PASSWORD: Optional[str] = Field(
+    OPENGAUSS_PASSWORD: str | None = Field(
         description="Password for authenticating with the OpenGauss database",
         default=None,
     )
 
-    OPENGAUSS_DATABASE: Optional[str] = Field(
+    OPENGAUSS_DATABASE: str | None = Field(
         description="Name of the OpenGauss database to connect to",
         default=None,
     )

api/configs/middleware/vdb/opensearch_config.py

@@ -1,24 +1,25 @@
-import enum
-from typing import Literal, Optional
+from enum import StrEnum
+from typing import Literal
 
 from pydantic import Field, PositiveInt
 from pydantic_settings import BaseSettings
 
 
+class AuthMethod(StrEnum):
+    """
+    Authentication method for OpenSearch
+    """
+
+    BASIC = "basic"
+    AWS_MANAGED_IAM = "aws_managed_iam"
+
+
 class OpenSearchConfig(BaseSettings):
     """
     Configuration settings for OpenSearch
     """
 
-    class AuthMethod(enum.StrEnum):
-        """
-        Authentication method for OpenSearch
-        """
-
-        BASIC = "basic"
-        AWS_MANAGED_IAM = "aws_managed_iam"
-
-    OPENSEARCH_HOST: Optional[str] = Field(
+    OPENSEARCH_HOST: str | None = Field(
         description="Hostname or IP address of the OpenSearch server (e.g., 'localhost' or 'opensearch.example.com')",
         default=None,
     )
@@ -43,21 +44,21 @@ class OpenSearchConfig(BaseSettings):
         default=AuthMethod.BASIC,
     )
 
-    OPENSEARCH_USER: Optional[str] = Field(
+    OPENSEARCH_USER: str | None = Field(
         description="Username for authenticating with OpenSearch",
         default=None,
     )
 
-    OPENSEARCH_PASSWORD: Optional[str] = Field(
+    OPENSEARCH_PASSWORD: str | None = Field(
         description="Password for authenticating with OpenSearch",
         default=None,
     )
 
-    OPENSEARCH_AWS_REGION: Optional[str] = Field(
+    OPENSEARCH_AWS_REGION: str | None = Field(
         description="AWS region for OpenSearch (e.g. 'us-west-2')",
         default=None,
     )
 
-    OPENSEARCH_AWS_SERVICE: Optional[Literal["es", "aoss"]] = Field(
+    OPENSEARCH_AWS_SERVICE: Literal["es", "aoss"] | None = Field(
         description="AWS service for OpenSearch (e.g. 'aoss' for OpenSearch Serverless)", default=None
     )

api/configs/middleware/vdb/oracle_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field
 from pydantic_settings import BaseSettings
 
@@ -9,33 +7,33 @@ class OracleConfig(BaseSettings):
     Configuration settings for Oracle database
     """
 
-    ORACLE_USER: Optional[str] = Field(
+    ORACLE_USER: str | None = Field(
         description="Username for authenticating with the Oracle database",
         default=None,
     )
 
-    ORACLE_PASSWORD: Optional[str] = Field(
+    ORACLE_PASSWORD: str | None = Field(
         description="Password for authenticating with the Oracle database",
         default=None,
     )
 
-    ORACLE_DSN: Optional[str] = Field(
+    ORACLE_DSN: str | None = Field(
         description="Oracle database connection string. For traditional database, use format 'host:port/service_name'. "
         "For autonomous database, use the service name from tnsnames.ora in the wallet",
         default=None,
     )
 
-    ORACLE_CONFIG_DIR: Optional[str] = Field(
+    ORACLE_CONFIG_DIR: str | None = Field(
         description="Directory containing the tnsnames.ora configuration file. Only used in thin mode connection",
         default=None,
     )
 
-    ORACLE_WALLET_LOCATION: Optional[str] = Field(
+    ORACLE_WALLET_LOCATION: str | None = Field(
         description="Oracle wallet directory path containing the wallet files for secure connection",
         default=None,
     )
 
-    ORACLE_WALLET_PASSWORD: Optional[str] = Field(
+    ORACLE_WALLET_PASSWORD: str | None = Field(
         description="Password to decrypt the Oracle wallet, if it is encrypted",
         default=None,
     )

api/configs/middleware/vdb/pgvector_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field, PositiveInt
 from pydantic_settings import BaseSettings
 
@@ -9,7 +7,7 @@ class PGVectorConfig(BaseSettings):
     Configuration settings for PGVector (PostgreSQL with vector extension)
     """
 
-    PGVECTOR_HOST: Optional[str] = Field(
+    PGVECTOR_HOST: str | None = Field(
         description="Hostname or IP address of the PostgreSQL server with PGVector extension (e.g., 'localhost')",
         default=None,
     )
@@ -19,17 +17,17 @@ class PGVectorConfig(BaseSettings):
         default=5433,
     )
 
-    PGVECTOR_USER: Optional[str] = Field(
+    PGVECTOR_USER: str | None = Field(
         description="Username for authenticating with the PostgreSQL database",
         default=None,
     )
 
-    PGVECTOR_PASSWORD: Optional[str] = Field(
+    PGVECTOR_PASSWORD: str | None = Field(
         description="Password for authenticating with the PostgreSQL database",
         default=None,
     )
 
-    PGVECTOR_DATABASE: Optional[str] = Field(
+    PGVECTOR_DATABASE: str | None = Field(
         description="Name of the PostgreSQL database to connect to",
         default=None,
     )

api/configs/middleware/vdb/pgvectors_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field, PositiveInt
 from pydantic_settings import BaseSettings
 
@@ -9,7 +7,7 @@ class PGVectoRSConfig(BaseSettings):
     Configuration settings for PGVecto.RS (Rust-based vector extension for PostgreSQL)
     """
 
-    PGVECTO_RS_HOST: Optional[str] = Field(
+    PGVECTO_RS_HOST: str | None = Field(
         description="Hostname or IP address of the PostgreSQL server with PGVecto.RS extension (e.g., 'localhost')",
         default=None,
     )
@@ -19,17 +17,17 @@ class PGVectoRSConfig(BaseSettings):
         default=5431,
     )
 
-    PGVECTO_RS_USER: Optional[str] = Field(
+    PGVECTO_RS_USER: str | None = Field(
         description="Username for authenticating with the PostgreSQL database using PGVecto.RS",
         default=None,
     )
 
-    PGVECTO_RS_PASSWORD: Optional[str] = Field(
+    PGVECTO_RS_PASSWORD: str | None = Field(
         description="Password for authenticating with the PostgreSQL database using PGVecto.RS",
         default=None,
     )
 
-    PGVECTO_RS_DATABASE: Optional[str] = Field(
+    PGVECTO_RS_DATABASE: str | None = Field(
         description="Name of the PostgreSQL database with PGVecto.RS extension to connect to",
         default=None,
     )

api/configs/middleware/vdb/qdrant_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field, NonNegativeInt, PositiveInt
 from pydantic_settings import BaseSettings
 
@@ -9,12 +7,12 @@ class QdrantConfig(BaseSettings):
     Configuration settings for Qdrant vector database
     """
 
-    QDRANT_URL: Optional[str] = Field(
+    QDRANT_URL: str | None = Field(
         description="URL of the Qdrant server (e.g., 'http://localhost:6333' or 'https://qdrant.example.com')",
         default=None,
     )
 
-    QDRANT_API_KEY: Optional[str] = Field(
+    QDRANT_API_KEY: str | None = Field(
         description="API key for authenticating with the Qdrant server",
         default=None,
     )

api/configs/middleware/vdb/relyt_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field, PositiveInt
 from pydantic_settings import BaseSettings
 
@@ -9,7 +7,7 @@ class RelytConfig(BaseSettings):
     Configuration settings for Relyt database
     """
 
-    RELYT_HOST: Optional[str] = Field(
+    RELYT_HOST: str | None = Field(
         description="Hostname or IP address of the Relyt server (e.g., 'localhost' or 'relyt.example.com')",
         default=None,
     )
@@ -19,17 +17,17 @@ class RelytConfig(BaseSettings):
         default=9200,
     )
 
-    RELYT_USER: Optional[str] = Field(
+    RELYT_USER: str | None = Field(
         description="Username for authenticating with the Relyt database",
         default=None,
     )
 
-    RELYT_PASSWORD: Optional[str] = Field(
+    RELYT_PASSWORD: str | None = Field(
         description="Password for authenticating with the Relyt database",
         default=None,
     )
 
-    RELYT_DATABASE: Optional[str] = Field(
+    RELYT_DATABASE: str | None = Field(
         description="Name of the Relyt database to connect to (default is 'default')",
         default="default",
     )

api/configs/middleware/vdb/tablestore_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field
 from pydantic_settings import BaseSettings
 
@@ -9,22 +7,22 @@ class TableStoreConfig(BaseSettings):
     Configuration settings for TableStore.
     """
 
-    TABLESTORE_ENDPOINT: Optional[str] = Field(
+    TABLESTORE_ENDPOINT: str | None = Field(
         description="Endpoint address of the TableStore server (e.g. 'https://instance-name.cn-hangzhou.ots.aliyuncs.com')",
         default=None,
     )
 
-    TABLESTORE_INSTANCE_NAME: Optional[str] = Field(
+    TABLESTORE_INSTANCE_NAME: str | None = Field(
         description="Instance name to access TableStore server (eg. 'instance-name')",
         default=None,
     )
 
-    TABLESTORE_ACCESS_KEY_ID: Optional[str] = Field(
+    TABLESTORE_ACCESS_KEY_ID: str | None = Field(
         description="AccessKey id for the instance name",
         default=None,
     )
 
-    TABLESTORE_ACCESS_KEY_SECRET: Optional[str] = Field(
+    TABLESTORE_ACCESS_KEY_SECRET: str | None = Field(
         description="AccessKey secret for the instance name",
         default=None,
     )

api/configs/middleware/vdb/tencent_vector_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field, NonNegativeInt, PositiveInt
 from pydantic_settings import BaseSettings
 
@@ -9,12 +7,12 @@ class TencentVectorDBConfig(BaseSettings):
     Configuration settings for Tencent Vector Database
     """
 
-    TENCENT_VECTOR_DB_URL: Optional[str] = Field(
+    TENCENT_VECTOR_DB_URL: str | None = Field(
         description="URL of the Tencent Vector Database service (e.g., 'https://vectordb.tencentcloudapi.com')",
         default=None,
     )
 
-    TENCENT_VECTOR_DB_API_KEY: Optional[str] = Field(
+    TENCENT_VECTOR_DB_API_KEY: str | None = Field(
         description="API key for authenticating with the Tencent Vector Database service",
         default=None,
     )
@@ -24,12 +22,12 @@ class TencentVectorDBConfig(BaseSettings):
         default=30,
     )
 
-    TENCENT_VECTOR_DB_USERNAME: Optional[str] = Field(
+    TENCENT_VECTOR_DB_USERNAME: str | None = Field(
         description="Username for authenticating with the Tencent Vector Database (if required)",
         default=None,
     )
 
-    TENCENT_VECTOR_DB_PASSWORD: Optional[str] = Field(
+    TENCENT_VECTOR_DB_PASSWORD: str | None = Field(
         description="Password for authenticating with the Tencent Vector Database (if required)",
         default=None,
     )
@@ -44,7 +42,7 @@ class TencentVectorDBConfig(BaseSettings):
         default=2,
     )
 
-    TENCENT_VECTOR_DB_DATABASE: Optional[str] = Field(
+    TENCENT_VECTOR_DB_DATABASE: str | None = Field(
         description="Name of the specific Tencent Vector Database to connect to",
         default=None,
     )

api/configs/middleware/vdb/tidb_on_qdrant_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field, NonNegativeInt, PositiveInt
 from pydantic_settings import BaseSettings
 
@@ -9,12 +7,12 @@ class TidbOnQdrantConfig(BaseSettings):
     Tidb on Qdrant configs
     """
 
-    TIDB_ON_QDRANT_URL: Optional[str] = Field(
+    TIDB_ON_QDRANT_URL: str | None = Field(
         description="Tidb on Qdrant url",
         default=None,
     )
 
-    TIDB_ON_QDRANT_API_KEY: Optional[str] = Field(
+    TIDB_ON_QDRANT_API_KEY: str | None = Field(
         description="Tidb on Qdrant api key",
         default=None,
     )
@@ -34,37 +32,37 @@ class TidbOnQdrantConfig(BaseSettings):
         default=6334,
     )
 
-    TIDB_PUBLIC_KEY: Optional[str] = Field(
+    TIDB_PUBLIC_KEY: str | None = Field(
         description="Tidb account public key",
         default=None,
     )
 
-    TIDB_PRIVATE_KEY: Optional[str] = Field(
+    TIDB_PRIVATE_KEY: str | None = Field(
         description="Tidb account private key",
         default=None,
     )
 
-    TIDB_API_URL: Optional[str] = Field(
+    TIDB_API_URL: str | None = Field(
         description="Tidb API url",
         default=None,
     )
 
-    TIDB_IAM_API_URL: Optional[str] = Field(
+    TIDB_IAM_API_URL: str | None = Field(
         description="Tidb IAM API url",
         default=None,
     )
 
-    TIDB_REGION: Optional[str] = Field(
+    TIDB_REGION: str | None = Field(
         description="Tidb serverless region",
         default="regions/aws-us-east-1",
     )
 
-    TIDB_PROJECT_ID: Optional[str] = Field(
+    TIDB_PROJECT_ID: str | None = Field(
         description="Tidb project id",
         default=None,
     )
 
-    TIDB_SPEND_LIMIT: Optional[int] = Field(
+    TIDB_SPEND_LIMIT: int | None = Field(
         description="Tidb spend limit",
         default=100,
     )

api/configs/middleware/vdb/tidb_vector_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field, PositiveInt
 from pydantic_settings import BaseSettings
 
@@ -9,27 +7,27 @@ class TiDBVectorConfig(BaseSettings):
     Configuration settings for TiDB Vector database
     """
 
-    TIDB_VECTOR_HOST: Optional[str] = Field(
+    TIDB_VECTOR_HOST: str | None = Field(
         description="Hostname or IP address of the TiDB Vector server (e.g., 'localhost' or 'tidb.example.com')",
         default=None,
     )
 
-    TIDB_VECTOR_PORT: Optional[PositiveInt] = Field(
+    TIDB_VECTOR_PORT: PositiveInt | None = Field(
         description="Port number on which the TiDB Vector server is listening (default is 4000)",
         default=4000,
     )
 
-    TIDB_VECTOR_USER: Optional[str] = Field(
+    TIDB_VECTOR_USER: str | None = Field(
         description="Username for authenticating with the TiDB Vector database",
         default=None,
     )
 
-    TIDB_VECTOR_PASSWORD: Optional[str] = Field(
+    TIDB_VECTOR_PASSWORD: str | None = Field(
         description="Password for authenticating with the TiDB Vector database",
         default=None,
     )
 
-    TIDB_VECTOR_DATABASE: Optional[str] = Field(
+    TIDB_VECTOR_DATABASE: str | None = Field(
         description="Name of the TiDB Vector database to connect to",
         default=None,
     )

api/configs/middleware/vdb/upstash_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field
 from pydantic_settings import BaseSettings
 
@@ -9,12 +7,12 @@ class UpstashConfig(BaseSettings):
     Configuration settings for Upstash vector database
     """
 
-    UPSTASH_VECTOR_URL: Optional[str] = Field(
+    UPSTASH_VECTOR_URL: str | None = Field(
         description="URL of the upstash server (e.g., 'https://vector.upstash.io')",
         default=None,
     )
 
-    UPSTASH_VECTOR_TOKEN: Optional[str] = Field(
+    UPSTASH_VECTOR_TOKEN: str | None = Field(
         description="Token for authenticating with the upstash server",
         default=None,
     )

api/configs/middleware/vdb/vastbase_vector_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field, PositiveInt
 from pydantic_settings import BaseSettings
 
@@ -9,7 +7,7 @@ class VastbaseVectorConfig(BaseSettings):
     Configuration settings for Vector (Vastbase with vector extension)
     """
 
-    VASTBASE_HOST: Optional[str] = Field(
+    VASTBASE_HOST: str | None = Field(
         description="Hostname or IP address of the Vastbase server with Vector extension (e.g., 'localhost')",
         default=None,
     )
@@ -19,17 +17,17 @@ class VastbaseVectorConfig(BaseSettings):
         default=5432,
     )
 
-    VASTBASE_USER: Optional[str] = Field(
+    VASTBASE_USER: str | None = Field(
         description="Username for authenticating with the Vastbase database",
         default=None,
     )
 
-    VASTBASE_PASSWORD: Optional[str] = Field(
+    VASTBASE_PASSWORD: str | None = Field(
         description="Password for authenticating with the Vastbase database",
         default=None,
     )
 
-    VASTBASE_DATABASE: Optional[str] = Field(
+    VASTBASE_DATABASE: str | None = Field(
         description="Name of the Vastbase database to connect to",
         default=None,
     )

api/configs/middleware/vdb/vikingdb_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field
 from pydantic_settings import BaseSettings
 
@@ -11,14 +9,14 @@ class VikingDBConfig(BaseSettings):
     https://www.volcengine.com/docs/6291/65568
     """
 
-    VIKINGDB_ACCESS_KEY: Optional[str] = Field(
+    VIKINGDB_ACCESS_KEY: str | None = Field(
         description="The Access Key provided by Volcengine VikingDB for API authentication."
         "Refer to the following documentation for details on obtaining credentials:"
         "https://www.volcengine.com/docs/6291/65568",
         default=None,
     )
 
-    VIKINGDB_SECRET_KEY: Optional[str] = Field(
+    VIKINGDB_SECRET_KEY: str | None = Field(
         description="The Secret Key provided by Volcengine VikingDB for API authentication.",
         default=None,
     )

api/configs/middleware/vdb/weaviate_config.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from pydantic import Field, PositiveInt
 from pydantic_settings import BaseSettings
 
@@ -9,12 +7,12 @@ class WeaviateConfig(BaseSettings):
     Configuration settings for Weaviate vector database
     """
 
-    WEAVIATE_ENDPOINT: Optional[str] = Field(
+    WEAVIATE_ENDPOINT: str | None = Field(
         description="URL of the Weaviate server (e.g., 'http://localhost:8080' or 'https://weaviate.example.com')",
         default=None,
     )
 
-    WEAVIATE_API_KEY: Optional[str] = Field(
+    WEAVIATE_API_KEY: str | None = Field(
         description="API key for authenticating with the Weaviate server",
         default=None,
     )

api/configs/packaging/__init__.py

@@ -1,6 +1,6 @@
 from pydantic import Field
 
-from configs.packaging.pyproject import PyProjectConfig, PyProjectTomlConfig
+from configs.packaging.pyproject import PyProjectTomlConfig
 
 
 class PackagingInfo(PyProjectTomlConfig):

api/configs/remote_settings_sources/apollo/__init__.py

@@ -1,5 +1,5 @@
 from collections.abc import Mapping
-from typing import Any, Optional
+from typing import Any
 
 from pydantic import Field
 from pydantic.fields import FieldInfo
@@ -15,22 +15,22 @@ class ApolloSettingsSourceInfo(BaseSettings):
     Packaging build information
     """
 
-    APOLLO_APP_ID: Optional[str] = Field(
+    APOLLO_APP_ID: str | None = Field(
         description="apollo app_id",
         default=None,
     )
 
-    APOLLO_CLUSTER: Optional[str] = Field(
+    APOLLO_CLUSTER: str | None = Field(
         description="apollo cluster",
         default=None,
     )
 
-    APOLLO_CONFIG_URL: Optional[str] = Field(
+    APOLLO_CONFIG_URL: str | None = Field(
         description="apollo config url",
         default=None,
     )
 
-    APOLLO_NAMESPACE: Optional[str] = Field(
+    APOLLO_NAMESPACE: str | None = Field(
         description="apollo namespace",
         default=None,
     )

api/configs/remote_settings_sources/apollo/client.py

@@ -4,8 +4,9 @@ import logging
 import os
 import threading
 import time
-from collections.abc import Mapping
+from collections.abc import Callable, Mapping
 from pathlib import Path
+from typing import Any
 
 from .python_3x import http_request, makedirs_wrapper
 from .utils import (
@@ -25,13 +26,13 @@ logger = logging.getLogger(__name__)
 class ApolloClient:
     def __init__(
         self,
-        config_url,
-        app_id,
-        cluster="default",
-        secret="",
-        start_hot_update=True,
-        change_listener=None,
-        _notification_map=None,
+        config_url: str,
+        app_id: str,
+        cluster: str = "default",
+        secret: str = "",
+        start_hot_update: bool = True,
+        change_listener: Callable[[str, str, str, Any], None] | None = None,
+        _notification_map: dict[str, int] | None = None,
     ):
         # Core routing parameters
         self.config_url = config_url
@@ -47,17 +48,17 @@ class ApolloClient:
         # Private control variables
         self._cycle_time = 5
         self._stopping = False
-        self._cache = {}
-        self._no_key = {}
-        self._hash = {}
+        self._cache: dict[str, dict[str, Any]] = {}
+        self._no_key: dict[str, str] = {}
+        self._hash: dict[str, str] = {}
         self._pull_timeout = 75
         self._cache_file_path = os.path.expanduser("~") + "/.dify/config/remote-settings/apollo/cache/"
-        self._long_poll_thread = None
+        self._long_poll_thread: threading.Thread | None = None
         self._change_listener = change_listener  # "add" "delete" "update"
         if _notification_map is None:
             _notification_map = {"application": -1}
         self._notification_map = _notification_map
-        self.last_release_key = None
+        self.last_release_key: str | None = None
         # Private startup method
         self._path_checker()
         if start_hot_update:
@@ -68,7 +69,7 @@ class ApolloClient:
         heartbeat.daemon = True
         heartbeat.start()
 
-    def get_json_from_net(self, namespace="application"):
+    def get_json_from_net(self, namespace: str = "application") -> dict[str, Any] | None:
         url = "{}/configs/{}/{}/{}?releaseKey={}&ip={}".format(
             self.config_url, self.app_id, self.cluster, namespace, "", self.ip
         )
@@ -88,7 +89,7 @@ class ApolloClient:
             logger.exception("an error occurred in get_json_from_net")
             return None
 
-    def get_value(self, key, default_val=None, namespace="application"):
+    def get_value(self, key: str, default_val: Any = None, namespace: str = "application") -> Any:
         try:
             # read memory configuration
             namespace_cache = self._cache.get(namespace)
@@ -104,7 +105,8 @@ class ApolloClient:
             namespace_data = self.get_json_from_net(namespace)
             val = get_value_from_dict(namespace_data, key)
             if val is not None:
-                self._update_cache_and_file(namespace_data, namespace)
+                if namespace_data is not None:
+                    self._update_cache_and_file(namespace_data, namespace)
                 return val
 
             # read the file configuration
@@ -126,23 +128,23 @@ class ApolloClient:
     # to ensure the real-time correctness of the function call.
     # If the user does not have the same default val twice
     # and the default val is used here, there may be a problem.
-    def _set_local_cache_none(self, namespace, key):
+    def _set_local_cache_none(self, namespace: str, key: str) -> None:
         no_key = no_key_cache_key(namespace, key)
         self._no_key[no_key] = key
 
-    def _start_hot_update(self):
+    def _start_hot_update(self) -> None:
         self._long_poll_thread = threading.Thread(target=self._listener)
         # When the asynchronous thread is started, the daemon thread will automatically exit
         # when the main thread is launched.
         self._long_poll_thread.daemon = True
         self._long_poll_thread.start()
 
-    def stop(self):
+    def stop(self) -> None:
         self._stopping = True
         logger.info("Stopping listener...")
 
     # Call the set callback function, and if it is abnormal, try it out
-    def _call_listener(self, namespace, old_kv, new_kv):
+    def _call_listener(self, namespace: str, old_kv: dict[str, Any] | None, new_kv: dict[str, Any] | None) -> None:
         if self._change_listener is None:
             return
         if old_kv is None:
@@ -168,12 +170,12 @@ class ApolloClient:
         except BaseException as e:
             logger.warning(str(e))
 
-    def _path_checker(self):
+    def _path_checker(self) -> None:
         if not os.path.isdir(self._cache_file_path):
             makedirs_wrapper(self._cache_file_path)
 
     # update the local cache and file cache
-    def _update_cache_and_file(self, namespace_data, namespace="application"):
+    def _update_cache_and_file(self, namespace_data: dict[str, Any], namespace: str = "application") -> None:
         # update the local cache
         self._cache[namespace] = namespace_data
         # update the file cache
@@ -187,7 +189,7 @@ class ApolloClient:
             self._hash[namespace] = new_hash
 
     # get the configuration from the local file
-    def _get_local_cache(self, namespace="application"):
+    def _get_local_cache(self, namespace: str = "application") -> dict[str, Any]:
         cache_file_path = os.path.join(self._cache_file_path, f"{self.app_id}_configuration_{namespace}.txt")
         if os.path.isfile(cache_file_path):
             with open(cache_file_path) as f:
@@ -195,8 +197,8 @@ class ApolloClient:
             return result
         return {}
 
-    def _long_poll(self):
-        notifications = []
+    def _long_poll(self) -> None:
+        notifications: list[dict[str, Any]] = []
         for key in self._cache:
             namespace_data = self._cache[key]
             notification_id = -1
@@ -236,7 +238,7 @@ class ApolloClient:
         except Exception as e:
             logger.warning(str(e))
 
-    def _get_net_and_set_local(self, namespace, n_id, call_change=False):
+    def _get_net_and_set_local(self, namespace: str, n_id: int, call_change: bool = False) -> None:
         namespace_data = self.get_json_from_net(namespace)
         if not namespace_data:
             return
@@ -248,7 +250,7 @@ class ApolloClient:
             new_kv = namespace_data.get(CONFIGURATIONS)
             self._call_listener(namespace, old_kv, new_kv)
 
-    def _listener(self):
+    def _listener(self) -> None:
         logger.info("start long_poll")
         while not self._stopping:
             self._long_poll()
@@ -266,13 +268,13 @@ class ApolloClient:
         headers["Timestamp"] = time_unix_now
         return headers
 
-    def _heart_beat(self):
+    def _heart_beat(self) -> None:
         while not self._stopping:
             for namespace in self._notification_map:
                 self._do_heart_beat(namespace)
             time.sleep(60 * 10)  # 10 minutes
 
-    def _do_heart_beat(self, namespace):
+    def _do_heart_beat(self, namespace: str) -> None:
         url = f"{self.config_url}/configs/{self.app_id}/{self.cluster}/{namespace}?ip={self.ip}"
         try:
             code, body = http_request(url, timeout=3, headers=self._sign_headers(url))
@@ -292,7 +294,7 @@ class ApolloClient:
             logger.exception("an error occurred in _do_heart_beat")
             return None
 
-    def get_all_dicts(self, namespace):
+    def get_all_dicts(self, namespace: str) -> dict[str, Any] | None:
         namespace_data = self._cache.get(namespace)
         if namespace_data is None:
             net_namespace_data = self.get_json_from_net(namespace)

api/configs/remote_settings_sources/apollo/python_3x.py

@@ -2,6 +2,8 @@ import logging
 import os
 import ssl
 import urllib.request
+from collections.abc import Mapping
+from typing import Any
 from urllib import parse
 from urllib.error import HTTPError
 
@@ -19,9 +21,9 @@ urllib.request.install_opener(opener)
 logger = logging.getLogger(__name__)
 
 
-def http_request(url, timeout, headers={}):
+def http_request(url: str, timeout: int | float, headers: Mapping[str, str] = {}) -> tuple[int, str | None]:
     try:
-        request = urllib.request.Request(url, headers=headers)
+        request = urllib.request.Request(url, headers=dict(headers))
         res = urllib.request.urlopen(request, timeout=timeout)
         body = res.read().decode("utf-8")
         return res.code, body
@@ -33,9 +35,9 @@ def http_request(url, timeout, headers={}):
         raise e
 
 
-def url_encode(params):
+def url_encode(params: dict[str, Any]) -> str:
     return parse.urlencode(params)
 
 
-def makedirs_wrapper(path):
+def makedirs_wrapper(path: str) -> None:
     os.makedirs(path, exist_ok=True)

api/configs/remote_settings_sources/apollo/utils.py

@@ -1,5 +1,6 @@
 import hashlib
 import socket
+from typing import Any
 
 from .python_3x import url_encode
 
@@ -10,7 +11,7 @@ NAMESPACE_NAME = "namespaceName"
 
 
 # add timestamps uris and keys
-def signature(timestamp, uri, secret):
+def signature(timestamp: str, uri: str, secret: str) -> str:
     import base64
     import hmac
 
@@ -19,16 +20,16 @@ def signature(timestamp, uri, secret):
     return base64.b64encode(hmac_code).decode()
 
 
-def url_encode_wrapper(params):
+def url_encode_wrapper(params: dict[str, Any]) -> str:
     return url_encode(params)
 
 
-def no_key_cache_key(namespace, key):
+def no_key_cache_key(namespace: str, key: str) -> str:
     return f"{namespace}{len(namespace)}{key}"
 
 
 # Returns whether the obtained value is obtained, and None if it does not
-def get_value_from_dict(namespace_cache, key):
+def get_value_from_dict(namespace_cache: dict[str, Any] | None, key: str) -> Any:
     if namespace_cache:
         kv_data = namespace_cache.get(CONFIGURATIONS)
         if kv_data is None:
@@ -38,7 +39,7 @@ def get_value_from_dict(namespace_cache, key):
     return None
 
 
-def init_ip():
+def init_ip() -> str:
     ip = ""
     s = None
     try:

api/configs/remote_settings_sources/base.py

@@ -11,5 +11,5 @@ class RemoteSettingsSource:
     def get_field_value(self, field: FieldInfo, field_name: str) -> tuple[Any, str, bool]:
         raise NotImplementedError
 
-    def prepare_field_value(self, field_name: str, field: FieldInfo, value: Any, value_is_complex: bool) -> Any:
+    def prepare_field_value(self, field_name: str, field: FieldInfo, value: Any, value_is_complex: bool):
         return value

api/configs/remote_settings_sources/nacos/__init__.py

@@ -11,16 +11,16 @@ logger = logging.getLogger(__name__)
 
 from configs.remote_settings_sources.base import RemoteSettingsSource
 
-from .utils import _parse_config
+from .utils import parse_config
 
 
 class NacosSettingsSource(RemoteSettingsSource):
     def __init__(self, configs: Mapping[str, Any]):
         self.configs = configs
-        self.remote_configs: dict[str, Any] = {}
+        self.remote_configs: dict[str, str] = {}
         self.async_init()
 
-    def async_init(self):
+    def async_init(self) -> None:
         data_id = os.getenv("DIFY_ENV_NACOS_DATA_ID", "dify-api-env.properties")
         group = os.getenv("DIFY_ENV_NACOS_GROUP", "nacos-dify")
         tenant = os.getenv("DIFY_ENV_NACOS_NAMESPACE", "")
@@ -29,22 +29,19 @@ class NacosSettingsSource(RemoteSettingsSource):
         try:
             content = NacosHttpClient().http_request("/nacos/v1/cs/configs", method="GET", headers={}, params=params)
             self.remote_configs = self._parse_config(content)
-        except Exception as e:
+        except Exception:
             logger.exception("[get-access-token] exception occurred")
             raise
 
-    def _parse_config(self, content: str) -> dict:
+    def _parse_config(self, content: str) -> dict[str, str]:
         if not content:
             return {}
         try:
-            return _parse_config(self, content)
+            return parse_config(content)
         except Exception as e:
             raise RuntimeError(f"Failed to parse config: {e}")
 
     def get_field_value(self, field: FieldInfo, field_name: str) -> tuple[Any, str, bool]:
-        if not isinstance(self.remote_configs, dict):
-            raise ValueError(f"remote configs is not dict, but {type(self.remote_configs)}")
-
         field_value = self.remote_configs.get(field_name)
         if field_value is None:
             return None, field_name, False

api/configs/remote_settings_sources/nacos/http_request.py

@@ -5,7 +5,7 @@ import logging
 import os
 import time
 
-import requests
+import httpx
 
 logger = logging.getLogger(__name__)
 
@@ -17,20 +17,26 @@ class NacosHttpClient:
         self.ak = os.getenv("DIFY_ENV_NACOS_ACCESS_KEY")
         self.sk = os.getenv("DIFY_ENV_NACOS_SECRET_KEY")
         self.server = os.getenv("DIFY_ENV_NACOS_SERVER_ADDR", "localhost:8848")
-        self.token = None
+        self.token: str | None = None
         self.token_ttl = 18000
         self.token_expire_time: float = 0
 
-    def http_request(self, url, method="GET", headers=None, params=None):
+    def http_request(
+        self, url: str, method: str = "GET", headers: dict[str, str] | None = None, params: dict[str, str] | None = None
+    ) -> str:
+        if headers is None:
+            headers = {}
+        if params is None:
+            params = {}
         try:
             self._inject_auth_info(headers, params)
-            response = requests.request(method, url="http://" + self.server + url, headers=headers, params=params)
+            response = httpx.request(method, url="http://" + self.server + url, headers=headers, params=params)
             response.raise_for_status()
             return response.text
-        except requests.exceptions.RequestException as e:
+        except httpx.RequestError as e:
             return f"Request to Nacos failed: {e}"
 
-    def _inject_auth_info(self, headers, params, module="config"):
+    def _inject_auth_info(self, headers: dict[str, str], params: dict[str, str], module: str = "config") -> None:
         headers.update({"User-Agent": "Nacos-Http-Client-In-Dify:v0.0.1"})
 
         if module == "login":
@@ -45,16 +51,17 @@ class NacosHttpClient:
             headers["timeStamp"] = ts
         if self.username and self.password:
             self.get_access_token(force_refresh=False)
-            params["accessToken"] = self.token
+            if self.token is not None:
+                params["accessToken"] = self.token
 
-    def __do_sign(self, sign_str, sk):
+    def __do_sign(self, sign_str: str, sk: str) -> str:
         return (
             base64.encodebytes(hmac.new(sk.encode(), sign_str.encode(), digestmod=hashlib.sha1).digest())
             .decode()
             .strip()
         )
 
-    def get_sign_str(self, group, tenant, ts):
+    def get_sign_str(self, group: str, tenant: str, ts: str) -> str:
         sign_str = ""
         if tenant:
             sign_str = tenant + "+"
@@ -63,7 +70,7 @@ class NacosHttpClient:
         sign_str += ts  # Directly concatenate ts without conditional checks, because the nacos auth header forced it.
         return sign_str
 
-    def get_access_token(self, force_refresh=False):
+    def get_access_token(self, force_refresh: bool = False) -> str | None:
         current_time = time.time()
         if self.token and not force_refresh and self.token_expire_time > current_time:
             return self.token
@@ -71,12 +78,13 @@ class NacosHttpClient:
         params = {"username": self.username, "password": self.password}
         url = "http://" + self.server + "/nacos/v1/auth/login"
         try:
-            resp = requests.request("POST", url, headers=None, params=params)
+            resp = httpx.request("POST", url, headers=None, params=params)
             resp.raise_for_status()
             response_data = resp.json()
             self.token = response_data.get("accessToken")
             self.token_ttl = response_data.get("tokenTtl", 18000)
             self.token_expire_time = current_time + self.token_ttl - 10
-        except Exception as e:
+            return self.token
+        except Exception:
             logger.exception("[get-access-token] exception occur")
             raise

api/configs/remote_settings_sources/nacos/utils.py

@@ -1,4 +1,4 @@
-def _parse_config(self, content: str) -> dict[str, str]:
+def parse_config(content: str) -> dict[str, str]:
     config: dict[str, str] = {}
     if not content:
         return config

api/constants/__init__.py

@@ -1,4 +1,5 @@
 from configs import dify_config
+from libs.collection_utils import convert_to_lower_and_upper_set
 
 HIDDEN_VALUE = "[__HIDDEN__]"
 UNKNOWN_VALUE = "[__UNKNOWN__]"
@@ -6,24 +7,39 @@ UUID_NIL = "00000000-0000-0000-0000-000000000000"
 
 DEFAULT_FILE_NUMBER_LIMITS = 3
 
-IMAGE_EXTENSIONS = ["jpg", "jpeg", "png", "webp", "gif", "svg"]
-IMAGE_EXTENSIONS.extend([ext.upper() for ext in IMAGE_EXTENSIONS])
+IMAGE_EXTENSIONS = convert_to_lower_and_upper_set({"jpg", "jpeg", "png", "webp", "gif", "svg"})
 
-VIDEO_EXTENSIONS = ["mp4", "mov", "mpeg", "webm"]
-VIDEO_EXTENSIONS.extend([ext.upper() for ext in VIDEO_EXTENSIONS])
-
-AUDIO_EXTENSIONS = ["mp3", "m4a", "wav", "amr", "mpga"]
-AUDIO_EXTENSIONS.extend([ext.upper() for ext in AUDIO_EXTENSIONS])
+VIDEO_EXTENSIONS = convert_to_lower_and_upper_set({"mp4", "mov", "mpeg", "webm"})
 
+AUDIO_EXTENSIONS = convert_to_lower_and_upper_set({"mp3", "m4a", "wav", "amr", "mpga"})
 
+_doc_extensions: set[str]
 if dify_config.ETL_TYPE == "Unstructured":
-    DOCUMENT_EXTENSIONS = ["txt", "markdown", "md", "mdx", "pdf", "html", "htm", "xlsx", "xls", "vtt", "properties"]
-    DOCUMENT_EXTENSIONS.extend(("doc", "docx", "csv", "eml", "msg", "pptx", "xml", "epub"))
+    _doc_extensions = {
+        "txt",
+        "markdown",
+        "md",
+        "mdx",
+        "pdf",
+        "html",
+        "htm",
+        "xlsx",
+        "xls",
+        "vtt",
+        "properties",
+        "doc",
+        "docx",
+        "csv",
+        "eml",
+        "msg",
+        "pptx",
+        "xml",
+        "epub",
+    }
     if dify_config.UNSTRUCTURED_API_URL:
-        DOCUMENT_EXTENSIONS.append("ppt")
-    DOCUMENT_EXTENSIONS.extend([ext.upper() for ext in DOCUMENT_EXTENSIONS])
+        _doc_extensions.add("ppt")
 else:
-    DOCUMENT_EXTENSIONS = [
+    _doc_extensions = {
         "txt",
         "markdown",
         "md",
@@ -37,5 +53,14 @@ else:
         "csv",
         "vtt",
         "properties",
-    ]
-    DOCUMENT_EXTENSIONS.extend([ext.upper() for ext in DOCUMENT_EXTENSIONS])
+    }
+DOCUMENT_EXTENSIONS: set[str] = convert_to_lower_and_upper_set(_doc_extensions)
+
+COOKIE_NAME_ACCESS_TOKEN = "access_token"
+COOKIE_NAME_REFRESH_TOKEN = "refresh_token"
+COOKIE_NAME_PASSPORT = "passport"
+COOKIE_NAME_CSRF_TOKEN = "csrf_token"
+
+HEADER_NAME_CSRF_TOKEN = "X-CSRF-Token"
+HEADER_NAME_APP_CODE = "X-App-Code"
+HEADER_NAME_PASSPORT = "X-App-Passport"

api/constants/languages.py

@@ -19,6 +19,7 @@ language_timezone_mapping = {
     "fa-IR": "Asia/Tehran",
     "sl-SI": "Europe/Ljubljana",
     "th-TH": "Asia/Bangkok",
+    "id-ID": "Asia/Jakarta",
 }
 
 languages = list(language_timezone_mapping.keys())

api/constants/model_template.py

@@ -7,7 +7,7 @@ default_app_templates: Mapping[AppMode, Mapping] = {
     # workflow default mode
     AppMode.WORKFLOW: {
         "app": {
-            "mode": AppMode.WORKFLOW.value,
+            "mode": AppMode.WORKFLOW,
             "enable_site": True,
             "enable_api": True,
         }
@@ -15,7 +15,7 @@ default_app_templates: Mapping[AppMode, Mapping] = {
     # completion default mode
     AppMode.COMPLETION: {
         "app": {
-            "mode": AppMode.COMPLETION.value,
+            "mode": AppMode.COMPLETION,
             "enable_site": True,
             "enable_api": True,
         },
@@ -44,7 +44,7 @@ default_app_templates: Mapping[AppMode, Mapping] = {
     # chat default mode
     AppMode.CHAT: {
         "app": {
-            "mode": AppMode.CHAT.value,
+            "mode": AppMode.CHAT,
             "enable_site": True,
             "enable_api": True,
         },
@@ -60,7 +60,7 @@ default_app_templates: Mapping[AppMode, Mapping] = {
     # advanced-chat default mode
     AppMode.ADVANCED_CHAT: {
         "app": {
-            "mode": AppMode.ADVANCED_CHAT.value,
+            "mode": AppMode.ADVANCED_CHAT,
             "enable_site": True,
             "enable_api": True,
         },
@@ -68,7 +68,7 @@ default_app_templates: Mapping[AppMode, Mapping] = {
     # agent-chat default mode
     AppMode.AGENT_CHAT: {
         "app": {
-            "mode": AppMode.AGENT_CHAT.value,
+            "mode": AppMode.AGENT_CHAT,
             "enable_site": True,
             "enable_api": True,
         },

api/contexts/__init__.py

@@ -5,10 +5,10 @@ from typing import TYPE_CHECKING
 from contexts.wrapper import RecyclableContextVar
 
 if TYPE_CHECKING:
+    from core.datasource.__base.datasource_provider import DatasourcePluginProviderController
     from core.model_runtime.entities.model_entities import AIModelEntity
     from core.plugin.entities.plugin_daemon import PluginModelProviderEntity
     from core.tools.plugin_tool.provider import PluginToolProviderController
-    from core.workflow.entities.variable_pool import VariablePool
 
 
 """
@@ -33,3 +33,11 @@ plugin_model_schema_lock: RecyclableContextVar[Lock] = RecyclableContextVar(Cont
 plugin_model_schemas: RecyclableContextVar[dict[str, "AIModelEntity"]] = RecyclableContextVar(
     ContextVar("plugin_model_schemas")
 )
+
+datasource_plugin_providers: RecyclableContextVar[dict[str, "DatasourcePluginProviderController"]] = (
+    RecyclableContextVar(ContextVar("datasource_plugin_providers"))
+)
+
+datasource_plugin_providers_lock: RecyclableContextVar[Lock] = RecyclableContextVar(
+    ContextVar("datasource_plugin_providers_lock")
+)

api/controllers/common/errors.py

@@ -1,5 +1,7 @@
 from werkzeug.exceptions import HTTPException
 
+from libs.exception import BaseHTTPException
+
 
 class FilenameNotExistsError(HTTPException):
     code = 400
@@ -9,3 +11,27 @@ class FilenameNotExistsError(HTTPException):
 class RemoteFileUploadError(HTTPException):
     code = 400
     description = "Error uploading remote file."
+
+
+class FileTooLargeError(BaseHTTPException):
+    error_code = "file_too_large"
+    description = "File size exceeded. {message}"
+    code = 413
+
+
+class UnsupportedFileTypeError(BaseHTTPException):
+    error_code = "unsupported_file_type"
+    description = "File type not allowed."
+    code = 415
+
+
+class TooManyFilesError(BaseHTTPException):
+    error_code = "too_many_files"
+    description = "Only one file is allowed."
+    code = 400
+
+
+class NoFileUploadedError(BaseHTTPException):
+    error_code = "no_file_uploaded"
+    description = "Please upload your file."
+    code = 400