feat: add node id in agent log

Co-authored-by: laipz8200 <16485841+laipz8200@users.noreply.github.com>
Co-authored-by: crazywoola <100913391+crazywoola@users.noreply.github.com>

.devcontainer/post_create_command.sh

@@ -1,12 +1,11 @@
 #!/bin/bash
 
-npm add -g pnpm@9.12.2
-cd web && pnpm install
+cd web && npm install
 pipx install poetry
 
 echo 'alias start-api="cd /workspaces/dify/api && poetry run python -m flask run --host 0.0.0.0 --port=5001 --debug"' >> ~/.bashrc
 echo 'alias start-worker="cd /workspaces/dify/api && poetry run python -m celery -A app.celery worker -P gevent -c 1 --loglevel INFO -Q dataset,generation,mail,ops_trace,app_deletion"' >> ~/.bashrc
-echo 'alias start-web="cd /workspaces/dify/web && pnpm dev"' >> ~/.bashrc
+echo 'alias start-web="cd /workspaces/dify/web && npm run dev"' >> ~/.bashrc
 echo 'alias start-containers="cd /workspaces/dify/docker && docker-compose -f docker-compose.middleware.yaml -p dify up -d"' >> ~/.bashrc
 echo 'alias stop-containers="cd /workspaces/dify/docker && docker-compose -f docker-compose.middleware.yaml -p dify down"' >> ~/.bashrc
 

.github/actions/setup-poetry/action.yml

@@ -8,7 +8,7 @@ inputs:
   poetry-version:
     description: Poetry version to set up
     required: true
-    default: '2.0.1'
+    default: '1.8.4'
   poetry-lockfile:
     description: Path to the Poetry lockfile to restore cache from
     required: true

.github/workflows/api-tests.yml

@@ -26,9 +26,6 @@ jobs:
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-          persist-credentials: false
 
       - name: Setup Poetry and Python ${{ matrix.python-version }}
         uses: ./.github/actions/setup-poetry
@@ -45,17 +42,25 @@ jobs:
         run: poetry install -C api --with dev
 
       - name: Check dependencies in pyproject.toml
-        run: poetry run -P api bash dev/pytest/pytest_artifacts.sh
+        run: poetry run -C api bash dev/pytest/pytest_artifacts.sh
 
       - name: Run Unit tests
-        run: poetry run -P api bash dev/pytest/pytest_unit_tests.sh
+        run: poetry run -C api bash dev/pytest/pytest_unit_tests.sh
+
+      - name: Run ModelRuntime
+        run: poetry run -C api bash dev/pytest/pytest_model_runtime.sh
 
       - name: Run dify config tests
-        run: poetry run -P api python dev/pytest/pytest_config_tests.py
+        run: poetry run -C api python dev/pytest/pytest_config_tests.py
+
+      - name: Run Tool
+        run: poetry run -C api bash dev/pytest/pytest_tools.sh
 
       - name: Run mypy
         run: |
-          poetry run -C api python -m mypy --install-types --non-interactive .
+          pushd api
+          poetry run python -m mypy --install-types --non-interactive .
+          popd
 
       - name: Set up dotenvs
         run: |
@@ -75,4 +80,4 @@ jobs:
             ssrf_proxy
 
       - name: Run Workflow
-        run: poetry run -P api bash dev/pytest/pytest_workflow.sh
+        run: poetry run -C api bash dev/pytest/pytest_workflow.sh

.github/workflows/build-push.yml

@@ -5,7 +5,6 @@ on:
     branches:
       - "main"
       - "deploy/dev"
-      - "dev/plugin-deploy"
   release:
     types: [published]
 
@@ -80,12 +79,10 @@ jobs:
           cache-to: type=gha,mode=max,scope=${{ matrix.service_name }}
 
       - name: Export digest
-        env:
-          DIGEST: ${{ steps.build.outputs.digest }}
         run: |
           mkdir -p /tmp/digests
-          sanitized_digest=${DIGEST#sha256:}
-          touch "/tmp/digests/${sanitized_digest}"
+          digest="${{ steps.build.outputs.digest }}"
+          touch "/tmp/digests/${digest#sha256:}"
 
       - name: Upload digest
         uses: actions/upload-artifact@v4
@@ -135,15 +132,10 @@ jobs:
 
       - name: Create manifest list and push
         working-directory: /tmp/digests
-        env:
-          IMAGE_NAME: ${{ env[matrix.image_name_env] }}
         run: |
           docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
-            $(printf "$IMAGE_NAME@sha256:%s " *)
+            $(printf '${{ env[matrix.image_name_env] }}@sha256:%s ' *)
 
       - name: Inspect image
-        env:
-          IMAGE_NAME: ${{ env[matrix.image_name_env] }}
-          IMAGE_VERSION: ${{ steps.meta.outputs.version }}
         run: |
-          docker buildx imagetools inspect "$IMAGE_NAME:$IMAGE_VERSION"
+          docker buildx imagetools inspect ${{ env[matrix.image_name_env] }}:${{ steps.meta.outputs.version }}

.github/workflows/db-migration-test.yml

@@ -4,7 +4,6 @@ on:
   pull_request:
     branches:
       - main
-      - plugins/beta
     paths:
       - api/migrations/**
       - .github/workflows/db-migration-test.yml
@@ -20,9 +19,6 @@ jobs:
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-          persist-credentials: false
 
       - name: Setup Poetry and Python
         uses: ./.github/actions/setup-poetry

.github/workflows/docker-build.yml

@@ -1,47 +0,0 @@
-name: Build docker image
-
-on:
-  pull_request:
-    branches:
-      - "main"
-    paths:
-      - api/Dockerfile
-      - web/Dockerfile
-
-concurrency:
-  group: docker-build-${{ github.head_ref || github.run_id }}
-  cancel-in-progress: true
-
-jobs:
-  build-docker:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        include:
-          - service_name: "api-amd64"
-            platform: linux/amd64
-            context: "api"
-          - service_name: "api-arm64"
-            platform: linux/arm64
-            context: "api"
-          - service_name: "web-amd64"
-            platform: linux/amd64
-            context: "web"
-          - service_name: "web-arm64"
-            platform: linux/arm64
-            context: "web"
-    steps:
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-
-      - name: Build Docker Image
-        uses: docker/build-push-action@v6
-        with:
-          push: false
-          context: "{{defaultContext}}:${{ matrix.context }}"
-          platforms: ${{ matrix.platform }}
-          cache-from: type=gha
-          cache-to: type=gha,mode=max

.github/workflows/expose_service_ports.sh

@@ -9,6 +9,6 @@ yq eval '.services["pgvecto-rs"].ports += ["5431:5432"]' -i docker/docker-compos
 yq eval '.services["elasticsearch"].ports += ["9200:9200"]' -i docker/docker-compose.yaml
 yq eval '.services.couchbase-server.ports += ["8091-8096:8091-8096"]' -i docker/docker-compose.yaml
 yq eval '.services.couchbase-server.ports += ["11210:11210"]' -i docker/docker-compose.yaml
-yq eval '.services.tidb.ports += ["4000:4000"]' -i docker/tidb/docker-compose.yaml
+yq eval '.services.tidb.ports += ["4000:4000"]' -i docker/docker-compose.yaml
 
 echo "Ports exposed for sandbox, weaviate, tidb, qdrant, chroma, milvus, pgvector, pgvecto-rs, elasticsearch, couchbase"

.github/workflows/style.yml

@@ -17,9 +17,6 @@ jobs:
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-          persist-credentials: false
 
       - name: Check changed files
         id: changed-files
@@ -41,12 +38,12 @@ jobs:
         if: steps.changed-files.outputs.any_changed == 'true'
         run: |
           poetry run -C api ruff --version
-          poetry run -C api ruff check ./
-          poetry run -C api ruff format --check ./
+          poetry run -C api ruff check ./api
+          poetry run -C api ruff format --check ./api
 
       - name: Dotenv check
         if: steps.changed-files.outputs.any_changed == 'true'
-        run: poetry run -P api dotenv-linter ./api/.env.example ./web/.env.example
+        run: poetry run -C api dotenv-linter ./api/.env.example ./web/.env.example
 
       - name: Lint hints
         if: failure()
@@ -62,9 +59,6 @@ jobs:
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-          persist-credentials: false
 
       - name: Check changed files
         id: changed-files
@@ -72,58 +66,22 @@ jobs:
         with:
           files: web/**
 
-      - name: Install pnpm
-        uses: pnpm/action-setup@v4
-        with:
-          version: 10
-          run_install: false
-
       - name: Setup NodeJS
         uses: actions/setup-node@v4
         if: steps.changed-files.outputs.any_changed == 'true'
         with:
           node-version: 20
-          cache: pnpm
+          cache: yarn
           cache-dependency-path: ./web/package.json
 
       - name: Web dependencies
         if: steps.changed-files.outputs.any_changed == 'true'
-        run: pnpm install --frozen-lockfile
+        run: yarn install --frozen-lockfile
 
       - name: Web style check
         if: steps.changed-files.outputs.any_changed == 'true'
-        run: pnpm run lint
+        run: yarn run lint
 
-  docker-compose-template:
-    name: Docker Compose Template
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-          persist-credentials: false
-
-      - name: Check changed files
-        id: changed-files
-        uses: tj-actions/changed-files@v45
-        with:
-          files: |
-            docker/generate_docker_compose
-            docker/.env.example
-            docker/docker-compose-template.yaml
-            docker/docker-compose.yaml
-
-      - name: Generate Docker Compose
-        if: steps.changed-files.outputs.any_changed == 'true'
-        run: |
-          cd docker
-          ./generate_docker_compose
-
-      - name: Check for changes
-        if: steps.changed-files.outputs.any_changed == 'true'
-        run: git diff --exit-code
 
   superlinter:
     name: SuperLinter
@@ -132,9 +90,6 @@ jobs:
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-          persist-credentials: false
 
       - name: Check changed files
         id: changed-files

.github/workflows/tool-test-sdks.yaml

@@ -26,19 +26,16 @@ jobs:
 
     steps:
       - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-          persist-credentials: false
 
       - name: Use Node.js ${{ matrix.node-version }}
         uses: actions/setup-node@v4
         with:
           node-version: ${{ matrix.node-version }}
           cache: ''
-          cache-dependency-path: 'pnpm-lock.yaml'
+          cache-dependency-path: 'yarn.lock'
 
       - name: Install Dependencies
-        run: pnpm install --frozen-lockfile
+        run: yarn install
 
       - name: Test
-        run: pnpm test
+        run: yarn test

.github/workflows/translate-i18n-base-on-english.yml

@@ -16,7 +16,6 @@ jobs:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 2 # last 2 commits
-          persist-credentials: false
 
       - name: Check for file changes in i18n/en-US
         id: check_files
@@ -39,11 +38,11 @@ jobs:
 
       - name: Install dependencies
         if: env.FILES_CHANGED == 'true'
-        run: pnpm install --frozen-lockfile
+        run: yarn install --frozen-lockfile
 
       - name: Run npm script
         if: env.FILES_CHANGED == 'true'
-        run: pnpm run auto-gen-i18n
+        run: npm run auto-gen-i18n
 
       - name: Create Pull Request
         if: env.FILES_CHANGED == 'true'

.github/workflows/vdb-tests.yml

@@ -28,9 +28,6 @@ jobs:
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-          persist-credentials: false
 
       - name: Setup Poetry and Python ${{ matrix.python-version }}
         uses: ./.github/actions/setup-poetry
@@ -54,15 +51,7 @@ jobs:
       - name: Expose Service Ports
         run: sh .github/workflows/expose_service_ports.sh
 
-      - name: Set up Vector Store (TiDB)
-        uses: hoverkraft-tech/compose-action@v2.0.2
-        with:
-          compose-file: docker/tidb/docker-compose.yaml
-          services: |
-            tidb
-            tiflash
-
-      - name: Set up Vector Stores (Weaviate, Qdrant, PGVector, Milvus, PgVecto-RS, Chroma, MyScale, ElasticSearch, Couchbase)
+      - name: Set up Vector Stores (TiDB, Weaviate, Qdrant, PGVector, Milvus, PgVecto-RS, Chroma, MyScale, ElasticSearch, Couchbase)
         uses: hoverkraft-tech/compose-action@v2.0.2
         with:
           compose-file: |
@@ -78,9 +67,7 @@ jobs:
             pgvector
             chroma
             elasticsearch
-
-      - name: Check TiDB Ready
-        run: poetry run -P api python api/tests/integration_tests/vdb/tidb_vector/check_tiflash_ready.py
+            tidb
 
       - name: Test Vector Stores
-        run: poetry run -P api bash dev/pytest/pytest_vdb.sh
+        run: poetry run -C api bash dev/pytest/pytest_vdb.sh

.github/workflows/web-tests.yml

@@ -22,34 +22,25 @@ jobs:
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-          persist-credentials: false
 
       - name: Check changed files
         id: changed-files
         uses: tj-actions/changed-files@v45
         with:
           files: web/**
-      # to run pnpm, should install package canvas, but it always install failed on amd64 under ubuntu-latest
-      # - name: Install pnpm
-      #   uses: pnpm/action-setup@v4
-      #   with:
-      #     version: 10
-      #     run_install: false
 
-      # - name: Setup Node.js
-      #   uses: actions/setup-node@v4
-      #   if: steps.changed-files.outputs.any_changed == 'true'
-      #   with:
-      #     node-version: 20
-      #     cache: pnpm
-      #     cache-dependency-path: ./web/package.json
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        if: steps.changed-files.outputs.any_changed == 'true'
+        with:
+          node-version: 20
+          cache: yarn
+          cache-dependency-path: ./web/package.json
 
-      # - name: Install dependencies
-      #   if: steps.changed-files.outputs.any_changed == 'true'
-      #   run: pnpm install --frozen-lockfile
+      - name: Install dependencies
+        if: steps.changed-files.outputs.any_changed == 'true'
+        run: yarn install --frozen-lockfile
 
-      # - name: Run tests
-      #   if: steps.changed-files.outputs.any_changed == 'true'
-      #   run: pnpm test
+      - name: Run tests
+        if: steps.changed-files.outputs.any_changed == 'true'
+        run: yarn test

.gitignore

@@ -163,7 +163,6 @@ docker/volumes/db/data/*
 docker/volumes/redis/data/*
 docker/volumes/weaviate/*
 docker/volumes/qdrant/*
-docker/tidb/volumes/*
 docker/volumes/etcd/*
 docker/volumes/minio/*
 docker/volumes/milvus/*
@@ -195,9 +194,3 @@ api/.vscode
 
 .idea/
 .vscode
-
-# pnpm
-/.pnpm-store
-
-# plugin migrate
-plugins.jsonl

CONTRIBUTING.md

@@ -73,7 +73,7 @@ Dify requires the following dependencies to build, make sure they're installed o
 * [Docker](https://www.docker.com/)
 * [Docker Compose](https://docs.docker.com/compose/install/)
 * [Node.js v18.x (LTS)](http://nodejs.org)
-* [pnpm](https://pnpm.io/)
+* [npm](https://www.npmjs.com/) version 8.x.x or [Yarn](https://yarnpkg.com/)
 * [Python](https://www.python.org/) version 3.11.x or 3.12.x
 
 ### 4. Installations

CONTRIBUTING_CN.md

@@ -70,7 +70,7 @@ Dify 依赖以下工具和库：
 - [Docker](https://www.docker.com/)
 - [Docker Compose](https://docs.docker.com/compose/install/)
 - [Node.js v18.x (LTS)](http://nodejs.org)
-- [pnpm](https://pnpm.io/)
+- [npm](https://www.npmjs.com/) version 8.x.x or [Yarn](https://yarnpkg.com/)
 - [Python](https://www.python.org/) version 3.11.x or 3.12.x
 
 ### 4. 安装

CONTRIBUTING_JA.md

@@ -73,7 +73,7 @@ Dify を構築するには次の依存関係が必要です。それらがシス
 - [Docker](https://www.docker.com/)
 - [Docker Compose](https://docs.docker.com/compose/install/)
 - [Node.js v18.x (LTS)](http://nodejs.org)
-- [pnpm](https://pnpm.io/)
+- [npm](https://www.npmjs.com/) version 8.x.x or [Yarn](https://yarnpkg.com/)
 - [Python](https://www.python.org/) version 3.11.x or 3.12.x
 
 ### 4. インストール

CONTRIBUTING_VI.md

@@ -72,7 +72,7 @@ Dify yêu cầu các phụ thuộc sau để build, hãy đảm bảo chúng đ
 - [Docker](https://www.docker.com/)
 - [Docker Compose](https://docs.docker.com/compose/install/)
 - [Node.js v18.x (LTS)](http://nodejs.org)
-- [pnpm](https://pnpm.io/)
+- [npm](https://www.npmjs.com/) phiên bản 8.x.x hoặc [Yarn](https://yarnpkg.com/)
 - [Python](https://www.python.org/) phiên bản 3.11.x hoặc 3.12.x
 
 ### 4. Cài đặt

LICENSE

@@ -1,12 +1,12 @@
 # Open Source License
 
-Dify is licensed under a modified version of the Apache License 2.0, with the following additional conditions:
+Dify is licensed under the Apache License 2.0, with the following additional conditions:
 
 1. Dify may be utilized commercially, including as a backend service for other applications or as an application development platform for enterprises. Should the conditions below be met, a commercial license must be obtained from the producer:
 
-a. Multi-tenant service: Unless explicitly authorized by Dify in writing, you may not use the Dify source code to operate a multi-tenant environment.
+a. Multi-tenant service: Unless explicitly authorized by Dify in writing, you may not use the Dify source code to operate a multi-tenant environment. 
     - Tenant Definition: Within the context of Dify, one tenant corresponds to one workspace. The workspace provides a separated area for each tenant's data and configurations.
-
+    
 b. LOGO and copyright information: In the process of using Dify's frontend, you may not remove or modify the LOGO or copyright information in the Dify console or applications. This restriction is inapplicable to uses of Dify that do not involve its frontend.
     - Frontend Definition: For the purposes of this license, the "frontend" of Dify includes all components located in the `web/` directory when running Dify from the raw source code, or the "web" image when running Dify with Docker.
 
@@ -21,4 +21,19 @@ Apart from the specific conditions mentioned above, all other rights and restric
 
 The interactive design of this product is protected by appearance patent.
 
-© 2025 LangGenius, Inc.
+© 2024 LangGenius, Inc.
+
+
+----------
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

README.md

@@ -25,9 +25,6 @@
     <a href="https://twitter.com/intent/follow?screen_name=dify_ai" target="_blank">
         <img src="https://img.shields.io/twitter/follow/dify_ai?logo=X&color=%20%23f5f5f5"
             alt="follow on X(Twitter)"></a>
-    <a href="https://www.linkedin.com/company/langgenius/" target="_blank">
-        <img src="https://custom-icon-badges.demolab.com/badge/LinkedIn-0A66C2?logo=linkedin-white&logoColor=fff"
-            alt="follow on LinkedIn"></a>
     <a href="https://hub.docker.com/u/langgenius" target="_blank">
         <img alt="Docker Pulls" src="https://img.shields.io/docker/pulls/langgenius/dify-web?labelColor=%20%23FDB062&color=%20%23f79009"></a>
     <a href="https://github.com/langgenius/dify/graphs/commit-activity" target="_blank">
@@ -108,72 +105,6 @@ Please refer to our [FAQ](https://docs.dify.ai/getting-started/install-self-host
 **7. Backend-as-a-Service**: 
   All of Dify's offerings come with corresponding APIs, so you could effortlessly integrate Dify into your own business logic.
 
-## Feature Comparison
-<table style="width: 100%;">
-  <tr>
-    <th align="center">Feature</th>
-    <th align="center">Dify.AI</th>
-    <th align="center">LangChain</th>
-    <th align="center">Flowise</th>
-    <th align="center">OpenAI Assistants API</th>
-  </tr>
-  <tr>
-    <td align="center">Programming Approach</td>
-    <td align="center">API + App-oriented</td>
-    <td align="center">Python Code</td>
-    <td align="center">App-oriented</td>
-    <td align="center">API-oriented</td>
-  </tr>
-  <tr>
-    <td align="center">Supported LLMs</td>
-    <td align="center">Rich Variety</td>
-    <td align="center">Rich Variety</td>
-    <td align="center">Rich Variety</td>
-    <td align="center">OpenAI-only</td>
-  </tr>
-  <tr>
-    <td align="center">RAG Engine</td>
-    <td align="center">✅</td>
-    <td align="center">✅</td>
-    <td align="center">✅</td>
-    <td align="center">✅</td>
-  </tr>
-  <tr>
-    <td align="center">Agent</td>
-    <td align="center">✅</td>
-    <td align="center">✅</td>
-    <td align="center">❌</td>
-    <td align="center">✅</td>
-  </tr>
-  <tr>
-    <td align="center">Workflow</td>
-    <td align="center">✅</td>
-    <td align="center">❌</td>
-    <td align="center">✅</td>
-    <td align="center">❌</td>
-  </tr>
-  <tr>
-    <td align="center">Observability</td>
-    <td align="center">✅</td>
-    <td align="center">✅</td>
-    <td align="center">❌</td>
-    <td align="center">❌</td>
-  </tr>
-  <tr>
-    <td align="center">Enterprise Feature (SSO/Access control)</td>
-    <td align="center">✅</td>
-    <td align="center">❌</td>
-    <td align="center">❌</td>
-    <td align="center">❌</td>
-  </tr>
-  <tr>
-    <td align="center">Local Deployment</td>
-    <td align="center">✅</td>
-    <td align="center">✅</td>
-    <td align="center">✅</td>
-    <td align="center">❌</td>
-  </tr>
-</table>
 
 ## Using Dify
 

README_AR.md

@@ -21,9 +21,6 @@
     <a href="https://twitter.com/intent/follow?screen_name=dify_ai" target="_blank">
         <img src="https://img.shields.io/twitter/follow/dify_ai?logo=X&color=%20%23f5f5f5"
             alt="follow on X(Twitter)"></a>
-    <a href="https://www.linkedin.com/company/langgenius/" target="_blank">
-        <img src="https://custom-icon-badges.demolab.com/badge/LinkedIn-0A66C2?logo=linkedin-white&logoColor=fff"
-            alt="follow on LinkedIn"></a>
     <a href="https://hub.docker.com/u/langgenius" target="_blank">
         <img alt="Docker Pulls" src="https://img.shields.io/docker/pulls/langgenius/dify-web?labelColor=%20%23FDB062&color=%20%23f79009"></a>
     <a href="https://github.com/langgenius/dify/graphs/commit-activity" target="_blank">

README_CN.md

@@ -21,9 +21,6 @@
     <a href="https://twitter.com/intent/follow?screen_name=dify_ai" target="_blank">
         <img src="https://img.shields.io/twitter/follow/dify_ai?logo=X&color=%20%23f5f5f5"
             alt="follow on X(Twitter)"></a>
-    <a href="https://www.linkedin.com/company/langgenius/" target="_blank">
-        <img src="https://custom-icon-badges.demolab.com/badge/LinkedIn-0A66C2?logo=linkedin-white&logoColor=fff"
-            alt="follow on LinkedIn"></a>
     <a href="https://hub.docker.com/u/langgenius" target="_blank">
         <img alt="Docker Pulls" src="https://img.shields.io/docker/pulls/langgenius/dify-web?labelColor=%20%23FDB062&color=%20%23f79009"></a>
     <a href="https://github.com/langgenius/dify/graphs/commit-activity" target="_blank">

README_ES.md

@@ -21,9 +21,6 @@
     <a href="https://twitter.com/intent/follow?screen_name=dify_ai" target="_blank">
         <img src="https://img.shields.io/twitter/follow/dify_ai?logo=X&color=%20%23f5f5f5"
             alt="seguir en X(Twitter)"></a>
-    <a href="https://www.linkedin.com/company/langgenius/" target="_blank">
-        <img src="https://custom-icon-badges.demolab.com/badge/LinkedIn-0A66C2?logo=linkedin-white&logoColor=fff"
-            alt="seguir en LinkedIn"></a>
     <a href="https://hub.docker.com/u/langgenius" target="_blank">
         <img alt="Descargas de Docker" src="https://img.shields.io/docker/pulls/langgenius/dify-web?labelColor=%20%23FDB062&color=%20%23f79009"></a>
     <a href="https://github.com/langgenius/dify/graphs/commit-activity" target="_blank">

README_FR.md

@@ -21,9 +21,6 @@
     <a href="https://twitter.com/intent/follow?screen_name=dify_ai" target="_blank">
         <img src="https://img.shields.io/twitter/follow/dify_ai?logo=X&color=%20%23f5f5f5"
             alt="suivre sur X(Twitter)"></a>
-    <a href="https://www.linkedin.com/company/langgenius/" target="_blank">
-        <img src="https://custom-icon-badges.demolab.com/badge/LinkedIn-0A66C2?logo=linkedin-white&logoColor=fff"
-            alt="suivre sur LinkedIn"></a>
     <a href="https://hub.docker.com/u/langgenius" target="_blank">
         <img alt="Tirages Docker" src="https://img.shields.io/docker/pulls/langgenius/dify-web?labelColor=%20%23FDB062&color=%20%23f79009"></a>
     <a href="https://github.com/langgenius/dify/graphs/commit-activity" target="_blank">
@@ -55,7 +52,7 @@
 Dify est une plateforme de développement d'applications LLM open source. Son interface intuitive combine un flux de travail d'IA, un pipeline RAG, des capacités d'agent, une gestion de modèles, des fonctionnalités d'observabilité, et plus encore, vous permettant de passer rapidement du prototype à la production. Voici une liste des fonctionnalités principales:
 </br> </br>
 
-**1. Flux de travail** : 
+**1. Flux de travail**: 
   Construisez et testez des flux de travail d'IA puissants sur un canevas visuel, en utilisant toutes les fonctionnalités suivantes et plus encore.
 
 
@@ -63,25 +60,27 @@ Dify est une plateforme de développement d'applications LLM open source. Son in
 
 
 
-**2. Prise en charge complète des modèles** : 
+**2. Prise en charge complète des modèles**: 
   Intégration transparente avec des centaines de LLM propriétaires / open source provenant de dizaines de fournisseurs d'inférence et de solutions auto-hébergées, couvrant GPT, Mistral, Llama3, et tous les modèles compatibles avec l'API OpenAI. Une liste complète des fournisseurs de modèles pris en charge se trouve [ici](https://docs.dify.ai/getting-started/readme/model-providers).
 
 ![providers-v5](https://github.com/langgenius/dify/assets/13230914/5a17bdbe-097a-4100-8363-40255b70f6e3)
 
 
-**3. IDE de prompt** : 
+**3. IDE de prompt**: 
   Interface intuitive pour créer des prompts, comparer les performances des modèles et ajouter des fonctionnalités supplémentaires telles que la synthèse vocale à une application basée sur des chats. 
 
-**4. Pipeline RAG** : 
+**4. Pipeline RAG**: 
   Des capacités RAG étendues qui couvrent tout, de l'ingestion de documents à la récupération, avec un support prêt à l'emploi pour l'extraction de texte à partir de PDF, PPT et autres formats de document courants.
 
-**5. Capacités d'agent** : 
+**5. Capac
+
+ités d'agent**: 
   Vous pouvez définir des agents basés sur l'appel de fonction LLM ou ReAct, et ajouter des outils pré-construits ou personnalisés pour l'agent. Dify fournit plus de 50 outils intégrés pour les agents d'IA, tels que la recherche Google, DALL·E, Stable Diffusion et WolframAlpha.
 
-**6. LLMOps** : 
+**6. LLMOps**: 
   Surveillez et analysez les journaux d'application et les performances au fil du temps. Vous pouvez continuellement améliorer les prompts, les ensembles de données et les modèles en fonction des données de production et des annotations.
 
-**7. Backend-as-a-Service** : 
+**7. Backend-as-a-Service**: 
   Toutes les offres de Dify sont accompagnées d'API correspondantes, vous permettant d'intégrer facilement Dify dans votre propre logique métier.
 
 

README_JA.md

@@ -21,9 +21,6 @@
     <a href="https://twitter.com/intent/follow?screen_name=dify_ai" target="_blank">
         <img src="https://img.shields.io/twitter/follow/dify_ai?logo=X&color=%20%23f5f5f5"
             alt="X(Twitter)でフォロー"></a>
-    <a href="https://www.linkedin.com/company/langgenius/" target="_blank">
-        <img src="https://custom-icon-badges.demolab.com/badge/LinkedIn-0A66C2?logo=linkedin-white&logoColor=fff"
-            alt="LinkedInでフォロー"></a>
     <a href="https://hub.docker.com/u/langgenius" target="_blank">
         <img alt="Docker Pulls" src="https://img.shields.io/docker/pulls/langgenius/dify-web?labelColor=%20%23FDB062&color=%20%23f79009"></a>
     <a href="https://github.com/langgenius/dify/graphs/commit-activity" target="_blank">
@@ -164,7 +161,7 @@ DifyはオープンソースのLLMアプリケーション開発プラットフ
 
 - **企業/組織向けのDify</br>**
 企業中心の機能を提供しています。[メールを送信](mailto:business@dify.ai?subject=[GitHub]Business%20License%20Inquiry)して企業のニーズについて相談してください。 </br>
-  > AWSを使用しているスタートアップ企業や中小企業の場合は、[AWS Marketplace](https://aws.amazon.com/marketplace/pp/prodview-t23mebxzwjhu6)のDify Premiumをチェックして、ワンクリックで自分のAWS VPCにデプロイできます。さらに、手頃な価格のAMIオファリングとして、ロゴやブランディングをカスタマイズしてアプリケーションを作成するオプションがあります。
+  > AWSを使用しているスタートアップ企業や中小企業の場合は、[AWS Marketplace](https://aws.amazon.com/marketplace/pp/prodview-t22mebxzwjhu6)のDify Premiumをチェックして、ワンクリックで自分のAWS VPCにデプロイできます。さらに、手頃な価格のAMIオファリングどして、ロゴやブランディングをカスタマイズしてアプリケーションを作成するオプションがあります。
 
 
 ## 最新の情報を入手

README_KL.md

@@ -21,9 +21,6 @@
     <a href="https://twitter.com/intent/follow?screen_name=dify_ai" target="_blank">
         <img src="https://img.shields.io/twitter/follow/dify_ai?logo=X&color=%20%23f5f5f5"
             alt="follow on X(Twitter)"></a>
-    <a href="https://www.linkedin.com/company/langgenius/" target="_blank">
-        <img src="https://custom-icon-badges.demolab.com/badge/LinkedIn-0A66C2?logo=linkedin-white&logoColor=fff"
-            alt="follow on LinkedIn"></a>
     <a href="https://hub.docker.com/u/langgenius" target="_blank">
         <img alt="Docker Pulls" src="https://img.shields.io/docker/pulls/langgenius/dify-web?labelColor=%20%23FDB062&color=%20%23f79009"></a>
     <a href="https://github.com/langgenius/dify/graphs/commit-activity" target="_blank">
@@ -87,7 +84,9 @@ Dify is an open-source LLM app development platform. Its intuitive interface com
 
 ## Feature Comparison
 <table style="width: 100%;">
-  <tr>
+  <tr
+
+>
     <th align="center">Feature</th>
     <th align="center">Dify.AI</th>
     <th align="center">LangChain</th>

README_KR.md

@@ -21,9 +21,6 @@
     <a href="https://twitter.com/intent/follow?screen_name=dify_ai" target="_blank">
         <img src="https://img.shields.io/twitter/follow/dify_ai?logo=X&color=%20%23f5f5f5"
             alt="follow on X(Twitter)"></a>
-    <a href="https://www.linkedin.com/company/langgenius/" target="_blank">
-        <img src="https://custom-icon-badges.demolab.com/badge/LinkedIn-0A66C2?logo=linkedin-white&logoColor=fff"
-            alt="follow on LinkedIn"></a>
     <a href="https://hub.docker.com/u/langgenius" target="_blank">
         <img alt="Docker Pulls" src="https://img.shields.io/docker/pulls/langgenius/dify-web?labelColor=%20%23FDB062&color=%20%23f79009"></a>
     <a href="https://github.com/langgenius/dify/graphs/commit-activity" target="_blank">

README_PT.md

@@ -25,9 +25,6 @@
     <a href="https://twitter.com/intent/follow?screen_name=dify_ai" target="_blank">
         <img src="https://img.shields.io/twitter/follow/dify_ai?logo=X&color=%20%23f5f5f5"
             alt="follow on X(Twitter)"></a>
-    <a href="https://www.linkedin.com/company/langgenius/" target="_blank">
-        <img src="https://custom-icon-badges.demolab.com/badge/LinkedIn-0A66C2?logo=linkedin-white&logoColor=fff"
-            alt="follow on LinkedIn"></a>
     <a href="https://hub.docker.com/u/langgenius" target="_blank">
         <img alt="Docker Pulls" src="https://img.shields.io/docker/pulls/langgenius/dify-web?labelColor=%20%23FDB062&color=%20%23f79009"></a>
     <a href="https://github.com/langgenius/dify/graphs/commit-activity" target="_blank">

README_SI.md

@@ -22,9 +22,6 @@
     <a href="https://twitter.com/intent/follow?screen_name=dify_ai" target="_blank">
         <img src="https://img.shields.io/twitter/follow/dify_ai?logo=X&color=%20%23f5f5f5"
             alt="follow on X(Twitter)"></a>
-    <a href="https://www.linkedin.com/company/langgenius/" target="_blank">
-        <img src="https://custom-icon-badges.demolab.com/badge/LinkedIn-0A66C2?logo=linkedin-white&logoColor=fff"
-            alt="follow on LinkedIn"></a>
     <a href="https://hub.docker.com/u/langgenius" target="_blank">
         <img alt="Docker Pulls" src="https://img.shields.io/docker/pulls/langgenius/dify-web?labelColor=%20%23FDB062&color=%20%23f79009"></a>
     <a href="https://github.com/langgenius/dify/graphs/commit-activity" target="_blank">
@@ -106,73 +103,6 @@ Prosimo, glejte naša pogosta vprašanja [FAQ](https://docs.dify.ai/getting-star
 **7. Backend-as-a-Service**: 
   AVse ponudbe Difyja so opremljene z ustreznimi API-ji, tako da lahko Dify brez težav integrirate v svojo poslovno logiko.
 
-## Primerjava Funkcij
-
-<table style="width: 100%;">
-  <tr>
-    <th align="center">Funkcija</th>
-    <th align="center">Dify.AI</th>
-    <th align="center">LangChain</th>
-    <th align="center">Flowise</th>
-    <th align="center">OpenAI Assistants API</th>
-  </tr>
-  <tr>
-    <td align="center">Programski pristop</td>
-    <td align="center">API + usmerjeno v aplikacije</td>
-    <td align="center">Python koda</td>
-    <td align="center">Usmerjeno v aplikacije</td>
-    <td align="center">Usmerjeno v API</td>
-  </tr>
-  <tr>
-    <td align="center">Podprti LLM-ji</td>
-    <td align="center">Bogata izbira</td>
-    <td align="center">Bogata izbira</td>
-    <td align="center">Bogata izbira</td>
-    <td align="center">Samo OpenAI</td>
-  </tr>
-  <tr>
-    <td align="center">RAG pogon</td>
-    <td align="center">✅</td>
-    <td align="center">✅</td>
-    <td align="center">✅</td>
-    <td align="center">✅</td>
-  </tr>
-  <tr>
-    <td align="center">Agent</td>
-    <td align="center">✅</td>
-    <td align="center">✅</td>
-    <td align="center">❌</td>
-    <td align="center">✅</td>
-  </tr>
-  <tr>
-    <td align="center">Potek dela</td>
-    <td align="center">✅</td>
-    <td align="center">❌</td>
-    <td align="center">✅</td>
-    <td align="center">❌</td>
-  </tr>
-  <tr>
-    <td align="center">Spremljanje</td>
-    <td align="center">✅</td>
-    <td align="center">✅</td>
-    <td align="center">❌</td>
-    <td align="center">❌</td>
-  </tr>
-  <tr>
-    <td align="center">Funkcija za podjetja (SSO/nadzor dostopa)</td>
-    <td align="center">✅</td>
-    <td align="center">❌</td>
-    <td align="center">❌</td>
-    <td align="center">❌</td>
-  </tr>
-  <tr>
-    <td align="center">Lokalna namestitev</td>
-    <td align="center">✅</td>
-    <td align="center">✅</td>
-    <td align="center">✅</td>
-    <td align="center">❌</td>
-  </tr>
-</table>
 
 ## Uporaba Dify
 
@@ -254,4 +184,4 @@ Zaradi zaščite vaše zasebnosti se izogibajte objavljanju varnostnih vprašanj
 
 ## Licenca
 
-To skladišče je na voljo pod [odprtokodno licenco Dify](LICENSE) , ki je v bistvu Apache 2.0 z nekaj dodatnimi omejitvami.
+To skladišče je na voljo pod [odprtokodno licenco Dify](LICENSE) , ki je v bistvu Apache 2.0 z nekaj dodatnimi omejitvami.

README_TR.md

@@ -21,9 +21,6 @@
     <a href="https://twitter.com/intent/follow?screen_name=dify_ai" target="_blank">
         <img src="https://img.shields.io/twitter/follow/dify_ai?logo=X&color=%20%23f5f5f5"
             alt="X(Twitter)'da takip et"></a>
-    <a href="https://www.linkedin.com/company/langgenius/" target="_blank">
-        <img src="https://custom-icon-badges.demolab.com/badge/LinkedIn-0A66C2?logo=linkedin-white&logoColor=fff"
-            alt="LinkedIn'da takip et"></a>
     <a href="https://hub.docker.com/u/langgenius" target="_blank">
         <img alt="Docker Çekmeleri" src="https://img.shields.io/docker/pulls/langgenius/dify-web?labelColor=%20%23FDB062&color=%20%23f79009"></a>
     <a href="https://github.com/langgenius/dify/graphs/commit-activity" target="_blank">
@@ -65,6 +62,8 @@ Görsel bir arayüz üzerinde güçlü AI iş akışları oluşturun ve test edi
 ![providers-v5](https://github.com/langgenius/dify/assets/13230914/5a17bdbe-097a-4100-8363-40255b70f6e3)
 
 
+Özür dilerim, haklısınız. Daha anlamlı ve akıcı bir çeviri yapmaya çalışayım. İşte güncellenmiş çeviri:
+
 **3. Prompt IDE**: 
   Komut istemlerini oluşturmak, model performansını karşılaştırmak ve sohbet tabanlı uygulamalara metin-konuşma gibi ek özellikler eklemek için kullanıcı dostu bir arayüz.
 
@@ -151,6 +150,8 @@ Görsel bir arayüz üzerinde güçlü AI iş akışları oluşturun ve test edi
 ## Dify'ı Kullanma
 
 - **Cloud </br>**
+İşte verdiğiniz metnin Türkçe çevirisi, kod bloğu içinde:
+- 
 Herkesin sıfır kurulumla denemesi için bir [Dify Cloud](https://dify.ai) hizmeti sunuyoruz. Bu hizmet, kendi kendine dağıtılan versiyonun tüm yeteneklerini sağlar ve sandbox planında 200 ücretsiz GPT-4 çağrısı içerir.
 
 - **Dify Topluluk Sürümünü Kendi Sunucunuzda Barındırma</br>**
@@ -176,6 +177,8 @@ GitHub'da Dify'a yıldız verin ve yeni sürümlerden anında haberdar olun.
 >- RAM >= 4GB
 
 </br>
+İşte verdiğiniz metnin Türkçe çevirisi, kod bloğu içinde:
+
 Dify sunucusunu başlatmanın en kolay yolu, [docker-compose.yml](docker/docker-compose.yaml) dosyamızı çalıştırmaktır. Kurulum komutunu çalıştırmadan önce, makinenizde [Docker](https://docs.docker.com/get-docker/) ve [Docker Compose](https://docs.docker.com/compose/install/)'un kurulu olduğundan emin olun:
 
 ```bash

README_VI.md

@@ -21,9 +21,6 @@
     <a href="https://twitter.com/intent/follow?screen_name=dify_ai" target="_blank">
         <img src="https://img.shields.io/twitter/follow/dify_ai?logo=X&color=%20%23f5f5f5"
             alt="theo dõi trên X(Twitter)"></a>
-    <a href="https://www.linkedin.com/company/langgenius/" target="_blank">
-        <img src="https://custom-icon-badges.demolab.com/badge/LinkedIn-0A66C2?logo=linkedin-white&logoColor=fff"
-            alt="theo dõi trên LinkedIn"></a>
     <a href="https://hub.docker.com/u/langgenius" target="_blank">
         <img alt="Docker Pulls" src="https://img.shields.io/docker/pulls/langgenius/dify-web?labelColor=%20%23FDB062&color=%20%23f79009"></a>
     <a href="https://github.com/langgenius/dify/graphs/commit-activity" target="_blank">

api/.env.example

@@ -422,8 +422,8 @@ POSITION_PROVIDER_INCLUDES=
 POSITION_PROVIDER_EXCLUDES=
 
 # Plugin configuration
-PLUGIN_DAEMON_KEY=lYkiYYT6owG+71oLerGzA7GXCgOT++6ovaezWAjpCjf+Sjc3ZtU+qUEi
-PLUGIN_DAEMON_URL=http://127.0.0.1:5002
+PLUGIN_API_KEY=lYkiYYT6owG+71oLerGzA7GXCgOT++6ovaezWAjpCjf+Sjc3ZtU+qUEi+vRjI/+XbV1AaFy691iy+kGDv2Jvy0/eAh8Y1
+PLUGIN_API_URL=http://127.0.0.1:5002
 PLUGIN_REMOTE_INSTALL_PORT=5003
 PLUGIN_REMOTE_INSTALL_HOST=localhost
 PLUGIN_MAX_PACKAGE_SIZE=15728640
@@ -435,7 +435,7 @@ MARKETPLACE_ENABLED=true
 MARKETPLACE_API_URL=https://marketplace.dify.ai
 
 # Endpoint configuration
-ENDPOINT_URL_TEMPLATE=http://localhost:5002/e/{hook_id}
+ENDPOINT_URL_TEMPLATE=http://localhost/e/{hook_id}
 
 # Reset password token expiry minutes
 RESET_PASSWORD_TOKEN_EXPIRY_MINUTES=5

api/.ruff.toml

@@ -53,12 +53,10 @@ ignore = [
     "FURB152", # math-constant
     "UP007", # non-pep604-annotation
     "UP032", # f-string
-    "UP045", # non-pep604-annotation-optional
     "B005", # strip-with-multi-characters
     "B006", # mutable-argument-default
     "B007", # unused-loop-control-variable
     "B026", # star-arg-unpacking-after-keyword-arg
-    "B903", # class-as-data-structure
     "B904", # raise-without-from-inside-except
     "B905", # zip-without-explicit-strict
     "N806", # non-lowercase-variable-in-function

api/Dockerfile

@@ -4,7 +4,7 @@ FROM python:3.12-slim-bookworm AS base
 WORKDIR /app/api
 
 # Install Poetry
-ENV POETRY_VERSION=2.0.1
+ENV POETRY_VERSION=1.8.4
 
 # if you located in China, you can use aliyun mirror to speed up
 # RUN pip install --no-cache-dir poetry==${POETRY_VERSION} -i https://mirrors.aliyun.com/pypi/simple/
@@ -48,20 +48,16 @@ ENV TZ=UTC
 
 WORKDIR /app/api
 
-RUN \
-    apt-get update \
-    # Install dependencies
-    && apt-get install -y --no-install-recommends \
-        # basic environment
-        curl nodejs libgmp-dev libmpfr-dev libmpc-dev \
-        # For Security
-        expat libldap-2.5-0 perl libsqlite3-0 zlib1g \
-        # install a chinese font to support the use of tools like matplotlib
-        fonts-noto-cjk \
-        # install a package to improve the accuracy of guessing mime type and file extension
-        media-types \
-        # install libmagic to support the use of python-magic guess MIMETYPE
-        libmagic1 \
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends curl nodejs libgmp-dev libmpfr-dev libmpc-dev \
+    # if you located in China, you can use aliyun mirror to speed up
+    # && echo "deb http://mirrors.aliyun.com/debian testing main" > /etc/apt/sources.list \
+    && echo "deb http://deb.debian.org/debian testing main" > /etc/apt/sources.list \
+    && apt-get update \
+    # For Security
+    && apt-get install -y --no-install-recommends expat=2.6.4-1 libldap-2.5-0=2.5.19+dfsg-1 perl=5.40.0-8 libsqlite3-0=3.46.1-1 zlib1g=1:1.3.dfsg+really1.3.1-1+b1 \
+    # install a chinese font to support the use of tools like matplotlib
+    && apt-get install -y fonts-noto-cjk \
     && apt-get autoremove -y \
     && rm -rf /var/lib/apt/lists/*
 
@@ -84,6 +80,7 @@ COPY . /app/api/
 COPY docker/entrypoint.sh /entrypoint.sh
 RUN chmod +x /entrypoint.sh
 
+
 ARG COMMIT_SHA
 ENV COMMIT_SHA=${COMMIT_SHA}
 

api/README.md

@@ -37,13 +37,7 @@
 
 4. Create environment.
 
-   Dify API service uses [Poetry](https://python-poetry.org/docs/) to manage dependencies. First, you need to add the poetry shell plugin, if you don't have it already, in order to run in a virtual environment. [Note: Poetry shell is no longer a native command so you need to install the poetry plugin beforehand]
-
-   ```bash
-   poetry self add poetry-plugin-shell
-   ```
-   
-   Then, You can execute `poetry shell` to activate the environment.
+   Dify API service uses [Poetry](https://python-poetry.org/docs/) to manage dependencies. You can execute `poetry shell` to activate the environment.
 
 5. Install dependencies
 
@@ -85,5 +79,5 @@
 2. Run the tests locally with mocked system environment variables in `tool.pytest_env` section in `pyproject.toml`
 
    ```bash
-   poetry run -P api bash dev/pytest/pytest_all_tests.sh
+   poetry run -C api bash dev/pytest/pytest_all_tests.sh
    ```

api/app_factory.py

@@ -2,7 +2,6 @@ import logging
 import time
 
 from configs import dify_config
-from contexts.wrapper import RecyclableContextVar
 from dify_app import DifyApp
 
 
@@ -17,12 +16,6 @@ def create_flask_app_with_configs() -> DifyApp:
     dify_app = DifyApp(__name__)
     dify_app.config.from_mapping(dify_config.model_dump())
 
-    # add before request hook
-    @dify_app.before_request
-    def before_request():
-        # add an unique identifier to each request
-        RecyclableContextVar.increment_thread_recycles()
-
     return dify_app
 
 

api/commands.py

@@ -707,13 +707,12 @@ def extract_unique_plugins(output_file: str, input_file: str):
 @click.option(
     "--output_file", prompt=True, help="The file to store the installed plugins.", default="installed_plugins.jsonl"
 )
-@click.option("--workers", prompt=True, help="The number of workers to install plugins.", default=100)
-def install_plugins(input_file: str, output_file: str, workers: int):
+def install_plugins(input_file: str, output_file: str):
     """
     Install plugins.
     """
     click.echo(click.style("Starting install plugins.", fg="white"))
 
-    PluginMigration.install_plugins(input_file, output_file, workers)
+    PluginMigration.install_plugins(input_file, output_file)
 
     click.echo(click.style("Install plugins completed.", fg="green"))

api/configs/feature/__init__.py

@@ -141,10 +141,10 @@ class PluginConfig(BaseSettings):
 
     PLUGIN_DAEMON_URL: HttpUrl = Field(
         description="Plugin API URL",
-        default="http://localhost:5002",
+        default="http://plugin:5002",
     )
 
-    PLUGIN_DAEMON_KEY: str = Field(
+    PLUGIN_API_KEY: str = Field(
         description="Plugin API key",
         default="plugin-api-key",
     )
@@ -200,7 +200,7 @@ class EndpointConfig(BaseSettings):
     )
 
     CONSOLE_WEB_URL: str = Field(
-        description="Base URL for the console web interface,used for frontend references and CORS configuration",
+        description="Base URL for the console web interface," "used for frontend references and CORS configuration",
         default="",
     )
 
@@ -373,8 +373,8 @@ class HttpConfig(BaseSettings):
     )
 
     RESPECT_XFORWARD_HEADERS_ENABLED: bool = Field(
-        description="Enable handling of X-Forwarded-For, X-Forwarded-Proto, and X-Forwarded-Port headers"
-        " when the app is behind a single trusted reverse proxy.",
+        description="Enable or disable the X-Forwarded-For Proxy Fix middleware from Werkzeug"
+        " to respect X-* headers to redirect clients",
         default=False,
     )
 
@@ -556,11 +556,6 @@ class AuthConfig(BaseSettings):
         default=86400,
     )
 
-    FORGOT_PASSWORD_LOCKOUT_DURATION: PositiveInt = Field(
-        description="Time (in seconds) a user must wait before retrying password reset after exceeding the rate limit.",
-        default=86400,
-    )
-
 
 class ModerationConfig(BaseSettings):
     """

api/configs/feature/hosted_service/__init__.py

@@ -1,40 +1,9 @@
 from typing import Optional
 
-from pydantic import Field, NonNegativeInt, computed_field
+from pydantic import Field, NonNegativeInt
 from pydantic_settings import BaseSettings
 
 
-class HostedCreditConfig(BaseSettings):
-    HOSTED_MODEL_CREDIT_CONFIG: str = Field(
-        description="Model credit configuration in format 'model:credits,model:credits', e.g., 'gpt-4:20,gpt-4o:10'",
-        default="",
-    )
-
-    def get_model_credits(self, model_name: str) -> int:
-        """
-        Get credit value for a specific model name.
-        Returns 1 if model is not found in configuration (default credit).
-
-        :param model_name: The name of the model to search for
-        :return: The credit value for the model
-        """
-        if not self.HOSTED_MODEL_CREDIT_CONFIG:
-            return 1
-
-        try:
-            credit_map = dict(
-                item.strip().split(":", 1) for item in self.HOSTED_MODEL_CREDIT_CONFIG.split(",") if ":" in item
-            )
-
-            # Search for matching model pattern
-            for pattern, credit in credit_map.items():
-                if pattern.strip() == model_name:
-                    return int(credit)
-            return 1  # Default quota if no match found
-        except (ValueError, AttributeError):
-            return 1  # Return default quota if parsing fails
-
-
 class HostedOpenAiConfig(BaseSettings):
     """
     Configuration for hosted OpenAI service
@@ -212,7 +181,7 @@ class HostedFetchAppTemplateConfig(BaseSettings):
     """
 
     HOSTED_FETCH_APP_TEMPLATES_MODE: str = Field(
-        description="Mode for fetching app templates: remote, db, or builtin default to remote,",
+        description="Mode for fetching app templates: remote, db, or builtin" " default to remote,",
         default="remote",
     )
 
@@ -233,7 +202,5 @@ class HostedServiceConfig(
     HostedZhipuAIConfig,
     # moderation
     HostedModerationConfig,
-    # credit config
-    HostedCreditConfig,
 ):
     pass

api/configs/middleware/__init__.py

@@ -1,4 +1,3 @@
-import os
 from typing import Any, Literal, Optional
 from urllib.parse import quote_plus
 
@@ -167,11 +166,6 @@ class DatabaseConfig(BaseSettings):
         default=False,
     )
 
-    RETRIEVAL_SERVICE_EXECUTORS: NonNegativeInt = Field(
-        description="Number of processes for the retrieval service, default to CPU cores.",
-        default=os.cpu_count(),
-    )
-
     @computed_field
     def SQLALCHEMY_ENGINE_OPTIONS(self) -> dict[str, Any]:
         return {

api/configs/packaging/__init__.py

@@ -9,7 +9,7 @@ class PackagingInfo(BaseSettings):
 
     CURRENT_VERSION: str = Field(
         description="Dify version",
-        default="1.0.0",
+        default="0.15.0",
     )
 
     COMMIT_SHA: str = Field(

api/constants/__init__.py

@@ -15,7 +15,7 @@ AUDIO_EXTENSIONS.extend([ext.upper() for ext in AUDIO_EXTENSIONS])
 
 if dify_config.ETL_TYPE == "Unstructured":
     DOCUMENT_EXTENSIONS = ["txt", "markdown", "md", "mdx", "pdf", "html", "htm", "xlsx", "xls"]
-    DOCUMENT_EXTENSIONS.extend(("doc", "docx", "csv", "eml", "msg", "pptx", "xml", "epub"))
+    DOCUMENT_EXTENSIONS.extend(("docx", "csv", "eml", "msg", "pptx", "xml", "epub"))
     if dify_config.UNSTRUCTURED_API_URL:
         DOCUMENT_EXTENSIONS.append("ppt")
     DOCUMENT_EXTENSIONS.extend([ext.upper() for ext in DOCUMENT_EXTENSIONS])

api/contexts/__init__.py

@@ -2,8 +2,6 @@ from contextvars import ContextVar
 from threading import Lock
 from typing import TYPE_CHECKING
 
-from contexts.wrapper import RecyclableContextVar
-
 if TYPE_CHECKING:
     from core.plugin.entities.plugin_daemon import PluginModelProviderEntity
     from core.tools.plugin_tool.provider import PluginToolProviderController
@@ -14,17 +12,8 @@ tenant_id: ContextVar[str] = ContextVar("tenant_id")
 
 workflow_variable_pool: ContextVar["VariablePool"] = ContextVar("workflow_variable_pool")
 
-"""
-To avoid race-conditions caused by gunicorn thread recycling, using RecyclableContextVar to replace with
-"""
-plugin_tool_providers: RecyclableContextVar[dict[str, "PluginToolProviderController"]] = RecyclableContextVar(
-    ContextVar("plugin_tool_providers")
-)
-plugin_tool_providers_lock: RecyclableContextVar[Lock] = RecyclableContextVar(ContextVar("plugin_tool_providers_lock"))
+plugin_tool_providers: ContextVar[dict[str, "PluginToolProviderController"]] = ContextVar("plugin_tool_providers")
+plugin_tool_providers_lock: ContextVar[Lock] = ContextVar("plugin_tool_providers_lock")
 
-plugin_model_providers: RecyclableContextVar[list["PluginModelProviderEntity"] | None] = RecyclableContextVar(
-    ContextVar("plugin_model_providers")
-)
-plugin_model_providers_lock: RecyclableContextVar[Lock] = RecyclableContextVar(
-    ContextVar("plugin_model_providers_lock")
-)
+plugin_model_providers: ContextVar[list["PluginModelProviderEntity"] | None] = ContextVar("plugin_model_providers")
+plugin_model_providers_lock: ContextVar[Lock] = ContextVar("plugin_model_providers_lock")

api/contexts/wrapper.py

@@ -1,65 +0,0 @@
-from contextvars import ContextVar
-from typing import Generic, TypeVar
-
-T = TypeVar("T")
-
-
-class HiddenValue:
-    pass
-
-
-_default = HiddenValue()
-
-
-class RecyclableContextVar(Generic[T]):
-    """
-    RecyclableContextVar is a wrapper around ContextVar
-    It's safe to use in gunicorn with thread recycling, but features like `reset` are not available for now
-
-    NOTE: you need to call `increment_thread_recycles` before requests
-    """
-
-    _thread_recycles: ContextVar[int] = ContextVar("thread_recycles")
-
-    @classmethod
-    def increment_thread_recycles(cls):
-        try:
-            recycles = cls._thread_recycles.get()
-            cls._thread_recycles.set(recycles + 1)
-        except LookupError:
-            cls._thread_recycles.set(0)
-
-    def __init__(self, context_var: ContextVar[T]):
-        self._context_var = context_var
-        self._updates = ContextVar[int](context_var.name + "_updates", default=0)
-
-    def get(self, default: T | HiddenValue = _default) -> T:
-        thread_recycles = self._thread_recycles.get(0)
-        self_updates = self._updates.get()
-        if thread_recycles > self_updates:
-            self._updates.set(thread_recycles)
-
-        # check if thread is recycled and should be updated
-        if thread_recycles < self_updates:
-            return self._context_var.get()
-        else:
-            # thread_recycles >= self_updates, means current context is invalid
-            if isinstance(default, HiddenValue) or default is _default:
-                raise LookupError
-            else:
-                return default
-
-    def set(self, value: T):
-        # it leads to a situation that self.updates is less than cls.thread_recycles if `set` was never called before
-        # increase it manually
-        thread_recycles = self._thread_recycles.get(0)
-        self_updates = self._updates.get()
-        if thread_recycles > self_updates:
-            self._updates.set(thread_recycles)
-
-        if self._updates.get() == self._thread_recycles.get(0):
-            # after increment,
-            self._updates.set(self._updates.get() + 1)
-
-        # set the context
-        self._context_var.set(value)

api/controllers/common/helpers.py

@@ -1,32 +1,12 @@
 import mimetypes
 import os
-import platform
 import re
 import urllib.parse
-import warnings
 from collections.abc import Mapping
 from typing import Any
 from uuid import uuid4
 
 import httpx
-
-try:
-    import magic
-except ImportError:
-    if platform.system() == "Windows":
-        warnings.warn(
-            "To use python-magic guess MIMETYPE, you need to run `pip install python-magic-bin`", stacklevel=2
-        )
-    elif platform.system() == "Darwin":
-        warnings.warn("To use python-magic guess MIMETYPE, you need to run `brew install libmagic`", stacklevel=2)
-    elif platform.system() == "Linux":
-        warnings.warn(
-            "To use python-magic guess MIMETYPE, you need to run `sudo apt-get install libmagic1`", stacklevel=2
-        )
-    else:
-        warnings.warn("To use python-magic guess MIMETYPE, you need to install `libmagic`", stacklevel=2)
-    magic = None  # type: ignore
-
 from pydantic import BaseModel
 
 from configs import dify_config
@@ -67,13 +47,6 @@ def guess_file_info_from_response(response: httpx.Response):
         # If guessing fails, use Content-Type from response headers
         mimetype = response.headers.get("Content-Type", "application/octet-stream")
 
-    # Use python-magic to guess MIME type if still unknown or generic
-    if mimetype == "application/octet-stream" and magic is not None:
-        try:
-            mimetype = magic.from_buffer(response.content[:1024], mime=True)
-        except magic.MagicException:
-            pass
-
     extension = os.path.splitext(filename)[1]
 
     # Ensure filename has an extension

api/controllers/console/admin.py

@@ -59,7 +59,7 @@ class InsertExploreAppListApi(Resource):
         with Session(db.engine) as session:
             app = session.execute(select(App).filter(App.id == args["app_id"])).scalar_one_or_none()
         if not app:
-            raise NotFound(f"App '{args['app_id']}' is not found")
+            raise NotFound(f'App \'{args["app_id"]}\' is not found')
 
         site = app.site
         if not site:

api/controllers/console/app/audio.py

@@ -22,7 +22,7 @@ from controllers.console.wraps import account_initialization_required, setup_req
 from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError
 from core.model_runtime.errors.invoke import InvokeError
 from libs.login import login_required
-from models import App, AppMode
+from models.model import AppMode
 from services.audio_service import AudioService
 from services.errors.audio import (
     AudioTooLargeServiceError,
@@ -79,7 +79,7 @@ class ChatMessageTextApi(Resource):
     @login_required
     @account_initialization_required
     @get_app_model
-    def post(self, app_model: App):
+    def post(self, app_model):
         from werkzeug.exceptions import InternalServerError
 
         try:
@@ -98,13 +98,9 @@ class ChatMessageTextApi(Resource):
                 and app_model.workflow.features_dict
             ):
                 text_to_speech = app_model.workflow.features_dict.get("text_to_speech")
-                if text_to_speech is None:
-                    raise ValueError("TTS is not enabled")
                 voice = args.get("voice") or text_to_speech.get("voice")
             else:
                 try:
-                    if app_model.app_model_config is None:
-                        raise ValueError("AppModelConfig not found")
                     voice = args.get("voice") or app_model.app_model_config.text_to_speech_dict.get("voice")
                 except Exception:
                     voice = None

api/controllers/console/auth/error.py

@@ -59,9 +59,3 @@ class EmailCodeAccountDeletionRateLimitExceededError(BaseHTTPException):
     error_code = "email_code_account_deletion_rate_limit_exceeded"
     description = "Too many account deletion emails have been sent. Please try again in 5 minutes."
     code = 429
-
-
-class EmailPasswordResetLimitError(BaseHTTPException):
-    error_code = "email_password_reset_limit"
-    description = "Too many failed password reset attempts. Please try again in 24 hours."
-    code = 429

api/controllers/console/auth/forgot_password.py

@@ -8,13 +8,7 @@ from sqlalchemy.orm import Session
 
 from constants.languages import languages
 from controllers.console import api
-from controllers.console.auth.error import (
-    EmailCodeError,
-    EmailPasswordResetLimitError,
-    InvalidEmailError,
-    InvalidTokenError,
-    PasswordMismatchError,
-)
+from controllers.console.auth.error import EmailCodeError, InvalidEmailError, InvalidTokenError, PasswordMismatchError
 from controllers.console.error import AccountInFreezeError, AccountNotFound, EmailSendIpLimitError
 from controllers.console.wraps import setup_required
 from events.tenant_event import tenant_was_created
@@ -71,10 +65,6 @@ class ForgotPasswordCheckApi(Resource):
 
         user_email = args["email"]
 
-        is_forgot_password_error_rate_limit = AccountService.is_forgot_password_error_rate_limit(args["email"])
-        if is_forgot_password_error_rate_limit:
-            raise EmailPasswordResetLimitError()
-
         token_data = AccountService.get_reset_password_data(args["token"])
         if token_data is None:
             raise InvalidTokenError()
@@ -83,10 +73,8 @@ class ForgotPasswordCheckApi(Resource):
             raise InvalidEmailError()
 
         if args["code"] != token_data.get("code"):
-            AccountService.add_forgot_password_error_rate_limit(args["email"])
             raise EmailCodeError()
 
-        AccountService.reset_forgot_password_error_rate_limit(args["email"])
         return {"is_valid": True, "email": token_data.get("email")}
 
 

api/controllers/console/datasets/data_source.py

@@ -135,7 +135,7 @@ class DataSourceNotionListApi(Resource):
                         data_source_info = json.loads(document.data_source_info)
                         exist_page_ids.append(data_source_info["notion_page_id"])
             # get all authorized pages
-            data_source_bindings = session.scalars(
+            data_source_bindings = session.execute(
                 select(DataSourceOauthBinding).filter_by(
                     tenant_id=current_user.current_tenant_id, provider="notion", disabled=False
                 )

api/controllers/console/datasets/datasets.py

@@ -14,7 +14,6 @@ from controllers.console.wraps import account_initialization_required, enterpris
 from core.errors.error import LLMBadRequestError, ProviderTokenNotInitError
 from core.indexing_runner import IndexingRunner
 from core.model_runtime.entities.model_entities import ModelType
-from core.plugin.entities.plugin import ModelProviderID
 from core.provider_manager import ProviderManager
 from core.rag.datasource.vdb.vector_type import VectorType
 from core.rag.extractor.entity.extract_setting import ExtractSetting
@@ -53,12 +52,12 @@ class DatasetListApi(Resource):
         # provider = request.args.get("provider", default="vendor")
         search = request.args.get("keyword", default=None, type=str)
         tag_ids = request.args.getlist("tag_ids")
-        include_all = request.args.get("include_all", default="false").lower() == "true"
+
         if ids:
             datasets, total = DatasetService.get_datasets_by_ids(ids, current_user.current_tenant_id)
         else:
             datasets, total = DatasetService.get_datasets(
-                page, limit, current_user.current_tenant_id, current_user, search, tag_ids, include_all
+                page, limit, current_user.current_tenant_id, current_user, search, tag_ids
             )
 
         # check embedding setting
@@ -73,9 +72,7 @@ class DatasetListApi(Resource):
 
         data = marshal(datasets, dataset_detail_fields)
         for item in data:
-            # convert embedding_model_provider to plugin standard format
             if item["indexing_technique"] == "high_quality":
-                item["embedding_model_provider"] = str(ModelProviderID(item["embedding_model_provider"]))
                 item_model = f"{item['embedding_model']}:{item['embedding_model_provider']}"
                 if item_model in model_names:
                     item["embedding_available"] = True
@@ -460,7 +457,7 @@ class DatasetIndexingEstimateApi(Resource):
             )
         except LLMBadRequestError:
             raise ProviderNotInitializeError(
-                "No Embedding Model available. Please configure a valid provider in the Settings -> Model Provider."
+                "No Embedding Model available. Please configure a valid provider " "in the Settings -> Model Provider."
             )
         except ProviderTokenNotInitError as ex:
             raise ProviderNotInitializeError(ex.description)
@@ -622,7 +619,9 @@ class DatasetRetrievalSettingApi(Resource):
         vector_type = dify_config.VECTOR_STORE
         match vector_type:
             case (
-                VectorType.RELYT
+                VectorType.MILVUS
+                | VectorType.RELYT
+                | VectorType.PGVECTOR
                 | VectorType.TIDB_VECTOR
                 | VectorType.CHROMA
                 | VectorType.TENCENT
@@ -646,7 +645,6 @@ class DatasetRetrievalSettingApi(Resource):
                 | VectorType.TIDB_ON_QDRANT
                 | VectorType.LINDORM
                 | VectorType.COUCHBASE
-                | VectorType.MILVUS
             ):
                 return {
                     "retrieval_method": [

api/controllers/console/datasets/datasets_document.py

@@ -362,7 +362,8 @@ class DatasetInitApi(Resource):
                 )
             except InvokeAuthorizationError:
                 raise ProviderNotInitializeError(
-                    "No Embedding Model available. Please configure a valid provider in the Settings -> Model Provider."
+                    "No Embedding Model available. Please configure a valid provider "
+                    "in the Settings -> Model Provider."
                 )
             except ProviderTokenNotInitError as ex:
                 raise ProviderNotInitializeError(ex.description)
@@ -539,7 +540,8 @@ class DocumentBatchIndexingEstimateApi(DocumentResource):
                 return response.model_dump(), 200
             except LLMBadRequestError:
                 raise ProviderNotInitializeError(
-                    "No Embedding Model available. Please configure a valid provider in the Settings -> Model Provider."
+                    "No Embedding Model available. Please configure a valid provider "
+                    "in the Settings -> Model Provider."
                 )
             except ProviderTokenNotInitError as ex:
                 raise ProviderNotInitializeError(ex.description)
@@ -617,7 +619,7 @@ class DocumentDetailApi(DocumentResource):
             raise InvalidMetadataError(f"Invalid metadata value: {metadata}")
 
         if metadata == "only":
-            response = {"id": document.id, "doc_type": document.doc_type, "doc_metadata": document.doc_metadata_details}
+            response = {"id": document.id, "doc_type": document.doc_type, "doc_metadata": document.doc_metadata}
         elif metadata == "without":
             dataset_process_rules = DatasetService.get_process_rules(dataset_id)
             document_process_rules = document.dataset_process_rule.to_dict()
@@ -678,7 +680,7 @@ class DocumentDetailApi(DocumentResource):
                 "disabled_by": document.disabled_by,
                 "archived": document.archived,
                 "doc_type": document.doc_type,
-                "doc_metadata": document.doc_metadata_details,
+                "doc_metadata": document.doc_metadata,
                 "segment_count": document.segment_count,
                 "average_segment_length": document.average_segment_length,
                 "hit_count": document.hit_count,

api/controllers/console/datasets/datasets_segments.py

@@ -168,7 +168,8 @@ class DatasetDocumentSegmentApi(Resource):
                 )
             except LLMBadRequestError:
                 raise ProviderNotInitializeError(
-                    "No Embedding Model available. Please configure a valid provider in the Settings -> Model Provider."
+                    "No Embedding Model available. Please configure a valid provider "
+                    "in the Settings -> Model Provider."
                 )
             except ProviderTokenNotInitError as ex:
                 raise ProviderNotInitializeError(ex.description)
@@ -216,7 +217,8 @@ class DatasetDocumentSegmentAddApi(Resource):
                 )
             except LLMBadRequestError:
                 raise ProviderNotInitializeError(
-                    "No Embedding Model available. Please configure a valid provider in the Settings -> Model Provider."
+                    "No Embedding Model available. Please configure a valid provider "
+                    "in the Settings -> Model Provider."
                 )
             except ProviderTokenNotInitError as ex:
                 raise ProviderNotInitializeError(ex.description)
@@ -265,7 +267,8 @@ class DatasetDocumentSegmentUpdateApi(Resource):
                 )
             except LLMBadRequestError:
                 raise ProviderNotInitializeError(
-                    "No Embedding Model available. Please configure a valid provider in the Settings -> Model Provider."
+                    "No Embedding Model available. Please configure a valid provider "
+                    "in the Settings -> Model Provider."
                 )
             except ProviderTokenNotInitError as ex:
                 raise ProviderNotInitializeError(ex.description)
@@ -365,9 +368,9 @@ class DatasetDocumentSegmentBatchImportApi(Resource):
             result = []
             for index, row in df.iterrows():
                 if document.doc_form == "qa_model":
-                    data = {"content": row.iloc[0], "answer": row.iloc[1]}
+                    data = {"content": row[0], "answer": row[1]}
                 else:
-                    data = {"content": row.iloc[0]}
+                    data = {"content": row[0]}
                 result.append(data)
             if len(result) == 0:
                 raise ValueError("The CSV file is empty.")
@@ -434,7 +437,8 @@ class ChildChunkAddApi(Resource):
                 )
             except LLMBadRequestError:
                 raise ProviderNotInitializeError(
-                    "No Embedding Model available. Please configure a valid provider in the Settings -> Model Provider."
+                    "No Embedding Model available. Please configure a valid provider "
+                    "in the Settings -> Model Provider."
                 )
             except ProviderTokenNotInitError as ex:
                 raise ProviderNotInitializeError(ex.description)

api/controllers/console/datasets/metadata.py

@@ -1,143 +0,0 @@
-from flask_login import current_user  # type: ignore  # type: ignore
-from flask_restful import Resource, marshal_with, reqparse  # type: ignore
-from werkzeug.exceptions import NotFound
-
-from controllers.console import api
-from controllers.console.wraps import account_initialization_required, enterprise_license_required, setup_required
-from fields.dataset_fields import dataset_metadata_fields
-from libs.login import login_required
-from services.dataset_service import DatasetService
-from services.entities.knowledge_entities.knowledge_entities import (
-    MetadataArgs,
-    MetadataOperationData,
-)
-from services.metadata_service import MetadataService
-
-
-def _validate_name(name):
-    if not name or len(name) < 1 or len(name) > 40:
-        raise ValueError("Name must be between 1 to 40 characters.")
-    return name
-
-
-def _validate_description_length(description):
-    if len(description) > 400:
-        raise ValueError("Description cannot exceed 400 characters.")
-    return description
-
-
-class DatasetListApi(Resource):
-    @setup_required
-    @login_required
-    @account_initialization_required
-    @enterprise_license_required
-    @marshal_with(dataset_metadata_fields)
-    def post(self, dataset_id):
-        parser = reqparse.RequestParser()
-        parser.add_argument("type", type=str, required=True, nullable=True, location="json")
-        parser.add_argument("name", type=str, required=True, nullable=True, location="json")
-        args = parser.parse_args()
-        metadata_args = MetadataArgs(**args)
-
-        dataset_id_str = str(dataset_id)
-        dataset = DatasetService.get_dataset(dataset_id_str)
-        if dataset is None:
-            raise NotFound("Dataset not found.")
-        DatasetService.check_dataset_permission(dataset, current_user)
-
-        metadata = MetadataService.create_metadata(dataset_id_str, metadata_args)
-        return metadata, 201
-
-
-class DatasetMetadataApi(Resource):
-    @setup_required
-    @login_required
-    @account_initialization_required
-    @enterprise_license_required
-    def patch(self, dataset_id, metadata_id):
-        parser = reqparse.RequestParser()
-        parser.add_argument("name", type=str, required=True, nullable=True, location="json")
-        args = parser.parse_args()
-
-        dataset_id_str = str(dataset_id)
-        metadata_id_str = str(metadata_id)
-        dataset = DatasetService.get_dataset(dataset_id_str)
-        if dataset is None:
-            raise NotFound("Dataset not found.")
-        DatasetService.check_dataset_permission(dataset, current_user)
-
-        metadata = MetadataService.update_metadata_name(dataset_id_str, metadata_id_str, args.get("name"))
-        return metadata, 200
-
-    @setup_required
-    @login_required
-    @account_initialization_required
-    @enterprise_license_required
-    def delete(self, dataset_id, metadata_id):
-        dataset_id_str = str(dataset_id)
-        metadata_id_str = str(metadata_id)
-        dataset = DatasetService.get_dataset(dataset_id_str)
-        if dataset is None:
-            raise NotFound("Dataset not found.")
-        DatasetService.check_dataset_permission(dataset, current_user)
-
-        MetadataService.delete_metadata(dataset_id_str, metadata_id_str)
-        return 200
-
-
-class DatasetMetadataBuiltInFieldApi(Resource):
-    @setup_required
-    @login_required
-    @account_initialization_required
-    @enterprise_license_required
-    def get(self):
-        built_in_fields = MetadataService.get_built_in_fields()
-        return built_in_fields, 200
-
-
-class DatasetMetadataBuiltInFieldActionApi(Resource):
-    @setup_required
-    @login_required
-    @account_initialization_required
-    @enterprise_license_required
-    def post(self, dataset_id, action):
-        dataset_id_str = str(dataset_id)
-        dataset = DatasetService.get_dataset(dataset_id_str)
-        if dataset is None:
-            raise NotFound("Dataset not found.")
-        DatasetService.check_dataset_permission(dataset, current_user)
-
-        if action == "enable":
-            MetadataService.enable_built_in_field(dataset)
-        elif action == "disable":
-            MetadataService.disable_built_in_field(dataset)
-        return 200
-
-
-class DocumentMetadataApi(Resource):
-    @setup_required
-    @login_required
-    @account_initialization_required
-    @enterprise_license_required
-    def post(self, dataset_id):
-        dataset_id_str = str(dataset_id)
-        dataset = DatasetService.get_dataset(dataset_id_str)
-        if dataset is None:
-            raise NotFound("Dataset not found.")
-        DatasetService.check_dataset_permission(dataset, current_user)
-
-        parser = reqparse.RequestParser()
-        parser.add_argument("operation_data", type=list, required=True, nullable=True, location="json")
-        args = parser.parse_args()
-        metadata_args = MetadataOperationData(**args)
-
-        MetadataService.update_documents_metadata(dataset, metadata_args)
-
-        return 200
-
-
-api.add_resource(DatasetListApi, "/datasets/<uuid:dataset_id>/metadata")
-api.add_resource(DatasetMetadataApi, "/datasets/<uuid:dataset_id>/metadata/<uuid:metadata_id>")
-api.add_resource(DatasetMetadataBuiltInFieldApi, "/datasets/metadata/built-in")
-api.add_resource(DatasetMetadataBuiltInFieldActionApi, "/datasets/metadata/built-in/<string:action>")
-api.add_resource(DocumentMetadataApi, "/datasets/<uuid:dataset_id>/documents/metadata")

api/controllers/console/explore/conversation.py

@@ -32,7 +32,7 @@ class ConversationListApi(InstalledAppResource):
 
         pinned = None
         if "pinned" in args and args["pinned"] is not None:
-            pinned = args["pinned"] == "true"
+            pinned = True if args["pinned"] == "true" else False
 
         try:
             with Session(db.engine) as session:

api/controllers/console/explore/message.py

@@ -50,7 +50,7 @@ class MessageListApi(InstalledAppResource):
 
         try:
             return MessageService.pagination_by_first_id(
-                app_model, current_user, args["conversation_id"], args["first_id"], args["limit"]
+                app_model, current_user, args["conversation_id"], args["first_id"], args["limit"], "desc"
             )
         except services.errors.conversation.ConversationNotExistsError:
             raise NotFound("Conversation Not Exists.")

api/controllers/files/image_preview.py

@@ -1,5 +1,3 @@
-from urllib.parse import quote
-
 from flask import Response, request
 from flask_restful import Resource, reqparse  # type: ignore
 from werkzeug.exceptions import NotFound
@@ -73,8 +71,7 @@ class FilePreviewApi(Resource):
         if upload_file.size > 0:
             response.headers["Content-Length"] = str(upload_file.size)
         if args["as_attachment"]:
-            encoded_filename = quote(upload_file.name)
-            response.headers["Content-Disposition"] = f"attachment; filename*=UTF-8''{encoded_filename}"
+            response.headers["Content-Disposition"] = f"attachment; filename={upload_file.name}"
 
         return response
 

api/controllers/inner_api/workspace/workspace.py

@@ -1,5 +1,3 @@
-import json
-
 from flask_restful import Resource, reqparse  # type: ignore
 
 from controllers.console.wraps import setup_required
@@ -31,34 +29,4 @@ class EnterpriseWorkspace(Resource):
         return {"message": "enterprise workspace created."}
 
 
-class EnterpriseWorkspaceNoOwnerEmail(Resource):
-    @setup_required
-    @enterprise_inner_api_only
-    def post(self):
-        parser = reqparse.RequestParser()
-        parser.add_argument("name", type=str, required=True, location="json")
-        args = parser.parse_args()
-
-        tenant = TenantService.create_tenant(args["name"], is_from_dashboard=True)
-
-        tenant_was_created.send(tenant)
-
-        resp = {
-            "id": tenant.id,
-            "name": tenant.name,
-            "encrypt_public_key": tenant.encrypt_public_key,
-            "plan": tenant.plan,
-            "status": tenant.status,
-            "custom_config": json.loads(tenant.custom_config) if tenant.custom_config else {},
-            "created_at": tenant.created_at.isoformat() + "Z" if tenant.created_at else None,
-            "updated_at": tenant.updated_at.isoformat() + "Z" if tenant.updated_at else None,
-        }
-
-        return {
-            "message": "enterprise workspace created.",
-            "tenant": resp,
-        }
-
-
 api.add_resource(EnterpriseWorkspace, "/enterprise/workspace")
-api.add_resource(EnterpriseWorkspaceNoOwnerEmail, "/enterprise/workspace/ownerless")

api/controllers/inner_api/wraps.py

@@ -65,7 +65,7 @@ def enterprise_inner_api_user_auth(view):
 def plugin_inner_api_only(view):
     @wraps(view)
     def decorated(*args, **kwargs):
-        if not dify_config.PLUGIN_DAEMON_KEY:
+        if not dify_config.PLUGIN_API_KEY:
             abort(404)
 
         # get header 'X-Inner-Api-Key'

api/controllers/service_api/__init__.py

@@ -7,4 +7,4 @@ api = ExternalApi(bp)
 
 from . import index
 from .app import app, audio, completion, conversation, file, message, workflow
-from .dataset import dataset, document, hit_testing, segment, upload_file
+from .dataset import dataset, document, hit_testing, segment

api/controllers/service_api/app/message.py

@@ -10,7 +10,6 @@ from controllers.service_api.app.error import NotChatAppError
 from controllers.service_api.wraps import FetchUserArg, WhereisUserArg, validate_app_token
 from core.app.entities.app_invoke_entities import InvokeFrom
 from fields.conversation_fields import message_file_fields
-from fields.message_fields import feedback_fields, retriever_resource_fields
 from fields.raws import FilesContainedField
 from libs.helper import TimestampField, uuid_value
 from models.model import App, AppMode, EndUser
@@ -19,6 +18,26 @@ from services.message_service import MessageService
 
 
 class MessageListApi(Resource):
+    feedback_fields = {"rating": fields.String}
+    retriever_resource_fields = {
+        "id": fields.String,
+        "message_id": fields.String,
+        "position": fields.Integer,
+        "dataset_id": fields.String,
+        "dataset_name": fields.String,
+        "document_id": fields.String,
+        "document_name": fields.String,
+        "data_source_type": fields.String,
+        "segment_id": fields.String,
+        "score": fields.Float,
+        "hit_count": fields.Integer,
+        "word_count": fields.Integer,
+        "segment_position": fields.Integer,
+        "index_node_hash": fields.String,
+        "content": fields.String,
+        "created_at": TimestampField,
+    }
+
     agent_thought_fields = {
         "id": fields.String,
         "chain_id": fields.String,
@@ -70,7 +89,7 @@ class MessageListApi(Resource):
 
         try:
             return MessageService.pagination_by_first_id(
-                app_model, end_user, args["conversation_id"], args["first_id"], args["limit"], "desc"
+                app_model, end_user, args["conversation_id"], args["first_id"], args["limit"]
             )
         except services.errors.conversation.ConversationNotExistsError:
             raise NotFound("Conversation Not Exists.")

api/controllers/service_api/dataset/dataset.py

@@ -31,11 +31,8 @@ class DatasetListApi(DatasetApiResource):
         # provider = request.args.get("provider", default="vendor")
         search = request.args.get("keyword", default=None, type=str)
         tag_ids = request.args.getlist("tag_ids")
-        include_all = request.args.get("include_all", default="false").lower() == "true"
 
-        datasets, total = DatasetService.get_datasets(
-            page, limit, tenant_id, current_user, search, tag_ids, include_all
-        )
+        datasets, total = DatasetService.get_datasets(page, limit, tenant_id, current_user, search, tag_ids)
         # check embedding setting
         provider_manager = ProviderManager()
         configurations = provider_manager.get_configurations(tenant_id=current_user.current_tenant_id)

api/controllers/service_api/dataset/document.py

@@ -18,7 +18,6 @@ from controllers.service_api.app.error import (
 from controllers.service_api.dataset.error import (
     ArchivedDocumentImmutableError,
     DocumentIndexingError,
-    InvalidMetadataError,
 )
 from controllers.service_api.wraps import DatasetApiResource, cloud_edition_billing_resource_check
 from core.errors.error import ProviderTokenNotInitError
@@ -51,9 +50,6 @@ class DocumentAddByTextApi(DatasetApiResource):
             "indexing_technique", type=str, choices=Dataset.INDEXING_TECHNIQUE_LIST, nullable=False, location="json"
         )
         parser.add_argument("retrieval_model", type=dict, required=False, nullable=False, location="json")
-        parser.add_argument("doc_type", type=str, required=False, nullable=True, location="json")
-        parser.add_argument("doc_metadata", type=dict, required=False, nullable=True, location="json")
-
         args = parser.parse_args()
         dataset_id = str(dataset_id)
         tenant_id = str(tenant_id)
@@ -65,28 +61,6 @@ class DocumentAddByTextApi(DatasetApiResource):
         if not dataset.indexing_technique and not args["indexing_technique"]:
             raise ValueError("indexing_technique is required.")
 
-        # Validate metadata if provided
-        if args.get("doc_type") or args.get("doc_metadata"):
-            if not args.get("doc_type") or not args.get("doc_metadata"):
-                raise InvalidMetadataError("Both doc_type and doc_metadata must be provided when adding metadata")
-
-            if args["doc_type"] not in DocumentService.DOCUMENT_METADATA_SCHEMA:
-                raise InvalidMetadataError(
-                    "Invalid doc_type. Must be one of: " + ", ".join(DocumentService.DOCUMENT_METADATA_SCHEMA.keys())
-                )
-
-            if not isinstance(args["doc_metadata"], dict):
-                raise InvalidMetadataError("doc_metadata must be a dictionary")
-
-            # Validate metadata schema based on doc_type
-            if args["doc_type"] != "others":
-                metadata_schema = DocumentService.DOCUMENT_METADATA_SCHEMA[args["doc_type"]]
-                for key, value in args["doc_metadata"].items():
-                    if key in metadata_schema and not isinstance(value, metadata_schema[key]):
-                        raise InvalidMetadataError(f"Invalid type for metadata field {key}")
-            # set to MetaDataConfig
-            args["metadata"] = {"doc_type": args["doc_type"], "doc_metadata": args["doc_metadata"]}
-
         text = args.get("text")
         name = args.get("name")
         if text is None or name is None:
@@ -133,8 +107,6 @@ class DocumentUpdateByTextApi(DatasetApiResource):
             "doc_language", type=str, default="English", required=False, nullable=False, location="json"
         )
         parser.add_argument("retrieval_model", type=dict, required=False, nullable=False, location="json")
-        parser.add_argument("doc_type", type=str, required=False, nullable=True, location="json")
-        parser.add_argument("doc_metadata", type=dict, required=False, nullable=True, location="json")
         args = parser.parse_args()
         dataset_id = str(dataset_id)
         tenant_id = str(tenant_id)
@@ -143,32 +115,6 @@ class DocumentUpdateByTextApi(DatasetApiResource):
         if not dataset:
             raise ValueError("Dataset is not exist.")
 
-        # indexing_technique is already set in dataset since this is an update
-        args["indexing_technique"] = dataset.indexing_technique
-
-        # Validate metadata if provided
-        if args.get("doc_type") or args.get("doc_metadata"):
-            if not args.get("doc_type") or not args.get("doc_metadata"):
-                raise InvalidMetadataError("Both doc_type and doc_metadata must be provided when adding metadata")
-
-            if args["doc_type"] not in DocumentService.DOCUMENT_METADATA_SCHEMA:
-                raise InvalidMetadataError(
-                    "Invalid doc_type. Must be one of: " + ", ".join(DocumentService.DOCUMENT_METADATA_SCHEMA.keys())
-                )
-
-            if not isinstance(args["doc_metadata"], dict):
-                raise InvalidMetadataError("doc_metadata must be a dictionary")
-
-            # Validate metadata schema based on doc_type
-            if args["doc_type"] != "others":
-                metadata_schema = DocumentService.DOCUMENT_METADATA_SCHEMA[args["doc_type"]]
-                for key, value in args["doc_metadata"].items():
-                    if key in metadata_schema and not isinstance(value, metadata_schema[key]):
-                        raise InvalidMetadataError(f"Invalid type for metadata field {key}")
-
-            # set to MetaDataConfig
-            args["metadata"] = {"doc_type": args["doc_type"], "doc_metadata": args["doc_metadata"]}
-
         if args["text"]:
             text = args.get("text")
             name = args.get("name")
@@ -215,30 +161,6 @@ class DocumentAddByFileApi(DatasetApiResource):
             args["doc_form"] = "text_model"
         if "doc_language" not in args:
             args["doc_language"] = "English"
-
-        # Validate metadata if provided
-        if args.get("doc_type") or args.get("doc_metadata"):
-            if not args.get("doc_type") or not args.get("doc_metadata"):
-                raise InvalidMetadataError("Both doc_type and doc_metadata must be provided when adding metadata")
-
-            if args["doc_type"] not in DocumentService.DOCUMENT_METADATA_SCHEMA:
-                raise InvalidMetadataError(
-                    "Invalid doc_type. Must be one of: " + ", ".join(DocumentService.DOCUMENT_METADATA_SCHEMA.keys())
-                )
-
-            if not isinstance(args["doc_metadata"], dict):
-                raise InvalidMetadataError("doc_metadata must be a dictionary")
-
-            # Validate metadata schema based on doc_type
-            if args["doc_type"] != "others":
-                metadata_schema = DocumentService.DOCUMENT_METADATA_SCHEMA[args["doc_type"]]
-                for key, value in args["doc_metadata"].items():
-                    if key in metadata_schema and not isinstance(value, metadata_schema[key]):
-                        raise InvalidMetadataError(f"Invalid type for metadata field {key}")
-
-            # set to MetaDataConfig
-            args["metadata"] = {"doc_type": args["doc_type"], "doc_metadata": args["doc_metadata"]}
-
         # get dataset info
         dataset_id = str(dataset_id)
         tenant_id = str(tenant_id)
@@ -306,29 +228,6 @@ class DocumentUpdateByFileApi(DatasetApiResource):
         if "doc_language" not in args:
             args["doc_language"] = "English"
 
-        # Validate metadata if provided
-        if args.get("doc_type") or args.get("doc_metadata"):
-            if not args.get("doc_type") or not args.get("doc_metadata"):
-                raise InvalidMetadataError("Both doc_type and doc_metadata must be provided when adding metadata")
-
-            if args["doc_type"] not in DocumentService.DOCUMENT_METADATA_SCHEMA:
-                raise InvalidMetadataError(
-                    "Invalid doc_type. Must be one of: " + ", ".join(DocumentService.DOCUMENT_METADATA_SCHEMA.keys())
-                )
-
-            if not isinstance(args["doc_metadata"], dict):
-                raise InvalidMetadataError("doc_metadata must be a dictionary")
-
-            # Validate metadata schema based on doc_type
-            if args["doc_type"] != "others":
-                metadata_schema = DocumentService.DOCUMENT_METADATA_SCHEMA[args["doc_type"]]
-                for key, value in args["doc_metadata"].items():
-                    if key in metadata_schema and not isinstance(value, metadata_schema[key]):
-                        raise InvalidMetadataError(f"Invalid type for metadata field {key}")
-
-            # set to MetaDataConfig
-            args["metadata"] = {"doc_type": args["doc_type"], "doc_metadata": args["doc_metadata"]}
-
         # get dataset info
         dataset_id = str(dataset_id)
         tenant_id = str(tenant_id)
@@ -336,10 +235,6 @@ class DocumentUpdateByFileApi(DatasetApiResource):
 
         if not dataset:
             raise ValueError("Dataset is not exist.")
-
-        # indexing_technique is already set in dataset since this is an update
-        args["indexing_technique"] = dataset.indexing_technique
-
         if "file" in request.files:
             # save file info
             file = request.files["file"]

api/controllers/service_api/dataset/segment.py

@@ -53,7 +53,8 @@ class SegmentApi(DatasetApiResource):
                 )
             except LLMBadRequestError:
                 raise ProviderNotInitializeError(
-                    "No Embedding Model available. Please configure a valid provider in the Settings -> Model Provider."
+                    "No Embedding Model available. Please configure a valid provider "
+                    "in the Settings -> Model Provider."
                 )
             except ProviderTokenNotInitError as ex:
                 raise ProviderNotInitializeError(ex.description)
@@ -94,7 +95,8 @@ class SegmentApi(DatasetApiResource):
                 )
             except LLMBadRequestError:
                 raise ProviderNotInitializeError(
-                    "No Embedding Model available. Please configure a valid provider in the Settings -> Model Provider."
+                    "No Embedding Model available. Please configure a valid provider "
+                    "in the Settings -> Model Provider."
                 )
             except ProviderTokenNotInitError as ex:
                 raise ProviderNotInitializeError(ex.description)
@@ -173,7 +175,8 @@ class DatasetSegmentApi(DatasetApiResource):
                 )
             except LLMBadRequestError:
                 raise ProviderNotInitializeError(
-                    "No Embedding Model available. Please configure a valid provider in the Settings -> Model Provider."
+                    "No Embedding Model available. Please configure a valid provider "
+                    "in the Settings -> Model Provider."
                 )
             except ProviderTokenNotInitError as ex:
                 raise ProviderNotInitializeError(ex.description)

api/controllers/service_api/dataset/upload_file.py

@@ -1,54 +0,0 @@
-from werkzeug.exceptions import NotFound
-
-from controllers.service_api import api
-from controllers.service_api.wraps import (
-    DatasetApiResource,
-)
-from core.file import helpers as file_helpers
-from extensions.ext_database import db
-from models.dataset import Dataset
-from models.model import UploadFile
-from services.dataset_service import DocumentService
-
-
-class UploadFileApi(DatasetApiResource):
-    def get(self, tenant_id, dataset_id, document_id):
-        """Get upload file."""
-        # check dataset
-        dataset_id = str(dataset_id)
-        tenant_id = str(tenant_id)
-        dataset = db.session.query(Dataset).filter(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).first()
-        if not dataset:
-            raise NotFound("Dataset not found.")
-        # check document
-        document_id = str(document_id)
-        document = DocumentService.get_document(dataset.id, document_id)
-        if not document:
-            raise NotFound("Document not found.")
-        # check upload file
-        if document.data_source_type != "upload_file":
-            raise ValueError(f"Document data source type ({document.data_source_type}) is not upload_file.")
-        data_source_info = document.data_source_info_dict
-        if data_source_info and "upload_file_id" in data_source_info:
-            file_id = data_source_info["upload_file_id"]
-            upload_file = db.session.query(UploadFile).filter(UploadFile.id == file_id).first()
-            if not upload_file:
-                raise NotFound("UploadFile not found.")
-        else:
-            raise ValueError("Upload file id not found in document data source info.")
-
-        url = file_helpers.get_signed_file_url(upload_file_id=upload_file.id)
-        return {
-            "id": upload_file.id,
-            "name": upload_file.name,
-            "size": upload_file.size,
-            "extension": upload_file.extension,
-            "url": url,
-            "download_url": f"{url}&as_attachment=true",
-            "mime_type": upload_file.mime_type,
-            "created_by": upload_file.created_by,
-            "created_at": upload_file.created_at.timestamp(),
-        }, 200
-
-
-api.add_resource(UploadFileApi, "/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/upload-file")

api/controllers/service_api/wraps.py

@@ -154,7 +154,7 @@ def validate_dataset_token(view=None):
             )  # TODO: only owner information is required, so only one is returned.
             if tenant_account_join:
                 tenant, ta = tenant_account_join
-                account = db.session.query(Account).filter(Account.id == ta.account_id).first()
+                account = Account.query.filter_by(id=ta.account_id).first()
                 # Login admin
                 if account:
                     account.current_tenant = tenant
@@ -195,11 +195,7 @@ def validate_and_get_api_token(scope: str | None = None):
     with Session(db.engine, expire_on_commit=False) as session:
         update_stmt = (
             update(ApiToken)
-            .where(
-                ApiToken.token == auth_token,
-                (ApiToken.last_used_at.is_(None) | (ApiToken.last_used_at < cutoff_time)),
-                ApiToken.type == scope,
-            )
+            .where(ApiToken.token == auth_token, ApiToken.last_used_at < cutoff_time, ApiToken.type == scope)
             .values(last_used_at=current_time)
             .returning(ApiToken)
         )
@@ -240,7 +236,7 @@ def create_or_update_end_user_for_user_id(app_model: App, user_id: Optional[str]
             tenant_id=app_model.tenant_id,
             app_id=app_model.id,
             type="service_api",
-            is_anonymous=user_id == "DEFAULT-USER",
+            is_anonymous=True if user_id == "DEFAULT-USER" else False,
             session_id=user_id,
         )
         db.session.add(end_user)

api/controllers/web/conversation.py

@@ -39,7 +39,7 @@ class ConversationListApi(WebApiResource):
 
         pinned = None
         if "pinned" in args and args["pinned"] is not None:
-            pinned = args["pinned"] == "true"
+            pinned = True if args["pinned"] == "true" else False
 
         try:
             with Session(db.engine) as session:

api/controllers/web/message.py

@@ -21,7 +21,7 @@ from core.app.entities.app_invoke_entities import InvokeFrom
 from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError
 from core.model_runtime.errors.invoke import InvokeError
 from fields.conversation_fields import message_file_fields
-from fields.message_fields import agent_thought_fields, feedback_fields, retriever_resource_fields
+from fields.message_fields import agent_thought_fields
 from fields.raws import FilesContainedField
 from libs import helper
 from libs.helper import TimestampField, uuid_value
@@ -34,6 +34,27 @@ from services.message_service import MessageService
 
 
 class MessageListApi(WebApiResource):
+    feedback_fields = {"rating": fields.String}
+
+    retriever_resource_fields = {
+        "id": fields.String,
+        "message_id": fields.String,
+        "position": fields.Integer,
+        "dataset_id": fields.String,
+        "dataset_name": fields.String,
+        "document_id": fields.String,
+        "document_name": fields.String,
+        "data_source_type": fields.String,
+        "segment_id": fields.String,
+        "score": fields.Float,
+        "hit_count": fields.Integer,
+        "word_count": fields.Integer,
+        "segment_position": fields.Integer,
+        "index_node_hash": fields.String,
+        "content": fields.String,
+        "created_at": TimestampField,
+    }
+
     message_fields = {
         "id": fields.String,
         "conversation_id": fields.String,
@@ -70,7 +91,7 @@ class MessageListApi(WebApiResource):
 
         try:
             return MessageService.pagination_by_first_id(
-                app_model, end_user, args["conversation_id"], args["first_id"], args["limit"]
+                app_model, end_user, args["conversation_id"], args["first_id"], args["limit"], "desc"
             )
         except services.errors.conversation.ConversationNotExistsError:
             raise NotFound("Conversation Not Exists.")

api/core/agent/base_agent_runner.py

@@ -329,7 +329,6 @@ class BaseAgentRunner(AppRunner):
         )
         if not updated_agent_thought:
             raise ValueError("agent thought not found")
-        agent_thought = updated_agent_thought
 
         if thought:
             agent_thought.thought = thought

api/core/agent/cot_agent_runner.py

@@ -168,7 +168,7 @@ class CotAgentRunner(BaseAgentRunner, ABC):
 
             self.save_agent_thought(
                 agent_thought=agent_thought,
-                tool_name=(scratchpad.action.action_name if scratchpad.action and not scratchpad.is_final() else ""),
+                tool_name=scratchpad.action.action_name if scratchpad.action else "",
                 tool_input={scratchpad.action.action_name: scratchpad.action.action_input} if scratchpad.action else {},
                 tool_invoke_meta={},
                 thought=scratchpad.thought or "",

api/core/agent/entities.py

@@ -1,7 +1,7 @@
 from enum import StrEnum
 from typing import Any, Optional, Union
 
-from pydantic import BaseModel, Field
+from pydantic import BaseModel
 
 from core.tools.entities.tool_entities import ToolInvokeMessage, ToolProviderType
 
@@ -14,7 +14,7 @@ class AgentToolEntity(BaseModel):
     provider_type: ToolProviderType
     provider_id: str
     tool_name: str
-    tool_parameters: dict[str, Any] = Field(default_factory=dict)
+    tool_parameters: dict[str, Any] = {}
     plugin_unique_identifier: str | None = None
 
 

api/core/app/app_config/easy_ui_based_app/model_config/manager.py

@@ -2,9 +2,9 @@ from collections.abc import Mapping
 from typing import Any
 
 from core.app.app_config.entities import ModelConfigEntity
+from core.entities import DEFAULT_PLUGIN_ID
 from core.model_runtime.entities.model_entities import ModelPropertyKey, ModelType
 from core.model_runtime.model_providers.model_provider_factory import ModelProviderFactory
-from core.plugin.entities.plugin import ModelProviderID
 from core.provider_manager import ProviderManager
 
 
@@ -61,7 +61,9 @@ class ModelConfigManager:
             raise ValueError(f"model.provider is required and must be in {str(model_provider_names)}")
 
         if "/" not in config["model"]["provider"]:
-            config["model"]["provider"] = str(ModelProviderID(config["model"]["provider"]))
+            config["model"]["provider"] = (
+                f"{DEFAULT_PLUGIN_ID}/{config['model']['provider']}/{config['model']['provider']}"
+            )
 
         if config["model"]["provider"] not in model_provider_names:
             raise ValueError(f"model.provider is required and must be in {str(model_provider_names)}")

api/core/app/app_config/entities.py

@@ -17,8 +17,8 @@ class ModelConfigEntity(BaseModel):
     provider: str
     model: str
     mode: Optional[str] = None
-    parameters: dict[str, Any] = Field(default_factory=dict)
-    stop: list[str] = Field(default_factory=list)
+    parameters: dict[str, Any] = {}
+    stop: list[str] = []
 
 
 class AdvancedChatMessageEntity(BaseModel):
@@ -132,7 +132,7 @@ class ExternalDataVariableEntity(BaseModel):
 
     variable: str
     type: str
-    config: dict[str, Any] = Field(default_factory=dict)
+    config: dict[str, Any] = {}
 
 
 class DatasetRetrieveConfigEntity(BaseModel):
@@ -188,7 +188,7 @@ class SensitiveWordAvoidanceEntity(BaseModel):
     """
 
     type: str
-    config: dict[str, Any] = Field(default_factory=dict)
+    config: dict[str, Any] = {}
 
 
 class TextToSpeechEntity(BaseModel):

api/core/app/apps/advanced_chat/app_generator.py

@@ -140,7 +140,9 @@ class AdvancedChatAppGenerator(MessageBasedAppGenerator):
             app_config=app_config,
             file_upload_config=file_extra_config,
             conversation_id=conversation.id if conversation else None,
-            inputs=self._prepare_user_inputs(
+            inputs=conversation.inputs
+            if conversation
+            else self._prepare_user_inputs(
                 user_inputs=inputs, variables=app_config.variables, tenant_id=app_model.tenant_id
             ),
             query=query,

api/core/app/apps/agent_chat/app_generator.py

@@ -149,7 +149,9 @@ class AgentChatAppGenerator(MessageBasedAppGenerator):
             model_conf=ModelConfigConverter.convert(app_config),
             file_upload_config=file_extra_config,
             conversation_id=conversation.id if conversation else None,
-            inputs=self._prepare_user_inputs(
+            inputs=conversation.inputs
+            if conversation
+            else self._prepare_user_inputs(
                 user_inputs=inputs, variables=app_config.variables, tenant_id=app_model.tenant_id
             ),
             query=query,

api/core/app/apps/agent_chat/app_runner.py

@@ -8,16 +8,16 @@ from core.agent.fc_agent_runner import FunctionCallAgentRunner
 from core.app.apps.agent_chat.app_config_manager import AgentChatAppConfig
 from core.app.apps.base_app_queue_manager import AppQueueManager, PublishFrom
 from core.app.apps.base_app_runner import AppRunner
-from core.app.entities.app_invoke_entities import AgentChatAppGenerateEntity
+from core.app.entities.app_invoke_entities import AgentChatAppGenerateEntity, ModelConfigWithCredentialsEntity
 from core.app.entities.queue_entities import QueueAnnotationReplyEvent
 from core.memory.token_buffer_memory import TokenBufferMemory
 from core.model_manager import ModelInstance
-from core.model_runtime.entities.llm_entities import LLMMode
+from core.model_runtime.entities.llm_entities import LLMMode, LLMUsage
 from core.model_runtime.entities.model_entities import ModelFeature, ModelPropertyKey
 from core.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel
 from core.moderation.base import ModerationError
 from extensions.ext_database import db
-from models.model import App, Conversation, Message
+from models.model import App, Conversation, Message, MessageAgentThought
 
 logger = logging.getLogger(__name__)
 
@@ -191,8 +191,7 @@ class AgentChatAppRunner(AppRunner):
         # change function call strategy based on LLM model
         llm_model = cast(LargeLanguageModel, model_instance.model_type_instance)
         model_schema = llm_model.get_model_schema(model_instance.model, model_instance.credentials)
-        if not model_schema:
-            raise ValueError("Model schema not found")
+        assert model_schema is not None
 
         if {ModelFeature.MULTI_TOOL_CALL, ModelFeature.TOOL_CALL}.intersection(model_schema.features or []):
             agent_entity.strategy = AgentEntity.Strategy.FUNCTION_CALLING
@@ -248,3 +247,29 @@ class AgentChatAppRunner(AppRunner):
             stream=application_generate_entity.stream,
             agent=True,
         )
+
+    def _get_usage_of_all_agent_thoughts(
+        self, model_config: ModelConfigWithCredentialsEntity, message: Message
+    ) -> LLMUsage:
+        """
+        Get usage of all agent thoughts
+        :param model_config: model config
+        :param message: message
+        :return:
+        """
+        agent_thoughts = (
+            db.session.query(MessageAgentThought).filter(MessageAgentThought.message_id == message.id).all()
+        )
+
+        all_message_tokens = 0
+        all_answer_tokens = 0
+        for agent_thought in agent_thoughts:
+            all_message_tokens += agent_thought.message_tokens
+            all_answer_tokens += agent_thought.answer_tokens
+
+        model_type_instance = model_config.provider_model_bundle.model_type_instance
+        model_type_instance = cast(LargeLanguageModel, model_type_instance)
+
+        return model_type_instance._calc_response_usage(
+            model_config.model, model_config.credentials, all_message_tokens, all_answer_tokens
+        )

api/core/app/apps/base_app_queue_manager.py

@@ -167,7 +167,8 @@ class AppQueueManager:
         else:
             if isinstance(data, DeclarativeMeta) or hasattr(data, "_sa_instance_state"):
                 raise TypeError(
-                    "Critical Error: Passing SQLAlchemy Model instances that cause thread safety issues is not allowed."
+                    "Critical Error: Passing SQLAlchemy Model instances "
+                    "that cause thread safety issues is not allowed."
                 )
 
 

api/core/app/apps/chat/app_generator.py

@@ -141,7 +141,9 @@ class ChatAppGenerator(MessageBasedAppGenerator):
             model_conf=ModelConfigConverter.convert(app_config),
             file_upload_config=file_extra_config,
             conversation_id=conversation.id if conversation else None,
-            inputs=self._prepare_user_inputs(
+            inputs=conversation.inputs
+            if conversation
+            else self._prepare_user_inputs(
                 user_inputs=inputs, variables=app_config.variables, tenant_id=app_model.tenant_id
             ),
             query=query,

api/core/app/apps/message_based_app_generator.py

@@ -42,6 +42,7 @@ class MessageBasedAppGenerator(BaseAppGenerator):
             ChatAppGenerateEntity,
             CompletionAppGenerateEntity,
             AgentChatAppGenerateEntity,
+            AgentChatAppGenerateEntity,
         ],
         queue_manager: AppQueueManager,
         conversation: Conversation,
@@ -88,7 +89,6 @@ class MessageBasedAppGenerator(BaseAppGenerator):
             Conversation.id == conversation_id,
             Conversation.app_id == app_model.id,
             Conversation.status == "normal",
-            Conversation.is_deleted.is_(False),
         ]
 
         if isinstance(user, Account):

api/core/app/apps/workflow_app_runner.py

@@ -241,7 +241,6 @@ class WorkflowBasedAppRunner(AppRunner):
                     predecessor_node_id=event.predecessor_node_id,
                     in_iteration_id=event.in_iteration_id,
                     parallel_mode_run_id=event.parallel_mode_run_id,
-                    agent_strategy=event.agent_strategy,
                 )
             )
         elif isinstance(event, NodeRunSucceededEvent):
@@ -387,6 +386,7 @@ class WorkflowBasedAppRunner(AppRunner):
                     status=event.status,
                     data=event.data,
                     metadata=event.metadata,
+                    node_id=event.node_id,
                 )
             )
         elif isinstance(event, ParallelBranchRunStartedEvent):

api/core/app/entities/app_invoke_entities.py

@@ -63,9 +63,9 @@ class ModelConfigWithCredentialsEntity(BaseModel):
     model_schema: AIModelEntity
     mode: str
     provider_model_bundle: ProviderModelBundle
-    credentials: dict[str, Any] = Field(default_factory=dict)
-    parameters: dict[str, Any] = Field(default_factory=dict)
-    stop: list[str] = Field(default_factory=list)
+    credentials: dict[str, Any] = {}
+    parameters: dict[str, Any] = {}
+    stop: list[str] = []
 
     # pydantic configs
     model_config = ConfigDict(protected_namespaces=())
@@ -94,7 +94,7 @@ class AppGenerateEntity(BaseModel):
     call_depth: int = 0
 
     # extra parameters, like: auto_generate_conversation_name
-    extras: dict[str, Any] = Field(default_factory=dict)
+    extras: dict[str, Any] = {}
 
     # tracing instance
     trace_manager: Optional[TraceQueueManager] = None

api/core/app/entities/queue_entities.py

@@ -6,7 +6,7 @@ from typing import Any, Optional
 from pydantic import BaseModel
 
 from core.model_runtime.entities.llm_entities import LLMResult, LLMResultChunk
-from core.workflow.entities.node_entities import AgentNodeStrategyInit, NodeRunMetadataKey
+from core.workflow.entities.node_entities import NodeRunMetadataKey
 from core.workflow.graph_engine.entities.graph_runtime_state import GraphRuntimeState
 from core.workflow.nodes import NodeType
 from core.workflow.nodes.base import BaseNodeData
@@ -281,7 +281,6 @@ class QueueNodeStartedEvent(AppQueueEvent):
     start_at: datetime
     parallel_mode_run_id: Optional[str] = None
     """iteratoin run in parallel mode run id"""
-    agent_strategy: Optional[AgentNodeStrategyInit] = None
 
 
 class QueueNodeSucceededEvent(AppQueueEvent):
@@ -331,6 +330,7 @@ class QueueAgentLogEvent(AppQueueEvent):
     status: str
     data: Mapping[str, Any]
     metadata: Optional[Mapping[str, Any]] = None
+    node_id: str
 
 
 class QueueNodeRetryEvent(QueueNodeStartedEvent):

api/core/app/entities/task_entities.py

@@ -6,7 +6,6 @@ from pydantic import BaseModel, ConfigDict
 
 from core.model_runtime.entities.llm_entities import LLMResult
 from core.model_runtime.utils.encoders import jsonable_encoder
-from core.workflow.entities.node_entities import AgentNodeStrategyInit
 from models.workflow import WorkflowNodeExecutionStatus
 
 
@@ -249,7 +248,6 @@ class NodeStartStreamResponse(StreamResponse):
         parent_parallel_start_node_id: Optional[str] = None
         iteration_id: Optional[str] = None
         parallel_run_id: Optional[str] = None
-        agent_strategy: Optional[AgentNodeStrategyInit] = None
 
     event: StreamEvent = StreamEvent.NODE_STARTED
     workflow_run_id: str
@@ -719,6 +717,7 @@ class AgentLogStreamResponse(StreamResponse):
         status: str
         data: Mapping[str, Any]
         metadata: Optional[Mapping[str, Any]] = None
+        node_id: str
 
     event: StreamEvent = StreamEvent.AGENT_LOG
     data: Data

api/core/app/task_pipeline/message_cycle_manage.py

@@ -145,7 +145,7 @@ class MessageCycleManage:
 
             # get extension
             if "." in message_file.url:
-                extension = f".{message_file.url.split('.')[-1]}"
+                extension = f'.{message_file.url.split(".")[-1]}'
                 if len(extension) > 10:
                     extension = ".bin"
             else:

api/core/app/task_pipeline/workflow_cycle_manage.py

@@ -541,7 +541,6 @@ class WorkflowCycleManage:
                 parent_parallel_start_node_id=event.parent_parallel_start_node_id,
                 iteration_id=event.in_iteration_id,
                 parallel_run_id=event.parallel_mode_run_id,
-                agent_strategy=event.agent_strategy,
             ),
         )
 
@@ -844,7 +843,7 @@ class WorkflowCycleManage:
         if node_execution_id not in self._workflow_node_executions:
             raise ValueError(f"Workflow node execution not found: {node_execution_id}")
         cached_workflow_node_execution = self._workflow_node_executions[node_execution_id]
-        return session.merge(cached_workflow_node_execution)
+        return cached_workflow_node_execution
 
     def _handle_agent_log(self, task_id: str, event: QueueAgentLogEvent) -> AgentLogStreamResponse:
         """
@@ -864,5 +863,6 @@ class WorkflowCycleManage:
                 status=event.status,
                 data=event.data,
                 metadata=event.metadata,
+                node_id=event.node_id,
             ),
         )

api/core/entities/provider_configuration.py

@@ -6,10 +6,10 @@ from collections.abc import Iterator, Sequence
 from json import JSONDecodeError
 from typing import Optional
 
-from pydantic import BaseModel, ConfigDict, Field
-from sqlalchemy import or_
+from pydantic import BaseModel, ConfigDict
 
 from constants import HIDDEN_VALUE
+from core.entities import DEFAULT_PLUGIN_ID
 from core.entities.model_entities import ModelStatus, ModelWithProviderEntity, SimpleModelProviderEntity
 from core.entities.provider_entities import (
     CustomConfiguration,
@@ -28,7 +28,6 @@ from core.model_runtime.entities.provider_entities import (
 )
 from core.model_runtime.model_providers.__base.ai_model import AIModel
 from core.model_runtime.model_providers.model_provider_factory import ModelProviderFactory
-from core.plugin.entities.plugin import ModelProviderID
 from extensions.ext_database import db
 from models.provider import (
     LoadBalancingModelConfig,
@@ -191,11 +190,8 @@ class ProviderConfiguration(BaseModel):
             db.session.query(Provider)
             .filter(
                 Provider.tenant_id == self.tenant_id,
+                Provider.provider_name == self.provider.provider,
                 Provider.provider_type == ProviderType.CUSTOM.value,
-                or_(
-                    Provider.provider_name == ModelProviderID(self.provider.provider).plugin_name,
-                    Provider.provider_name == self.provider.provider,
-                ),
             )
             .first()
         )
@@ -283,10 +279,7 @@ class ProviderConfiguration(BaseModel):
             db.session.query(Provider)
             .filter(
                 Provider.tenant_id == self.tenant_id,
-                or_(
-                    Provider.provider_name == ModelProviderID(self.provider.provider).plugin_name,
-                    Provider.provider_name == self.provider.provider,
-                ),
+                Provider.provider_name == self.provider.provider,
                 Provider.provider_type == ProviderType.CUSTOM.value,
             )
             .first()
@@ -1003,7 +996,7 @@ class ProviderConfigurations(BaseModel):
     """
 
     tenant_id: str
-    configurations: dict[str, ProviderConfiguration] = Field(default_factory=dict)
+    configurations: dict[str, ProviderConfiguration] = {}
 
     def __init__(self, tenant_id: str):
         super().__init__(tenant_id=tenant_id)
@@ -1059,7 +1052,7 @@ class ProviderConfigurations(BaseModel):
 
     def __getitem__(self, key):
         if "/" not in key:
-            key = str(ModelProviderID(key))
+            key = f"{DEFAULT_PLUGIN_ID}/{key}/{key}"
 
         return self.configurations[key]
 
@@ -1074,7 +1067,7 @@ class ProviderConfigurations(BaseModel):
 
     def get(self, key, default=None) -> ProviderConfiguration | None:
         if "/" not in key:
-            key = str(ModelProviderID(key))
+            key = f"{DEFAULT_PLUGIN_ID}/{key}/{key}"
 
         return self.configurations.get(key, default)  # type: ignore
 

api/core/external_data_tool/api/api.py

@@ -62,9 +62,8 @@ class ApiExternalDataTool(ExternalDataTool):
 
         if not api_based_extension:
             raise ValueError(
-                "[External data tool] API query failed, variable: {}, error: api_based_extension_id is invalid".format(
-                    self.variable
-                )
+                "[External data tool] API query failed, variable: {}, "
+                "error: api_based_extension_id is invalid".format(self.variable)
             )
 
         # decrypt api_key

api/core/file/helpers.py

@@ -33,7 +33,7 @@ def get_signed_file_url_for_plugin(filename: str, mimetype: str, tenant_id: str,
     sign = hmac.new(key, msg.encode(), hashlib.sha256).digest()
     encoded_sign = base64.urlsafe_b64encode(sign).decode()
 
-    return f"{url}?timestamp={timestamp}&nonce={nonce}&sign={encoded_sign}&user_id={user_id}&tenant_id={tenant_id}"
+    return f"{url}?timestamp={timestamp}&nonce={nonce}&sign={encoded_sign}&user_id={user_id}"
 
 
 def verify_plugin_file_signature(

api/core/file/models.py

@@ -90,7 +90,7 @@ class File(BaseModel):
     def markdown(self) -> str:
         url = self.generate_url()
         if self.type == FileType.IMAGE:
-            text = f"![{self.filename or ''}]({url})"
+            text = f'![{self.filename or ""}]({url})'
         else:
             text = f"[{self.filename or url}]({url})"
 

api/core/helper/ssrf_proxy.py

@@ -11,6 +11,15 @@ from configs import dify_config
 
 SSRF_DEFAULT_MAX_RETRIES = dify_config.SSRF_DEFAULT_MAX_RETRIES
 
+proxy_mounts = (
+    {
+        "http://": httpx.HTTPTransport(proxy=dify_config.SSRF_PROXY_HTTP_URL),
+        "https://": httpx.HTTPTransport(proxy=dify_config.SSRF_PROXY_HTTPS_URL),
+    }
+    if dify_config.SSRF_PROXY_HTTP_URL and dify_config.SSRF_PROXY_HTTPS_URL
+    else None
+)
+
 BACKOFF_FACTOR = 0.5
 STATUS_FORCELIST = [429, 500, 502, 503, 504]
 
@@ -41,11 +50,7 @@ def make_request(method, url, max_retries=SSRF_DEFAULT_MAX_RETRIES, **kwargs):
             if dify_config.SSRF_PROXY_ALL_URL:
                 with httpx.Client(proxy=dify_config.SSRF_PROXY_ALL_URL) as client:
                     response = client.request(method=method, url=url, **kwargs)
-            elif dify_config.SSRF_PROXY_HTTP_URL and dify_config.SSRF_PROXY_HTTPS_URL:
-                proxy_mounts = {
-                    "http://": httpx.HTTPTransport(proxy=dify_config.SSRF_PROXY_HTTP_URL),
-                    "https://": httpx.HTTPTransport(proxy=dify_config.SSRF_PROXY_HTTPS_URL),
-                }
+            elif proxy_mounts:
                 with httpx.Client(mounts=proxy_mounts) as client:
                     response = client.request(method=method, url=url, **kwargs)
             else:

api/core/hosting_configuration.py

@@ -41,13 +41,9 @@ class HostedModerationConfig(BaseModel):
 
 
 class HostingConfiguration:
-    provider_map: dict[str, HostingProvider]
+    provider_map: dict[str, HostingProvider] = {}
     moderation_config: Optional[HostedModerationConfig] = None
 
-    def __init__(self) -> None:
-        self.provider_map = {}
-        self.moderation_config = None
-
     def init_app(self, app: Flask) -> None:
         if dify_config.EDITION != "CLOUD":
             return

api/core/indexing_runner.py

@@ -530,6 +530,7 @@ class IndexingRunner:
         # chunk nodes by chunk size
         indexing_start_at = time.perf_counter()
         tokens = 0
+        chunk_size = 10
         if dataset_document.doc_form != IndexType.PARENT_CHILD_INDEX:
             # create keyword index
             create_keyword_thread = threading.Thread(
@@ -538,22 +539,11 @@ class IndexingRunner:
             )
             create_keyword_thread.start()
 
-        max_workers = 10
         if dataset.indexing_technique == "high_quality":
-            with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
+            with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor:
                 futures = []
-
-                # Distribute documents into multiple groups based on the hash values of page_content
-                # This is done to prevent multiple threads from processing the same document,
-                # Thereby avoiding potential database insertion deadlocks
-                document_groups: list[list[Document]] = [[] for _ in range(max_workers)]
-                for document in documents:
-                    hash = helper.generate_text_hash(document.page_content)
-                    group_index = int(hash, 16) % max_workers
-                    document_groups[group_index].append(document)
-                for chunk_documents in document_groups:
-                    if len(chunk_documents) == 0:
-                        continue
+                for i in range(0, len(documents), chunk_size):
+                    chunk_documents = documents[i : i + chunk_size]
                     futures.append(
                         executor.submit(
                             self._process_chunk,

api/core/llm_generator/prompts.py

@@ -131,7 +131,7 @@ JAVASCRIPT_CODE_GENERATOR_PROMPT_TEMPLATE = (
 SUGGESTED_QUESTIONS_AFTER_ANSWER_INSTRUCTION_PROMPT = (
     "Please help me predict the three most likely questions that human would ask, "
     "and keeping each question under 20 characters.\n"
-    "MAKE SURE your output is the SAME language as the Assistant's latest response. "
+    "MAKE SURE your output is the SAME language as the Assistant's latest response"
     "The output must be an array in JSON format following the specified schema:\n"
     '["question1","question2","question3"]\n'
 )

api/core/model_runtime/entities/__init__.py

@@ -1,4 +1,4 @@
-from .llm_entities import LLMMode, LLMResult, LLMResultChunk, LLMResultChunkDelta, LLMUsage
+from .llm_entities import LLMResult, LLMResultChunk, LLMResultChunkDelta, LLMUsage
 from .message_entities import (
     AssistantPromptMessage,
     AudioPromptMessageContent,
@@ -23,7 +23,6 @@ __all__ = [
     "AudioPromptMessageContent",
     "DocumentPromptMessageContent",
     "ImagePromptMessageContent",
-    "LLMMode",
     "LLMResult",
     "LLMResultChunk",
     "LLMResultChunkDelta",

api/core/model_runtime/entities/llm_entities.py

@@ -1,5 +1,5 @@
 from decimal import Decimal
-from enum import StrEnum
+from enum import Enum
 from typing import Optional
 
 from pydantic import BaseModel
@@ -8,7 +8,7 @@ from core.model_runtime.entities.message_entities import AssistantPromptMessage,
 from core.model_runtime.entities.model_entities import ModelUsage, PriceInfo
 
 
-class LLMMode(StrEnum):
+class LLMMode(Enum):
     """
     Enum class for large language model mode.
     """

api/core/model_runtime/model_providers/__base/ai_model.py

@@ -3,11 +3,8 @@ from typing import Optional
 
 from pydantic import BaseModel, ConfigDict, Field
 
-from core.model_runtime.entities.common_entities import I18nObject
-from core.model_runtime.entities.defaults import PARAMETER_RULE_TEMPLATE
 from core.model_runtime.entities.model_entities import (
     AIModelEntity,
-    DefaultParameterName,
     ModelType,
     PriceConfig,
     PriceInfo,
@@ -21,7 +18,6 @@ from core.model_runtime.errors.invoke import (
     InvokeRateLimitError,
     InvokeServerUnavailableError,
 )
-from core.model_runtime.model_providers.__base.tokenizers.gpt2_tokenzier import GPT2Tokenizer
 from core.plugin.entities.plugin_daemon import PluginDaemonInnerError, PluginModelProviderEntity
 from core.plugin.manager.model import PluginModelManager
 
@@ -148,102 +144,3 @@ class AIModel(BaseModel):
             model=model,
             credentials=credentials or {},
         )
-
-    def get_customizable_model_schema_from_credentials(self, model: str, credentials: dict) -> Optional[AIModelEntity]:
-        """
-        Get customizable model schema from credentials
-
-        :param model: model name
-        :param credentials: model credentials
-        :return: model schema
-        """
-        return self._get_customizable_model_schema(model, credentials)
-
-    def _get_customizable_model_schema(self, model: str, credentials: dict) -> Optional[AIModelEntity]:
-        """
-        Get customizable model schema and fill in the template
-        """
-        schema = self.get_customizable_model_schema(model, credentials)
-
-        if not schema:
-            return None
-
-        # fill in the template
-        new_parameter_rules = []
-        for parameter_rule in schema.parameter_rules:
-            if parameter_rule.use_template:
-                try:
-                    default_parameter_name = DefaultParameterName.value_of(parameter_rule.use_template)
-                    default_parameter_rule = self._get_default_parameter_rule_variable_map(default_parameter_name)
-                    if not parameter_rule.max and "max" in default_parameter_rule:
-                        parameter_rule.max = default_parameter_rule["max"]
-                    if not parameter_rule.min and "min" in default_parameter_rule:
-                        parameter_rule.min = default_parameter_rule["min"]
-                    if not parameter_rule.default and "default" in default_parameter_rule:
-                        parameter_rule.default = default_parameter_rule["default"]
-                    if not parameter_rule.precision and "precision" in default_parameter_rule:
-                        parameter_rule.precision = default_parameter_rule["precision"]
-                    if not parameter_rule.required and "required" in default_parameter_rule:
-                        parameter_rule.required = default_parameter_rule["required"]
-                    if not parameter_rule.help and "help" in default_parameter_rule:
-                        parameter_rule.help = I18nObject(
-                            en_US=default_parameter_rule["help"]["en_US"],
-                        )
-                    if (
-                        parameter_rule.help
-                        and not parameter_rule.help.en_US
-                        and ("help" in default_parameter_rule and "en_US" in default_parameter_rule["help"])
-                    ):
-                        parameter_rule.help.en_US = default_parameter_rule["help"]["en_US"]
-                    if (
-                        parameter_rule.help
-                        and not parameter_rule.help.zh_Hans
-                        and ("help" in default_parameter_rule and "zh_Hans" in default_parameter_rule["help"])
-                    ):
-                        parameter_rule.help.zh_Hans = default_parameter_rule["help"].get(
-                            "zh_Hans", default_parameter_rule["help"]["en_US"]
-                        )
-                except ValueError:
-                    pass
-
-            new_parameter_rules.append(parameter_rule)
-
-        schema.parameter_rules = new_parameter_rules
-
-        return schema
-
-    def get_customizable_model_schema(self, model: str, credentials: dict) -> Optional[AIModelEntity]:
-        """
-        Get customizable model schema
-
-        :param model: model name
-        :param credentials: model credentials
-        :return: model schema
-        """
-        return None
-
-    def _get_default_parameter_rule_variable_map(self, name: DefaultParameterName) -> dict:
-        """
-        Get default parameter rule for given name
-
-        :param name: parameter name
-        :return: parameter rule
-        """
-        default_parameter_rule = PARAMETER_RULE_TEMPLATE.get(name)
-
-        if not default_parameter_rule:
-            raise Exception(f"Invalid model parameter rule name {name}")
-
-        return default_parameter_rule
-
-    def _get_num_tokens_by_gpt2(self, text: str) -> int:
-        """
-        Get number of tokens for given prompt messages by gpt2
-        Some provider models do not provide an interface for obtaining the number of tokens.
-        Here, the gpt2 tokenizer is used to calculate the number of tokens.
-        This method can be executed offline, and the gpt2 tokenizer has been cached in the project.
-
-        :param text: plain text of prompt. You need to convert the original message to plain text
-        :return: number of tokens
-        """
-        return GPT2Tokenizer.get_num_tokens(text)

api/core/model_runtime/model_providers/__base/large_language_model.py

@@ -107,46 +107,11 @@ class LargeLanguageModel(AIModel):
                 content_list = []
                 usage = LLMUsage.empty_usage()
                 system_fingerprint = None
-                tools_calls: list[AssistantPromptMessage.ToolCall] = []
-
-                def increase_tool_call(new_tool_calls: list[AssistantPromptMessage.ToolCall]):
-                    def get_tool_call(tool_name: str):
-                        if not tool_name:
-                            return tools_calls[-1]
-
-                        tool_call = next(
-                            (tool_call for tool_call in tools_calls if tool_call.function.name == tool_name), None
-                        )
-                        if tool_call is None:
-                            tool_call = AssistantPromptMessage.ToolCall(
-                                id="",
-                                type="",
-                                function=AssistantPromptMessage.ToolCall.ToolCallFunction(name=tool_name, arguments=""),
-                            )
-                            tools_calls.append(tool_call)
-
-                        return tool_call
-
-                    for new_tool_call in new_tool_calls:
-                        # get tool call
-                        tool_call = get_tool_call(new_tool_call.function.name)
-                        # update tool call
-                        if new_tool_call.id:
-                            tool_call.id = new_tool_call.id
-                        if new_tool_call.type:
-                            tool_call.type = new_tool_call.type
-                        if new_tool_call.function.name:
-                            tool_call.function.name = new_tool_call.function.name
-                        if new_tool_call.function.arguments:
-                            tool_call.function.arguments += new_tool_call.function.arguments
-
                 for chunk in result:
                     if isinstance(chunk.delta.message.content, str):
                         content += chunk.delta.message.content
                     elif isinstance(chunk.delta.message.content, list):
                         content_list.extend(chunk.delta.message.content)
-                    if chunk.delta.message.tool_calls:
-                        increase_tool_call(chunk.delta.message.tool_calls)
 
                     usage = chunk.delta.usage or LLMUsage.empty_usage()
                     system_fingerprint = chunk.system_fingerprint
@@ -155,10 +120,7 @@ class LargeLanguageModel(AIModel):
                 result = LLMResult(
                     model=model,
                     prompt_messages=prompt_messages,
-                    message=AssistantPromptMessage(
-                        content=content or content_list,
-                        tool_calls=tools_calls,
-                    ),
+                    message=AssistantPromptMessage(content=content or content_list),
                     usage=usage,
                     system_fingerprint=system_fingerprint,
                 )
@@ -228,7 +190,7 @@ class LargeLanguageModel(AIModel):
         :return: result generator
         """
         callbacks = callbacks or []
-        assistant_message = AssistantPromptMessage(content="")
+        prompt_message = AssistantPromptMessage(content="")
         usage = None
         system_fingerprint = None
         real_model = model
@@ -250,7 +212,7 @@ class LargeLanguageModel(AIModel):
                     callbacks=callbacks,
                 )
 
-                assistant_message.content += chunk.delta.message.content
+                prompt_message.content += chunk.delta.message.content
                 real_model = chunk.model
                 if chunk.delta.usage:
                     usage = chunk.delta.usage
@@ -265,7 +227,7 @@ class LargeLanguageModel(AIModel):
             result=LLMResult(
                 model=real_model,
                 prompt_messages=prompt_messages,
-                message=assistant_message,
+                message=prompt_message,
                 usage=usage or LLMUsage.empty_usage(),
                 system_fingerprint=system_fingerprint,
             ),

api/core/model_runtime/model_providers/__base/tokenizers/gpt2_tokenzier.py

@@ -1,9 +1,6 @@
-import logging
 from threading import Lock
 from typing import Any
 
-logger = logging.getLogger(__name__)
-
 _tokenizer: Any = None
 _lock = Lock()
 
@@ -46,6 +43,5 @@ class GPT2Tokenizer:
                     base_path = abspath(__file__)
                     gpt2_tokenizer_path = join(dirname(base_path), "gpt2")
                     _tokenizer = TransformerGPT2Tokenizer.from_pretrained(gpt2_tokenizer_path)
-                    logger.info("Fallback to Transformers' GPT-2 tokenizer from tiktoken")
 
             return _tokenizer

api/core/model_runtime/model_providers/_position.yaml

@@ -1,5 +1,4 @@
 - openai
-- deepseek
 - anthropic
 - azure_openai
 - google
@@ -33,6 +32,7 @@
 - localai
 - volcengine_maas
 - openai_api_compatible
+- deepseek
 - hunyuan
 - siliconflow
 - perfxcloud

api/core/model_runtime/model_providers/model_provider_factory.py

@@ -7,6 +7,7 @@ from typing import Optional
 from pydantic import BaseModel
 
 import contexts
+from core.entities import DEFAULT_PLUGIN_ID
 from core.helper.position_helper import get_provider_position_map, sort_to_dict_by_position_map
 from core.model_runtime.entities.model_entities import AIModelEntity, ModelType
 from core.model_runtime.entities.provider_entities import ProviderConfig, ProviderEntity, SimpleProviderEntity
@@ -19,7 +20,6 @@ from core.model_runtime.model_providers.__base.text_embedding_model import TextE
 from core.model_runtime.model_providers.__base.tts_model import TTSModel
 from core.model_runtime.schema_validators.model_credential_schema_validator import ModelCredentialSchemaValidator
 from core.model_runtime.schema_validators.provider_credential_schema_validator import ProviderCredentialSchemaValidator
-from core.plugin.entities.plugin import ModelProviderID
 from core.plugin.entities.plugin_daemon import PluginModelProviderEntity
 from core.plugin.manager.asset import PluginAssetManager
 from core.plugin.manager.model import PluginModelManager
@@ -33,11 +33,9 @@ class ModelProviderExtension(BaseModel):
 
 
 class ModelProviderFactory:
-    provider_position_map: dict[str, int]
+    provider_position_map: dict[str, int] = {}
 
     def __init__(self, tenant_id: str) -> None:
-        self.provider_position_map = {}
-
         self.tenant_id = tenant_id
         self.plugin_model_manager = PluginModelManager()
 
@@ -114,9 +112,6 @@ class ModelProviderFactory:
         :param provider: provider name
         :return: provider schema
         """
-        if "/" not in provider:
-            provider = str(ModelProviderID(provider))
-
         # fetch plugin model providers
         plugin_model_provider_entities = self.get_plugin_model_providers()
 
@@ -361,5 +356,11 @@ class ModelProviderFactory:
         :param provider: provider name
         :return: plugin id and provider name
         """
-        provider_id = ModelProviderID(provider)
-        return provider_id.plugin_id, provider_id.provider_name
+        plugin_id = DEFAULT_PLUGIN_ID
+        provider_name = provider
+        if "/" in provider:
+            # get the plugin_id before provider
+            plugin_id = "/".join(provider.split("/")[:-1])
+            provider_name = provider.split("/")[-1]
+
+        return plugin_id, provider_name

api/core/model_runtime/model_providers/vertex_ai/llm/_position.yaml

@@ -1,22 +0,0 @@
-- claude-3-haiku@20240307
-- claude-3-opus@20240229
-- claude-3-sonnet@20240229
-- claude-3-5-sonnet-v2@20241022
-- claude-3-5-sonnet@20240620
-- gemini-1.0-pro-vision-001
-- gemini-1.0-pro-002
-- gemini-1.5-flash-001
-- gemini-1.5-flash-002
-- gemini-1.5-pro-001
-- gemini-1.5-pro-002
-- gemini-2.0-flash-001
-- gemini-2.0-flash-exp
-- gemini-2.0-flash-lite-preview-02-05
-- gemini-2.0-flash-thinking-exp-01-21
-- gemini-2.0-flash-thinking-exp-1219
-- gemini-2.0-pro-exp-02-05
-- gemini-exp-1114
-- gemini-exp-1121
-- gemini-exp-1206
-- gemini-flash-experimental
-- gemini-pro-experimental