add download file method

.devcontainer/Dockerfile

@@ -1,5 +1,5 @@
-FROM mcr.microsoft.com/devcontainers/python:3.12
+FROM mcr.microsoft.com/devcontainers/python:3.10
 
 # [Optional] Uncomment this section to install additional OS packages.
 # RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
-#     && apt-get -y install --no-install-recommends <your-package-list-here>
+#     && apt-get -y install --no-install-recommends <your-package-list-here>

.devcontainer/devcontainer.json

@@ -1,7 +1,7 @@
 // For format details, see https://aka.ms/devcontainer.json. For config options, see the
 // README at: https://github.com/devcontainers/templates/tree/main/src/anaconda
 {
-	"name": "Python 3.12",
+	"name": "Python 3.10",
 	"build": { 
 		"context": "..",
 		"dockerfile": "Dockerfile"

.devcontainer/post_create_command.sh

@@ -7,6 +7,5 @@ echo 'alias start-api="cd /workspaces/dify/api && poetry run python -m flask run
 echo 'alias start-worker="cd /workspaces/dify/api && poetry run python -m celery -A app.celery worker -P gevent -c 1 --loglevel INFO -Q dataset,generation,mail,ops_trace,app_deletion"' >> ~/.bashrc
 echo 'alias start-web="cd /workspaces/dify/web && npm run dev"' >> ~/.bashrc
 echo 'alias start-containers="cd /workspaces/dify/docker && docker-compose -f docker-compose.middleware.yaml -p dify up -d"' >> ~/.bashrc
-echo 'alias stop-containers="cd /workspaces/dify/docker && docker-compose -f docker-compose.middleware.yaml -p dify down"' >> ~/.bashrc
 
-source /home/vscode/.bashrc
+source /home/vscode/.bashrc

.github/actions/setup-poetry/action.yml

@@ -4,7 +4,7 @@ inputs:
   python-version:
     description: Python version to use and the Poetry installed with
     required: true
-    default: '3.11'
+    default: '3.10'
   poetry-version:
     description: Poetry version to set up
     required: true

.github/pull_request_template.md

@@ -8,9 +8,16 @@ Please include a summary of the change and which issue is fixed. Please also inc
 
 # Screenshots
 
-| Before | After |
-|--------|-------|
-| ...    | ...   |
+<table>
+  <tr>
+  <td>Before: </td>
+  <td>After: </td>
+  </tr>
+  <tr>
+  <td>...</td>
+  <td>...</td>
+  </tr>
+</table>
 
 # Checklist
 

.github/workflows/api-tests.yml

@@ -20,6 +20,7 @@ jobs:
     strategy:
       matrix:
         python-version:
+          - "3.10"
           - "3.11"
           - "3.12"
 
@@ -50,9 +51,6 @@ jobs:
       - name: Run ModelRuntime
         run: poetry run -C api bash dev/pytest/pytest_model_runtime.sh
 
-      - name: Run dify config tests
-        run: poetry run -C api python dev/pytest/pytest_config_tests.py
-
       - name: Run Tool
         run: poetry run -C api bash dev/pytest/pytest_tools.sh
 

.github/workflows/db-migration-test.yml

@@ -48,8 +48,6 @@ jobs:
           cp .env.example .env
 
       - name: Run DB Migration
-        env:
-          DEBUG: true
         run: |
           cd api
           poetry run python -m flask upgrade-db

.github/workflows/expose_service_ports.sh

@@ -9,6 +9,5 @@ yq eval '.services["pgvecto-rs"].ports += ["5431:5432"]' -i docker/docker-compos
 yq eval '.services["elasticsearch"].ports += ["9200:9200"]' -i docker/docker-compose.yaml
 yq eval '.services.couchbase-server.ports += ["8091-8096:8091-8096"]' -i docker/docker-compose.yaml
 yq eval '.services.couchbase-server.ports += ["11210:11210"]' -i docker/docker-compose.yaml
-yq eval '.services.tidb.ports += ["4000:4000"]' -i docker/docker-compose.yaml
 
-echo "Ports exposed for sandbox, weaviate, tidb, qdrant, chroma, milvus, pgvector, pgvecto-rs, elasticsearch, couchbase"
+echo "Ports exposed for sandbox, weaviate, qdrant, chroma, milvus, pgvector, pgvecto-rs, elasticsearch, couchbase"

.github/workflows/style.yml

@@ -37,7 +37,6 @@ jobs:
       - name: Ruff check
         if: steps.changed-files.outputs.any_changed == 'true'
         run: |
-          poetry run -C api ruff --version
           poetry run -C api ruff check ./api
           poetry run -C api ruff format --check ./api
 

.github/workflows/vdb-tests.yml

@@ -8,8 +8,6 @@ on:
       - api/core/rag/datasource/**
       - docker/**
       - .github/workflows/vdb-tests.yml
-      - api/poetry.lock
-      - api/pyproject.toml
 
 concurrency:
   group: vdb-tests-${{ github.head_ref || github.run_id }}
@@ -22,6 +20,7 @@ jobs:
     strategy:
       matrix:
         python-version:
+          - "3.10"
           - "3.11"
           - "3.12"
 
@@ -51,7 +50,7 @@ jobs:
       - name: Expose Service Ports
         run: sh .github/workflows/expose_service_ports.sh
 
-      - name: Set up Vector Stores (TiDB, Weaviate, Qdrant, PGVector, Milvus, PgVecto-RS, Chroma, MyScale, ElasticSearch, Couchbase)
+      - name: Set up Vector Stores (Weaviate, Qdrant, PGVector, Milvus, PgVecto-RS, Chroma, MyScale, ElasticSearch, Couchbase)
         uses: hoverkraft-tech/compose-action@v2.0.2
         with:
           compose-file: |
@@ -67,7 +66,6 @@ jobs:
             pgvector
             chroma
             elasticsearch
-            tidb
 
       - name: Test Vector Stores
         run: poetry run -C api bash dev/pytest/pytest_vdb.sh

CONTRIBUTING_CN.md

@@ -71,7 +71,7 @@ Dify 依赖以下工具和库：
 - [Docker Compose](https://docs.docker.com/compose/install/)
 - [Node.js v18.x (LTS)](http://nodejs.org)
 - [npm](https://www.npmjs.com/) version 8.x.x or [Yarn](https://yarnpkg.com/)
-- [Python](https://www.python.org/) version 3.11.x or 3.12.x
+- [Python](https://www.python.org/) version 3.10.x
 
 ### 4. 安装
 

CONTRIBUTING_JA.md

@@ -74,7 +74,7 @@ Dify を構築するには次の依存関係が必要です。それらがシス
 - [Docker Compose](https://docs.docker.com/compose/install/)
 - [Node.js v18.x (LTS)](http://nodejs.org)
 - [npm](https://www.npmjs.com/) version 8.x.x or [Yarn](https://yarnpkg.com/)
-- [Python](https://www.python.org/) version 3.11.x or 3.12.x
+- [Python](https://www.python.org/) version 3.10.x
 
 ### 4. インストール
 

CONTRIBUTING_VI.md

@@ -73,7 +73,7 @@ Dify yêu cầu các phụ thuộc sau để build, hãy đảm bảo chúng đ
 - [Docker Compose](https://docs.docker.com/compose/install/)
 - [Node.js v18.x (LTS)](http://nodejs.org)
 - [npm](https://www.npmjs.com/) phiên bản 8.x.x hoặc [Yarn](https://yarnpkg.com/)
-- [Python](https://www.python.org/) phiên bản 3.11.x hoặc 3.12.x
+- [Python](https://www.python.org/) phiên bản 3.10.x
 
 ### 4. Cài đặt
 
@@ -153,4 +153,4 @@ Và thế là xong! Khi PR của bạn được merge, bạn sẽ được giớ
 
 ## Nhận trợ giúp
 
-Nếu bạn gặp khó khăn hoặc có câu hỏi cấp bách trong quá trình đóng góp, hãy đặt câu hỏi của bạn trong vấn đề GitHub liên quan, hoặc tham gia [Discord](https://discord.gg/8Tpq4AcN9c) của chúng tôi để trò chuyện nhanh chóng.
+Nếu bạn gặp khó khăn hoặc có câu hỏi cấp bách trong quá trình đóng góp, hãy đặt câu hỏi của bạn trong vấn đề GitHub liên quan, hoặc tham gia [Discord](https://discord.gg/8Tpq4AcN9c) của chúng tôi để trò chuyện nhanh chóng.

README.md

@@ -147,13 +147,6 @@ Deploy Dify to Cloud Platform with a single click using [terraform](https://www.
 ##### Google Cloud
 - [Google Cloud Terraform by @sotazum](https://github.com/DeNA/dify-google-cloud-terraform)
 
-#### Using AWS CDK for Deployment
-
-Deploy Dify to AWS with [CDK](https://aws.amazon.com/cdk/)
-
-##### AWS 
-- [AWS CDK by @KevinZhao](https://github.com/aws-samples/solution-for-deploying-dify-on-aws)
-
 ## Contributing
 
 For those who'd like to contribute code, see our [Contribution Guide](https://github.com/langgenius/dify/blob/main/CONTRIBUTING.md). 

README_AR.md

@@ -190,13 +190,6 @@ docker compose up -d
 ##### Google Cloud
 - [Google Cloud Terraform بواسطة @sotazum](https://github.com/DeNA/dify-google-cloud-terraform)
 
-#### استخدام AWS CDK للنشر
-
-انشر Dify على AWS باستخدام [CDK](https://aws.amazon.com/cdk/)
-
-##### AWS 
-- [AWS CDK بواسطة @KevinZhao](https://github.com/aws-samples/solution-for-deploying-dify-on-aws)
-
 ## المساهمة
 
 لأولئك الذين يرغبون في المساهمة، انظر إلى [دليل المساهمة](https://github.com/langgenius/dify/blob/main/CONTRIBUTING.md) لدينا. 
@@ -229,10 +222,3 @@ docker compose up -d
 ## الرخصة
 
 هذا المستودع متاح تحت [رخصة البرنامج الحر Dify](LICENSE)، والتي تعتبر بشكل أساسي Apache 2.0 مع بعض القيود الإضافية.
-## الكشف عن الأمان
-
-لحماية خصوصيتك، يرجى تجنب نشر مشكلات الأمان على GitHub. بدلاً من ذلك، أرسل أسئلتك إلى security@dify.ai وسنقدم لك إجابة أكثر تفصيلاً.
-
-## الرخصة
-
-هذا المستودع متاح تحت [رخصة البرنامج الحر Dify](LICENSE)، والتي تعتبر بشكل أساسي Apache 2.0 مع بعض القيود الإضافية.

README_CN.md

@@ -213,13 +213,6 @@ docker compose up -d
 ##### Google Cloud
 - [Google Cloud Terraform by @sotazum](https://github.com/DeNA/dify-google-cloud-terraform)
 
-#### 使用 AWS CDK 部署
-
-使用 [CDK](https://aws.amazon.com/cdk/) 将 Dify 部署到 AWS
-
-##### AWS 
-- [AWS CDK by @KevinZhao](https://github.com/aws-samples/solution-for-deploying-dify-on-aws)
-
 ## Star History
 
 [![Star History Chart](https://api.star-history.com/svg?repos=langgenius/dify&type=Date)](https://star-history.com/#langgenius/dify&Date)

README_ES.md

@@ -215,13 +215,6 @@ Despliega Dify en una plataforma en la nube con un solo clic utilizando [terrafo
 ##### Google Cloud
 - [Google Cloud Terraform por @sotazum](https://github.com/DeNA/dify-google-cloud-terraform)
 
-#### Usando AWS CDK para el Despliegue
-
-Despliegue Dify en AWS usando [CDK](https://aws.amazon.com/cdk/)
-
-##### AWS 
-- [AWS CDK por @KevinZhao](https://github.com/aws-samples/solution-for-deploying-dify-on-aws)
-
 ## Contribuir
 
 Para aquellos que deseen contribuir con código, consulten nuestra [Guía de contribución](https://github.com/langgenius/dify/blob/main/CONTRIBUTING.md). 
@@ -255,10 +248,3 @@ Para proteger tu privacidad, evita publicar problemas de seguridad en GitHub. En
 ## Licencia
 
 Este repositorio está disponible bajo la [Licencia de Código Abierto de Dify](LICENSE), que es esencialmente Apache 2.0 con algunas restricciones adicionales.
-## Divulgación de Seguridad
-
-Para proteger tu privacidad, evita publicar problemas de seguridad en GitHub. En su lugar, envía tus preguntas a security@dify.ai y te proporcionaremos una respuesta más detallada.
-
-## Licencia
-
-Este repositorio está disponible bajo la [Licencia de Código Abierto de Dify](LICENSE), que es esencialmente Apache 2.0 con algunas restricciones adicionales.

README_FR.md

@@ -213,13 +213,6 @@ Déployez Dify sur une plateforme cloud en un clic en utilisant [terraform](http
 ##### Google Cloud
 - [Google Cloud Terraform par @sotazum](https://github.com/DeNA/dify-google-cloud-terraform)
 
-#### Utilisation d'AWS CDK pour le déploiement
-
-Déployez Dify sur AWS en utilisant [CDK](https://aws.amazon.com/cdk/)
-
-##### AWS 
-- [AWS CDK par @KevinZhao](https://github.com/aws-samples/solution-for-deploying-dify-on-aws)
-
 ## Contribuer
 
 Pour ceux qui souhaitent contribuer du code, consultez notre [Guide de contribution](https://github.com/langgenius/dify/blob/main/CONTRIBUTING.md). 
@@ -253,10 +246,3 @@ Pour protéger votre vie privée, veuillez éviter de publier des problèmes de
 ## Licence
 
 Ce référentiel est disponible sous la [Licence open source Dify](LICENSE), qui est essentiellement l'Apache 2.0 avec quelques restrictions supplémentaires.
-## Divulgation de sécurité
-
-Pour protéger votre vie privée, veuillez éviter de publier des problèmes de sécurité sur GitHub. Au lieu de cela, envoyez vos questions à security@dify.ai et nous vous fournirons une réponse plus détaillée.
-
-## Licence
-
-Ce référentiel est disponible sous la [Licence open source Dify](LICENSE), qui est essentiellement l'Apache 2.0 avec quelques restrictions supplémentaires.

README_JA.md

@@ -212,13 +212,6 @@ docker compose up -d
 ##### Google Cloud
 - [@sotazumによるGoogle Cloud Terraform](https://github.com/DeNA/dify-google-cloud-terraform)
 
-#### AWS CDK を使用したデプロイ
-
-[CDK](https://aws.amazon.com/cdk/) を使用して、DifyをAWSにデプロイします
-
-##### AWS 
-- [@KevinZhaoによるAWS CDK](https://github.com/aws-samples/solution-for-deploying-dify-on-aws)
-
 ## 貢献
 
 コードに貢献したい方は、[Contribution Guide](https://github.com/langgenius/dify/blob/main/CONTRIBUTING.md)を参照してください。

README_KL.md

@@ -213,13 +213,6 @@ wa'logh nIqHom neH ghun deployment toy'wI' [terraform](https://www.terraform.io/
 ##### Google Cloud
 - [Google Cloud Terraform qachlot @sotazum](https://github.com/DeNA/dify-google-cloud-terraform)
 
-#### AWS CDK atorlugh pilersitsineq
-
-wa'logh nIqHom neH ghun deployment toy'wI' [CDK](https://aws.amazon.com/cdk/) lo'laH.
-
-##### AWS 
-- [AWS CDK qachlot @KevinZhao](https://github.com/aws-samples/solution-for-deploying-dify-on-aws)
-
 ## Contributing
 
 For those who'd like to contribute code, see our [Contribution Guide](https://github.com/langgenius/dify/blob/main/CONTRIBUTING.md). 

README_KR.md

@@ -205,13 +205,6 @@ Dify를 Kubernetes에 배포하고 프리미엄 스케일링 설정을 구성했
 ##### Google Cloud
 - [sotazum의 Google Cloud Terraform](https://github.com/DeNA/dify-google-cloud-terraform)
 
-#### AWS CDK를 사용한 배포
-
-[CDK](https://aws.amazon.com/cdk/)를 사용하여 AWS에 Dify 배포
-
-##### AWS 
-- [KevinZhao의 AWS CDK](https://github.com/aws-samples/solution-for-deploying-dify-on-aws)
-
 ## 기여
 
 코드에 기여하고 싶은 분들은 [기여 가이드](https://github.com/langgenius/dify/blob/main/CONTRIBUTING.md)를 참조하세요.

README_PT.md

@@ -211,13 +211,6 @@ Implante o Dify na Plataforma Cloud com um único clique usando [terraform](http
 ##### Google Cloud
 - [Google Cloud Terraform por @sotazum](https://github.com/DeNA/dify-google-cloud-terraform)
 
-#### Usando AWS CDK para Implantação
-
-Implante o Dify na AWS usando [CDK](https://aws.amazon.com/cdk/)
-
-##### AWS 
-- [AWS CDK por @KevinZhao](https://github.com/aws-samples/solution-for-deploying-dify-on-aws)
-
 ## Contribuindo
 
 Para aqueles que desejam contribuir com código, veja nosso [Guia de Contribuição](https://github.com/langgenius/dify/blob/main/CONTRIBUTING.md). 

README_SI.md

@@ -145,13 +145,6 @@ namestite Dify v Cloud Platform z enim klikom z uporabo [terraform](https://www.
 ##### Google Cloud
 - [Google Cloud Terraform by @sotazum](https://github.com/DeNA/dify-google-cloud-terraform)
 
-#### Uporaba AWS CDK za uvajanje
-
-Uvedite Dify v AWS z uporabo [CDK](https://aws.amazon.com/cdk/)
-
-##### AWS 
-- [AWS CDK by @KevinZhao](https://github.com/aws-samples/solution-for-deploying-dify-on-aws)
-
 ## Prispevam
 
 Za tiste, ki bi radi prispevali kodo, si oglejte naš vodnik za prispevke . Hkrati vas prosimo, da podprete Dify tako, da ga delite na družbenih medijih ter na dogodkih in konferencah. 

README_TR.md

@@ -211,13 +211,6 @@ Dify'ı bulut platformuna tek tıklamayla dağıtın [terraform](https://www.ter
 ##### Google Cloud
 - [Google Cloud Terraform tarafından @sotazum](https://github.com/DeNA/dify-google-cloud-terraform)
 
-#### AWS CDK ile Dağıtım
-
-[CDK](https://aws.amazon.com/cdk/) kullanarak Dify'ı AWS'ye dağıtın
-
-##### AWS 
-- [AWS CDK tarafından @KevinZhao](https://github.com/aws-samples/solution-for-deploying-dify-on-aws)
-
 ## Katkıda Bulunma
 
 Kod katkısında bulunmak isteyenler için [Katkı Kılavuzumuza](https://github.com/langgenius/dify/blob/main/CONTRIBUTING.md) bakabilirsiniz.

README_VI.md

@@ -207,13 +207,6 @@ Triển khai Dify lên nền tảng đám mây với một cú nhấp chuột b
 ##### Google Cloud
 - [Google Cloud Terraform bởi @sotazum](https://github.com/DeNA/dify-google-cloud-terraform)
 
-#### Sử dụng AWS CDK để Triển khai
-
-Triển khai Dify trên AWS bằng [CDK](https://aws.amazon.com/cdk/)
-
-##### AWS 
-- [AWS CDK bởi @KevinZhao](https://github.com/aws-samples/solution-for-deploying-dify-on-aws)
-
 ## Đóng góp
 
 Đối với những người muốn đóng góp mã, xem [Hướng dẫn Đóng góp](https://github.com/langgenius/dify/blob/main/CONTRIBUTING.md) của chúng tôi. 

api/.env.example

@@ -56,27 +56,20 @@ DB_DATABASE=dify
 
 # Storage configuration
 # use for store upload files, private keys...
-# storage type: opendal, s3, aliyun-oss, azure-blob, baidu-obs, google-storage, huawei-obs, oci-storage, tencent-cos, volcengine-tos, supabase
-STORAGE_TYPE=opendal
-
-# Apache OpenDAL storage configuration, refer to https://github.com/apache/opendal
-OPENDAL_SCHEME=fs
-OPENDAL_FS_ROOT=storage
-
-# S3 Storage configuration
+# storage type: local, s3, aliyun-oss, azure-blob, baidu-obs, google-storage, huawei-obs, oci-storage, tencent-cos, volcengine-tos, supabase
+STORAGE_TYPE=local
+STORAGE_LOCAL_PATH=storage
 S3_USE_AWS_MANAGED_IAM=false
 S3_ENDPOINT=https://your-bucket-name.storage.s3.clooudflare.com
 S3_BUCKET_NAME=your-bucket-name
 S3_ACCESS_KEY=your-access-key
 S3_SECRET_KEY=your-secret-key
 S3_REGION=your-region
-
 # Azure Blob Storage configuration
 AZURE_BLOB_ACCOUNT_NAME=your-account-name
 AZURE_BLOB_ACCOUNT_KEY=your-account-key
 AZURE_BLOB_CONTAINER_NAME=yout-container-name
 AZURE_BLOB_ACCOUNT_URL=https://<your_account_name>.blob.core.windows.net
-
 # Aliyun oss Storage configuration
 ALIYUN_OSS_BUCKET_NAME=your-bucket-name
 ALIYUN_OSS_ACCESS_KEY=your-access-key
@@ -86,7 +79,6 @@ ALIYUN_OSS_AUTH_VERSION=v1
 ALIYUN_OSS_REGION=your-region
 # Don't start with '/'. OSS doesn't support leading slash in object names.
 ALIYUN_OSS_PATH=your-path
-
 # Google Storage configuration
 GOOGLE_STORAGE_BUCKET_NAME=yout-bucket-name
 GOOGLE_STORAGE_SERVICE_ACCOUNT_JSON_BASE64=your-google-service-account-json-base64-string
@@ -133,8 +125,8 @@ SUPABASE_URL=your-server-url
 WEB_API_CORS_ALLOW_ORIGINS=http://127.0.0.1:3000,*
 CONSOLE_CORS_ALLOW_ORIGINS=http://127.0.0.1:3000,*
 
-# Vector database configuration
-# support: weaviate, qdrant, milvus, myscale, relyt, pgvecto_rs, pgvector, pgvector, chroma, opensearch, tidb_vector, couchbase, vikingdb, upstash, lindorm, oceanbase
+
+# Vector database configuration, support: weaviate, qdrant, milvus, myscale, relyt, pgvecto_rs, pgvector, pgvector, chroma, opensearch, tidb_vector, couchbase, vikingdb, upstash, lindorm, oceanbase
 VECTOR_STORE=weaviate
 
 # Weaviate configuration
@@ -285,7 +277,6 @@ VIKINGDB_SOCKET_TIMEOUT=30
 LINDORM_URL=http://ld-*******************-proxy-search-pub.lindorm.aliyuncs.com:30070
 LINDORM_USERNAME=admin
 LINDORM_PASSWORD=admin
-USING_UGC_INDEX=False
 
 # OceanBase Vector configuration
 OCEANBASE_VECTOR_HOST=127.0.0.1
@@ -304,7 +295,8 @@ UPLOAD_VIDEO_FILE_SIZE_LIMIT=100
 UPLOAD_AUDIO_FILE_SIZE_LIMIT=50
 
 # Model configuration
-MULTIMODAL_SEND_FORMAT=base64
+MULTIMODAL_SEND_IMAGE_FORMAT=base64
+MULTIMODAL_SEND_VIDEO_FORMAT=base64
 PROMPT_GENERATION_MAX_TOKENS=512
 CODE_GENERATION_MAX_TOKENS=1024
 
@@ -337,7 +329,6 @@ NOTION_INTERNAL_SECRET=you-internal-secret
 ETL_TYPE=dify
 UNSTRUCTURED_API_URL=
 UNSTRUCTURED_API_KEY=
-SCARF_NO_ANALYTICS=true
 
 #ssrf
 SSRF_PROXY_HTTP_URL=
@@ -389,11 +380,9 @@ LOG_FILE_BACKUP_COUNT=5
 LOG_DATEFORMAT=%Y-%m-%d %H:%M:%S
 # Log Timezone
 LOG_TZ=UTC
-# Log format
-LOG_FORMAT=%(asctime)s,%(msecs)d %(levelname)-2s [%(filename)s:%(lineno)d] %(req_id)s %(message)s
 
 # Indexing configuration
-INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH=4000
+INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH=1000
 
 # Workflow runtime configuration
 WORKFLOW_MAX_EXECUTION_STEPS=500
@@ -421,9 +410,4 @@ POSITION_PROVIDER_EXCLUDES=
 # Reset password token expiry minutes
 RESET_PASSWORD_TOKEN_EXPIRY_MINUTES=5
 
-CREATE_TIDB_SERVICE_JOB_ENABLED=false
-
-# Maximum number of submitted thread count in a ThreadPool for parallel node execution
-MAX_SUBMIT_COUNT=100
-# Lockout duration in seconds
-LOGIN_LOCKOUT_DURATION=86400
+CREATE_TIDB_SERVICE_JOB_ENABLED=false

api/.ruff.toml

@@ -1,95 +0,0 @@
-exclude = [
-    "migrations/*",
-]
-line-length = 120
-
-[format]
-quote-style = "double"
-
-[lint]
-preview = true
-select = [
-    "B", # flake8-bugbear rules
-    "C4", # flake8-comprehensions
-    "E", # pycodestyle E rules
-    "F", # pyflakes rules
-    "FURB", # refurb rules
-    "I", # isort rules
-    "N", # pep8-naming
-    "PT", # flake8-pytest-style rules
-    "PLC0208", # iteration-over-set
-    "PLC2801", # unnecessary-dunder-call
-    "PLC0414", # useless-import-alias
-    "PLE0604", # invalid-all-object
-    "PLE0605", # invalid-all-format
-    "PLR0402", # manual-from-import
-    "PLR1711", # useless-return
-    "PLR1714", # repeated-equality-comparison
-    "RUF013", # implicit-optional
-    "RUF019", # unnecessary-key-check
-    "RUF100", # unused-noqa
-    "RUF101", # redirected-noqa
-    "RUF200", # invalid-pyproject-toml
-    "RUF022", # unsorted-dunder-all
-    "S506", # unsafe-yaml-load
-    "SIM", # flake8-simplify rules
-    "TRY400", # error-instead-of-exception
-    "TRY401", # verbose-log-message
-    "UP", # pyupgrade rules
-    "W191", # tab-indentation
-    "W605", # invalid-escape-sequence
-]
-
-ignore = [
-    "E402", # module-import-not-at-top-of-file
-    "E711", # none-comparison
-    "E712", # true-false-comparison
-    "E721", # type-comparison
-    "E722", # bare-except
-    "E731", # lambda-assignment
-    "F821", # undefined-name
-    "F841", # unused-variable
-    "FURB113", # repeated-append
-    "FURB152", # math-constant
-    "UP007", # non-pep604-annotation
-    "UP032", # f-string
-    "B005", # strip-with-multi-characters
-    "B006", # mutable-argument-default
-    "B007", # unused-loop-control-variable
-    "B026", # star-arg-unpacking-after-keyword-arg
-    "B904", # raise-without-from-inside-except
-    "B905", # zip-without-explicit-strict
-    "N806", # non-lowercase-variable-in-function
-    "N815", # mixed-case-variable-in-class-scope
-    "PT011", # pytest-raises-too-broad
-    "SIM102", # collapsible-if
-    "SIM103", # needless-bool
-    "SIM105", # suppressible-exception
-    "SIM107", # return-in-try-except-finally
-    "SIM108", # if-else-block-instead-of-if-exp
-    "SIM113", # eumerate-for-loop
-    "SIM117", # multiple-with-statements
-    "SIM210", # if-expr-with-true-false
-]
-
-[lint.per-file-ignores]
-"__init__.py" = [
-    "F401", # unused-import
-    "F811", # redefined-while-unused
-]
-"configs/*" = [
-    "N802", # invalid-function-name
-]
-"libs/gmpy2_pkcs10aep_cipher.py" = [
-    "N803", # invalid-argument-name
-]
-"tests/*" = [
-    "F811", # redefined-while-unused
-    "F401", # unused-import
-]
-
-[lint.pyflakes]
-extend-generics = [
-    "_pytest.monkeypatch",
-    "tests.integration_tests",
-]

api/Dockerfile

@@ -55,7 +55,7 @@ RUN apt-get update \
     && echo "deb http://deb.debian.org/debian testing main" > /etc/apt/sources.list \
     && apt-get update \
     # For Security
-    && apt-get install -y --no-install-recommends expat=2.6.4-1 libldap-2.5-0=2.5.18+dfsg-3+b1 perl=5.40.0-8 libsqlite3-0=3.46.1-1 zlib1g=1:1.3.dfsg+really1.3.1-1+b1 \
+    && apt-get install -y --no-install-recommends expat=2.6.4-1 libldap-2.5-0=2.5.18+dfsg-3+b1 perl=5.40.0-7 libsqlite3-0=3.46.1-1 zlib1g=1:1.3.dfsg+really1.3.1-1+b1 \
     # install a chinese font to support the use of tools like matplotlib
     && apt-get install -y fonts-noto-cjk \
     && apt-get autoremove -y \

api/app.py

@@ -1,30 +1,111 @@
-from libs import version_utils
+import os
+import sys
 
-# preparation before creating app
-version_utils.check_supported_python_version()
+from configs import dify_config
+
+if not dify_config.DEBUG:
+    from gevent import monkey
+
+    monkey.patch_all()
+
+    import grpc.experimental.gevent
+
+    grpc.experimental.gevent.init_gevent()
+
+import json
+import threading
+import time
+import warnings
+
+from flask import Response
+
+from app_factory import create_app
+
+# DO NOT REMOVE BELOW
+from events import event_handlers  # noqa: F401
+from extensions.ext_database import db
+
+# TODO: Find a way to avoid importing models here
+from models import account, dataset, model, source, task, tool, tools, web  # noqa: F401
+
+# DO NOT REMOVE ABOVE
+
+if sys.version_info[:2] == (3, 10):
+    print("Warning: Python 3.10 will not be supported in the next version.")
 
 
-def is_db_command():
-    import sys
+warnings.simplefilter("ignore", ResourceWarning)
 
-    if len(sys.argv) > 1 and sys.argv[0].endswith("flask") and sys.argv[1] == "db":
-        return True
-    return False
+os.environ["TZ"] = "UTC"
+# windows platform not support tzset
+if hasattr(time, "tzset"):
+    time.tzset()
 
 
 # create app
-if is_db_command():
-    from app_factory import create_migrations_app
+app = create_app()
+celery = app.extensions["celery"]
 
-    app = create_migrations_app()
-else:
-    from app_factory import create_app
-    from libs import threadings_utils
+if dify_config.TESTING:
+    print("App is running in TESTING mode")
 
-    threadings_utils.apply_gevent_threading_patch()
 
-    app = create_app()
-    celery = app.extensions["celery"]
+@app.after_request
+def after_request(response):
+    """Add Version headers to the response."""
+    response.headers.add("X-Version", dify_config.CURRENT_VERSION)
+    response.headers.add("X-Env", dify_config.DEPLOY_ENV)
+    return response
+
+
+@app.route("/health")
+def health():
+    return Response(
+        json.dumps({"pid": os.getpid(), "status": "ok", "version": dify_config.CURRENT_VERSION}),
+        status=200,
+        content_type="application/json",
+    )
+
+
+@app.route("/threads")
+def threads():
+    num_threads = threading.active_count()
+    threads = threading.enumerate()
+
+    thread_list = []
+    for thread in threads:
+        thread_name = thread.name
+        thread_id = thread.ident
+        is_alive = thread.is_alive()
+
+        thread_list.append(
+            {
+                "name": thread_name,
+                "id": thread_id,
+                "is_alive": is_alive,
+            }
+        )
+
+    return {
+        "pid": os.getpid(),
+        "thread_num": num_threads,
+        "threads": thread_list,
+    }
+
+
+@app.route("/db-pool-stat")
+def pool_stat():
+    engine = db.engine
+    return {
+        "pid": os.getpid(),
+        "pool_size": engine.pool.size(),
+        "checked_in_connections": engine.pool.checkedin(),
+        "checked_out_connections": engine.pool.checkedout(),
+        "overflow_connections": engine.pool.overflow(),
+        "connection_timeout": engine.pool.timeout(),
+        "recycle_time": db.engine.pool._recycle,
+    }
+
 
 if __name__ == "__main__":
     app.run(host="0.0.0.0", port=5001)

api/app_factory.py

@@ -1,14 +1,54 @@
-import logging
-import time
+import os
 
 from configs import dify_config
-from dify_app import DifyApp
+
+if not dify_config.DEBUG:
+    from gevent import monkey
+
+    monkey.patch_all()
+
+    import grpc.experimental.gevent
+
+    grpc.experimental.gevent.init_gevent()
+
+import json
+
+from flask import Flask, Response, request
+from flask_cors import CORS
+from werkzeug.exceptions import Unauthorized
+
+import contexts
+from commands import register_commands
+from configs import dify_config
+from extensions import (
+    ext_celery,
+    ext_code_based_extension,
+    ext_compress,
+    ext_database,
+    ext_hosting_provider,
+    ext_logging,
+    ext_login,
+    ext_mail,
+    ext_migrate,
+    ext_proxy_fix,
+    ext_redis,
+    ext_sentry,
+    ext_storage,
+)
+from extensions.ext_database import db
+from extensions.ext_login import login_manager
+from libs.passport import PassportService
+from services.account_service import AccountService
+
+
+class DifyApp(Flask):
+    pass
 
 
 # ----------------------------
 # Application Factory Function
 # ----------------------------
-def create_flask_app_with_configs() -> DifyApp:
+def create_flask_app_with_configs() -> Flask:
     """
     create a raw flask app
     with configs loaded from .env file
@@ -16,86 +56,123 @@ def create_flask_app_with_configs() -> DifyApp:
     dify_app = DifyApp(__name__)
     dify_app.config.from_mapping(dify_config.model_dump())
 
+    # populate configs into system environment variables
+    for key, value in dify_app.config.items():
+        if isinstance(value, str):
+            os.environ[key] = value
+        elif isinstance(value, int | float | bool):
+            os.environ[key] = str(value)
+        elif value is None:
+            os.environ[key] = ""
+
     return dify_app
 
 
-def create_app() -> DifyApp:
-    start_time = time.perf_counter()
+def create_app() -> Flask:
     app = create_flask_app_with_configs()
+    app.secret_key = dify_config.SECRET_KEY
     initialize_extensions(app)
-    end_time = time.perf_counter()
-    if dify_config.DEBUG:
-        logging.info(f"Finished create_app ({round((end_time - start_time) * 1000, 2)} ms)")
+    register_blueprints(app)
+    register_commands(app)
+
     return app
 
 
-def initialize_extensions(app: DifyApp):
-    from extensions import (
-        ext_app_metrics,
-        ext_blueprints,
-        ext_celery,
-        ext_code_based_extension,
-        ext_commands,
-        ext_compress,
-        ext_database,
-        ext_hosting_provider,
-        ext_import_modules,
-        ext_logging,
-        ext_login,
-        ext_mail,
-        ext_migrate,
-        ext_proxy_fix,
-        ext_redis,
-        ext_sentry,
-        ext_set_secretkey,
-        ext_storage,
-        ext_timezone,
-        ext_warnings,
+def initialize_extensions(app):
+    # Since the application instance is now created, pass it to each Flask
+    # extension instance to bind it to the Flask application instance (app)
+    ext_logging.init_app(app)
+    ext_compress.init_app(app)
+    ext_code_based_extension.init()
+    ext_database.init_app(app)
+    ext_migrate.init(app, db)
+    ext_redis.init_app(app)
+    ext_storage.init_app(app)
+    ext_celery.init_app(app)
+    ext_login.init_app(app)
+    ext_mail.init_app(app)
+    ext_hosting_provider.init_app(app)
+    ext_sentry.init_app(app)
+    ext_proxy_fix.init_app(app)
+
+
+# Flask-Login configuration
+@login_manager.request_loader
+def load_user_from_request(request_from_flask_login):
+    """Load user based on the request."""
+    if request.blueprint not in {"console", "inner_api"}:
+        return None
+    # Check if the user_id contains a dot, indicating the old format
+    auth_header = request.headers.get("Authorization", "")
+    if not auth_header:
+        auth_token = request.args.get("_token")
+        if not auth_token:
+            raise Unauthorized("Invalid Authorization token.")
+    else:
+        if " " not in auth_header:
+            raise Unauthorized("Invalid Authorization header format. Expected 'Bearer <api-key>' format.")
+        auth_scheme, auth_token = auth_header.split(None, 1)
+        auth_scheme = auth_scheme.lower()
+        if auth_scheme != "bearer":
+            raise Unauthorized("Invalid Authorization header format. Expected 'Bearer <api-key>' format.")
+
+    decoded = PassportService().verify(auth_token)
+    user_id = decoded.get("user_id")
+
+    logged_in_account = AccountService.load_logged_in_account(account_id=user_id)
+    if logged_in_account:
+        contexts.tenant_id.set(logged_in_account.current_tenant_id)
+    return logged_in_account
+
+
+@login_manager.unauthorized_handler
+def unauthorized_handler():
+    """Handle unauthorized requests."""
+    return Response(
+        json.dumps({"code": "unauthorized", "message": "Unauthorized."}),
+        status=401,
+        content_type="application/json",
     )
 
-    extensions = [
-        ext_timezone,
-        ext_logging,
-        ext_warnings,
-        ext_import_modules,
-        ext_set_secretkey,
-        ext_compress,
-        ext_code_based_extension,
-        ext_database,
-        ext_app_metrics,
-        ext_migrate,
-        ext_redis,
-        ext_storage,
-        ext_celery,
-        ext_login,
-        ext_mail,
-        ext_hosting_provider,
-        ext_sentry,
-        ext_proxy_fix,
-        ext_blueprints,
-        ext_commands,
-    ]
-    for ext in extensions:
-        short_name = ext.__name__.split(".")[-1]
-        is_enabled = ext.is_enabled() if hasattr(ext, "is_enabled") else True
-        if not is_enabled:
-            if dify_config.DEBUG:
-                logging.info(f"Skipped {short_name}")
-            continue
 
-        start_time = time.perf_counter()
-        ext.init_app(app)
-        end_time = time.perf_counter()
-        if dify_config.DEBUG:
-            logging.info(f"Loaded {short_name} ({round((end_time - start_time) * 1000, 2)} ms)")
+# register blueprint routers
+def register_blueprints(app):
+    from controllers.console import bp as console_app_bp
+    from controllers.files import bp as files_bp
+    from controllers.inner_api import bp as inner_api_bp
+    from controllers.service_api import bp as service_api_bp
+    from controllers.web import bp as web_bp
 
+    CORS(
+        service_api_bp,
+        allow_headers=["Content-Type", "Authorization", "X-App-Code"],
+        methods=["GET", "PUT", "POST", "DELETE", "OPTIONS", "PATCH"],
+    )
+    app.register_blueprint(service_api_bp)
 
-def create_migrations_app():
-    app = create_flask_app_with_configs()
-    from extensions import ext_database, ext_migrate
+    CORS(
+        web_bp,
+        resources={r"/*": {"origins": dify_config.WEB_API_CORS_ALLOW_ORIGINS}},
+        supports_credentials=True,
+        allow_headers=["Content-Type", "Authorization", "X-App-Code"],
+        methods=["GET", "PUT", "POST", "DELETE", "OPTIONS", "PATCH"],
+        expose_headers=["X-Version", "X-Env"],
+    )
 
-    # Initialize only required extensions
-    ext_database.init_app(app)
-    ext_migrate.init_app(app)
+    app.register_blueprint(web_bp)
 
-    return app
+    CORS(
+        console_app_bp,
+        resources={r"/*": {"origins": dify_config.CONSOLE_CORS_ALLOW_ORIGINS}},
+        supports_credentials=True,
+        allow_headers=["Content-Type", "Authorization"],
+        methods=["GET", "PUT", "POST", "DELETE", "OPTIONS", "PATCH"],
+        expose_headers=["X-Version", "X-Env"],
+    )
+
+    app.register_blueprint(console_app_bp)
+
+    CORS(files_bp, allow_headers=["Content-Type"], methods=["GET", "PUT", "POST", "DELETE", "OPTIONS", "PATCH"])
+    app.register_blueprint(files_bp)
+
+    app.register_blueprint(inner_api_bp)

api/commands.py

@@ -259,7 +259,7 @@ def migrate_knowledge_vector_database():
     skipped_count = 0
     total_count = 0
     vector_type = dify_config.VECTOR_STORE
-    upper_collection_vector_types = {
+    upper_colletion_vector_types = {
         VectorType.MILVUS,
         VectorType.PGVECTOR,
         VectorType.RELYT,
@@ -267,7 +267,7 @@ def migrate_knowledge_vector_database():
         VectorType.ORACLE,
         VectorType.ELASTICSEARCH,
     }
-    lower_collection_vector_types = {
+    lower_colletion_vector_types = {
         VectorType.ANALYTICDB,
         VectorType.CHROMA,
         VectorType.MYSCALE,
@@ -307,7 +307,7 @@ def migrate_knowledge_vector_database():
                         continue
                 collection_name = ""
                 dataset_id = dataset.id
-                if vector_type in upper_collection_vector_types:
+                if vector_type in upper_colletion_vector_types:
                     collection_name = Dataset.gen_collection_name_by_id(dataset_id)
                 elif vector_type == VectorType.QDRANT:
                     if dataset.collection_binding_id:
@@ -323,7 +323,7 @@ def migrate_knowledge_vector_database():
                     else:
                         collection_name = Dataset.gen_collection_name_by_id(dataset_id)
 
-                elif vector_type in lower_collection_vector_types:
+                elif vector_type in lower_colletion_vector_types:
                     collection_name = Dataset.gen_collection_name_by_id(dataset_id).lower()
                 else:
                     raise ValueError(f"Vector store {vector_type} is not supported.")
@@ -640,3 +640,15 @@ where sites.id is null limit 1000"""
                 break
 
     click.echo(click.style("Fix for missing app-related sites completed successfully!", fg="green"))
+
+
+def register_commands(app):
+    app.cli.add_command(reset_password)
+    app.cli.add_command(reset_email)
+    app.cli.add_command(reset_encrypt_key_pair)
+    app.cli.add_command(vdb_migrate)
+    app.cli.add_command(convert_to_agent_apps)
+    app.cli.add_command(add_qdrant_doc_id_index)
+    app.cli.add_command(create_tenant)
+    app.cli.add_command(upgrade_db)
+    app.cli.add_command(fix_app_site_missing)

api/configs/app_config.py

@@ -1,51 +1,11 @@
-import logging
-from typing import Any
+from pydantic_settings import SettingsConfigDict
 
-from pydantic.fields import FieldInfo
-from pydantic_settings import BaseSettings, PydanticBaseSettingsSource, SettingsConfigDict
-
-from .deploy import DeploymentConfig
-from .enterprise import EnterpriseFeatureConfig
-from .extra import ExtraServiceConfig
-from .feature import FeatureConfig
-from .middleware import MiddlewareConfig
-from .packaging import PackagingInfo
-from .remote_settings_sources import RemoteSettingsSource, RemoteSettingsSourceConfig, RemoteSettingsSourceName
-from .remote_settings_sources.apollo import ApolloSettingsSource
-
-logger = logging.getLogger(__name__)
-
-
-class RemoteSettingsSourceFactory(PydanticBaseSettingsSource):
-    def __init__(self, settings_cls: type[BaseSettings]):
-        super().__init__(settings_cls)
-
-    def get_field_value(self, field: FieldInfo, field_name: str) -> tuple[Any, str, bool]:
-        raise NotImplementedError
-
-    def __call__(self) -> dict[str, Any]:
-        current_state = self.current_state
-        remote_source_name = current_state.get("REMOTE_SETTINGS_SOURCE_NAME")
-        if not remote_source_name:
-            return {}
-
-        remote_source: RemoteSettingsSource | None = None
-        match remote_source_name:
-            case RemoteSettingsSourceName.APOLLO:
-                remote_source = ApolloSettingsSource(current_state)
-            case _:
-                logger.warning(f"Unsupported remote source: {remote_source_name}")
-                return {}
-
-        d: dict[str, Any] = {}
-
-        for field_name, field in self.settings_cls.model_fields.items():
-            field_value, field_key, value_is_complex = remote_source.get_field_value(field, field_name)
-            field_value = remote_source.prepare_field_value(field_name, field, field_value, value_is_complex)
-            if field_value is not None:
-                d[field_key] = field_value
-
-        return d
+from configs.deploy import DeploymentConfig
+from configs.enterprise import EnterpriseFeatureConfig
+from configs.extra import ExtraServiceConfig
+from configs.feature import FeatureConfig
+from configs.middleware import MiddlewareConfig
+from configs.packaging import PackagingInfo
 
 
 class DifyConfig(
@@ -59,8 +19,6 @@ class DifyConfig(
     MiddlewareConfig,
     # Extra service configs
     ExtraServiceConfig,
-    # Remote source configs
-    RemoteSettingsSourceConfig,
     # Enterprise feature configs
     # **Before using, please contact business@dify.ai by email to inquire about licensing matters.**
     EnterpriseFeatureConfig,
@@ -77,20 +35,3 @@ class DifyConfig(
     # please consider to arrange it in the proper config group of existed or added
     # for better readability and maintainability.
     # Thanks for your concentration and consideration.
-
-    @classmethod
-    def settings_customise_sources(
-        cls,
-        settings_cls: type[BaseSettings],
-        init_settings: PydanticBaseSettingsSource,
-        env_settings: PydanticBaseSettingsSource,
-        dotenv_settings: PydanticBaseSettingsSource,
-        file_secret_settings: PydanticBaseSettingsSource,
-    ) -> tuple[PydanticBaseSettingsSource, ...]:
-        return (
-            init_settings,
-            env_settings,
-            RemoteSettingsSourceFactory(settings_cls),
-            dotenv_settings,
-            file_secret_settings,
-        )

api/configs/deploy/__init__.py

@@ -17,6 +17,11 @@ class DeploymentConfig(BaseSettings):
         default=False,
     )
 
+    TESTING: bool = Field(
+        description="Enable testing mode for running automated tests",
+        default=False,
+    )
+
     EDITION: str = Field(
         description="Deployment edition of the application (e.g., 'SELF_HOSTED', 'CLOUD')",
         default="SELF_HOSTED",

api/configs/feature/__init__.py

@@ -439,17 +439,6 @@ class WorkflowConfig(BaseSettings):
     )
 
 
-class WorkflowNodeExecutionConfig(BaseSettings):
-    """
-    Configuration for workflow node execution
-    """
-
-    MAX_SUBMIT_COUNT: PositiveInt = Field(
-        description="Maximum number of submitted thread count in a ThreadPool for parallel node execution",
-        default=100,
-    )
-
-
 class AuthConfig(BaseSettings):
     """
     Configuration for authentication and OAuth
@@ -485,11 +474,6 @@ class AuthConfig(BaseSettings):
         default=60,
     )
 
-    LOGIN_LOCKOUT_DURATION: PositiveInt = Field(
-        description="Time (in seconds) a user must wait before retrying login after exceeding the rate limit.",
-        default=86400,
-    )
-
 
 class ModerationConfig(BaseSettings):
     """
@@ -601,11 +585,6 @@ class RagEtlConfig(BaseSettings):
         default=None,
     )
 
-    SCARF_NO_ANALYTICS: Optional[str] = Field(
-        description="This is about whether to disable Scarf analytics in Unstructured library.",
-        default="false",
-    )
-
 
 class DataSetConfig(BaseSettings):
     """
@@ -661,13 +640,18 @@ class IndexingConfig(BaseSettings):
 
     INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH: PositiveInt = Field(
         description="Maximum token length for text segmentation during indexing",
-        default=4000,
+        default=1000,
     )
 
 
-class MultiModalTransferConfig(BaseSettings):
-    MULTIMODAL_SEND_FORMAT: Literal["base64", "url"] = Field(
-        description="Format for sending files in multimodal contexts ('base64' or 'url'), default is base64",
+class VisionFormatConfig(BaseSettings):
+    MULTIMODAL_SEND_IMAGE_FORMAT: Literal["base64", "url"] = Field(
+        description="Format for sending images in multimodal contexts ('base64' or 'url'), default is base64",
+        default="base64",
+    )
+
+    MULTIMODAL_SEND_VIDEO_FORMAT: Literal["base64", "url"] = Field(
+        description="Format for sending videos in multimodal contexts ('base64' or 'url'), default is base64",
         default="base64",
     )
 
@@ -773,20 +757,19 @@ class FeatureConfig(
     FileAccessConfig,
     FileUploadConfig,
     HttpConfig,
+    VisionFormatConfig,
     InnerAPIConfig,
     IndexingConfig,
     LoggingConfig,
     MailConfig,
     ModelLoadBalanceConfig,
     ModerationConfig,
-    MultiModalTransferConfig,
     PositionConfig,
     RagEtlConfig,
     SecurityConfig,
     ToolConfig,
     UpdateConfig,
     WorkflowConfig,
-    WorkflowNodeExecutionConfig,
     WorkspaceConfig,
     LoginConfig,
     # hosted services config

api/configs/middleware/__init__.py

@@ -1,69 +1,54 @@
-from typing import Any, Literal, Optional
+from typing import Any, Optional
 from urllib.parse import quote_plus
 
 from pydantic import Field, NonNegativeInt, PositiveFloat, PositiveInt, computed_field
 from pydantic_settings import BaseSettings
 
-from .cache.redis_config import RedisConfig
-from .storage.aliyun_oss_storage_config import AliyunOSSStorageConfig
-from .storage.amazon_s3_storage_config import S3StorageConfig
-from .storage.azure_blob_storage_config import AzureBlobStorageConfig
-from .storage.baidu_obs_storage_config import BaiduOBSStorageConfig
-from .storage.google_cloud_storage_config import GoogleCloudStorageConfig
-from .storage.huawei_obs_storage_config import HuaweiCloudOBSStorageConfig
-from .storage.oci_storage_config import OCIStorageConfig
-from .storage.opendal_storage_config import OpenDALStorageConfig
-from .storage.supabase_storage_config import SupabaseStorageConfig
-from .storage.tencent_cos_storage_config import TencentCloudCOSStorageConfig
-from .storage.volcengine_tos_storage_config import VolcengineTOSStorageConfig
-from .vdb.analyticdb_config import AnalyticdbConfig
-from .vdb.baidu_vector_config import BaiduVectorDBConfig
-from .vdb.chroma_config import ChromaConfig
-from .vdb.couchbase_config import CouchbaseConfig
-from .vdb.elasticsearch_config import ElasticsearchConfig
-from .vdb.lindorm_config import LindormConfig
-from .vdb.milvus_config import MilvusConfig
-from .vdb.myscale_config import MyScaleConfig
-from .vdb.oceanbase_config import OceanBaseVectorConfig
-from .vdb.opensearch_config import OpenSearchConfig
-from .vdb.oracle_config import OracleConfig
-from .vdb.pgvector_config import PGVectorConfig
-from .vdb.pgvectors_config import PGVectoRSConfig
-from .vdb.qdrant_config import QdrantConfig
-from .vdb.relyt_config import RelytConfig
-from .vdb.tencent_vector_config import TencentVectorDBConfig
-from .vdb.tidb_on_qdrant_config import TidbOnQdrantConfig
-from .vdb.tidb_vector_config import TiDBVectorConfig
-from .vdb.upstash_config import UpstashConfig
-from .vdb.vikingdb_config import VikingDBConfig
-from .vdb.weaviate_config import WeaviateConfig
+from configs.middleware.cache.redis_config import RedisConfig
+from configs.middleware.storage.aliyun_oss_storage_config import AliyunOSSStorageConfig
+from configs.middleware.storage.amazon_s3_storage_config import S3StorageConfig
+from configs.middleware.storage.azure_blob_storage_config import AzureBlobStorageConfig
+from configs.middleware.storage.baidu_obs_storage_config import BaiduOBSStorageConfig
+from configs.middleware.storage.google_cloud_storage_config import GoogleCloudStorageConfig
+from configs.middleware.storage.huawei_obs_storage_config import HuaweiCloudOBSStorageConfig
+from configs.middleware.storage.oci_storage_config import OCIStorageConfig
+from configs.middleware.storage.supabase_storage_config import SupabaseStorageConfig
+from configs.middleware.storage.tencent_cos_storage_config import TencentCloudCOSStorageConfig
+from configs.middleware.storage.volcengine_tos_storage_config import VolcengineTOSStorageConfig
+from configs.middleware.vdb.analyticdb_config import AnalyticdbConfig
+from configs.middleware.vdb.baidu_vector_config import BaiduVectorDBConfig
+from configs.middleware.vdb.chroma_config import ChromaConfig
+from configs.middleware.vdb.couchbase_config import CouchbaseConfig
+from configs.middleware.vdb.elasticsearch_config import ElasticsearchConfig
+from configs.middleware.vdb.lindorm_config import LindormConfig
+from configs.middleware.vdb.milvus_config import MilvusConfig
+from configs.middleware.vdb.myscale_config import MyScaleConfig
+from configs.middleware.vdb.oceanbase_config import OceanBaseVectorConfig
+from configs.middleware.vdb.opensearch_config import OpenSearchConfig
+from configs.middleware.vdb.oracle_config import OracleConfig
+from configs.middleware.vdb.pgvector_config import PGVectorConfig
+from configs.middleware.vdb.pgvectors_config import PGVectoRSConfig
+from configs.middleware.vdb.qdrant_config import QdrantConfig
+from configs.middleware.vdb.relyt_config import RelytConfig
+from configs.middleware.vdb.tencent_vector_config import TencentVectorDBConfig
+from configs.middleware.vdb.tidb_on_qdrant_config import TidbOnQdrantConfig
+from configs.middleware.vdb.tidb_vector_config import TiDBVectorConfig
+from configs.middleware.vdb.upstash_config import UpstashConfig
+from configs.middleware.vdb.vikingdb_config import VikingDBConfig
+from configs.middleware.vdb.weaviate_config import WeaviateConfig
 
 
 class StorageConfig(BaseSettings):
-    STORAGE_TYPE: Literal[
-        "opendal",
-        "s3",
-        "aliyun-oss",
-        "azure-blob",
-        "baidu-obs",
-        "google-storage",
-        "huawei-obs",
-        "oci-storage",
-        "tencent-cos",
-        "volcengine-tos",
-        "supabase",
-        "local",
-    ] = Field(
+    STORAGE_TYPE: str = Field(
         description="Type of storage to use."
-        " Options: 'opendal', '(deprecated) local', 's3', 'aliyun-oss', 'azure-blob', 'baidu-obs', 'google-storage', "
-        "'huawei-obs', 'oci-storage', 'tencent-cos', 'volcengine-tos', 'supabase'. Default is 'opendal'.",
-        default="opendal",
+        " Options: 'local', 's3', 'aliyun-oss', 'azure-blob', 'baidu-obs', 'google-storage', 'huawei-obs', "
+        "'oci-storage', 'tencent-cos', 'volcengine-tos', 'supabase'. Default is 'local'.",
+        default="local",
     )
 
     STORAGE_LOCAL_PATH: str = Field(
         description="Path for local storage when STORAGE_TYPE is set to 'local'.",
         default="storage",
-        deprecated=True,
     )
 
 
@@ -88,7 +73,7 @@ class KeywordStoreConfig(BaseSettings):
     )
 
 
-class DatabaseConfig(BaseSettings):
+class DatabaseConfig:
     DB_HOST: str = Field(
         description="Hostname or IP address of the database server.",
         default="localhost",
@@ -250,7 +235,6 @@ class MiddlewareConfig(
     GoogleCloudStorageConfig,
     HuaweiCloudOBSStorageConfig,
     OCIStorageConfig,
-    OpenDALStorageConfig,
     S3StorageConfig,
     SupabaseStorageConfig,
     TencentCloudCOSStorageConfig,

api/configs/middleware/storage/baidu_obs_storage_config.py

@@ -1,10 +1,9 @@
 from typing import Optional
 
-from pydantic import Field
-from pydantic_settings import BaseSettings
+from pydantic import BaseModel, Field
 
 
-class BaiduOBSStorageConfig(BaseSettings):
+class BaiduOBSStorageConfig(BaseModel):
     """
     Configuration settings for Baidu Object Storage Service (OBS)
     """

api/configs/middleware/storage/huawei_obs_storage_config.py

@@ -1,10 +1,9 @@
 from typing import Optional
 
-from pydantic import Field
-from pydantic_settings import BaseSettings
+from pydantic import BaseModel, Field
 
 
-class HuaweiCloudOBSStorageConfig(BaseSettings):
+class HuaweiCloudOBSStorageConfig(BaseModel):
     """
     Configuration settings for Huawei Cloud Object Storage Service (OBS)
     """

api/configs/middleware/storage/opendal_storage_config.py

@@ -1,9 +0,0 @@
-from pydantic import Field
-from pydantic_settings import BaseSettings
-
-
-class OpenDALStorageConfig(BaseSettings):
-    OPENDAL_SCHEME: str = Field(
-        default="fs",
-        description="OpenDAL scheme.",
-    )

api/configs/middleware/storage/supabase_storage_config.py

@@ -1,10 +1,9 @@
 from typing import Optional
 
-from pydantic import Field
-from pydantic_settings import BaseSettings
+from pydantic import BaseModel, Field
 
 
-class SupabaseStorageConfig(BaseSettings):
+class SupabaseStorageConfig(BaseModel):
     """
     Configuration settings for Supabase Object Storage Service
     """

api/configs/middleware/storage/volcengine_tos_storage_config.py

@@ -1,10 +1,9 @@
 from typing import Optional
 
-from pydantic import Field
-from pydantic_settings import BaseSettings
+from pydantic import BaseModel, Field
 
 
-class VolcengineTOSStorageConfig(BaseSettings):
+class VolcengineTOSStorageConfig(BaseModel):
     """
     Configuration settings for Volcengine Tinder Object Storage (TOS)
     """

api/configs/middleware/vdb/analyticdb_config.py

@@ -1,10 +1,9 @@
 from typing import Optional
 
-from pydantic import Field, PositiveInt
-from pydantic_settings import BaseSettings
+from pydantic import BaseModel, Field, PositiveInt
 
 
-class AnalyticdbConfig(BaseSettings):
+class AnalyticdbConfig(BaseModel):
     """
     Configuration for connecting to Alibaba Cloud AnalyticDB for PostgreSQL.
     Refer to the following documentation for details on obtaining credentials:

api/configs/middleware/vdb/couchbase_config.py

@@ -1,10 +1,9 @@
 from typing import Optional
 
-from pydantic import Field
-from pydantic_settings import BaseSettings
+from pydantic import BaseModel, Field
 
 
-class CouchbaseConfig(BaseSettings):
+class CouchbaseConfig(BaseModel):
     """
     Couchbase configs
     """

api/configs/middleware/vdb/lindorm_config.py

@@ -21,14 +21,3 @@ class LindormConfig(BaseSettings):
         description="Lindorm password",
         default=None,
     )
-    DEFAULT_INDEX_TYPE: Optional[str] = Field(
-        description="Lindorm Vector Index Type, hnsw or flat is available in dify",
-        default="hnsw",
-    )
-    DEFAULT_DISTANCE_TYPE: Optional[str] = Field(
-        description="Vector Distance Type, support l2, cosinesimil, innerproduct", default="l2"
-    )
-    USING_UGC_INDEX: Optional[bool] = Field(
-        description="Using UGC index will store the same type of Index in a single index but can retrieve separately.",
-        default=False,
-    )

api/configs/middleware/vdb/myscale_config.py

@@ -1,8 +1,7 @@
-from pydantic import Field, PositiveInt
-from pydantic_settings import BaseSettings
+from pydantic import BaseModel, Field, PositiveInt
 
 
-class MyScaleConfig(BaseSettings):
+class MyScaleConfig(BaseModel):
     """
     Configuration settings for MyScale vector database
     """

api/configs/middleware/vdb/vikingdb_config.py

@@ -1,10 +1,9 @@
 from typing import Optional
 
-from pydantic import Field
-from pydantic_settings import BaseSettings
+from pydantic import BaseModel, Field
 
 
-class VikingDBConfig(BaseSettings):
+class VikingDBConfig(BaseModel):
     """
     Configuration for connecting to Volcengine VikingDB.
     Refer to the following documentation for details on obtaining credentials:

api/configs/packaging/__init__.py

@@ -9,7 +9,7 @@ class PackagingInfo(BaseSettings):
 
     CURRENT_VERSION: str = Field(
         description="Dify version",
-        default="0.14.1",
+        default="0.11.2",
     )
 
     COMMIT_SHA: str = Field(

api/configs/remote_settings_sources/__init__.py

@@ -1,17 +0,0 @@
-from typing import Optional
-
-from pydantic import Field
-
-from .apollo import ApolloSettingsSourceInfo
-from .base import RemoteSettingsSource
-from .enums import RemoteSettingsSourceName
-
-
-class RemoteSettingsSourceConfig(ApolloSettingsSourceInfo):
-    REMOTE_SETTINGS_SOURCE_NAME: RemoteSettingsSourceName | str = Field(
-        description="name of remote config source",
-        default="",
-    )
-
-
-__all__ = ["RemoteSettingsSource", "RemoteSettingsSourceConfig", "RemoteSettingsSourceName"]

api/configs/remote_settings_sources/apollo/__init__.py

@@ -1,55 +0,0 @@
-from collections.abc import Mapping
-from typing import Any, Optional
-
-from pydantic import Field
-from pydantic.fields import FieldInfo
-from pydantic_settings import BaseSettings
-
-from configs.remote_settings_sources.base import RemoteSettingsSource
-
-from .client import ApolloClient
-
-
-class ApolloSettingsSourceInfo(BaseSettings):
-    """
-    Packaging build information
-    """
-
-    APOLLO_APP_ID: Optional[str] = Field(
-        description="apollo app_id",
-        default=None,
-    )
-
-    APOLLO_CLUSTER: Optional[str] = Field(
-        description="apollo cluster",
-        default=None,
-    )
-
-    APOLLO_CONFIG_URL: Optional[str] = Field(
-        description="apollo config url",
-        default=None,
-    )
-
-    APOLLO_NAMESPACE: Optional[str] = Field(
-        description="apollo namespace",
-        default=None,
-    )
-
-
-class ApolloSettingsSource(RemoteSettingsSource):
-    def __init__(self, configs: Mapping[str, Any]):
-        self.client = ApolloClient(
-            app_id=configs["APOLLO_APP_ID"],
-            cluster=configs["APOLLO_CLUSTER"],
-            config_url=configs["APOLLO_CONFIG_URL"],
-            start_hot_update=False,
-            _notification_map={configs["APOLLO_NAMESPACE"]: -1},
-        )
-        self.namespace = configs["APOLLO_NAMESPACE"]
-        self.remote_configs = self.client.get_all_dicts(self.namespace)
-
-    def get_field_value(self, field: FieldInfo, field_name: str) -> tuple[Any, str, bool]:
-        if not isinstance(self.remote_configs, dict):
-            raise ValueError(f"remote configs is not dict, but {type(self.remote_configs)}")
-        field_value = self.remote_configs.get(field_name)
-        return field_value, field_name, False

api/configs/remote_settings_sources/apollo/client.py

@@ -1,303 +0,0 @@
-import hashlib
-import json
-import logging
-import os
-import threading
-import time
-from pathlib import Path
-
-from .python_3x import http_request, makedirs_wrapper
-from .utils import (
-    CONFIGURATIONS,
-    NAMESPACE_NAME,
-    NOTIFICATION_ID,
-    get_value_from_dict,
-    init_ip,
-    no_key_cache_key,
-    signature,
-    url_encode_wrapper,
-)
-
-logger = logging.getLogger(__name__)
-
-
-class ApolloClient:
-    def __init__(
-        self,
-        config_url,
-        app_id,
-        cluster="default",
-        secret="",
-        start_hot_update=True,
-        change_listener=None,
-        _notification_map=None,
-    ):
-        # Core routing parameters
-        self.config_url = config_url
-        self.cluster = cluster
-        self.app_id = app_id
-
-        # Non-core parameters
-        self.ip = init_ip()
-        self.secret = secret
-
-        # Check the parameter variables
-
-        # Private control variables
-        self._cycle_time = 5
-        self._stopping = False
-        self._cache = {}
-        self._no_key = {}
-        self._hash = {}
-        self._pull_timeout = 75
-        self._cache_file_path = os.path.expanduser("~") + "/.dify/config/remote-settings/apollo/cache/"
-        self._long_poll_thread = None
-        self._change_listener = change_listener  # "add" "delete" "update"
-        if _notification_map is None:
-            _notification_map = {"application": -1}
-        self._notification_map = _notification_map
-        self.last_release_key = None
-        # Private startup method
-        self._path_checker()
-        if start_hot_update:
-            self._start_hot_update()
-
-        # start the heartbeat thread
-        heartbeat = threading.Thread(target=self._heart_beat)
-        heartbeat.daemon = True
-        heartbeat.start()
-
-    def get_json_from_net(self, namespace="application"):
-        url = "{}/configs/{}/{}/{}?releaseKey={}&ip={}".format(
-            self.config_url, self.app_id, self.cluster, namespace, "", self.ip
-        )
-        try:
-            code, body = http_request(url, timeout=3, headers=self._sign_headers(url))
-            if code == 200:
-                if not body:
-                    logger.error(f"get_json_from_net load configs failed, body is {body}")
-                    return None
-                data = json.loads(body)
-                data = data["configurations"]
-                return_data = {CONFIGURATIONS: data}
-                return return_data
-            else:
-                return None
-        except Exception:
-            logger.exception("an error occurred in get_json_from_net")
-            return None
-
-    def get_value(self, key, default_val=None, namespace="application"):
-        try:
-            # read memory configuration
-            namespace_cache = self._cache.get(namespace)
-            val = get_value_from_dict(namespace_cache, key)
-            if val is not None:
-                return val
-
-            no_key = no_key_cache_key(namespace, key)
-            if no_key in self._no_key:
-                return default_val
-
-            # read the network configuration
-            namespace_data = self.get_json_from_net(namespace)
-            val = get_value_from_dict(namespace_data, key)
-            if val is not None:
-                self._update_cache_and_file(namespace_data, namespace)
-                return val
-
-            # read the file configuration
-            namespace_cache = self._get_local_cache(namespace)
-            val = get_value_from_dict(namespace_cache, key)
-            if val is not None:
-                self._update_cache_and_file(namespace_cache, namespace)
-                return val
-
-            # If all of them are not obtained, the default value is returned
-            # and the local cache is set to None
-            self._set_local_cache_none(namespace, key)
-            return default_val
-        except Exception:
-            logger.exception("get_value has error, [key is %s], [namespace is %s]", key, namespace)
-            return default_val
-
-    # Set the key of a namespace to none, and do not set default val
-    # to ensure the real-time correctness of the function call.
-    # If the user does not have the same default val twice
-    # and the default val is used here, there may be a problem.
-    def _set_local_cache_none(self, namespace, key):
-        no_key = no_key_cache_key(namespace, key)
-        self._no_key[no_key] = key
-
-    def _start_hot_update(self):
-        self._long_poll_thread = threading.Thread(target=self._listener)
-        # When the asynchronous thread is started, the daemon thread will automatically exit
-        # when the main thread is launched.
-        self._long_poll_thread.daemon = True
-        self._long_poll_thread.start()
-
-    def stop(self):
-        self._stopping = True
-        logger.info("Stopping listener...")
-
-    # Call the set callback function, and if it is abnormal, try it out
-    def _call_listener(self, namespace, old_kv, new_kv):
-        if self._change_listener is None:
-            return
-        if old_kv is None:
-            old_kv = {}
-        if new_kv is None:
-            new_kv = {}
-        try:
-            for key in old_kv:
-                new_value = new_kv.get(key)
-                old_value = old_kv.get(key)
-                if new_value is None:
-                    # If newValue is empty, it means key, and the value is deleted.
-                    self._change_listener("delete", namespace, key, old_value)
-                    continue
-                if new_value != old_value:
-                    self._change_listener("update", namespace, key, new_value)
-                    continue
-            for key in new_kv:
-                new_value = new_kv.get(key)
-                old_value = old_kv.get(key)
-                if old_value is None:
-                    self._change_listener("add", namespace, key, new_value)
-        except BaseException as e:
-            logger.warning(str(e))
-
-    def _path_checker(self):
-        if not os.path.isdir(self._cache_file_path):
-            makedirs_wrapper(self._cache_file_path)
-
-    # update the local cache and file cache
-    def _update_cache_and_file(self, namespace_data, namespace="application"):
-        # update the local cache
-        self._cache[namespace] = namespace_data
-        # update the file cache
-        new_string = json.dumps(namespace_data)
-        new_hash = hashlib.md5(new_string.encode("utf-8")).hexdigest()
-        if self._hash.get(namespace) == new_hash:
-            pass
-        else:
-            file_path = Path(self._cache_file_path) / f"{self.app_id}_configuration_{namespace}.txt"
-            file_path.write_text(new_string)
-            self._hash[namespace] = new_hash
-
-    # get the configuration from the local file
-    def _get_local_cache(self, namespace="application"):
-        cache_file_path = os.path.join(self._cache_file_path, f"{self.app_id}_configuration_{namespace}.txt")
-        if os.path.isfile(cache_file_path):
-            with open(cache_file_path) as f:
-                result = json.loads(f.readline())
-            return result
-        return {}
-
-    def _long_poll(self):
-        notifications = []
-        for key in self._cache:
-            namespace_data = self._cache[key]
-            notification_id = -1
-            if NOTIFICATION_ID in namespace_data:
-                notification_id = self._cache[key][NOTIFICATION_ID]
-            notifications.append({NAMESPACE_NAME: key, NOTIFICATION_ID: notification_id})
-        try:
-            # if the length is 0 it is returned directly
-            if len(notifications) == 0:
-                return
-            url = "{}/notifications/v2".format(self.config_url)
-            params = {
-                "appId": self.app_id,
-                "cluster": self.cluster,
-                "notifications": json.dumps(notifications, ensure_ascii=False),
-            }
-            param_str = url_encode_wrapper(params)
-            url = url + "?" + param_str
-            code, body = http_request(url, self._pull_timeout, headers=self._sign_headers(url))
-            http_code = code
-            if http_code == 304:
-                logger.debug("No change, loop...")
-                return
-            if http_code == 200:
-                if not body:
-                    logger.error(f"_long_poll load configs failed,body is {body}")
-                    return
-                data = json.loads(body)
-                for entry in data:
-                    namespace = entry[NAMESPACE_NAME]
-                    n_id = entry[NOTIFICATION_ID]
-                    logger.info("%s has changes: notificationId=%d", namespace, n_id)
-                    self._get_net_and_set_local(namespace, n_id, call_change=True)
-                    return
-            else:
-                logger.warning("Sleep...")
-        except Exception as e:
-            logger.warning(str(e))
-
-    def _get_net_and_set_local(self, namespace, n_id, call_change=False):
-        namespace_data = self.get_json_from_net(namespace)
-        if not namespace_data:
-            return
-        namespace_data[NOTIFICATION_ID] = n_id
-        old_namespace = self._cache.get(namespace)
-        self._update_cache_and_file(namespace_data, namespace)
-        if self._change_listener is not None and call_change and old_namespace:
-            old_kv = old_namespace.get(CONFIGURATIONS)
-            new_kv = namespace_data.get(CONFIGURATIONS)
-            self._call_listener(namespace, old_kv, new_kv)
-
-    def _listener(self):
-        logger.info("start long_poll")
-        while not self._stopping:
-            self._long_poll()
-            time.sleep(self._cycle_time)
-        logger.info("stopped, long_poll")
-
-    # add the need for endorsement to the header
-    def _sign_headers(self, url):
-        headers = {}
-        if self.secret == "":
-            return headers
-        uri = url[len(self.config_url) : len(url)]
-        time_unix_now = str(int(round(time.time() * 1000)))
-        headers["Authorization"] = "Apollo " + self.app_id + ":" + signature(time_unix_now, uri, self.secret)
-        headers["Timestamp"] = time_unix_now
-        return headers
-
-    def _heart_beat(self):
-        while not self._stopping:
-            for namespace in self._notification_map:
-                self._do_heart_beat(namespace)
-            time.sleep(60 * 10)  # 10分钟
-
-    def _do_heart_beat(self, namespace):
-        url = "{}/configs/{}/{}/{}?ip={}".format(self.config_url, self.app_id, self.cluster, namespace, self.ip)
-        try:
-            code, body = http_request(url, timeout=3, headers=self._sign_headers(url))
-            if code == 200:
-                if not body:
-                    logger.error(f"_do_heart_beat load configs failed,body is {body}")
-                    return None
-                data = json.loads(body)
-                if self.last_release_key == data["releaseKey"]:
-                    return None
-                self.last_release_key = data["releaseKey"]
-                data = data["configurations"]
-                self._update_cache_and_file(data, namespace)
-            else:
-                return None
-        except Exception:
-            logger.exception("an error occurred in _do_heart_beat")
-            return None
-
-    def get_all_dicts(self, namespace):
-        namespace_data = self._cache.get(namespace)
-        if namespace_data is None:
-            net_namespace_data = self.get_json_from_net(namespace)
-            if not net_namespace_data:
-                return namespace_data
-            namespace_data = net_namespace_data.get(CONFIGURATIONS)
-            if namespace_data:
-                self._update_cache_and_file(namespace_data, namespace)
-        return namespace_data

api/configs/remote_settings_sources/apollo/python_3x.py

@@ -1,41 +0,0 @@
-import logging
-import os
-import ssl
-import urllib.request
-from urllib import parse
-from urllib.error import HTTPError
-
-# Create an SSL context that allows for a lower level of security
-ssl_context = ssl.create_default_context()
-ssl_context.set_ciphers("HIGH:!DH:!aNULL")
-ssl_context.check_hostname = False
-ssl_context.verify_mode = ssl.CERT_NONE
-
-# Create an opener object and pass in a custom SSL context
-opener = urllib.request.build_opener(urllib.request.HTTPSHandler(context=ssl_context))
-
-urllib.request.install_opener(opener)
-
-logger = logging.getLogger(__name__)
-
-
-def http_request(url, timeout, headers={}):
-    try:
-        request = urllib.request.Request(url, headers=headers)
-        res = urllib.request.urlopen(request, timeout=timeout)
-        body = res.read().decode("utf-8")
-        return res.code, body
-    except HTTPError as e:
-        if e.code == 304:
-            logger.warning("http_request error,code is 304, maybe you should check secret")
-            return 304, None
-        logger.warning("http_request error,code is %d, msg is %s", e.code, e.msg)
-        raise e
-
-
-def url_encode(params):
-    return parse.urlencode(params)
-
-
-def makedirs_wrapper(path):
-    os.makedirs(path, exist_ok=True)

api/configs/remote_settings_sources/apollo/utils.py

@@ -1,51 +0,0 @@
-import hashlib
-import socket
-
-from .python_3x import url_encode
-
-# define constants
-CONFIGURATIONS = "configurations"
-NOTIFICATION_ID = "notificationId"
-NAMESPACE_NAME = "namespaceName"
-
-
-# add timestamps uris and keys
-def signature(timestamp, uri, secret):
-    import base64
-    import hmac
-
-    string_to_sign = "" + timestamp + "\n" + uri
-    hmac_code = hmac.new(secret.encode(), string_to_sign.encode(), hashlib.sha1).digest()
-    return base64.b64encode(hmac_code).decode()
-
-
-def url_encode_wrapper(params):
-    return url_encode(params)
-
-
-def no_key_cache_key(namespace, key):
-    return "{}{}{}".format(namespace, len(namespace), key)
-
-
-# Returns whether the obtained value is obtained, and None if it does not
-def get_value_from_dict(namespace_cache, key):
-    if namespace_cache:
-        kv_data = namespace_cache.get(CONFIGURATIONS)
-        if kv_data is None:
-            return None
-        if key in kv_data:
-            return kv_data[key]
-    return None
-
-
-def init_ip():
-    ip = ""
-    s = None
-    try:
-        s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
-        s.connect(("8.8.8.8", 53))
-        ip = s.getsockname()[0]
-    finally:
-        if s:
-            s.close()
-    return ip

api/configs/remote_settings_sources/base.py

@@ -1,15 +0,0 @@
-from collections.abc import Mapping
-from typing import Any
-
-from pydantic.fields import FieldInfo
-
-
-class RemoteSettingsSource:
-    def __init__(self, configs: Mapping[str, Any]):
-        pass
-
-    def get_field_value(self, field: FieldInfo, field_name: str) -> tuple[Any, str, bool]:
-        raise NotImplementedError
-
-    def prepare_field_value(self, field_name: str, field: FieldInfo, value: Any, value_is_complex: bool) -> Any:
-        return value

api/configs/remote_settings_sources/enums.py

@@ -1,5 +0,0 @@
-from enum import StrEnum
-
-
-class RemoteSettingsSourceName(StrEnum):
-    APOLLO = "apollo"

api/constants/__init__.py

@@ -14,11 +14,11 @@ AUDIO_EXTENSIONS.extend([ext.upper() for ext in AUDIO_EXTENSIONS])
 
 
 if dify_config.ETL_TYPE == "Unstructured":
-    DOCUMENT_EXTENSIONS = ["txt", "markdown", "md", "mdx", "pdf", "html", "htm", "xlsx", "xls"]
+    DOCUMENT_EXTENSIONS = ["txt", "markdown", "md", "pdf", "html", "htm", "xlsx", "xls"]
     DOCUMENT_EXTENSIONS.extend(("docx", "csv", "eml", "msg", "pptx", "xml", "epub"))
     if dify_config.UNSTRUCTURED_API_URL:
         DOCUMENT_EXTENSIONS.append("ppt")
     DOCUMENT_EXTENSIONS.extend([ext.upper() for ext in DOCUMENT_EXTENSIONS])
 else:
-    DOCUMENT_EXTENSIONS = ["txt", "markdown", "md", "mdx", "pdf", "html", "htm", "xlsx", "xls", "docx", "csv"]
+    DOCUMENT_EXTENSIONS = ["txt", "markdown", "md", "pdf", "html", "htm", "xlsx", "xls", "docx", "csv"]
     DOCUMENT_EXTENSIONS.extend([ext.upper() for ext in DOCUMENT_EXTENSIONS])

api/constants/languages.py

@@ -18,7 +18,6 @@ language_timezone_mapping = {
     "tr-TR": "Europe/Istanbul",
     "fa-IR": "Asia/Tehran",
     "sl-SI": "Europe/Ljubljana",
-    "th-TH": "Asia/Bangkok",
 }
 
 languages = list(language_timezone_mapping.keys())

api/controllers/console/__init__.py

@@ -62,6 +62,7 @@ from .datasets import (
     external,
     hit_testing,
     website,
+    fta_test,
 )
 
 # Import explore controllers

api/controllers/console/admin.py

@@ -31,7 +31,7 @@ def admin_required(view):
         if auth_scheme != "bearer":
             raise Unauthorized("Invalid Authorization header format. Expected 'Bearer <api-key>' format.")
 
-        if auth_token != dify_config.ADMIN_API_KEY:
+        if dify_config.ADMIN_API_KEY != auth_token:
             raise Unauthorized("API key is invalid.")
 
         return view(*args, **kwargs)

api/controllers/console/app/app.py

@@ -190,7 +190,7 @@ class AppCopyApi(Resource):
             )
             session.commit()
 
-            stmt = select(App).where(App.id == result.app_id)
+            stmt = select(App).where(App.id == result.app.id)
             app = session.scalar(stmt)
 
         return app, 201

api/controllers/console/app/conversation.py

@@ -1,4 +1,4 @@
-from datetime import UTC, datetime
+from datetime import datetime, timezone
 
 import pytz
 from flask_login import current_user
@@ -314,7 +314,7 @@ def _get_conversation(app_model, conversation_id):
         raise NotFound("Conversation Not Exists.")
 
     if not conversation.read_at:
-        conversation.read_at = datetime.now(UTC).replace(tzinfo=None)
+        conversation.read_at = datetime.now(timezone.utc).replace(tzinfo=None)
         conversation.read_account_id = current_user.id
         db.session.commit()
 

api/controllers/console/app/model_config.py

@@ -65,7 +65,7 @@ class ModelConfigResource(Resource):
                         provider_type=agent_tool_entity.provider_type,
                         identity_id=f"AGENT.{app_model.id}",
                     )
-                except Exception:
+                except Exception as e:
                     continue
 
                 # get decrypted parameters
@@ -97,7 +97,7 @@ class ModelConfigResource(Resource):
                             app_id=app_model.id,
                             agent_tool=agent_tool_entity,
                         )
-                    except Exception:
+                    except Exception as e:
                         continue
 
                 manager = ToolParameterConfigurationManager(

api/controllers/console/app/ops_trace.py

@@ -1,5 +1,4 @@
 from flask_restful import Resource, reqparse
-from werkzeug.exceptions import BadRequest
 
 from controllers.console import api
 from controllers.console.app.error import TracingConfigCheckError, TracingConfigIsExist, TracingConfigNotExist
@@ -27,7 +26,7 @@ class TraceAppConfigApi(Resource):
                 return {"has_not_configured": True}
             return trace_config
         except Exception as e:
-            raise BadRequest(str(e))
+            raise e
 
     @setup_required
     @login_required
@@ -49,7 +48,7 @@ class TraceAppConfigApi(Resource):
                 raise TracingConfigCheckError()
             return result
         except Exception as e:
-            raise BadRequest(str(e))
+            raise e
 
     @setup_required
     @login_required
@@ -69,7 +68,7 @@ class TraceAppConfigApi(Resource):
                 raise TracingConfigNotExist()
             return {"result": "success"}
         except Exception as e:
-            raise BadRequest(str(e))
+            raise e
 
     @setup_required
     @login_required
@@ -86,7 +85,7 @@ class TraceAppConfigApi(Resource):
                 raise TracingConfigNotExist()
             return {"result": "success"}
         except Exception as e:
-            raise BadRequest(str(e))
+            raise e
 
 
 api.add_resource(TraceAppConfigApi, "/apps/<uuid:app_id>/trace-config")

api/controllers/console/app/site.py

@@ -1,4 +1,4 @@
-from datetime import UTC, datetime
+from datetime import datetime, timezone
 
 from flask_login import current_user
 from flask_restful import Resource, marshal_with, reqparse
@@ -75,7 +75,7 @@ class AppSite(Resource):
                 setattr(site, attr_name, value)
 
         site.updated_by = current_user.id
-        site.updated_at = datetime.now(UTC).replace(tzinfo=None)
+        site.updated_at = datetime.now(timezone.utc).replace(tzinfo=None)
         db.session.commit()
 
         return site
@@ -99,7 +99,7 @@ class AppSiteAccessTokenReset(Resource):
 
         site.code = Site.generate_code(16)
         site.updated_by = current_user.id
-        site.updated_at = datetime.now(UTC).replace(tzinfo=None)
+        site.updated_at = datetime.now(timezone.utc).replace(tzinfo=None)
         db.session.commit()
 
         return site

api/controllers/console/app/workflow.py

@@ -100,11 +100,11 @@ class DraftWorkflowApi(Resource):
         try:
             environment_variables_list = args.get("environment_variables") or []
             environment_variables = [
-                variable_factory.build_environment_variable_from_mapping(obj) for obj in environment_variables_list
+                variable_factory.build_variable_from_mapping(obj) for obj in environment_variables_list
             ]
             conversation_variables_list = args.get("conversation_variables") or []
             conversation_variables = [
-                variable_factory.build_conversation_variable_from_mapping(obj) for obj in conversation_variables_list
+                variable_factory.build_variable_from_mapping(obj) for obj in conversation_variables_list
             ]
             workflow = workflow_service.sync_draft_workflow(
                 app_model=app_model,
@@ -382,7 +382,7 @@ class DefaultBlockConfigApi(Resource):
         filters = None
         if args.get("q"):
             try:
-                filters = json.loads(args.get("q", ""))
+                filters = json.loads(args.get("q"))
             except json.JSONDecodeError:
                 raise ValueError("Invalid filters")
 

api/controllers/console/auth/activate.py

@@ -65,7 +65,7 @@ class ActivateApi(Resource):
         account.timezone = args["timezone"]
         account.interface_theme = "light"
         account.status = AccountStatus.ACTIVE.value
-        account.initialized_at = datetime.datetime.now(datetime.UTC).replace(tzinfo=None)
+        account.initialized_at = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None)
         db.session.commit()
 
         token_pair = AccountService.login(account, ip_address=extract_remote_ip(request))

api/controllers/console/auth/data_source_oauth.py

@@ -34,6 +34,7 @@ class OAuthDataSource(Resource):
         OAUTH_DATASOURCE_PROVIDERS = get_oauth_providers()
         with current_app.app_context():
             oauth_provider = OAUTH_DATASOURCE_PROVIDERS.get(provider)
+            print(vars(oauth_provider))
         if not oauth_provider:
             return {"error": "Invalid provider"}, 400
         if dify_config.NOTION_INTEGRATION_TYPE == "internal":

api/controllers/console/auth/oauth.py

@@ -1,5 +1,5 @@
 import logging
-from datetime import UTC, datetime
+from datetime import datetime, timezone
 from typing import Optional
 
 import requests
@@ -52,6 +52,7 @@ class OAuthLogin(Resource):
         OAUTH_PROVIDERS = get_oauth_providers()
         with current_app.app_context():
             oauth_provider = OAUTH_PROVIDERS.get(provider)
+            print(vars(oauth_provider))
         if not oauth_provider:
             return {"error": "Invalid provider"}, 400
 
@@ -105,7 +106,7 @@ class OAuthCallback(Resource):
 
         if account.status == AccountStatus.PENDING.value:
             account.status = AccountStatus.ACTIVE.value
-            account.initialized_at = datetime.now(UTC).replace(tzinfo=None)
+            account.initialized_at = datetime.now(timezone.utc).replace(tzinfo=None)
             db.session.commit()
 
         try:

api/controllers/console/datasets/data_source.py

@@ -83,7 +83,7 @@ class DataSourceApi(Resource):
         if action == "enable":
             if data_source_binding.disabled:
                 data_source_binding.disabled = False
-                data_source_binding.updated_at = datetime.datetime.now(datetime.UTC).replace(tzinfo=None)
+                data_source_binding.updated_at = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None)
                 db.session.add(data_source_binding)
                 db.session.commit()
             else:
@@ -92,7 +92,7 @@ class DataSourceApi(Resource):
         if action == "disable":
             if not data_source_binding.disabled:
                 data_source_binding.disabled = True
-                data_source_binding.updated_at = datetime.datetime.now(datetime.UTC).replace(tzinfo=None)
+                data_source_binding.updated_at = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None)
                 db.session.add(data_source_binding)
                 db.session.commit()
             else:

api/controllers/console/datasets/datasets_document.py

@@ -1,6 +1,6 @@
 import logging
 from argparse import ArgumentTypeError
-from datetime import UTC, datetime
+from datetime import datetime, timezone
 
 from flask import request
 from flask_login import current_user
@@ -106,7 +106,6 @@ class GetProcessRuleApi(Resource):
         # get default rules
         mode = DocumentService.DEFAULT_RULES["mode"]
         rules = DocumentService.DEFAULT_RULES["rules"]
-        limits = DocumentService.DEFAULT_RULES["limits"]
         if document_id:
             # get the latest process rule
             document = Document.query.get_or_404(document_id)
@@ -133,7 +132,7 @@ class GetProcessRuleApi(Resource):
                 mode = dataset_process_rule.mode
                 rules = dataset_process_rule.rules_dict
 
-        return {"mode": mode, "rules": rules, "limits": limits}
+        return {"mode": mode, "rules": rules}
 
 
 class DatasetDocumentListApi(Resource):
@@ -666,7 +665,7 @@ class DocumentProcessingApi(DocumentResource):
                 raise InvalidActionError("Document not in indexing state.")
 
             document.paused_by = current_user.id
-            document.paused_at = datetime.now(UTC).replace(tzinfo=None)
+            document.paused_at = datetime.now(timezone.utc).replace(tzinfo=None)
             document.is_paused = True
             db.session.commit()
 
@@ -746,7 +745,7 @@ class DocumentMetadataApi(DocumentResource):
                     document.doc_metadata[key] = value
 
         document.doc_type = doc_type
-        document.updated_at = datetime.now(UTC).replace(tzinfo=None)
+        document.updated_at = datetime.now(timezone.utc).replace(tzinfo=None)
         db.session.commit()
 
         return {"result": "success", "message": "Document metadata updated."}, 200
@@ -788,7 +787,7 @@ class DocumentStatusApi(DocumentResource):
             document.enabled = True
             document.disabled_at = None
             document.disabled_by = None
-            document.updated_at = datetime.now(UTC).replace(tzinfo=None)
+            document.updated_at = datetime.now(timezone.utc).replace(tzinfo=None)
             db.session.commit()
 
             # Set cache to prevent indexing the same document multiple times
@@ -805,9 +804,9 @@ class DocumentStatusApi(DocumentResource):
                 raise InvalidActionError("Document already disabled.")
 
             document.enabled = False
-            document.disabled_at = datetime.now(UTC).replace(tzinfo=None)
+            document.disabled_at = datetime.now(timezone.utc).replace(tzinfo=None)
             document.disabled_by = current_user.id
-            document.updated_at = datetime.now(UTC).replace(tzinfo=None)
+            document.updated_at = datetime.now(timezone.utc).replace(tzinfo=None)
             db.session.commit()
 
             # Set cache to prevent indexing the same document multiple times
@@ -822,9 +821,9 @@ class DocumentStatusApi(DocumentResource):
                 raise InvalidActionError("Document already archived.")
 
             document.archived = True
-            document.archived_at = datetime.now(UTC).replace(tzinfo=None)
+            document.archived_at = datetime.now(timezone.utc).replace(tzinfo=None)
             document.archived_by = current_user.id
-            document.updated_at = datetime.now(UTC).replace(tzinfo=None)
+            document.updated_at = datetime.now(timezone.utc).replace(tzinfo=None)
             db.session.commit()
 
             if document.enabled:
@@ -841,7 +840,7 @@ class DocumentStatusApi(DocumentResource):
             document.archived = False
             document.archived_at = None
             document.archived_by = None
-            document.updated_at = datetime.now(UTC).replace(tzinfo=None)
+            document.updated_at = datetime.now(timezone.utc).replace(tzinfo=None)
             db.session.commit()
 
             # Set cache to prevent indexing the same document multiple times
@@ -948,7 +947,7 @@ class DocumentRetryApi(DocumentResource):
                 if document.indexing_status == "completed":
                     raise DocumentAlreadyFinishedError()
                 retry_documents.append(document)
-            except Exception:
+            except Exception as e:
                 logging.exception(f"Failed to retry document, document id: {document_id}")
                 continue
         # retry document

api/controllers/console/datasets/datasets_segments.py

@@ -1,5 +1,5 @@
 import uuid
-from datetime import UTC, datetime
+from datetime import datetime, timezone
 
 import pandas as pd
 from flask import request
@@ -188,7 +188,7 @@ class DatasetDocumentSegmentApi(Resource):
                 raise InvalidActionError("Segment is already disabled.")
 
             segment.enabled = False
-            segment.disabled_at = datetime.now(UTC).replace(tzinfo=None)
+            segment.disabled_at = datetime.now(timezone.utc).replace(tzinfo=None)
             segment.disabled_by = current_user.id
             db.session.commit()
 

api/controllers/console/datasets/fta_test.py

@@ -0,0 +1,145 @@
+import json
+
+import requests
+from flask import Response
+from flask_restful import Resource, reqparse
+from sqlalchemy import text
+
+from controllers.console import api
+from extensions.ext_database import db
+from extensions.ext_storage import storage
+from models.fta import ComponentFailure, ComponentFailureStats
+
+
+class FATTestApi(Resource):
+    def post(self):
+        parser = reqparse.RequestParser()
+        parser.add_argument("log_process_data", nullable=False, required=True, type=str, location="args")
+        args = parser.parse_args()
+        print(args["log_process_data"])
+        # Extract the JSON string from the text field
+        json_str = args["log_process_data"].strip("```json\\n").strip("```").strip().replace("\\n", "")
+        log_data = json.loads(json_str)
+        db.session.query(ComponentFailure).delete()
+        for data in log_data:
+            if not isinstance(data, dict):
+                raise TypeError("Data must be a dictionary.")
+
+            required_keys = {"Date", "Component", "FailureMode", "Cause", "RepairAction", "Technician"}
+            if not required_keys.issubset(data.keys()):
+                raise ValueError(f"Data dictionary must contain the following keys: {required_keys}")
+
+            try:
+                # Clear existing stats
+                component_failure = ComponentFailure(
+                    Date=data["Date"],
+                    Component=data["Component"],
+                    FailureMode=data["FailureMode"],
+                    Cause=data["Cause"],
+                    RepairAction=data["RepairAction"],
+                    Technician=data["Technician"],
+                )
+                db.session.add(component_failure)
+                db.session.commit()
+            except Exception as e:
+                print(e)
+        # Clear existing stats
+        db.session.query(ComponentFailureStats).delete()
+
+        # Insert calculated statistics
+        try:
+            db.session.execute(
+                text("""
+                    INSERT INTO component_failure_stats ("Component", "FailureMode", "Cause", "PossibleAction", "Probability", "MTBF")
+                    SELECT 
+                        cf."Component",
+                        cf."FailureMode",
+                        cf."Cause",
+                        cf."RepairAction" as "PossibleAction",
+                        COUNT(*) * 1.0 / (SELECT COUNT(*) FROM component_failure WHERE "Component" = cf."Component") AS "Probability",
+                        COALESCE(AVG(EXTRACT(EPOCH FROM (next_failure_date::timestamp - cf."Date"::timestamp)) / 86400.0),0)AS "MTBF"
+                    FROM (
+                        SELECT 
+                            "Component",
+                            "FailureMode",
+                            "Cause",
+                            "RepairAction",
+                            "Date", 
+                            LEAD("Date") OVER (PARTITION BY "Component", "FailureMode", "Cause" ORDER BY "Date") AS next_failure_date
+                        FROM 
+                            component_failure
+                    ) cf
+                    GROUP BY 
+                        cf."Component", cf."FailureMode", cf."Cause", cf."RepairAction";
+            """)
+            )
+            db.session.commit()
+        except Exception as e:
+            db.session.rollback()
+            print(f"Error during stats calculation: {e}")
+        # output format
+        # [
+        #     (17, 'Hydraulic system', 'Leak', 'Hose rupture', 'Replaced hydraulic hose', 0.3333333333333333, None),
+        #     (18, 'Hydraulic system', 'Leak', 'Seal Wear', 'Replaced the faulty seal', 0.3333333333333333, None),
+        #     (19, 'Hydraulic system', 'Pressure drop', 'Fluid leak', 'Replaced hydraulic fluid and seals', 0.3333333333333333, None)
+        # ]
+
+        component_failure_stats = db.session.query(ComponentFailureStats).all()
+        # Convert stats to list of tuples format
+        stats_list = []
+        for stat in component_failure_stats:
+            stats_list.append(
+                (
+                    stat.StatID,
+                    stat.Component,
+                    stat.FailureMode,
+                    stat.Cause,
+                    stat.PossibleAction,
+                    stat.Probability,
+                    stat.MTBF,
+                )
+            )
+        return {"data": stats_list}, 200
+
+
+# generate-fault-tree
+class GenerateFaultTreeApi(Resource):
+    def post(self):
+        parser = reqparse.RequestParser()
+        parser.add_argument("llm_text", nullable=False, required=True, type=str, location="args")
+        args = parser.parse_args()
+        entities = args["llm_text"].replace("```", "").replace("\\n", "\n")
+        print(entities)
+        request_data = {"fault_tree_text": entities}
+        url = "https://fta.cognitech-dev.live/generate-fault-tree"
+        headers = {"accept": "application/json", "Content-Type": "application/json"}
+
+        response = requests.post(url, json=request_data, headers=headers)
+        print(response.json())
+        return {"data": response.json()}, 200
+
+
+class ExtractSVGApi(Resource):
+    def post(self):
+        parser = reqparse.RequestParser()
+        parser.add_argument("svg_text", nullable=False, required=True, type=str, location="args")
+        args = parser.parse_args()
+        # svg_text = ''.join(args["svg_text"].splitlines())
+        svg_text = args["svg_text"].replace("\n", "")
+        svg_text = svg_text.replace('"', '"')
+        print(svg_text)
+        svg_text_json = json.loads(svg_text)
+        svg_content = svg_text_json.get("data").get("svg_content")[0]
+        svg_content = svg_content.replace("\n", "").replace('"', '"')
+        file_key = "fta_svg/" + "fat.svg"
+        if storage.exists(file_key):
+            storage.delete(file_key)
+        storage.save(file_key, svg_content.encode("utf-8"))
+        generator = storage.load(file_key, stream=True)
+
+        return Response(generator, mimetype="image/svg+xml")
+
+
+api.add_resource(FATTestApi, "/fta/db-handler")
+api.add_resource(GenerateFaultTreeApi, "/fta/generate-fault-tree")
+api.add_resource(ExtractSVGApi, "/fta/extract-svg")

api/controllers/console/explore/completion.py

@@ -1,5 +1,5 @@
 import logging
-from datetime import UTC, datetime
+from datetime import datetime, timezone
 
 from flask_login import current_user
 from flask_restful import reqparse
@@ -46,7 +46,7 @@ class CompletionApi(InstalledAppResource):
         streaming = args["response_mode"] == "streaming"
         args["auto_generate_name"] = False
 
-        installed_app.last_used_at = datetime.now(UTC).replace(tzinfo=None)
+        installed_app.last_used_at = datetime.now(timezone.utc).replace(tzinfo=None)
         db.session.commit()
 
         try:
@@ -106,7 +106,7 @@ class ChatApi(InstalledAppResource):
 
         args["auto_generate_name"] = False
 
-        installed_app.last_used_at = datetime.now(UTC).replace(tzinfo=None)
+        installed_app.last_used_at = datetime.now(timezone.utc).replace(tzinfo=None)
         db.session.commit()
 
         try:

api/controllers/console/explore/installed_app.py

@@ -1,6 +1,5 @@
-from datetime import UTC, datetime
+from datetime import datetime, timezone
 
-from flask import request
 from flask_login import current_user
 from flask_restful import Resource, inputs, marshal_with, reqparse
 from sqlalchemy import and_
@@ -21,17 +20,8 @@ class InstalledAppsListApi(Resource):
     @account_initialization_required
     @marshal_with(installed_app_list_fields)
     def get(self):
-        app_id = request.args.get("app_id", default=None, type=str)
         current_tenant_id = current_user.current_tenant_id
-
-        if app_id:
-            installed_apps = (
-                db.session.query(InstalledApp)
-                .filter(and_(InstalledApp.tenant_id == current_tenant_id, InstalledApp.app_id == app_id))
-                .all()
-            )
-        else:
-            installed_apps = db.session.query(InstalledApp).filter(InstalledApp.tenant_id == current_tenant_id).all()
+        installed_apps = db.session.query(InstalledApp).filter(InstalledApp.tenant_id == current_tenant_id).all()
 
         current_user.role = TenantService.get_user_role(current_user, current_user.current_tenant)
         installed_apps = [
@@ -91,7 +81,7 @@ class InstalledAppsListApi(Resource):
                 tenant_id=current_tenant_id,
                 app_owner_tenant_id=app.tenant_id,
                 is_pinned=False,
-                last_used_at=datetime.now(UTC).replace(tzinfo=None),
+                last_used_at=datetime.now(timezone.utc).replace(tzinfo=None),
             )
             db.session.add(new_installed_app)
             db.session.commit()

api/controllers/console/explore/recommended_app.py

@@ -4,7 +4,6 @@ from flask_restful import Resource, fields, marshal_with, reqparse
 from constants.languages import languages
 from controllers.console import api
 from controllers.console.wraps import account_initialization_required
-from libs.helper import AppIconUrlField
 from libs.login import login_required
 from services.recommended_app_service import RecommendedAppService
 
@@ -13,8 +12,6 @@ app_fields = {
     "name": fields.String,
     "mode": fields.String,
     "icon": fields.String,
-    "icon_type": fields.String,
-    "icon_url": AppIconUrlField,
     "icon_background": fields.String,
 }
 

api/controllers/console/files.py

@@ -1,7 +1,6 @@
 from flask import request
 from flask_login import current_user
 from flask_restful import Resource, marshal_with
-from werkzeug.exceptions import Forbidden
 
 import services
 from configs import dify_config
@@ -59,9 +58,6 @@ class FileApi(Resource):
         if not file.filename:
             raise FilenameNotExistsError
 
-        if source == "datasets" and not current_user.is_dataset_editor:
-            raise Forbidden()
-
         if source not in ("datasets", None):
             source = None
 

api/controllers/console/workspace/account.py

@@ -60,7 +60,7 @@ class AccountInitApi(Resource):
                 raise InvalidInvitationCodeError()
 
             invitation_code.status = "used"
-            invitation_code.used_at = datetime.datetime.now(datetime.UTC).replace(tzinfo=None)
+            invitation_code.used_at = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None)
             invitation_code.used_by_tenant_id = account.current_tenant_id
             invitation_code.used_by_account_id = account.id
 
@@ -68,7 +68,7 @@ class AccountInitApi(Resource):
         account.timezone = args["timezone"]
         account.interface_theme = "light"
         account.status = "active"
-        account.initialized_at = datetime.datetime.now(datetime.UTC).replace(tzinfo=None)
+        account.initialized_at = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None)
         db.session.commit()
 
         return {"result": "success"}

api/controllers/console/workspace/tool_providers.py

@@ -368,7 +368,6 @@ class ToolWorkflowProviderCreateApi(Resource):
             description=args["description"],
             parameters=args["parameters"],
             privacy_policy=args["privacy_policy"],
-            labels=args["labels"],
         )
 
 

api/controllers/service_api/app/app.py

@@ -48,8 +48,7 @@ class AppInfoApi(Resource):
     @validate_app_token
     def get(self, app_model: App):
         """Get app information"""
-        tags = [tag.name for tag in app_model.tags]
-        return {"name": app_model.name, "description": app_model.description, "tags": tags}
+        return {"name": app_model.name, "description": app_model.description}
 
 
 api.add_resource(AppParameterApi, "/parameters")

api/controllers/service_api/wraps.py

@@ -1,5 +1,5 @@
 from collections.abc import Callable
-from datetime import UTC, datetime
+from datetime import datetime, timezone
 from enum import Enum
 from functools import wraps
 from typing import Optional
@@ -198,7 +198,7 @@ def validate_and_get_api_token(scope=None):
     if not api_token:
         raise Unauthorized("Access token is invalid")
 
-    api_token.last_used_at = datetime.now(UTC).replace(tzinfo=None)
+    api_token.last_used_at = datetime.now(timezone.utc).replace(tzinfo=None)
     db.session.commit()
 
     return api_token

api/core/agent/base_agent_runner.py

@@ -2,7 +2,7 @@ import json
 import logging
 import uuid
 from collections.abc import Mapping, Sequence
-from datetime import UTC, datetime
+from datetime import datetime, timezone
 from typing import Optional, Union, cast
 
 from core.agent.entities import AgentEntity, AgentToolEntity
@@ -412,7 +412,7 @@ class BaseAgentRunner(AppRunner):
             .first()
         )
 
-        db_variables.updated_at = datetime.now(UTC).replace(tzinfo=None)
+        db_variables.updated_at = datetime.now(timezone.utc).replace(tzinfo=None)
         db_variables.variables_str = json.dumps(jsonable_encoder(tool_variables.pool))
         db.session.commit()
         db.session.close()

api/core/app/app_config/easy_ui_based_app/dataset/manager.py

@@ -1,4 +1,3 @@
-import uuid
 from typing import Optional
 
 from core.app.app_config.entities import DatasetEntity, DatasetRetrieveConfigEntity

api/core/app/app_config/easy_ui_based_app/model_config/manager.py

@@ -1,6 +1,3 @@
-from collections.abc import Mapping
-from typing import Any
-
 from core.app.app_config.entities import ModelConfigEntity
 from core.model_runtime.entities.model_entities import ModelPropertyKey, ModelType
 from core.model_runtime.model_providers import model_provider_factory
@@ -39,7 +36,7 @@ class ModelConfigManager:
         )
 
     @classmethod
-    def validate_and_set_defaults(cls, tenant_id: str, config: Mapping[str, Any]) -> tuple[dict, list[str]]:
+    def validate_and_set_defaults(cls, tenant_id: str, config: dict) -> tuple[dict, list[str]]:
         """
         Validate and set defaults for model config
 

api/core/app/app_config/easy_ui_based_app/prompt_template/manager.py

@@ -1,5 +1,4 @@
 from core.app.app_config.entities import (
-    AdvancedChatMessageEntity,
     AdvancedChatPromptTemplateEntity,
     AdvancedCompletionPromptTemplateEntity,
     PromptTemplateEntity,
@@ -26,9 +25,7 @@ class PromptTemplateConfigManager:
                 chat_prompt_messages = []
                 for message in chat_prompt_config.get("prompt", []):
                     chat_prompt_messages.append(
-                        AdvancedChatMessageEntity(
-                            **{"text": message["text"], "role": PromptMessageRole.value_of(message["role"])}
-                        )
+                        {"text": message["text"], "role": PromptMessageRole.value_of(message["role"])}
                     )
 
                 advanced_chat_prompt_template = AdvancedChatPromptTemplateEntity(messages=chat_prompt_messages)

api/core/app/app_config/entities.py

@@ -1,5 +1,5 @@
 from collections.abc import Sequence
-from enum import Enum, StrEnum
+from enum import Enum
 from typing import Any, Optional
 
 from pydantic import BaseModel, Field, field_validator
@@ -88,7 +88,7 @@ class PromptTemplateEntity(BaseModel):
     advanced_completion_prompt_template: Optional[AdvancedCompletionPromptTemplateEntity] = None
 
 
-class VariableEntityType(StrEnum):
+class VariableEntityType(str, Enum):
     TEXT_INPUT = "text-input"
     SELECT = "select"
     PARAGRAPH = "paragraph"

api/core/app/apps/README.md

@@ -2,7 +2,7 @@
 
 Due to the presence of tasks in App Runner that require long execution times, such as LLM generation and external requests, Flask-Sqlalchemy's strategy for database connection pooling is to allocate one connection (transaction) per request. This approach keeps a connection occupied even during non-DB tasks, leading to the inability to acquire new connections during high concurrency requests due to multiple long-running tasks.
 
-Therefore, the database operations in App Runner and Task Pipeline must ensure connections are closed immediately after use, and it's better to pass IDs rather than Model objects to avoid detach errors.
+Therefore, the database operations in App Runner and Task Pipeline must ensure connections are closed immediately after use, and it's better to pass IDs rather than Model objects to avoid deattach errors.
 
 Examples:
 

api/core/app/apps/advanced_chat/app_generator.py

@@ -2,7 +2,7 @@ import contextvars
 import logging
 import threading
 import uuid
-from collections.abc import Generator, Mapping
+from collections.abc import Generator
 from typing import Any, Literal, Optional, Union, overload
 
 from flask import Flask, current_app
@@ -23,7 +23,6 @@ from core.app.entities.app_invoke_entities import AdvancedChatAppGenerateEntity,
 from core.app.entities.task_entities import ChatbotAppBlockingResponse, ChatbotAppStreamResponse
 from core.model_runtime.errors.invoke import InvokeAuthorizationError, InvokeError
 from core.ops.ops_trace_manager import TraceQueueManager
-from core.prompt.utils.get_thread_messages_length import get_thread_messages_length
 from extensions.ext_database import db
 from factories import file_factory
 from models.account import Account
@@ -34,17 +33,15 @@ logger = logging.getLogger(__name__)
 
 
 class AdvancedChatAppGenerator(MessageBasedAppGenerator):
-    _dialogue_count: int
-
     @overload
     def generate(
         self,
         app_model: App,
         workflow: Workflow,
         user: Union[Account, EndUser],
-        args: Mapping[str, Any],
+        args: dict,
         invoke_from: InvokeFrom,
-        streaming: Literal[True],
+        stream: Literal[True] = True,
     ) -> Generator[str, None, None]: ...
 
     @overload
@@ -53,31 +50,20 @@ class AdvancedChatAppGenerator(MessageBasedAppGenerator):
         app_model: App,
         workflow: Workflow,
         user: Union[Account, EndUser],
-        args: Mapping[str, Any],
+        args: dict,
         invoke_from: InvokeFrom,
-        streaming: Literal[False],
-    ) -> Mapping[str, Any]: ...
-
-    @overload
-    def generate(
-        self,
-        app_model: App,
-        workflow: Workflow,
-        user: Union[Account, EndUser],
-        args: Mapping[str, Any],
-        invoke_from: InvokeFrom,
-        streaming: bool = True,
-    ) -> Union[Mapping[str, Any], Generator[str, None, None]]: ...
+        stream: Literal[False] = False,
+    ) -> dict: ...
 
     def generate(
         self,
         app_model: App,
         workflow: Workflow,
         user: Union[Account, EndUser],
-        args: Mapping[str, Any],
+        args: dict,
         invoke_from: InvokeFrom,
-        streaming: bool = True,
-    ):
+        stream: bool = True,
+    ) -> dict[str, Any] | Generator[str, Any, None]:
         """
         Generate App response.
 
@@ -141,14 +127,12 @@ class AdvancedChatAppGenerator(MessageBasedAppGenerator):
             conversation_id=conversation.id if conversation else None,
             inputs=conversation.inputs
             if conversation
-            else self._prepare_user_inputs(
-                user_inputs=inputs, variables=app_config.variables, tenant_id=app_model.tenant_id
-            ),
+            else self._prepare_user_inputs(user_inputs=inputs, app_config=app_config),
             query=query,
             files=file_objs,
             parent_message_id=args.get("parent_message_id") if invoke_from != InvokeFrom.SERVICE_API else UUID_NIL,
             user_id=user.id,
-            stream=streaming,
+            stream=stream,
             invoke_from=invoke_from,
             extras=extras,
             trace_manager=trace_manager,
@@ -162,12 +146,12 @@ class AdvancedChatAppGenerator(MessageBasedAppGenerator):
             invoke_from=invoke_from,
             application_generate_entity=application_generate_entity,
             conversation=conversation,
-            stream=streaming,
+            stream=stream,
         )
 
     def single_iteration_generate(
-        self, app_model: App, workflow: Workflow, node_id: str, user: Account, args: dict, streaming: bool = True
-    ) -> Mapping[str, Any] | Generator[str, None, None]:
+        self, app_model: App, workflow: Workflow, node_id: str, user: Account, args: dict, stream: bool = True
+    ) -> dict[str, Any] | Generator[str, Any, None]:
         """
         Generate App response.
 
@@ -196,7 +180,7 @@ class AdvancedChatAppGenerator(MessageBasedAppGenerator):
             query="",
             files=[],
             user_id=user.id,
-            stream=streaming,
+            stream=stream,
             invoke_from=InvokeFrom.DEBUGGER,
             extras={"auto_generate_conversation_name": False},
             single_iteration_run=AdvancedChatAppGenerateEntity.SingleIterationRunEntity(
@@ -211,7 +195,7 @@ class AdvancedChatAppGenerator(MessageBasedAppGenerator):
             invoke_from=InvokeFrom.DEBUGGER,
             application_generate_entity=application_generate_entity,
             conversation=None,
-            stream=streaming,
+            stream=stream,
         )
 
     def _generate(
@@ -223,7 +207,7 @@ class AdvancedChatAppGenerator(MessageBasedAppGenerator):
         application_generate_entity: AdvancedChatAppGenerateEntity,
         conversation: Optional[Conversation] = None,
         stream: bool = True,
-    ) -> Mapping[str, Any] | Generator[str, None, None]:
+    ) -> dict[str, Any] | Generator[str, Any, None]:
         """
         Generate App response.
 
@@ -247,9 +231,6 @@ class AdvancedChatAppGenerator(MessageBasedAppGenerator):
             db.session.commit()
             db.session.refresh(conversation)
 
-        # get conversation dialogue count
-        self._dialogue_count = get_thread_messages_length(conversation.id)
-
         # init queue manager
         queue_manager = MessageBasedAppQueueManager(
             task_id=application_generate_entity.task_id,
@@ -320,7 +301,6 @@ class AdvancedChatAppGenerator(MessageBasedAppGenerator):
                     queue_manager=queue_manager,
                     conversation=conversation,
                     message=message,
-                    dialogue_count=self._dialogue_count,
                 )
 
                 runner.run()
@@ -374,7 +354,6 @@ class AdvancedChatAppGenerator(MessageBasedAppGenerator):
             message=message,
             user=user,
             stream=stream,
-            dialogue_count=self._dialogue_count,
         )
 
         try:

api/core/app/apps/advanced_chat/app_runner.py

@@ -39,14 +39,12 @@ class AdvancedChatAppRunner(WorkflowBasedAppRunner):
         queue_manager: AppQueueManager,
         conversation: Conversation,
         message: Message,
-        dialogue_count: int,
     ) -> None:
         super().__init__(queue_manager)
 
         self.application_generate_entity = application_generate_entity
         self.conversation = conversation
         self.message = message
-        self._dialogue_count = dialogue_count
 
     def run(self) -> None:
         app_config = self.application_generate_entity.app_config
@@ -124,13 +122,19 @@ class AdvancedChatAppRunner(WorkflowBasedAppRunner):
 
                 session.commit()
 
+            # Increment dialogue count.
+            self.conversation.dialogue_count += 1
+
+            conversation_dialogue_count = self.conversation.dialogue_count
+            db.session.commit()
+
             # Create a variable pool.
             system_inputs = {
                 SystemVariableKey.QUERY: query,
                 SystemVariableKey.FILES: files,
                 SystemVariableKey.CONVERSATION_ID: self.conversation.id,
                 SystemVariableKey.USER_ID: user_id,
-                SystemVariableKey.DIALOGUE_COUNT: self._dialogue_count,
+                SystemVariableKey.DIALOGUE_COUNT: conversation_dialogue_count,
                 SystemVariableKey.APP_ID: app_config.app_id,
                 SystemVariableKey.WORKFLOW_ID: app_config.workflow_id,
                 SystemVariableKey.WORKFLOW_RUN_ID: self.application_generate_entity.workflow_run_id,

api/core/app/apps/advanced_chat/generate_task_pipeline.py

@@ -19,10 +19,8 @@ from core.app.entities.queue_entities import (
     QueueIterationNextEvent,
     QueueIterationStartEvent,
     QueueMessageReplaceEvent,
-    QueueNodeExceptionEvent,
     QueueNodeFailedEvent,
     QueueNodeInIterationFailedEvent,
-    QueueNodeRetryEvent,
     QueueNodeStartedEvent,
     QueueNodeSucceededEvent,
     QueueParallelBranchRunFailedEvent,
@@ -33,7 +31,6 @@ from core.app.entities.queue_entities import (
     QueueStopEvent,
     QueueTextChunkEvent,
     QueueWorkflowFailedEvent,
-    QueueWorkflowPartialSuccessEvent,
     QueueWorkflowStartedEvent,
     QueueWorkflowSucceededEvent,
 )
@@ -91,7 +88,6 @@ class AdvancedChatAppGenerateTaskPipeline(BasedGenerateTaskPipeline, WorkflowCyc
         message: Message,
         user: Union[Account, EndUser],
         stream: bool,
-        dialogue_count: int,
     ) -> None:
         """
         Initialize AdvancedChatAppGenerateTaskPipeline.
@@ -102,7 +98,6 @@ class AdvancedChatAppGenerateTaskPipeline(BasedGenerateTaskPipeline, WorkflowCyc
         :param message: message
         :param user: user
         :param stream: stream
-        :param dialogue_count: dialogue count
         """
         super().__init__(application_generate_entity, queue_manager, user, stream)
 
@@ -119,7 +114,7 @@ class AdvancedChatAppGenerateTaskPipeline(BasedGenerateTaskPipeline, WorkflowCyc
             SystemVariableKey.FILES: application_generate_entity.files,
             SystemVariableKey.CONVERSATION_ID: conversation.id,
             SystemVariableKey.USER_ID: user_id,
-            SystemVariableKey.DIALOGUE_COUNT: dialogue_count,
+            SystemVariableKey.DIALOGUE_COUNT: conversation.dialogue_count,
             SystemVariableKey.APP_ID: application_generate_entity.app_config.app_id,
             SystemVariableKey.WORKFLOW_ID: workflow.id,
             SystemVariableKey.WORKFLOW_RUN_ID: application_generate_entity.workflow_run_id,
@@ -320,7 +315,7 @@ class AdvancedChatAppGenerateTaskPipeline(BasedGenerateTaskPipeline, WorkflowCyc
 
                 if response:
                     yield response
-            elif isinstance(event, QueueNodeFailedEvent | QueueNodeInIterationFailedEvent | QueueNodeExceptionEvent):
+            elif isinstance(event, QueueNodeFailedEvent | QueueNodeInIterationFailedEvent):
                 workflow_node_execution = self._handle_workflow_node_execution_failed(event)
 
                 response = self._workflow_node_finish_to_stream_response(
@@ -329,22 +324,6 @@ class AdvancedChatAppGenerateTaskPipeline(BasedGenerateTaskPipeline, WorkflowCyc
                     workflow_node_execution=workflow_node_execution,
                 )
 
-                if response:
-                    yield response
-            elif isinstance(
-                event,
-                QueueNodeRetryEvent,
-            ):
-                workflow_node_execution = self._handle_workflow_node_execution_retried(
-                    workflow_run=workflow_run, event=event
-                )
-
-                response = self._workflow_node_retry_to_stream_response(
-                    event=event,
-                    task_id=self._application_generate_entity.task_id,
-                    workflow_node_execution=workflow_node_execution,
-                )
-
                 if response:
                     yield response
             elif isinstance(event, QueueParallelBranchRunStartedEvent):
@@ -403,29 +382,6 @@ class AdvancedChatAppGenerateTaskPipeline(BasedGenerateTaskPipeline, WorkflowCyc
                     task_id=self._application_generate_entity.task_id, workflow_run=workflow_run
                 )
 
-                self._queue_manager.publish(QueueAdvancedChatMessageEndEvent(), PublishFrom.TASK_PIPELINE)
-            elif isinstance(event, QueueWorkflowPartialSuccessEvent):
-                if not workflow_run:
-                    raise Exception("Workflow run not initialized.")
-
-                if not graph_runtime_state:
-                    raise Exception("Graph runtime state not initialized.")
-
-                workflow_run = self._handle_workflow_run_partial_success(
-                    workflow_run=workflow_run,
-                    start_at=graph_runtime_state.start_at,
-                    total_tokens=graph_runtime_state.total_tokens,
-                    total_steps=graph_runtime_state.node_run_steps,
-                    outputs=event.outputs,
-                    exceptions_count=event.exceptions_count,
-                    conversation_id=None,
-                    trace_manager=trace_manager,
-                )
-
-                yield self._workflow_finish_to_stream_response(
-                    task_id=self._application_generate_entity.task_id, workflow_run=workflow_run
-                )
-
                 self._queue_manager.publish(QueueAdvancedChatMessageEndEvent(), PublishFrom.TASK_PIPELINE)
             elif isinstance(event, QueueWorkflowFailedEvent):
                 if not workflow_run:
@@ -443,7 +399,6 @@ class AdvancedChatAppGenerateTaskPipeline(BasedGenerateTaskPipeline, WorkflowCyc
                     error=event.error,
                     conversation_id=self._conversation.id,
                     trace_manager=trace_manager,
-                    exceptions_count=event.exceptions_count,
                 )
 
                 yield self._workflow_finish_to_stream_response(

api/core/app/apps/agent_chat/app_config_manager.py

@@ -1,6 +1,5 @@
 import uuid
-from collections.abc import Mapping
-from typing import Any, Optional
+from typing import Optional
 
 from core.agent.entities import AgentEntity
 from core.app.app_config.base_app_config_manager import BaseAppConfigManager
@@ -86,7 +85,7 @@ class AgentChatAppConfigManager(BaseAppConfigManager):
         return app_config
 
     @classmethod
-    def config_validate(cls, tenant_id: str, config: Mapping[str, Any]) -> dict:
+    def config_validate(cls, tenant_id: str, config: dict) -> dict:
         """
         Validate for agent chat app model config
 

api/core/app/apps/agent_chat/app_generator.py

@@ -1,7 +1,7 @@
 import logging
 import threading
 import uuid
-from collections.abc import Generator, Mapping
+from collections.abc import Generator
 from typing import Any, Literal, Union, overload
 
 from flask import Flask, current_app
@@ -31,45 +31,31 @@ class AgentChatAppGenerator(MessageBasedAppGenerator):
     @overload
     def generate(
         self,
-        *,
         app_model: App,
         user: Union[Account, EndUser],
-        args: Mapping[str, Any],
+        args: dict,
         invoke_from: InvokeFrom,
-        streaming: Literal[True],
-    ) -> Generator[str, None, None]: ...
+        stream: Literal[True] = True,
+    ) -> Generator[dict, None, None]: ...
 
     @overload
     def generate(
         self,
-        *,
         app_model: App,
         user: Union[Account, EndUser],
-        args: Mapping[str, Any],
+        args: dict,
         invoke_from: InvokeFrom,
-        streaming: Literal[False],
-    ) -> Mapping[str, Any]: ...
-
-    @overload
-    def generate(
-        self,
-        *,
-        app_model: App,
-        user: Union[Account, EndUser],
-        args: Mapping[str, Any],
-        invoke_from: InvokeFrom,
-        streaming: bool,
-    ) -> Mapping[str, Any] | Generator[str, None, None]: ...
+        stream: Literal[False] = False,
+    ) -> dict: ...
 
     def generate(
         self,
-        *,
         app_model: App,
         user: Union[Account, EndUser],
-        args: Mapping[str, Any],
+        args: Any,
         invoke_from: InvokeFrom,
-        streaming: bool = True,
-    ):
+        stream: bool = True,
+    ) -> Union[dict, Generator[dict, None, None]]:
         """
         Generate App response.
 
@@ -79,7 +65,7 @@ class AgentChatAppGenerator(MessageBasedAppGenerator):
         :param invoke_from: invoke from source
         :param stream: is stream
         """
-        if not streaming:
+        if not stream:
             raise ValueError("Agent Chat App does not support blocking mode")
 
         if not args.get("query"):
@@ -110,8 +96,7 @@ class AgentChatAppGenerator(MessageBasedAppGenerator):
 
             # validate config
             override_model_config_dict = AgentChatAppConfigManager.config_validate(
-                tenant_id=app_model.tenant_id,
-                config=args["model_config"],
+                tenant_id=app_model.tenant_id, config=args.get("model_config")
             )
 
             # always enable retriever resource in debugger mode
@@ -149,14 +134,12 @@ class AgentChatAppGenerator(MessageBasedAppGenerator):
             conversation_id=conversation.id if conversation else None,
             inputs=conversation.inputs
             if conversation
-            else self._prepare_user_inputs(
-                user_inputs=inputs, variables=app_config.variables, tenant_id=app_model.tenant_id
-            ),
+            else self._prepare_user_inputs(user_inputs=inputs, app_config=app_config),
             query=query,
             files=file_objs,
             parent_message_id=args.get("parent_message_id") if invoke_from != InvokeFrom.SERVICE_API else UUID_NIL,
             user_id=user.id,
-            stream=streaming,
+            stream=stream,
             invoke_from=invoke_from,
             extras=extras,
             call_depth=0,
@@ -197,7 +180,7 @@ class AgentChatAppGenerator(MessageBasedAppGenerator):
             conversation=conversation,
             message=message,
             user=user,
-            stream=streaming,
+            stream=stream,
         )
 
         return AgentChatAppGenerateResponseConverter.convert(response=response, invoke_from=invoke_from)

api/core/app/apps/base_app_generate_response_converter.py

@@ -1,6 +1,6 @@
 import logging
 from abc import ABC, abstractmethod
-from collections.abc import Generator, Mapping
+from collections.abc import Generator
 from typing import Any, Union
 
 from core.app.entities.app_invoke_entities import InvokeFrom
@@ -14,10 +14,8 @@ class AppGenerateResponseConverter(ABC):
 
     @classmethod
     def convert(
-        cls,
-        response: Union[AppBlockingResponse, Generator[AppStreamResponse, Any, None]],
-        invoke_from: InvokeFrom,
-    ) -> Mapping[str, Any] | Generator[str, None, None]:
+        cls, response: Union[AppBlockingResponse, Generator[AppStreamResponse, Any, None]], invoke_from: InvokeFrom
+    ) -> dict[str, Any] | Generator[str, Any, None]:
         if invoke_from in {InvokeFrom.DEBUGGER, InvokeFrom.SERVICE_API}:
             if isinstance(response, AppBlockingResponse):
                 return cls.convert_blocking_full_response(response)
@@ -82,7 +80,7 @@ class AppGenerateResponseConverter(ABC):
             for resource in metadata["retriever_resources"]:
                 updated_resources.append(
                     {
-                        "segment_id": resource.get("segment_id", ""),
+                        "segment_id": resource["segment_id"],
                         "position": resource["position"],
                         "document_name": resource["document_name"],
                         "score": resource["score"],

api/core/app/apps/base_app_generator.py

@@ -1,4 +1,4 @@
-from collections.abc import Mapping, Sequence
+from collections.abc import Mapping
 from typing import TYPE_CHECKING, Any, Optional
 
 from core.app.app_config.entities import VariableEntityType
@@ -6,7 +6,7 @@ from core.file import File, FileUploadConfig
 from factories import file_factory
 
 if TYPE_CHECKING:
-    from core.app.app_config.entities import VariableEntity
+    from core.app.app_config.entities import AppConfig, VariableEntity
 
 
 class BaseAppGenerator:
@@ -14,23 +14,23 @@ class BaseAppGenerator:
         self,
         *,
         user_inputs: Optional[Mapping[str, Any]],
-        variables: Sequence["VariableEntity"],
-        tenant_id: str,
+        app_config: "AppConfig",
     ) -> Mapping[str, Any]:
         user_inputs = user_inputs or {}
         # Filter input variables from form configuration, handle required fields, default values, and option values
+        variables = app_config.variables
         user_inputs = {
             var.variable: self._validate_inputs(value=user_inputs.get(var.variable), variable_entity=var)
             for var in variables
         }
         user_inputs = {k: self._sanitize_value(v) for k, v in user_inputs.items()}
         # Convert files in inputs to File
-        entity_dictionary = {item.variable: item for item in variables}
+        entity_dictionary = {item.variable: item for item in app_config.variables}
         # Convert single file to File
         files_inputs = {
             k: file_factory.build_from_mapping(
                 mapping=v,
-                tenant_id=tenant_id,
+                tenant_id=app_config.tenant_id,
                 config=FileUploadConfig(
                     allowed_file_types=entity_dictionary[k].allowed_file_types,
                     allowed_file_extensions=entity_dictionary[k].allowed_file_extensions,
@@ -44,7 +44,7 @@ class BaseAppGenerator:
         file_list_inputs = {
             k: file_factory.build_from_mappings(
                 mappings=v,
-                tenant_id=tenant_id,
+                tenant_id=app_config.tenant_id,
                 config=FileUploadConfig(
                     allowed_file_types=entity_dictionary[k].allowed_file_types,
                     allowed_file_extensions=entity_dictionary[k].allowed_file_extensions,

api/core/app/apps/chat/app_generator.py

@@ -1,7 +1,7 @@
 import logging
 import threading
 import uuid
-from collections.abc import Generator, Mapping
+from collections.abc import Generator
 from typing import Any, Literal, Union, overload
 
 from flask import Flask, current_app
@@ -34,9 +34,9 @@ class ChatAppGenerator(MessageBasedAppGenerator):
         self,
         app_model: App,
         user: Union[Account, EndUser],
-        args: Mapping[str, Any],
+        args: Any,
         invoke_from: InvokeFrom,
-        streaming: Literal[True],
+        stream: Literal[True] = True,
     ) -> Generator[str, None, None]: ...
 
     @overload
@@ -44,29 +44,19 @@ class ChatAppGenerator(MessageBasedAppGenerator):
         self,
         app_model: App,
         user: Union[Account, EndUser],
-        args: Mapping[str, Any],
+        args: Any,
         invoke_from: InvokeFrom,
-        streaming: Literal[False],
-    ) -> Mapping[str, Any]: ...
-
-    @overload
-    def generate(
-        self,
-        app_model: App,
-        user: Union[Account, EndUser],
-        args: Mapping[str, Any],
-        invoke_from: InvokeFrom,
-        streaming: bool,
-    ) -> Union[Mapping[str, Any], Generator[str, None, None]]: ...
+        stream: Literal[False] = False,
+    ) -> dict: ...
 
     def generate(
         self,
         app_model: App,
         user: Union[Account, EndUser],
-        args: Mapping[str, Any],
+        args: Any,
         invoke_from: InvokeFrom,
-        streaming: bool = True,
-    ):
+        stream: bool = True,
+    ) -> Union[dict, Generator[str, None, None]]:
         """
         Generate App response.
 
@@ -142,9 +132,7 @@ class ChatAppGenerator(MessageBasedAppGenerator):
             conversation_id=conversation.id if conversation else None,
             inputs=conversation.inputs
             if conversation
-            else self._prepare_user_inputs(
-                user_inputs=inputs, variables=app_config.variables, tenant_id=app_model.tenant_id
-            ),
+            else self._prepare_user_inputs(user_inputs=inputs, app_config=app_config),
             query=query,
             files=file_objs,
             parent_message_id=args.get("parent_message_id") if invoke_from != InvokeFrom.SERVICE_API else UUID_NIL,
@@ -152,7 +140,7 @@ class ChatAppGenerator(MessageBasedAppGenerator):
             invoke_from=invoke_from,
             extras=extras,
             trace_manager=trace_manager,
-            stream=streaming,
+            stream=stream,
         )
 
         # init generate records
@@ -189,7 +177,7 @@ class ChatAppGenerator(MessageBasedAppGenerator):
             conversation=conversation,
             message=message,
             user=user,
-            stream=streaming,
+            stream=stream,
         )
 
         return ChatAppGenerateResponseConverter.convert(response=response, invoke_from=invoke_from)

api/core/app/apps/completion/app_generator.py

@@ -1,7 +1,7 @@
 import logging
 import threading
 import uuid
-from collections.abc import Generator, Mapping
+from collections.abc import Generator
 from typing import Any, Literal, Union, overload
 
 from flask import Flask, current_app
@@ -34,9 +34,9 @@ class CompletionAppGenerator(MessageBasedAppGenerator):
         self,
         app_model: App,
         user: Union[Account, EndUser],
-        args: Mapping[str, Any],
+        args: dict,
         invoke_from: InvokeFrom,
-        streaming: Literal[True],
+        stream: Literal[True] = True,
     ) -> Generator[str, None, None]: ...
 
     @overload
@@ -44,29 +44,14 @@ class CompletionAppGenerator(MessageBasedAppGenerator):
         self,
         app_model: App,
         user: Union[Account, EndUser],
-        args: Mapping[str, Any],
+        args: dict,
         invoke_from: InvokeFrom,
-        streaming: Literal[False],
-    ) -> Mapping[str, Any]: ...
-
-    @overload
-    def generate(
-        self,
-        app_model: App,
-        user: Union[Account, EndUser],
-        args: Mapping[str, Any],
-        invoke_from: InvokeFrom,
-        streaming: bool,
-    ) -> Mapping[str, Any] | Generator[str, None, None]: ...
+        stream: Literal[False] = False,
+    ) -> dict: ...
 
     def generate(
-        self,
-        app_model: App,
-        user: Union[Account, EndUser],
-        args: Mapping[str, Any],
-        invoke_from: InvokeFrom,
-        streaming: bool = True,
-    ):
+        self, app_model: App, user: Union[Account, EndUser], args: Any, invoke_from: InvokeFrom, stream: bool = True
+    ) -> Union[dict, Generator[str, None, None]]:
         """
         Generate App response.
 
@@ -128,13 +113,11 @@ class CompletionAppGenerator(MessageBasedAppGenerator):
             app_config=app_config,
             model_conf=ModelConfigConverter.convert(app_config),
             file_upload_config=file_extra_config,
-            inputs=self._prepare_user_inputs(
-                user_inputs=inputs, variables=app_config.variables, tenant_id=app_model.tenant_id
-            ),
+            inputs=self._prepare_user_inputs(user_inputs=inputs, app_config=app_config),
             query=query,
             files=file_objs,
             user_id=user.id,
-            stream=streaming,
+            stream=stream,
             invoke_from=invoke_from,
             extras=extras,
             trace_manager=trace_manager,
@@ -173,7 +156,7 @@ class CompletionAppGenerator(MessageBasedAppGenerator):
             conversation=conversation,
             message=message,
             user=user,
-            stream=streaming,
+            stream=stream,
         )
 
         return CompletionAppGenerateResponseConverter.convert(response=response, invoke_from=invoke_from)

api/core/app/apps/message_based_app_generator.py

@@ -1,7 +1,7 @@
 import json
 import logging
 from collections.abc import Generator
-from datetime import UTC, datetime
+from datetime import datetime, timezone
 from typing import Optional, Union
 
 from sqlalchemy import and_
@@ -200,7 +200,7 @@ class MessageBasedAppGenerator(BaseAppGenerator):
             db.session.commit()
             db.session.refresh(conversation)
         else:
-            conversation.updated_at = datetime.now(UTC).replace(tzinfo=None)
+            conversation.updated_at = datetime.now(timezone.utc).replace(tzinfo=None)
             db.session.commit()
 
         message = Message(

api/core/app/apps/workflow/app_generator.py

@@ -33,13 +33,12 @@ class WorkflowAppGenerator(BaseAppGenerator):
     @overload
     def generate(
         self,
-        *,
         app_model: App,
         workflow: Workflow,
-        user: Account | EndUser,
-        args: Mapping[str, Any],
+        user: Union[Account, EndUser],
+        args: dict,
         invoke_from: InvokeFrom,
-        streaming: Literal[True],
+        stream: Literal[True] = True,
         call_depth: int = 0,
         workflow_thread_pool_id: Optional[str] = None,
     ) -> Generator[str, None, None]: ...
@@ -47,40 +46,24 @@ class WorkflowAppGenerator(BaseAppGenerator):
     @overload
     def generate(
         self,
-        *,
         app_model: App,
         workflow: Workflow,
-        user: Account | EndUser,
-        args: Mapping[str, Any],
+        user: Union[Account, EndUser],
+        args: dict,
         invoke_from: InvokeFrom,
-        streaming: Literal[False],
+        stream: Literal[False] = False,
         call_depth: int = 0,
         workflow_thread_pool_id: Optional[str] = None,
-    ) -> Mapping[str, Any]: ...
-
-    @overload
-    def generate(
-        self,
-        *,
-        app_model: App,
-        workflow: Workflow,
-        user: Account | EndUser,
-        args: Mapping[str, Any],
-        invoke_from: InvokeFrom,
-        streaming: bool = True,
-        call_depth: int = 0,
-        workflow_thread_pool_id: Optional[str] = None,
-    ) -> Mapping[str, Any] | Generator[str, None, None]: ...
+    ) -> dict: ...
 
     def generate(
         self,
-        *,
         app_model: App,
         workflow: Workflow,
-        user: Account | EndUser,
+        user: Union[Account, EndUser],
         args: Mapping[str, Any],
         invoke_from: InvokeFrom,
-        streaming: bool = True,
+        stream: bool = True,
         call_depth: int = 0,
         workflow_thread_pool_id: Optional[str] = None,
     ):
@@ -113,12 +96,10 @@ class WorkflowAppGenerator(BaseAppGenerator):
             task_id=str(uuid.uuid4()),
             app_config=app_config,
             file_upload_config=file_extra_config,
-            inputs=self._prepare_user_inputs(
-                user_inputs=inputs, variables=app_config.variables, tenant_id=app_model.tenant_id
-            ),
+            inputs=self._prepare_user_inputs(user_inputs=inputs, app_config=app_config),
             files=system_files,
             user_id=user.id,
-            stream=streaming,
+            stream=stream,
             invoke_from=invoke_from,
             call_depth=call_depth,
             trace_manager=trace_manager,
@@ -132,7 +113,7 @@ class WorkflowAppGenerator(BaseAppGenerator):
             user=user,
             application_generate_entity=application_generate_entity,
             invoke_from=invoke_from,
-            streaming=streaming,
+            stream=stream,
             workflow_thread_pool_id=workflow_thread_pool_id,
         )
 
@@ -144,9 +125,20 @@ class WorkflowAppGenerator(BaseAppGenerator):
         user: Union[Account, EndUser],
         application_generate_entity: WorkflowAppGenerateEntity,
         invoke_from: InvokeFrom,
-        streaming: bool = True,
+        stream: bool = True,
         workflow_thread_pool_id: Optional[str] = None,
-    ) -> Mapping[str, Any] | Generator[str, None, None]:
+    ) -> dict[str, Any] | Generator[str, None, None]:
+        """
+        Generate App response.
+
+        :param app_model: App
+        :param workflow: Workflow
+        :param user: account or end user
+        :param application_generate_entity: application generate entity
+        :param invoke_from: invoke from source
+        :param stream: is stream
+        :param workflow_thread_pool_id: workflow thread pool id
+        """
         # init queue manager
         queue_manager = WorkflowAppQueueManager(
             task_id=application_generate_entity.task_id,
@@ -175,20 +167,14 @@ class WorkflowAppGenerator(BaseAppGenerator):
             workflow=workflow,
             queue_manager=queue_manager,
             user=user,
-            stream=streaming,
+            stream=stream,
         )
 
         return WorkflowAppGenerateResponseConverter.convert(response=response, invoke_from=invoke_from)
 
     def single_iteration_generate(
-        self,
-        app_model: App,
-        workflow: Workflow,
-        node_id: str,
-        user: Account,
-        args: Mapping[str, Any],
-        streaming: bool = True,
-    ) -> Mapping[str, Any] | Generator[str, None, None]:
+        self, app_model: App, workflow: Workflow, node_id: str, user: Account, args: dict, stream: bool = True
+    ) -> dict[str, Any] | Generator[str, Any, None]:
         """
         Generate App response.
 
@@ -215,7 +201,7 @@ class WorkflowAppGenerator(BaseAppGenerator):
             inputs={},
             files=[],
             user_id=user.id,
-            stream=streaming,
+            stream=stream,
             invoke_from=InvokeFrom.DEBUGGER,
             extras={"auto_generate_conversation_name": False},
             single_iteration_run=WorkflowAppGenerateEntity.SingleIterationRunEntity(
@@ -230,7 +216,7 @@ class WorkflowAppGenerator(BaseAppGenerator):
             user=user,
             invoke_from=InvokeFrom.DEBUGGER,
             application_generate_entity=application_generate_entity,
-            streaming=streaming,
+            stream=stream,
         )
 
     def _generate_worker(

api/core/app/apps/workflow/app_queue_manager.py

@@ -6,7 +6,6 @@ from core.app.entities.queue_entities import (
     QueueMessageEndEvent,
     QueueStopEvent,
     QueueWorkflowFailedEvent,
-    QueueWorkflowPartialSuccessEvent,
     QueueWorkflowSucceededEvent,
     WorkflowQueueMessage,
 )
@@ -35,8 +34,7 @@ class WorkflowAppQueueManager(AppQueueManager):
             | QueueErrorEvent
             | QueueMessageEndEvent
             | QueueWorkflowSucceededEvent
-            | QueueWorkflowFailedEvent
-            | QueueWorkflowPartialSuccessEvent,
+            | QueueWorkflowFailedEvent,
         ):
             self.stop_listen()
 

api/core/app/apps/workflow/generate_task_pipeline.py

@@ -15,10 +15,8 @@ from core.app.entities.queue_entities import (
     QueueIterationCompletedEvent,
     QueueIterationNextEvent,
     QueueIterationStartEvent,
-    QueueNodeExceptionEvent,
     QueueNodeFailedEvent,
     QueueNodeInIterationFailedEvent,
-    QueueNodeRetryEvent,
     QueueNodeStartedEvent,
     QueueNodeSucceededEvent,
     QueueParallelBranchRunFailedEvent,
@@ -28,7 +26,6 @@ from core.app.entities.queue_entities import (
     QueueStopEvent,
     QueueTextChunkEvent,
     QueueWorkflowFailedEvent,
-    QueueWorkflowPartialSuccessEvent,
     QueueWorkflowStartedEvent,
     QueueWorkflowSucceededEvent,
 )
@@ -260,44 +257,7 @@ class WorkflowAppGenerateTaskPipeline(BasedGenerateTaskPipeline, WorkflowCycleMa
 
                 workflow_node_execution = self._handle_node_execution_start(workflow_run=workflow_run, event=event)
 
-                node_start_response = self._workflow_node_start_to_stream_response(
-                    event=event,
-                    task_id=self._application_generate_entity.task_id,
-                    workflow_node_execution=workflow_node_execution,
-                )
-
-                if node_start_response:
-                    yield node_start_response
-            elif isinstance(event, QueueNodeSucceededEvent):
-                workflow_node_execution = self._handle_workflow_node_execution_success(event)
-
-                node_success_response = self._workflow_node_finish_to_stream_response(
-                    event=event,
-                    task_id=self._application_generate_entity.task_id,
-                    workflow_node_execution=workflow_node_execution,
-                )
-
-                if node_success_response:
-                    yield node_success_response
-            elif isinstance(event, QueueNodeFailedEvent | QueueNodeInIterationFailedEvent | QueueNodeExceptionEvent):
-                workflow_node_execution = self._handle_workflow_node_execution_failed(event)
-
-                node_failed_response = self._workflow_node_finish_to_stream_response(
-                    event=event,
-                    task_id=self._application_generate_entity.task_id,
-                    workflow_node_execution=workflow_node_execution,
-                )
-                if node_failed_response:
-                    yield node_failed_response
-            elif isinstance(
-                event,
-                QueueNodeRetryEvent,
-            ):
-                workflow_node_execution = self._handle_workflow_node_execution_retried(
-                    workflow_run=workflow_run, event=event
-                )
-
-                response = self._workflow_node_retry_to_stream_response(
+                response = self._workflow_node_start_to_stream_response(
                     event=event,
                     task_id=self._application_generate_entity.task_id,
                     workflow_node_execution=workflow_node_execution,
@@ -305,7 +265,28 @@ class WorkflowAppGenerateTaskPipeline(BasedGenerateTaskPipeline, WorkflowCycleMa
 
                 if response:
                     yield response
+            elif isinstance(event, QueueNodeSucceededEvent):
+                workflow_node_execution = self._handle_workflow_node_execution_success(event)
 
+                response = self._workflow_node_finish_to_stream_response(
+                    event=event,
+                    task_id=self._application_generate_entity.task_id,
+                    workflow_node_execution=workflow_node_execution,
+                )
+
+                if response:
+                    yield response
+            elif isinstance(event, QueueNodeFailedEvent | QueueNodeInIterationFailedEvent):
+                workflow_node_execution = self._handle_workflow_node_execution_failed(event)
+
+                response = self._workflow_node_finish_to_stream_response(
+                    event=event,
+                    task_id=self._application_generate_entity.task_id,
+                    workflow_node_execution=workflow_node_execution,
+                )
+
+                if response:
+                    yield response
             elif isinstance(event, QueueParallelBranchRunStartedEvent):
                 if not workflow_run:
                     raise Exception("Workflow run not initialized.")
@@ -361,30 +342,6 @@ class WorkflowAppGenerateTaskPipeline(BasedGenerateTaskPipeline, WorkflowCycleMa
                 # save workflow app log
                 self._save_workflow_app_log(workflow_run)
 
-                yield self._workflow_finish_to_stream_response(
-                    task_id=self._application_generate_entity.task_id, workflow_run=workflow_run
-                )
-            elif isinstance(event, QueueWorkflowPartialSuccessEvent):
-                if not workflow_run:
-                    raise Exception("Workflow run not initialized.")
-
-                if not graph_runtime_state:
-                    raise Exception("Graph runtime state not initialized.")
-
-                workflow_run = self._handle_workflow_run_partial_success(
-                    workflow_run=workflow_run,
-                    start_at=graph_runtime_state.start_at,
-                    total_tokens=graph_runtime_state.total_tokens,
-                    total_steps=graph_runtime_state.node_run_steps,
-                    outputs=event.outputs,
-                    exceptions_count=event.exceptions_count,
-                    conversation_id=None,
-                    trace_manager=trace_manager,
-                )
-
-                # save workflow app log
-                self._save_workflow_app_log(workflow_run)
-
                 yield self._workflow_finish_to_stream_response(
                     task_id=self._application_generate_entity.task_id, workflow_run=workflow_run
                 )
@@ -394,6 +351,7 @@ class WorkflowAppGenerateTaskPipeline(BasedGenerateTaskPipeline, WorkflowCycleMa
 
                 if not graph_runtime_state:
                     raise Exception("Graph runtime state not initialized.")
+
                 workflow_run = self._handle_workflow_run_failed(
                     workflow_run=workflow_run,
                     start_at=graph_runtime_state.start_at,
@@ -405,7 +363,6 @@ class WorkflowAppGenerateTaskPipeline(BasedGenerateTaskPipeline, WorkflowCycleMa
                     error=event.error if isinstance(event, QueueWorkflowFailedEvent) else event.get_stop_reason(),
                     conversation_id=None,
                     trace_manager=trace_manager,
-                    exceptions_count=event.exceptions_count if isinstance(event, QueueWorkflowFailedEvent) else 0,
                 )
 
                 # save workflow app log

api/core/app/apps/workflow_app_runner.py

@@ -8,10 +8,8 @@ from core.app.entities.queue_entities import (
     QueueIterationCompletedEvent,
     QueueIterationNextEvent,
     QueueIterationStartEvent,
-    QueueNodeExceptionEvent,
     QueueNodeFailedEvent,
     QueueNodeInIterationFailedEvent,
-    QueueNodeRetryEvent,
     QueueNodeStartedEvent,
     QueueNodeSucceededEvent,
     QueueParallelBranchRunFailedEvent,
@@ -20,7 +18,6 @@ from core.app.entities.queue_entities import (
     QueueRetrieverResourcesEvent,
     QueueTextChunkEvent,
     QueueWorkflowFailedEvent,
-    QueueWorkflowPartialSuccessEvent,
     QueueWorkflowStartedEvent,
     QueueWorkflowSucceededEvent,
 )
@@ -28,7 +25,6 @@ from core.workflow.entities.variable_pool import VariablePool
 from core.workflow.graph_engine.entities.event import (
     GraphEngineEvent,
     GraphRunFailedEvent,
-    GraphRunPartialSucceededEvent,
     GraphRunStartedEvent,
     GraphRunSucceededEvent,
     IterationRunFailedEvent,
@@ -36,10 +32,8 @@ from core.workflow.graph_engine.entities.event import (
     IterationRunStartedEvent,
     IterationRunSucceededEvent,
     NodeInIterationFailedEvent,
-    NodeRunExceptionEvent,
     NodeRunFailedEvent,
     NodeRunRetrieverResourceEvent,
-    NodeRunRetryEvent,
     NodeRunStartedEvent,
     NodeRunStreamChunkEvent,
     NodeRunSucceededEvent,
@@ -49,7 +43,8 @@ from core.workflow.graph_engine.entities.event import (
 )
 from core.workflow.graph_engine.entities.graph import Graph
 from core.workflow.nodes import NodeType
-from core.workflow.nodes.node_mapping import NODE_TYPE_CLASSES_MAPPING
+from core.workflow.nodes.iteration import IterationNodeData
+from core.workflow.nodes.node_mapping import node_type_classes_mapping
 from core.workflow.workflow_entry import WorkflowEntry
 from extensions.ext_database import db
 from models.model import App
@@ -144,8 +139,7 @@ class WorkflowBasedAppRunner(AppRunner):
 
         # Get node class
         node_type = NodeType(iteration_node_config.get("data", {}).get("type"))
-        node_version = iteration_node_config.get("data", {}).get("version", "1")
-        node_cls = NODE_TYPE_CLASSES_MAPPING[node_type][node_version]
+        node_cls = node_type_classes_mapping[node_type]
 
         # init variable pool
         variable_pool = VariablePool(
@@ -166,6 +160,8 @@ class WorkflowBasedAppRunner(AppRunner):
             user_inputs=user_inputs,
             variable_pool=variable_pool,
             tenant_id=workflow.tenant_id,
+            node_type=node_type,
+            node_data=IterationNodeData(**iteration_node_config.get("data", {})),
         )
 
         return graph, variable_pool
@@ -182,12 +178,8 @@ class WorkflowBasedAppRunner(AppRunner):
             )
         elif isinstance(event, GraphRunSucceededEvent):
             self._publish_event(QueueWorkflowSucceededEvent(outputs=event.outputs))
-        elif isinstance(event, GraphRunPartialSucceededEvent):
-            self._publish_event(
-                QueueWorkflowPartialSuccessEvent(outputs=event.outputs, exceptions_count=event.exceptions_count)
-            )
         elif isinstance(event, GraphRunFailedEvent):
-            self._publish_event(QueueWorkflowFailedEvent(error=event.error, exceptions_count=event.exceptions_count))
+            self._publish_event(QueueWorkflowFailedEvent(error=event.error))
         elif isinstance(event, NodeRunStartedEvent):
             self._publish_event(
                 QueueNodeStartedEvent(
@@ -263,36 +255,6 @@ class WorkflowBasedAppRunner(AppRunner):
                     in_iteration_id=event.in_iteration_id,
                 )
             )
-        elif isinstance(event, NodeRunExceptionEvent):
-            self._publish_event(
-                QueueNodeExceptionEvent(
-                    node_execution_id=event.id,
-                    node_id=event.node_id,
-                    node_type=event.node_type,
-                    node_data=event.node_data,
-                    parallel_id=event.parallel_id,
-                    parallel_start_node_id=event.parallel_start_node_id,
-                    parent_parallel_id=event.parent_parallel_id,
-                    parent_parallel_start_node_id=event.parent_parallel_start_node_id,
-                    start_at=event.route_node_state.start_at,
-                    inputs=event.route_node_state.node_run_result.inputs
-                    if event.route_node_state.node_run_result
-                    else {},
-                    process_data=event.route_node_state.node_run_result.process_data
-                    if event.route_node_state.node_run_result
-                    else {},
-                    outputs=event.route_node_state.node_run_result.outputs
-                    if event.route_node_state.node_run_result
-                    else {},
-                    error=event.route_node_state.node_run_result.error
-                    if event.route_node_state.node_run_result and event.route_node_state.node_run_result.error
-                    else "Unknown error",
-                    execution_metadata=event.route_node_state.node_run_result.metadata
-                    if event.route_node_state.node_run_result
-                    else {},
-                    in_iteration_id=event.in_iteration_id,
-                )
-            )
         elif isinstance(event, NodeInIterationFailedEvent):
             self._publish_event(
                 QueueNodeInIterationFailedEvent(
@@ -422,36 +384,6 @@ class WorkflowBasedAppRunner(AppRunner):
                     error=event.error if isinstance(event, IterationRunFailedEvent) else None,
                 )
             )
-        elif isinstance(event, NodeRunRetryEvent):
-            self._publish_event(
-                QueueNodeRetryEvent(
-                    node_execution_id=event.id,
-                    node_id=event.node_id,
-                    node_type=event.node_type,
-                    node_data=event.node_data,
-                    parallel_id=event.parallel_id,
-                    parallel_start_node_id=event.parallel_start_node_id,
-                    parent_parallel_id=event.parent_parallel_id,
-                    parent_parallel_start_node_id=event.parent_parallel_start_node_id,
-                    start_at=event.start_at,
-                    inputs=event.route_node_state.node_run_result.inputs
-                    if event.route_node_state.node_run_result
-                    else {},
-                    process_data=event.route_node_state.node_run_result.process_data
-                    if event.route_node_state.node_run_result
-                    else {},
-                    outputs=event.route_node_state.node_run_result.outputs
-                    if event.route_node_state.node_run_result
-                    else {},
-                    error=event.error,
-                    execution_metadata=event.route_node_state.node_run_result.metadata
-                    if event.route_node_state.node_run_result
-                    else {},
-                    in_iteration_id=event.in_iteration_id,
-                    retry_index=event.retry_index,
-                    start_index=event.start_index,
-                )
-            )
 
     def get_workflow(self, app_model: App, workflow_id: str) -> Optional[Workflow]:
         """

api/core/app/entities/queue_entities.py

@@ -1,8 +1,8 @@
 from datetime import datetime
-from enum import Enum, StrEnum
+from enum import Enum
 from typing import Any, Optional
 
-from pydantic import BaseModel
+from pydantic import BaseModel, field_validator
 
 from core.model_runtime.entities.llm_entities import LLMResult, LLMResultChunk
 from core.workflow.entities.node_entities import NodeRunMetadataKey
@@ -11,7 +11,7 @@ from core.workflow.nodes import NodeType
 from core.workflow.nodes.base import BaseNodeData
 
 
-class QueueEvent(StrEnum):
+class QueueEvent(str, Enum):
     """
     QueueEvent enum
     """
@@ -25,14 +25,12 @@ class QueueEvent(StrEnum):
     WORKFLOW_STARTED = "workflow_started"
     WORKFLOW_SUCCEEDED = "workflow_succeeded"
     WORKFLOW_FAILED = "workflow_failed"
-    WORKFLOW_PARTIAL_SUCCEEDED = "workflow_partial_succeeded"
     ITERATION_START = "iteration_start"
     ITERATION_NEXT = "iteration_next"
     ITERATION_COMPLETED = "iteration_completed"
     NODE_STARTED = "node_started"
     NODE_SUCCEEDED = "node_succeeded"
     NODE_FAILED = "node_failed"
-    NODE_EXCEPTION = "node_exception"
     RETRIEVER_RESOURCES = "retriever_resources"
     ANNOTATION_REPLY = "annotation_reply"
     AGENT_THOUGHT = "agent_thought"
@@ -43,7 +41,6 @@ class QueueEvent(StrEnum):
     ERROR = "error"
     PING = "ping"
     STOP = "stop"
-    RETRY = "retry"
 
 
 class AppQueueEvent(BaseModel):
@@ -116,6 +113,18 @@ class QueueIterationNextEvent(AppQueueEvent):
     output: Optional[Any] = None  # output for the current iteration
     duration: Optional[float] = None
 
+    @field_validator("output", mode="before")
+    @classmethod
+    def set_output(cls, v):
+        """
+        Set output
+        """
+        if v is None:
+            return None
+        if isinstance(v, int | float | str | bool | dict | list):
+            return v
+        raise ValueError("output must be a valid type")
+
 
 class QueueIterationCompletedEvent(AppQueueEvent):
     """
@@ -240,17 +249,6 @@ class QueueWorkflowFailedEvent(AppQueueEvent):
 
     event: QueueEvent = QueueEvent.WORKFLOW_FAILED
     error: str
-    exceptions_count: int
-
-
-class QueueWorkflowPartialSuccessEvent(AppQueueEvent):
-    """
-    QueueWorkflowFailedEvent entity
-    """
-
-    event: QueueEvent = QueueEvent.WORKFLOW_PARTIAL_SUCCEEDED
-    exceptions_count: int
-    outputs: Optional[dict[str, Any]] = None
 
 
 class QueueNodeStartedEvent(AppQueueEvent):
@@ -314,37 +312,6 @@ class QueueNodeSucceededEvent(AppQueueEvent):
     iteration_duration_map: Optional[dict[str, float]] = None
 
 
-class QueueNodeRetryEvent(AppQueueEvent):
-    """QueueNodeRetryEvent entity"""
-
-    event: QueueEvent = QueueEvent.RETRY
-
-    node_execution_id: str
-    node_id: str
-    node_type: NodeType
-    node_data: BaseNodeData
-    parallel_id: Optional[str] = None
-    """parallel id if node is in parallel"""
-    parallel_start_node_id: Optional[str] = None
-    """parallel start node id if node is in parallel"""
-    parent_parallel_id: Optional[str] = None
-    """parent parallel id if node is in parallel"""
-    parent_parallel_start_node_id: Optional[str] = None
-    """parent parallel start node id if node is in parallel"""
-    in_iteration_id: Optional[str] = None
-    """iteration id if node is in iteration"""
-    start_at: datetime
-
-    inputs: Optional[dict[str, Any]] = None
-    process_data: Optional[dict[str, Any]] = None
-    outputs: Optional[dict[str, Any]] = None
-    execution_metadata: Optional[dict[NodeRunMetadataKey, Any]] = None
-
-    error: str
-    retry_index: int  # retry index
-    start_index: int  # start index
-
-
 class QueueNodeInIterationFailedEvent(AppQueueEvent):
     """
     QueueNodeInIterationFailedEvent entity
@@ -376,37 +343,6 @@ class QueueNodeInIterationFailedEvent(AppQueueEvent):
     error: str
 
 
-class QueueNodeExceptionEvent(AppQueueEvent):
-    """
-    QueueNodeExceptionEvent entity
-    """
-
-    event: QueueEvent = QueueEvent.NODE_EXCEPTION
-
-    node_execution_id: str
-    node_id: str
-    node_type: NodeType
-    node_data: BaseNodeData
-    parallel_id: Optional[str] = None
-    """parallel id if node is in parallel"""
-    parallel_start_node_id: Optional[str] = None
-    """parallel start node id if node is in parallel"""
-    parent_parallel_id: Optional[str] = None
-    """parent parallel id if node is in parallel"""
-    parent_parallel_start_node_id: Optional[str] = None
-    """parent parallel start node id if node is in parallel"""
-    in_iteration_id: Optional[str] = None
-    """iteration id if node is in iteration"""
-    start_at: datetime
-
-    inputs: Optional[dict[str, Any]] = None
-    process_data: Optional[dict[str, Any]] = None
-    outputs: Optional[dict[str, Any]] = None
-    execution_metadata: Optional[dict[NodeRunMetadataKey, Any]] = None
-
-    error: str
-
-
 class QueueNodeFailedEvent(AppQueueEvent):
     """
     QueueNodeFailedEvent entity

api/core/app/entities/task_entities.py

@@ -52,7 +52,6 @@ class StreamEvent(Enum):
     WORKFLOW_FINISHED = "workflow_finished"
     NODE_STARTED = "node_started"
     NODE_FINISHED = "node_finished"
-    NODE_RETRY = "node_retry"
     PARALLEL_BRANCH_STARTED = "parallel_branch_started"
     PARALLEL_BRANCH_FINISHED = "parallel_branch_finished"
     ITERATION_STARTED = "iteration_started"
@@ -214,7 +213,6 @@ class WorkflowFinishStreamResponse(StreamResponse):
         created_by: Optional[dict] = None
         created_at: int
         finished_at: int
-        exceptions_count: Optional[int] = 0
         files: Optional[Sequence[Mapping[str, Any]]] = []
 
     event: StreamEvent = StreamEvent.WORKFLOW_FINISHED
@@ -343,75 +341,6 @@ class NodeFinishStreamResponse(StreamResponse):
         }
 
 
-class NodeRetryStreamResponse(StreamResponse):
-    """
-    NodeFinishStreamResponse entity
-    """
-
-    class Data(BaseModel):
-        """
-        Data entity
-        """
-
-        id: str
-        node_id: str
-        node_type: str
-        title: str
-        index: int
-        predecessor_node_id: Optional[str] = None
-        inputs: Optional[dict] = None
-        process_data: Optional[dict] = None
-        outputs: Optional[dict] = None
-        status: str
-        error: Optional[str] = None
-        elapsed_time: float
-        execution_metadata: Optional[dict] = None
-        created_at: int
-        finished_at: int
-        files: Optional[Sequence[Mapping[str, Any]]] = []
-        parallel_id: Optional[str] = None
-        parallel_start_node_id: Optional[str] = None
-        parent_parallel_id: Optional[str] = None
-        parent_parallel_start_node_id: Optional[str] = None
-        iteration_id: Optional[str] = None
-        retry_index: int = 0
-
-    event: StreamEvent = StreamEvent.NODE_RETRY
-    workflow_run_id: str
-    data: Data
-
-    def to_ignore_detail_dict(self):
-        return {
-            "event": self.event.value,
-            "task_id": self.task_id,
-            "workflow_run_id": self.workflow_run_id,
-            "data": {
-                "id": self.data.id,
-                "node_id": self.data.node_id,
-                "node_type": self.data.node_type,
-                "title": self.data.title,
-                "index": self.data.index,
-                "predecessor_node_id": self.data.predecessor_node_id,
-                "inputs": None,
-                "process_data": None,
-                "outputs": None,
-                "status": self.data.status,
-                "error": None,
-                "elapsed_time": self.data.elapsed_time,
-                "execution_metadata": None,
-                "created_at": self.data.created_at,
-                "finished_at": self.data.finished_at,
-                "files": [],
-                "parallel_id": self.data.parallel_id,
-                "parallel_start_node_id": self.data.parallel_start_node_id,
-                "parent_parallel_id": self.data.parent_parallel_id,
-                "parent_parallel_start_node_id": self.data.parent_parallel_start_node_id,
-                "iteration_id": self.data.iteration_id,
-                "retry_index": self.data.retry_index,
-            },
-        }
-
-
 class ParallelBranchStartStreamResponse(StreamResponse):
     """
     ParallelBranchStartStreamResponse entity