Merge branch 'feat/enhance-multi-modal-support' into release/0.10.0-beta

- Use more precise type annotations with Sequence and Mapping for task entities.
- Ensure raw_prompt is assigned properly after replacement in advanced prompt transform.
- Remove unused generator return type from _fetch_context method.
- Refactor tool node file handling to retrieve more comprehensive file attributes, ensuring file existence validation in the database.

.devcontainer/post_start_command.sh

@@ -1,3 +1,3 @@
 #!/bin/bash
 
-cd api && poetry install
+poetry install -C api

.github/workflows/api-tests.yml

@@ -27,17 +27,18 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v4
 
+      - name: Install Poetry
+        uses: abatilo/actions-poetry@v3
+
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
+          cache: 'poetry'
           cache-dependency-path: |
             api/pyproject.toml
             api/poetry.lock
 
-      - name: Install Poetry
-        uses: abatilo/actions-poetry@v3
-
       - name: Check Poetry lockfile
         run: |
           poetry check -C api --lock
@@ -78,7 +79,7 @@ jobs:
       - name: Run Workflow
         run: poetry run -C api bash dev/pytest/pytest_workflow.sh
 
-      - name: Set up Vector Stores (Weaviate, Qdrant, PGVector, Milvus, PgVecto-RS, Chroma, MyScale, ElasticSearch, Couchbase)
+      - name: Set up Vector Stores (Weaviate, Qdrant, PGVector, Milvus, PgVecto-RS, Chroma, MyScale, ElasticSearch)
         uses: hoverkraft-tech/compose-action@v2.0.0
         with:
           compose-file: |
@@ -86,7 +87,6 @@ jobs:
           services: |
             weaviate
             qdrant
-            couchbase-server
             etcd
             minio
             milvus-standalone

.github/workflows/build-push.yml

@@ -5,6 +5,7 @@ on:
     branches:
       - "main"
       - "deploy/dev"
+      - "release/0.10.0-beta"
   release:
     types: [published]
 
@@ -49,7 +50,7 @@ jobs:
           echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV
 
       - name: Login to Docker Hub
-        uses: docker/login-action@v3
+        uses: docker/login-action@v2
         with:
           username: ${{ env.DOCKERHUB_USER }}
           password: ${{ env.DOCKERHUB_TOKEN }}
@@ -114,7 +115,7 @@ jobs:
           merge-multiple: true
 
       - name: Login to Docker Hub
-        uses: docker/login-action@v3
+        uses: docker/login-action@v2
         with:
           username: ${{ env.DOCKERHUB_USER }}
           password: ${{ env.DOCKERHUB_TOKEN }}

.github/workflows/db-migration-test.yml

@@ -23,17 +23,18 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v4
 
+      - name: Install Poetry
+        uses: abatilo/actions-poetry@v3
+
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
+          cache: 'poetry'
           cache-dependency-path: |
             api/pyproject.toml
             api/poetry.lock
 
-      - name: Install Poetry
-        uses: abatilo/actions-poetry@v3
-
       - name: Install dependencies
         run: poetry install -C api
 

.github/workflows/expose_service_ports.sh

@@ -7,7 +7,5 @@ yq eval '.services["milvus-standalone"].ports += ["19530:19530"]' -i docker/dock
 yq eval '.services.pgvector.ports += ["5433:5432"]' -i docker/docker-compose.yaml
 yq eval '.services["pgvecto-rs"].ports += ["5431:5432"]' -i docker/docker-compose.yaml
 yq eval '.services["elasticsearch"].ports += ["9200:9200"]' -i docker/docker-compose.yaml
-yq eval '.services.couchbase-server.ports += ["8091-8096:8091-8096"]' -i docker/docker-compose.yaml
-yq eval '.services.couchbase-server.ports += ["11210:11210"]' -i docker/docker-compose.yaml
 
-echo "Ports exposed for sandbox, weaviate, qdrant, chroma, milvus, pgvector, pgvecto-rs, elasticsearch, couchbase"
+echo "Ports exposed for sandbox, weaviate, qdrant, chroma, milvus, pgvector, pgvecto-rs, elasticsearch"

.github/workflows/style.yml

@@ -24,16 +24,15 @@ jobs:
         with:
           files: api/**
 
+      - name: Install Poetry
+        uses: abatilo/actions-poetry@v3
+
       - name: Set up Python
         uses: actions/setup-python@v5
         if: steps.changed-files.outputs.any_changed == 'true'
         with:
           python-version: '3.10'
 
-      - name: Install Poetry
-        if: steps.changed-files.outputs.any_changed == 'true'
-        uses: abatilo/actions-poetry@v3
-
       - name: Python dependencies
         if: steps.changed-files.outputs.any_changed == 'true'
         run: poetry install -C api --only lint

.gitignore

@@ -173,12 +173,8 @@ docker/volumes/myscale/log/*
 docker/volumes/unstructured/*
 docker/volumes/pgvector/data/*
 docker/volumes/pgvecto_rs/data/*
-docker/volumes/couchbase/*
-docker/volumes/oceanbase/*
 
 docker/nginx/conf.d/default.conf
-docker/nginx/ssl/*
-!docker/nginx/ssl/.gitkeep
 docker/middleware.env
 
 sdks/python-client/build
@@ -191,4 +187,4 @@ pyrightconfig.json
 api/.vscode
 
 .idea/
-.vscode
+.vscode

CONTRIBUTING.md

@@ -81,7 +81,7 @@ Dify requires the following dependencies to build, make sure they're installed o
 
 Dify is composed of a backend and a frontend. Navigate to the backend directory by `cd api/`, then follow the [Backend README](api/README.md) to install it. In a separate terminal, navigate to the frontend directory by `cd web/`, then follow the [Frontend README](web/README.md) to install.
 
-Check the [installation FAQ](https://docs.dify.ai/learn-more/faq/install-faq) for a list of common issues and steps to troubleshoot.
+Check the [installation FAQ](https://docs.dify.ai/learn-more/faq/self-host-faq) for a list of common issues and steps to troubleshoot.
 
 ### 5. Visit dify in your browser
 

CONTRIBUTING_VI.md

@@ -79,7 +79,7 @@ Dify yêu cầu các phụ thuộc sau để build, hãy đảm bảo chúng đ
 
 Dify bao gồm một backend và một frontend. Đi đến thư mục backend bằng lệnh `cd api/`, sau đó làm theo hướng dẫn trong [README của Backend](api/README.md) để cài đặt. Trong một terminal khác, đi đến thư mục frontend bằng lệnh `cd web/`, sau đó làm theo hướng dẫn trong [README của Frontend](web/README.md) để cài đặt.
 
-Kiểm tra [FAQ về cài đặt](https://docs.dify.ai/learn-more/faq/install-faq) để xem danh sách các vấn đề thường gặp và các bước khắc phục.
+Kiểm tra [FAQ về cài đặt](https://docs.dify.ai/learn-more/faq/self-host-faq) để xem danh sách các vấn đề thường gặp và các bước khắc phục.
 
 ### 5. Truy cập Dify trong trình duyệt của bạn
 

LICENSE

@@ -6,9 +6,8 @@ Dify is licensed under the Apache License 2.0, with the following additional con
 
 a. Multi-tenant service: Unless explicitly authorized by Dify in writing, you may not use the Dify source code to operate a multi-tenant environment. 
     - Tenant Definition: Within the context of Dify, one tenant corresponds to one workspace. The workspace provides a separated area for each tenant's data and configurations.
-    
-b. LOGO and copyright information: In the process of using Dify's frontend, you may not remove or modify the LOGO or copyright information in the Dify console or applications. This restriction is inapplicable to uses of Dify that do not involve its frontend.
-    - Frontend Definition: For the purposes of this license, the "frontend" of Dify includes all components located in the `web/` directory when running Dify from the raw source code, or the "web" image when running Dify with Docker.
+
+b. LOGO and copyright information: In the process of using Dify's frontend components, you may not remove or modify the LOGO or copyright information in the Dify console or applications. This restriction is inapplicable to uses of Dify that do not involve its frontend components.
 
 Please contact business@dify.ai by email to inquire about licensing matters.
 

README.md

@@ -1,9 +1,5 @@
 ![cover-v5-optimized](https://github.com/langgenius/dify/assets/13230914/f9e19af5-61ba-4119-b926-d10c4c06ebab)
 
-<p align="center">
-  📌 <a href="https://dify.ai/blog/introducing-dify-workflow-file-upload-a-demo-on-ai-podcast">Introducing Dify Workflow File Upload: Recreate Google NotebookLM Podcast</a>
-</p>
-
 <p align="center">
   <a href="https://cloud.dify.ai">Dify Cloud</a> ·
   <a href="https://docs.dify.ai/getting-started/install-self-hosted">Self-hosting</a> ·
@@ -46,56 +42,6 @@
 </p>
 
 
-## Table of Content
-0. [Quick-Start🚀](https://github.com/langgenius/dify?tab=readme-ov-file#quick-start)
-
-1. [Intro📖](https://github.com/langgenius/dify?tab=readme-ov-file#intro)
-
-2. [How to use🔧](https://github.com/langgenius/dify?tab=readme-ov-file#using-dify)
-
-3. [Stay Ahead🏃](https://github.com/langgenius/dify?tab=readme-ov-file#staying-ahead)
-
-4. [Next Steps🏹](https://github.com/langgenius/dify?tab=readme-ov-file#next-steps)
-
-5. [Contributing💪](https://github.com/langgenius/dify?tab=readme-ov-file#contributing)
-
-6. [Community and Contact🏠](https://github.com/langgenius/dify?tab=readme-ov-file#community--contact)
-
-7. [Star-History📈](https://github.com/langgenius/dify?tab=readme-ov-file#star-history)
-
-8. [Security🔒](https://github.com/langgenius/dify?tab=readme-ov-file#security-disclosure)
-
-9. [License🤝](https://github.com/langgenius/dify?tab=readme-ov-file#license)
-
-> Make sure you read through this README before you start utilizing Dify😊
-
-
-## Quick start
-The quickest way to deploy Dify locally is to run our [docker-compose.yml](https://github.com/langgenius/dify/blob/main/docker/docker-compose.yaml). Follow the instructions to start in 5 minutes.
-
-> Before installing Dify, make sure your machine meets the following minimum system requirements:
-> 
->- CPU >= 2 Core
->- RAM >= 4 GiB
->- Docker and Docker Compose Installed
-</br>
-
-Run the following command in your terminal to clone the whole repo.
-```bash
-git clone https://github.com/langgenius/dify.git
-```
-After cloning,run the following command one by one.
-```bash
-cd dify
-cd docker
-cp .env.example .env
-docker compose up -d
-```
-
-After running, you can access the Dify dashboard in your browser at [http://localhost/install](http://localhost/install) and start the initialization process. You will be asked to setup an admin account.
-For more info of quick setup, check [here](https://docs.dify.ai/getting-started/install-self-hosted/docker-compose)
-
-## Intro
 Dify is an open-source LLM app development platform. Its intuitive interface combines AI workflow, RAG pipeline, agent capabilities, model management, observability features and more, letting you quickly go from prototype to production. Here's a list of the core features:
 </br> </br>
 
@@ -129,6 +75,73 @@ Dify is an open-source LLM app development platform. Its intuitive interface com
   All of Dify's offerings come with corresponding APIs, so you could effortlessly integrate Dify into your own business logic.
 
 
+## Feature comparison
+<table style="width: 100%;">
+  <tr>
+    <th align="center">Feature</th>
+    <th align="center">Dify.AI</th>
+    <th align="center">LangChain</th>
+    <th align="center">Flowise</th>
+    <th align="center">OpenAI Assistants API</th>
+  </tr>
+  <tr>
+    <td align="center">Programming Approach</td>
+    <td align="center">API + App-oriented</td>
+    <td align="center">Python Code</td>
+    <td align="center">App-oriented</td>
+    <td align="center">API-oriented</td>
+  </tr>
+  <tr>
+    <td align="center">Supported LLMs</td>
+    <td align="center">Rich Variety</td>
+    <td align="center">Rich Variety</td>
+    <td align="center">Rich Variety</td>
+    <td align="center">OpenAI-only</td>
+  </tr>
+  <tr>
+    <td align="center">RAG Engine</td>
+    <td align="center">✅</td>
+    <td align="center">✅</td>
+    <td align="center">✅</td>
+    <td align="center">✅</td>
+  </tr>
+  <tr>
+    <td align="center">Agent</td>
+    <td align="center">✅</td>
+    <td align="center">✅</td>
+    <td align="center">❌</td>
+    <td align="center">✅</td>
+  </tr>
+  <tr>
+    <td align="center">Workflow</td>
+    <td align="center">✅</td>
+    <td align="center">❌</td>
+    <td align="center">✅</td>
+    <td align="center">❌</td>
+  </tr>
+  <tr>
+    <td align="center">Observability</td>
+    <td align="center">✅</td>
+    <td align="center">✅</td>
+    <td align="center">❌</td>
+    <td align="center">❌</td>
+  </tr>
+  <tr>
+    <td align="center">Enterprise Features (SSO/Access control)</td>
+    <td align="center">✅</td>
+    <td align="center">❌</td>
+    <td align="center">❌</td>
+    <td align="center">❌</td>
+  </tr>
+  <tr>
+    <td align="center">Local Deployment</td>
+    <td align="center">✅</td>
+    <td align="center">✅</td>
+    <td align="center">✅</td>
+    <td align="center">❌</td>
+  </tr>
+</table>
+
 ## Using Dify
 
 - **Cloud </br>**
@@ -149,21 +162,30 @@ Star Dify on GitHub and be instantly notified of new releases.
 
 ![star-us](https://github.com/langgenius/dify/assets/13230914/b823edc1-6388-4e25-ad45-2f6b187adbb4)
 
+
+
+## Quick start
+> Before installing Dify, make sure your machine meets the following minimum system requirements:
+> 
+>- CPU >= 2 Core
+>- RAM >= 4GB
+
+</br>
+
+The easiest way to start the Dify server is to run our [docker-compose.yml](docker/docker-compose.yaml) file. Before running the installation command, make sure that [Docker](https://docs.docker.com/get-docker/) and [Docker Compose](https://docs.docker.com/compose/install/) are installed on your machine:
+
+```bash
+cd docker
+cp .env.example .env
+docker compose up -d
+```
+
+After running, you can access the Dify dashboard in your browser at [http://localhost/install](http://localhost/install) and start the initialization process.
+
+> If you'd like to contribute to Dify or do additional development, refer to our [guide to deploying from source code](https://docs.dify.ai/getting-started/install-self-hosted/local-source-code)
+
 ## Next steps
 
-Go to [quick-start](https://github.com/langgenius/dify?tab=readme-ov-file#quick-start) to setup your Dify or setup by source code.
-
-#### If you......
-If you forget your admin account, you can refer to this [guide](https://docs.dify.ai/getting-started/install-self-hosted/faqs#id-4.-how-to-reset-the-password-of-the-admin-account) to reset the password.
-
-> Use docker compose up without "-d" to enable logs printing out in your terminal. This might be useful if you have encountered unknow problems when using Dify.
-
-If you encountered system error and would like to acquire help in Github issues, make sure you always paste logs of the error in the request to accerate the conversation. Go to [Community & contact](https://github.com/langgenius/dify?tab=readme-ov-file#community--contact) for more information.
-
-> Please read the [Dify Documentation](https://docs.dify.ai/) for detailed how-to-use guidance. Most of the potential problems are explained in the doc.
-
-> If you'd like to contribute to Dify or make additional development, refer to our [guide to deploying from source code](https://docs.dify.ai/getting-started/install-self-hosted/local-source-code)
-
 If you need to customize the configuration, please refer to the comments in our [.env.example](docker/.env.example) file and update the corresponding values in your `.env` file. Additionally, you might need to make adjustments to the `docker-compose.yaml` file itself, such as changing image versions, port mappings, or volume mounts, based on your specific deployment environment and requirements. After making any changes, please re-run `docker-compose up -d`. You can find the full list of available environment variables [here](https://docs.dify.ai/getting-started/install-self-hosted/environments).
 
 If you'd like to configure a highly-available setup, there are community-contributed [Helm Charts](https://helm.sh/) and YAML files which allow Dify to be deployed on Kubernetes.
@@ -202,7 +224,6 @@ At the same time, please consider supporting Dify by sharing it on social media
 * [GitHub Issues](https://github.com/langgenius/dify/issues). Best for: bugs you encounter using Dify.AI, and feature proposals. See our [Contribution Guide](https://github.com/langgenius/dify/blob/main/CONTRIBUTING.md).
 * [Discord](https://discord.gg/FngNHpbcY7). Best for: sharing your applications and hanging out with the community.
 * [X(Twitter)](https://twitter.com/dify_ai). Best for: sharing your applications and hanging out with the community.
-* Make sure a log, if possible, is attached to an error reported to maximize solution efficiency.
 
 ## Star history
 

README_CN.md

@@ -154,7 +154,7 @@ Dify 是一个开源的 LLM 应用开发平台。其直观的界面结合了 AI
 我们提供[ Dify 云服务](https://dify.ai)，任何人都可以零设置尝试。它提供了自部署版本的所有功能，并在沙盒计划中包含 200 次免费的 GPT-4 调用。
 
 - **自托管 Dify 社区版</br>**
-使用这个[入门指南](#快速启动)快速在您的环境中运行 Dify。
+使用这个[入门指南](#quick-start)快速在您的环境中运行 Dify。
 使用我们的[文档](https://docs.dify.ai)进行进一步的参考和更深入的说明。
 
 - **面向企业/组织的 Dify</br>**
@@ -174,7 +174,7 @@ Dify 是一个开源的 LLM 应用开发平台。其直观的界面结合了 AI
 在安装 Dify 之前，请确保您的机器满足以下最低系统要求：
 
 - CPU >= 2 Core
-- RAM >= 4 GiB
+- RAM >= 4GB
 
 ### 快速启动
 

README_PT.md

@@ -1,241 +0,0 @@
-![cover-v5-optimized](https://github.com/langgenius/dify/assets/13230914/f9e19af5-61ba-4119-b926-d10c4c06ebab)
-
-<p align="center">
-  📌 <a href="https://dify.ai/blog/introducing-dify-workflow-file-upload-a-demo-on-ai-podcast">Introduzindo o Dify Workflow com Upload de Arquivo: Recrie o Podcast Google NotebookLM</a>
-</p>
-
-<p align="center">
-  <a href="https://cloud.dify.ai">Dify Cloud</a> ·
-  <a href="https://docs.dify.ai/getting-started/install-self-hosted">Auto-hospedagem</a> ·
-  <a href="https://docs.dify.ai">Documentação</a> ·
-  <a href="https://udify.app/chat/22L1zSxg6yW1cWQg">Consultas empresariais</a>
-</p>
-
-<p align="center">
-    <a href="https://dify.ai" target="_blank">
-        <img alt="Static Badge" src="https://img.shields.io/badge/Product-F04438"></a>
-    <a href="https://dify.ai/pricing" target="_blank">
-        <img alt="Static Badge" src="https://img.shields.io/badge/free-pricing?logo=free&color=%20%23155EEF&label=pricing&labelColor=%20%23528bff"></a>
-    <a href="https://discord.gg/FngNHpbcY7" target="_blank">
-        <img src="https://img.shields.io/discord/1082486657678311454?logo=discord&labelColor=%20%235462eb&logoColor=%20%23f5f5f5&color=%20%235462eb"
-            alt="chat on Discord"></a>
-    <a href="https://twitter.com/intent/follow?screen_name=dify_ai" target="_blank">
-        <img src="https://img.shields.io/twitter/follow/dify_ai?logo=X&color=%20%23f5f5f5"
-            alt="follow on X(Twitter)"></a>
-    <a href="https://hub.docker.com/u/langgenius" target="_blank">
-        <img alt="Docker Pulls" src="https://img.shields.io/docker/pulls/langgenius/dify-web?labelColor=%20%23FDB062&color=%20%23f79009"></a>
-    <a href="https://github.com/langgenius/dify/graphs/commit-activity" target="_blank">
-        <img alt="Commits last month" src="https://img.shields.io/github/commit-activity/m/langgenius/dify?labelColor=%20%2332b583&color=%20%2312b76a"></a>
-    <a href="https://github.com/langgenius/dify/" target="_blank">
-        <img alt="Issues closed" src="https://img.shields.io/github/issues-search?query=repo%3Alanggenius%2Fdify%20is%3Aclosed&label=issues%20closed&labelColor=%20%237d89b0&color=%20%235d6b98"></a>
-    <a href="https://github.com/langgenius/dify/discussions/" target="_blank">
-        <img alt="Discussion posts" src="https://img.shields.io/github/discussions/langgenius/dify?labelColor=%20%239b8afb&color=%20%237a5af8"></a>
-</p>
-
-<p align="center">
-  <a href="./README.md"><img alt="README em Inglês" src="https://img.shields.io/badge/English-d9d9d9"></a>
-  <a href="./README_CN.md"><img alt="简体中文版自述文件" src="https://img.shields.io/badge/简体中文-d9d9d9"></a>
-  <a href="./README_JA.md"><img alt="日本語のREADME" src="https://img.shields.io/badge/日本語-d9d9d9"></a>
-  <a href="./README_ES.md"><img alt="README em Espanhol" src="https://img.shields.io/badge/Español-d9d9d9"></a>
-  <a href="./README_FR.md"><img alt="README em Francês" src="https://img.shields.io/badge/Français-d9d9d9"></a>
-  <a href="./README_KL.md"><img alt="README tlhIngan Hol" src="https://img.shields.io/badge/Klingon-d9d9d9"></a>
-  <a href="./README_KR.md"><img alt="README em Coreano" src="https://img.shields.io/badge/한국어-d9d9d9"></a>
-  <a href="./README_AR.md"><img alt="README em Árabe" src="https://img.shields.io/badge/العربية-d9d9d9"></a>
-  <a href="./README_TR.md"><img alt="README em Turco" src="https://img.shields.io/badge/Türkçe-d9d9d9"></a>
-  <a href="./README_VI.md"><img alt="README em Vietnamita" src="https://img.shields.io/badge/Ti%E1%BA%BFng%20Vi%E1%BB%87t-d9d9d9"></a>
-  <a href="./README_PT.md"><img alt="README em Português - BR" src="https://img.shields.io/badge/Portugu%C3%AAs-BR?style=flat&label=BR&color=d9d9d9"></a>
-</p>
-
-Dify é uma plataforma de desenvolvimento de aplicativos LLM de código aberto. Sua interface intuitiva combina workflow de IA, pipeline RAG, capacidades de agente, gerenciamento de modelos, recursos de observabilidade e muito mais, permitindo que você vá rapidamente do protótipo à produção. Aqui está uma lista das principais funcionalidades:
-</br> </br>
-
-**1. Workflow**: 
-  Construa e teste workflows poderosos de IA em uma interface visual, aproveitando todos os recursos a seguir e muito mais.
-
-
-  https://github.com/langgenius/dify/assets/13230914/356df23e-1604-483d-80a6-9517ece318aa
-
-
-
-**2. Suporte abrangente a modelos**: 
-  Integração perfeita com centenas de LLMs proprietários e de código aberto de diversas provedoras e soluções auto-hospedadas, abrangendo GPT, Mistral, Llama3 e qualquer modelo compatível com a API da OpenAI. A lista completa de provedores suportados pode ser encontrada [aqui](https://docs.dify.ai/getting-started/readme/model-providers).
-
-![providers-v5](https://github.com/langgenius/dify/assets/13230914/5a17bdbe-097a-4100-8363-40255b70f6e3)
-
-
-**3. IDE de Prompt**: 
-  Interface intuitiva para criação de prompts, comparação de desempenho de modelos e adição de recursos como conversão de texto para fala em um aplicativo baseado em chat. 
-
-**4. Pipeline RAG**: 
-  Extensas capacidades de RAG que cobrem desde a ingestão de documentos até a recuperação, com suporte nativo para extração de texto de PDFs, PPTs e outros formatos de documentos comuns.
-
-**5. Capacidades de agente**: 
-  Você pode definir agentes com base em LLM Function Calling ou ReAct e adicionar ferramentas pré-construídas ou personalizadas para o agente. O Dify oferece mais de 50 ferramentas integradas para agentes de IA, como Google Search, DALL·E, Stable Diffusion e WolframAlpha.
-
-**6. LLMOps**: 
-  Monitore e analise os registros e o desempenho do aplicativo ao longo do tempo. É possível melhorar continuamente prompts, conjuntos de dados e modelos com base nos dados de produção e anotações.
-
-**7. Backend como Serviço**: 
-  Todas os recursos do Dify vêm com APIs correspondentes, permitindo que você integre o Dify sem esforço na lógica de negócios da sua empresa.
-
-
-## Comparação de recursos
-<table style="width: 100%;">
-  <tr>
-    <th align="center">Recurso</th>
-    <th align="center">Dify.AI</th>
-    <th align="center">LangChain</th>
-    <th align="center">Flowise</th>
-    <th align="center">OpenAI Assistants API</th>
-  </tr>
-  <tr>
-    <td align="center">Abordagem de Programação</td>
-    <td align="center">Orientada a API + Aplicativo</td>
-    <td align="center">Código Python</td>
-    <td align="center">Orientada a Aplicativo</td>
-    <td align="center">Orientada a API</td>
-  </tr>
-  <tr>
-    <td align="center">LLMs Suportados</td>
-    <td align="center">Variedade Rica</td>
-    <td align="center">Variedade Rica</td>
-    <td align="center">Variedade Rica</td>
-    <td align="center">Apenas OpenAI</td>
-  </tr>
-  <tr>
-    <td align="center">RAG Engine</td>
-    <td align="center">✅</td>
-    <td align="center">✅</td>
-    <td align="center">✅</td>
-    <td align="center">✅</td>
-  </tr>
-  <tr>
-    <td align="center">Agente</td>
-    <td align="center">✅</td>
-    <td align="center">✅</td>
-    <td align="center">❌</td>
-    <td align="center">✅</td>
-  </tr>
-  <tr>
-    <td align="center">Workflow</td>
-    <td align="center">✅</td>
-    <td align="center">❌</td>
-    <td align="center">✅</td>
-    <td align="center">❌</td>
-  </tr>
-  <tr>
-    <td align="center">Observabilidade</td>
-    <td align="center">✅</td>
-    <td align="center">✅</td>
-    <td align="center">❌</td>
-    <td align="center">❌</td>
-  </tr>
-  <tr>
-    <td align="center">Recursos Empresariais (SSO/Controle de Acesso)</td>
-    <td align="center">✅</td>
-    <td align="center">❌</td>
-    <td align="center">❌</td>
-    <td align="center">❌</td>
-  </tr>
-  <tr>
-    <td align="center">Implantação Local</td>
-    <td align="center">✅</td>
-    <td align="center">✅</td>
-    <td align="center">✅</td>
-    <td align="center">❌</td>
-  </tr>
-</table>
-
-## Usando o Dify
-
-- **Nuvem </br>**
-Oferecemos o serviço [Dify Cloud](https://dify.ai) para qualquer pessoa experimentar sem nenhuma configuração. Ele fornece todas as funcionalidades da versão auto-hospedada, incluindo 200 chamadas GPT-4 gratuitas no plano sandbox.
-
-- **Auto-hospedagem do Dify Community Edition</br>**
-Configure rapidamente o Dify no seu ambiente com este [guia inicial](#quick-start).
-Use nossa [documentação](https://docs.dify.ai) para referências adicionais e instruções mais detalhadas.
-
-- **Dify para empresas/organizações</br>**
-Oferecemos recursos adicionais voltados para empresas. [Envie suas perguntas através deste chatbot](https://udify.app/chat/22L1zSxg6yW1cWQg) ou [envie-nos um e-mail](mailto:business@dify.ai?subject=[GitHub]Business%20License%20Inquiry) para discutir necessidades empresariais. </br>
-  > Para startups e pequenas empresas que utilizam AWS, confira o [Dify Premium no AWS Marketplace](https://aws.amazon.com/marketplace/pp/prodview-t22mebxzwjhu6) e implemente no seu próprio AWS VPC com um clique. É uma oferta AMI acessível com a opção de criar aplicativos com logotipo e marca personalizados.
-
-
-## Mantendo-se atualizado
-
-Dê uma estrela no Dify no GitHub e seja notificado imediatamente sobre novos lançamentos.
-
-![star-us](https://github.com/langgenius/dify/assets/13230914/b823edc1-6388-4e25-ad45-2f6b187adbb4)
-
-
-
-## Início rápido
-> Antes de instalar o Dify, certifique-se de que sua máquina atenda aos seguintes requisitos mínimos de sistema:
-> 
->- CPU >= 2 Núcleos
->- RAM >= 4 GiB
-
-</br>
-
-A maneira mais fácil de iniciar o servidor Dify é executar nosso arquivo [docker-compose.yml](docker/docker-compose.yaml). Antes de rodar o comando de instalação, certifique-se de que o [Docker](https://docs.docker.com/get-docker/) e o [Docker Compose](https://docs.docker.com/compose/install/) estão instalados na sua máquina:
-
-```bash
-cd docker
-cp .env.example .env
-docker compose up -d
-```
-
-Após a execução, você pode acessar o painel do Dify no navegador em [http://localhost/install](http://localhost/install) e iniciar o processo de inicialização.
-
-> Se você deseja contribuir com o Dify ou fazer desenvolvimento adicional, consulte nosso [guia para implantar a partir do código fonte](https://docs.dify.ai/getting-started/install-self-hosted/local-source-code).
-
-## Próximos passos
-
-Se precisar personalizar a configuração, consulte os comentários no nosso arquivo [.env.example](docker/.env.example) e atualize os valores correspondentes no seu arquivo `.env`. Além disso, talvez seja necessário fazer ajustes no próprio arquivo `docker-compose.yaml`, como alterar versões de imagem, mapeamentos de portas ou montagens de volumes, com base no seu ambiente de implantação específico e nas suas necessidades. Após fazer quaisquer alterações, execute novamente `docker-compose up -d`. Você pode encontrar a lista completa de variáveis de ambiente disponíveis [aqui](https://docs.dify.ai/getting-started/install-self-hosted/environments).
-
-Se deseja configurar uma instalação de alta disponibilidade, há [Helm Charts](https://helm.sh/) e arquivos YAML contribuídos pela comunidade que permitem a implantação do Dify no Kubernetes.
-
-- [Helm Chart de @LeoQuote](https://github.com/douban/charts/tree/master/charts/dify)
-- [Helm Chart de @BorisPolonsky](https://github.com/BorisPolonsky/dify-helm)
-- [Arquivo YAML de @Winson-030](https://github.com/Winson-030/dify-kubernetes)
-
-#### Usando o Terraform para Implantação
-
-Implante o Dify na Plataforma Cloud com um único clique usando [terraform](https://www.terraform.io/)
-
-##### Azure Global
-- [Azure Terraform por @nikawang](https://github.com/nikawang/dify-azure-terraform)
-
-##### Google Cloud
-- [Google Cloud Terraform por @sotazum](https://github.com/DeNA/dify-google-cloud-terraform)
-
-## Contribuindo
-
-Para aqueles que desejam contribuir com código, veja nosso [Guia de Contribuição](https://github.com/langgenius/dify/blob/main/CONTRIBUTING.md). 
-Ao mesmo tempo, considere apoiar o Dify compartilhando-o nas redes sociais e em eventos e conferências.
-
-> Estamos buscando contribuidores para ajudar na tradução do Dify para idiomas além de Mandarim e Inglês. Se você tiver interesse em ajudar, consulte o [README i18n](https://github.com/langgenius/dify/blob/main/web/i18n/README.md) para mais informações e deixe-nos um comentário no canal `global-users` em nosso [Servidor da Comunidade no Discord](https://discord.gg/8Tpq4AcN9c).
-
-**Contribuidores**
-
-<a href="https://github.com/langgenius/dify/graphs/contributors">
-  <img src="https://contrib.rocks/image?repo=langgenius/dify" />
-</a>
-
-## Comunidade e contato
-
-* [Discussões no GitHub](https://github.com/langgenius/dify/discussions). Melhor para: compartilhar feedback e fazer perguntas.
-* [Problemas no GitHub](https://github.com/langgenius/dify/issues). Melhor para: relatar bugs encontrados no Dify.AI e propor novos recursos. Veja nosso [Guia de Contribuição](https://github.com/langgenius/dify/blob/main/CONTRIBUTING.md).
-* [Discord](https://discord.gg/FngNHpbcY7). Melhor para: compartilhar suas aplicações e interagir com a comunidade.
-* [X(Twitter)](https://twitter.com/dify_ai). Melhor para: compartilhar suas aplicações e interagir com a comunidade.
-
-## Histórico de estrelas
-
-[![Gráfico de Histórico de Estrelas](https://api.star-history.com/svg?repos=langgenius/dify&type=Date)](https://star-history.com/#langgenius/dify&Date)
-
-## Divulgação de segurança
-
-Para proteger sua privacidade, evite postar problemas de segurança no GitHub. Em vez disso, envie suas perguntas para security@dify.ai e forneceremos uma resposta mais detalhada.
-
-## Licença
-
-Este repositório está disponível sob a [Licença de Código Aberto Dify](LICENSE), que é essencialmente Apache 2.0 com algumas restrições adicionais.

api/.env.example

@@ -31,17 +31,8 @@ REDIS_HOST=localhost
 REDIS_PORT=6379
 REDIS_USERNAME=
 REDIS_PASSWORD=difyai123456
-REDIS_USE_SSL=false
 REDIS_DB=0
 
-# redis Sentinel configuration.
-REDIS_USE_SENTINEL=false
-REDIS_SENTINELS=
-REDIS_SENTINEL_SERVICE_NAME=
-REDIS_SENTINEL_USERNAME=
-REDIS_SENTINEL_PASSWORD=
-REDIS_SENTINEL_SOCKET_TIMEOUT=0.1
-
 # PostgreSQL database configuration
 DB_USERNAME=postgres
 DB_PASSWORD=difyai123456
@@ -51,7 +42,7 @@ DB_DATABASE=dify
 
 # Storage configuration
 # use for store upload files, private keys...
-# storage type: local, s3, aliyun-oss, azure-blob, baidu-obs, google-storage, huawei-obs, oci-storage, tencent-cos, volcengine-tos, supabase
+# storage type: local, s3, azure-blob, google-storage, tencent-cos, huawei-obs, volcengine-tos, baidu-obs, supabase
 STORAGE_TYPE=local
 STORAGE_LOCAL_PATH=storage
 S3_USE_AWS_MANAGED_IAM=false
@@ -120,8 +111,7 @@ SUPABASE_URL=your-server-url
 WEB_API_CORS_ALLOW_ORIGINS=http://127.0.0.1:3000,*
 CONSOLE_CORS_ALLOW_ORIGINS=http://127.0.0.1:3000,*
 
-
-# Vector database configuration, support: weaviate, qdrant, milvus, myscale, relyt, pgvecto_rs, pgvector, pgvector, chroma, opensearch, tidb_vector, couchbase, vikingdb, upstash, lindorm
+# Vector database configuration, support: weaviate, qdrant, milvus, myscale, relyt, pgvecto_rs, pgvector, pgvector, chroma, opensearch, tidb_vector, vikingdb
 VECTOR_STORE=weaviate
 
 # Weaviate configuration
@@ -137,13 +127,6 @@ QDRANT_CLIENT_TIMEOUT=20
 QDRANT_GRPC_ENABLED=false
 QDRANT_GRPC_PORT=6334
 
-#Couchbase configuration
-COUCHBASE_CONNECTION_STRING=127.0.0.1
-COUCHBASE_USER=Administrator
-COUCHBASE_PASSWORD=password
-COUCHBASE_BUCKET_NAME=Embeddings
-COUCHBASE_SCOPE_NAME=_default
-
 # Milvus configuration
 MILVUS_URI=http://127.0.0.1:19530
 MILVUS_TOKEN=
@@ -203,20 +186,6 @@ TIDB_VECTOR_USER=xxx.root
 TIDB_VECTOR_PASSWORD=xxxxxx
 TIDB_VECTOR_DATABASE=dify
 
-# Tidb on qdrant configuration
-TIDB_ON_QDRANT_URL=http://127.0.0.1
-TIDB_ON_QDRANT_API_KEY=dify
-TIDB_ON_QDRANT_CLIENT_TIMEOUT=20
-TIDB_ON_QDRANT_GRPC_ENABLED=false
-TIDB_ON_QDRANT_GRPC_PORT=6334
-TIDB_PUBLIC_KEY=dify
-TIDB_PRIVATE_KEY=dify
-TIDB_API_URL=http://127.0.0.1
-TIDB_IAM_API_URL=http://127.0.0.1
-TIDB_REGION=regions/aws-us-east-1
-TIDB_PROJECT_ID=dify
-TIDB_SPEND_LIMIT=100
-
 # Chroma configuration
 CHROMA_HOST=127.0.0.1
 CHROMA_PORT=8000
@@ -251,10 +220,6 @@ BAIDU_VECTOR_DB_DATABASE=dify
 BAIDU_VECTOR_DB_SHARD=1
 BAIDU_VECTOR_DB_REPLICAS=3
 
-# Upstash configuration
-UPSTASH_VECTOR_URL=your-server-url
-UPSTASH_VECTOR_TOKEN=your-access-token
-
 # ViKingDB configuration
 VIKINGDB_ACCESS_KEY=your-ak
 VIKINGDB_SECRET_KEY=your-sk
@@ -264,20 +229,6 @@ VIKINGDB_SCHEMA=http
 VIKINGDB_CONNECTION_TIMEOUT=30
 VIKINGDB_SOCKET_TIMEOUT=30
 
-# Lindorm configuration
-LINDORM_URL=http://ld-*******************-proxy-search-pub.lindorm.aliyuncs.com:30070
-LINDORM_USERNAME=admin
-LINDORM_PASSWORD=admin
-
-# OceanBase Vector configuration
-OCEANBASE_VECTOR_HOST=127.0.0.1
-OCEANBASE_VECTOR_PORT=2881
-OCEANBASE_VECTOR_USER=root@test
-OCEANBASE_VECTOR_PASSWORD=
-OCEANBASE_VECTOR_DATABASE=test
-OCEANBASE_MEMORY_LIMIT=6G
-
-
 # Upload configuration
 UPLOAD_FILE_SIZE_LIMIT=15
 UPLOAD_FILE_BATCH_LIMIT=5
@@ -288,7 +239,6 @@ UPLOAD_AUDIO_FILE_SIZE_LIMIT=50
 # Model Configuration
 MULTIMODAL_SEND_IMAGE_FORMAT=base64
 PROMPT_GENERATION_MAX_TOKENS=512
-CODE_GENERATION_MAX_TOKENS=1024
 
 # Mail configuration, support: resend, smtp
 MAIL_TYPE=
@@ -327,9 +277,6 @@ SSRF_DEFAULT_MAX_RETRIES=3
 BATCH_UPLOAD_LIMIT=10
 KEYWORD_DATA_SOURCE_TYPE=database
 
-# Workflow file upload limit
-WORKFLOW_FILE_UPLOAD_LIMIT=10
-
 # CODE EXECUTION CONFIGURATION
 CODE_EXECUTION_ENDPOINT=http://127.0.0.1:8194
 CODE_EXECUTION_API_KEY=dify-sandbox
@@ -357,10 +304,6 @@ RESPECT_XFORWARD_HEADERS_ENABLED=false
 
 # Log file path
 LOG_FILE=
-# Log file max size, the unit is MB
-LOG_FILE_MAX_SIZE=20
-# Log file max backup count
-LOG_FILE_BACKUP_COUNT=5
 
 # Indexing configuration
 INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH=1000
@@ -387,6 +330,3 @@ POSITION_TOOL_EXCLUDES=
 POSITION_PROVIDER_PINS=
 POSITION_PROVIDER_INCLUDES=
 POSITION_PROVIDER_EXCLUDES=
-
-# Reset password token expiry minutes
-RESET_PASSWORD_TOKEN_EXPIRY_MINUTES=5

api/Dockerfile

@@ -55,9 +55,7 @@ RUN apt-get update \
     && echo "deb http://deb.debian.org/debian testing main" > /etc/apt/sources.list \
     && apt-get update \
     # For Security
-    && apt-get install -y --no-install-recommends expat=2.6.3-2 libldap-2.5-0=2.5.18+dfsg-3+b1 perl=5.40.0-6 libsqlite3-0=3.46.1-1 zlib1g=1:1.3.dfsg+really1.3.1-1+b1 \
-    # install a chinese font to support the use of tools like matplotlib
-    && apt-get install -y fonts-noto-cjk \
+    && apt-get install -y --no-install-recommends zlib1g=1:1.3.dfsg+really1.3.1-1 expat=2.6.3-1 libldap-2.5-0=2.5.18+dfsg-3 perl=5.38.2-5 libsqlite3-0=3.46.0-1 \
     && apt-get autoremove -y \
     && rm -rf /var/lib/apt/lists/*
 

api/README.md

@@ -85,4 +85,3 @@
    cd ../
    poetry run -C api bash dev/pytest/pytest_all_tests.sh
    ```
-

api/app.py

@@ -1,7 +1,5 @@
 import os
 
-from configs import dify_config
-
 if os.environ.get("DEBUG", "false").lower() != "true":
     from gevent import monkey
 
@@ -12,20 +10,44 @@ if os.environ.get("DEBUG", "false").lower() != "true":
     grpc.experimental.gevent.init_gevent()
 
 import json
+import logging
+import sys
 import threading
 import time
 import warnings
+from logging.handlers import RotatingFileHandler
 
-from flask import Response
+from flask import Flask, Response, request
+from flask_cors import CORS
+from werkzeug.exceptions import Unauthorized
 
-from app_factory import create_app
+import contexts
+from commands import register_commands
+from configs import dify_config
 
 # DO NOT REMOVE BELOW
 from events import event_handlers  # noqa: F401
+from extensions import (
+    ext_celery,
+    ext_code_based_extension,
+    ext_compress,
+    ext_database,
+    ext_hosting_provider,
+    ext_login,
+    ext_mail,
+    ext_migrate,
+    ext_proxy_fix,
+    ext_redis,
+    ext_sentry,
+    ext_storage,
+)
 from extensions.ext_database import db
+from extensions.ext_login import login_manager
+from libs.passport import PassportService
 
 # TODO: Find a way to avoid importing models here
 from models import account, dataset, model, source, task, tool, tools, web  # noqa: F401
+from services.account_service import AccountService
 
 # DO NOT REMOVE ABOVE
 
@@ -38,11 +60,194 @@ if hasattr(time, "tzset"):
     time.tzset()
 
 
+class DifyApp(Flask):
+    pass
+
+
+# -------------
+# Configuration
+# -------------
+
+
+config_type = os.getenv("EDITION", default="SELF_HOSTED")  # ce edition first
+
+
+# ----------------------------
+# Application Factory Function
+# ----------------------------
+
+
+def create_flask_app_with_configs() -> Flask:
+    """
+    create a raw flask app
+    with configs loaded from .env file
+    """
+    dify_app = DifyApp(__name__)
+    dify_app.config.from_mapping(dify_config.model_dump())
+
+    # populate configs into system environment variables
+    for key, value in dify_app.config.items():
+        if isinstance(value, str):
+            os.environ[key] = value
+        elif isinstance(value, int | float | bool):
+            os.environ[key] = str(value)
+        elif value is None:
+            os.environ[key] = ""
+
+    return dify_app
+
+
+def create_app() -> Flask:
+    app = create_flask_app_with_configs()
+
+    app.secret_key = app.config["SECRET_KEY"]
+
+    log_handlers = None
+    log_file = app.config.get("LOG_FILE")
+    if log_file:
+        log_dir = os.path.dirname(log_file)
+        os.makedirs(log_dir, exist_ok=True)
+        log_handlers = [
+            RotatingFileHandler(
+                filename=log_file,
+                maxBytes=1024 * 1024 * 1024,
+                backupCount=5,
+            ),
+            logging.StreamHandler(sys.stdout),
+        ]
+
+    logging.basicConfig(
+        level=app.config.get("LOG_LEVEL"),
+        format=app.config["LOG_FORMAT"],
+        datefmt=app.config.get("LOG_DATEFORMAT"),
+        handlers=log_handlers,
+        force=True,
+    )
+    log_tz = app.config.get("LOG_TZ")
+    if log_tz:
+        from datetime import datetime
+
+        import pytz
+
+        timezone = pytz.timezone(log_tz)
+
+        def time_converter(seconds):
+            return datetime.utcfromtimestamp(seconds).astimezone(timezone).timetuple()
+
+        for handler in logging.root.handlers:
+            assert handler.formatter
+            handler.formatter.converter = time_converter
+    initialize_extensions(app)
+    register_blueprints(app)
+    register_commands(app)
+
+    return app
+
+
+def initialize_extensions(app):
+    # Since the application instance is now created, pass it to each Flask
+    # extension instance to bind it to the Flask application instance (app)
+    ext_compress.init_app(app)
+    ext_code_based_extension.init()
+    ext_database.init_app(app)
+    ext_migrate.init(app, db)
+    ext_redis.init_app(app)
+    ext_storage.init_app(app)
+    ext_celery.init_app(app)
+    ext_login.init_app(app)
+    ext_mail.init_app(app)
+    ext_hosting_provider.init_app(app)
+    ext_sentry.init_app(app)
+    ext_proxy_fix.init_app(app)
+
+
+# Flask-Login configuration
+@login_manager.request_loader
+def load_user_from_request(request_from_flask_login):
+    """Load user based on the request."""
+    if request.blueprint not in {"console", "inner_api"}:
+        return None
+    # Check if the user_id contains a dot, indicating the old format
+    auth_header = request.headers.get("Authorization", "")
+    if not auth_header:
+        auth_token = request.args.get("_token")
+        if not auth_token:
+            raise Unauthorized("Invalid Authorization token.")
+    else:
+        if " " not in auth_header:
+            raise Unauthorized("Invalid Authorization header format. Expected 'Bearer <api-key>' format.")
+        auth_scheme, auth_token = auth_header.split(None, 1)
+        auth_scheme = auth_scheme.lower()
+        if auth_scheme != "bearer":
+            raise Unauthorized("Invalid Authorization header format. Expected 'Bearer <api-key>' format.")
+
+    decoded = PassportService().verify(auth_token)
+    user_id = decoded.get("user_id")
+
+    logged_in_account = AccountService.load_logged_in_account(account_id=user_id)
+    if logged_in_account:
+        contexts.tenant_id.set(logged_in_account.current_tenant_id)
+    return logged_in_account
+
+
+@login_manager.unauthorized_handler
+def unauthorized_handler():
+    """Handle unauthorized requests."""
+    return Response(
+        json.dumps({"code": "unauthorized", "message": "Unauthorized."}),
+        status=401,
+        content_type="application/json",
+    )
+
+
+# register blueprint routers
+def register_blueprints(app):
+    from controllers.console import bp as console_app_bp
+    from controllers.files import bp as files_bp
+    from controllers.inner_api import bp as inner_api_bp
+    from controllers.service_api import bp as service_api_bp
+    from controllers.web import bp as web_bp
+
+    CORS(
+        service_api_bp,
+        allow_headers=["Content-Type", "Authorization", "X-App-Code"],
+        methods=["GET", "PUT", "POST", "DELETE", "OPTIONS", "PATCH"],
+    )
+    app.register_blueprint(service_api_bp)
+
+    CORS(
+        web_bp,
+        resources={r"/*": {"origins": app.config["WEB_API_CORS_ALLOW_ORIGINS"]}},
+        supports_credentials=True,
+        allow_headers=["Content-Type", "Authorization", "X-App-Code"],
+        methods=["GET", "PUT", "POST", "DELETE", "OPTIONS", "PATCH"],
+        expose_headers=["X-Version", "X-Env"],
+    )
+
+    app.register_blueprint(web_bp)
+
+    CORS(
+        console_app_bp,
+        resources={r"/*": {"origins": app.config["CONSOLE_CORS_ALLOW_ORIGINS"]}},
+        supports_credentials=True,
+        allow_headers=["Content-Type", "Authorization"],
+        methods=["GET", "PUT", "POST", "DELETE", "OPTIONS", "PATCH"],
+        expose_headers=["X-Version", "X-Env"],
+    )
+
+    app.register_blueprint(console_app_bp)
+
+    CORS(files_bp, allow_headers=["Content-Type"], methods=["GET", "PUT", "POST", "DELETE", "OPTIONS", "PATCH"])
+    app.register_blueprint(files_bp)
+
+    app.register_blueprint(inner_api_bp)
+
+
 # create app
 app = create_app()
 celery = app.extensions["celery"]
 
-if dify_config.TESTING:
+if app.config.get("TESTING"):
     print("App is running in TESTING mode")
 
 
@@ -50,15 +255,15 @@ if dify_config.TESTING:
 def after_request(response):
     """Add Version headers to the response."""
     response.set_cookie("remember_token", "", expires=0)
-    response.headers.add("X-Version", dify_config.CURRENT_VERSION)
-    response.headers.add("X-Env", dify_config.DEPLOY_ENV)
+    response.headers.add("X-Version", app.config["CURRENT_VERSION"])
+    response.headers.add("X-Env", app.config["DEPLOY_ENV"])
     return response
 
 
 @app.route("/health")
 def health():
     return Response(
-        json.dumps({"pid": os.getpid(), "status": "ok", "version": dify_config.CURRENT_VERSION}),
+        json.dumps({"pid": os.getpid(), "status": "ok", "version": app.config["CURRENT_VERSION"]}),
         status=200,
         content_type="application/json",
     )

api/app_factory.py

@@ -1,176 +0,0 @@
-import os
-
-if os.environ.get("DEBUG", "false").lower() != "true":
-    from gevent import monkey
-
-    monkey.patch_all()
-
-    import grpc.experimental.gevent
-
-    grpc.experimental.gevent.init_gevent()
-
-import json
-
-from flask import Flask, Response, request
-from flask_cors import CORS
-from werkzeug.exceptions import Unauthorized
-
-import contexts
-from commands import register_commands
-from configs import dify_config
-from extensions import (
-    ext_celery,
-    ext_code_based_extension,
-    ext_compress,
-    ext_database,
-    ext_hosting_provider,
-    ext_logging,
-    ext_login,
-    ext_mail,
-    ext_migrate,
-    ext_proxy_fix,
-    ext_redis,
-    ext_sentry,
-    ext_storage,
-)
-from extensions.ext_database import db
-from extensions.ext_login import login_manager
-from libs.passport import PassportService
-from services.account_service import AccountService
-
-
-class DifyApp(Flask):
-    pass
-
-
-# ----------------------------
-# Application Factory Function
-# ----------------------------
-def create_flask_app_with_configs() -> Flask:
-    """
-    create a raw flask app
-    with configs loaded from .env file
-    """
-    dify_app = DifyApp(__name__)
-    dify_app.config.from_mapping(dify_config.model_dump())
-
-    # populate configs into system environment variables
-    for key, value in dify_app.config.items():
-        if isinstance(value, str):
-            os.environ[key] = value
-        elif isinstance(value, int | float | bool):
-            os.environ[key] = str(value)
-        elif value is None:
-            os.environ[key] = ""
-
-    return dify_app
-
-
-def create_app() -> Flask:
-    app = create_flask_app_with_configs()
-    app.secret_key = dify_config.SECRET_KEY
-    initialize_extensions(app)
-    register_blueprints(app)
-    register_commands(app)
-
-    return app
-
-
-def initialize_extensions(app):
-    # Since the application instance is now created, pass it to each Flask
-    # extension instance to bind it to the Flask application instance (app)
-    ext_logging.init_app(app)
-    ext_compress.init_app(app)
-    ext_code_based_extension.init()
-    ext_database.init_app(app)
-    ext_migrate.init(app, db)
-    ext_redis.init_app(app)
-    ext_storage.init_app(app)
-    ext_celery.init_app(app)
-    ext_login.init_app(app)
-    ext_mail.init_app(app)
-    ext_hosting_provider.init_app(app)
-    ext_sentry.init_app(app)
-    ext_proxy_fix.init_app(app)
-
-
-# Flask-Login configuration
-@login_manager.request_loader
-def load_user_from_request(request_from_flask_login):
-    """Load user based on the request."""
-    if request.blueprint not in {"console", "inner_api"}:
-        return None
-    # Check if the user_id contains a dot, indicating the old format
-    auth_header = request.headers.get("Authorization", "")
-    if not auth_header:
-        auth_token = request.args.get("_token")
-        if not auth_token:
-            raise Unauthorized("Invalid Authorization token.")
-    else:
-        if " " not in auth_header:
-            raise Unauthorized("Invalid Authorization header format. Expected 'Bearer <api-key>' format.")
-        auth_scheme, auth_token = auth_header.split(None, 1)
-        auth_scheme = auth_scheme.lower()
-        if auth_scheme != "bearer":
-            raise Unauthorized("Invalid Authorization header format. Expected 'Bearer <api-key>' format.")
-
-    decoded = PassportService().verify(auth_token)
-    user_id = decoded.get("user_id")
-
-    logged_in_account = AccountService.load_logged_in_account(account_id=user_id)
-    if logged_in_account:
-        contexts.tenant_id.set(logged_in_account.current_tenant_id)
-    return logged_in_account
-
-
-@login_manager.unauthorized_handler
-def unauthorized_handler():
-    """Handle unauthorized requests."""
-    return Response(
-        json.dumps({"code": "unauthorized", "message": "Unauthorized."}),
-        status=401,
-        content_type="application/json",
-    )
-
-
-# register blueprint routers
-def register_blueprints(app):
-    from controllers.console import bp as console_app_bp
-    from controllers.files import bp as files_bp
-    from controllers.inner_api import bp as inner_api_bp
-    from controllers.service_api import bp as service_api_bp
-    from controllers.web import bp as web_bp
-
-    CORS(
-        service_api_bp,
-        allow_headers=["Content-Type", "Authorization", "X-App-Code"],
-        methods=["GET", "PUT", "POST", "DELETE", "OPTIONS", "PATCH"],
-    )
-    app.register_blueprint(service_api_bp)
-
-    CORS(
-        web_bp,
-        resources={r"/*": {"origins": dify_config.WEB_API_CORS_ALLOW_ORIGINS}},
-        supports_credentials=True,
-        allow_headers=["Content-Type", "Authorization", "X-App-Code"],
-        methods=["GET", "PUT", "POST", "DELETE", "OPTIONS", "PATCH"],
-        expose_headers=["X-Version", "X-Env"],
-    )
-
-    app.register_blueprint(web_bp)
-
-    CORS(
-        console_app_bp,
-        resources={r"/*": {"origins": dify_config.CONSOLE_CORS_ALLOW_ORIGINS}},
-        supports_credentials=True,
-        allow_headers=["Content-Type", "Authorization"],
-        methods=["GET", "PUT", "POST", "DELETE", "OPTIONS", "PATCH"],
-        expose_headers=["X-Version", "X-Env"],
-    )
-
-    app.register_blueprint(console_app_bp)
-
-    CORS(files_bp, allow_headers=["Content-Type"], methods=["GET", "PUT", "POST", "DELETE", "OPTIONS", "PATCH"])
-    app.register_blueprint(files_bp)
-
-    app.register_blueprint(inner_api_bp)

api/commands.py

@@ -259,28 +259,6 @@ def migrate_knowledge_vector_database():
     skipped_count = 0
     total_count = 0
     vector_type = dify_config.VECTOR_STORE
-    upper_colletion_vector_types = {
-        VectorType.MILVUS,
-        VectorType.PGVECTOR,
-        VectorType.RELYT,
-        VectorType.WEAVIATE,
-        VectorType.ORACLE,
-        VectorType.ELASTICSEARCH,
-    }
-    lower_colletion_vector_types = {
-        VectorType.ANALYTICDB,
-        VectorType.CHROMA,
-        VectorType.MYSCALE,
-        VectorType.PGVECTO_RS,
-        VectorType.TIDB_VECTOR,
-        VectorType.OPENSEARCH,
-        VectorType.TENCENT,
-        VectorType.BAIDU,
-        VectorType.VIKINGDB,
-        VectorType.UPSTASH,
-        VectorType.COUCHBASE,
-        VectorType.OCEANBASE,
-    }
     page = 1
     while True:
         try:
@@ -306,9 +284,11 @@ def migrate_knowledge_vector_database():
                         skipped_count = skipped_count + 1
                         continue
                 collection_name = ""
-                dataset_id = dataset.id
-                if vector_type in upper_colletion_vector_types:
+                if vector_type == VectorType.WEAVIATE:
+                    dataset_id = dataset.id
                     collection_name = Dataset.gen_collection_name_by_id(dataset_id)
+                    index_struct_dict = {"type": VectorType.WEAVIATE, "vector_store": {"class_prefix": collection_name}}
+                    dataset.index_struct = json.dumps(index_struct_dict)
                 elif vector_type == VectorType.QDRANT:
                     if dataset.collection_binding_id:
                         dataset_collection_binding = (
@@ -321,15 +301,63 @@ def migrate_knowledge_vector_database():
                         else:
                             raise ValueError("Dataset Collection Binding not found")
                     else:
+                        dataset_id = dataset.id
                         collection_name = Dataset.gen_collection_name_by_id(dataset_id)
+                    index_struct_dict = {"type": VectorType.QDRANT, "vector_store": {"class_prefix": collection_name}}
+                    dataset.index_struct = json.dumps(index_struct_dict)
 
-                elif vector_type in lower_colletion_vector_types:
-                    collection_name = Dataset.gen_collection_name_by_id(dataset_id).lower()
+                elif vector_type == VectorType.MILVUS:
+                    dataset_id = dataset.id
+                    collection_name = Dataset.gen_collection_name_by_id(dataset_id)
+                    index_struct_dict = {"type": VectorType.MILVUS, "vector_store": {"class_prefix": collection_name}}
+                    dataset.index_struct = json.dumps(index_struct_dict)
+                elif vector_type == VectorType.RELYT:
+                    dataset_id = dataset.id
+                    collection_name = Dataset.gen_collection_name_by_id(dataset_id)
+                    index_struct_dict = {"type": "relyt", "vector_store": {"class_prefix": collection_name}}
+                    dataset.index_struct = json.dumps(index_struct_dict)
+                elif vector_type == VectorType.TENCENT:
+                    dataset_id = dataset.id
+                    collection_name = Dataset.gen_collection_name_by_id(dataset_id)
+                    index_struct_dict = {"type": VectorType.TENCENT, "vector_store": {"class_prefix": collection_name}}
+                    dataset.index_struct = json.dumps(index_struct_dict)
+                elif vector_type == VectorType.PGVECTOR:
+                    dataset_id = dataset.id
+                    collection_name = Dataset.gen_collection_name_by_id(dataset_id)
+                    index_struct_dict = {"type": VectorType.PGVECTOR, "vector_store": {"class_prefix": collection_name}}
+                    dataset.index_struct = json.dumps(index_struct_dict)
+                elif vector_type == VectorType.OPENSEARCH:
+                    dataset_id = dataset.id
+                    collection_name = Dataset.gen_collection_name_by_id(dataset_id)
+                    index_struct_dict = {
+                        "type": VectorType.OPENSEARCH,
+                        "vector_store": {"class_prefix": collection_name},
+                    }
+                    dataset.index_struct = json.dumps(index_struct_dict)
+                elif vector_type == VectorType.ANALYTICDB:
+                    dataset_id = dataset.id
+                    collection_name = Dataset.gen_collection_name_by_id(dataset_id)
+                    index_struct_dict = {
+                        "type": VectorType.ANALYTICDB,
+                        "vector_store": {"class_prefix": collection_name},
+                    }
+                    dataset.index_struct = json.dumps(index_struct_dict)
+                elif vector_type == VectorType.ELASTICSEARCH:
+                    dataset_id = dataset.id
+                    index_name = Dataset.gen_collection_name_by_id(dataset_id)
+                    index_struct_dict = {"type": "elasticsearch", "vector_store": {"class_prefix": index_name}}
+                    dataset.index_struct = json.dumps(index_struct_dict)
+                elif vector_type == VectorType.BAIDU:
+                    dataset_id = dataset.id
+                    collection_name = Dataset.gen_collection_name_by_id(dataset_id)
+                    index_struct_dict = {
+                        "type": VectorType.BAIDU,
+                        "vector_store": {"class_prefix": collection_name},
+                    }
+                    dataset.index_struct = json.dumps(index_struct_dict)
                 else:
                     raise ValueError(f"Vector store {vector_type} is not supported.")
 
-                index_struct_dict = {"type": vector_type, "vector_store": {"class_prefix": collection_name}}
-                dataset.index_struct = json.dumps(index_struct_dict)
                 vector = Vector(dataset)
                 click.echo(f"Migrating dataset {dataset.id}.")
 

api/configs/feature/__init__.py

@@ -1,15 +1,6 @@
 from typing import Annotated, Literal, Optional
 
-from pydantic import (
-    AliasChoices,
-    Field,
-    HttpUrl,
-    NegativeInt,
-    NonNegativeInt,
-    PositiveFloat,
-    PositiveInt,
-    computed_field,
-)
+from pydantic import AliasChoices, Field, HttpUrl, NegativeInt, NonNegativeInt, PositiveInt, computed_field
 from pydantic_settings import BaseSettings
 
 from configs.feature.hosted_service import HostedServiceConfig
@@ -27,24 +18,9 @@ class SecurityConfig(BaseSettings):
         default="",
     )
 
-    RESET_PASSWORD_TOKEN_EXPIRY_MINUTES: PositiveInt = Field(
-        description="Duration in minutes for which a password reset token remains valid",
-        default=5,
-    )
-
-    LOGIN_DISABLED: bool = Field(
-        description="Whether to disable login checks",
-        default=False,
-    )
-
-    ADMIN_API_KEY_ENABLE: bool = Field(
-        description="Whether to enable admin api key for authentication",
-        default=False,
-    )
-
-    ADMIN_API_KEY: Optional[str] = Field(
-        description="admin api key for authentication",
-        default=None,
+    RESET_PASSWORD_TOKEN_EXPIRY_HOURS: PositiveInt = Field(
+        description="Duration in hours for which a password reset token remains valid",
+        default=24,
     )
 
 
@@ -216,11 +192,6 @@ class FileUploadConfig(BaseSettings):
         default=20,
     )
 
-    WORKFLOW_FILE_UPLOAD_LIMIT: PositiveInt = Field(
-        description="Maximum number of files allowed in a workflow upload operation",
-        default=10,
-    )
-
 
 class HttpConfig(BaseSettings):
     """
@@ -324,16 +295,6 @@ class LoggingConfig(BaseSettings):
         default=None,
     )
 
-    LOG_FILE_MAX_SIZE: PositiveInt = Field(
-        description="Maximum file size for file rotation retention, the unit is megabytes (MB)",
-        default=20,
-    )
-
-    LOG_FILE_BACKUP_COUNT: PositiveInt = Field(
-        description="Maximum file backup count file rotation retention",
-        default=5,
-    )
-
     LOG_FORMAT: str = Field(
         description="Format string for log messages",
         default="%(asctime)s.%(msecs)03d %(levelname)s [%(threadName)s] [%(filename)s:%(lineno)d] - %(message)s",
@@ -522,11 +483,6 @@ class MailConfig(BaseSettings):
         default=False,
     )
 
-    EMAIL_SEND_IP_LIMIT_PER_MINUTE: PositiveInt = Field(
-        description="Maximum number of emails allowed to be sent from the same IP address in a minute",
-        default=50,
-    )
-
 
 class RagEtlConfig(BaseSettings):
     """
@@ -561,26 +517,16 @@ class DataSetConfig(BaseSettings):
     Configuration for dataset management
     """
 
-    PLAN_SANDBOX_CLEAN_DAY_SETTING: PositiveInt = Field(
-        description="Interval in days for dataset cleanup operations - plan: sandbox",
+    CLEAN_DAY_SETTING: PositiveInt = Field(
+        description="Interval in days for dataset cleanup operations",
         default=30,
     )
 
-    PLAN_PRO_CLEAN_DAY_SETTING: PositiveInt = Field(
-        description="Interval in days for dataset cleanup operations - plan: pro and team",
-        default=7,
-    )
-
     DATASET_OPERATOR_ENABLED: bool = Field(
         description="Enable or disable dataset operator functionality",
         default=False,
     )
 
-    TIDB_SERVERLESS_NUMBER: PositiveInt = Field(
-        description="number of tidb serverless cluster",
-        default=500,
-    )
-
 
 class WorkspaceConfig(BaseSettings):
     """
@@ -674,33 +620,6 @@ class PositionConfig(BaseSettings):
         return {item.strip() for item in self.POSITION_TOOL_EXCLUDES.split(",") if item.strip() != ""}
 
 
-class LoginConfig(BaseSettings):
-    ENABLE_EMAIL_CODE_LOGIN: bool = Field(
-        description="whether to enable email code login",
-        default=False,
-    )
-    ENABLE_EMAIL_PASSWORD_LOGIN: bool = Field(
-        description="whether to enable email password login",
-        default=True,
-    )
-    ENABLE_SOCIAL_OAUTH_LOGIN: bool = Field(
-        description="whether to enable github/google oauth login",
-        default=False,
-    )
-    EMAIL_CODE_LOGIN_TOKEN_EXPIRY_MINUTES: PositiveInt = Field(
-        description="expiry time in minutes for email code login token",
-        default=5,
-    )
-    ALLOW_REGISTER: bool = Field(
-        description="whether to enable register",
-        default=False,
-    )
-    ALLOW_CREATE_WORKSPACE: bool = Field(
-        description="whether to enable create workspace",
-        default=False,
-    )
-
-
 class FeatureConfig(
     # place the configs in alphabet order
     AppExecutionConfig,
@@ -726,7 +645,6 @@ class FeatureConfig(
     UpdateConfig,
     WorkflowConfig,
     WorkspaceConfig,
-    LoginConfig,
     # hosted services config
     HostedServiceConfig,
     CeleryBeatConfig,

api/configs/middleware/__init__.py

@@ -16,14 +16,10 @@ from configs.middleware.storage.supabase_storage_config import SupabaseStorageCo
 from configs.middleware.storage.tencent_cos_storage_config import TencentCloudCOSStorageConfig
 from configs.middleware.storage.volcengine_tos_storage_config import VolcengineTOSStorageConfig
 from configs.middleware.vdb.analyticdb_config import AnalyticdbConfig
-from configs.middleware.vdb.baidu_vector_config import BaiduVectorDBConfig
 from configs.middleware.vdb.chroma_config import ChromaConfig
-from configs.middleware.vdb.couchbase_config import CouchbaseConfig
 from configs.middleware.vdb.elasticsearch_config import ElasticsearchConfig
-from configs.middleware.vdb.lindorm_config import LindormConfig
 from configs.middleware.vdb.milvus_config import MilvusConfig
 from configs.middleware.vdb.myscale_config import MyScaleConfig
-from configs.middleware.vdb.oceanbase_config import OceanBaseVectorConfig
 from configs.middleware.vdb.opensearch_config import OpenSearchConfig
 from configs.middleware.vdb.oracle_config import OracleConfig
 from configs.middleware.vdb.pgvector_config import PGVectorConfig
@@ -31,9 +27,7 @@ from configs.middleware.vdb.pgvectors_config import PGVectoRSConfig
 from configs.middleware.vdb.qdrant_config import QdrantConfig
 from configs.middleware.vdb.relyt_config import RelytConfig
 from configs.middleware.vdb.tencent_vector_config import TencentVectorDBConfig
-from configs.middleware.vdb.tidb_on_qdrant_config import TidbOnQdrantConfig
 from configs.middleware.vdb.tidb_vector_config import TiDBVectorConfig
-from configs.middleware.vdb.upstash_config import UpstashConfig
 from configs.middleware.vdb.vikingdb_config import VikingDBConfig
 from configs.middleware.vdb.weaviate_config import WeaviateConfig
 
@@ -41,8 +35,7 @@ from configs.middleware.vdb.weaviate_config import WeaviateConfig
 class StorageConfig(BaseSettings):
     STORAGE_TYPE: str = Field(
         description="Type of storage to use."
-        " Options: 'local', 's3', 'aliyun-oss', 'azure-blob', 'baidu-obs', 'google-storage', 'huawei-obs', "
-        "'oci-storage', 'tencent-cos', 'volcengine-tos', 'supabase'. Default is 'local'.",
+        " Options: 'local', 's3', 'azure-blob', 'aliyun-oss', 'google-storage'. Default is 'local'.",
         default="local",
     )
 
@@ -59,11 +52,6 @@ class VectorStoreConfig(BaseSettings):
         default=None,
     )
 
-    VECTOR_STORE_WHITELIST_ENABLE: Optional[bool] = Field(
-        description="Enable whitelist for vector store.",
-        default=False,
-    )
-
 
 class KeywordStoreConfig(BaseSettings):
     KEYWORD_STORE: str = Field(
@@ -255,13 +243,7 @@ class MiddlewareConfig(
     TiDBVectorConfig,
     WeaviateConfig,
     ElasticsearchConfig,
-    CouchbaseConfig,
     InternalTestConfig,
     VikingDBConfig,
-    UpstashConfig,
-    TidbOnQdrantConfig,
-    LindormConfig,
-    OceanBaseVectorConfig,
-    BaiduVectorDBConfig,
 ):
     pass

api/configs/middleware/vdb/couchbase_config.py

@@ -1,34 +0,0 @@
-from typing import Optional
-
-from pydantic import BaseModel, Field
-
-
-class CouchbaseConfig(BaseModel):
-    """
-    Couchbase configs
-    """
-
-    COUCHBASE_CONNECTION_STRING: Optional[str] = Field(
-        description="COUCHBASE connection string",
-        default=None,
-    )
-
-    COUCHBASE_USER: Optional[str] = Field(
-        description="COUCHBASE user",
-        default=None,
-    )
-
-    COUCHBASE_PASSWORD: Optional[str] = Field(
-        description="COUCHBASE password",
-        default=None,
-    )
-
-    COUCHBASE_BUCKET_NAME: Optional[str] = Field(
-        description="COUCHBASE bucket name",
-        default=None,
-    )
-
-    COUCHBASE_SCOPE_NAME: Optional[str] = Field(
-        description="COUCHBASE scope name",
-        default=None,
-    )

api/configs/middleware/vdb/lindorm_config.py

@@ -1,23 +0,0 @@
-from typing import Optional
-
-from pydantic import Field
-from pydantic_settings import BaseSettings
-
-
-class LindormConfig(BaseSettings):
-    """
-    Lindorm configs
-    """
-
-    LINDORM_URL: Optional[str] = Field(
-        description="Lindorm url",
-        default=None,
-    )
-    LINDORM_USERNAME: Optional[str] = Field(
-        description="Lindorm user",
-        default=None,
-    )
-    LINDORM_PASSWORD: Optional[str] = Field(
-        description="Lindorm password",
-        default=None,
-    )

api/configs/middleware/vdb/oceanbase_config.py

@@ -1,35 +0,0 @@
-from typing import Optional
-
-from pydantic import Field, PositiveInt
-from pydantic_settings import BaseSettings
-
-
-class OceanBaseVectorConfig(BaseSettings):
-    """
-    Configuration settings for OceanBase Vector database
-    """
-
-    OCEANBASE_VECTOR_HOST: Optional[str] = Field(
-        description="Hostname or IP address of the OceanBase Vector server (e.g. 'localhost')",
-        default=None,
-    )
-
-    OCEANBASE_VECTOR_PORT: Optional[PositiveInt] = Field(
-        description="Port number on which the OceanBase Vector server is listening (default is 2881)",
-        default=2881,
-    )
-
-    OCEANBASE_VECTOR_USER: Optional[str] = Field(
-        description="Username for authenticating with the OceanBase Vector database",
-        default=None,
-    )
-
-    OCEANBASE_VECTOR_PASSWORD: Optional[str] = Field(
-        description="Password for authenticating with the OceanBase Vector database",
-        default=None,
-    )
-
-    OCEANBASE_VECTOR_DATABASE: Optional[str] = Field(
-        description="Name of the OceanBase Vector database to connect to",
-        default=None,
-    )

api/configs/middleware/vdb/oracle_config.py

@@ -14,7 +14,7 @@ class OracleConfig(BaseSettings):
         default=None,
     )
 
-    ORACLE_PORT: PositiveInt = Field(
+    ORACLE_PORT: Optional[PositiveInt] = Field(
         description="Port number on which the Oracle database server is listening (default is 1521)",
         default=1521,
     )

api/configs/middleware/vdb/pgvector_config.py

@@ -14,7 +14,7 @@ class PGVectorConfig(BaseSettings):
         default=None,
     )
 
-    PGVECTOR_PORT: PositiveInt = Field(
+    PGVECTOR_PORT: Optional[PositiveInt] = Field(
         description="Port number on which the PostgreSQL server is listening (default is 5433)",
         default=5433,
     )

api/configs/middleware/vdb/pgvectors_config.py

@@ -14,7 +14,7 @@ class PGVectoRSConfig(BaseSettings):
         default=None,
     )
 
-    PGVECTO_RS_PORT: PositiveInt = Field(
+    PGVECTO_RS_PORT: Optional[PositiveInt] = Field(
         description="Port number on which the PostgreSQL server with PGVecto.RS is listening (default is 5431)",
         default=5431,
     )

api/configs/middleware/vdb/tidb_on_qdrant_config.py

@@ -1,70 +0,0 @@
-from typing import Optional
-
-from pydantic import Field, NonNegativeInt, PositiveInt
-from pydantic_settings import BaseSettings
-
-
-class TidbOnQdrantConfig(BaseSettings):
-    """
-    Tidb on Qdrant configs
-    """
-
-    TIDB_ON_QDRANT_URL: Optional[str] = Field(
-        description="Tidb on Qdrant url",
-        default=None,
-    )
-
-    TIDB_ON_QDRANT_API_KEY: Optional[str] = Field(
-        description="Tidb on Qdrant api key",
-        default=None,
-    )
-
-    TIDB_ON_QDRANT_CLIENT_TIMEOUT: NonNegativeInt = Field(
-        description="Tidb on Qdrant client timeout in seconds",
-        default=20,
-    )
-
-    TIDB_ON_QDRANT_GRPC_ENABLED: bool = Field(
-        description="whether enable grpc support for Tidb on Qdrant connection",
-        default=False,
-    )
-
-    TIDB_ON_QDRANT_GRPC_PORT: PositiveInt = Field(
-        description="Tidb on Qdrant grpc port",
-        default=6334,
-    )
-
-    TIDB_PUBLIC_KEY: Optional[str] = Field(
-        description="Tidb account public key",
-        default=None,
-    )
-
-    TIDB_PRIVATE_KEY: Optional[str] = Field(
-        description="Tidb account private key",
-        default=None,
-    )
-
-    TIDB_API_URL: Optional[str] = Field(
-        description="Tidb API url",
-        default=None,
-    )
-
-    TIDB_IAM_API_URL: Optional[str] = Field(
-        description="Tidb IAM API url",
-        default=None,
-    )
-
-    TIDB_REGION: Optional[str] = Field(
-        description="Tidb serverless region",
-        default="regions/aws-us-east-1",
-    )
-
-    TIDB_PROJECT_ID: Optional[str] = Field(
-        description="Tidb project id",
-        default=None,
-    )
-
-    TIDB_SPEND_LIMIT: Optional[int] = Field(
-        description="Tidb spend limit",
-        default=100,
-    )

api/configs/middleware/vdb/upstash_config.py

@@ -1,20 +0,0 @@
-from typing import Optional
-
-from pydantic import Field
-from pydantic_settings import BaseSettings
-
-
-class UpstashConfig(BaseSettings):
-    """
-    Configuration settings for Upstash vector database
-    """
-
-    UPSTASH_VECTOR_URL: Optional[str] = Field(
-        description="URL of the upstash server (e.g., 'https://vector.upstash.io')",
-        default=None,
-    )
-
-    UPSTASH_VECTOR_TOKEN: Optional[str] = Field(
-        description="Token for authenticating with the upstash server",
-        default=None,
-    )

api/configs/middleware/vdb/vikingdb_config.py

@@ -11,39 +11,27 @@ class VikingDBConfig(BaseModel):
     """
 
     VIKINGDB_ACCESS_KEY: Optional[str] = Field(
-        description="The Access Key provided by Volcengine VikingDB for API authentication."
-        "Refer to the following documentation for details on obtaining credentials:"
-        "https://www.volcengine.com/docs/6291/65568",
-        default=None,
+        default=None, description="The Access Key provided by Volcengine VikingDB for API authentication."
     )
-
     VIKINGDB_SECRET_KEY: Optional[str] = Field(
-        description="The Secret Key provided by Volcengine VikingDB for API authentication.",
-        default=None,
+        default=None, description="The Secret Key provided by Volcengine VikingDB for API authentication."
     )
-
-    VIKINGDB_REGION: str = Field(
-        description="The region of the Volcengine VikingDB service.(e.g., 'cn-shanghai', 'cn-beijing').",
+    VIKINGDB_REGION: Optional[str] = Field(
         default="cn-shanghai",
+        description="The region of the Volcengine VikingDB service.(e.g., 'cn-shanghai', 'cn-beijing').",
     )
-
-    VIKINGDB_HOST: str = Field(
+    VIKINGDB_HOST: Optional[str] = Field(
+        default="api-vikingdb.mlp.cn-shanghai.volces.com",
         description="The host of the Volcengine VikingDB service.(e.g., 'api-vikingdb.volces.com', \
             'api-vikingdb.mlp.cn-shanghai.volces.com')",
-        default="api-vikingdb.mlp.cn-shanghai.volces.com",
     )
-
-    VIKINGDB_SCHEME: str = Field(
-        description="The scheme of the Volcengine VikingDB service.(e.g., 'http', 'https').",
+    VIKINGDB_SCHEME: Optional[str] = Field(
         default="http",
+        description="The scheme of the Volcengine VikingDB service.(e.g., 'http', 'https').",
     )
-
-    VIKINGDB_CONNECTION_TIMEOUT: int = Field(
-        description="The connection timeout of the Volcengine VikingDB service.",
-        default=30,
+    VIKINGDB_CONNECTION_TIMEOUT: Optional[int] = Field(
+        default=30, description="The connection timeout of the Volcengine VikingDB service."
     )
-
-    VIKINGDB_SOCKET_TIMEOUT: int = Field(
-        description="The socket timeout of the Volcengine VikingDB service.",
-        default=30,
+    VIKINGDB_SOCKET_TIMEOUT: Optional[int] = Field(
+        default=30, description="The socket timeout of the Volcengine VikingDB service."
     )

api/configs/packaging/__init__.py

@@ -9,7 +9,7 @@ class PackagingInfo(BaseSettings):
 
     CURRENT_VERSION: str = Field(
         description="Dify version",
-        default="0.10.2",
+        default="0.10.0-beta3",
     )
 
     COMMIT_SHA: str = Field(

api/constants/__init__.py

@@ -12,13 +12,10 @@ VIDEO_EXTENSIONS.extend([ext.upper() for ext in VIDEO_EXTENSIONS])
 AUDIO_EXTENSIONS = ["mp3", "m4a", "wav", "webm", "amr"]
 AUDIO_EXTENSIONS.extend([ext.upper() for ext in AUDIO_EXTENSIONS])
 
+DOCUMENT_EXTENSIONS = ["txt", "markdown", "md", "pdf", "html", "htm", "xlsx", "xls", "docx", "csv"]
+DOCUMENT_EXTENSIONS.extend([ext.upper() for ext in DOCUMENT_EXTENSIONS])
 
 if dify_config.ETL_TYPE == "Unstructured":
     DOCUMENT_EXTENSIONS = ["txt", "markdown", "md", "pdf", "html", "htm", "xlsx", "xls"]
-    DOCUMENT_EXTENSIONS.extend(("docx", "csv", "eml", "msg", "pptx", "xml", "epub"))
-    if dify_config.UNSTRUCTURED_API_URL:
-        DOCUMENT_EXTENSIONS.append("ppt")
-    DOCUMENT_EXTENSIONS.extend([ext.upper() for ext in DOCUMENT_EXTENSIONS])
-else:
-    DOCUMENT_EXTENSIONS = ["txt", "markdown", "md", "pdf", "html", "htm", "xlsx", "xls", "docx", "csv"]
+    DOCUMENT_EXTENSIONS.extend(("docx", "csv", "eml", "msg", "pptx", "ppt", "xml", "epub"))
     DOCUMENT_EXTENSIONS.extend([ext.upper() for ext in DOCUMENT_EXTENSIONS])

api/controllers/common/errors.py

@@ -1,6 +0,0 @@
-from werkzeug.exceptions import HTTPException
-
-
-class FilenameNotExistsError(HTTPException):
-    code = 400
-    description = "The specified filename does not exist."

api/controllers/common/fields.py

@@ -1,24 +0,0 @@
-from flask_restful import fields
-
-parameters__system_parameters = {
-    "image_file_size_limit": fields.Integer,
-    "video_file_size_limit": fields.Integer,
-    "audio_file_size_limit": fields.Integer,
-    "file_size_limit": fields.Integer,
-    "workflow_file_upload_limit": fields.Integer,
-}
-
-parameters_fields = {
-    "opening_statement": fields.String,
-    "suggested_questions": fields.Raw,
-    "suggested_questions_after_answer": fields.Raw,
-    "speech_to_text": fields.Raw,
-    "text_to_speech": fields.Raw,
-    "retriever_resource": fields.Raw,
-    "annotation_reply": fields.Raw,
-    "more_like_this": fields.Raw,
-    "user_input_form": fields.Raw,
-    "sensitive_word_avoidance": fields.Raw,
-    "file_upload": fields.Raw,
-    "system_parameters": fields.Nested(parameters__system_parameters),
-}

api/controllers/common/helpers.py

@@ -1,97 +0,0 @@
-import mimetypes
-import os
-import re
-import urllib.parse
-from collections.abc import Mapping
-from typing import Any
-from uuid import uuid4
-
-import httpx
-from pydantic import BaseModel
-
-from configs import dify_config
-
-
-class FileInfo(BaseModel):
-    filename: str
-    extension: str
-    mimetype: str
-    size: int
-
-
-def guess_file_info_from_response(response: httpx.Response):
-    url = str(response.url)
-    # Try to extract filename from URL
-    parsed_url = urllib.parse.urlparse(url)
-    url_path = parsed_url.path
-    filename = os.path.basename(url_path)
-
-    # If filename couldn't be extracted, use Content-Disposition header
-    if not filename:
-        content_disposition = response.headers.get("Content-Disposition")
-        if content_disposition:
-            filename_match = re.search(r'filename="?(.+)"?', content_disposition)
-            if filename_match:
-                filename = filename_match.group(1)
-
-    # If still no filename, generate a unique one
-    if not filename:
-        unique_name = str(uuid4())
-        filename = f"{unique_name}"
-
-    # Guess MIME type from filename first, then URL
-    mimetype, _ = mimetypes.guess_type(filename)
-    if mimetype is None:
-        mimetype, _ = mimetypes.guess_type(url)
-    if mimetype is None:
-        # If guessing fails, use Content-Type from response headers
-        mimetype = response.headers.get("Content-Type", "application/octet-stream")
-
-    extension = os.path.splitext(filename)[1]
-
-    # Ensure filename has an extension
-    if not extension:
-        extension = mimetypes.guess_extension(mimetype) or ".bin"
-        filename = f"{filename}{extension}"
-
-    return FileInfo(
-        filename=filename,
-        extension=extension,
-        mimetype=mimetype,
-        size=int(response.headers.get("Content-Length", -1)),
-    )
-
-
-def get_parameters_from_feature_dict(*, features_dict: Mapping[str, Any], user_input_form: list[dict[str, Any]]):
-    return {
-        "opening_statement": features_dict.get("opening_statement"),
-        "suggested_questions": features_dict.get("suggested_questions", []),
-        "suggested_questions_after_answer": features_dict.get("suggested_questions_after_answer", {"enabled": False}),
-        "speech_to_text": features_dict.get("speech_to_text", {"enabled": False}),
-        "text_to_speech": features_dict.get("text_to_speech", {"enabled": False}),
-        "retriever_resource": features_dict.get("retriever_resource", {"enabled": False}),
-        "annotation_reply": features_dict.get("annotation_reply", {"enabled": False}),
-        "more_like_this": features_dict.get("more_like_this", {"enabled": False}),
-        "user_input_form": user_input_form,
-        "sensitive_word_avoidance": features_dict.get(
-            "sensitive_word_avoidance", {"enabled": False, "type": "", "configs": []}
-        ),
-        "file_upload": features_dict.get(
-            "file_upload",
-            {
-                "image": {
-                    "enabled": False,
-                    "number_limits": 3,
-                    "detail": "high",
-                    "transfer_methods": ["remote_url", "local_file"],
-                }
-            },
-        ),
-        "system_parameters": {
-            "image_file_size_limit": dify_config.UPLOAD_IMAGE_FILE_SIZE_LIMIT,
-            "video_file_size_limit": dify_config.UPLOAD_VIDEO_FILE_SIZE_LIMIT,
-            "audio_file_size_limit": dify_config.UPLOAD_AUDIO_FILE_SIZE_LIMIT,
-            "file_size_limit": dify_config.UPLOAD_FILE_SIZE_LIMIT,
-            "workflow_file_upload_limit": dify_config.WORKFLOW_FILE_UPLOAD_LIMIT,
-        },
-    }

api/controllers/console/__init__.py

@@ -2,21 +2,9 @@ from flask import Blueprint
 
 from libs.external_api import ExternalApi
 
-from .files import FileApi, FilePreviewApi, FileSupportTypeApi
-from .remote_files import RemoteFileInfoApi, RemoteFileUploadApi
-
 bp = Blueprint("console", __name__, url_prefix="/console/api")
 api = ExternalApi(bp)
 
-# File
-api.add_resource(FileApi, "/files/upload")
-api.add_resource(FilePreviewApi, "/files/<uuid:file_id>/preview")
-api.add_resource(FileSupportTypeApi, "/files/support-type")
-
-# Remote files
-api.add_resource(RemoteFileInfoApi, "/remote-files/<path:url>")
-api.add_resource(RemoteFileUploadApi, "/remote-files/upload")
-
 # Import other controllers
 from . import admin, apikey, extension, feature, ping, setup, version
 
@@ -55,7 +43,7 @@ from .datasets import (
     datasets_document,
     datasets_segments,
     external,
-    fta_test,
+    file,
     hit_testing,
     website,
 )

api/controllers/console/admin.py

@@ -1,10 +1,10 @@
+import os
 from functools import wraps
 
 from flask import request
 from flask_restful import Resource, reqparse
 from werkzeug.exceptions import NotFound, Unauthorized
 
-from configs import dify_config
 from constants.languages import supported_language
 from controllers.console import api
 from controllers.console.wraps import only_edition_cloud
@@ -15,7 +15,7 @@ from models.model import App, InstalledApp, RecommendedApp
 def admin_required(view):
     @wraps(view)
     def decorated(*args, **kwargs):
-        if not dify_config.ADMIN_API_KEY:
+        if not os.getenv("ADMIN_API_KEY"):
             raise Unauthorized("API key is invalid.")
 
         auth_header = request.headers.get("Authorization")
@@ -31,7 +31,7 @@ def admin_required(view):
         if auth_scheme != "bearer":
             raise Unauthorized("Invalid Authorization header format. Expected 'Bearer <api-key>' format.")
 
-        if dify_config.ADMIN_API_KEY != auth_token:
+        if os.getenv("ADMIN_API_KEY") != auth_token:
             raise Unauthorized("API key is invalid.")
 
         return view(*args, **kwargs)

api/controllers/console/apikey.py

@@ -10,7 +10,8 @@ from models.dataset import Dataset
 from models.model import ApiToken, App
 
 from . import api
-from .wraps import account_initialization_required, setup_required
+from .setup import setup_required
+from .wraps import account_initialization_required
 
 api_key_fields = {
     "id": fields.String,

api/controllers/console/app/advanced_prompt_template.py

@@ -1,7 +1,8 @@
 from flask_restful import Resource, reqparse
 
 from controllers.console import api
-from controllers.console.wraps import account_initialization_required, setup_required
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required
 from libs.login import login_required
 from services.advanced_prompt_template_service import AdvancedPromptTemplateService
 

api/controllers/console/app/agent.py

@@ -2,7 +2,8 @@ from flask_restful import Resource, reqparse
 
 from controllers.console import api
 from controllers.console.app.wraps import get_app_model
-from controllers.console.wraps import account_initialization_required, setup_required
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required
 from libs.helper import uuid_value
 from libs.login import login_required
 from models.model import AppMode

api/controllers/console/app/annotation.py

@@ -6,11 +6,8 @@ from werkzeug.exceptions import Forbidden
 from controllers.console import api
 from controllers.console.app.error import NoFileUploadedError
 from controllers.console.datasets.error import TooManyFilesError
-from controllers.console.wraps import (
-    account_initialization_required,
-    cloud_edition_billing_resource_check,
-    setup_required,
-)
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required, cloud_edition_billing_resource_check
 from extensions.ext_redis import redis_client
 from fields.annotation_fields import (
     annotation_fields,

api/controllers/console/app/app.py

@@ -6,11 +6,8 @@ from werkzeug.exceptions import BadRequest, Forbidden, abort
 
 from controllers.console import api
 from controllers.console.app.wraps import get_app_model
-from controllers.console.wraps import (
-    account_initialization_required,
-    cloud_edition_billing_resource_check,
-    setup_required,
-)
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required, cloud_edition_billing_resource_check
 from core.ops.ops_trace_manager import OpsTraceManager
 from fields.app_fields import (
     app_detail_fields,

api/controllers/console/app/audio.py

@@ -18,7 +18,8 @@ from controllers.console.app.error import (
     UnsupportedAudioTypeError,
 )
 from controllers.console.app.wraps import get_app_model
-from controllers.console.wraps import account_initialization_required, setup_required
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required
 from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError
 from core.model_runtime.errors.invoke import InvokeError
 from libs.login import login_required

api/controllers/console/app/completion.py

@@ -15,7 +15,8 @@ from controllers.console.app.error import (
     ProviderQuotaExceededError,
 )
 from controllers.console.app.wraps import get_app_model
-from controllers.console.wraps import account_initialization_required, setup_required
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required
 from controllers.web.error import InvokeRateLimitError as InvokeRateLimitHttpError
 from core.app.apps.base_app_queue_manager import AppQueueManager
 from core.app.entities.app_invoke_entities import InvokeFrom

api/controllers/console/app/conversation.py

@@ -10,7 +10,8 @@ from werkzeug.exceptions import Forbidden, NotFound
 
 from controllers.console import api
 from controllers.console.app.wraps import get_app_model
-from controllers.console.wraps import account_initialization_required, setup_required
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required
 from core.app.entities.app_invoke_entities import InvokeFrom
 from extensions.ext_database import db
 from fields.conversation_fields import (

api/controllers/console/app/conversation_variables.py

@@ -4,7 +4,8 @@ from sqlalchemy.orm import Session
 
 from controllers.console import api
 from controllers.console.app.wraps import get_app_model
-from controllers.console.wraps import account_initialization_required, setup_required
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required
 from extensions.ext_database import db
 from fields.conversation_variable_fields import paginated_conversation_variable_fields
 from libs.login import login_required

api/controllers/console/app/generator.py

@@ -10,7 +10,8 @@ from controllers.console.app.error import (
     ProviderNotInitializeError,
     ProviderQuotaExceededError,
 )
-from controllers.console.wraps import account_initialization_required, setup_required
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required
 from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError
 from core.llm_generator.llm_generator import LLMGenerator
 from core.model_runtime.errors.invoke import InvokeError
@@ -51,39 +52,4 @@ class RuleGenerateApi(Resource):
         return rules
 
 
-class RuleCodeGenerateApi(Resource):
-    @setup_required
-    @login_required
-    @account_initialization_required
-    def post(self):
-        parser = reqparse.RequestParser()
-        parser.add_argument("instruction", type=str, required=True, nullable=False, location="json")
-        parser.add_argument("model_config", type=dict, required=True, nullable=False, location="json")
-        parser.add_argument("no_variable", type=bool, required=True, default=False, location="json")
-        parser.add_argument("code_language", type=str, required=False, default="javascript", location="json")
-        args = parser.parse_args()
-
-        account = current_user
-        CODE_GENERATION_MAX_TOKENS = int(os.getenv("CODE_GENERATION_MAX_TOKENS", "1024"))
-        try:
-            code_result = LLMGenerator.generate_code(
-                tenant_id=account.current_tenant_id,
-                instruction=args["instruction"],
-                model_config=args["model_config"],
-                code_language=args["code_language"],
-                max_tokens=CODE_GENERATION_MAX_TOKENS,
-            )
-        except ProviderTokenNotInitError as ex:
-            raise ProviderNotInitializeError(ex.description)
-        except QuotaExceededError:
-            raise ProviderQuotaExceededError()
-        except ModelCurrentlyNotSupportError:
-            raise ProviderModelCurrentlyNotSupportError()
-        except InvokeError as e:
-            raise CompletionRequestError(e.description)
-
-        return code_result
-
-
 api.add_resource(RuleGenerateApi, "/rule-generate")
-api.add_resource(RuleCodeGenerateApi, "/rule-code-generate")

api/controllers/console/app/message.py

@@ -14,11 +14,8 @@ from controllers.console.app.error import (
 )
 from controllers.console.app.wraps import get_app_model
 from controllers.console.explore.error import AppSuggestedQuestionsAfterAnswerDisabledError
-from controllers.console.wraps import (
-    account_initialization_required,
-    cloud_edition_billing_resource_check,
-    setup_required,
-)
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required, cloud_edition_billing_resource_check
 from core.app.entities.app_invoke_entities import InvokeFrom
 from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError
 from core.model_runtime.errors.invoke import InvokeError
@@ -108,8 +105,6 @@ class ChatMessageListApi(Resource):
             if rest_count > 0:
                 has_more = True
 
-        history_messages = list(reversed(history_messages))
-
         return InfiniteScrollPagination(data=history_messages, limit=args["limit"], has_more=has_more)
 
 

api/controllers/console/app/model_config.py

@@ -6,7 +6,8 @@ from flask_restful import Resource
 
 from controllers.console import api
 from controllers.console.app.wraps import get_app_model
-from controllers.console.wraps import account_initialization_required, setup_required
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required
 from core.agent.entities import AgentToolEntity
 from core.tools.tool_manager import ToolManager
 from core.tools.utils.configuration import ToolParameterConfigurationManager

api/controllers/console/app/ops_trace.py

@@ -2,7 +2,8 @@ from flask_restful import Resource, reqparse
 
 from controllers.console import api
 from controllers.console.app.error import TracingConfigCheckError, TracingConfigIsExist, TracingConfigNotExist
-from controllers.console.wraps import account_initialization_required, setup_required
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required
 from libs.login import login_required
 from services.ops_service import OpsService
 

api/controllers/console/app/site.py

@@ -7,7 +7,8 @@ from werkzeug.exceptions import Forbidden, NotFound
 from constants.languages import supported_language
 from controllers.console import api
 from controllers.console.app.wraps import get_app_model
-from controllers.console.wraps import account_initialization_required, setup_required
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required
 from extensions.ext_database import db
 from fields.app_fields import app_site_fields
 from libs.login import login_required

api/controllers/console/app/statistic.py

@@ -8,7 +8,8 @@ from flask_restful import Resource, reqparse
 
 from controllers.console import api
 from controllers.console.app.wraps import get_app_model
-from controllers.console.wraps import account_initialization_required, setup_required
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required
 from extensions.ext_database import db
 from libs.helper import DatetimeString
 from libs.login import login_required

api/controllers/console/app/workflow.py

@@ -9,7 +9,8 @@ import services
 from controllers.console import api
 from controllers.console.app.error import ConversationCompletedError, DraftWorkflowNotExist, DraftWorkflowNotSync
 from controllers.console.app.wraps import get_app_model
-from controllers.console.wraps import account_initialization_required, setup_required
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required
 from core.app.apps.base_app_queue_manager import AppQueueManager
 from core.app.entities.app_invoke_entities import InvokeFrom
 from factories import variable_factory

api/controllers/console/app/workflow_app_log.py

@@ -3,7 +3,8 @@ from flask_restful.inputs import int_range
 
 from controllers.console import api
 from controllers.console.app.wraps import get_app_model
-from controllers.console.wraps import account_initialization_required, setup_required
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required
 from fields.workflow_app_log_fields import workflow_app_log_pagination_fields
 from libs.login import login_required
 from models import App

api/controllers/console/app/workflow_run.py

@@ -3,7 +3,8 @@ from flask_restful.inputs import int_range
 
 from controllers.console import api
 from controllers.console.app.wraps import get_app_model
-from controllers.console.wraps import account_initialization_required, setup_required
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required
 from fields.workflow_run_fields import (
     advanced_chat_workflow_run_pagination_fields,
     workflow_run_detail_fields,

api/controllers/console/app/workflow_statistic.py

@@ -8,11 +8,12 @@ from flask_restful import Resource, reqparse
 
 from controllers.console import api
 from controllers.console.app.wraps import get_app_model
-from controllers.console.wraps import account_initialization_required, setup_required
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required
+from enums import WorkflowRunTriggeredFrom
 from extensions.ext_database import db
 from libs.helper import DatetimeString
 from libs.login import login_required
-from models.enums import WorkflowRunTriggeredFrom
 from models.model import AppMode
 
 

api/controllers/console/auth/activate.py

@@ -1,15 +1,17 @@
+import base64
 import datetime
+import secrets
 
-from flask import request
 from flask_restful import Resource, reqparse
 
 from constants.languages import supported_language
 from controllers.console import api
 from controllers.console.error import AlreadyActivateError
 from extensions.ext_database import db
-from libs.helper import StrLen, email, extract_remote_ip, timezone
-from models.account import AccountStatus, Tenant
-from services.account_service import AccountService, RegisterService
+from libs.helper import StrLen, email, timezone
+from libs.password import hash_password, valid_password
+from models.account import AccountStatus
+from services.account_service import RegisterService
 
 
 class ActivateCheckApi(Resource):
@@ -25,18 +27,8 @@ class ActivateCheckApi(Resource):
         token = args["token"]
 
         invitation = RegisterService.get_invitation_if_token_valid(workspaceId, reg_email, token)
-        if invitation:
-            data = invitation.get("data", {})
-            tenant: Tenant = invitation.get("tenant", None)
-            workspace_name = tenant.name if tenant else None
-            workspace_id = tenant.id if tenant else None
-            invitee_email = data.get("email") if data else None
-            return {
-                "is_valid": invitation is not None,
-                "data": {"workspace_name": workspace_name, "workspace_id": workspace_id, "email": invitee_email},
-            }
-        else:
-            return {"is_valid": False}
+
+        return {"is_valid": invitation is not None, "workspace_name": invitation["tenant"].name if invitation else None}
 
 
 class ActivateApi(Resource):
@@ -46,6 +38,7 @@ class ActivateApi(Resource):
         parser.add_argument("email", type=email, required=False, nullable=True, location="json")
         parser.add_argument("token", type=str, required=True, nullable=False, location="json")
         parser.add_argument("name", type=StrLen(30), required=True, nullable=False, location="json")
+        parser.add_argument("password", type=valid_password, required=True, nullable=False, location="json")
         parser.add_argument(
             "interface_language", type=supported_language, required=True, nullable=False, location="json"
         )
@@ -61,6 +54,15 @@ class ActivateApi(Resource):
         account = invitation["account"]
         account.name = args["name"]
 
+        # generate password salt
+        salt = secrets.token_bytes(16)
+        base64_salt = base64.b64encode(salt).decode()
+
+        # encrypt password with salt
+        password_hashed = hash_password(args["password"], salt)
+        base64_password_hashed = base64.b64encode(password_hashed).decode()
+        account.password = base64_password_hashed
+        account.password_salt = base64_salt
         account.interface_language = args["interface_language"]
         account.timezone = args["timezone"]
         account.interface_theme = "light"
@@ -68,9 +70,7 @@ class ActivateApi(Resource):
         account.initialized_at = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None)
         db.session.commit()
 
-        token_pair = AccountService.login(account, ip_address=extract_remote_ip(request))
-
-        return {"result": "success", "data": token_pair.model_dump()}
+        return {"result": "success"}
 
 
 api.add_resource(ActivateCheckApi, "/activate/check")

api/controllers/console/auth/data_source_bearer_auth.py

@@ -7,7 +7,8 @@ from controllers.console.auth.error import ApiKeyAuthFailedError
 from libs.login import login_required
 from services.auth.api_key_auth_service import ApiKeyAuthService
 
-from ..wraps import account_initialization_required, setup_required
+from ..setup import setup_required
+from ..wraps import account_initialization_required
 
 
 class ApiKeyAuthDataSource(Resource):

api/controllers/console/auth/data_source_oauth.py

@@ -11,7 +11,8 @@ from controllers.console import api
 from libs.login import login_required
 from libs.oauth_data_source import NotionOAuth
 
-from ..wraps import account_initialization_required, setup_required
+from ..setup import setup_required
+from ..wraps import account_initialization_required
 
 
 def get_oauth_providers():

api/controllers/console/auth/error.py

@@ -27,29 +27,5 @@ class InvalidTokenError(BaseHTTPException):
 
 class PasswordResetRateLimitExceededError(BaseHTTPException):
     error_code = "password_reset_rate_limit_exceeded"
-    description = "Too many password reset emails have been sent. Please try again in 1 minutes."
-    code = 429
-
-
-class EmailCodeError(BaseHTTPException):
-    error_code = "email_code_error"
-    description = "Email code is invalid or expired."
-    code = 400
-
-
-class EmailOrPasswordMismatchError(BaseHTTPException):
-    error_code = "email_or_password_mismatch"
-    description = "The email or password is mismatched."
-    code = 400
-
-
-class EmailPasswordLoginLimitError(BaseHTTPException):
-    error_code = "email_code_login_limit"
-    description = "Too many incorrect password attempts. Please try again later."
-    code = 429
-
-
-class EmailCodeLoginRateLimitExceededError(BaseHTTPException):
-    error_code = "email_code_login_rate_limit_exceeded"
-    description = "Too many login emails have been sent. Please try again in 5 minutes."
+    description = "Password reset rate limit exceeded. Try again later."
     code = 429

api/controllers/console/auth/forgot_password.py

@@ -1,82 +1,65 @@
 import base64
+import logging
 import secrets
 
-from flask import request
 from flask_restful import Resource, reqparse
 
-from constants.languages import languages
 from controllers.console import api
 from controllers.console.auth.error import (
-    EmailCodeError,
     InvalidEmailError,
     InvalidTokenError,
     PasswordMismatchError,
+    PasswordResetRateLimitExceededError,
 )
-from controllers.console.error import EmailSendIpLimitError, NotAllowedRegister
-from controllers.console.wraps import setup_required
-from events.tenant_event import tenant_was_created
+from controllers.console.setup import setup_required
 from extensions.ext_database import db
-from libs.helper import email, extract_remote_ip
+from libs.helper import email as email_validate
 from libs.password import hash_password, valid_password
-from models.account import Account
-from services.account_service import AccountService, TenantService
-from services.errors.workspace import WorkSpaceNotAllowedCreateError
-from services.feature_service import FeatureService
+from models import Account
+from services.account_service import AccountService
+from services.errors.account import RateLimitExceededError
 
 
 class ForgotPasswordSendEmailApi(Resource):
     @setup_required
     def post(self):
         parser = reqparse.RequestParser()
-        parser.add_argument("email", type=email, required=True, location="json")
-        parser.add_argument("language", type=str, required=False, location="json")
+        parser.add_argument("email", type=str, required=True, location="json")
         args = parser.parse_args()
 
-        ip_address = extract_remote_ip(request)
-        if AccountService.is_email_send_ip_limit(ip_address):
-            raise EmailSendIpLimitError()
+        email = args["email"]
 
-        if args["language"] is not None and args["language"] == "zh-Hans":
-            language = "zh-Hans"
+        if not email_validate(email):
+            raise InvalidEmailError()
+
+        account = Account.query.filter_by(email=email).first()
+
+        if account:
+            try:
+                AccountService.send_reset_password_email(account=account)
+            except RateLimitExceededError:
+                logging.warning(f"Rate limit exceeded for email: {account.email}")
+                raise PasswordResetRateLimitExceededError()
         else:
-            language = "en-US"
+            # Return success to avoid revealing email registration status
+            logging.warning(f"Attempt to reset password for unregistered email: {email}")
 
-        account = Account.query.filter_by(email=args["email"]).first()
-        token = None
-        if account is None:
-            if FeatureService.get_system_features().is_allow_register:
-                token = AccountService.send_reset_password_email(email=args["email"], language=language)
-                return {"result": "fail", "data": token, "code": "account_not_found"}
-            else:
-                raise NotAllowedRegister()
-        else:
-            token = AccountService.send_reset_password_email(account=account, email=args["email"], language=language)
-
-        return {"result": "success", "data": token}
+        return {"result": "success"}
 
 
 class ForgotPasswordCheckApi(Resource):
     @setup_required
     def post(self):
         parser = reqparse.RequestParser()
-        parser.add_argument("email", type=str, required=True, location="json")
-        parser.add_argument("code", type=str, required=True, location="json")
         parser.add_argument("token", type=str, required=True, nullable=False, location="json")
         args = parser.parse_args()
+        token = args["token"]
 
-        user_email = args["email"]
+        reset_data = AccountService.get_reset_password_data(token)
 
-        token_data = AccountService.get_reset_password_data(args["token"])
-        if token_data is None:
-            raise InvalidTokenError()
-
-        if user_email != token_data.get("email"):
-            raise InvalidEmailError()
-
-        if args["code"] != token_data.get("code"):
-            raise EmailCodeError()
-
-        return {"is_valid": True, "email": token_data.get("email")}
+        if reset_data is None:
+            return {"is_valid": False, "email": None}
+        return {"is_valid": True, "email": reset_data.get("email")}
 
 
 class ForgotPasswordResetApi(Resource):
@@ -109,26 +92,9 @@ class ForgotPasswordResetApi(Resource):
         base64_password_hashed = base64.b64encode(password_hashed).decode()
 
         account = Account.query.filter_by(email=reset_data.get("email")).first()
-        if account:
-            account.password = base64_password_hashed
-            account.password_salt = base64_salt
-            db.session.commit()
-            tenant = TenantService.get_join_tenants(account)
-            if not tenant and not FeatureService.get_system_features().is_allow_create_workspace:
-                tenant = TenantService.create_tenant(f"{account.name}'s Workspace")
-                TenantService.create_tenant_member(tenant, account, role="owner")
-                account.current_tenant = tenant
-                tenant_was_created.send(tenant)
-        else:
-            try:
-                account = AccountService.create_account_and_tenant(
-                    email=reset_data.get("email"),
-                    name=reset_data.get("email"),
-                    password=password_confirm,
-                    interface_language=languages[0],
-                )
-            except WorkSpaceNotAllowedCreateError:
-                pass
+        account.password = base64_password_hashed
+        account.password_salt = base64_salt
+        db.session.commit()
 
         return {"result": "success"}
 

api/controllers/console/auth/login.py

@@ -5,29 +5,12 @@ from flask import request
 from flask_restful import Resource, reqparse
 
 import services
-from constants.languages import languages
 from controllers.console import api
-from controllers.console.auth.error import (
-    EmailCodeError,
-    EmailOrPasswordMismatchError,
-    EmailPasswordLoginLimitError,
-    InvalidEmailError,
-    InvalidTokenError,
-)
-from controllers.console.error import (
-    AccountBannedError,
-    EmailSendIpLimitError,
-    NotAllowedCreateWorkspace,
-    NotAllowedRegister,
-)
-from controllers.console.wraps import setup_required
-from events.tenant_event import tenant_was_created
+from controllers.console.setup import setup_required
 from libs.helper import email, extract_remote_ip
 from libs.password import valid_password
-from models.account import Account
-from services.account_service import AccountService, RegisterService, TenantService
-from services.errors.workspace import WorkSpaceNotAllowedCreateError
-from services.feature_service import FeatureService
+from models import Account
+from services.account_service import AccountService, TenantService
 
 
 class LoginApi(Resource):
@@ -40,43 +23,15 @@ class LoginApi(Resource):
         parser.add_argument("email", type=email, required=True, location="json")
         parser.add_argument("password", type=valid_password, required=True, location="json")
         parser.add_argument("remember_me", type=bool, required=False, default=False, location="json")
-        parser.add_argument("invite_token", type=str, required=False, default=None, location="json")
-        parser.add_argument("language", type=str, required=False, default="en-US", location="json")
         args = parser.parse_args()
 
-        is_login_error_rate_limit = AccountService.is_login_error_rate_limit(args["email"])
-        if is_login_error_rate_limit:
-            raise EmailPasswordLoginLimitError()
-
-        invitation = args["invite_token"]
-        if invitation:
-            invitation = RegisterService.get_invitation_if_token_valid(None, args["email"], invitation)
-
-        if args["language"] is not None and args["language"] == "zh-Hans":
-            language = "zh-Hans"
-        else:
-            language = "en-US"
+        # todo: Verify the recaptcha
 
         try:
-            if invitation:
-                data = invitation.get("data", {})
-                invitee_email = data.get("email") if data else None
-                if invitee_email != args["email"]:
-                    raise InvalidEmailError()
-                account = AccountService.authenticate(args["email"], args["password"], args["invite_token"])
-            else:
-                account = AccountService.authenticate(args["email"], args["password"])
-        except services.errors.account.AccountLoginError:
-            raise AccountBannedError()
-        except services.errors.account.AccountPasswordError:
-            AccountService.add_login_error_rate_limit(args["email"])
-            raise EmailOrPasswordMismatchError()
-        except services.errors.account.AccountNotFoundError:
-            if FeatureService.get_system_features().is_allow_register:
-                token = AccountService.send_reset_password_email(email=args["email"], language=language)
-                return {"result": "fail", "data": token, "code": "account_not_found"}
-            else:
-                raise NotAllowedRegister()
+            account = AccountService.authenticate(args["email"], args["password"])
+        except services.errors.account.AccountLoginError as e:
+            return {"code": "unauthorized", "message": str(e)}, 401
+
         # SELF_HOSTED only have one workspace
         tenants = TenantService.get_join_tenants(account)
         if len(tenants) == 0:
@@ -86,7 +41,7 @@ class LoginApi(Resource):
             }
 
         token_pair = AccountService.login(account=account, ip_address=extract_remote_ip(request))
-        AccountService.reset_login_error_rate_limit(args["email"])
+
         return {"result": "success", "data": token_pair.model_dump()}
 
 
@@ -94,111 +49,60 @@ class LogoutApi(Resource):
     @setup_required
     def get(self):
         account = cast(Account, flask_login.current_user)
-        if isinstance(account, flask_login.AnonymousUserMixin):
-            return {"result": "success"}
         AccountService.logout(account=account)
         flask_login.logout_user()
         return {"result": "success"}
 
 
-class ResetPasswordSendEmailApi(Resource):
+class ResetPasswordApi(Resource):
     @setup_required
-    def post(self):
-        parser = reqparse.RequestParser()
-        parser.add_argument("email", type=email, required=True, location="json")
-        parser.add_argument("language", type=str, required=False, location="json")
-        args = parser.parse_args()
+    def get(self):
+        # parser = reqparse.RequestParser()
+        # parser.add_argument('email', type=email, required=True, location='json')
+        # args = parser.parse_args()
 
-        if args["language"] is not None and args["language"] == "zh-Hans":
-            language = "zh-Hans"
-        else:
-            language = "en-US"
+        # import mailchimp_transactional as MailchimpTransactional
+        # from mailchimp_transactional.api_client import ApiClientError
 
-        account = AccountService.get_user_through_email(args["email"])
-        if account is None:
-            if FeatureService.get_system_features().is_allow_register:
-                token = AccountService.send_reset_password_email(email=args["email"], language=language)
-            else:
-                raise NotAllowedRegister()
-        else:
-            token = AccountService.send_reset_password_email(account=account, language=language)
+        # account = {'email': args['email']}
+        # account = AccountService.get_by_email(args['email'])
+        # if account is None:
+        #     raise ValueError('Email not found')
+        # new_password = AccountService.generate_password()
+        # AccountService.update_password(account, new_password)
 
-        return {"result": "success", "data": token}
+        # todo: Send email
+        # MAILCHIMP_API_KEY = dify_config.MAILCHIMP_TRANSACTIONAL_API_KEY
+        # mailchimp = MailchimpTransactional(MAILCHIMP_API_KEY)
 
+        # message = {
+        #     'from_email': 'noreply@example.com',
+        #     'to': [{'email': account['email']}],
+        #     'subject': 'Reset your Dify password',
+        #     'html': """
+        #         <p>Dear User,</p>
+        #         <p>The Dify team has generated a new password for you, details as follows:</p>
+        #         <p><strong>{new_password}</strong></p>
+        #         <p>Please change your password to log in as soon as possible.</p>
+        #         <p>Regards,</p>
+        #         <p>The Dify Team</p>
+        #     """
+        # }
 
-class EmailCodeLoginSendEmailApi(Resource):
-    @setup_required
-    def post(self):
-        parser = reqparse.RequestParser()
-        parser.add_argument("email", type=email, required=True, location="json")
-        parser.add_argument("language", type=str, required=False, location="json")
-        args = parser.parse_args()
+        # response = mailchimp.messages.send({
+        #     'message': message,
+        #     # required for transactional email
+        #     ' settings': {
+        #         'sandbox_mode': dify_config.MAILCHIMP_SANDBOX_MODE,
+        #     },
+        # })
 
-        ip_address = extract_remote_ip(request)
-        if AccountService.is_email_send_ip_limit(ip_address):
-            raise EmailSendIpLimitError()
+        # Check if MSG was sent
+        # if response.status_code != 200:
+        #     # handle error
+        #     pass
 
-        if args["language"] is not None and args["language"] == "zh-Hans":
-            language = "zh-Hans"
-        else:
-            language = "en-US"
-
-        account = AccountService.get_user_through_email(args["email"])
-        if account is None:
-            if FeatureService.get_system_features().is_allow_register:
-                token = AccountService.send_email_code_login_email(email=args["email"], language=language)
-            else:
-                raise NotAllowedRegister()
-        else:
-            token = AccountService.send_email_code_login_email(account=account, language=language)
-
-        return {"result": "success", "data": token}
-
-
-class EmailCodeLoginApi(Resource):
-    @setup_required
-    def post(self):
-        parser = reqparse.RequestParser()
-        parser.add_argument("email", type=str, required=True, location="json")
-        parser.add_argument("code", type=str, required=True, location="json")
-        parser.add_argument("token", type=str, required=True, location="json")
-        args = parser.parse_args()
-
-        user_email = args["email"]
-
-        token_data = AccountService.get_email_code_login_data(args["token"])
-        if token_data is None:
-            raise InvalidTokenError()
-
-        if token_data["email"] != args["email"]:
-            raise InvalidEmailError()
-
-        if token_data["code"] != args["code"]:
-            raise EmailCodeError()
-
-        AccountService.revoke_email_code_login_token(args["token"])
-        account = AccountService.get_user_through_email(user_email)
-        if account:
-            tenant = TenantService.get_join_tenants(account)
-            if not tenant:
-                if not FeatureService.get_system_features().is_allow_create_workspace:
-                    raise NotAllowedCreateWorkspace()
-                else:
-                    tenant = TenantService.create_tenant(f"{account.name}'s Workspace")
-                    TenantService.create_tenant_member(tenant, account, role="owner")
-                    account.current_tenant = tenant
-                    tenant_was_created.send(tenant)
-
-        if account is None:
-            try:
-                account = AccountService.create_account_and_tenant(
-                    email=user_email, name=user_email, interface_language=languages[0]
-                )
-            except WorkSpaceNotAllowedCreateError:
-                return NotAllowedCreateWorkspace()
-        token_pair = AccountService.login(account, ip_address=extract_remote_ip(request))
-        AccountService.reset_login_error_rate_limit(args["email"])
-        return {"result": "success", "data": token_pair.model_dump()}
+        return {"result": "success"}
 
 
 class RefreshTokenApi(Resource):
@@ -216,7 +120,4 @@ class RefreshTokenApi(Resource):
 
 api.add_resource(LoginApi, "/login")
 api.add_resource(LogoutApi, "/logout")
-api.add_resource(EmailCodeLoginSendEmailApi, "/email-code-login")
-api.add_resource(EmailCodeLoginApi, "/email-code-login/validity")
-api.add_resource(ResetPasswordSendEmailApi, "/reset-password")
 api.add_resource(RefreshTokenApi, "/refresh-token")

api/controllers/console/auth/oauth.py

@@ -5,20 +5,15 @@ from typing import Optional
 import requests
 from flask import current_app, redirect, request
 from flask_restful import Resource
-from werkzeug.exceptions import Unauthorized
 
 from configs import dify_config
 from constants.languages import languages
-from events.tenant_event import tenant_was_created
 from extensions.ext_database import db
 from libs.helper import extract_remote_ip
 from libs.oauth import GitHubOAuth, GoogleOAuth, OAuthUserInfo
 from models import Account
 from models.account import AccountStatus
 from services.account_service import AccountService, RegisterService, TenantService
-from services.errors.account import AccountNotFoundError
-from services.errors.workspace import WorkSpaceNotAllowedCreateError, WorkSpaceNotFoundError
-from services.feature_service import FeatureService
 
 from .. import api
 
@@ -48,7 +43,6 @@ def get_oauth_providers():
 
 class OAuthLogin(Resource):
     def get(self, provider: str):
-        invite_token = request.args.get("invite_token") or None
         OAUTH_PROVIDERS = get_oauth_providers()
         with current_app.app_context():
             oauth_provider = OAUTH_PROVIDERS.get(provider)
@@ -56,7 +50,7 @@ class OAuthLogin(Resource):
         if not oauth_provider:
             return {"error": "Invalid provider"}, 400
 
-        auth_url = oauth_provider.get_authorization_url(invite_token=invite_token)
+        auth_url = oauth_provider.get_authorization_url()
         return redirect(auth_url)
 
 
@@ -69,11 +63,6 @@ class OAuthCallback(Resource):
             return {"error": "Invalid provider"}, 400
 
         code = request.args.get("code")
-        state = request.args.get("state")
-        invite_token = None
-        if state:
-            invite_token = state
-
         try:
             token = oauth_provider.get_access_token(code)
             user_info = oauth_provider.get_user_info(token)
@@ -81,43 +70,17 @@ class OAuthCallback(Resource):
             logging.exception(f"An error occurred during the OAuth process with {provider}: {e.response.text}")
             return {"error": "OAuth process failed"}, 400
 
-        if invite_token and RegisterService.is_valid_invite_token(invite_token):
-            invitation = RegisterService._get_invitation_by_token(token=invite_token)
-            if invitation:
-                invitation_email = invitation.get("email", None)
-                if invitation_email != user_info.email:
-                    return redirect(f"{dify_config.CONSOLE_WEB_URL}/signin?message=Invalid invitation token.")
-
-            return redirect(f"{dify_config.CONSOLE_WEB_URL}/signin/invite-settings?invite_token={invite_token}")
-
-        try:
-            account = _generate_account(provider, user_info)
-        except AccountNotFoundError:
-            return redirect(f"{dify_config.CONSOLE_WEB_URL}/signin?message=Account not found.")
-        except (WorkSpaceNotFoundError, WorkSpaceNotAllowedCreateError):
-            return redirect(
-                f"{dify_config.CONSOLE_WEB_URL}/signin"
-                "?message=Workspace not found, please contact system admin to invite you to join in a workspace."
-            )
-
+        account = _generate_account(provider, user_info)
         # Check account status
-        if account.status == AccountStatus.BANNED.value:
-            return redirect(f"{dify_config.CONSOLE_WEB_URL}/signin?message=Account is banned.")
+        if account.status in {AccountStatus.BANNED.value, AccountStatus.CLOSED.value}:
+            return {"error": "Account is banned or closed."}, 403
 
         if account.status == AccountStatus.PENDING.value:
             account.status = AccountStatus.ACTIVE.value
             account.initialized_at = datetime.now(timezone.utc).replace(tzinfo=None)
             db.session.commit()
 
-        try:
-            TenantService.create_owner_tenant_if_not_exist(account)
-        except Unauthorized:
-            return redirect(f"{dify_config.CONSOLE_WEB_URL}/signin?message=Workspace not found.")
-        except WorkSpaceNotAllowedCreateError:
-            return redirect(
-                f"{dify_config.CONSOLE_WEB_URL}/signin"
-                "?message=Workspace not found, please contact system admin to invite you to join in a workspace."
-            )
+        TenantService.create_owner_tenant_if_not_exist(account)
 
         token_pair = AccountService.login(
             account=account,
@@ -142,20 +105,8 @@ def _generate_account(provider: str, user_info: OAuthUserInfo):
     # Get account by openid or email.
     account = _get_account_by_openid_or_email(provider, user_info)
 
-    if account:
-        tenant = TenantService.get_join_tenants(account)
-        if not tenant:
-            if not FeatureService.get_system_features().is_allow_create_workspace:
-                raise WorkSpaceNotAllowedCreateError()
-            else:
-                tenant = TenantService.create_tenant(f"{account.name}'s Workspace")
-                TenantService.create_tenant_member(tenant, account, role="owner")
-                account.current_tenant = tenant
-                tenant_was_created.send(tenant)
-
     if not account:
-        if not FeatureService.get_system_features().is_allow_register:
-            raise AccountNotFoundError()
+        # Create account
         account_name = user_info.name or "Dify"
         account = RegisterService.register(
             email=user_info.email, name=account_name, password=None, open_id=user_info.id, provider=provider

api/controllers/console/billing/billing.py

@@ -2,7 +2,8 @@ from flask_login import current_user
 from flask_restful import Resource, reqparse
 
 from controllers.console import api
-from controllers.console.wraps import account_initialization_required, only_edition_cloud, setup_required
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required, only_edition_cloud
 from libs.login import login_required
 from services.billing_service import BillingService
 

api/controllers/console/datasets/data_source.py

@@ -7,7 +7,8 @@ from flask_restful import Resource, marshal_with, reqparse
 from werkzeug.exceptions import NotFound
 
 from controllers.console import api
-from controllers.console.wraps import account_initialization_required, setup_required
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required
 from core.indexing_runner import IndexingRunner
 from core.rag.extractor.entity.extract_setting import ExtractSetting
 from core.rag.extractor.notion_extractor import NotionExtractor

api/controllers/console/datasets/datasets.py

@@ -10,7 +10,8 @@ from controllers.console import api
 from controllers.console.apikey import api_key_fields, api_key_list
 from controllers.console.app.error import ProviderNotInitializeError
 from controllers.console.datasets.error import DatasetInUseError, DatasetNameDuplicateError, IndexingEstimateError
-from controllers.console.wraps import account_initialization_required, setup_required
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required
 from core.errors.error import LLMBadRequestError, ProviderTokenNotInitError
 from core.indexing_runner import IndexingRunner
 from core.model_runtime.entities.model_entities import ModelType
@@ -101,13 +102,6 @@ class DatasetListApi(Resource):
             help="type is required. Name must be between 1 to 40 characters.",
             type=_validate_name,
         )
-        parser.add_argument(
-            "description",
-            type=str,
-            nullable=True,
-            required=False,
-            default="",
-        )
         parser.add_argument(
             "indexing_technique",
             type=str,
@@ -146,7 +140,6 @@ class DatasetListApi(Resource):
             dataset = DatasetService.create_empty_dataset(
                 tenant_id=current_user.current_tenant_id,
                 name=args["name"],
-                description=args["description"],
                 indexing_technique=args["indexing_technique"],
                 account=current_user,
                 permission=DatasetPermissionEnum.ONLY_ME,
@@ -456,7 +449,7 @@ class DatasetIndexingEstimateApi(Resource):
             )
         except LLMBadRequestError:
             raise ProviderNotInitializeError(
-                "No Embedding Model available. Please configure a valid provider " "in the Settings -> Model Provider."
+                "No Embedding Model available. Please configure a valid provider in the Settings -> Model Provider."
             )
         except ProviderTokenNotInitError as ex:
             raise ProviderNotInitializeError(ex.description)
@@ -620,15 +613,12 @@ class DatasetRetrievalSettingApi(Resource):
             case (
                 VectorType.MILVUS
                 | VectorType.RELYT
-                | VectorType.PGVECTOR
                 | VectorType.TIDB_VECTOR
                 | VectorType.CHROMA
                 | VectorType.TENCENT
                 | VectorType.PGVECTO_RS
                 | VectorType.BAIDU
                 | VectorType.VIKINGDB
-                | VectorType.UPSTASH
-                | VectorType.OCEANBASE
             ):
                 return {"retrieval_method": [RetrievalMethod.SEMANTIC_SEARCH.value]}
             case (
@@ -640,9 +630,6 @@ class DatasetRetrievalSettingApi(Resource):
                 | VectorType.ORACLE
                 | VectorType.ELASTICSEARCH
                 | VectorType.PGVECTOR
-                | VectorType.TIDB_ON_QDRANT
-                | VectorType.LINDORM
-                | VectorType.COUCHBASE
             ):
                 return {
                     "retrieval_method": [
@@ -670,8 +657,6 @@ class DatasetRetrievalSettingMockApi(Resource):
                 | VectorType.PGVECTO_RS
                 | VectorType.BAIDU
                 | VectorType.VIKINGDB
-                | VectorType.UPSTASH
-                | VectorType.OCEANBASE
             ):
                 return {"retrieval_method": [RetrievalMethod.SEMANTIC_SEARCH.value]}
             case (
@@ -682,9 +667,7 @@ class DatasetRetrievalSettingMockApi(Resource):
                 | VectorType.MYSCALE
                 | VectorType.ORACLE
                 | VectorType.ELASTICSEARCH
-                | VectorType.COUCHBASE
                 | VectorType.PGVECTOR
-                | VectorType.LINDORM
             ):
                 return {
                     "retrieval_method": [

api/controllers/console/datasets/datasets_document.py

@@ -24,11 +24,8 @@ from controllers.console.datasets.error import (
     InvalidActionError,
     InvalidMetadataError,
 )
-from controllers.console.wraps import (
-    account_initialization_required,
-    cloud_edition_billing_resource_check,
-    setup_required,
-)
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required, cloud_edition_billing_resource_check
 from core.errors.error import (
     LLMBadRequestError,
     ModelCurrentlyNotSupportError,

api/controllers/console/datasets/datasets_segments.py

@@ -11,11 +11,11 @@ import services
 from controllers.console import api
 from controllers.console.app.error import ProviderNotInitializeError
 from controllers.console.datasets.error import InvalidActionError, NoFileUploadedError, TooManyFilesError
+from controllers.console.setup import setup_required
 from controllers.console.wraps import (
     account_initialization_required,
     cloud_edition_billing_knowledge_limit_check,
     cloud_edition_billing_resource_check,
-    setup_required,
 )
 from core.errors.error import LLMBadRequestError, ProviderTokenNotInitError
 from core.model_manager import ModelManager

api/controllers/console/datasets/external.py

@@ -6,7 +6,8 @@ from werkzeug.exceptions import Forbidden, InternalServerError, NotFound
 import services
 from controllers.console import api
 from controllers.console.datasets.error import DatasetNameDuplicateError
-from controllers.console.wraps import account_initialization_required, setup_required
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required
 from fields.dataset_fields import dataset_detail_fields
 from libs.login import login_required
 from services.dataset_service import DatasetService

api/controllers/console/datasets/file.py

@@ -1,3 +1,5 @@
+import urllib.parse
+
 from flask import request
 from flask_login import current_user
 from flask_restful import Resource, marshal_with
@@ -5,22 +7,19 @@ from flask_restful import Resource, marshal_with
 import services
 from configs import dify_config
 from constants import DOCUMENT_EXTENSIONS
-from controllers.common.errors import FilenameNotExistsError
-from controllers.console.wraps import (
-    account_initialization_required,
-    cloud_edition_billing_resource_check,
-    setup_required,
-)
-from fields.file_fields import file_fields, upload_config_fields
-from libs.login import login_required
-from services.file_service import FileService
-
-from .errors import (
+from controllers.console import api
+from controllers.console.datasets.error import (
     FileTooLargeError,
     NoFileUploadedError,
     TooManyFilesError,
     UnsupportedFileTypeError,
 )
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required, cloud_edition_billing_resource_check
+from core.helper import ssrf_proxy
+from fields.file_fields import file_fields, remote_file_info_fields, upload_config_fields
+from libs.login import login_required
+from services.file_service import FileService
 
 PREVIEW_WORDS_LIMIT = 3000
 
@@ -31,13 +30,13 @@ class FileApi(Resource):
     @account_initialization_required
     @marshal_with(upload_config_fields)
     def get(self):
+        file_size_limit = dify_config.UPLOAD_FILE_SIZE_LIMIT
+        batch_count_limit = dify_config.UPLOAD_FILE_BATCH_LIMIT
+        image_file_size_limit = dify_config.UPLOAD_IMAGE_FILE_SIZE_LIMIT
         return {
-            "file_size_limit": dify_config.UPLOAD_FILE_SIZE_LIMIT,
-            "batch_count_limit": dify_config.UPLOAD_FILE_BATCH_LIMIT,
-            "image_file_size_limit": dify_config.UPLOAD_IMAGE_FILE_SIZE_LIMIT,
-            "video_file_size_limit": dify_config.UPLOAD_VIDEO_FILE_SIZE_LIMIT,
-            "audio_file_size_limit": dify_config.UPLOAD_AUDIO_FILE_SIZE_LIMIT,
-            "workflow_file_upload_limit": dify_config.WORKFLOW_FILE_UPLOAD_LIMIT,
+            "file_size_limit": file_size_limit,
+            "batch_count_limit": batch_count_limit,
+            "image_file_size_limit": image_file_size_limit,
         }, 200
 
     @setup_required
@@ -46,29 +45,17 @@ class FileApi(Resource):
     @marshal_with(file_fields)
     @cloud_edition_billing_resource_check("documents")
     def post(self):
+        # get file from request
         file = request.files["file"]
-        source = request.form.get("source")
 
+        # check file
         if "file" not in request.files:
             raise NoFileUploadedError()
 
         if len(request.files) > 1:
             raise TooManyFilesError()
-
-        if not file.filename:
-            raise FilenameNotExistsError
-
-        if source not in ("datasets", None):
-            source = None
-
         try:
-            upload_file = FileService.upload_file(
-                filename=file.filename,
-                content=file.read(),
-                mimetype=file.mimetype,
-                user=current_user,
-                source=source,
-            )
+            upload_file = FileService.upload_file(file=file, user=current_user)
         except services.errors.file.FileTooLargeError as file_too_large_error:
             raise FileTooLargeError(file_too_large_error.description)
         except services.errors.file.UnsupportedFileTypeError:
@@ -93,3 +80,23 @@ class FileSupportTypeApi(Resource):
     @account_initialization_required
     def get(self):
         return {"allowed_extensions": DOCUMENT_EXTENSIONS}
+
+
+class RemoteFileInfoApi(Resource):
+    @marshal_with(remote_file_info_fields)
+    def get(self, url):
+        decoded_url = urllib.parse.unquote(url)
+        try:
+            response = ssrf_proxy.head(decoded_url)
+            return {
+                "file_type": response.headers.get("Content-Type", "application/octet-stream"),
+                "file_length": int(response.headers.get("Content-Length", 0)),
+            }
+        except Exception as e:
+            return {"error": str(e)}, 400
+
+
+api.add_resource(FileApi, "/files/upload")
+api.add_resource(FilePreviewApi, "/files/<uuid:file_id>/preview")
+api.add_resource(FileSupportTypeApi, "/files/support-type")
+api.add_resource(RemoteFileInfoApi, "/remote-files/<path:url>")

api/controllers/console/datasets/fta_test.py

@@ -1,145 +0,0 @@
-import json
-
-import requests
-from flask import Response
-from flask_restful import Resource, reqparse
-from sqlalchemy import text
-
-from controllers.console import api
-from extensions.ext_database import db
-from extensions.ext_storage import storage
-from models.fta import ComponentFailure, ComponentFailureStats
-
-
-class FATTestApi(Resource):
-    def post(self):
-        parser = reqparse.RequestParser()
-        parser.add_argument("log_process_data", nullable=False, required=True, type=str, location="args")
-        args = parser.parse_args()
-        print(args["log_process_data"])
-        # Extract the JSON string from the text field
-        json_str = args["log_process_data"].strip("```json\\n").strip("```").strip().replace("\\n", "")
-        log_data = json.loads(json_str)
-        db.session.query(ComponentFailure).delete()
-        for data in log_data:
-            if not isinstance(data, dict):
-                raise TypeError("Data must be a dictionary.")
-
-            required_keys = {"Date", "Component", "FailureMode", "Cause", "RepairAction", "Technician"}
-            if not required_keys.issubset(data.keys()):
-                raise ValueError(f"Data dictionary must contain the following keys: {required_keys}")
-
-            try:
-                # Clear existing stats
-                component_failure = ComponentFailure(
-                    Date=data["Date"],
-                    Component=data["Component"],
-                    FailureMode=data["FailureMode"],
-                    Cause=data["Cause"],
-                    RepairAction=data["RepairAction"],
-                    Technician=data["Technician"],
-                )
-                db.session.add(component_failure)
-                db.session.commit()
-            except Exception as e:
-                print(e)
-        # Clear existing stats
-        db.session.query(ComponentFailureStats).delete()
-
-        # Insert calculated statistics
-        try:
-            db.session.execute(
-                text("""
-                    INSERT INTO component_failure_stats ("Component", "FailureMode", "Cause", "PossibleAction", "Probability", "MTBF")
-                    SELECT 
-                        cf."Component",
-                        cf."FailureMode",
-                        cf."Cause",
-                        cf."RepairAction" as "PossibleAction",
-                        COUNT(*) * 1.0 / (SELECT COUNT(*) FROM component_failure WHERE "Component" = cf."Component") AS "Probability",
-                        COALESCE(AVG(EXTRACT(EPOCH FROM (next_failure_date::timestamp - cf."Date"::timestamp)) / 86400.0),0)AS "MTBF"
-                    FROM (
-                        SELECT 
-                            "Component",
-                            "FailureMode",
-                            "Cause",
-                            "RepairAction",
-                            "Date", 
-                            LEAD("Date") OVER (PARTITION BY "Component", "FailureMode", "Cause" ORDER BY "Date") AS next_failure_date
-                        FROM 
-                            component_failure
-                    ) cf
-                    GROUP BY 
-                        cf."Component", cf."FailureMode", cf."Cause", cf."RepairAction";
-            """)
-            )
-            db.session.commit()
-        except Exception as e:
-            db.session.rollback()
-            print(f"Error during stats calculation: {e}")
-        # output format
-        # [
-        #     (17, 'Hydraulic system', 'Leak', 'Hose rupture', 'Replaced hydraulic hose', 0.3333333333333333, None),
-        #     (18, 'Hydraulic system', 'Leak', 'Seal Wear', 'Replaced the faulty seal', 0.3333333333333333, None),
-        #     (19, 'Hydraulic system', 'Pressure drop', 'Fluid leak', 'Replaced hydraulic fluid and seals', 0.3333333333333333, None)
-        # ]
-
-        component_failure_stats = db.session.query(ComponentFailureStats).all()
-        # Convert stats to list of tuples format
-        stats_list = []
-        for stat in component_failure_stats:
-            stats_list.append(
-                (
-                    stat.StatID,
-                    stat.Component,
-                    stat.FailureMode,
-                    stat.Cause,
-                    stat.PossibleAction,
-                    stat.Probability,
-                    stat.MTBF,
-                )
-            )
-        return {"data": stats_list}, 200
-
-
-# generate-fault-tree
-class GenerateFaultTreeApi(Resource):
-    def post(self):
-        parser = reqparse.RequestParser()
-        parser.add_argument("llm_text", nullable=False, required=True, type=str, location="args")
-        args = parser.parse_args()
-        entities = args["llm_text"].replace("```", "").replace("\\n", "\n")
-        print(entities)
-        request_data = {"fault_tree_text": entities}
-        url = "https://fta.cognitech-dev.live/generate-fault-tree"
-        headers = {"accept": "application/json", "Content-Type": "application/json"}
-
-        response = requests.post(url, json=request_data, headers=headers)
-        print(response.json())
-        return {"data": response.json()}, 200
-
-
-class ExtractSVGApi(Resource):
-    def post(self):
-        parser = reqparse.RequestParser()
-        parser.add_argument("svg_text", nullable=False, required=True, type=str, location="args")
-        args = parser.parse_args()
-        # svg_text = ''.join(args["svg_text"].splitlines())
-        svg_text = args["svg_text"].replace("\n", "")
-        svg_text = svg_text.replace('"', '"')
-        print(svg_text)
-        svg_text_json = json.loads(svg_text)
-        svg_content = svg_text_json.get("data").get("svg_content")[0]
-        svg_content = svg_content.replace("\n", "").replace('"', '"')
-        file_key = "fta_svg/" + "fat.svg"
-        if storage.exists(file_key):
-            storage.delete(file_key)
-        storage.save(file_key, svg_content.encode("utf-8"))
-        generator = storage.load(file_key, stream=True)
-
-        return Response(generator, mimetype="image/svg+xml")
-
-
-api.add_resource(FATTestApi, "/fta/db-handler")
-api.add_resource(GenerateFaultTreeApi, "/fta/generate-fault-tree")
-api.add_resource(ExtractSVGApi, "/fta/extract-svg")

api/controllers/console/datasets/hit_testing.py

@@ -1,23 +1,88 @@
-from flask_restful import Resource
+import logging
 
+from flask_login import current_user
+from flask_restful import Resource, marshal, reqparse
+from werkzeug.exceptions import Forbidden, InternalServerError, NotFound
+
+import services
 from controllers.console import api
-from controllers.console.datasets.hit_testing_base import DatasetsHitTestingBase
-from controllers.console.wraps import account_initialization_required, setup_required
+from controllers.console.app.error import (
+    CompletionRequestError,
+    ProviderModelCurrentlyNotSupportError,
+    ProviderNotInitializeError,
+    ProviderQuotaExceededError,
+)
+from controllers.console.datasets.error import DatasetNotInitializedError
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required
+from core.errors.error import (
+    LLMBadRequestError,
+    ModelCurrentlyNotSupportError,
+    ProviderTokenNotInitError,
+    QuotaExceededError,
+)
+from core.model_runtime.errors.invoke import InvokeError
+from fields.hit_testing_fields import hit_testing_record_fields
 from libs.login import login_required
+from services.dataset_service import DatasetService
+from services.hit_testing_service import HitTestingService
 
 
-class HitTestingApi(Resource, DatasetsHitTestingBase):
+class HitTestingApi(Resource):
     @setup_required
     @login_required
     @account_initialization_required
     def post(self, dataset_id):
         dataset_id_str = str(dataset_id)
 
-        dataset = self.get_and_validate_dataset(dataset_id_str)
-        args = self.parse_args()
-        self.hit_testing_args_check(args)
+        dataset = DatasetService.get_dataset(dataset_id_str)
+        if dataset is None:
+            raise NotFound("Dataset not found.")
 
-        return self.perform_hit_testing(dataset, args)
+        try:
+            DatasetService.check_dataset_permission(dataset, current_user)
+        except services.errors.account.NoPermissionError as e:
+            raise Forbidden(str(e))
+
+        parser = reqparse.RequestParser()
+        parser.add_argument("query", type=str, location="json")
+        parser.add_argument("retrieval_model", type=dict, required=False, location="json")
+        parser.add_argument("external_retrieval_model", type=dict, required=False, location="json")
+        args = parser.parse_args()
+
+        HitTestingService.hit_testing_args_check(args)
+
+        try:
+            response = HitTestingService.retrieve(
+                dataset=dataset,
+                query=args["query"],
+                account=current_user,
+                retrieval_model=args["retrieval_model"],
+                external_retrieval_model=args["external_retrieval_model"],
+                limit=10,
+            )
+
+            return {"query": response["query"], "records": marshal(response["records"], hit_testing_record_fields)}
+        except services.errors.index.IndexNotInitializedError:
+            raise DatasetNotInitializedError()
+        except ProviderTokenNotInitError as ex:
+            raise ProviderNotInitializeError(ex.description)
+        except QuotaExceededError:
+            raise ProviderQuotaExceededError()
+        except ModelCurrentlyNotSupportError:
+            raise ProviderModelCurrentlyNotSupportError()
+        except LLMBadRequestError:
+            raise ProviderNotInitializeError(
+                "No Embedding Model or Reranking Model available. Please configure a valid provider "
+                "in the Settings -> Model Provider."
+            )
+        except InvokeError as e:
+            raise CompletionRequestError(e.description)
+        except ValueError as e:
+            raise ValueError(str(e))
+        except Exception as e:
+            logging.exception("Hit testing failed.")
+            raise InternalServerError(str(e))
 
 
 api.add_resource(HitTestingApi, "/datasets/<uuid:dataset_id>/hit-testing")

api/controllers/console/datasets/hit_testing_base.py

@@ -1,85 +0,0 @@
-import logging
-
-from flask_login import current_user
-from flask_restful import marshal, reqparse
-from werkzeug.exceptions import Forbidden, InternalServerError, NotFound
-
-import services.dataset_service
-from controllers.console.app.error import (
-    CompletionRequestError,
-    ProviderModelCurrentlyNotSupportError,
-    ProviderNotInitializeError,
-    ProviderQuotaExceededError,
-)
-from controllers.console.datasets.error import DatasetNotInitializedError
-from core.errors.error import (
-    LLMBadRequestError,
-    ModelCurrentlyNotSupportError,
-    ProviderTokenNotInitError,
-    QuotaExceededError,
-)
-from core.model_runtime.errors.invoke import InvokeError
-from fields.hit_testing_fields import hit_testing_record_fields
-from services.dataset_service import DatasetService
-from services.hit_testing_service import HitTestingService
-
-
-class DatasetsHitTestingBase:
-    @staticmethod
-    def get_and_validate_dataset(dataset_id: str):
-        dataset = DatasetService.get_dataset(dataset_id)
-        if dataset is None:
-            raise NotFound("Dataset not found.")
-
-        try:
-            DatasetService.check_dataset_permission(dataset, current_user)
-        except services.errors.account.NoPermissionError as e:
-            raise Forbidden(str(e))
-
-        return dataset
-
-    @staticmethod
-    def hit_testing_args_check(args):
-        HitTestingService.hit_testing_args_check(args)
-
-    @staticmethod
-    def parse_args():
-        parser = reqparse.RequestParser()
-
-        parser.add_argument("query", type=str, location="json")
-        parser.add_argument("retrieval_model", type=dict, required=False, location="json")
-        parser.add_argument("external_retrieval_model", type=dict, required=False, location="json")
-        return parser.parse_args()
-
-    @staticmethod
-    def perform_hit_testing(dataset, args):
-        try:
-            response = HitTestingService.retrieve(
-                dataset=dataset,
-                query=args["query"],
-                account=current_user,
-                retrieval_model=args["retrieval_model"],
-                external_retrieval_model=args["external_retrieval_model"],
-                limit=10,
-            )
-            return {"query": response["query"], "records": marshal(response["records"], hit_testing_record_fields)}
-        except services.errors.index.IndexNotInitializedError:
-            raise DatasetNotInitializedError()
-        except ProviderTokenNotInitError as ex:
-            raise ProviderNotInitializeError(ex.description)
-        except QuotaExceededError:
-            raise ProviderQuotaExceededError()
-        except ModelCurrentlyNotSupportError:
-            raise ProviderModelCurrentlyNotSupportError()
-        except LLMBadRequestError:
-            raise ProviderNotInitializeError(
-                "No Embedding Model or Reranking Model available. Please configure a valid provider "
-                "in the Settings -> Model Provider."
-            )
-        except InvokeError as e:
-            raise CompletionRequestError(e.description)
-        except ValueError as e:
-            raise ValueError(str(e))
-        except Exception as e:
-            logging.exception("Hit testing failed.")
-            raise InternalServerError(str(e))

api/controllers/console/datasets/website.py

@@ -2,7 +2,8 @@ from flask_restful import Resource, reqparse
 
 from controllers.console import api
 from controllers.console.datasets.error import WebsiteCrawlError
-from controllers.console.wraps import account_initialization_required, setup_required
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required
 from libs.login import login_required
 from services.website_service import WebsiteService
 

api/controllers/console/error.py

@@ -38,27 +38,3 @@ class AlreadyActivateError(BaseHTTPException):
     error_code = "already_activate"
     description = "Auth Token is invalid or account already activated, please check again."
     code = 403
-
-
-class NotAllowedCreateWorkspace(BaseHTTPException):
-    error_code = "not_allowed_create_workspace"
-    description = "Workspace not found, please contact system admin to invite you to join in a workspace."
-    code = 400
-
-
-class AccountBannedError(BaseHTTPException):
-    error_code = "account_banned"
-    description = "Account is banned."
-    code = 400
-
-
-class NotAllowedRegister(BaseHTTPException):
-    error_code = "unauthorized"
-    description = "Account not found."
-    code = 400
-
-
-class EmailSendIpLimitError(BaseHTTPException):
-    error_code = "email_send_ip_limit"
-    description = "Too many emails have been sent from this IP address recently. Please try again later."
-    code = 429

api/controllers/console/explore/parameter.py

@@ -1,7 +1,6 @@
-from flask_restful import marshal_with
+from flask_restful import fields, marshal_with
 
-from controllers.common import fields
-from controllers.common import helpers as controller_helpers
+from configs import dify_config
 from controllers.console import api
 from controllers.console.app.error import AppUnavailableError
 from controllers.console.explore.wraps import InstalledAppResource
@@ -12,14 +11,38 @@ from services.app_service import AppService
 class AppParameterApi(InstalledAppResource):
     """Resource for app variables."""
 
-    @marshal_with(fields.parameters_fields)
+    variable_fields = {
+        "key": fields.String,
+        "name": fields.String,
+        "description": fields.String,
+        "type": fields.String,
+        "default": fields.String,
+        "max_length": fields.Integer,
+        "options": fields.List(fields.String),
+    }
+
+    system_parameters_fields = {"image_file_size_limit": fields.String}
+
+    parameters_fields = {
+        "opening_statement": fields.String,
+        "suggested_questions": fields.Raw,
+        "suggested_questions_after_answer": fields.Raw,
+        "speech_to_text": fields.Raw,
+        "text_to_speech": fields.Raw,
+        "retriever_resource": fields.Raw,
+        "annotation_reply": fields.Raw,
+        "more_like_this": fields.Raw,
+        "user_input_form": fields.Raw,
+        "sensitive_word_avoidance": fields.Raw,
+        "file_upload": fields.Raw,
+        "system_parameters": fields.Nested(system_parameters_fields),
+    }
+
+    @marshal_with(parameters_fields)
     def get(self, installed_app: InstalledApp):
         """Retrieve app parameters."""
         app_model = installed_app.app
 
-        if app_model is None:
-            raise AppUnavailableError()
-
         if app_model.mode in {AppMode.ADVANCED_CHAT.value, AppMode.WORKFLOW.value}:
             workflow = app_model.workflow
             if workflow is None:
@@ -29,16 +52,38 @@ class AppParameterApi(InstalledAppResource):
             user_input_form = workflow.user_input_form(to_old_structure=True)
         else:
             app_model_config = app_model.app_model_config
-            if app_model_config is None:
-                raise AppUnavailableError()
-
             features_dict = app_model_config.to_dict()
 
             user_input_form = features_dict.get("user_input_form", [])
 
-        return controller_helpers.get_parameters_from_feature_dict(
-            features_dict=features_dict, user_input_form=user_input_form
-        )
+        return {
+            "opening_statement": features_dict.get("opening_statement"),
+            "suggested_questions": features_dict.get("suggested_questions", []),
+            "suggested_questions_after_answer": features_dict.get(
+                "suggested_questions_after_answer", {"enabled": False}
+            ),
+            "speech_to_text": features_dict.get("speech_to_text", {"enabled": False}),
+            "text_to_speech": features_dict.get("text_to_speech", {"enabled": False}),
+            "retriever_resource": features_dict.get("retriever_resource", {"enabled": False}),
+            "annotation_reply": features_dict.get("annotation_reply", {"enabled": False}),
+            "more_like_this": features_dict.get("more_like_this", {"enabled": False}),
+            "user_input_form": user_input_form,
+            "sensitive_word_avoidance": features_dict.get(
+                "sensitive_word_avoidance", {"enabled": False, "type": "", "configs": []}
+            ),
+            "file_upload": features_dict.get(
+                "file_upload",
+                {
+                    "image": {
+                        "enabled": False,
+                        "number_limits": 3,
+                        "detail": "high",
+                        "transfer_methods": ["remote_url", "local_file"],
+                    }
+                },
+            ),
+            "system_parameters": {"image_file_size_limit": dify_config.UPLOAD_IMAGE_FILE_SIZE_LIMIT},
+        }
 
 
 class ExploreAppMetaApi(InstalledAppResource):

api/controllers/console/extension.py

@@ -3,7 +3,8 @@ from flask_restful import Resource, marshal_with, reqparse
 
 from constants import HIDDEN_VALUE
 from controllers.console import api
-from controllers.console.wraps import account_initialization_required, setup_required
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required
 from fields.api_based_extension_fields import api_based_extension_fields
 from libs.login import login_required
 from models.api_based_extension import APIBasedExtension

api/controllers/console/feature.py

@@ -5,7 +5,8 @@ from libs.login import login_required
 from services.feature_service import FeatureService
 
 from . import api
-from .wraps import account_initialization_required, cloud_utm_record, setup_required
+from .setup import setup_required
+from .wraps import account_initialization_required, cloud_utm_record
 
 
 class FeatureApi(Resource):

api/controllers/console/files/errors.py

@@ -1,25 +0,0 @@
-from libs.exception import BaseHTTPException
-
-
-class FileTooLargeError(BaseHTTPException):
-    error_code = "file_too_large"
-    description = "File size exceeded. {message}"
-    code = 413
-
-
-class UnsupportedFileTypeError(BaseHTTPException):
-    error_code = "unsupported_file_type"
-    description = "File type not allowed."
-    code = 415
-
-
-class TooManyFilesError(BaseHTTPException):
-    error_code = "too_many_files"
-    description = "Only one file is allowed."
-    code = 400
-
-
-class NoFileUploadedError(BaseHTTPException):
-    error_code = "no_file_uploaded"
-    description = "Please upload your file."
-    code = 400

api/controllers/console/remote_files.py

@@ -1,71 +0,0 @@
-import urllib.parse
-from typing import cast
-
-from flask_login import current_user
-from flask_restful import Resource, marshal_with, reqparse
-
-from controllers.common import helpers
-from core.file import helpers as file_helpers
-from core.helper import ssrf_proxy
-from fields.file_fields import file_fields_with_signed_url, remote_file_info_fields
-from models.account import Account
-from services.file_service import FileService
-
-
-class RemoteFileInfoApi(Resource):
-    @marshal_with(remote_file_info_fields)
-    def get(self, url):
-        decoded_url = urllib.parse.unquote(url)
-        try:
-            response = ssrf_proxy.head(decoded_url)
-            return {
-                "file_type": response.headers.get("Content-Type", "application/octet-stream"),
-                "file_length": int(response.headers.get("Content-Length", 0)),
-            }
-        except Exception as e:
-            return {"error": str(e)}, 400
-
-
-class RemoteFileUploadApi(Resource):
-    @marshal_with(file_fields_with_signed_url)
-    def post(self):
-        parser = reqparse.RequestParser()
-        parser.add_argument("url", type=str, required=True, help="URL is required")
-        args = parser.parse_args()
-
-        url = args["url"]
-
-        response = ssrf_proxy.head(url)
-        response.raise_for_status()
-
-        file_info = helpers.guess_file_info_from_response(response)
-
-        if not FileService.is_file_size_within_limit(extension=file_info.extension, file_size=file_info.size):
-            return {"error": "File size exceeded"}, 400
-
-        response = ssrf_proxy.get(url)
-        response.raise_for_status()
-        content = response.content
-
-        try:
-            user = cast(Account, current_user)
-            upload_file = FileService.upload_file(
-                filename=file_info.filename,
-                content=content,
-                mimetype=file_info.mimetype,
-                user=user,
-                source_url=url,
-            )
-        except Exception as e:
-            return {"error": str(e)}, 400
-
-        return {
-            "id": upload_file.id,
-            "name": upload_file.name,
-            "size": upload_file.size,
-            "extension": upload_file.extension,
-            "url": file_helpers.get_signed_file_url(upload_file_id=upload_file.id),
-            "mime_type": upload_file.mime_type,
-            "created_by": upload_file.created_by,
-            "created_at": upload_file.created_at,
-        }, 201

api/controllers/console/setup.py

@@ -1,3 +1,5 @@
+from functools import wraps
+
 from flask import request
 from flask_restful import Resource, reqparse
 
@@ -8,7 +10,7 @@ from models.model import DifySetup
 from services.account_service import RegisterService, TenantService
 
 from . import api
-from .error import AlreadySetupError, NotInitValidateError
+from .error import AlreadySetupError, NotInitValidateError, NotSetupError
 from .init_validate import get_init_validate_status
 from .wraps import only_edition_self_hosted
 
@@ -50,10 +52,26 @@ class SetupApi(Resource):
         return {"result": "success"}, 201
 
 
+def setup_required(view):
+    @wraps(view)
+    def decorated(*args, **kwargs):
+        # check setup
+        if not get_init_validate_status():
+            raise NotInitValidateError()
+
+        elif not get_setup_status():
+            raise NotSetupError()
+
+        return view(*args, **kwargs)
+
+    return decorated
+
+
 def get_setup_status():
     if dify_config.EDITION == "SELF_HOSTED":
         return DifySetup.query.first()
-    return True
+    else:
+        return True
 
 
 api.add_resource(SetupApi, "/setup")

api/controllers/console/tag/tags.py

@@ -4,7 +4,8 @@ from flask_restful import Resource, marshal_with, reqparse
 from werkzeug.exceptions import Forbidden
 
 from controllers.console import api
-from controllers.console.wraps import account_initialization_required, setup_required
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required
 from fields.tag_fields import tag_fields
 from libs.login import login_required
 from models.model import Tag

api/controllers/console/version.py

@@ -3,7 +3,6 @@ import logging
 
 import requests
 from flask_restful import Resource, reqparse
-from packaging import version
 
 from configs import dify_config
 
@@ -48,15 +47,43 @@ class VersionApi(Resource):
 
 
 def _has_new_version(*, latest_version: str, current_version: str) -> bool:
-    try:
-        latest = version.parse(latest_version)
-        current = version.parse(current_version)
+    def parse_version(version: str) -> tuple:
+        # Split version into parts and pre-release suffix if any
+        parts = version.split("-")
+        version_parts = parts[0].split(".")
+        pre_release = parts[1] if len(parts) > 1 else None
 
-        # Compare versions
-        return latest > current
-    except version.InvalidVersion:
-        logging.warning(f"Invalid version format: latest={latest_version}, current={current_version}")
+        # Validate version format
+        if len(version_parts) != 3:
+            raise ValueError(f"Invalid version format: {version}")
+
+        try:
+            # Convert version parts to integers
+            major, minor, patch = map(int, version_parts)
+            return (major, minor, patch, pre_release)
+        except ValueError:
+            raise ValueError(f"Invalid version format: {version}")
+
+    latest = parse_version(latest_version)
+    current = parse_version(current_version)
+
+    # Compare major, minor, and patch versions
+    for latest_part, current_part in zip(latest[:3], current[:3]):
+        if latest_part > current_part:
+            return True
+        elif latest_part < current_part:
+            return False
+
+    # If versions are equal, check pre-release suffixes
+    if latest[3] is None and current[3] is not None:
+        return True
+    elif latest[3] is not None and current[3] is None:
         return False
+    elif latest[3] is not None and current[3] is not None:
+        # Simple string comparison for pre-release versions
+        return latest[3] > current[3]
+
+    return False
 
 
 api.add_resource(VersionApi, "/version")

api/controllers/console/workspace/account.py

@@ -8,13 +8,14 @@ from flask_restful import Resource, fields, marshal_with, reqparse
 from configs import dify_config
 from constants.languages import supported_language
 from controllers.console import api
+from controllers.console.setup import setup_required
 from controllers.console.workspace.error import (
     AccountAlreadyInitedError,
     CurrentPasswordIncorrectError,
     InvalidInvitationCodeError,
     RepeatPasswordNotMatchError,
 )
-from controllers.console.wraps import account_initialization_required, setup_required
+from controllers.console.wraps import account_initialization_required
 from extensions.ext_database import db
 from fields.member_fields import account_fields
 from libs.helper import TimestampField, timezone

api/controllers/console/workspace/load_balancing_config.py

@@ -2,7 +2,8 @@ from flask_restful import Resource, reqparse
 from werkzeug.exceptions import Forbidden
 
 from controllers.console import api
-from controllers.console.wraps import account_initialization_required, setup_required
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required
 from core.model_runtime.entities.model_entities import ModelType
 from core.model_runtime.errors.validate import CredentialsValidateFailedError
 from libs.login import current_user, login_required

api/controllers/console/workspace/members.py

@@ -4,11 +4,8 @@ from flask_restful import Resource, abort, marshal_with, reqparse
 import services
 from configs import dify_config
 from controllers.console import api
-from controllers.console.wraps import (
-    account_initialization_required,
-    cloud_edition_billing_resource_check,
-    setup_required,
-)
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required, cloud_edition_billing_resource_check
 from extensions.ext_database import db
 from fields.member_fields import account_with_role_list_fields
 from libs.login import login_required

api/controllers/console/workspace/model_providers.py

@@ -6,7 +6,8 @@ from flask_restful import Resource, reqparse
 from werkzeug.exceptions import Forbidden
 
 from controllers.console import api
-from controllers.console.wraps import account_initialization_required, setup_required
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required
 from core.model_runtime.entities.model_entities import ModelType
 from core.model_runtime.errors.validate import CredentialsValidateFailedError
 from core.model_runtime.utils.encoders import jsonable_encoder

api/controllers/console/workspace/models.py

@@ -5,7 +5,8 @@ from flask_restful import Resource, reqparse
 from werkzeug.exceptions import Forbidden
 
 from controllers.console import api
-from controllers.console.wraps import account_initialization_required, setup_required
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required
 from core.model_runtime.entities.model_entities import ModelType
 from core.model_runtime.errors.validate import CredentialsValidateFailedError
 from core.model_runtime.utils.encoders import jsonable_encoder

api/controllers/console/workspace/tool_providers.py

@@ -7,7 +7,8 @@ from werkzeug.exceptions import Forbidden
 
 from configs import dify_config
 from controllers.console import api
-from controllers.console.wraps import account_initialization_required, setup_required
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required
 from core.model_runtime.utils.encoders import jsonable_encoder
 from libs.helper import alphanumeric, uuid_value
 from libs.login import login_required

api/controllers/console/workspace/workspace.py

@@ -6,7 +6,6 @@ from flask_restful import Resource, fields, inputs, marshal, marshal_with, reqpa
 from werkzeug.exceptions import Unauthorized
 
 import services
-from controllers.common.errors import FilenameNotExistsError
 from controllers.console import api
 from controllers.console.admin import admin_required
 from controllers.console.datasets.error import (
@@ -16,11 +15,8 @@ from controllers.console.datasets.error import (
     UnsupportedFileTypeError,
 )
 from controllers.console.error import AccountNotLinkTenantError
-from controllers.console.wraps import (
-    account_initialization_required,
-    cloud_edition_billing_resource_check,
-    setup_required,
-)
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required, cloud_edition_billing_resource_check
 from extensions.ext_database import db
 from libs.helper import TimestampField
 from libs.login import login_required
@@ -197,20 +193,12 @@ class WebappLogoWorkspaceApi(Resource):
         if len(request.files) > 1:
             raise TooManyFilesError()
 
-        if not file.filename:
-            raise FilenameNotExistsError
-
         extension = file.filename.split(".")[-1]
         if extension.lower() not in {"svg", "png"}:
             raise UnsupportedFileTypeError()
 
         try:
-            upload_file = FileService.upload_file(
-                filename=file.filename,
-                content=file.read(),
-                mimetype=file.mimetype,
-                user=current_user,
-            )
+            upload_file = FileService.upload_file(file=file, user=current_user)
 
         except services.errors.file.FileTooLargeError as file_too_large_error:
             raise FileTooLargeError(file_too_large_error.description)

api/controllers/console/wraps.py

@@ -1,5 +1,4 @@
 import json
-import os
 from functools import wraps
 
 from flask import abort, request
@@ -7,12 +6,9 @@ from flask_login import current_user
 
 from configs import dify_config
 from controllers.console.workspace.error import AccountNotInitializedError
-from models.model import DifySetup
 from services.feature_service import FeatureService
 from services.operation_service import OperationService
 
-from .error import NotInitValidateError, NotSetupError
-
 
 def account_initialization_required(view):
     @wraps(view)
@@ -128,17 +124,3 @@ def cloud_utm_record(view):
         return view(*args, **kwargs)
 
     return decorated
-
-
-def setup_required(view):
-    @wraps(view)
-    def decorated(*args, **kwargs):
-        # check setup
-        if dify_config.EDITION == "SELF_HOSTED" and os.environ.get("INIT_PASSWORD") and not DifySetup.query.first():
-            raise NotInitValidateError()
-        elif dify_config.EDITION == "SELF_HOSTED" and not DifySetup.query.first():
-            raise NotSetupError()
-
-        return view(*args, **kwargs)
-
-    return decorated

api/controllers/files/image_preview.py

@@ -1,5 +1,5 @@
 from flask import Response, request
-from flask_restful import Resource, reqparse
+from flask_restful import Resource
 from werkzeug.exceptions import NotFound
 
 import services
@@ -10,10 +10,6 @@ from services.file_service import FileService
 
 
 class ImagePreviewApi(Resource):
-    """
-    Deprecated
-    """
-
     def get(self, file_id):
         file_id = str(file_id)
 
@@ -41,39 +37,24 @@ class FilePreviewApi(Resource):
     def get(self, file_id):
         file_id = str(file_id)
 
-        parser = reqparse.RequestParser()
-        parser.add_argument("timestamp", type=str, required=True, location="args")
-        parser.add_argument("nonce", type=str, required=True, location="args")
-        parser.add_argument("sign", type=str, required=True, location="args")
-        parser.add_argument("as_attachment", type=bool, required=False, default=False, location="args")
+        timestamp = request.args.get("timestamp")
+        nonce = request.args.get("nonce")
+        sign = request.args.get("sign")
 
-        args = parser.parse_args()
-
-        if not args["timestamp"] or not args["nonce"] or not args["sign"]:
+        if not timestamp or not nonce or not sign:
             return {"content": "Invalid request."}, 400
 
         try:
-            generator, upload_file = FileService.get_file_generator_by_file_id(
+            generator, mimetype = FileService.get_signed_file_preview(
                 file_id=file_id,
-                timestamp=args["timestamp"],
-                nonce=args["nonce"],
-                sign=args["sign"],
+                timestamp=timestamp,
+                nonce=nonce,
+                sign=sign,
             )
         except services.errors.file.UnsupportedFileTypeError:
             raise UnsupportedFileTypeError()
 
-        response = Response(
-            generator,
-            mimetype=upload_file.mime_type,
-            direct_passthrough=True,
-            headers={},
-        )
-        if upload_file.size > 0:
-            response.headers["Content-Length"] = str(upload_file.size)
-        if args["as_attachment"]:
-            response.headers["Content-Disposition"] = f"attachment; filename={upload_file.name}"
-
-        return response
+        return Response(generator, mimetype=mimetype)
 
 
 class WorkspaceWebappLogoApi(Resource):

api/controllers/files/tool_files.py

@@ -16,7 +16,6 @@ class ToolFilePreviewApi(Resource):
         parser.add_argument("timestamp", type=str, required=True, location="args")
         parser.add_argument("nonce", type=str, required=True, location="args")
         parser.add_argument("sign", type=str, required=True, location="args")
-        parser.add_argument("as_attachment", type=bool, required=False, default=False, location="args")
 
         args = parser.parse_args()
 
@@ -29,27 +28,18 @@ class ToolFilePreviewApi(Resource):
             raise Forbidden("Invalid request.")
 
         try:
-            stream, tool_file = ToolFileManager.get_file_generator_by_tool_file_id(
+            result = ToolFileManager.get_file_generator_by_tool_file_id(
                 file_id,
             )
 
-            if not stream or not tool_file:
+            if not result:
                 raise NotFound("file is not found")
+
+            generator, mimetype = result
         except Exception:
             raise UnsupportedFileTypeError()
 
-        response = Response(
-            stream,
-            mimetype=tool_file.mimetype,
-            direct_passthrough=True,
-            headers={},
-        )
-        if tool_file.size > 0:
-            response.headers["Content-Length"] = str(tool_file.size)
-        if args["as_attachment"]:
-            response.headers["Content-Disposition"] = f"attachment; filename={tool_file.name}"
-
-        return response
+        return Response(generator, mimetype=mimetype)
 
 
 api.add_resource(ToolFilePreviewApi, "/files/tools/<uuid:file_id>.<string:extension>")

api/controllers/inner_api/workspace/workspace.py

@@ -1,6 +1,6 @@
 from flask_restful import Resource, reqparse
 
-from controllers.console.wraps import setup_required
+from controllers.console.setup import setup_required
 from controllers.inner_api import api
 from controllers.inner_api.wraps import inner_api_only
 from events.tenant_event import tenant_was_created
@@ -21,7 +21,7 @@ class EnterpriseWorkspace(Resource):
         if account is None:
             return {"message": "owner account not found."}, 404
 
-        tenant = TenantService.create_tenant(args["name"], is_from_dashboard=True)
+        tenant = TenantService.create_tenant(args["name"])
         TenantService.create_tenant_member(tenant, account, role="owner")
 
         tenant_was_created.send(tenant)

api/controllers/service_api/__init__.py

@@ -5,6 +5,7 @@ from libs.external_api import ExternalApi
 bp = Blueprint("service_api", __name__, url_prefix="/v1")
 api = ExternalApi(bp)
 
+
 from . import index
 from .app import app, audio, completion, conversation, file, message, workflow
-from .dataset import dataset, document, hit_testing, segment
+from .dataset import dataset, document, segment

api/controllers/service_api/app/app.py

@@ -1,7 +1,6 @@
-from flask_restful import Resource, marshal_with
+from flask_restful import Resource, fields, marshal_with
 
-from controllers.common import fields
-from controllers.common import helpers as controller_helpers
+from configs import dify_config
 from controllers.service_api import api
 from controllers.service_api.app.error import AppUnavailableError
 from controllers.service_api.wraps import validate_app_token
@@ -12,8 +11,35 @@ from services.app_service import AppService
 class AppParameterApi(Resource):
     """Resource for app variables."""
 
+    variable_fields = {
+        "key": fields.String,
+        "name": fields.String,
+        "description": fields.String,
+        "type": fields.String,
+        "default": fields.String,
+        "max_length": fields.Integer,
+        "options": fields.List(fields.String),
+    }
+
+    system_parameters_fields = {"image_file_size_limit": fields.String}
+
+    parameters_fields = {
+        "opening_statement": fields.String,
+        "suggested_questions": fields.Raw,
+        "suggested_questions_after_answer": fields.Raw,
+        "speech_to_text": fields.Raw,
+        "text_to_speech": fields.Raw,
+        "retriever_resource": fields.Raw,
+        "annotation_reply": fields.Raw,
+        "more_like_this": fields.Raw,
+        "user_input_form": fields.Raw,
+        "sensitive_word_avoidance": fields.Raw,
+        "file_upload": fields.Raw,
+        "system_parameters": fields.Nested(system_parameters_fields),
+    }
+
     @validate_app_token
-    @marshal_with(fields.parameters_fields)
+    @marshal_with(parameters_fields)
     def get(self, app_model: App):
         """Retrieve app parameters."""
         if app_model.mode in {AppMode.ADVANCED_CHAT.value, AppMode.WORKFLOW.value}:
@@ -25,16 +51,38 @@ class AppParameterApi(Resource):
             user_input_form = workflow.user_input_form(to_old_structure=True)
         else:
             app_model_config = app_model.app_model_config
-            if app_model_config is None:
-                raise AppUnavailableError()
-
             features_dict = app_model_config.to_dict()
 
             user_input_form = features_dict.get("user_input_form", [])
 
-        return controller_helpers.get_parameters_from_feature_dict(
-            features_dict=features_dict, user_input_form=user_input_form
-        )
+        return {
+            "opening_statement": features_dict.get("opening_statement"),
+            "suggested_questions": features_dict.get("suggested_questions", []),
+            "suggested_questions_after_answer": features_dict.get(
+                "suggested_questions_after_answer", {"enabled": False}
+            ),
+            "speech_to_text": features_dict.get("speech_to_text", {"enabled": False}),
+            "text_to_speech": features_dict.get("text_to_speech", {"enabled": False}),
+            "retriever_resource": features_dict.get("retriever_resource", {"enabled": False}),
+            "annotation_reply": features_dict.get("annotation_reply", {"enabled": False}),
+            "more_like_this": features_dict.get("more_like_this", {"enabled": False}),
+            "user_input_form": user_input_form,
+            "sensitive_word_avoidance": features_dict.get(
+                "sensitive_word_avoidance", {"enabled": False, "type": "", "configs": []}
+            ),
+            "file_upload": features_dict.get(
+                "file_upload",
+                {
+                    "image": {
+                        "enabled": False,
+                        "number_limits": 3,
+                        "detail": "high",
+                        "transfer_methods": ["remote_url", "local_file"],
+                    }
+                },
+            ),
+            "system_parameters": {"image_file_size_limit": dify_config.UPLOAD_IMAGE_FILE_SIZE_LIMIT},
+        }
 
 
 class AppMetaApi(Resource):

api/controllers/service_api/app/completion.py

@@ -4,6 +4,7 @@ from flask_restful import Resource, reqparse
 from werkzeug.exceptions import InternalServerError, NotFound
 
 import services
+from constants import UUID_NIL
 from controllers.service_api import api
 from controllers.service_api.app.error import (
     AppUnavailableError,
@@ -107,6 +108,7 @@ class ChatApi(Resource):
         parser.add_argument("conversation_id", type=uuid_value, location="json")
         parser.add_argument("retriever_from", type=str, required=False, default="dev", location="json")
         parser.add_argument("auto_generate_name", type=bool, required=False, default=True, location="json")
+        parser.add_argument("parent_message_id", type=uuid_value, required=False, default=UUID_NIL, location="json")
 
         args = parser.parse_args()
 

api/controllers/service_api/app/file.py

@@ -2,7 +2,6 @@ from flask import request
 from flask_restful import Resource, marshal_with
 
 import services
-from controllers.common.errors import FilenameNotExistsError
 from controllers.service_api import api
 from controllers.service_api.app.error import (
     FileTooLargeError,
@@ -32,17 +31,8 @@ class FileApi(Resource):
         if len(request.files) > 1:
             raise TooManyFilesError()
 
-        if not file.filename:
-            raise FilenameNotExistsError
-
         try:
-            upload_file = FileService.upload_file(
-                filename=file.filename,
-                content=file.read(),
-                mimetype=file.mimetype,
-                user=end_user,
-                source="datasets",
-            )
+            upload_file = FileService.upload_file(file, end_user)
         except services.errors.file.FileTooLargeError as file_too_large_error:
             raise FileTooLargeError(file_too_large_error.description)
         except services.errors.file.UnsupportedFileTypeError:

api/controllers/service_api/app/message.py

@@ -48,7 +48,7 @@ class MessageListApi(Resource):
         "tool_input": fields.String,
         "created_at": TimestampField,
         "observation": fields.String,
-        "message_files": fields.List(fields.Nested(message_file_fields)),
+        "message_files": fields.List(fields.String),
     }
 
     message_fields = {

api/controllers/service_api/dataset/dataset.py

@@ -66,13 +66,6 @@ class DatasetListApi(DatasetApiResource):
             help="type is required. Name must be between 1 to 40 characters.",
             type=_validate_name,
         )
-        parser.add_argument(
-            "description",
-            type=str,
-            nullable=True,
-            required=False,
-            default="",
-        )
         parser.add_argument(
             "indexing_technique",
             type=str,
@@ -115,7 +108,6 @@ class DatasetListApi(DatasetApiResource):
             dataset = DatasetService.create_empty_dataset(
                 tenant_id=tenant_id,
                 name=args["name"],
-                description=args["description"],
                 indexing_technique=args["indexing_technique"],
                 account=current_user,
                 permission=args["permission"],