bump to 0.6.14 (#6294)

Co-authored-by: jinqi.guo <jinqi.guo@ubtrobot.com>

.devcontainer/post_create_command.sh

@@ -1,9 +1,10 @@
 #!/bin/bash
 
 cd web && npm install
+pipx install poetry
 
 echo 'alias start-api="cd /workspaces/dify/api && flask run --host 0.0.0.0 --port=5001 --debug"' >> ~/.bashrc
-echo 'alias start-worker="cd /workspaces/dify/api && celery -A app.celery worker -P gevent -c 1 --loglevel INFO -Q dataset,generation,mail"' >> ~/.bashrc
+echo 'alias start-worker="cd /workspaces/dify/api && celery -A app.celery worker -P gevent -c 1 --loglevel INFO -Q dataset,generation,mail,ops_trace,app_deletion"' >> ~/.bashrc
 echo 'alias start-web="cd /workspaces/dify/web && npm run dev"' >> ~/.bashrc
 echo 'alias start-containers="cd /workspaces/dify/docker && docker-compose -f docker-compose.middleware.yaml -p dify up -d"' >> ~/.bashrc
 

.devcontainer/post_start_command.sh

@@ -1,3 +1,3 @@
 #!/bin/bash
 
-cd api && pip install -r requirements.txt
+poetry install -C api

.gitattributes

@@ -0,0 +1,7 @@
+# Ensure that .sh scripts use LF as line separator, even if they are checked out
+# to Windows(NTFS) file-system, by a user of Docker for Window. 
+# These .sh scripts will be run from the Container after `docker compose up -d`.
+# If they appear to be CRLF style, Dash from the Container will fail to execute
+# them.
+
+*.sh      text eol=lf

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -14,13 +14,15 @@ body:
           required: true
         - label: I confirm that I am using English to submit this report (我已阅读并同意 [Language Policy](https://github.com/langgenius/dify/issues/1542)).
           required: true
+        - label: "请务必使用英文提交 Issue，否则会被关闭。谢谢！:）"
+          required: true
         - label: "Please do not modify this template :) and fill in all the required fields."
           required: true
 
   - type: input
     attributes:
       label: Dify version
-      placeholder: 0.3.21
+      placeholder: 0.6.11
       description: See about section in Dify console
     validations:
       required: true
@@ -40,7 +42,7 @@ body:
   - type: textarea
     attributes:
       label: Steps to reproduce
-      description: We highly suggest including screenshots and a bug report log.
+      description: We highly suggest including screenshots and a bug report log. Please use the right markdown syntax for code blocks.
       placeholder: Having detailed steps helps us reproduce the bug.
     validations:
       required: true

.github/ISSUE_TEMPLATE/document_issue.yml

@@ -12,6 +12,8 @@ body:
           required: true
         - label: I confirm that I am using English to submit report (我已阅读并同意 [Language Policy](https://github.com/langgenius/dify/issues/1542)).
           required: true
+        - label: "请务必使用英文提交 Issue，否则会被关闭。谢谢！:）"
+          required: true
         - label: "Please do not modify this template :) and fill in all the required fields."
           required: true
   - type: textarea

.github/ISSUE_TEMPLATE/feature_request.yml

@@ -12,35 +12,25 @@ body:
           required: true
         - label: I confirm that I am using English to submit this report (我已阅读并同意 [Language Policy](https://github.com/langgenius/dify/issues/1542)).
           required: true
+        - label: "请务必使用英文提交 Issue，否则会被关闭。谢谢！:）"
+          required: true
         - label: "Please do not modify this template :) and fill in all the required fields."
           required: true
   - type: textarea
     attributes:
-      label: 1. Is this request related to a challenge you're experiencing?
+      label: 1. Is this request related to a challenge you're experiencing? Tell me about your story.
       placeholder: Please describe the specific scenario or problem you're facing as clearly as possible. For instance "I was trying to use [feature] for [specific task], and [what happened]... It was frustrating because...."
     validations:
       required: true
   - type: textarea
     attributes:
-      label: 2. Describe the feature you'd like to see
-      placeholder: Think about what you want to achieve and how this feature will help you. Sketches, flow diagrams, or any visual representation will be a major plus.
-    validations:
-      required: true
-  - type: textarea
-    attributes:
-      label: 3. How will this feature improve your workflow or experience?
-      placeholder: Tell us how this change will benefit your work. This helps us prioritize based on user impact.
-    validations:
-      required: true
-  - type: textarea
-    attributes:
-      label: 4. Additional context or comments
+      label: 2. Additional context or comments
       placeholder: (Any other information, comments, documentations, links, or screenshots that would provide more clarity. This is the place to add anything else not covered above.)
     validations:
       required: false
   - type: checkboxes
     attributes:
-      label: 5. Can you help us with this feature?
+      label: 3. Can you help us with this feature?
       description: Let us know! This is not a commitment, but a starting point for collaboration.
       options:
         - label: I am interested in contributing to this feature.

.github/ISSUE_TEMPLATE/translation_issue.yml

@@ -12,6 +12,8 @@ body:
           required: true
         - label: I confirm that I am using English to submit this report (我已阅读并同意 [Language Policy](https://github.com/langgenius/dify/issues/1542)).
           required: true
+        - label: "请务必使用英文提交 Issue，否则会被关闭。谢谢！:）"
+          required: true
         - label: "Please do not modify this template :) and fill in all the required fields."
           required: true
   - type: input

.github/pull_request_template.md

@@ -1,13 +1,21 @@
+# Checklist:
+
+> [!IMPORTANT]  
+> Please review the checklist below before submitting your pull request.
+
+- [ ] Please open an issue before creating a PR or link to an existing issue
+- [ ] I have performed a self-review of my own code
+- [ ] I have commented my code, particularly in hard-to-understand areas
+- [ ] I ran `dev/reformat`(backend) and `cd web && npx lint-staged`(frontend) to appease the lint gods
+
 # Description
 
-Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. List any dependencies that are required for this change.
+Describe the big picture of your changes here to communicate to the maintainers why we should accept this pull request. If it fixes a bug or resolves a feature request, be sure to link to that issue. Close issue syntax: `Fixes #<issue number>`, see [documentation](https://docs.github.com/en/issues/tracking-your-work-with-issues/linking-a-pull-request-to-an-issue#linking-a-pull-request-to-an-issue-using-a-keyword) for more details.
 
-Fixes # (issue)
+Fixes 
 
 ## Type of Change
 
-Please delete options that are not relevant.
-
 - [ ] Bug fix (non-breaking change which fixes an issue)
 - [ ] New feature (non-breaking change which adds functionality)
 - [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected)
@@ -15,18 +23,12 @@ Please delete options that are not relevant.
 - [ ] Improvement, including but not limited to code refactoring, performance optimization, and UI/UX improvement
 - [ ] Dependency upgrade
 
-# How Has This Been Tested?
+# Testing Instructions
 
 Please describe the tests that you ran to verify your changes. Provide instructions so we can reproduce. Please also list any relevant details for your test configuration
 
-- [ ] TODO
+- [ ] Test A
+- [ ] Test B
+
 
-# Suggested Checklist:
 
-- [ ] I have performed a self-review of my own code
-- [ ] I have commented my code, particularly in hard-to-understand areas
-- [ ] My changes generate no new warnings
-- [ ] I ran `dev/reformat`(backend) and `cd web && npx lint-staged`(frontend) to appease the lint gods
-- [ ] `optional` I have made corresponding changes to the documentation 
-- [ ] `optional` I have added tests that prove my fix is effective or that my feature works
-- [ ] `optional` New and existing unit tests pass locally with my changes

.github/workflows/api-tests.yml

@@ -14,74 +14,6 @@ concurrency:
 
 jobs:
   test:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version:
-          - "3.10"
-          - "3.11"
-
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: 'pip'
-          cache-dependency-path: |
-            ./api/requirements.txt
-            ./api/requirements-dev.txt
-
-      - name: Install dependencies
-        run: pip install -r ./api/requirements.txt -r ./api/requirements-dev.txt
-
-      - name: Run Unit tests
-        run: dev/pytest/pytest_unit_tests.sh
-
-      - name: Run ModelRuntime
-        run: dev/pytest/pytest_model_runtime.sh
-
-      - name: Run Tool
-        run: dev/pytest/pytest_tools.sh
-
-      - name: Set up Sandbox
-        uses: hoverkraft-tech/compose-action@v2.0.0
-        with:
-          compose-file: |
-            docker/docker-compose.middleware.yaml
-          services: |
-            sandbox
-            ssrf_proxy
-
-      - name: Run Workflow
-        run: dev/pytest/pytest_workflow.sh
-
-      - name: Set up Vector Stores (Weaviate, Qdrant, PGVector, Milvus, PgVecto-RS, Chroma)
-        uses: hoverkraft-tech/compose-action@v2.0.0
-        with:
-          compose-file: |
-            docker/docker-compose.middleware.yaml
-            docker/docker-compose.qdrant.yaml
-            docker/docker-compose.milvus.yaml
-            docker/docker-compose.pgvecto-rs.yaml
-            docker/docker-compose.pgvector.yaml
-            docker/docker-compose.chroma.yaml
-          services: |
-            weaviate
-            qdrant
-            etcd
-            minio
-            milvus-standalone
-            pgvecto-rs
-            pgvector
-            chroma
-
-      - name: Test Vector Stores
-        run: dev/pytest/pytest_vdb.sh
-
-  test-in-poetry:
     name: API Tests
     runs-on: ubuntu-latest
     strategy:
@@ -108,7 +40,7 @@ jobs:
 
       - name: Poetry check
         run: |
-          poetry check -C api
+          poetry check -C api --lock
           poetry show -C api
 
       - name: Install dependencies
@@ -123,6 +55,14 @@ jobs:
       - name: Run Tool
         run: poetry run -C api bash dev/pytest/pytest_tools.sh
 
+      - name: Set up dotenvs
+        run: |
+          cp docker/.env.example docker/.env
+          cp docker/middleware.env.example docker/middleware.env
+
+      - name: Expose Service Ports
+        run: sh .github/workflows/expose_service_ports.sh
+
       - name: Set up Sandbox
         uses: hoverkraft-tech/compose-action@v2.0.0
         with:
@@ -135,16 +75,11 @@ jobs:
       - name: Run Workflow
         run: poetry run -C api bash dev/pytest/pytest_workflow.sh
 
-      - name: Set up Vector Stores (Weaviate, Qdrant, PGVector, Milvus, PgVecto-RS, Chroma)
+      - name: Set up Vector Stores (Weaviate, Qdrant, PGVector, Milvus, PgVecto-RS, Chroma, MyScale)
         uses: hoverkraft-tech/compose-action@v2.0.0
         with:
           compose-file: |
-            docker/docker-compose.middleware.yaml
-            docker/docker-compose.qdrant.yaml
-            docker/docker-compose.milvus.yaml
-            docker/docker-compose.pgvecto-rs.yaml
-            docker/docker-compose.pgvector.yaml
-            docker/docker-compose.chroma.yaml
+            docker/docker-compose.yaml
           services: |
             weaviate
             qdrant
@@ -154,6 +89,6 @@ jobs:
             pgvecto-rs
             pgvector
             chroma
-
+            myscale
       - name: Test Vector Stores
         run: poetry run -C api bash dev/pytest/pytest_vdb.sh

.github/workflows/build-push.yml

@@ -8,6 +8,10 @@ on:
   release:
     types: [published]
 
+concurrency:
+  group: build-push-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
 env:
   DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
   DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
@@ -15,24 +19,34 @@ env:
   DIFY_API_IMAGE_NAME: ${{ vars.DIFY_API_IMAGE_NAME || 'langgenius/dify-api' }}
 
 jobs:
-  build-and-push:
-    runs-on: ubuntu-latest
+  build:
+    runs-on: ${{ matrix.platform == 'linux/arm64' && 'arm64_runner' || 'ubuntu-latest' }}
     if: github.repository == 'langgenius/dify'
     strategy:
       matrix:
         include:
-          - service_name: "web"
-            image_name_env: "DIFY_WEB_IMAGE_NAME"
-            context: "web"
-          - service_name: "api"
+          - service_name: "build-api-amd64"
             image_name_env: "DIFY_API_IMAGE_NAME"
             context: "api"
-    steps:
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
+            platform: linux/amd64
+          - service_name: "build-api-arm64"
+            image_name_env: "DIFY_API_IMAGE_NAME"
+            context: "api"
+            platform: linux/arm64
+          - service_name: "build-web-amd64"
+            image_name_env: "DIFY_WEB_IMAGE_NAME"
+            context: "web"
+            platform: linux/amd64
+          - service_name: "build-web-arm64"
+            image_name_env: "DIFY_WEB_IMAGE_NAME"
+            context: "web"
+            platform: linux/arm64
 
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
+    steps:
+      - name: Prepare
+        run: |
+          platform=${{ matrix.platform }}
+          echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV
 
       - name: Login to Docker Hub
         uses: docker/login-action@v2
@@ -40,7 +54,72 @@ jobs:
           username: ${{ env.DOCKERHUB_USER }}
           password: ${{ env.DOCKERHUB_TOKEN }}
 
-      - name: Extract metadata (tags, labels) for Docker
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Extract metadata for Docker
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env[matrix.image_name_env] }}
+
+      - name: Build Docker image
+        id: build
+        uses: docker/build-push-action@v6
+        with:
+          context: "{{defaultContext}}:${{ matrix.context }}"
+          platforms: ${{ matrix.platform }}
+          build-args: COMMIT_SHA=${{ fromJSON(steps.meta.outputs.json).labels['org.opencontainers.image.revision'] }}
+          labels: ${{ steps.meta.outputs.labels }}
+          outputs: type=image,name=${{ env[matrix.image_name_env] }},push-by-digest=true,name-canonical=true,push=true
+          cache-from: type=gha,scope=${{ matrix.service_name }}
+          cache-to: type=gha,mode=max,scope=${{ matrix.service_name }}
+
+      - name: Export digest
+        run: |
+          mkdir -p /tmp/digests
+          digest="${{ steps.build.outputs.digest }}"
+          touch "/tmp/digests/${digest#sha256:}"
+
+      - name: Upload digest
+        uses: actions/upload-artifact@v4
+        with:
+          name: digests-${{ matrix.context }}-${{ env.PLATFORM_PAIR }}
+          path: /tmp/digests/*
+          if-no-files-found: error
+          retention-days: 1
+
+  create-manifest:
+    needs: build
+    runs-on: ubuntu-latest
+    if: github.repository == 'langgenius/dify'
+    strategy:
+      matrix:
+        include:
+          - service_name: "merge-api-images"
+            image_name_env: "DIFY_API_IMAGE_NAME"
+            context: "api"
+          - service_name: "merge-web-images"
+            image_name_env: "DIFY_WEB_IMAGE_NAME"
+            context: "web"
+    steps:
+      - name: Download digests
+        uses: actions/download-artifact@v4
+        with:
+          path: /tmp/digests
+          pattern: digests-${{ matrix.context }}-*
+          merge-multiple: true
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ env.DOCKERHUB_USER }}
+          password: ${{ env.DOCKERHUB_TOKEN }}
+
+      - name: Extract metadata for Docker
         id: meta
         uses: docker/metadata-action@v5
         with:
@@ -51,14 +130,12 @@ jobs:
             type=sha,enable=true,priority=100,prefix=,suffix=,format=long
             type=raw,value=${{ github.ref_name }},enable=${{ startsWith(github.ref, 'refs/tags/') }}
 
-      - name: Build and push
-        uses: docker/build-push-action@v5
-        with:
-          context: "{{defaultContext}}:${{ matrix.context }}"
-          platforms: ${{ startsWith(github.ref, 'refs/tags/') && 'linux/amd64,linux/arm64' || 'linux/amd64' }}
-          build-args: COMMIT_SHA=${{ fromJSON(steps.meta.outputs.json).labels['org.opencontainers.image.revision'] }}
-          push: true
-          tags: ${{ steps.meta.outputs.tags }}
-          labels: ${{ steps.meta.outputs.labels }}
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
+      - name: Create manifest list and push
+        working-directory: /tmp/digests
+        run: |
+          docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
+            $(printf '${{ env[matrix.image_name_env] }}@sha256:%s ' *)
+
+      - name: Inspect image
+        run: |
+          docker buildx imagetools inspect ${{ env[matrix.image_name_env] }}:${{ steps.meta.outputs.version }}

.github/workflows/db-migration-test.yml

@@ -38,6 +38,11 @@ jobs:
       - name: Install dependencies
         run: poetry install -C api
 
+      - name: Prepare middleware env
+        run: |
+          cd docker
+          cp middleware.env.example middleware.env
+
       - name: Set up Middlewares
         uses: hoverkraft-tech/compose-action@v2.0.0
         with:

.github/workflows/expose_service_ports.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+yq eval '.services.weaviate.ports += ["8080:8080"]' -i docker/docker-compose.yaml
+yq eval '.services.qdrant.ports += ["6333:6333"]' -i docker/docker-compose.yaml
+yq eval '.services.chroma.ports += ["8000:8000"]' -i docker/docker-compose.yaml
+yq eval '.services["milvus-standalone"].ports += ["19530:19530"]' -i docker/docker-compose.yaml
+yq eval '.services.pgvector.ports += ["5433:5432"]' -i docker/docker-compose.yaml
+yq eval '.services["pgvecto-rs"].ports += ["5431:5432"]' -i docker/docker-compose.yaml
+
+echo "Ports exposed for sandbox, weaviate, qdrant, chroma, milvus, pgvector, pgvecto-rs."

.github/workflows/style.yml

@@ -99,7 +99,7 @@ jobs:
             **.sh
             **.yaml
             **.yml
-            Dockerfile
+            **Dockerfile
             dev/**
 
       - name: Super-linter
@@ -113,7 +113,8 @@ jobs:
           IGNORE_GITIGNORED_FILES: true
           VALIDATE_BASH: true
           VALIDATE_BASH_EXEC: true
-          VALIDATE_GITHUB_ACTIONS: true
+          # FIXME: temporarily disabled until api-docker.yaml's run script is fixed for shellcheck
+          # VALIDATE_GITHUB_ACTIONS: true
           VALIDATE_DOCKERFILE_HADOLINT: true
           VALIDATE_XML: true
           VALIDATE_YAML: true

.gitignore

@@ -138,10 +138,22 @@ web/.vscode/settings.json
 !.idea/vcs.xml
 !.idea/icon.png
 .ideaDataSources/
+*.iml
+api/.idea
 
 api/.env
 api/storage/*
 
+docker-legacy/volumes/app/storage/*
+docker-legacy/volumes/db/data/*
+docker-legacy/volumes/redis/data/*
+docker-legacy/volumes/weaviate/*
+docker-legacy/volumes/qdrant/*
+docker-legacy/volumes/etcd/*
+docker-legacy/volumes/minio/*
+docker-legacy/volumes/milvus/*
+docker-legacy/volumes/chroma/*
+
 docker/volumes/app/storage/*
 docker/volumes/db/data/*
 docker/volumes/redis/data/*
@@ -152,6 +164,9 @@ docker/volumes/minio/*
 docker/volumes/milvus/*
 docker/volumes/chroma/*
 
+docker/nginx/conf.d/default.conf
+docker/middleware.env
+
 sdks/python-client/build
 sdks/python-client/dist
 sdks/python-client/dify_client.egg-info
@@ -159,3 +174,5 @@ sdks/python-client/dify_client.egg-info
 .vscode/*
 !.vscode/launch.json
 pyrightconfig.json
+
+.idea/

.vscode/launch.json

@@ -1,44 +1,54 @@
 {
-    // Use IntelliSense to learn about possible attributes.
-    // Hover to view descriptions of existing attributes.
-    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
     "version": "0.2.0",
     "configurations": [
-        {
-            "name": "Python: Celery",
-            "type": "debugpy",
-            "request": "launch",
-            "module": "celery",
-            "justMyCode": true,
-            "args": ["-A", "app.celery", "worker", "-P", "gevent", "-c", "1", "--loglevel", "info", "-Q", "dataset,generation,mail"],
-            "envFile": "${workspaceFolder}/.env",
-            "env": {
-                "FLASK_APP": "app.py",
-                "FLASK_DEBUG": "1",
-                "GEVENT_SUPPORT": "True"
-            },
-            "console": "integratedTerminal",
-            "python": "${command:python.interpreterPath}"
-        },
         {
             "name": "Python: Flask",
             "type": "debugpy",
             "request": "launch",
+            "python": "${workspaceFolder}/api/.venv/bin/python",
+            "cwd": "${workspaceFolder}/api",
+            "envFile": ".env",
             "module": "flask",
+            "justMyCode": true,
+            "jinja": true,
             "env": {
                 "FLASK_APP": "app.py",
-                "FLASK_DEBUG": "1",
                 "GEVENT_SUPPORT": "True"
             },
             "args": [
                 "run",
                 "--host=0.0.0.0",
                 "--port=5001",
-                "--debug"
-            ],
-            "jinja": true,
+            ]
+        },
+        {
+            "name": "Python: Celery",
+            "type": "debugpy",
+            "request": "launch",
+            "python": "${workspaceFolder}/api/.venv/bin/python",
+            "cwd": "${workspaceFolder}/api",
+            "module": "celery",
             "justMyCode": true,
-            "python": "${command:python.interpreterPath}"
-        }
+            "envFile": ".env",
+            "console": "integratedTerminal",
+            "env": {
+                "FLASK_APP": "app.py",
+                "FLASK_DEBUG": "1",
+                "GEVENT_SUPPORT": "True"
+            },
+            "args": [
+                "-A",
+                "app.celery",
+                "worker",
+                "-P",
+                "gevent",
+                "-c",
+                "1",
+                "--loglevel",
+                "info",
+                "-Q",
+                "dataset,generation,mail,ops_trace,app_deletion"
+            ]
+        },
     ]
 }

README.md

@@ -174,6 +174,7 @@ The easiest way to start the Dify server is to run our [docker-compose.yml](dock
 
 ```bash
 cd docker
+cp .env.example .env
 docker compose up -d
 ```
 
@@ -183,7 +184,7 @@ After running, you can access the Dify dashboard in your browser at [http://loca
 
 ## Next steps
 
-If you need to customize the configuration, please refer to the comments in our [docker-compose.yml](docker/docker-compose.yaml) file and manually set the environment configuration. After making the changes, please run `docker-compose up -d` again. You can see the full list of environment variables [here](https://docs.dify.ai/getting-started/install-self-hosted/environments).
+If you need to customize the configuration, please refer to the comments in our [.env.example](docker/.env.example) file and update the corresponding values in your `.env` file. Additionally, you might need to make adjustments to the `docker-compose.yaml` file itself, such as changing image versions, port mappings, or volume mounts, based on your specific deployment environment and requirements. After making any changes, please re-run `docker-compose up -d`. You can find the full list of available environment variables [here](https://docs.dify.ai/getting-started/install-self-hosted/environments).
 
 If you'd like to configure a highly-available setup, there are community-contributed [Helm Charts](https://helm.sh/) and YAML files which allow Dify to be deployed on Kubernetes.
 
@@ -191,6 +192,11 @@ If you'd like to configure a highly-available setup, there are community-contrib
 - [Helm Chart by @BorisPolonsky](https://github.com/BorisPolonsky/dify-helm)
 - [YAML file by @Winson-030](https://github.com/Winson-030/dify-kubernetes)
 
+#### Using Terraform for Deployment
+
+##### Azure Global
+Deploy Dify to Azure with a single click using [terraform](https://www.terraform.io/).
+- [Azure Terraform by @nikawang](https://github.com/nikawang/dify-azure-terraform)
 
 ## Contributing
 

README_AR.md

@@ -157,15 +157,17 @@
 
 ```bash
 cd docker
+cp .env.example .env
 docker compose up -d
 ```
+
 بعد التشغيل، يمكنك الوصول إلى لوحة تحكم Dify في متصفحك على [http://localhost/install](http://localhost/install) وبدء عملية التهيئة.
 
 > إذا كنت ترغب في المساهمة في Dify أو القيام بتطوير إضافي، فانظر إلى [دليلنا للنشر من الشفرة (code) المصدرية](https://docs.dify.ai/getting-started/install-self-hosted/local-source-code)
 
 ## الخطوات التالية
 
-إذا كنت بحاجة إلى تخصيص التكوين، يرجى الرجوع إلى التعليقات في ملف [docker-compose.yml](docker/docker-compose.yaml) لدينا وتعيين التكوينات البيئية يدويًا. بعد إجراء التغييرات، يرجى تشغيل `docker-compose up -d` مرة أخرى. يمكنك رؤية قائمة كاملة بالمتغيرات البيئية [هنا](https://docs.dify.ai/getting-started/install-self-hosted/environments).
+إذا كنت بحاجة إلى تخصيص الإعدادات، فيرجى الرجوع إلى التعليقات في ملف [.env.example](docker/.env.example) وتحديث القيم المقابلة في ملف `.env`. بالإضافة إلى ذلك، قد تحتاج إلى إجراء تعديلات على ملف `docker-compose.yaml` نفسه، مثل تغيير إصدارات الصور أو تعيينات المنافذ أو نقاط تحميل وحدات التخزين، بناءً على بيئة النشر ومتطلباتك الخاصة. بعد إجراء أي تغييرات، يرجى إعادة تشغيل `docker-compose up -d`. يمكنك العثور على قائمة كاملة بمتغيرات البيئة المتاحة [هنا](https://docs.dify.ai/getting-started/install-self-hosted/environments).
 
 يوجد مجتمع خاص بـ [Helm Charts](https://helm.sh/) وملفات YAML التي تسمح بتنفيذ Dify على Kubernetes للنظام من الإيجابيات العلوية.
 
@@ -173,6 +175,12 @@ docker compose up -d
 - [رسم بياني Helm من قبل @BorisPolonsky](https://github.com/BorisPolonsky/dify-helm)
 - [ملف YAML من قبل @Winson-030](https://github.com/Winson-030/dify-kubernetes)
 
+#### استخدام Terraform للتوزيع
+
+##### Azure Global
+استخدم [terraform](https://www.terraform.io/) لنشر Dify على Azure بنقرة واحدة.
+- [Azure Terraform بواسطة @nikawang](https://github.com/nikawang/dify-azure-terraform)
+
 
 ## المساهمة
 

README_CN.md

@@ -179,11 +179,16 @@ Dify 是一个开源的 LLM 应用开发平台。其直观的界面结合了 AI
 
 ```bash
 cd docker
+cp .env.example .env
 docker compose up -d
 ```
 
 运行后，可以在浏览器上访问 [http://localhost/install](http://localhost/install) 进入 Dify 控制台并开始初始化安装操作。
 
+### 自定义配置
+
+如果您需要自定义配置，请参考 [.env.example](docker/.env.example) 文件中的注释，并更新 `.env` 文件中对应的值。此外，您可能需要根据您的具体部署环境和需求对 `docker-compose.yaml` 文件本身进行调整，例如更改镜像版本、端口映射或卷挂载。完成任何更改后，请重新运行 `docker-compose up -d`。您可以在[此处](https://docs.dify.ai/getting-started/install-self-hosted/environments)找到可用环境变量的完整列表。
+
 #### 使用 Helm Chart 部署
 
 使用 [Helm Chart](https://helm.sh/) 版本或者 YAML 文件，可以在 Kubernetes 上部署 Dify。
@@ -192,9 +197,11 @@ docker compose up -d
 - [Helm Chart by @BorisPolonsky](https://github.com/BorisPolonsky/dify-helm)
 - [YAML 文件 by @Winson-030](https://github.com/Winson-030/dify-kubernetes)
 
-### 配置
+#### 使用 Terraform 部署
 
-如果您需要自定义配置，请参考我们的 [docker-compose.yml](docker/docker-compose.yaml) 文件中的注释，并手动设置环境配置。更改后，请再次运行 `docker-compose up -d`。您可以在我们的[文档](https://docs.dify.ai/getting-started/install-self-hosted/environments)中查看所有环境变量的完整列表。
+##### Azure Global
+使用 [terraform](https://www.terraform.io/) 一键部署 Dify 到 Azure。
+- [Azure Terraform by @nikawang](https://github.com/nikawang/dify-azure-terraform)
 
 ## Star History
 

README_ES.md

@@ -179,6 +179,7 @@ La forma más fácil de iniciar el servidor de Dify es ejecutar nuestro archivo
 
 ```bash
 cd docker
+cp .env.example .env
 docker compose up -d
 ```
 
@@ -188,7 +189,7 @@ Después de ejecutarlo, puedes acceder al panel de control de Dify en tu navegad
 
 ## Próximos pasos
 
-Si necesitas personalizar la configuración, consulta los comentarios en nuestro archivo [docker-compose.yml](docker/docker-compose.yaml) y configura manualmente la configuración del entorno
+Si necesita personalizar la configuración, consulte los comentarios en nuestro archivo [.env.example](docker/.env.example) y actualice los valores correspondientes en su archivo `.env`. Además, es posible que deba realizar ajustes en el propio archivo `docker-compose.yaml`, como cambiar las versiones de las imágenes, las asignaciones de puertos o los montajes de volúmenes, según su entorno de implementación y requisitos específicos. Después de realizar cualquier cambio, vuelva a ejecutar `docker-compose up -d`. Puede encontrar la lista completa de variables de entorno disponibles [aquí](https://docs.dify.ai/getting-started/install-self-hosted/environments).
 
 . Después de realizar los cambios, ejecuta `docker-compose up -d` nuevamente. Puedes ver la lista completa de variables de entorno [aquí](https://docs.dify.ai/getting-started/install-self-hosted/environments).
 
@@ -198,6 +199,12 @@ Si desea configurar una configuración de alta disponibilidad, la comunidad prop
 - [Gráfico Helm por @BorisPolonsky](https://github.com/BorisPolonsky/dify-helm)
 - [Ficheros YAML por @Winson-030](https://github.com/Winson-030/dify-kubernetes)
 
+#### Uso de Terraform para el despliegue
+
+##### Azure Global
+Utiliza [terraform](https://www.terraform.io/) para desplegar Dify en Azure con un solo clic.
+- [Azure Terraform por @nikawang](https://github.com/nikawang/dify-azure-terraform)
+
 
 ## Contribuir
 

README_FR.md

@@ -179,6 +179,7 @@ La manière la plus simple de démarrer le serveur Dify est d'exécuter notre fi
 
 ```bash
 cd docker
+cp .env.example .env
 docker compose up -d
 ```
 
@@ -188,9 +189,7 @@ Après l'exécution, vous pouvez accéder au tableau de bord Dify dans votre nav
 
 ## Prochaines étapes
 
-Si vous devez personnaliser la configuration, veuillez
-
- vous référer aux commentaires dans notre fichier [docker-compose.yml](docker/docker-compose.yaml) et définir manuellement la configuration de l'environnement. Après avoir apporté les modifications, veuillez exécuter à nouveau `docker-compose up -d`. Vous pouvez voir la liste complète des variables d'environnement [ici](https://docs.dify.ai/getting-started/install-self-hosted/environments).
+Si vous devez personnaliser la configuration, veuillez vous référer aux commentaires dans notre fichier [.env.example](docker/.env.example) et mettre à jour les valeurs correspondantes dans votre fichier `.env`. De plus, vous devrez peut-être apporter des modifications au fichier `docker-compose.yaml` lui-même, comme changer les versions d'image, les mappages de ports ou les montages de volumes, en fonction de votre environnement de déploiement et de vos exigences spécifiques. Après avoir effectué des modifications, veuillez réexécuter `docker-compose up -d`. Vous pouvez trouver la liste complète des variables d'environnement disponibles [ici](https://docs.dify.ai/getting-started/install-self-hosted/environments).
 
 Si vous souhaitez configurer une configuration haute disponibilité, la communauté fournit des [Helm Charts](https://helm.sh/) et des fichiers YAML, à travers lesquels vous pouvez déployer Dify sur Kubernetes.
 
@@ -198,6 +197,12 @@ Si vous souhaitez configurer une configuration haute disponibilité, la communau
 - [Helm Chart par @BorisPolonsky](https://github.com/BorisPolonsky/dify-helm)
 - [Fichier YAML par @Winson-030](https://github.com/Winson-030/dify-kubernetes)
 
+#### Utilisation de Terraform pour le déploiement
+
+##### Azure Global
+Utilisez [terraform](https://www.terraform.io/) pour déployer Dify sur Azure en un clic.
+- [Azure Terraform par @nikawang](https://github.com/nikawang/dify-azure-terraform)
+
 
 ## Contribuer
 

README_JA.md

@@ -178,6 +178,7 @@ Difyサーバーを起動する最も簡単な方法は、[docker-compose.yml](d
 
 ```bash
 cd docker
+cp .env.example .env
 docker compose up -d
 ```
 
@@ -187,7 +188,7 @@ docker compose up -d
 
 ## 次のステップ
 
-環境設定をカスタマイズする場合は、[docker-compose.yml](docker/docker-compose.yaml)ファイル内のコメントを参照して、環境設定を手動で設定してください。変更を加えた後は、再び `docker-compose up -d` を実行してください。環境変数の完全なリストは[こちら](https://docs.dify.ai/getting-started/install-self-hosted/environments)をご覧ください。
+設定をカスタマイズする必要がある場合は、[.env.example](docker/.env.example) ファイルのコメントを参照し、`.env` ファイルの対応する値を更新してください。さらに、デプロイ環境や要件に応じて、`docker-compose.yaml` ファイル自体を調整する必要がある場合があります。たとえば、イメージのバージョン、ポートのマッピング、ボリュームのマウントなどを変更します。変更を加えた後は、`docker-compose up -d` を再実行してください。利用可能な環境変数の全一覧は、[こちら](https://docs.dify.ai/getting-started/install-self-hosted/environments)で確認できます。
 
 高可用性設定を設定する必要がある場合、コミュニティは[Helm Charts](https://helm.sh/)とYAMLファイルにより、DifyをKubernetesにデプロイすることができます。
 
@@ -195,6 +196,12 @@ docker compose up -d
 - [Helm Chart by @BorisPolonsky](https://github.com/BorisPolonsky/dify-helm)
 - [YAML file by @Winson-030](https://github.com/Winson-030/dify-kubernetes)
 
+#### Terraformを使用したデプロイ
+
+##### Azure Global
+[terraform](https://www.terraform.io/) を使用して、AzureにDifyをワンクリックでデプロイします。
+- [nikawangのAzure Terraform](https://github.com/nikawang/dify-azure-terraform)
+
 
 ## 貢献
 

README_KL.md

@@ -179,6 +179,7 @@ The easiest way to start the Dify server is to run our [docker-compose.yml](dock
 
 ```bash
 cd docker
+cp .env.example .env
 docker compose up -d
 ```
 
@@ -188,7 +189,7 @@ After running, you can access the Dify dashboard in your browser at [http://loca
 
 ## Next steps
 
-If you need to customize the configuration, please refer to the comments in our [docker-compose.yml](docker/docker-compose.yaml) file and manually set the environment configuration. After making the changes, please run `docker-compose up -d` again. You can see the full list of environment variables [here](https://docs.dify.ai/getting-started/install-self-hosted/environments).
+If you need to customize the configuration, please refer to the comments in our [.env.example](docker/.env.example) file and update the corresponding values in your `.env` file. Additionally, you might need to make adjustments to the `docker-compose.yaml` file itself, such as changing image versions, port mappings, or volume mounts, based on your specific deployment environment and requirements. After making any changes, please re-run `docker-compose up -d`. You can find the full list of available environment variables [here](https://docs.dify.ai/getting-started/install-self-hosted/environments).
 
 If you'd like to configure a highly-available setup, there are community-contributed [Helm Charts](https://helm.sh/) and YAML files which allow Dify to be deployed on Kubernetes.
 
@@ -196,6 +197,13 @@ If you'd like to configure a highly-available setup, there are community-contrib
 - [Helm Chart by @BorisPolonsky](https://github.com/BorisPolonsky/dify-helm)
 - [YAML file by @Winson-030](https://github.com/Winson-030/dify-kubernetes)
 
+#### Terraform atorlugu pilersitsineq
+
+##### Azure Global
+Atoruk [terraform](https://www.terraform.io/) Dify-mik Azure-mut ataatsikkut ikkussuilluarlugu.
+- [Azure Terraform atorlugu @nikawang](https://github.com/nikawang/dify-azure-terraform)
+
+
 ## Contributing
 
 For those who'd like to contribute code, see our [Contribution Guide](https://github.com/langgenius/dify/blob/main/CONTRIBUTING.md). 

README_KR.md

@@ -172,6 +172,7 @@ Dify 서버를 시작하는 가장 쉬운 방법은 [docker-compose.yml](docker/
 
 ```bash
 cd docker
+cp .env.example .env
 docker compose up -d
 ```
 
@@ -181,8 +182,7 @@ docker compose up -d
 
 ## 다음 단계
 
-구성 커스터마이징이 필요한 경우, [docker-compose.yml](docker/docker-compose.yaml) 파일의 코멘트를 참조하여 환경 구성을 수동으로 설정하십시오. 변경 후 `docker-compose up -d` 를 다시 실행하십시오. 환경 변수의 전체 목록은 [여기](https://docs.dify.ai/getting-started/install-self-hosted/environments)에서 확인할 수 있습니다.
-
+구성을 사용자 정의해야 하는 경우 [.env.example](docker/.env.example) 파일의 주석을 참조하고 `.env` 파일에서 해당 값을 업데이트하십시오. 또한 특정 배포 환경 및 요구 사항에 따라 `docker-compose.yaml` 파일 자체를 조정해야 할 수도 있습니다. 예를 들어 이미지 버전, 포트 매핑 또는 볼륨 마운트를 변경합니다. 변경 한 후 `docker-compose up -d`를 다시 실행하십시오. 사용 가능한 환경 변수의 전체 목록은 [여기](https://docs.dify.ai/getting-started/install-self-hosted/environments)에서 찾을 수 있습니다.
 
 Dify를 Kubernetes에 배포하고 프리미엄 스케일링 설정을 구성했다는 커뮤니티가 제공하는 [Helm Charts](https://helm.sh/)와 YAML 파일이 존재합니다.
 
@@ -190,6 +190,12 @@ Dify를 Kubernetes에 배포하고 프리미엄 스케일링 설정을 구성했
 - [Helm Chart by @BorisPolonsky](https://github.com/BorisPolonsky/dify-helm)
 - [YAML file by @Winson-030](https://github.com/Winson-030/dify-kubernetes)
 
+#### Terraform을 사용한 배포
+
+##### Azure Global
+[terraform](https://www.terraform.io/)을 사용하여 Azure에 Dify를 원클릭으로 배포하세요.
+- [nikawang의 Azure Terraform](https://github.com/nikawang/dify-azure-terraform)
+
 ## 기여
 
 코드에 기여하고 싶은 분들은 [기여 가이드](https://github.com/langgenius/dify/blob/main/CONTRIBUTING.md)를 참조하세요.

api/.dockerignore

@@ -8,4 +8,7 @@ logs
 *.log*
 
 # jetbrains
-.idea
+.idea
+
+# venv
+.venv

api/.env.example

@@ -39,7 +39,7 @@ DB_DATABASE=dify
 
 # Storage configuration
 # use for store upload files, private keys...
-# storage type: local, s3, azure-blob
+# storage type: local, s3, azure-blob, google-storage
 STORAGE_TYPE=local
 STORAGE_LOCAL_PATH=storage
 S3_USE_AWS_MANAGED_IAM=false
@@ -63,7 +63,7 @@ ALIYUN_OSS_REGION=your-region
 
 # Google Storage configuration
 GOOGLE_STORAGE_BUCKET_NAME=yout-bucket-name
-GOOGLE_STORAGE_SERVICE_ACCOUNT_JSON=your-google-service-account-json-base64-string
+GOOGLE_STORAGE_SERVICE_ACCOUNT_JSON_BASE64=your-google-service-account-json-base64-string
 
 # Tencent COS Storage configuration
 TENCENT_COS_BUCKET_NAME=your-bucket-name
@@ -72,11 +72,18 @@ TENCENT_COS_SECRET_ID=your-secret-id
 TENCENT_COS_REGION=your-region
 TENCENT_COS_SCHEME=your-scheme
 
+# OCI Storage configuration
+OCI_ENDPOINT=your-endpoint
+OCI_BUCKET_NAME=your-bucket-name
+OCI_ACCESS_KEY=your-access-key
+OCI_SECRET_KEY=your-secret-key
+OCI_REGION=your-region
+
 # CORS configuration
 WEB_API_CORS_ALLOW_ORIGINS=http://127.0.0.1:3000,*
 CONSOLE_CORS_ALLOW_ORIGINS=http://127.0.0.1:3000,*
 
-# Vector database configuration, support: weaviate, qdrant, milvus, relyt, pgvecto_rs, pgvector
+# Vector database configuration, support: weaviate, qdrant, milvus, myscale, relyt, pgvecto_rs, pgvector, pgvector, chroma, opensearch, tidb_vector
 VECTOR_STORE=weaviate
 
 # Weaviate configuration
@@ -99,6 +106,14 @@ MILVUS_USER=root
 MILVUS_PASSWORD=Milvus
 MILVUS_SECURE=false
 
+# MyScale configuration
+MYSCALE_HOST=127.0.0.1
+MYSCALE_PORT=8123
+MYSCALE_USER=default
+MYSCALE_PASSWORD=
+MYSCALE_DATABASE=default
+MYSCALE_FTS_PARAMS=
+
 # Relyt configuration
 RELYT_HOST=127.0.0.1
 RELYT_PORT=5432
@@ -144,6 +159,23 @@ CHROMA_DATABASE=default_database
 CHROMA_AUTH_PROVIDER=chromadb.auth.token_authn.TokenAuthenticationServerProvider
 CHROMA_AUTH_CREDENTIALS=difyai123456
 
+# AnalyticDB configuration
+ANALYTICDB_KEY_ID=your-ak
+ANALYTICDB_KEY_SECRET=your-sk
+ANALYTICDB_REGION_ID=cn-hangzhou
+ANALYTICDB_INSTANCE_ID=gp-ab123456
+ANALYTICDB_ACCOUNT=testaccount
+ANALYTICDB_PASSWORD=testpassword
+ANALYTICDB_NAMESPACE=dify
+ANALYTICDB_NAMESPACE_PASSWORD=difypassword
+
+# OpenSearch configuration
+OPENSEARCH_HOST=127.0.0.1
+OPENSEARCH_PORT=9200
+OPENSEARCH_USER=admin
+OPENSEARCH_PASSWORD=admin
+OPENSEARCH_SECURE=true
+
 # Upload configuration
 UPLOAD_FILE_SIZE_LIMIT=15
 UPLOAD_FILE_BATCH_LIMIT=5
@@ -223,4 +255,4 @@ WORKFLOW_CALL_MAX_DEPTH=5
 
 # App configuration
 APP_MAX_EXECUTION_TIME=1200
-
+APP_MAX_ACTIVE_REQUESTS=0

api/Dockerfile

@@ -1,34 +1,42 @@
 # base image
 FROM python:3.10-slim-bookworm AS base
 
-LABEL maintainer="takatost@gmail.com"
+WORKDIR /app/api
 
-# install packages
-FROM base as packages
+# Install Poetry
+ENV POETRY_VERSION=1.8.3
+RUN pip install --no-cache-dir poetry==${POETRY_VERSION}
+
+# Configure Poetry
+ENV POETRY_CACHE_DIR=/tmp/poetry_cache
+ENV POETRY_NO_INTERACTION=1
+ENV POETRY_VIRTUALENVS_IN_PROJECT=true
+ENV POETRY_VIRTUALENVS_CREATE=true
+
+FROM base AS packages
 
 RUN apt-get update \
     && apt-get install -y --no-install-recommends gcc g++ libc-dev libffi-dev libgmp-dev libmpfr-dev libmpc-dev
 
-COPY requirements.txt /requirements.txt
-
-RUN --mount=type=cache,target=/root/.cache/pip \
-    pip install --prefix=/pkg -r requirements.txt
+# Install Python dependencies
+COPY pyproject.toml poetry.lock ./
+RUN poetry install --sync --no-cache --no-root
 
 # production stage
 FROM base AS production
 
-ENV FLASK_APP app.py
-ENV EDITION SELF_HOSTED
-ENV DEPLOY_ENV PRODUCTION
-ENV CONSOLE_API_URL http://127.0.0.1:5001
-ENV CONSOLE_WEB_URL http://127.0.0.1:3000
-ENV SERVICE_API_URL http://127.0.0.1:5001
-ENV APP_WEB_URL http://127.0.0.1:3000
+ENV FLASK_APP=app.py
+ENV EDITION=SELF_HOSTED
+ENV DEPLOY_ENV=PRODUCTION
+ENV CONSOLE_API_URL=http://127.0.0.1:5001
+ENV CONSOLE_WEB_URL=http://127.0.0.1:3000
+ENV SERVICE_API_URL=http://127.0.0.1:5001
+ENV APP_WEB_URL=http://127.0.0.1:3000
 
 EXPOSE 5001
 
 # set timezone
-ENV TZ UTC
+ENV TZ=UTC
 
 WORKDIR /app/api
 
@@ -37,13 +45,20 @@ RUN apt-get update \
     && apt-get autoremove \
     && rm -rf /var/lib/apt/lists/*
 
-COPY --from=packages /pkg /usr/local
+# Copy Python environment and packages
+ENV VIRTUAL_ENV=/app/api/.venv
+COPY --from=packages ${VIRTUAL_ENV} ${VIRTUAL_ENV}
+ENV PATH="${VIRTUAL_ENV}/bin:${PATH}"
+
+# Copy source code
 COPY . /app/api/
 
+# Copy entrypoint
 COPY docker/entrypoint.sh /entrypoint.sh
 RUN chmod +x /entrypoint.sh
 
-ARG COMMIT_SHA
-ENV COMMIT_SHA ${COMMIT_SHA}
 
-ENTRYPOINT ["/bin/bash", "/entrypoint.sh"]
+ARG COMMIT_SHA
+ENV COMMIT_SHA=${COMMIT_SHA}
+
+ENTRYPOINT ["/bin/bash", "/entrypoint.sh"]

api/README.md

@@ -2,13 +2,17 @@
 
 ## Usage
 
+> [!IMPORTANT]
+> In the v0.6.12 release, we deprecated `pip` as the package management tool for Dify API Backend service and replaced it with `poetry`.
+
 1. Start the docker-compose stack
 
    The backend require some middleware, including PostgreSQL, Redis, and Weaviate, which can be started together using `docker-compose`.
 
    ```bash
    cd ../docker
-   docker-compose -f docker-compose.middleware.yaml -p dify up -d
+   cp middleware.env.example middleware.env
+   docker compose -f docker-compose.middleware.yaml -p dify up -d
    cd ../api
    ```
 
@@ -29,11 +33,8 @@
 
    Dify API service uses [Poetry](https://python-poetry.org/docs/) to manage dependencies. You can execute `poetry shell` to activate the environment.
 
-   > Using pip can be found [below](#usage-with-pip).
-
 5. Install dependencies
 
-=======
    ```bash
    poetry env use 3.10
    poetry install
@@ -66,7 +67,7 @@
 10. If you need to debug local async processing, please start the worker service.
 
    ```bash
-   poetry run python -m celery -A app.celery worker -P gevent -c 1 --loglevel INFO -Q dataset,generation,mail
+   poetry run python -m celery -A app.celery worker -P gevent -c 1 --loglevel INFO -Q dataset,generation,mail,ops_trace,app_deletion
    ```
 
    The started celery app handles the async tasks, e.g. dataset importing and documents indexing.
@@ -85,63 +86,3 @@
    cd ../
    poetry run -C api bash dev/pytest/pytest_all_tests.sh
    ```
-
-## Usage with pip
-
-> [!NOTE]  
-> In the next version, we will deprecate pip as the primary package management tool for dify api service, currently Poetry and pip coexist.
-
-1. Start the docker-compose stack
-
-   The backend require some middleware, including PostgreSQL, Redis, and Weaviate, which can be started together using `docker-compose`.
-
-   ```bash
-   cd ../docker
-   docker-compose -f docker-compose.middleware.yaml -p dify up -d
-   cd ../api
-   ```
-
-2. Copy `.env.example` to `.env`
-3. Generate a `SECRET_KEY` in the `.env` file.
-
-   ```bash
-   sed -i "/^SECRET_KEY=/c\SECRET_KEY=$(openssl rand -base64 42)" .env
-   ```
-
-4. Create environment.
-
-   If you use Anaconda, create a new environment and activate it
-  
-   ```bash
-   conda create --name dify python=3.10
-   conda activate dify
-   ```
-
-5. Install dependencies
-
-   ```bash
-   pip install -r requirements.txt
-   ```
-
-6. Run migrate
-
-   Before the first launch, migrate the database to the latest version.
-
-   ```bash
-   flask db upgrade
-   ```
-
-7. Start backend:
-
-   ```bash
-   flask run --host 0.0.0.0 --port=5001 --debug
-   ```
-
-8. Setup your application by visiting <http://localhost:5001/console/api/setup> or other apis...
-9. If you need to debug local async processing, please start the worker service.
-
-   ```bash
-   celery -A app.celery worker -P gevent -c 1 --loglevel INFO -Q dataset,generation,mail
-   ```
-
-   The started celery app handles the async tasks, e.g. dataset importing and documents indexing.

api/app.py

@@ -1,6 +1,8 @@
 import os
 
-if not os.environ.get("DEBUG") or os.environ.get("DEBUG").lower() != 'true':
+from configs import dify_config
+
+if os.environ.get("DEBUG", "false").lower() != 'true':
     from gevent import monkey
 
     monkey.patch_all()
@@ -22,7 +24,6 @@ from flask_cors import CORS
 from werkzeug.exceptions import Unauthorized
 
 from commands import register_commands
-from config import Config
 
 # DO NOT REMOVE BELOW
 from events import event_handlers
@@ -42,6 +43,8 @@ from extensions import (
 from extensions.ext_database import db
 from extensions.ext_login import login_manager
 from libs.passport import PassportService
+
+# TODO: Find a way to avoid importing models here
 from models import account, dataset, model, source, task, tool, tools, web
 from services.account_service import AccountService
 
@@ -74,10 +77,28 @@ config_type = os.getenv('EDITION', default='SELF_HOSTED')  # ce edition first
 # Application Factory Function
 # ----------------------------
 
+def create_flask_app_with_configs() -> Flask:
+    """
+    create a raw flask app
+    with configs loaded from .env file
+    """
+    dify_app = DifyApp(__name__)
+    dify_app.config.from_mapping(dify_config.model_dump())
+
+    # populate configs into system environment variables
+    for key, value in dify_app.config.items():
+        if isinstance(value, str):
+            os.environ[key] = value
+        elif isinstance(value, int | float | bool):
+            os.environ[key] = str(value)
+        elif value is None:
+            os.environ[key] = ''
+
+    return dify_app
+
 
 def create_app() -> Flask:
-    app = DifyApp(__name__)
-    app.config.from_object(Config())
+    app = create_flask_app_with_configs()
 
     app.secret_key = app.config['SECRET_KEY']
 
@@ -99,9 +120,21 @@ def create_app() -> Flask:
         level=app.config.get('LOG_LEVEL'),
         format=app.config.get('LOG_FORMAT'),
         datefmt=app.config.get('LOG_DATEFORMAT'),
-        handlers=log_handlers
+        handlers=log_handlers,
+        force=True
     )
+    log_tz = app.config.get('LOG_TZ')
+    if log_tz:
+        from datetime import datetime
 
+        import pytz
+        timezone = pytz.timezone(log_tz)
+
+        def time_converter(seconds):
+            return datetime.utcfromtimestamp(seconds).astimezone(timezone).timetuple()
+
+        for handler in logging.root.handlers:
+            handler.formatter.converter = time_converter
     initialize_extensions(app)
     register_blueprints(app)
     register_commands(app)
@@ -129,27 +162,26 @@ def initialize_extensions(app):
 @login_manager.request_loader
 def load_user_from_request(request_from_flask_login):
     """Load user based on the request."""
-    if request.blueprint in ['console', 'inner_api']:
-        # Check if the user_id contains a dot, indicating the old format
-        auth_header = request.headers.get('Authorization', '')
-        if not auth_header:
-            auth_token = request.args.get('_token')
-            if not auth_token:
-                raise Unauthorized('Invalid Authorization token.')
-        else:
-            if ' ' not in auth_header:
-                raise Unauthorized('Invalid Authorization header format. Expected \'Bearer <api-key>\' format.')
-            auth_scheme, auth_token = auth_header.split(None, 1)
-            auth_scheme = auth_scheme.lower()
-            if auth_scheme != 'bearer':
-                raise Unauthorized('Invalid Authorization header format. Expected \'Bearer <api-key>\' format.')
-
-        decoded = PassportService().verify(auth_token)
-        user_id = decoded.get('user_id')
-
-        return AccountService.load_user(user_id)
-    else:
+    if request.blueprint not in ['console', 'inner_api']:
         return None
+    # Check if the user_id contains a dot, indicating the old format
+    auth_header = request.headers.get('Authorization', '')
+    if not auth_header:
+        auth_token = request.args.get('_token')
+        if not auth_token:
+            raise Unauthorized('Invalid Authorization token.')
+    else:
+        if ' ' not in auth_header:
+            raise Unauthorized('Invalid Authorization header format. Expected \'Bearer <api-key>\' format.')
+        auth_scheme, auth_token = auth_header.split(None, 1)
+        auth_scheme = auth_scheme.lower()
+        if auth_scheme != 'bearer':
+            raise Unauthorized('Invalid Authorization header format. Expected \'Bearer <api-key>\' format.')
+
+    decoded = PassportService().verify(auth_token)
+    user_id = decoded.get('user_id')
+
+    return AccountService.load_logged_in_account(account_id=user_id, token=auth_token)
 
 
 @login_manager.unauthorized_handler
@@ -210,7 +242,7 @@ def register_blueprints(app):
 app = create_app()
 celery = app.extensions["celery"]
 
-if app.config['TESTING']:
+if app.config.get('TESTING'):
     print("App is running in TESTING mode")
 
 

api/commands.py

@@ -8,10 +8,12 @@ import click
 from flask import current_app
 from werkzeug.exceptions import NotFound
 
+from configs import dify_config
 from constants.languages import languages
 from core.rag.datasource.vdb.vector_factory import Vector
 from core.rag.datasource.vdb.vector_type import VectorType
 from core.rag.models.document import Document
+from events.app_event import app_was_created
 from extensions.ext_database import db
 from extensions.ext_redis import redis_client
 from libs.helper import email as email_validate
@@ -111,7 +113,7 @@ def reset_encrypt_key_pair():
     After the reset, all LLM credentials will become invalid, requiring re-entry.
     Only support SELF_HOSTED mode.
     """
-    if current_app.config['EDITION'] != 'SELF_HOSTED':
+    if dify_config.EDITION != 'SELF_HOSTED':
         click.echo(click.style('Sorry, only support SELF_HOSTED mode.', fg='red'))
         return
 
@@ -327,6 +329,22 @@ def migrate_knowledge_vector_database():
                         "vector_store": {"class_prefix": collection_name}
                     }
                     dataset.index_struct = json.dumps(index_struct_dict)
+                elif vector_type == VectorType.OPENSEARCH:
+                    dataset_id = dataset.id
+                    collection_name = Dataset.gen_collection_name_by_id(dataset_id)
+                    index_struct_dict = {
+                        "type": VectorType.OPENSEARCH,
+                        "vector_store": {"class_prefix": collection_name}
+                    }
+                    dataset.index_struct = json.dumps(index_struct_dict)
+                elif vector_type == VectorType.ANALYTICDB:
+                    dataset_id = dataset.id
+                    collection_name = Dataset.gen_collection_name_by_id(dataset_id)
+                    index_struct_dict = {
+                        "type": VectorType.ANALYTICDB,
+                        "vector_store": {"class_prefix": collection_name}
+                    }
+                    dataset.index_struct = json.dumps(index_struct_dict)
                 else:
                     raise ValueError(f"Vector store {vector_type} is not supported.")
 
@@ -577,6 +595,53 @@ def upgrade_db():
         click.echo('Database migration skipped')
 
 
+@click.command('fix-app-site-missing', help='Fix app related site missing issue.')
+def fix_app_site_missing():
+    """
+    Fix app related site missing issue.
+    """
+    click.echo(click.style('Start fix app related site missing issue.', fg='green'))
+
+    failed_app_ids = []
+    while True:
+        sql = """select apps.id as id from apps left join sites on sites.app_id=apps.id
+where sites.id is null limit 1000"""
+        with db.engine.begin() as conn:
+            rs = conn.execute(db.text(sql))
+
+            processed_count = 0
+            for i in rs:
+                processed_count += 1
+                app_id = str(i.id)
+
+                if app_id in failed_app_ids:
+                    continue
+
+                try:
+                    app = db.session.query(App).filter(App.id == app_id).first()
+                    tenant = app.tenant
+                    if tenant:
+                        accounts = tenant.get_accounts()
+                        if not accounts:
+                            print("Fix app {} failed.".format(app.id))
+                            continue
+
+                        account = accounts[0]
+                        print("Fix app {} related site missing issue.".format(app.id))
+                        app_was_created.send(app, account=account)
+                except Exception as e:
+                    failed_app_ids.append(app_id)
+                    click.echo(click.style('Fix app {} related site missing issue failed!'.format(app_id), fg='red'))
+                    logging.exception(f'Fix app related site missing issue failed, error: {e}')
+                    continue
+
+            if not processed_count:
+                break
+
+
+    click.echo(click.style('Congratulations! Fix app related site missing issue successful!', fg='green'))
+
+
 def register_commands(app):
     app.cli.add_command(reset_password)
     app.cli.add_command(reset_email)
@@ -586,3 +651,4 @@ def register_commands(app):
     app.cli.add_command(add_qdrant_doc_id_index)
     app.cli.add_command(create_tenant)
     app.cli.add_command(upgrade_db)
+    app.cli.add_command(fix_app_site_missing)

api/config.py

@@ -1,458 +0,0 @@
-import os
-
-import dotenv
-
-dotenv.load_dotenv()
-
-DEFAULTS = {
-    'EDITION': 'SELF_HOSTED',
-    'DB_USERNAME': 'postgres',
-    'DB_PASSWORD': '',
-    'DB_HOST': 'localhost',
-    'DB_PORT': '5432',
-    'DB_DATABASE': 'dify',
-    'DB_CHARSET': '',
-    'REDIS_HOST': 'localhost',
-    'REDIS_PORT': '6379',
-    'REDIS_DB': '0',
-    'REDIS_USE_SSL': 'False',
-    'OAUTH_REDIRECT_PATH': '/console/api/oauth/authorize',
-    'OAUTH_REDIRECT_INDEX_PATH': '/',
-    'CONSOLE_WEB_URL': 'https://cloud.dify.ai',
-    'CONSOLE_API_URL': 'https://cloud.dify.ai',
-    'SERVICE_API_URL': 'https://api.dify.ai',
-    'APP_WEB_URL': 'https://udify.app',
-    'FILES_URL': '',
-    'FILES_ACCESS_TIMEOUT': 300,
-    'S3_USE_AWS_MANAGED_IAM': 'False',
-    'S3_ADDRESS_STYLE': 'auto',
-    'STORAGE_TYPE': 'local',
-    'STORAGE_LOCAL_PATH': 'storage',
-    'CHECK_UPDATE_URL': 'https://updates.dify.ai',
-    'DEPLOY_ENV': 'PRODUCTION',
-    'SQLALCHEMY_DATABASE_URI_SCHEME': 'postgresql',
-    'SQLALCHEMY_POOL_SIZE': 30,
-    'SQLALCHEMY_MAX_OVERFLOW': 10,
-    'SQLALCHEMY_POOL_RECYCLE': 3600,
-    'SQLALCHEMY_POOL_PRE_PING': 'False',
-    'SQLALCHEMY_ECHO': 'False',
-    'SENTRY_TRACES_SAMPLE_RATE': 1.0,
-    'SENTRY_PROFILES_SAMPLE_RATE': 1.0,
-    'WEAVIATE_GRPC_ENABLED': 'True',
-    'WEAVIATE_BATCH_SIZE': 100,
-    'QDRANT_CLIENT_TIMEOUT': 20,
-    'QDRANT_GRPC_ENABLED': 'False',
-    'QDRANT_GRPC_PORT': '6334',
-    'CELERY_BACKEND': 'database',
-    'LOG_LEVEL': 'INFO',
-    'LOG_FILE': '',
-    'LOG_FORMAT': '%(asctime)s.%(msecs)03d %(levelname)s [%(threadName)s] [%(filename)s:%(lineno)d] - %(message)s',
-    'LOG_DATEFORMAT': '%Y-%m-%d %H:%M:%S',
-    'HOSTED_OPENAI_QUOTA_LIMIT': 200,
-    'HOSTED_OPENAI_TRIAL_ENABLED': 'False',
-    'HOSTED_OPENAI_TRIAL_MODELS': 'gpt-3.5-turbo,gpt-3.5-turbo-1106,gpt-3.5-turbo-instruct,gpt-3.5-turbo-16k,gpt-3.5-turbo-16k-0613,gpt-3.5-turbo-0613,gpt-3.5-turbo-0125,text-davinci-003',
-    'HOSTED_OPENAI_PAID_ENABLED': 'False',
-    'HOSTED_OPENAI_PAID_MODELS': 'gpt-4,gpt-4-turbo-preview,gpt-4-turbo-2024-04-09,gpt-4-1106-preview,gpt-4-0125-preview,gpt-3.5-turbo,gpt-3.5-turbo-16k,gpt-3.5-turbo-16k-0613,gpt-3.5-turbo-1106,gpt-3.5-turbo-0613,gpt-3.5-turbo-0125,gpt-3.5-turbo-instruct,text-davinci-003',
-    'HOSTED_AZURE_OPENAI_ENABLED': 'False',
-    'HOSTED_AZURE_OPENAI_QUOTA_LIMIT': 200,
-    'HOSTED_ANTHROPIC_QUOTA_LIMIT': 600000,
-    'HOSTED_ANTHROPIC_TRIAL_ENABLED': 'False',
-    'HOSTED_ANTHROPIC_PAID_ENABLED': 'False',
-    'HOSTED_MODERATION_ENABLED': 'False',
-    'HOSTED_MODERATION_PROVIDERS': '',
-    'HOSTED_FETCH_APP_TEMPLATES_MODE': 'remote',
-    'HOSTED_FETCH_APP_TEMPLATES_REMOTE_DOMAIN': 'https://tmpl.dify.ai',
-    'CLEAN_DAY_SETTING': 30,
-    'UPLOAD_FILE_SIZE_LIMIT': 15,
-    'UPLOAD_FILE_BATCH_LIMIT': 5,
-    'UPLOAD_IMAGE_FILE_SIZE_LIMIT': 10,
-    'OUTPUT_MODERATION_BUFFER_SIZE': 300,
-    'MULTIMODAL_SEND_IMAGE_FORMAT': 'base64',
-    'INVITE_EXPIRY_HOURS': 72,
-    'BILLING_ENABLED': 'False',
-    'CAN_REPLACE_LOGO': 'False',
-    'MODEL_LB_ENABLED': 'False',
-    'ETL_TYPE': 'dify',
-    'KEYWORD_STORE': 'jieba',
-    'BATCH_UPLOAD_LIMIT': 20,
-    'CODE_EXECUTION_ENDPOINT': 'http://sandbox:8194',
-    'CODE_EXECUTION_API_KEY': 'dify-sandbox',
-    'TOOL_ICON_CACHE_MAX_AGE': 3600,
-    'MILVUS_DATABASE': 'default',
-    'KEYWORD_DATA_SOURCE_TYPE': 'database',
-    'INNER_API': 'False',
-    'ENTERPRISE_ENABLED': 'False',
-    'INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH': 1000,
-    'WORKFLOW_MAX_EXECUTION_STEPS': 500,
-    'WORKFLOW_MAX_EXECUTION_TIME': 1200,
-    'WORKFLOW_CALL_MAX_DEPTH': 5,
-    'APP_MAX_EXECUTION_TIME': 1200,
-}
-
-
-def get_env(key):
-    return os.environ.get(key, DEFAULTS.get(key))
-
-
-def get_bool_env(key):
-    value = get_env(key)
-    return value.lower() == 'true' if value is not None else False
-
-
-def get_cors_allow_origins(env, default):
-    cors_allow_origins = []
-    if get_env(env):
-        for origin in get_env(env).split(','):
-            cors_allow_origins.append(origin)
-    else:
-        cors_allow_origins = [default]
-
-    return cors_allow_origins
-
-
-class Config:
-    """Application configuration class."""
-
-    def __init__(self):
-        # ------------------------
-        # General Configurations.
-        # ------------------------
-        self.CURRENT_VERSION = "0.6.11"
-        self.COMMIT_SHA = get_env('COMMIT_SHA')
-        self.EDITION = get_env('EDITION')
-        self.DEPLOY_ENV = get_env('DEPLOY_ENV')
-        self.TESTING = False
-        self.LOG_LEVEL = get_env('LOG_LEVEL')
-        self.LOG_FILE = get_env('LOG_FILE')
-        self.LOG_FORMAT = get_env('LOG_FORMAT')
-        self.LOG_DATEFORMAT = get_env('LOG_DATEFORMAT')
-        self.API_COMPRESSION_ENABLED = get_bool_env('API_COMPRESSION_ENABLED')
-
-        # The backend URL prefix of the console API.
-        # used to concatenate the login authorization callback or notion integration callback.
-        self.CONSOLE_API_URL = get_env('CONSOLE_API_URL')
-
-        # The front-end URL prefix of the console web.
-        # used to concatenate some front-end addresses and for CORS configuration use.
-        self.CONSOLE_WEB_URL = get_env('CONSOLE_WEB_URL')
-
-        # WebApp Url prefix.
-        # used to display WebAPP API Base Url to the front-end.
-        self.APP_WEB_URL = get_env('APP_WEB_URL')
-
-        # Service API Url prefix.
-        # used to display Service API Base Url to the front-end.
-        self.SERVICE_API_URL = get_env('SERVICE_API_URL')
-
-        # File preview or download Url prefix.
-        # used to display File preview or download Url to the front-end or as Multi-model inputs;
-        # Url is signed and has expiration time.
-        self.FILES_URL = get_env('FILES_URL') if get_env('FILES_URL') else self.CONSOLE_API_URL
-
-        # File Access Time specifies a time interval in seconds for the file to be accessed.
-        # The default value is 300 seconds.
-        self.FILES_ACCESS_TIMEOUT = int(get_env('FILES_ACCESS_TIMEOUT'))
-
-        # Your App secret key will be used for securely signing the session cookie
-        # Make sure you are changing this key for your deployment with a strong key.
-        # You can generate a strong key using `openssl rand -base64 42`.
-        # Alternatively you can set it with `SECRET_KEY` environment variable.
-        self.SECRET_KEY = get_env('SECRET_KEY')
-
-        # Enable or disable the inner API.
-        self.INNER_API = get_bool_env('INNER_API')
-        # The inner API key is used to authenticate the inner API.
-        self.INNER_API_KEY = get_env('INNER_API_KEY')
-
-        # cors settings
-        self.CONSOLE_CORS_ALLOW_ORIGINS = get_cors_allow_origins(
-            'CONSOLE_CORS_ALLOW_ORIGINS', self.CONSOLE_WEB_URL)
-        self.WEB_API_CORS_ALLOW_ORIGINS = get_cors_allow_origins(
-            'WEB_API_CORS_ALLOW_ORIGINS', '*')
-
-        # check update url
-        self.CHECK_UPDATE_URL = get_env('CHECK_UPDATE_URL')
-
-        # ------------------------
-        # Database Configurations.
-        # ------------------------
-        db_credentials = {
-            key: get_env(key) for key in
-            ['DB_USERNAME', 'DB_PASSWORD', 'DB_HOST', 'DB_PORT', 'DB_DATABASE', 'DB_CHARSET']
-        }
-        self.SQLALCHEMY_DATABASE_URI_SCHEME = get_env('SQLALCHEMY_DATABASE_URI_SCHEME')
-
-        db_extras = f"?client_encoding={db_credentials['DB_CHARSET']}" if db_credentials['DB_CHARSET'] else ""
-
-        self.SQLALCHEMY_DATABASE_URI = f"{self.SQLALCHEMY_DATABASE_URI_SCHEME}://{db_credentials['DB_USERNAME']}:{db_credentials['DB_PASSWORD']}@{db_credentials['DB_HOST']}:{db_credentials['DB_PORT']}/{db_credentials['DB_DATABASE']}{db_extras}"
-        self.SQLALCHEMY_ENGINE_OPTIONS = {
-            'pool_size': int(get_env('SQLALCHEMY_POOL_SIZE')),
-            'max_overflow': int(get_env('SQLALCHEMY_MAX_OVERFLOW')),
-            'pool_recycle': int(get_env('SQLALCHEMY_POOL_RECYCLE')),
-            'pool_pre_ping': get_bool_env('SQLALCHEMY_POOL_PRE_PING'),
-            'connect_args': {'options': '-c timezone=UTC'},
-        }
-
-        self.SQLALCHEMY_ECHO = get_bool_env('SQLALCHEMY_ECHO')
-
-        # ------------------------
-        # Redis Configurations.
-        # ------------------------
-        self.REDIS_HOST = get_env('REDIS_HOST')
-        self.REDIS_PORT = get_env('REDIS_PORT')
-        self.REDIS_USERNAME = get_env('REDIS_USERNAME')
-        self.REDIS_PASSWORD = get_env('REDIS_PASSWORD')
-        self.REDIS_DB = get_env('REDIS_DB')
-        self.REDIS_USE_SSL = get_bool_env('REDIS_USE_SSL')
-
-        # ------------------------
-        # Celery worker Configurations.
-        # ------------------------
-        self.CELERY_BROKER_URL = get_env('CELERY_BROKER_URL')
-        self.CELERY_BACKEND = get_env('CELERY_BACKEND')
-        self.CELERY_RESULT_BACKEND = 'db+{}'.format(self.SQLALCHEMY_DATABASE_URI) \
-            if self.CELERY_BACKEND == 'database' else self.CELERY_BROKER_URL
-        self.BROKER_USE_SSL = self.CELERY_BROKER_URL.startswith('rediss://')
-
-        # ------------------------
-        # Code Execution Sandbox Configurations.
-        # ------------------------
-        self.CODE_EXECUTION_ENDPOINT = get_env('CODE_EXECUTION_ENDPOINT')
-        self.CODE_EXECUTION_API_KEY = get_env('CODE_EXECUTION_API_KEY')
-
-        # ------------------------
-        # File Storage Configurations.
-        # ------------------------
-        self.STORAGE_TYPE = get_env('STORAGE_TYPE')
-        self.STORAGE_LOCAL_PATH = get_env('STORAGE_LOCAL_PATH')
-
-        # S3 Storage settings
-        self.S3_USE_AWS_MANAGED_IAM = get_bool_env('S3_USE_AWS_MANAGED_IAM')
-        self.S3_ENDPOINT = get_env('S3_ENDPOINT')
-        self.S3_BUCKET_NAME = get_env('S3_BUCKET_NAME')
-        self.S3_ACCESS_KEY = get_env('S3_ACCESS_KEY')
-        self.S3_SECRET_KEY = get_env('S3_SECRET_KEY')
-        self.S3_REGION = get_env('S3_REGION')
-        self.S3_ADDRESS_STYLE = get_env('S3_ADDRESS_STYLE')
-
-        # Azure Blob Storage settings
-        self.AZURE_BLOB_ACCOUNT_NAME = get_env('AZURE_BLOB_ACCOUNT_NAME')
-        self.AZURE_BLOB_ACCOUNT_KEY = get_env('AZURE_BLOB_ACCOUNT_KEY')
-        self.AZURE_BLOB_CONTAINER_NAME = get_env('AZURE_BLOB_CONTAINER_NAME')
-        self.AZURE_BLOB_ACCOUNT_URL = get_env('AZURE_BLOB_ACCOUNT_URL')
-
-        # Aliyun Storage settings
-        self.ALIYUN_OSS_BUCKET_NAME = get_env('ALIYUN_OSS_BUCKET_NAME')
-        self.ALIYUN_OSS_ACCESS_KEY = get_env('ALIYUN_OSS_ACCESS_KEY')
-        self.ALIYUN_OSS_SECRET_KEY = get_env('ALIYUN_OSS_SECRET_KEY')
-        self.ALIYUN_OSS_ENDPOINT = get_env('ALIYUN_OSS_ENDPOINT')
-        self.ALIYUN_OSS_REGION = get_env('ALIYUN_OSS_REGION')
-        self.ALIYUN_OSS_AUTH_VERSION = get_env('ALIYUN_OSS_AUTH_VERSION')
-
-        # Google Cloud Storage settings
-        self.GOOGLE_STORAGE_BUCKET_NAME = get_env('GOOGLE_STORAGE_BUCKET_NAME')
-        self.GOOGLE_STORAGE_SERVICE_ACCOUNT_JSON_BASE64 = get_env('GOOGLE_STORAGE_SERVICE_ACCOUNT_JSON_BASE64')
-
-        # Tencent Cos Storage settings
-        self.TENCENT_COS_BUCKET_NAME = get_env('TENCENT_COS_BUCKET_NAME')
-        self.TENCENT_COS_REGION = get_env('TENCENT_COS_REGION')
-        self.TENCENT_COS_SECRET_ID = get_env('TENCENT_COS_SECRET_ID')
-        self.TENCENT_COS_SECRET_KEY = get_env('TENCENT_COS_SECRET_KEY')
-        self.TENCENT_COS_SCHEME = get_env('TENCENT_COS_SCHEME')
-
-        # ------------------------
-        # Vector Store Configurations.
-        # Currently, only support: qdrant, milvus, zilliz, weaviate, relyt, pgvector
-        # ------------------------
-        self.VECTOR_STORE = get_env('VECTOR_STORE')
-        self.KEYWORD_STORE = get_env('KEYWORD_STORE')
-
-        # qdrant settings
-        self.QDRANT_URL = get_env('QDRANT_URL')
-        self.QDRANT_API_KEY = get_env('QDRANT_API_KEY')
-        self.QDRANT_CLIENT_TIMEOUT = get_env('QDRANT_CLIENT_TIMEOUT')
-        self.QDRANT_GRPC_ENABLED = get_env('QDRANT_GRPC_ENABLED')
-        self.QDRANT_GRPC_PORT = get_env('QDRANT_GRPC_PORT')
-
-        # milvus / zilliz setting
-        self.MILVUS_HOST = get_env('MILVUS_HOST')
-        self.MILVUS_PORT = get_env('MILVUS_PORT')
-        self.MILVUS_USER = get_env('MILVUS_USER')
-        self.MILVUS_PASSWORD = get_env('MILVUS_PASSWORD')
-        self.MILVUS_SECURE = get_env('MILVUS_SECURE')
-        self.MILVUS_DATABASE = get_env('MILVUS_DATABASE')
-
-        # weaviate settings
-        self.WEAVIATE_ENDPOINT = get_env('WEAVIATE_ENDPOINT')
-        self.WEAVIATE_API_KEY = get_env('WEAVIATE_API_KEY')
-        self.WEAVIATE_GRPC_ENABLED = get_bool_env('WEAVIATE_GRPC_ENABLED')
-        self.WEAVIATE_BATCH_SIZE = int(get_env('WEAVIATE_BATCH_SIZE'))
-
-        # relyt settings
-        self.RELYT_HOST = get_env('RELYT_HOST')
-        self.RELYT_PORT = get_env('RELYT_PORT')
-        self.RELYT_USER = get_env('RELYT_USER')
-        self.RELYT_PASSWORD = get_env('RELYT_PASSWORD')
-        self.RELYT_DATABASE = get_env('RELYT_DATABASE')
-
-
-        # tencent settings
-        self.TENCENT_VECTOR_DB_URL = get_env('TENCENT_VECTOR_DB_URL')
-        self.TENCENT_VECTOR_DB_API_KEY = get_env('TENCENT_VECTOR_DB_API_KEY')
-        self.TENCENT_VECTOR_DB_TIMEOUT = get_env('TENCENT_VECTOR_DB_TIMEOUT')
-        self.TENCENT_VECTOR_DB_USERNAME = get_env('TENCENT_VECTOR_DB_USERNAME')
-        self.TENCENT_VECTOR_DB_DATABASE = get_env('TENCENT_VECTOR_DB_DATABASE')
-        self.TENCENT_VECTOR_DB_SHARD = get_env('TENCENT_VECTOR_DB_SHARD')
-        self.TENCENT_VECTOR_DB_REPLICAS = get_env('TENCENT_VECTOR_DB_REPLICAS')
-
-        # pgvecto rs settings
-        self.PGVECTO_RS_HOST = get_env('PGVECTO_RS_HOST')
-        self.PGVECTO_RS_PORT = get_env('PGVECTO_RS_PORT')
-        self.PGVECTO_RS_USER = get_env('PGVECTO_RS_USER')
-        self.PGVECTO_RS_PASSWORD = get_env('PGVECTO_RS_PASSWORD')
-        self.PGVECTO_RS_DATABASE = get_env('PGVECTO_RS_DATABASE')
-
-        # pgvector settings
-        self.PGVECTOR_HOST = get_env('PGVECTOR_HOST')
-        self.PGVECTOR_PORT = get_env('PGVECTOR_PORT')
-        self.PGVECTOR_USER = get_env('PGVECTOR_USER')
-        self.PGVECTOR_PASSWORD = get_env('PGVECTOR_PASSWORD')
-        self.PGVECTOR_DATABASE = get_env('PGVECTOR_DATABASE')
-
-        # tidb-vector settings
-        self.TIDB_VECTOR_HOST = get_env('TIDB_VECTOR_HOST')
-        self.TIDB_VECTOR_PORT = get_env('TIDB_VECTOR_PORT')
-        self.TIDB_VECTOR_USER = get_env('TIDB_VECTOR_USER')
-        self.TIDB_VECTOR_PASSWORD = get_env('TIDB_VECTOR_PASSWORD')
-        self.TIDB_VECTOR_DATABASE = get_env('TIDB_VECTOR_DATABASE')
-
-        # chroma settings
-        self.CHROMA_HOST = get_env('CHROMA_HOST')
-        self.CHROMA_PORT = get_env('CHROMA_PORT')
-        self.CHROMA_TENANT = get_env('CHROMA_TENANT')
-        self.CHROMA_DATABASE = get_env('CHROMA_DATABASE')
-        self.CHROMA_AUTH_PROVIDER = get_env('CHROMA_AUTH_PROVIDER')
-        self.CHROMA_AUTH_CREDENTIALS = get_env('CHROMA_AUTH_CREDENTIALS')
-
-        # ------------------------
-        # Mail Configurations.
-        # ------------------------
-        self.MAIL_TYPE = get_env('MAIL_TYPE')
-        self.MAIL_DEFAULT_SEND_FROM = get_env('MAIL_DEFAULT_SEND_FROM')
-        self.RESEND_API_KEY = get_env('RESEND_API_KEY')
-        self.RESEND_API_URL = get_env('RESEND_API_URL')
-        # SMTP settings
-        self.SMTP_SERVER = get_env('SMTP_SERVER')
-        self.SMTP_PORT = get_env('SMTP_PORT')
-        self.SMTP_USERNAME = get_env('SMTP_USERNAME')
-        self.SMTP_PASSWORD = get_env('SMTP_PASSWORD')
-        self.SMTP_USE_TLS = get_bool_env('SMTP_USE_TLS')
-        self.SMTP_OPPORTUNISTIC_TLS = get_bool_env('SMTP_OPPORTUNISTIC_TLS')
-
-        # ------------------------
-        # Workspace Configurations.
-        # ------------------------
-        self.INVITE_EXPIRY_HOURS = int(get_env('INVITE_EXPIRY_HOURS'))
-
-        # ------------------------
-        # Sentry Configurations.
-        # ------------------------
-        self.SENTRY_DSN = get_env('SENTRY_DSN')
-        self.SENTRY_TRACES_SAMPLE_RATE = float(get_env('SENTRY_TRACES_SAMPLE_RATE'))
-        self.SENTRY_PROFILES_SAMPLE_RATE = float(get_env('SENTRY_PROFILES_SAMPLE_RATE'))
-
-        # ------------------------
-        # Business Configurations.
-        # ------------------------
-
-        # multi model send image format, support base64, url, default is base64
-        self.MULTIMODAL_SEND_IMAGE_FORMAT = get_env('MULTIMODAL_SEND_IMAGE_FORMAT')
-
-        # Dataset Configurations.
-        self.CLEAN_DAY_SETTING = get_env('CLEAN_DAY_SETTING')
-
-        # File upload Configurations.
-        self.UPLOAD_FILE_SIZE_LIMIT = int(get_env('UPLOAD_FILE_SIZE_LIMIT'))
-        self.UPLOAD_FILE_BATCH_LIMIT = int(get_env('UPLOAD_FILE_BATCH_LIMIT'))
-        self.UPLOAD_IMAGE_FILE_SIZE_LIMIT = int(get_env('UPLOAD_IMAGE_FILE_SIZE_LIMIT'))
-        self.BATCH_UPLOAD_LIMIT = get_env('BATCH_UPLOAD_LIMIT')
-
-        # RAG ETL Configurations.
-        self.ETL_TYPE = get_env('ETL_TYPE')
-        self.UNSTRUCTURED_API_URL = get_env('UNSTRUCTURED_API_URL')
-        self.UNSTRUCTURED_API_KEY = get_env('UNSTRUCTURED_API_KEY')
-        self.KEYWORD_DATA_SOURCE_TYPE = get_env('KEYWORD_DATA_SOURCE_TYPE')
-
-        # Indexing Configurations.
-        self.INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH = get_env('INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH')
-
-        # Tool Configurations.
-        self.TOOL_ICON_CACHE_MAX_AGE = get_env('TOOL_ICON_CACHE_MAX_AGE')
-
-        self.WORKFLOW_MAX_EXECUTION_STEPS = int(get_env('WORKFLOW_MAX_EXECUTION_STEPS'))
-        self.WORKFLOW_MAX_EXECUTION_TIME = int(get_env('WORKFLOW_MAX_EXECUTION_TIME'))
-        self.WORKFLOW_CALL_MAX_DEPTH = int(get_env('WORKFLOW_CALL_MAX_DEPTH'))
-        self.APP_MAX_EXECUTION_TIME = int(get_env('APP_MAX_EXECUTION_TIME'))
-
-        # Moderation in app Configurations.
-        self.OUTPUT_MODERATION_BUFFER_SIZE = int(get_env('OUTPUT_MODERATION_BUFFER_SIZE'))
-
-        # Notion integration setting
-        self.NOTION_CLIENT_ID = get_env('NOTION_CLIENT_ID')
-        self.NOTION_CLIENT_SECRET = get_env('NOTION_CLIENT_SECRET')
-        self.NOTION_INTEGRATION_TYPE = get_env('NOTION_INTEGRATION_TYPE')
-        self.NOTION_INTERNAL_SECRET = get_env('NOTION_INTERNAL_SECRET')
-        self.NOTION_INTEGRATION_TOKEN = get_env('NOTION_INTEGRATION_TOKEN')
-
-        # ------------------------
-        # Platform Configurations.
-        # ------------------------
-        self.GITHUB_CLIENT_ID = get_env('GITHUB_CLIENT_ID')
-        self.GITHUB_CLIENT_SECRET = get_env('GITHUB_CLIENT_SECRET')
-        self.GOOGLE_CLIENT_ID = get_env('GOOGLE_CLIENT_ID')
-        self.GOOGLE_CLIENT_SECRET = get_env('GOOGLE_CLIENT_SECRET')
-        self.OAUTH_REDIRECT_PATH = get_env('OAUTH_REDIRECT_PATH')
-
-        self.HOSTED_OPENAI_API_KEY = get_env('HOSTED_OPENAI_API_KEY')
-        self.HOSTED_OPENAI_API_BASE = get_env('HOSTED_OPENAI_API_BASE')
-        self.HOSTED_OPENAI_API_ORGANIZATION = get_env('HOSTED_OPENAI_API_ORGANIZATION')
-        self.HOSTED_OPENAI_TRIAL_ENABLED = get_bool_env('HOSTED_OPENAI_TRIAL_ENABLED')
-        self.HOSTED_OPENAI_TRIAL_MODELS = get_env('HOSTED_OPENAI_TRIAL_MODELS')
-        self.HOSTED_OPENAI_QUOTA_LIMIT = int(get_env('HOSTED_OPENAI_QUOTA_LIMIT'))
-        self.HOSTED_OPENAI_PAID_ENABLED = get_bool_env('HOSTED_OPENAI_PAID_ENABLED')
-        self.HOSTED_OPENAI_PAID_MODELS = get_env('HOSTED_OPENAI_PAID_MODELS')
-
-        self.HOSTED_AZURE_OPENAI_ENABLED = get_bool_env('HOSTED_AZURE_OPENAI_ENABLED')
-        self.HOSTED_AZURE_OPENAI_API_KEY = get_env('HOSTED_AZURE_OPENAI_API_KEY')
-        self.HOSTED_AZURE_OPENAI_API_BASE = get_env('HOSTED_AZURE_OPENAI_API_BASE')
-        self.HOSTED_AZURE_OPENAI_QUOTA_LIMIT = int(get_env('HOSTED_AZURE_OPENAI_QUOTA_LIMIT'))
-
-        self.HOSTED_ANTHROPIC_API_BASE = get_env('HOSTED_ANTHROPIC_API_BASE')
-        self.HOSTED_ANTHROPIC_API_KEY = get_env('HOSTED_ANTHROPIC_API_KEY')
-        self.HOSTED_ANTHROPIC_TRIAL_ENABLED = get_bool_env('HOSTED_ANTHROPIC_TRIAL_ENABLED')
-        self.HOSTED_ANTHROPIC_QUOTA_LIMIT = int(get_env('HOSTED_ANTHROPIC_QUOTA_LIMIT'))
-        self.HOSTED_ANTHROPIC_PAID_ENABLED = get_bool_env('HOSTED_ANTHROPIC_PAID_ENABLED')
-
-        self.HOSTED_MINIMAX_ENABLED = get_bool_env('HOSTED_MINIMAX_ENABLED')
-        self.HOSTED_SPARK_ENABLED = get_bool_env('HOSTED_SPARK_ENABLED')
-        self.HOSTED_ZHIPUAI_ENABLED = get_bool_env('HOSTED_ZHIPUAI_ENABLED')
-
-        self.HOSTED_MODERATION_ENABLED = get_bool_env('HOSTED_MODERATION_ENABLED')
-        self.HOSTED_MODERATION_PROVIDERS = get_env('HOSTED_MODERATION_PROVIDERS')
-
-        # fetch app templates mode, remote, builtin, db(only for dify SaaS), default: remote
-        self.HOSTED_FETCH_APP_TEMPLATES_MODE = get_env('HOSTED_FETCH_APP_TEMPLATES_MODE')
-        self.HOSTED_FETCH_APP_TEMPLATES_REMOTE_DOMAIN = get_env('HOSTED_FETCH_APP_TEMPLATES_REMOTE_DOMAIN')
-
-        # Model Load Balancing Configurations.
-        self.MODEL_LB_ENABLED = get_bool_env('MODEL_LB_ENABLED')
-
-        # Platform Billing Configurations.
-        self.BILLING_ENABLED = get_bool_env('BILLING_ENABLED')
-
-        # ------------------------
-        # Enterprise feature Configurations.
-        # **Before using, please contact business@dify.ai by email to inquire about licensing matters.**
-        # ------------------------
-        self.ENTERPRISE_ENABLED = get_bool_env('ENTERPRISE_ENABLED')
-        self.CAN_REPLACE_LOGO = get_bool_env('CAN_REPLACE_LOGO')

api/configs/__init__.py

@@ -0,0 +1,3 @@
+from .app_config import DifyConfig
+
+dify_config = DifyConfig()

api/configs/app_config.py

@@ -0,0 +1,67 @@
+from pydantic import Field, computed_field
+from pydantic_settings import SettingsConfigDict
+
+from configs.deploy import DeploymentConfig
+from configs.enterprise import EnterpriseFeatureConfig
+from configs.extra import ExtraServiceConfig
+from configs.feature import FeatureConfig
+from configs.middleware import MiddlewareConfig
+from configs.packaging import PackagingInfo
+
+
+class DifyConfig(
+    # Packaging info
+    PackagingInfo,
+
+    # Deployment configs
+    DeploymentConfig,
+
+    # Feature configs
+    FeatureConfig,
+
+    # Middleware configs
+    MiddlewareConfig,
+
+    # Extra service configs
+    ExtraServiceConfig,
+
+    # Enterprise feature configs
+    # **Before using, please contact business@dify.ai by email to inquire about licensing matters.**
+    EnterpriseFeatureConfig,
+):
+    DEBUG: bool = Field(default=False, description='whether to enable debug mode.')
+
+    model_config = SettingsConfigDict(
+        # read from dotenv format config file
+        env_file='.env',
+        env_file_encoding='utf-8',
+        frozen=True,
+
+        # ignore extra attributes
+        extra='ignore',
+    )
+
+    CODE_MAX_NUMBER: int = 9223372036854775807
+    CODE_MIN_NUMBER: int = -9223372036854775808
+    CODE_MAX_STRING_LENGTH: int = 80000
+    CODE_MAX_STRING_ARRAY_LENGTH: int = 30
+    CODE_MAX_OBJECT_ARRAY_LENGTH: int = 30
+    CODE_MAX_NUMBER_ARRAY_LENGTH: int = 1000
+
+    HTTP_REQUEST_MAX_CONNECT_TIMEOUT: int = 300
+    HTTP_REQUEST_MAX_READ_TIMEOUT: int = 600
+    HTTP_REQUEST_MAX_WRITE_TIMEOUT: int = 600
+    HTTP_REQUEST_NODE_MAX_BINARY_SIZE: int = 1024 * 1024 * 10
+
+    @computed_field
+    def HTTP_REQUEST_NODE_READABLE_MAX_BINARY_SIZE(self) -> str:
+        return f'{self.HTTP_REQUEST_NODE_MAX_BINARY_SIZE / 1024 / 1024:.2f}MB'
+
+    HTTP_REQUEST_NODE_MAX_TEXT_SIZE: int = 1024 * 1024
+
+    @computed_field
+    def HTTP_REQUEST_NODE_READABLE_MAX_TEXT_SIZE(self) -> str:
+        return f'{self.HTTP_REQUEST_NODE_MAX_TEXT_SIZE / 1024 / 1024:.2f}MB'
+
+    SSRF_PROXY_HTTP_URL: str | None = None
+    SSRF_PROXY_HTTPS_URL: str | None = None

api/configs/deploy/__init__.py

@@ -0,0 +1,27 @@
+from pydantic import Field
+from pydantic_settings import BaseSettings
+
+
+class DeploymentConfig(BaseSettings):
+    """
+    Deployment configs
+    """
+    APPLICATION_NAME: str = Field(
+        description='application name',
+        default='langgenius/dify',
+    )
+
+    TESTING: bool = Field(
+        description='',
+        default=False,
+    )
+
+    EDITION: str = Field(
+        description='deployment edition',
+        default='SELF_HOSTED',
+    )
+
+    DEPLOY_ENV: str = Field(
+        description='deployment environment, default to PRODUCTION.',
+        default='PRODUCTION',
+    )

api/configs/enterprise/__init__.py

@@ -0,0 +1,19 @@
+from pydantic import Field
+from pydantic_settings import BaseSettings
+
+
+class EnterpriseFeatureConfig(BaseSettings):
+    """
+    Enterprise feature configs.
+    **Before using, please contact business@dify.ai by email to inquire about licensing matters.**
+    """
+    ENTERPRISE_ENABLED: bool = Field(
+        description='whether to enable enterprise features.'
+                    'Before using, please contact business@dify.ai by email to inquire about licensing matters.',
+        default=False,
+    )
+
+    CAN_REPLACE_LOGO: bool = Field(
+        description='whether to allow replacing enterprise logo.',
+        default=False,
+    )

api/configs/extra/__init__.py

@@ -0,0 +1,10 @@
+from configs.extra.notion_config import NotionConfig
+from configs.extra.sentry_config import SentryConfig
+
+
+class ExtraServiceConfig(
+    # place the configs in alphabet order
+    NotionConfig,
+    SentryConfig,
+):
+    pass

api/configs/extra/notion_config.py

@@ -0,0 +1,34 @@
+from typing import Optional
+
+from pydantic import Field
+from pydantic_settings import BaseSettings
+
+
+class NotionConfig(BaseSettings):
+    """
+    Notion integration configs
+    """
+    NOTION_CLIENT_ID: Optional[str] = Field(
+        description='Notion client ID',
+        default=None,
+    )
+
+    NOTION_CLIENT_SECRET: Optional[str] = Field(
+        description='Notion client secret key',
+        default=None,
+    )
+
+    NOTION_INTEGRATION_TYPE: Optional[str] = Field(
+        description='Notion integration type, default to None, available values: internal.',
+        default=None,
+    )
+
+    NOTION_INTERNAL_SECRET: Optional[str] = Field(
+        description='Notion internal secret key',
+        default=None,
+    )
+
+    NOTION_INTEGRATION_TOKEN: Optional[str] = Field(
+        description='Notion integration token',
+        default=None,
+    )

api/configs/extra/sentry_config.py

@@ -0,0 +1,24 @@
+from typing import Optional
+
+from pydantic import Field, NonNegativeFloat
+from pydantic_settings import BaseSettings
+
+
+class SentryConfig(BaseSettings):
+    """
+    Sentry configs
+    """
+    SENTRY_DSN: Optional[str] = Field(
+        description='Sentry DSN',
+        default=None,
+    )
+
+    SENTRY_TRACES_SAMPLE_RATE: NonNegativeFloat = Field(
+        description='Sentry trace sample rate',
+        default=1.0,
+    )
+
+    SENTRY_PROFILES_SAMPLE_RATE: NonNegativeFloat = Field(
+        description='Sentry profiles sample rate',
+        default=1.0,
+    )

api/configs/feature/__init__.py

@@ -0,0 +1,466 @@
+from typing import Optional
+
+from pydantic import AliasChoices, Field, NonNegativeInt, PositiveInt, computed_field
+from pydantic_settings import BaseSettings
+
+from configs.feature.hosted_service import HostedServiceConfig
+
+
+class SecurityConfig(BaseSettings):
+    """
+    Secret Key configs
+    """
+    SECRET_KEY: Optional[str] = Field(
+        description='Your App secret key will be used for securely signing the session cookie'
+                    'Make sure you are changing this key for your deployment with a strong key.'
+                    'You can generate a strong key using `openssl rand -base64 42`.'
+                    'Alternatively you can set it with `SECRET_KEY` environment variable.',
+        default=None,
+    )
+
+    RESET_PASSWORD_TOKEN_EXPIRY_HOURS: PositiveInt = Field(
+        description='Expiry time in hours for reset token',
+        default=24,
+    )
+
+class AppExecutionConfig(BaseSettings):
+    """
+    App Execution configs
+    """
+    APP_MAX_EXECUTION_TIME: PositiveInt = Field(
+        description='execution timeout in seconds for app execution',
+        default=1200,
+    )
+    APP_MAX_ACTIVE_REQUESTS: NonNegativeInt = Field(
+        description='max active request per app, 0 means unlimited',
+        default=0,
+    )
+
+
+class CodeExecutionSandboxConfig(BaseSettings):
+    """
+    Code Execution Sandbox configs
+    """
+    CODE_EXECUTION_ENDPOINT: str = Field(
+        description='endpoint URL of code execution servcie',
+        default='http://sandbox:8194',
+    )
+
+    CODE_EXECUTION_API_KEY: str = Field(
+        description='API key for code execution service',
+        default='dify-sandbox',
+    )
+
+
+class EndpointConfig(BaseSettings):
+    """
+    Module URL configs
+    """
+    CONSOLE_API_URL: str = Field(
+        description='The backend URL prefix of the console API.'
+                    'used to concatenate the login authorization callback or notion integration callback.',
+        default='',
+    )
+
+    CONSOLE_WEB_URL: str = Field(
+        description='The front-end URL prefix of the console web.'
+                    'used to concatenate some front-end addresses and for CORS configuration use.',
+        default='',
+    )
+
+    SERVICE_API_URL: str = Field(
+        description='Service API Url prefix.'
+                    'used to display Service API Base Url to the front-end.',
+        default='',
+    )
+
+    APP_WEB_URL: str = Field(
+        description='WebApp Url prefix.'
+                    'used to display WebAPP API Base Url to the front-end.',
+        default='',
+    )
+
+
+class FileAccessConfig(BaseSettings):
+    """
+    File Access configs
+    """
+    FILES_URL: str = Field(
+        description='File preview or download Url prefix.'
+                    ' used to display File preview or download Url to the front-end or as Multi-model inputs;'
+                    'Url is signed and has expiration time.',
+        validation_alias=AliasChoices('FILES_URL', 'CONSOLE_API_URL'),
+        alias_priority=1,
+        default='',
+    )
+
+    FILES_ACCESS_TIMEOUT: int = Field(
+        description='timeout in seconds for file accessing',
+        default=300,
+    )
+
+
+class FileUploadConfig(BaseSettings):
+    """
+    File Uploading configs
+    """
+    UPLOAD_FILE_SIZE_LIMIT: NonNegativeInt = Field(
+        description='size limit in Megabytes for uploading files',
+        default=15,
+    )
+
+    UPLOAD_FILE_BATCH_LIMIT: NonNegativeInt = Field(
+        description='batch size limit for uploading files',
+        default=5,
+    )
+
+    UPLOAD_IMAGE_FILE_SIZE_LIMIT: NonNegativeInt = Field(
+        description='image file size limit in Megabytes for uploading files',
+        default=10,
+    )
+
+    BATCH_UPLOAD_LIMIT: NonNegativeInt = Field(
+        description='',  # todo: to be clarified
+        default=20,
+    )
+
+
+class HttpConfig(BaseSettings):
+    """
+    HTTP configs
+    """
+    API_COMPRESSION_ENABLED: bool = Field(
+        description='whether to enable HTTP response compression of gzip',
+        default=False,
+    )
+
+    inner_CONSOLE_CORS_ALLOW_ORIGINS: str = Field(
+        description='',
+        validation_alias=AliasChoices('CONSOLE_CORS_ALLOW_ORIGINS', 'CONSOLE_WEB_URL'),
+        default='',
+    )
+
+    @computed_field
+    @property
+    def CONSOLE_CORS_ALLOW_ORIGINS(self) -> list[str]:
+        return self.inner_CONSOLE_CORS_ALLOW_ORIGINS.split(',')
+
+    inner_WEB_API_CORS_ALLOW_ORIGINS: str = Field(
+        description='',
+        validation_alias=AliasChoices('WEB_API_CORS_ALLOW_ORIGINS'),
+        default='*',
+    )
+
+    @computed_field
+    @property
+    def WEB_API_CORS_ALLOW_ORIGINS(self) -> list[str]:
+        return self.inner_WEB_API_CORS_ALLOW_ORIGINS.split(',')
+
+
+class InnerAPIConfig(BaseSettings):
+    """
+    Inner API configs
+    """
+    INNER_API: bool = Field(
+        description='whether to enable the inner API',
+        default=False,
+    )
+
+    INNER_API_KEY: Optional[str] = Field(
+        description='The inner API key is used to authenticate the inner API',
+        default=None,
+    )
+
+
+class LoggingConfig(BaseSettings):
+    """
+    Logging configs
+    """
+
+    LOG_LEVEL: str = Field(
+        description='Log output level, default to INFO.'
+                    'It is recommended to set it to ERROR for production.',
+        default='INFO',
+    )
+
+    LOG_FILE: Optional[str] = Field(
+        description='logging output file path',
+        default=None,
+    )
+
+    LOG_FORMAT: str = Field(
+        description='log format',
+        default='%(asctime)s.%(msecs)03d %(levelname)s [%(threadName)s] [%(filename)s:%(lineno)d] - %(message)s',
+    )
+
+    LOG_DATEFORMAT: Optional[str] = Field(
+        description='log date format',
+        default=None,
+    )
+
+    LOG_TZ: Optional[str] = Field(
+        description='specify log timezone, eg: America/New_York',
+        default=None,
+    )
+
+
+class ModelLoadBalanceConfig(BaseSettings):
+    """
+    Model load balance configs
+    """
+    MODEL_LB_ENABLED: bool = Field(
+        description='whether to enable model load balancing',
+        default=False,
+    )
+
+
+class BillingConfig(BaseSettings):
+    """
+    Platform Billing Configurations
+    """
+    BILLING_ENABLED: bool = Field(
+        description='whether to enable billing',
+        default=False,
+    )
+
+
+class UpdateConfig(BaseSettings):
+    """
+    Update configs
+    """
+    CHECK_UPDATE_URL: str = Field(
+        description='url for checking updates',
+        default='https://updates.dify.ai',
+    )
+
+
+class WorkflowConfig(BaseSettings):
+    """
+    Workflow feature configs
+    """
+
+    WORKFLOW_MAX_EXECUTION_STEPS: PositiveInt = Field(
+        description='max execution steps in single workflow execution',
+        default=500,
+    )
+
+    WORKFLOW_MAX_EXECUTION_TIME: PositiveInt = Field(
+        description='max execution time in seconds in single workflow execution',
+        default=1200,
+    )
+
+    WORKFLOW_CALL_MAX_DEPTH: PositiveInt = Field(
+        description='max depth of calling in single workflow execution',
+        default=5,
+    )
+
+
+class OAuthConfig(BaseSettings):
+    """
+    oauth configs
+    """
+    OAUTH_REDIRECT_PATH: str = Field(
+        description='redirect path for OAuth',
+        default='/console/api/oauth/authorize',
+    )
+
+    GITHUB_CLIENT_ID: Optional[str] = Field(
+        description='GitHub client id for OAuth',
+        default=None,
+    )
+
+    GITHUB_CLIENT_SECRET: Optional[str] = Field(
+        description='GitHub client secret key for OAuth',
+        default=None,
+    )
+
+    GOOGLE_CLIENT_ID: Optional[str] = Field(
+        description='Google client id for OAuth',
+        default=None,
+    )
+
+    GOOGLE_CLIENT_SECRET: Optional[str] = Field(
+        description='Google client secret key for OAuth',
+        default=None,
+    )
+
+
+class ModerationConfig(BaseSettings):
+    """
+    Moderation in app configs.
+    """
+
+    # todo: to be clarified in usage and unit
+    OUTPUT_MODERATION_BUFFER_SIZE: PositiveInt = Field(
+        description='buffer size for moderation',
+        default=300,
+    )
+
+
+class ToolConfig(BaseSettings):
+    """
+    Tool configs
+    """
+
+    TOOL_ICON_CACHE_MAX_AGE: PositiveInt = Field(
+        description='max age in seconds for tool icon caching',
+        default=3600,
+    )
+
+
+class MailConfig(BaseSettings):
+    """
+    Mail Configurations
+    """
+
+    MAIL_TYPE: Optional[str] = Field(
+        description='Mail provider type name, default to None, availabile values are `smtp` and `resend`.',
+        default=None,
+    )
+
+    MAIL_DEFAULT_SEND_FROM: Optional[str] = Field(
+        description='default email address for sending from ',
+        default=None,
+    )
+
+    RESEND_API_KEY: Optional[str] = Field(
+        description='API key for Resend',
+        default=None,
+    )
+
+    RESEND_API_URL: Optional[str] = Field(
+        description='API URL for Resend',
+        default=None,
+    )
+
+    SMTP_SERVER: Optional[str] = Field(
+        description='smtp server host',
+        default=None,
+    )
+
+    SMTP_PORT: Optional[int] = Field(
+        description='smtp server port',
+        default=465,
+    )
+
+    SMTP_USERNAME: Optional[str] = Field(
+        description='smtp server username',
+        default=None,
+    )
+
+    SMTP_PASSWORD: Optional[str] = Field(
+        description='smtp server password',
+        default=None,
+    )
+
+    SMTP_USE_TLS: bool = Field(
+        description='whether to use TLS connection to smtp server',
+        default=False,
+    )
+
+    SMTP_OPPORTUNISTIC_TLS: bool = Field(
+        description='whether to use opportunistic TLS connection to smtp server',
+        default=False,
+    )
+
+
+class RagEtlConfig(BaseSettings):
+    """
+    RAG ETL Configurations.
+    """
+
+    ETL_TYPE: str = Field(
+        description='RAG ETL type name, default to `dify`, available values are `dify` and `Unstructured`. ',
+        default='dify',
+    )
+
+    KEYWORD_DATA_SOURCE_TYPE: str = Field(
+        description='source type for keyword data, default to `database`, available values are `database` .',
+        default='database',
+    )
+
+    UNSTRUCTURED_API_URL: Optional[str] = Field(
+        description='API URL for Unstructured',
+        default=None,
+    )
+
+    UNSTRUCTURED_API_KEY: Optional[str] = Field(
+        description='API key for Unstructured',
+        default=None,
+    )
+
+
+class DataSetConfig(BaseSettings):
+    """
+    Dataset configs
+    """
+
+    CLEAN_DAY_SETTING: PositiveInt = Field(
+        description='interval in days for cleaning up dataset',
+        default=30,
+    )
+
+    DATASET_OPERATOR_ENABLED: bool = Field(
+        description='whether to enable dataset operator',
+        default=False,
+    )
+
+
+class WorkspaceConfig(BaseSettings):
+    """
+    Workspace configs
+    """
+
+    INVITE_EXPIRY_HOURS: PositiveInt = Field(
+        description='workspaces invitation expiration in hours',
+        default=72,
+    )
+
+
+class IndexingConfig(BaseSettings):
+    """
+    Indexing configs.
+    """
+
+    INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH: PositiveInt = Field(
+        description='max segmentation token length for indexing',
+        default=1000,
+    )
+
+
+class ImageFormatConfig(BaseSettings):
+    MULTIMODAL_SEND_IMAGE_FORMAT: str = Field(
+        description='multi model send image format, support base64, url, default is base64',
+        default='base64',
+    )
+
+
+class FeatureConfig(
+    # place the configs in alphabet order
+    AppExecutionConfig,
+    BillingConfig,
+    CodeExecutionSandboxConfig,
+    DataSetConfig,
+    EndpointConfig,
+    FileAccessConfig,
+    FileUploadConfig,
+    HttpConfig,
+    ImageFormatConfig,
+    InnerAPIConfig,
+    IndexingConfig,
+    LoggingConfig,
+    MailConfig,
+    ModelLoadBalanceConfig,
+    ModerationConfig,
+    OAuthConfig,
+    RagEtlConfig,
+    SecurityConfig,
+    ToolConfig,
+    UpdateConfig,
+    WorkflowConfig,
+    WorkspaceConfig,
+
+    # hosted services config
+    HostedServiceConfig,
+):
+    pass

api/configs/feature/hosted_service/__init__.py

@@ -0,0 +1,209 @@
+from typing import Optional
+
+from pydantic import Field, NonNegativeInt
+from pydantic_settings import BaseSettings
+
+
+class HostedOpenAiConfig(BaseSettings):
+    """
+    Hosted OpenAI service config
+    """
+
+    HOSTED_OPENAI_API_KEY: Optional[str] = Field(
+        description='',
+        default=None,
+    )
+
+    HOSTED_OPENAI_API_BASE: Optional[str] = Field(
+        description='',
+        default=None,
+    )
+
+    HOSTED_OPENAI_API_ORGANIZATION: Optional[str] = Field(
+        description='',
+        default=None,
+    )
+
+    HOSTED_OPENAI_TRIAL_ENABLED: bool = Field(
+        description='',
+        default=False,
+    )
+
+    HOSTED_OPENAI_TRIAL_MODELS: str = Field(
+        description='',
+        default='gpt-3.5-turbo,'
+                'gpt-3.5-turbo-1106,'
+                'gpt-3.5-turbo-instruct,'
+                'gpt-3.5-turbo-16k,'
+                'gpt-3.5-turbo-16k-0613,'
+                'gpt-3.5-turbo-0613,'
+                'gpt-3.5-turbo-0125,'
+                'text-davinci-003',
+    )
+
+    HOSTED_OPENAI_QUOTA_LIMIT: NonNegativeInt = Field(
+        description='',
+        default=200,
+    )
+
+    HOSTED_OPENAI_PAID_ENABLED: bool = Field(
+        description='',
+        default=False,
+    )
+
+    HOSTED_OPENAI_PAID_MODELS: str = Field(
+        description='',
+        default='gpt-4,'
+                'gpt-4-turbo-preview,'
+                'gpt-4-turbo-2024-04-09,'
+                'gpt-4-1106-preview,'
+                'gpt-4-0125-preview,'
+                'gpt-3.5-turbo,'
+                'gpt-3.5-turbo-16k,'
+                'gpt-3.5-turbo-16k-0613,'
+                'gpt-3.5-turbo-1106,'
+                'gpt-3.5-turbo-0613,'
+                'gpt-3.5-turbo-0125,'
+                'gpt-3.5-turbo-instruct,'
+                'text-davinci-003',
+    )
+
+
+class HostedAzureOpenAiConfig(BaseSettings):
+    """
+    Hosted OpenAI service config
+    """
+
+    HOSTED_AZURE_OPENAI_ENABLED: bool = Field(
+        description='',
+        default=False,
+    )
+
+    HOSTED_OPENAI_API_KEY: Optional[str] = Field(
+        description='',
+        default=None,
+    )
+
+    HOSTED_AZURE_OPENAI_API_BASE: Optional[str] = Field(
+        description='',
+        default=None,
+    )
+
+    HOSTED_AZURE_OPENAI_QUOTA_LIMIT: NonNegativeInt = Field(
+        description='',
+        default=200,
+    )
+
+
+class HostedAnthropicConfig(BaseSettings):
+    """
+    Hosted Azure OpenAI service config
+    """
+
+    HOSTED_ANTHROPIC_API_BASE: Optional[str] = Field(
+        description='',
+        default=None,
+    )
+
+    HOSTED_ANTHROPIC_API_KEY: Optional[str] = Field(
+        description='',
+        default=None,
+    )
+
+    HOSTED_ANTHROPIC_TRIAL_ENABLED: bool = Field(
+        description='',
+        default=False,
+    )
+
+    HOSTED_ANTHROPIC_QUOTA_LIMIT: NonNegativeInt = Field(
+        description='',
+        default=600000,
+    )
+
+    HOSTED_ANTHROPIC_PAID_ENABLED: bool = Field(
+        description='',
+        default=False,
+    )
+
+
+class HostedMinmaxConfig(BaseSettings):
+    """
+    Hosted Minmax service config
+    """
+
+    HOSTED_MINIMAX_ENABLED: bool = Field(
+        description='',
+        default=False,
+    )
+
+
+class HostedSparkConfig(BaseSettings):
+    """
+    Hosted Spark service config
+    """
+
+    HOSTED_SPARK_ENABLED: bool = Field(
+        description='',
+        default=False,
+    )
+
+
+class HostedZhipuAIConfig(BaseSettings):
+    """
+    Hosted Minmax service config
+    """
+
+    HOSTED_ZHIPUAI_ENABLED: bool = Field(
+        description='',
+        default=False,
+    )
+
+
+class HostedModerationConfig(BaseSettings):
+    """
+    Hosted Moderation service config
+    """
+
+    HOSTED_MODERATION_ENABLED: bool = Field(
+        description='',
+        default=False,
+    )
+
+    HOSTED_MODERATION_PROVIDERS: str = Field(
+        description='',
+        default='',
+    )
+
+
+class HostedFetchAppTemplateConfig(BaseSettings):
+    """
+    Hosted Moderation service config
+    """
+
+    HOSTED_FETCH_APP_TEMPLATES_MODE: str = Field(
+        description='the mode for fetching app templates,'
+                    ' default to remote,'
+                    ' available values: remote, db, builtin',
+        default='remote',
+    )
+
+    HOSTED_FETCH_APP_TEMPLATES_REMOTE_DOMAIN: str = Field(
+        description='the domain for fetching remote app templates',
+        default='https://tmpl.dify.ai',
+    )
+
+
+class HostedServiceConfig(
+    # place the configs in alphabet order
+    HostedAnthropicConfig,
+    HostedAzureOpenAiConfig,
+    HostedFetchAppTemplateConfig,
+    HostedMinmaxConfig,
+    HostedOpenAiConfig,
+    HostedSparkConfig,
+    HostedZhipuAIConfig,
+
+    # moderation
+    HostedModerationConfig,
+):
+    pass

api/configs/middleware/__init__.py

@@ -0,0 +1,202 @@
+from typing import Any, Optional
+
+from pydantic import Field, NonNegativeInt, PositiveInt, computed_field
+from pydantic_settings import BaseSettings
+
+from configs.middleware.cache.redis_config import RedisConfig
+from configs.middleware.storage.aliyun_oss_storage_config import AliyunOSSStorageConfig
+from configs.middleware.storage.amazon_s3_storage_config import S3StorageConfig
+from configs.middleware.storage.azure_blob_storage_config import AzureBlobStorageConfig
+from configs.middleware.storage.google_cloud_storage_config import GoogleCloudStorageConfig
+from configs.middleware.storage.oci_storage_config import OCIStorageConfig
+from configs.middleware.storage.tencent_cos_storage_config import TencentCloudCOSStorageConfig
+from configs.middleware.vdb.analyticdb_config import AnalyticdbConfig
+from configs.middleware.vdb.chroma_config import ChromaConfig
+from configs.middleware.vdb.milvus_config import MilvusConfig
+from configs.middleware.vdb.myscale_config import MyScaleConfig
+from configs.middleware.vdb.opensearch_config import OpenSearchConfig
+from configs.middleware.vdb.oracle_config import OracleConfig
+from configs.middleware.vdb.pgvector_config import PGVectorConfig
+from configs.middleware.vdb.pgvectors_config import PGVectoRSConfig
+from configs.middleware.vdb.qdrant_config import QdrantConfig
+from configs.middleware.vdb.relyt_config import RelytConfig
+from configs.middleware.vdb.tencent_vector_config import TencentVectorDBConfig
+from configs.middleware.vdb.tidb_vector_config import TiDBVectorConfig
+from configs.middleware.vdb.weaviate_config import WeaviateConfig
+
+
+class StorageConfig(BaseSettings):
+    STORAGE_TYPE: str = Field(
+        description='storage type,'
+                    ' default to `local`,'
+                    ' available values are `local`, `s3`, `azure-blob`, `aliyun-oss`, `google-storage`.',
+        default='local',
+    )
+
+    STORAGE_LOCAL_PATH: str = Field(
+        description='local storage path',
+        default='storage',
+    )
+
+
+class VectorStoreConfig(BaseSettings):
+    VECTOR_STORE: Optional[str] = Field(
+        description='vector store type',
+        default=None,
+    )
+
+
+class KeywordStoreConfig(BaseSettings):
+    KEYWORD_STORE: str = Field(
+        description='keyword store type',
+        default='jieba',
+    )
+
+
+class DatabaseConfig:
+    DB_HOST: str = Field(
+        description='db host',
+        default='localhost',
+    )
+
+    DB_PORT: PositiveInt = Field(
+        description='db port',
+        default=5432,
+    )
+
+    DB_USERNAME: str = Field(
+        description='db username',
+        default='postgres',
+    )
+
+    DB_PASSWORD: str = Field(
+        description='db password',
+        default='',
+    )
+
+    DB_DATABASE: str = Field(
+        description='db database',
+        default='dify',
+    )
+
+    DB_CHARSET: str = Field(
+        description='db charset',
+        default='',
+    )
+
+    DB_EXTRAS: str = Field(
+        description='db extras options. Example: keepalives_idle=60&keepalives=1',
+        default='',
+    )
+
+    SQLALCHEMY_DATABASE_URI_SCHEME: str = Field(
+        description='db uri scheme',
+        default='postgresql',
+    )
+
+    @computed_field
+    @property
+    def SQLALCHEMY_DATABASE_URI(self) -> str:
+        db_extras = (
+            f"{self.DB_EXTRAS}&client_encoding={self.DB_CHARSET}"
+            if self.DB_CHARSET
+            else self.DB_EXTRAS
+        ).strip("&")
+        db_extras = f"?{db_extras}" if db_extras else ""
+        return (f"{self.SQLALCHEMY_DATABASE_URI_SCHEME}://"
+                f"{self.DB_USERNAME}:{self.DB_PASSWORD}@{self.DB_HOST}:{self.DB_PORT}/{self.DB_DATABASE}"
+                f"{db_extras}")
+
+    SQLALCHEMY_POOL_SIZE: NonNegativeInt = Field(
+        description='pool size of SqlAlchemy',
+        default=30,
+    )
+
+    SQLALCHEMY_MAX_OVERFLOW: NonNegativeInt = Field(
+        description='max overflows for SqlAlchemy',
+        default=10,
+    )
+
+    SQLALCHEMY_POOL_RECYCLE: NonNegativeInt = Field(
+        description='SqlAlchemy pool recycle',
+        default=3600,
+    )
+
+    SQLALCHEMY_POOL_PRE_PING: bool = Field(
+        description='whether to enable pool pre-ping in SqlAlchemy',
+        default=False,
+    )
+
+    SQLALCHEMY_ECHO: bool | str = Field(
+        description='whether to enable SqlAlchemy echo',
+        default=False,
+    )
+
+    @computed_field
+    @property
+    def SQLALCHEMY_ENGINE_OPTIONS(self) -> dict[str, Any]:
+        return {
+            'pool_size': self.SQLALCHEMY_POOL_SIZE,
+            'max_overflow': self.SQLALCHEMY_MAX_OVERFLOW,
+            'pool_recycle': self.SQLALCHEMY_POOL_RECYCLE,
+            'pool_pre_ping': self.SQLALCHEMY_POOL_PRE_PING,
+            'connect_args': {'options': '-c timezone=UTC'},
+        }
+
+
+class CeleryConfig(DatabaseConfig):
+    CELERY_BACKEND: str = Field(
+        description='Celery backend, available values are `database`, `redis`',
+        default='database',
+    )
+
+    CELERY_BROKER_URL: Optional[str] = Field(
+        description='CELERY_BROKER_URL',
+        default=None,
+    )
+
+    @computed_field
+    @property
+    def CELERY_RESULT_BACKEND(self) -> str | None:
+        return 'db+{}'.format(self.SQLALCHEMY_DATABASE_URI) \
+            if self.CELERY_BACKEND == 'database' else self.CELERY_BROKER_URL
+
+    @computed_field
+    @property
+    def BROKER_USE_SSL(self) -> bool:
+        return self.CELERY_BROKER_URL.startswith('rediss://') if self.CELERY_BROKER_URL else False
+
+
+class MiddlewareConfig(
+    # place the configs in alphabet order
+    CeleryConfig,
+    DatabaseConfig,
+    KeywordStoreConfig,
+    RedisConfig,
+
+    # configs of storage and storage providers
+    StorageConfig,
+    AliyunOSSStorageConfig,
+    AzureBlobStorageConfig,
+    GoogleCloudStorageConfig,
+    TencentCloudCOSStorageConfig,
+    S3StorageConfig,
+    OCIStorageConfig,
+
+    # configs of vdb and vdb providers
+    VectorStoreConfig,
+    AnalyticdbConfig,
+    ChromaConfig,
+    MilvusConfig,
+    MyScaleConfig,
+    OpenSearchConfig,
+    OracleConfig,
+    PGVectorConfig,
+    PGVectoRSConfig,
+    QdrantConfig,
+    RelytConfig,
+    TencentVectorDBConfig,
+    TiDBVectorConfig,
+    WeaviateConfig,
+):
+    pass

api/configs/middleware/cache/redis_config.py

@@ -0,0 +1,39 @@
+from typing import Optional
+
+from pydantic import Field, NonNegativeInt, PositiveInt
+from pydantic_settings import BaseSettings
+
+
+class RedisConfig(BaseSettings):
+    """
+    Redis configs
+    """
+    REDIS_HOST: str = Field(
+        description='Redis host',
+        default='localhost',
+    )
+
+    REDIS_PORT: PositiveInt = Field(
+        description='Redis port',
+        default=6379,
+    )
+
+    REDIS_USERNAME: Optional[str] = Field(
+        description='Redis username',
+        default=None,
+    )
+
+    REDIS_PASSWORD: Optional[str] = Field(
+        description='Redis password',
+        default=None,
+    )
+
+    REDIS_DB: NonNegativeInt = Field(
+        description='Redis database id, default to 0',
+        default=0,
+    )
+
+    REDIS_USE_SSL: bool = Field(
+        description='whether to use SSL for Redis connection',
+        default=False,
+    )

api/configs/middleware/storage/aliyun_oss_storage_config.py

@@ -0,0 +1,40 @@
+from typing import Optional
+
+from pydantic import Field
+from pydantic_settings import BaseSettings
+
+
+class AliyunOSSStorageConfig(BaseSettings):
+    """
+    Aliyun storage configs
+    """
+
+    ALIYUN_OSS_BUCKET_NAME: Optional[str] = Field(
+        description='Aliyun OSS bucket name',
+        default=None,
+    )
+
+    ALIYUN_OSS_ACCESS_KEY: Optional[str] = Field(
+        description='Aliyun OSS access key',
+        default=None,
+    )
+
+    ALIYUN_OSS_SECRET_KEY: Optional[str] = Field(
+        description='Aliyun OSS secret key',
+        default=None,
+    )
+
+    ALIYUN_OSS_ENDPOINT: Optional[str] = Field(
+        description='Aliyun OSS endpoint URL',
+        default=None,
+    )
+
+    ALIYUN_OSS_REGION: Optional[str] = Field(
+        description='Aliyun OSS region',
+        default=None,
+    )
+
+    ALIYUN_OSS_AUTH_VERSION: Optional[str] = Field(
+        description='Aliyun OSS authentication version',
+        default=None,
+    )

api/configs/middleware/storage/amazon_s3_storage_config.py

@@ -0,0 +1,45 @@
+from typing import Optional
+
+from pydantic import Field
+from pydantic_settings import BaseSettings
+
+
+class S3StorageConfig(BaseSettings):
+    """
+    S3 storage configs
+    """
+
+    S3_ENDPOINT: Optional[str] = Field(
+        description='S3 storage endpoint',
+        default=None,
+    )
+
+    S3_REGION: Optional[str] = Field(
+        description='S3 storage region',
+        default=None,
+    )
+
+    S3_BUCKET_NAME: Optional[str] = Field(
+        description='S3 storage bucket name',
+        default=None,
+    )
+
+    S3_ACCESS_KEY: Optional[str] = Field(
+        description='S3 storage access key',
+        default=None,
+    )
+
+    S3_SECRET_KEY: Optional[str] = Field(
+        description='S3 storage secret key',
+        default=None,
+    )
+
+    S3_ADDRESS_STYLE: str = Field(
+        description='S3 storage address style',
+        default='auto',
+    )
+
+    S3_USE_AWS_MANAGED_IAM: bool = Field(
+        description='whether to use aws managed IAM for S3',
+        default=False,
+    )

api/configs/middleware/storage/azure_blob_storage_config.py

@@ -0,0 +1,30 @@
+from typing import Optional
+
+from pydantic import Field
+from pydantic_settings import BaseSettings
+
+
+class AzureBlobStorageConfig(BaseSettings):
+    """
+    Azure Blob storage configs
+    """
+
+    AZURE_BLOB_ACCOUNT_NAME: Optional[str] = Field(
+        description='Azure Blob account name',
+        default=None,
+    )
+
+    AZURE_BLOB_ACCOUNT_KEY: Optional[str] = Field(
+        description='Azure Blob account key',
+        default=None,
+    )
+
+    AZURE_BLOB_CONTAINER_NAME: Optional[str] = Field(
+        description='Azure Blob container name',
+        default=None,
+    )
+
+    AZURE_BLOB_ACCOUNT_URL: Optional[str] = Field(
+        description='Azure Blob account URL',
+        default=None,
+    )

api/configs/middleware/storage/google_cloud_storage_config.py

@@ -0,0 +1,20 @@
+from typing import Optional
+
+from pydantic import Field
+from pydantic_settings import BaseSettings
+
+
+class GoogleCloudStorageConfig(BaseSettings):
+    """
+    Google Cloud storage configs
+    """
+
+    GOOGLE_STORAGE_BUCKET_NAME: Optional[str] = Field(
+        description='Google Cloud storage bucket name',
+        default=None,
+    )
+
+    GOOGLE_STORAGE_SERVICE_ACCOUNT_JSON_BASE64: Optional[str] = Field(
+        description='Google Cloud storage service account json base64',
+        default=None,
+    )

api/configs/middleware/storage/oci_storage_config.py

@@ -0,0 +1,36 @@
+from typing import Optional
+
+from pydantic import Field
+from pydantic_settings import BaseSettings
+
+
+class OCIStorageConfig(BaseSettings):
+    """
+    OCI storage configs
+    """
+
+    OCI_ENDPOINT: Optional[str] = Field(
+        description='OCI storage endpoint',
+        default=None,
+    )
+
+    OCI_REGION: Optional[str] = Field(
+        description='OCI storage region',
+        default=None,
+    )
+
+    OCI_BUCKET_NAME: Optional[str] = Field(
+        description='OCI storage bucket name',
+        default=None,
+    )
+
+    OCI_ACCESS_KEY: Optional[str] = Field(
+        description='OCI storage access key',
+        default=None,
+    )
+
+    OCI_SECRET_KEY: Optional[str] = Field(
+        description='OCI storage secret key',
+        default=None,
+    )
+

api/configs/middleware/storage/tencent_cos_storage_config.py

@@ -0,0 +1,35 @@
+from typing import Optional
+
+from pydantic import Field
+from pydantic_settings import BaseSettings
+
+
+class TencentCloudCOSStorageConfig(BaseSettings):
+    """
+    Tencent Cloud COS storage configs
+    """
+
+    TENCENT_COS_BUCKET_NAME: Optional[str] = Field(
+        description='Tencent Cloud COS bucket name',
+        default=None,
+    )
+
+    TENCENT_COS_REGION: Optional[str] = Field(
+        description='Tencent Cloud COS region',
+        default=None,
+    )
+
+    TENCENT_COS_SECRET_ID: Optional[str] = Field(
+        description='Tencent Cloud COS secret id',
+        default=None,
+    )
+
+    TENCENT_COS_SECRET_KEY: Optional[str] = Field(
+        description='Tencent Cloud COS secret key',
+        default=None,
+    )
+
+    TENCENT_COS_SCHEME: Optional[str] = Field(
+        description='Tencent Cloud COS scheme',
+        default=None,
+    )

api/configs/middleware/vdb/analyticdb_config.py

@@ -0,0 +1,44 @@
+from typing import Optional
+
+from pydantic import BaseModel, Field
+
+
+class AnalyticdbConfig(BaseModel):
+    """
+    Configuration for connecting to AnalyticDB.
+    Refer to the following documentation for details on obtaining credentials:
+    https://www.alibabacloud.com/help/en/analyticdb-for-postgresql/getting-started/create-an-instance-instances-with-vector-engine-optimization-enabled
+    """
+
+    ANALYTICDB_KEY_ID : Optional[str] = Field(
+        default=None,
+        description="The Access Key ID provided by Alibaba Cloud for authentication."
+    )
+    ANALYTICDB_KEY_SECRET : Optional[str] = Field(
+        default=None,
+        description="The Secret Access Key corresponding to the Access Key ID for secure access."
+    )
+    ANALYTICDB_REGION_ID : Optional[str] = Field(
+        default=None,
+        description="The region where the AnalyticDB instance is deployed (e.g., 'cn-hangzhou')."
+    )
+    ANALYTICDB_INSTANCE_ID : Optional[str] = Field(
+        default=None,
+        description="The unique identifier of the AnalyticDB instance you want to connect to (e.g., 'gp-ab123456').."
+    )
+    ANALYTICDB_ACCOUNT : Optional[str] = Field(
+        default=None,
+        description="The account name used to log in to the AnalyticDB instance."
+    )
+    ANALYTICDB_PASSWORD : Optional[str] = Field(
+        default=None,
+        description="The password associated with the AnalyticDB account for authentication."
+    )
+    ANALYTICDB_NAMESPACE : Optional[str] = Field(
+        default=None,
+        description="The namespace within AnalyticDB for schema isolation."
+    )
+    ANALYTICDB_NAMESPACE_PASSWORD : Optional[str] = Field(
+        default=None,
+        description="The password for accessing the specified namespace within the AnalyticDB instance."
+    )

api/configs/middleware/vdb/chroma_config.py

@@ -0,0 +1,40 @@
+from typing import Optional
+
+from pydantic import Field, PositiveInt
+from pydantic_settings import BaseSettings
+
+
+class ChromaConfig(BaseSettings):
+    """
+    Chroma configs
+    """
+
+    CHROMA_HOST: Optional[str] = Field(
+        description='Chroma host',
+        default=None,
+    )
+
+    CHROMA_PORT: PositiveInt = Field(
+        description='Chroma port',
+        default=8000,
+    )
+
+    CHROMA_TENANT: Optional[str] = Field(
+        description='Chroma database',
+        default=None,
+    )
+
+    CHROMA_DATABASE: Optional[str] = Field(
+        description='Chroma database',
+        default=None,
+    )
+
+    CHROMA_AUTH_PROVIDER: Optional[str] = Field(
+        description='Chroma authentication provider',
+        default=None,
+    )
+
+    CHROMA_AUTH_CREDENTIALS: Optional[str] = Field(
+        description='Chroma authentication credentials',
+        default=None,
+    )

api/configs/middleware/vdb/milvus_config.py

@@ -0,0 +1,40 @@
+from typing import Optional
+
+from pydantic import Field, PositiveInt
+from pydantic_settings import BaseSettings
+
+
+class MilvusConfig(BaseSettings):
+    """
+    Milvus configs
+    """
+
+    MILVUS_HOST: Optional[str] = Field(
+        description='Milvus host',
+        default=None,
+    )
+
+    MILVUS_PORT: PositiveInt = Field(
+        description='Milvus RestFul API port',
+        default=9091,
+    )
+
+    MILVUS_USER: Optional[str] = Field(
+        description='Milvus user',
+        default=None,
+    )
+
+    MILVUS_PASSWORD: Optional[str] = Field(
+        description='Milvus password',
+        default=None,
+    )
+
+    MILVUS_SECURE: bool = Field(
+        description='whether to use SSL connection for Milvus',
+        default=False,
+    )
+
+    MILVUS_DATABASE: str = Field(
+        description='Milvus database, default to `default`',
+        default='default',
+    )

api/configs/middleware/vdb/myscale_config.py

@@ -0,0 +1,39 @@
+from typing import Optional
+
+from pydantic import BaseModel, Field, PositiveInt
+
+
+class MyScaleConfig(BaseModel):
+    """
+    MyScale configs
+    """
+
+    MYSCALE_HOST: Optional[str] = Field(
+        description='MyScale host',
+        default=None,
+    )
+
+    MYSCALE_PORT: Optional[PositiveInt] = Field(
+        description='MyScale port',
+        default=8123,
+    )
+
+    MYSCALE_USER: Optional[str] = Field(
+        description='MyScale user',
+        default=None,
+    )
+
+    MYSCALE_PASSWORD: Optional[str] = Field(
+        description='MyScale password',
+        default=None,
+    )
+
+    MYSCALE_DATABASE: Optional[str] = Field(
+        description='MyScale database name',
+        default=None,
+    )
+
+    MYSCALE_FTS_PARAMS: Optional[str] = Field(
+        description='MyScale fts index parameters',
+        default=None,
+    )

api/configs/middleware/vdb/opensearch_config.py

@@ -0,0 +1,35 @@
+from typing import Optional
+
+from pydantic import Field, PositiveInt
+from pydantic_settings import BaseSettings
+
+
+class OpenSearchConfig(BaseSettings):
+    """
+    OpenSearch configs
+    """
+
+    OPENSEARCH_HOST: Optional[str] = Field(
+        description='OpenSearch host',
+        default=None,
+    )
+
+    OPENSEARCH_PORT: PositiveInt = Field(
+        description='OpenSearch port',
+        default=9200,
+    )
+
+    OPENSEARCH_USER: Optional[str] = Field(
+        description='OpenSearch user',
+        default=None,
+    )
+
+    OPENSEARCH_PASSWORD: Optional[str] = Field(
+        description='OpenSearch password',
+        default=None,
+    )
+
+    OPENSEARCH_SECURE: bool = Field(
+        description='whether to use SSL connection for OpenSearch',
+        default=False,
+    )

api/configs/middleware/vdb/oracle_config.py

@@ -0,0 +1,35 @@
+from typing import Optional
+
+from pydantic import Field, PositiveInt
+from pydantic_settings import BaseSettings
+
+
+class OracleConfig(BaseSettings):
+    """
+    ORACLE configs
+    """
+
+    ORACLE_HOST: Optional[str] = Field(
+        description='ORACLE host',
+        default=None,
+    )
+
+    ORACLE_PORT: Optional[PositiveInt] = Field(
+        description='ORACLE port',
+        default=1521,
+    )
+
+    ORACLE_USER: Optional[str] = Field(
+        description='ORACLE user',
+        default=None,
+    )
+
+    ORACLE_PASSWORD: Optional[str] = Field(
+        description='ORACLE password',
+        default=None,
+    )
+
+    ORACLE_DATABASE: Optional[str] = Field(
+        description='ORACLE database',
+        default=None,
+    )

api/configs/middleware/vdb/pgvector_config.py

@@ -0,0 +1,35 @@
+from typing import Optional
+
+from pydantic import Field, PositiveInt
+from pydantic_settings import BaseSettings
+
+
+class PGVectorConfig(BaseSettings):
+    """
+    PGVector configs
+    """
+
+    PGVECTOR_HOST: Optional[str] = Field(
+        description='PGVector host',
+        default=None,
+    )
+
+    PGVECTOR_PORT: Optional[PositiveInt] = Field(
+        description='PGVector port',
+        default=5433,
+    )
+
+    PGVECTOR_USER: Optional[str] = Field(
+        description='PGVector user',
+        default=None,
+    )
+
+    PGVECTOR_PASSWORD: Optional[str] = Field(
+        description='PGVector password',
+        default=None,
+    )
+
+    PGVECTOR_DATABASE: Optional[str] = Field(
+        description='PGVector database',
+        default=None,
+    )

api/configs/middleware/vdb/pgvectors_config.py

@@ -0,0 +1,35 @@
+from typing import Optional
+
+from pydantic import Field, PositiveInt
+from pydantic_settings import BaseSettings
+
+
+class PGVectoRSConfig(BaseSettings):
+    """
+    PGVectoRS configs
+    """
+
+    PGVECTO_RS_HOST: Optional[str] = Field(
+        description='PGVectoRS host',
+        default=None,
+    )
+
+    PGVECTO_RS_PORT: Optional[PositiveInt] = Field(
+        description='PGVectoRS port',
+        default=5431,
+    )
+
+    PGVECTO_RS_USER: Optional[str] = Field(
+        description='PGVectoRS user',
+        default=None,
+    )
+
+    PGVECTO_RS_PASSWORD: Optional[str] = Field(
+        description='PGVectoRS password',
+        default=None,
+    )
+
+    PGVECTO_RS_DATABASE: Optional[str] = Field(
+        description='PGVectoRS database',
+        default=None,
+    )

api/configs/middleware/vdb/qdrant_config.py

@@ -0,0 +1,35 @@
+from typing import Optional
+
+from pydantic import Field, NonNegativeInt, PositiveInt
+from pydantic_settings import BaseSettings
+
+
+class QdrantConfig(BaseSettings):
+    """
+    Qdrant configs
+    """
+
+    QDRANT_URL: Optional[str] = Field(
+        description='Qdrant url',
+        default=None,
+    )
+
+    QDRANT_API_KEY: Optional[str] = Field(
+        description='Qdrant api key',
+        default=None,
+    )
+
+    QDRANT_CLIENT_TIMEOUT: NonNegativeInt = Field(
+        description='Qdrant client timeout in seconds',
+        default=20,
+    )
+
+    QDRANT_GRPC_ENABLED: bool = Field(
+        description='whether enable grpc support for Qdrant connection',
+        default=False,
+    )
+
+    QDRANT_GRPC_PORT: PositiveInt = Field(
+        description='Qdrant grpc port',
+        default=6334,
+    )

api/configs/middleware/vdb/relyt_config.py

@@ -0,0 +1,35 @@
+from typing import Optional
+
+from pydantic import Field, PositiveInt
+from pydantic_settings import BaseSettings
+
+
+class RelytConfig(BaseSettings):
+    """
+    Relyt configs
+    """
+
+    RELYT_HOST: Optional[str] = Field(
+        description='Relyt host',
+        default=None,
+    )
+
+    RELYT_PORT: PositiveInt = Field(
+        description='Relyt port',
+        default=9200,
+    )
+
+    RELYT_USER: Optional[str] = Field(
+        description='Relyt user',
+        default=None,
+    )
+
+    RELYT_PASSWORD: Optional[str] = Field(
+        description='Relyt password',
+        default=None,
+    )
+
+    RELYT_DATABASE: Optional[str] = Field(
+        description='Relyt database',
+        default='default',
+    )

api/configs/middleware/vdb/tencent_vector_config.py

@@ -0,0 +1,50 @@
+from typing import Optional
+
+from pydantic import Field, NonNegativeInt, PositiveInt
+from pydantic_settings import BaseSettings
+
+
+class TencentVectorDBConfig(BaseSettings):
+    """
+    Tencent Vector configs
+    """
+
+    TENCENT_VECTOR_DB_URL: Optional[str] = Field(
+        description='Tencent Vector URL',
+        default=None,
+    )
+
+    TENCENT_VECTOR_DB_API_KEY: Optional[str] = Field(
+        description='Tencent Vector API key',
+        default=None,
+    )
+
+    TENCENT_VECTOR_DB_TIMEOUT: PositiveInt = Field(
+        description='Tencent Vector timeout in seconds',
+        default=30,
+    )
+
+    TENCENT_VECTOR_DB_USERNAME: Optional[str] = Field(
+        description='Tencent Vector username',
+        default=None,
+    )
+
+    TENCENT_VECTOR_DB_PASSWORD: Optional[str] = Field(
+        description='Tencent Vector password',
+        default=None,
+    )
+
+    TENCENT_VECTOR_DB_SHARD: PositiveInt = Field(
+        description='Tencent Vector sharding number',
+        default=1,
+    )
+
+    TENCENT_VECTOR_DB_REPLICAS: NonNegativeInt = Field(
+        description='Tencent Vector replicas',
+        default=2,
+    )
+
+    TENCENT_VECTOR_DB_DATABASE: Optional[str] = Field(
+        description='Tencent Vector Database',
+        default=None,
+    )

api/configs/middleware/vdb/tidb_vector_config.py

@@ -0,0 +1,35 @@
+from typing import Optional
+
+from pydantic import Field, PositiveInt
+from pydantic_settings import BaseSettings
+
+
+class TiDBVectorConfig(BaseSettings):
+    """
+    TiDB Vector configs
+    """
+
+    TIDB_VECTOR_HOST: Optional[str] = Field(
+        description='TiDB Vector host',
+        default=None,
+    )
+
+    TIDB_VECTOR_PORT: Optional[PositiveInt] = Field(
+        description='TiDB Vector port',
+        default=4000,
+    )
+
+    TIDB_VECTOR_USER: Optional[str] = Field(
+        description='TiDB Vector user',
+        default=None,
+    )
+
+    TIDB_VECTOR_PASSWORD: Optional[str] = Field(
+        description='TiDB Vector password',
+        default=None,
+    )
+
+    TIDB_VECTOR_DATABASE: Optional[str] = Field(
+        description='TiDB Vector database',
+        default=None,
+    )

api/configs/middleware/vdb/weaviate_config.py

@@ -0,0 +1,30 @@
+from typing import Optional
+
+from pydantic import Field, PositiveInt
+from pydantic_settings import BaseSettings
+
+
+class WeaviateConfig(BaseSettings):
+    """
+    Weaviate configs
+    """
+
+    WEAVIATE_ENDPOINT: Optional[str] = Field(
+        description='Weaviate endpoint URL',
+        default=None,
+    )
+
+    WEAVIATE_API_KEY: Optional[str] = Field(
+        description='Weaviate API key',
+        default=None,
+    )
+
+    WEAVIATE_GRPC_ENABLED: bool = Field(
+        description='whether to enable gRPC for Weaviate connection',
+        default=True,
+    )
+
+    WEAVIATE_BATCH_SIZE: PositiveInt = Field(
+        description='Weaviate batch size',
+        default=100,
+    )

api/configs/packaging/__init__.py

@@ -0,0 +1,18 @@
+from pydantic import Field
+from pydantic_settings import BaseSettings
+
+
+class PackagingInfo(BaseSettings):
+    """
+    Packaging build information
+    """
+
+    CURRENT_VERSION: str = Field(
+        description='Dify version',
+        default='0.6.14',
+    )
+
+    COMMIT_SHA: str = Field(
+        description="SHA-1 checksum of the git commit used to build the app",
+        default='',
+    )

api/constants/languages.py

@@ -1,7 +1,3 @@
-
-
-languages = ['en-US', 'zh-Hans', 'zh-Hant', 'pt-BR', 'es-ES', 'fr-FR', 'de-DE', 'ja-JP', 'ko-KR', 'ru-RU', 'it-IT', 'uk-UA', 'vi-VN', 'pl-PL', 'hi-IN']
-
 language_timezone_mapping = {
     'en-US': 'America/New_York',
     'zh-Hans': 'Asia/Shanghai',
@@ -18,9 +14,11 @@ language_timezone_mapping = {
     'vi-VN': 'Asia/Ho_Chi_Minh',
     'ro-RO': 'Europe/Bucharest',
     'pl-PL': 'Europe/Warsaw',
-    'hi-IN': 'Asia/Kolkata'
+    'hi-IN': 'Asia/Kolkata',
 }
 
+languages = list(language_timezone_mapping.keys())
+
 
 def supported_language(lang):
     if lang in languages:

api/constants/model_template.py

@@ -22,7 +22,7 @@ default_app_templates = {
         'model_config': {
             'model': {
                 "provider": "openai",
-                "name": "gpt-4",
+                "name": "gpt-4o",
                 "mode": "chat",
                 "completion_params": {}
             },
@@ -51,7 +51,7 @@ default_app_templates = {
         'model_config': {
             'model': {
                 "provider": "openai",
-                "name": "gpt-4",
+                "name": "gpt-4o",
                 "mode": "chat",
                 "completion_params": {}
             }
@@ -77,7 +77,7 @@ default_app_templates = {
         'model_config': {
             'model': {
                 "provider": "openai",
-                "name": "gpt-4",
+                "name": "gpt-4o",
                 "mode": "chat",
                 "completion_params": {}
             }

api/constants/tts_auto_play_timeout.py

@@ -0,0 +1,4 @@
+TTS_AUTO_PLAY_TIMEOUT = 5
+
+# sleep 20 ms ( 40ms => 1280 byte audio file,20ms => 640 byte audio file)
+TTS_AUTO_PLAY_YIELD_CPU_TIME = 0.02

api/controllers/console/__init__.py

@@ -20,6 +20,7 @@ from .app import (
     generator,
     message,
     model_config,
+    ops_trace,
     site,
     statistic,
     workflow,
@@ -29,7 +30,7 @@ from .app import (
 )
 
 # Import auth controllers
-from .auth import activate, data_source_bearer_auth, data_source_oauth, login, oauth
+from .auth import activate, data_source_bearer_auth, data_source_oauth, forgot_password, login, oauth
 
 # Import billing controllers
 from .billing import billing

api/controllers/console/app/app.py

@@ -1,4 +1,3 @@
-import json
 import uuid
 
 from flask_login import current_user
@@ -9,17 +8,15 @@ from controllers.console import api
 from controllers.console.app.wraps import get_app_model
 from controllers.console.setup import setup_required
 from controllers.console.wraps import account_initialization_required, cloud_edition_billing_resource_check
-from core.tools.tool_manager import ToolManager
-from core.tools.utils.configuration import ToolParameterConfigurationManager
+from core.ops.ops_trace_manager import OpsTraceManager
 from fields.app_fields import (
     app_detail_fields,
     app_detail_fields_with_site,
     app_pagination_fields,
 )
 from libs.login import login_required
-from models.model import App, AppMode, AppModelConfig
+from services.app_dsl_service import AppDslService
 from services.app_service import AppService
-from services.tag_service import TagService
 
 ALLOW_CREATE_APP_MODES = ['chat', 'agent-chat', 'advanced-chat', 'workflow', 'completion']
 
@@ -101,8 +98,42 @@ class AppImportApi(Resource):
         parser.add_argument('icon_background', type=str, location='json')
         args = parser.parse_args()
 
-        app_service = AppService()
-        app = app_service.import_app(current_user.current_tenant_id, args['data'], args, current_user)
+        app = AppDslService.import_and_create_new_app(
+            tenant_id=current_user.current_tenant_id,
+            data=args['data'],
+            args=args,
+            account=current_user
+        )
+
+        return app, 201
+
+
+class AppImportFromUrlApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @marshal_with(app_detail_fields_with_site)
+    @cloud_edition_billing_resource_check('apps')
+    def post(self):
+        """Import app from url"""
+        # The role of the current user in the ta table must be admin, owner, or editor
+        if not current_user.is_editor:
+            raise Forbidden()
+
+        parser = reqparse.RequestParser()
+        parser.add_argument('url', type=str, required=True, nullable=False, location='json')
+        parser.add_argument('name', type=str, location='json')
+        parser.add_argument('description', type=str, location='json')
+        parser.add_argument('icon', type=str, location='json')
+        parser.add_argument('icon_background', type=str, location='json')
+        args = parser.parse_args()
+
+        app = AppDslService.import_and_create_new_app_from_url(
+            tenant_id=current_user.current_tenant_id,
+            url=args['url'],
+            args=args,
+            account=current_user
+        )
 
         return app, 201
 
@@ -138,6 +169,7 @@ class AppApi(Resource):
         parser.add_argument('description', type=str, location='json')
         parser.add_argument('icon', type=str, location='json')
         parser.add_argument('icon_background', type=str, location='json')
+        parser.add_argument('max_active_requests', type=int, location='json')
         args = parser.parse_args()
 
         app_service = AppService()
@@ -180,9 +212,13 @@ class AppCopyApi(Resource):
         parser.add_argument('icon_background', type=str, location='json')
         args = parser.parse_args()
 
-        app_service = AppService()
-        data = app_service.export_app(app_model)
-        app = app_service.import_app(current_user.current_tenant_id, data, args, current_user)
+        data = AppDslService.export_dsl(app_model=app_model)
+        app = AppDslService.import_and_create_new_app(
+            tenant_id=current_user.current_tenant_id,
+            data=data,
+            args=args,
+            account=current_user
+        )
 
         return app, 201
 
@@ -194,10 +230,12 @@ class AppExportApi(Resource):
     @get_app_model
     def get(self, app_model):
         """Export app"""
-        app_service = AppService()
+        # The role of the current user in the ta table must be admin, owner, or editor
+        if not current_user.is_editor:
+            raise Forbidden()
 
         return {
-            "data": app_service.export_app(app_model)
+            "data": AppDslService.export_dsl(app_model=app_model)
         }
 
 
@@ -286,8 +324,42 @@ class AppApiStatus(Resource):
         return app_model
 
 
+class AppTraceApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def get(self, app_id):
+        """Get app trace"""
+        app_trace_config = OpsTraceManager.get_app_tracing_config(
+            app_id=app_id
+        )
+
+        return app_trace_config
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def post(self, app_id):
+        # add app trace
+        if not current_user.is_admin_or_owner:
+            raise Forbidden()
+        parser = reqparse.RequestParser()
+        parser.add_argument('enabled', type=bool, required=True, location='json')
+        parser.add_argument('tracing_provider', type=str, required=True, location='json')
+        args = parser.parse_args()
+
+        OpsTraceManager.update_app_tracing_config(
+            app_id=app_id,
+            enabled=args['enabled'],
+            tracing_provider=args['tracing_provider'],
+        )
+
+        return {"result": "success"}
+
+
 api.add_resource(AppListApi, '/apps')
 api.add_resource(AppImportApi, '/apps/import')
+api.add_resource(AppImportFromUrlApi, '/apps/import/url')
 api.add_resource(AppApi, '/apps/<uuid:app_id>')
 api.add_resource(AppCopyApi, '/apps/<uuid:app_id>/copy')
 api.add_resource(AppExportApi, '/apps/<uuid:app_id>/export')
@@ -295,3 +367,4 @@ api.add_resource(AppNameApi, '/apps/<uuid:app_id>/name')
 api.add_resource(AppIconApi, '/apps/<uuid:app_id>/icon')
 api.add_resource(AppSiteStatus, '/apps/<uuid:app_id>/site-enable')
 api.add_resource(AppApiStatus, '/apps/<uuid:app_id>/api-enable')
+api.add_resource(AppTraceApi, '/apps/<uuid:app_id>/trace')

api/controllers/console/app/audio.py

@@ -81,15 +81,36 @@ class ChatMessageTextApi(Resource):
     @account_initialization_required
     @get_app_model
     def post(self, app_model):
+        from werkzeug.exceptions import InternalServerError
+
         try:
+            parser = reqparse.RequestParser()
+            parser.add_argument('message_id', type=str, location='json')
+            parser.add_argument('text', type=str, location='json')
+            parser.add_argument('voice', type=str, location='json')
+            parser.add_argument('streaming', type=bool, location='json')
+            args = parser.parse_args()
+
+            message_id = args.get('message_id', None)
+            text = args.get('text', None)
+            if (app_model.mode in [AppMode.ADVANCED_CHAT.value, AppMode.WORKFLOW.value]
+                    and app_model.workflow
+                    and app_model.workflow.features_dict):
+                text_to_speech = app_model.workflow.features_dict.get('text_to_speech')
+                voice = args.get('voice') if args.get('voice') else text_to_speech.get('voice')
+            else:
+                try:
+                    voice = args.get('voice') if args.get('voice') else app_model.app_model_config.text_to_speech_dict.get(
+                        'voice')
+                except Exception:
+                    voice = None
             response = AudioService.transcript_tts(
                 app_model=app_model,
-                text=request.form['text'],
-                voice=request.form['voice'],
-                streaming=False
+                text=text,
+                message_id=message_id,
+                voice=voice
             )
-
-            return {'data': response.data.decode('latin1')}
+            return response
         except services.errors.app_model_config.AppModelConfigBrokenError:
             logging.exception("App model config broken.")
             raise AppUnavailableError()

api/controllers/console/app/completion.py

@@ -19,7 +19,12 @@ from controllers.console.setup import setup_required
 from controllers.console.wraps import account_initialization_required
 from core.app.apps.base_app_queue_manager import AppQueueManager
 from core.app.entities.app_invoke_entities import InvokeFrom
-from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError
+from core.errors.error import (
+    AppInvokeQuotaExceededError,
+    ModelCurrentlyNotSupportError,
+    ProviderTokenNotInitError,
+    QuotaExceededError,
+)
 from core.model_runtime.errors.invoke import InvokeError
 from libs import helper
 from libs.helper import uuid_value
@@ -75,7 +80,7 @@ class CompletionMessageApi(Resource):
             raise ProviderModelCurrentlyNotSupportError()
         except InvokeError as e:
             raise CompletionRequestError(e.description)
-        except ValueError as e:
+        except (ValueError, AppInvokeQuotaExceededError) as e:
             raise e
         except Exception as e:
             logging.exception("internal server error.")
@@ -141,7 +146,7 @@ class ChatMessageApi(Resource):
             raise ProviderModelCurrentlyNotSupportError()
         except InvokeError as e:
             raise CompletionRequestError(e.description)
-        except ValueError as e:
+        except (ValueError, AppInvokeQuotaExceededError) as e:
             raise e
         except Exception as e:
             logging.exception("internal server error.")

api/controllers/console/app/error.py

@@ -97,3 +97,21 @@ class DraftWorkflowNotSync(BaseHTTPException):
     error_code = 'draft_workflow_not_sync'
     description = "Workflow graph might have been modified, please refresh and resubmit."
     code = 400
+
+
+class TracingConfigNotExist(BaseHTTPException):
+    error_code = 'trace_config_not_exist'
+    description = "Trace config not exist."
+    code = 400
+
+
+class TracingConfigIsExist(BaseHTTPException):
+    error_code = 'trace_config_is_exist'
+    description = "Trace config is exist."
+    code = 400
+
+
+class TracingConfigCheckError(BaseHTTPException):
+    error_code = 'trace_config_check_error'
+    description = "Invalid Credentials."
+    code = 400

api/controllers/console/app/model_config.py

@@ -25,6 +25,7 @@ class ModelConfigResource(Resource):
     @account_initialization_required
     @get_app_model(mode=[AppMode.AGENT_CHAT, AppMode.CHAT, AppMode.COMPLETION])
     def post(self, app_model):
+        
         """Modify app model config"""
         # validate config
         model_configuration = AppModelConfigService.validate_configuration(

api/controllers/console/app/ops_trace.py

@@ -0,0 +1,101 @@
+from flask_restful import Resource, reqparse
+
+from controllers.console import api
+from controllers.console.app.error import TracingConfigCheckError, TracingConfigIsExist, TracingConfigNotExist
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required
+from libs.login import login_required
+from services.ops_service import OpsService
+
+
+class TraceAppConfigApi(Resource):
+    """
+    Manage trace app configurations
+    """
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def get(self, app_id):
+        parser = reqparse.RequestParser()
+        parser.add_argument('tracing_provider', type=str, required=True, location='args')
+        args = parser.parse_args()
+
+        try:
+            trace_config = OpsService.get_tracing_app_config(
+                app_id=app_id, tracing_provider=args['tracing_provider']
+                )
+            if not trace_config:
+                return {"has_not_configured": True}
+            return trace_config
+        except Exception as e:
+            raise e
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def post(self, app_id):
+        """Create a new trace app configuration"""
+        parser = reqparse.RequestParser()
+        parser.add_argument('tracing_provider', type=str, required=True, location='json')
+        parser.add_argument('tracing_config', type=dict, required=True, location='json')
+        args = parser.parse_args()
+
+        try:
+            result = OpsService.create_tracing_app_config(
+                app_id=app_id,
+                tracing_provider=args['tracing_provider'],
+                tracing_config=args['tracing_config']
+            )
+            if not result:
+                raise TracingConfigIsExist()
+            if result.get('error'):
+                raise TracingConfigCheckError()
+            return result
+        except Exception as e:
+            raise e
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def patch(self, app_id):
+        """Update an existing trace app configuration"""
+        parser = reqparse.RequestParser()
+        parser.add_argument('tracing_provider', type=str, required=True, location='json')
+        parser.add_argument('tracing_config', type=dict, required=True, location='json')
+        args = parser.parse_args()
+
+        try:
+            result = OpsService.update_tracing_app_config(
+                app_id=app_id,
+                tracing_provider=args['tracing_provider'],
+                tracing_config=args['tracing_config']
+            )
+            if not result:
+                raise TracingConfigNotExist()
+            return {"result": "success"}
+        except Exception as e:
+            raise e
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def delete(self, app_id):
+        """Delete an existing trace app configuration"""
+        parser = reqparse.RequestParser()
+        parser.add_argument('tracing_provider', type=str, required=True, location='args')
+        args = parser.parse_args()
+
+        try:
+            result = OpsService.delete_tracing_app_config(
+                app_id=app_id,
+                tracing_provider=args['tracing_provider']
+            )
+            if not result:
+                raise TracingConfigNotExist()
+            return {"result": "success"}
+        except Exception as e:
+            raise e
+
+
+api.add_resource(TraceAppConfigApi, '/apps/<uuid:app_id>/trace-config')

api/controllers/console/app/site.py

@@ -20,6 +20,8 @@ def parse_app_site_args():
     parser.add_argument('icon_background', type=str, required=False, location='json')
     parser.add_argument('description', type=str, required=False, location='json')
     parser.add_argument('default_language', type=supported_language, required=False, location='json')
+    parser.add_argument('chat_color_theme', type=str, required=False, location='json')
+    parser.add_argument('chat_color_theme_inverted', type=bool, required=False, location='json')
     parser.add_argument('customize_domain', type=str, required=False, location='json')
     parser.add_argument('copyright', type=str, required=False, location='json')
     parser.add_argument('privacy_policy', type=str, required=False, location='json')
@@ -28,6 +30,7 @@ def parse_app_site_args():
                         required=False,
                         location='json')
     parser.add_argument('prompt_public', type=bool, required=False, location='json')
+    parser.add_argument('show_workflow_steps', type=bool, required=False, location='json')
     return parser.parse_args()
 
 
@@ -54,12 +57,15 @@ class AppSite(Resource):
             'icon_background',
             'description',
             'default_language',
+            'chat_color_theme',
+            'chat_color_theme_inverted',
             'customize_domain',
             'copyright',
             'privacy_policy',
             'custom_disclaimer',
             'customize_token_strategy',
-            'prompt_public'
+            'prompt_public',
+            'show_workflow_steps'
         ]:
             value = args.get(attr_name)
             if value is not None:

api/controllers/console/app/workflow.py

@@ -13,12 +13,14 @@ from controllers.console.setup import setup_required
 from controllers.console.wraps import account_initialization_required
 from core.app.apps.base_app_queue_manager import AppQueueManager
 from core.app.entities.app_invoke_entities import InvokeFrom
+from core.errors.error import AppInvokeQuotaExceededError
 from fields.workflow_fields import workflow_fields
 from fields.workflow_run_fields import workflow_run_node_execution_fields
 from libs import helper
 from libs.helper import TimestampField, uuid_value
 from libs.login import current_user, login_required
 from models.model import App, AppMode
+from services.app_dsl_service import AppDslService
 from services.app_generate_service import AppGenerateService
 from services.errors.app import WorkflowHashNotEqualError
 from services.workflow_service import WorkflowService
@@ -109,6 +111,33 @@ class DraftWorkflowApi(Resource):
         }
 
 
+class DraftWorkflowImportApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @get_app_model(mode=[AppMode.ADVANCED_CHAT, AppMode.WORKFLOW])
+    @marshal_with(workflow_fields)
+    def post(self, app_model: App):
+        """
+        Import draft workflow
+        """
+        # The role of the current user in the ta table must be admin, owner, or editor
+        if not current_user.is_editor:
+            raise Forbidden()
+
+        parser = reqparse.RequestParser()
+        parser.add_argument('data', type=str, required=True, nullable=False, location='json')
+        args = parser.parse_args()
+
+        workflow = AppDslService.import_and_overwrite_workflow(
+            app_model=app_model,
+            data=args['data'],
+            account=current_user
+        )
+
+        return workflow
+
+
 class AdvancedChatDraftWorkflowRunApi(Resource):
     @setup_required
     @login_required
@@ -251,7 +280,7 @@ class DraftWorkflowRunApi(Resource):
             )
 
             return helper.compact_generate_response(response)
-        except ValueError as e:
+        except (ValueError, AppInvokeQuotaExceededError) as e:
             raise e
         except Exception as e:
             logging.exception("internal server error.")
@@ -439,6 +468,7 @@ class ConvertToWorkflowApi(Resource):
 
 
 api.add_resource(DraftWorkflowApi, '/apps/<uuid:app_id>/workflows/draft')
+api.add_resource(DraftWorkflowImportApi, '/apps/<uuid:app_id>/workflows/draft/import')
 api.add_resource(AdvancedChatDraftWorkflowRunApi, '/apps/<uuid:app_id>/advanced-chat/workflows/draft/run')
 api.add_resource(DraftWorkflowRunApi, '/apps/<uuid:app_id>/workflows/draft/run')
 api.add_resource(WorkflowTaskStopApi, '/apps/<uuid:app_id>/workflow-runs/tasks/<string:task_id>/stop')

api/controllers/console/auth/data_source_oauth.py

@@ -6,6 +6,7 @@ from flask_login import current_user
 from flask_restful import Resource
 from werkzeug.exceptions import Forbidden
 
+from configs import dify_config
 from controllers.console import api
 from libs.login import login_required
 from libs.oauth_data_source import NotionOAuth
@@ -16,11 +17,11 @@ from ..wraps import account_initialization_required
 
 def get_oauth_providers():
     with current_app.app_context():
-        notion_oauth = NotionOAuth(client_id=current_app.config.get('NOTION_CLIENT_ID'),
-                                   client_secret=current_app.config.get(
-                                       'NOTION_CLIENT_SECRET'),
-                                   redirect_uri=current_app.config.get(
-                                       'CONSOLE_API_URL') + '/console/api/oauth/data-source/callback/notion')
+        if not dify_config.NOTION_CLIENT_ID or not dify_config.NOTION_CLIENT_SECRET:
+            return {}
+        notion_oauth = NotionOAuth(client_id=dify_config.NOTION_CLIENT_ID,
+                                   client_secret=dify_config.NOTION_CLIENT_SECRET,
+                                   redirect_uri=dify_config.CONSOLE_API_URL + '/console/api/oauth/data-source/callback/notion')
 
         OAUTH_PROVIDERS = {
             'notion': notion_oauth
@@ -39,8 +40,10 @@ class OAuthDataSource(Resource):
             print(vars(oauth_provider))
         if not oauth_provider:
             return {'error': 'Invalid provider'}, 400
-        if current_app.config.get('NOTION_INTEGRATION_TYPE') == 'internal':
-            internal_secret = current_app.config.get('NOTION_INTERNAL_SECRET')
+        if dify_config.NOTION_INTEGRATION_TYPE == 'internal':
+            internal_secret = dify_config.NOTION_INTERNAL_SECRET
+            if not internal_secret:
+                return {'error': 'Internal secret is not set'},
             oauth_provider.save_internal_access_token(internal_secret)
             return { 'data': '' }
         else:
@@ -60,13 +63,13 @@ class OAuthDataSourceCallback(Resource):
         if 'code' in request.args:
             code = request.args.get('code')
 
-            return redirect(f'{current_app.config.get("CONSOLE_WEB_URL")}?type=notion&code={code}')
+            return redirect(f'{dify_config.CONSOLE_WEB_URL}?type=notion&code={code}')
         elif 'error' in request.args:
             error = request.args.get('error')
 
-            return redirect(f'{current_app.config.get("CONSOLE_WEB_URL")}?type=notion&error={error}')
+            return redirect(f'{dify_config.CONSOLE_WEB_URL}?type=notion&error={error}')
         else:
-            return redirect(f'{current_app.config.get("CONSOLE_WEB_URL")}?type=notion&error=Access denied')
+            return redirect(f'{dify_config.CONSOLE_WEB_URL}?type=notion&error=Access denied')
         
 
 class OAuthDataSourceBinding(Resource):

api/controllers/console/auth/error.py

@@ -5,3 +5,28 @@ class ApiKeyAuthFailedError(BaseHTTPException):
     error_code = 'auth_failed'
     description = "{message}"
     code = 500
+
+
+class InvalidEmailError(BaseHTTPException):
+    error_code = 'invalid_email'
+    description = "The email address is not valid."
+    code = 400
+
+
+class PasswordMismatchError(BaseHTTPException):
+    error_code = 'password_mismatch'
+    description = "The passwords do not match."
+    code = 400
+
+
+class InvalidTokenError(BaseHTTPException):
+    error_code = 'invalid_or_expired_token'
+    description = "The token is invalid or has expired."
+    code = 400
+
+
+class PasswordResetRateLimitExceededError(BaseHTTPException):
+    error_code = 'password_reset_rate_limit_exceeded'
+    description = "Password reset rate limit exceeded. Try again later."
+    code = 429
+

api/controllers/console/auth/forgot_password.py

@@ -0,0 +1,107 @@
+import base64
+import logging
+import secrets
+
+from flask_restful import Resource, reqparse
+
+from controllers.console import api
+from controllers.console.auth.error import (
+    InvalidEmailError,
+    InvalidTokenError,
+    PasswordMismatchError,
+    PasswordResetRateLimitExceededError,
+)
+from controllers.console.setup import setup_required
+from extensions.ext_database import db
+from libs.helper import email as email_validate
+from libs.password import hash_password, valid_password
+from models.account import Account
+from services.account_service import AccountService
+from services.errors.account import RateLimitExceededError
+
+
+class ForgotPasswordSendEmailApi(Resource):
+
+    @setup_required
+    def post(self):
+        parser = reqparse.RequestParser()
+        parser.add_argument('email', type=str, required=True, location='json')
+        args = parser.parse_args()
+
+        email = args['email']
+
+        if not email_validate(email):
+            raise InvalidEmailError()
+
+        account = Account.query.filter_by(email=email).first()
+
+        if account:
+            try:
+                AccountService.send_reset_password_email(account=account)
+            except RateLimitExceededError:
+                logging.warning(f"Rate limit exceeded for email: {account.email}")
+                raise PasswordResetRateLimitExceededError()
+        else:
+            # Return success to avoid revealing email registration status
+            logging.warning(f"Attempt to reset password for unregistered email: {email}")
+
+        return {"result": "success"}
+
+
+class ForgotPasswordCheckApi(Resource):
+
+    @setup_required
+    def post(self):
+        parser = reqparse.RequestParser()
+        parser.add_argument('token', type=str, required=True, nullable=False, location='json')
+        args = parser.parse_args()
+        token = args['token']
+
+        reset_data = AccountService.get_reset_password_data(token)
+
+        if reset_data is None:
+            return {'is_valid': False, 'email': None}
+        return {'is_valid': True, 'email': reset_data.get('email')}
+
+
+class ForgotPasswordResetApi(Resource):
+
+    @setup_required
+    def post(self):
+        parser = reqparse.RequestParser()
+        parser.add_argument('token', type=str, required=True, nullable=False, location='json')
+        parser.add_argument('new_password', type=valid_password, required=True, nullable=False, location='json')
+        parser.add_argument('password_confirm', type=valid_password, required=True, nullable=False, location='json')
+        args = parser.parse_args()
+
+        new_password = args['new_password']
+        password_confirm = args['password_confirm']
+
+        if str(new_password).strip() != str(password_confirm).strip():
+            raise PasswordMismatchError()
+
+        token = args['token']
+        reset_data = AccountService.get_reset_password_data(token)
+
+        if reset_data is None:
+            raise InvalidTokenError()
+
+        AccountService.revoke_reset_password_token(token)
+
+        salt = secrets.token_bytes(16)
+        base64_salt = base64.b64encode(salt).decode()
+
+        password_hashed = hash_password(new_password, salt)
+        base64_password_hashed = base64.b64encode(password_hashed).decode()
+
+        account = Account.query.filter_by(email=reset_data.get('email')).first()
+        account.password = base64_password_hashed
+        account.password_salt = base64_salt
+        db.session.commit()
+
+        return {'result': 'success'}
+
+
+api.add_resource(ForgotPasswordSendEmailApi, '/forgot-password')
+api.add_resource(ForgotPasswordCheckApi, '/forgot-password/validity')
+api.add_resource(ForgotPasswordResetApi, '/forgot-password/resets')

api/controllers/console/auth/login.py

@@ -1,12 +1,15 @@
+from typing import cast
+
 import flask_login
-from flask import current_app, request
+from flask import request
 from flask_restful import Resource, reqparse
 
 import services
 from controllers.console import api
 from controllers.console.setup import setup_required
-from libs.helper import email
+from libs.helper import email, get_remote_ip
 from libs.password import valid_password
+from models.account import Account
 from services.account_service import AccountService, TenantService
 
 
@@ -34,10 +37,7 @@ class LoginApi(Resource):
         if len(tenants) == 0:
             return {'result': 'fail', 'data': 'workspace not found, please contact system admin to invite you to join in a workspace'}
 
-        AccountService.update_last_login(account, request)
-
-        # todo: return the user info
-        token = AccountService.get_account_jwt_token(account)
+        token = AccountService.login(account, ip_address=get_remote_ip(request))
 
         return {'result': 'success', 'data': token}
 
@@ -46,6 +46,9 @@ class LogoutApi(Resource):
 
     @setup_required
     def get(self):
+        account = cast(Account, flask_login.current_user)
+        token = request.headers.get('Authorization', '').split(' ')[1]
+        AccountService.logout(account=account, token=token)
         flask_login.logout_user()
         return {'result': 'success'}
 
@@ -53,14 +56,14 @@ class LogoutApi(Resource):
 class ResetPasswordApi(Resource):
     @setup_required
     def get(self):
-        parser = reqparse.RequestParser()
-        parser.add_argument('email', type=email, required=True, location='json')
-        args = parser.parse_args()
+        # parser = reqparse.RequestParser()
+        # parser.add_argument('email', type=email, required=True, location='json')
+        # args = parser.parse_args()
 
         # import mailchimp_transactional as MailchimpTransactional
         # from mailchimp_transactional.api_client import ApiClientError
 
-        account = {'email': args['email']}
+        # account = {'email': args['email']}
         # account = AccountService.get_by_email(args['email'])
         # if account is None:
         #     raise ValueError('Email not found')
@@ -68,22 +71,22 @@ class ResetPasswordApi(Resource):
         # AccountService.update_password(account, new_password)
 
         # todo: Send email
-        MAILCHIMP_API_KEY = current_app.config['MAILCHIMP_TRANSACTIONAL_API_KEY']
+        # MAILCHIMP_API_KEY = current_app.config['MAILCHIMP_TRANSACTIONAL_API_KEY']
         # mailchimp = MailchimpTransactional(MAILCHIMP_API_KEY)
 
-        message = {
-            'from_email': 'noreply@example.com',
-            'to': [{'email': account.email}],
-            'subject': 'Reset your Dify password',
-            'html': """
-                <p>Dear User,</p>
-                <p>The Dify team has generated a new password for you, details as follows:</p> 
-                <p><strong>{new_password}</strong></p>
-                <p>Please change your password to log in as soon as possible.</p>
-                <p>Regards,</p>
-                <p>The Dify Team</p> 
-            """
-        }
+        # message = {
+        #     'from_email': 'noreply@example.com',
+        #     'to': [{'email': account['email']}],
+        #     'subject': 'Reset your Dify password',
+        #     'html': """
+        #         <p>Dear User,</p>
+        #         <p>The Dify team has generated a new password for you, details as follows:</p> 
+        #         <p><strong>{new_password}</strong></p>
+        #         <p>Please change your password to log in as soon as possible.</p>
+        #         <p>Regards,</p>
+        #         <p>The Dify Team</p> 
+        #     """
+        # }
 
         # response = mailchimp.messages.send({
         #     'message': message,

api/controllers/console/auth/oauth.py

@@ -6,8 +6,10 @@ import requests
 from flask import current_app, redirect, request
 from flask_restful import Resource
 
+from configs import dify_config
 from constants.languages import languages
 from extensions.ext_database import db
+from libs.helper import get_remote_ip
 from libs.oauth import GitHubOAuth, GoogleOAuth, OAuthUserInfo
 from models.account import Account, AccountStatus
 from services.account_service import AccountService, RegisterService, TenantService
@@ -17,22 +19,24 @@ from .. import api
 
 def get_oauth_providers():
     with current_app.app_context():
-        github_oauth = GitHubOAuth(client_id=current_app.config.get('GITHUB_CLIENT_ID'),
-                                   client_secret=current_app.config.get(
-                                       'GITHUB_CLIENT_SECRET'),
-                                   redirect_uri=current_app.config.get(
-                                       'CONSOLE_API_URL') + '/console/api/oauth/authorize/github')
+        if not dify_config.GITHUB_CLIENT_ID or not dify_config.GITHUB_CLIENT_SECRET:
+            github_oauth = None
+        else:
+            github_oauth = GitHubOAuth(
+                client_id=dify_config.GITHUB_CLIENT_ID,
+                client_secret=dify_config.GITHUB_CLIENT_SECRET,
+                redirect_uri=dify_config.CONSOLE_API_URL + '/console/api/oauth/authorize/github',
+            )
+        if not dify_config.GOOGLE_CLIENT_ID or not dify_config.GOOGLE_CLIENT_SECRET:
+            google_oauth = None
+        else:
+            google_oauth = GoogleOAuth(
+                client_id=dify_config.GOOGLE_CLIENT_ID,
+                client_secret=dify_config.GOOGLE_CLIENT_SECRET,
+                redirect_uri=dify_config.CONSOLE_API_URL + '/console/api/oauth/authorize/google',
+            )
 
-        google_oauth = GoogleOAuth(client_id=current_app.config.get('GOOGLE_CLIENT_ID'),
-                                   client_secret=current_app.config.get(
-                                       'GOOGLE_CLIENT_SECRET'),
-                                   redirect_uri=current_app.config.get(
-                                       'CONSOLE_API_URL') + '/console/api/oauth/authorize/google')
-
-        OAUTH_PROVIDERS = {
-            'github': github_oauth,
-            'google': google_oauth
-        }
+        OAUTH_PROVIDERS = {'github': github_oauth, 'google': google_oauth}
         return OAUTH_PROVIDERS
 
 
@@ -62,8 +66,7 @@ class OAuthCallback(Resource):
             token = oauth_provider.get_access_token(code)
             user_info = oauth_provider.get_user_info(token)
         except requests.exceptions.HTTPError as e:
-            logging.exception(
-                f"An error occurred during the OAuth process with {provider}: {e.response.text}")
+            logging.exception(f'An error occurred during the OAuth process with {provider}: {e.response.text}')
             return {'error': 'OAuth process failed'}, 400
 
         account = _generate_account(provider, user_info)
@@ -78,11 +81,9 @@ class OAuthCallback(Resource):
 
         TenantService.create_owner_tenant_if_not_exist(account)
 
-        AccountService.update_last_login(account, request)
+        token = AccountService.login(account, ip_address=get_remote_ip(request))
 
-        token = AccountService.get_account_jwt_token(account)
-
-        return redirect(f'{current_app.config.get("CONSOLE_WEB_URL")}?console_token={token}')
+        return redirect(f'{dify_config.CONSOLE_WEB_URL}?console_token={token}')
 
 
 def _get_account_by_openid_or_email(provider: str, user_info: OAuthUserInfo) -> Optional[Account]:
@@ -102,11 +103,7 @@ def _generate_account(provider: str, user_info: OAuthUserInfo):
         # Create account
         account_name = user_info.name if user_info.name else 'Dify'
         account = RegisterService.register(
-            email=user_info.email,
-            name=account_name,
-            password=None,
-            open_id=user_info.id,
-            provider=provider
+            email=user_info.email, name=account_name, password=None, open_id=user_info.id, provider=provider
         )
 
         # Set interface language

api/controllers/console/datasets/datasets.py

@@ -8,7 +8,7 @@ import services
 from controllers.console import api
 from controllers.console.apikey import api_key_fields, api_key_list
 from controllers.console.app.error import ProviderNotInitializeError
-from controllers.console.datasets.error import DatasetInUseError, DatasetNameDuplicateError
+from controllers.console.datasets.error import DatasetInUseError, DatasetNameDuplicateError, IndexingEstimateError
 from controllers.console.setup import setup_required
 from controllers.console.wraps import account_initialization_required
 from core.errors.error import LLMBadRequestError, ProviderTokenNotInitError
@@ -17,6 +17,7 @@ from core.model_runtime.entities.model_entities import ModelType
 from core.provider_manager import ProviderManager
 from core.rag.datasource.vdb.vector_type import VectorType
 from core.rag.extractor.entity.extract_setting import ExtractSetting
+from core.rag.retrieval.retrival_methods import RetrievalMethod
 from extensions.ext_database import db
 from fields.app_fields import related_app_list
 from fields.dataset_fields import dataset_detail_fields, dataset_query_detail_fields
@@ -24,7 +25,7 @@ from fields.document_fields import document_status_fields
 from libs.login import login_required
 from models.dataset import Dataset, Document, DocumentSegment
 from models.model import ApiToken, UploadFile
-from services.dataset_service import DatasetService, DocumentService
+from services.dataset_service import DatasetPermissionService, DatasetService, DocumentService
 
 
 def _validate_name(name):
@@ -84,6 +85,12 @@ class DatasetListApi(Resource):
             else:
                 item['embedding_available'] = True
 
+            if item.get('permission') == 'partial_members':
+                part_users_list = DatasetPermissionService.get_dataset_partial_member_list(item['id'])
+                item.update({'partial_member_list': part_users_list})
+            else:
+                item.update({'partial_member_list': []})
+
         response = {
             'data': data,
             'has_more': len(datasets) == limit,
@@ -107,8 +114,8 @@ class DatasetListApi(Resource):
                             help='Invalid indexing technique.')
         args = parser.parse_args()
 
-        # The role of the current user in the ta table must be admin, owner, or editor
-        if not current_user.is_editor:
+        # The role of the current user in the ta table must be admin, owner, or editor, or dataset_operator
+        if not current_user.is_dataset_editor:
             raise Forbidden()
 
         try:
@@ -139,6 +146,10 @@ class DatasetApi(Resource):
         except services.errors.account.NoPermissionError as e:
             raise Forbidden(str(e))
         data = marshal(dataset, dataset_detail_fields)
+        if data.get('permission') == 'partial_members':
+            part_users_list = DatasetPermissionService.get_dataset_partial_member_list(dataset_id_str)
+            data.update({'partial_member_list': part_users_list})
+
         # check embedding setting
         provider_manager = ProviderManager()
         configurations = provider_manager.get_configurations(
@@ -162,6 +173,11 @@ class DatasetApi(Resource):
                 data['embedding_available'] = False
         else:
             data['embedding_available'] = True
+
+        if data.get('permission') == 'partial_members':
+            part_users_list = DatasetPermissionService.get_dataset_partial_member_list(dataset_id_str)
+            data.update({'partial_member_list': part_users_list})
+
         return data, 200
 
     @setup_required
@@ -187,17 +203,21 @@ class DatasetApi(Resource):
                             nullable=True,
                             help='Invalid indexing technique.')
         parser.add_argument('permission', type=str, location='json', choices=(
-            'only_me', 'all_team_members'), help='Invalid permission.')
+            'only_me', 'all_team_members', 'partial_members'), help='Invalid permission.'
+                            )
         parser.add_argument('embedding_model', type=str,
                             location='json', help='Invalid embedding model.')
         parser.add_argument('embedding_model_provider', type=str,
                             location='json', help='Invalid embedding model provider.')
         parser.add_argument('retrieval_model', type=dict, location='json', help='Invalid retrieval model.')
+        parser.add_argument('partial_member_list', type=list, location='json', help='Invalid parent user list.')
         args = parser.parse_args()
+        data = request.get_json()
 
-        # The role of the current user in the ta table must be admin, owner, or editor
-        if not current_user.is_editor:
-            raise Forbidden()
+        # The role of the current user in the ta table must be admin, owner, editor, or dataset_operator
+        DatasetPermissionService.check_permission(
+            current_user, dataset, data.get('permission'), data.get('partial_member_list')
+        )
 
         dataset = DatasetService.update_dataset(
             dataset_id_str, args, current_user)
@@ -205,7 +225,20 @@ class DatasetApi(Resource):
         if dataset is None:
             raise NotFound("Dataset not found.")
 
-        return marshal(dataset, dataset_detail_fields), 200
+        result_data = marshal(dataset, dataset_detail_fields)
+        tenant_id = current_user.current_tenant_id
+
+        if data.get('partial_member_list') and data.get('permission') == 'partial_members':
+            DatasetPermissionService.update_partial_member_list(
+                tenant_id, dataset_id_str, data.get('partial_member_list')
+            )
+        else:
+            DatasetPermissionService.clear_partial_member_list(dataset_id_str)
+
+        partial_member_list = DatasetPermissionService.get_dataset_partial_member_list(dataset_id_str)
+        result_data.update({'partial_member_list': partial_member_list})
+
+        return result_data, 200
 
     @setup_required
     @login_required
@@ -214,17 +247,27 @@ class DatasetApi(Resource):
         dataset_id_str = str(dataset_id)
 
         # The role of the current user in the ta table must be admin, owner, or editor
-        if not current_user.is_editor:
+        if not current_user.is_editor or current_user.is_dataset_operator:
             raise Forbidden()
 
         try:
             if DatasetService.delete_dataset(dataset_id_str, current_user):
+                DatasetPermissionService.clear_partial_member_list(dataset_id_str)
                 return {'result': 'success'}, 204
             else:
                 raise NotFound("Dataset not found.")
         except services.errors.dataset.DatasetInUseError:
             raise DatasetInUseError()
 
+class DatasetUseCheckApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def get(self, dataset_id):
+        dataset_id_str = str(dataset_id)
+
+        dataset_is_using = DatasetService.dataset_use_check(dataset_id_str)
+        return {'is_using': dataset_is_using}, 200
 
 class DatasetQueryApi(Resource):
 
@@ -345,6 +388,8 @@ class DatasetIndexingEstimateApi(Resource):
                 "in the Settings -> Model Provider.")
         except ProviderTokenNotInitError as ex:
             raise ProviderNotInitializeError(ex.description)
+        except Exception as e:
+            raise IndexingEstimateError(str(e))
 
         return response, 200
 
@@ -497,16 +542,18 @@ class DatasetRetrievalSettingApi(Resource):
     def get(self):
         vector_type = current_app.config['VECTOR_STORE']
         match vector_type:
-            case VectorType.MILVUS | VectorType.RELYT | VectorType.PGVECTOR | VectorType.TIDB_VECTOR | VectorType.CHROMA | VectorType.TENCENT:
+            case VectorType.MILVUS | VectorType.RELYT | VectorType.PGVECTOR | VectorType.TIDB_VECTOR | VectorType.CHROMA | VectorType.TENCENT | VectorType.ORACLE:
                 return {
                     'retrieval_method': [
-                        'semantic_search'
+                        RetrievalMethod.SEMANTIC_SEARCH
                     ]
                 }
-            case VectorType.QDRANT | VectorType.WEAVIATE:
+            case VectorType.QDRANT | VectorType.WEAVIATE | VectorType.OPENSEARCH | VectorType.ANALYTICDB | VectorType.MYSCALE:
                 return {
                     'retrieval_method': [
-                        'semantic_search', 'full_text_search', 'hybrid_search'
+                        RetrievalMethod.SEMANTIC_SEARCH,
+                        RetrievalMethod.FULL_TEXT_SEARCH,
+                        RetrievalMethod.HYBRID_SEARCH,
                     ]
                 }
             case _:
@@ -519,16 +566,18 @@ class DatasetRetrievalSettingMockApi(Resource):
     @account_initialization_required
     def get(self, vector_type):
         match vector_type:
-            case VectorType.MILVUS | VectorType.RELYT | VectorType.PGVECTOR | VectorType.TIDB_VECTOR | VectorType.CHROMA | VectorType.TENCEN:
+            case VectorType.MILVUS | VectorType.RELYT | VectorType.PGVECTOR | VectorType.TIDB_VECTOR | VectorType.CHROMA | VectorType.TENCENT | VectorType.ORACLE:
                 return {
                     'retrieval_method': [
-                        'semantic_search'
+                        RetrievalMethod.SEMANTIC_SEARCH
                     ]
                 }
-            case VectorType.QDRANT | VectorType.WEAVIATE:
+            case VectorType.QDRANT | VectorType.WEAVIATE | VectorType.OPENSEARCH| VectorType.ANALYTICDB | VectorType.MYSCALE:
                 return {
                     'retrieval_method': [
-                        'semantic_search', 'full_text_search', 'hybrid_search'
+                        RetrievalMethod.SEMANTIC_SEARCH,
+                        RetrievalMethod.FULL_TEXT_SEARCH,
+                        RetrievalMethod.HYBRID_SEARCH,
                     ]
                 }
             case _:
@@ -553,8 +602,30 @@ class DatasetErrorDocs(Resource):
         }, 200
 
 
+class DatasetPermissionUserListApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def get(self, dataset_id):
+        dataset_id_str = str(dataset_id)
+        dataset = DatasetService.get_dataset(dataset_id_str)
+        if dataset is None:
+            raise NotFound("Dataset not found.")
+        try:
+            DatasetService.check_dataset_permission(dataset, current_user)
+        except services.errors.account.NoPermissionError as e:
+            raise Forbidden(str(e))
+
+        partial_members_list = DatasetPermissionService.get_dataset_partial_member_list(dataset_id_str)
+
+        return {
+            'data': partial_members_list,
+        }, 200
+
+
 api.add_resource(DatasetListApi, '/datasets')
 api.add_resource(DatasetApi, '/datasets/<uuid:dataset_id>')
+api.add_resource(DatasetUseCheckApi, '/datasets/<uuid:dataset_id>/use-check')
 api.add_resource(DatasetQueryApi, '/datasets/<uuid:dataset_id>/queries')
 api.add_resource(DatasetErrorDocs, '/datasets/<uuid:dataset_id>/error-docs')
 api.add_resource(DatasetIndexingEstimateApi, '/datasets/indexing-estimate')
@@ -565,3 +636,4 @@ api.add_resource(DatasetApiDeleteApi, '/datasets/api-keys/<uuid:api_key_id>')
 api.add_resource(DatasetApiBaseUrlApi, '/datasets/api-base-info')
 api.add_resource(DatasetRetrievalSettingApi, '/datasets/retrieval-setting')
 api.add_resource(DatasetRetrievalSettingMockApi, '/datasets/retrieval-setting/<string:vector_type>')
+api.add_resource(DatasetPermissionUserListApi, '/datasets/<uuid:dataset_id>/permission-part-users')

api/controllers/console/datasets/datasets_document.py

@@ -20,6 +20,7 @@ from controllers.console.datasets.error import (
     ArchivedDocumentImmutableError,
     DocumentAlreadyFinishedError,
     DocumentIndexingError,
+    IndexingEstimateError,
     InvalidActionError,
     InvalidMetadataError,
 )
@@ -227,7 +228,7 @@ class DatasetDocumentListApi(Resource):
             raise NotFound('Dataset not found.')
 
         # The role of the current user in the ta table must be admin, owner, or editor
-        if not current_user.is_editor:
+        if not current_user.is_dataset_editor:
             raise Forbidden()
 
         try:
@@ -293,6 +294,11 @@ class DatasetInitApi(Resource):
         parser.add_argument('retrieval_model', type=dict, required=False, nullable=False,
                             location='json')
         args = parser.parse_args()
+
+        # The role of the current user in the ta table must be admin, owner, or editor, or dataset_operator
+        if not current_user.is_dataset_editor:
+            raise Forbidden()
+
         if args['indexing_technique'] == 'high_quality':
             try:
                 model_manager = ModelManager()
@@ -388,6 +394,8 @@ class DocumentIndexingEstimateApi(DocumentResource):
                         "in the Settings -> Model Provider.")
                 except ProviderTokenNotInitError as ex:
                     raise ProviderNotInitializeError(ex.description)
+                except Exception as e:
+                    raise IndexingEstimateError(str(e))
 
         return response
 
@@ -493,6 +501,8 @@ class DocumentBatchIndexingEstimateApi(DocumentResource):
                     "in the Settings -> Model Provider.")
             except ProviderTokenNotInitError as ex:
                 raise ProviderNotInitializeError(ex.description)
+            except Exception as e:
+                raise IndexingEstimateError(str(e))
         return response
 
 
@@ -752,14 +762,18 @@ class DocumentStatusApi(DocumentResource):
         dataset = DatasetService.get_dataset(dataset_id)
         if dataset is None:
             raise NotFound("Dataset not found.")
+
+        # The role of the current user in the ta table must be admin, owner, or editor
+        if not current_user.is_dataset_editor:
+            raise Forbidden()
+
         # check user's model setting
         DatasetService.check_dataset_model_setting(dataset)
 
-        document = self.get_document(dataset_id, document_id)
+        # check user's permission
+        DatasetService.check_dataset_permission(dataset, current_user)
 
-        # The role of the current user in the ta table must be admin, owner, or editor
-        if not current_user.is_editor:
-            raise Forbidden()
+        document = self.get_document(dataset_id, document_id)
 
         indexing_cache_key = 'document_{}_indexing'.format(document.id)
         cache_result = redis_client.get(indexing_cache_key)
@@ -950,10 +964,11 @@ class DocumentRenameApi(DocumentResource):
     @account_initialization_required
     @marshal_with(document_fields)
     def post(self, dataset_id, document_id):
-        # The role of the current user in the ta table must be admin or owner
-        if not current_user.is_admin_or_owner:
+        # The role of the current user in the ta table must be admin, owner, editor, or dataset_operator
+        if not current_user.is_dataset_editor:
             raise Forbidden()
-
+        dataset = DatasetService.get_dataset(dataset_id)
+        DatasetService.check_dataset_operator_permission(current_user, dataset)
         parser = reqparse.RequestParser()
         parser.add_argument('name', type=str, required=True, nullable=False, location='json')
         args = parser.parse_args()

api/controllers/console/datasets/error.py

@@ -83,3 +83,9 @@ class DatasetInUseError(BaseHTTPException):
     error_code = 'dataset_in_use'
     description = "The dataset is being used by some apps. Please remove the dataset from the apps before deleting it."
     code = 409
+
+
+class IndexingEstimateError(BaseHTTPException):
+    error_code = 'indexing_estimate_error'
+    description = "Knowledge indexing estimate failed: {message}"
+    code = 500

api/controllers/console/datasets/file.py

@@ -14,7 +14,7 @@ from controllers.console.setup import setup_required
 from controllers.console.wraps import account_initialization_required, cloud_edition_billing_resource_check
 from fields.file_fields import file_fields, upload_config_fields
 from libs.login import login_required
-from services.file_service import ALLOWED_EXTENSIONS, UNSTRUSTURED_ALLOWED_EXTENSIONS, FileService
+from services.file_service import ALLOWED_EXTENSIONS, UNSTRUCTURED_ALLOWED_EXTENSIONS, FileService
 
 PREVIEW_WORDS_LIMIT = 3000
 
@@ -77,7 +77,7 @@ class FileSupportTypeApi(Resource):
     @account_initialization_required
     def get(self):
         etl_type = current_app.config['ETL_TYPE']
-        allowed_extensions = UNSTRUSTURED_ALLOWED_EXTENSIONS if etl_type == 'Unstructured' else ALLOWED_EXTENSIONS
+        allowed_extensions = UNSTRUCTURED_ALLOWED_EXTENSIONS if etl_type == 'Unstructured' else ALLOWED_EXTENSIONS
         return {'allowed_extensions': allowed_extensions}
 
 

api/controllers/console/explore/audio.py

@@ -19,6 +19,7 @@ from controllers.console.app.error import (
 from controllers.console.explore.wraps import InstalledAppResource
 from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError
 from core.model_runtime.errors.invoke import InvokeError
+from models.model import AppMode
 from services.audio_service import AudioService
 from services.errors.audio import (
     AudioTooLargeServiceError,
@@ -70,16 +71,33 @@ class ChatAudioApi(InstalledAppResource):
 
 class ChatTextApi(InstalledAppResource):
     def post(self, installed_app):
-        app_model = installed_app.app
+        from flask_restful import reqparse
 
+        app_model = installed_app.app
         try:
+            parser = reqparse.RequestParser()
+            parser.add_argument('message_id', type=str, required=False, location='json')
+            parser.add_argument('voice', type=str, location='json')
+            parser.add_argument('streaming', type=bool, location='json')
+            args = parser.parse_args()
+
+            message_id = args.get('message_id')
+            if (app_model.mode in [AppMode.ADVANCED_CHAT.value, AppMode.WORKFLOW.value]
+                    and app_model.workflow
+                    and app_model.workflow.features_dict):
+                text_to_speech = app_model.workflow.features_dict.get('text_to_speech')
+                voice = args.get('voice') if args.get('voice') else text_to_speech.get('voice')
+            else:
+                try:
+                    voice = args.get('voice') if args.get('voice') else app_model.app_model_config.text_to_speech_dict.get('voice')
+                except Exception:
+                    voice = None
             response = AudioService.transcript_tts(
                 app_model=app_model,
-                text=request.form['text'],
-                voice=request.form['voice'] if request.form.get('voice') else app_model.app_model_config.text_to_speech_dict.get('voice'),
-                streaming=False
+                message_id=message_id,
+                voice=voice
             )
-            return {'data': response.data.decode('latin1')}
+            return response
         except services.errors.app_model_config.AppModelConfigBrokenError:
             logging.exception("App model config broken.")
             raise AppUnavailableError()
@@ -108,3 +126,5 @@ class ChatTextApi(InstalledAppResource):
 
 api.add_resource(ChatAudioApi, '/installed-apps/<uuid:installed_app_id>/audio-to-text', endpoint='installed_app_audio')
 api.add_resource(ChatTextApi, '/installed-apps/<uuid:installed_app_id>/text-to-audio', endpoint='installed_app_text')
+# api.add_resource(ChatTextApiWithMessageId, '/installed-apps/<uuid:installed_app_id>/text-to-audio/message-id',
+#                  endpoint='installed_app_text_with_message_id')

api/controllers/console/setup.py

@@ -3,11 +3,10 @@ from functools import wraps
 from flask import current_app, request
 from flask_restful import Resource, reqparse
 
-from extensions.ext_database import db
-from libs.helper import email, str_len
+from libs.helper import email, get_remote_ip, str_len
 from libs.password import valid_password
 from models.model import DifySetup
-from services.account_service import AccountService, RegisterService, TenantService
+from services.account_service import RegisterService, TenantService
 
 from . import api
 from .error import AlreadySetupError, NotInitValidateError, NotSetupError
@@ -51,28 +50,17 @@ class SetupApi(Resource):
                             required=True, location='json')
         args = parser.parse_args()
 
-        # Register
-        account = RegisterService.register(
+        # setup
+        RegisterService.setup(
             email=args['email'],
             name=args['name'],
-            password=args['password']
+            password=args['password'],
+            ip_address=get_remote_ip(request)
         )
 
-        TenantService.create_owner_tenant_if_not_exist(account)
-
-        setup()
-        AccountService.update_last_login(account, request)
-
         return {'result': 'success'}, 201
 
 
-def setup():
-    dify_setup = DifySetup(
-        version=current_app.config['CURRENT_VERSION']
-    )
-    db.session.add(dify_setup)
-
-
 def setup_required(view):
     @wraps(view)
     def decorated(*args, **kwargs):

api/controllers/console/tag/tags.py

@@ -36,7 +36,7 @@ class TagListApi(Resource):
     @account_initialization_required
     def post(self):
         # The role of the current user in the ta table must be admin, owner, or editor
-        if not current_user.is_editor:
+        if not (current_user.is_editor or current_user.is_dataset_editor):
             raise Forbidden()
 
         parser = reqparse.RequestParser()
@@ -68,7 +68,7 @@ class TagUpdateDeleteApi(Resource):
     def patch(self, tag_id):
         tag_id = str(tag_id)
         # The role of the current user in the ta table must be admin, owner, or editor
-        if not current_user.is_editor:
+        if not (current_user.is_editor or current_user.is_dataset_editor):
             raise Forbidden()
 
         parser = reqparse.RequestParser()
@@ -109,8 +109,8 @@ class TagBindingCreateApi(Resource):
     @login_required
     @account_initialization_required
     def post(self):
-        # The role of the current user in the ta table must be admin, owner, or editor
-        if not current_user.is_editor:
+        # The role of the current user in the ta table must be admin, owner, editor, or dataset_operator
+        if not (current_user.is_editor or current_user.is_dataset_editor):
             raise Forbidden()
 
         parser = reqparse.RequestParser()
@@ -134,8 +134,8 @@ class TagBindingDeleteApi(Resource):
     @login_required
     @account_initialization_required
     def post(self):
-        # The role of the current user in the ta table must be admin, owner, or editor
-        if not current_user.is_editor:
+        # The role of the current user in the ta table must be admin, owner, editor, or dataset_operator
+        if not (current_user.is_editor or current_user.is_dataset_editor):
             raise Forbidden()
 
         parser = reqparse.RequestParser()

api/controllers/console/workspace/account.py

@@ -245,6 +245,8 @@ class AccountIntegrateApi(Resource):
         return {'data': integrate_data}
 
 
+
+
 # Register API resources
 api.add_resource(AccountInitApi, '/account/init')
 api.add_resource(AccountProfileApi, '/account/profile')

api/controllers/console/workspace/members.py

@@ -131,7 +131,20 @@ class MemberUpdateRoleApi(Resource):
         return {'result': 'success'}
 
 
+class DatasetOperatorMemberListApi(Resource):
+    """List all members of current tenant."""
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @marshal_with(account_with_role_list_fields)
+    def get(self):
+        members = TenantService.get_dataset_operator_members(current_user.current_tenant)
+        return {'result': 'success', 'accounts': members}, 200
+
+
 api.add_resource(MemberListApi, '/workspaces/current/members')
 api.add_resource(MemberInviteEmailApi, '/workspaces/current/members/invite-email')
 api.add_resource(MemberCancelInviteApi, '/workspaces/current/members/<uuid:member_id>')
 api.add_resource(MemberUpdateRoleApi, '/workspaces/current/members/<uuid:member_id>/update-role')
+api.add_resource(DatasetOperatorMemberListApi, '/workspaces/current/dataset-operators')

api/controllers/console/workspace/tool_providers.py

@@ -104,7 +104,7 @@ class ToolBuiltinProviderIconApi(Resource):
     @setup_required
     def get(self, provider):
         icon_bytes, mimetype = BuiltinToolManageService.get_builtin_tool_provider_icon(provider)
-        icon_cache_max_age = int(current_app.config.get('TOOL_ICON_CACHE_MAX_AGE'))
+        icon_cache_max_age = current_app.config.get('TOOL_ICON_CACHE_MAX_AGE')
         return send_file(io.BytesIO(icon_bytes), mimetype=mimetype, max_age=icon_cache_max_age)
 
 class ToolApiProviderAddApi(Resource):

api/controllers/inner_api/wraps.py

@@ -3,8 +3,9 @@ from functools import wraps
 from hashlib import sha1
 from hmac import new as hmac_new
 
-from flask import abort, current_app, request
+from flask import abort, request
 
+from configs import dify_config
 from extensions.ext_database import db
 from models.model import EndUser
 
@@ -12,12 +13,12 @@ from models.model import EndUser
 def inner_api_only(view):
     @wraps(view)
     def decorated(*args, **kwargs):
-        if not current_app.config['INNER_API']:
+        if not dify_config.INNER_API:
             abort(404)
 
         # get header 'X-Inner-Api-Key'
         inner_api_key = request.headers.get('X-Inner-Api-Key')
-        if not inner_api_key or inner_api_key != current_app.config['INNER_API_KEY']:
+        if not inner_api_key or inner_api_key != dify_config.INNER_API_KEY:
             abort(404)
 
         return view(*args, **kwargs)
@@ -28,7 +29,7 @@ def inner_api_only(view):
 def inner_api_user_auth(view):
     @wraps(view)
     def decorated(*args, **kwargs):
-        if not current_app.config['INNER_API']:
+        if not dify_config.INNER_API:
             return view(*args, **kwargs)
 
         # get header 'X-Inner-Api-Key'

api/controllers/service_api/app/app.py

@@ -1,7 +1,7 @@
 
-from flask import current_app
 from flask_restful import Resource, fields, marshal_with
 
+from configs import dify_config
 from controllers.service_api import api
 from controllers.service_api.app.error import AppUnavailableError
 from controllers.service_api.wraps import validate_app_token
@@ -78,7 +78,7 @@ class AppParameterApi(Resource):
                                                      "transfer_methods": ["remote_url", "local_file"]
                                                  }}),
             'system_parameters': {
-                'image_file_size_limit': current_app.config.get('UPLOAD_IMAGE_FILE_SIZE_LIMIT')
+                'image_file_size_limit': dify_config.UPLOAD_IMAGE_FILE_SIZE_LIMIT
             }
         }
 

api/controllers/service_api/app/audio.py

@@ -20,7 +20,7 @@ from controllers.service_api.app.error import (
 from controllers.service_api.wraps import FetchUserArg, WhereisUserArg, validate_app_token
 from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError
 from core.model_runtime.errors.invoke import InvokeError
-from models.model import App, EndUser
+from models.model import App, AppMode, EndUser
 from services.audio_service import AudioService
 from services.errors.audio import (
     AudioTooLargeServiceError,
@@ -72,19 +72,30 @@ class AudioApi(Resource):
 class TextApi(Resource):
     @validate_app_token(fetch_user_arg=FetchUserArg(fetch_from=WhereisUserArg.JSON))
     def post(self, app_model: App, end_user: EndUser):
-        parser = reqparse.RequestParser()
-        parser.add_argument('text', type=str, required=True, nullable=False, location='json')
-        parser.add_argument('voice', type=str, location='json')
-        parser.add_argument('streaming', type=bool, required=False, nullable=False, location='json')
-        args = parser.parse_args()
-
         try:
+            parser = reqparse.RequestParser()
+            parser.add_argument('message_id', type=str, required=False, location='json')
+            parser.add_argument('voice', type=str, location='json')
+            parser.add_argument('streaming', type=bool, location='json')
+            args = parser.parse_args()
+
+            message_id = args.get('message_id')
+            if (app_model.mode in [AppMode.ADVANCED_CHAT.value, AppMode.WORKFLOW.value]
+                    and app_model.workflow
+                    and app_model.workflow.features_dict):
+                text_to_speech = app_model.workflow.features_dict.get('text_to_speech')
+                voice = args.get('voice') if args.get('voice') else text_to_speech.get('voice')
+            else:
+                try:
+                    voice = args.get('voice') if args.get('voice') else app_model.app_model_config.text_to_speech_dict.get(
+                        'voice')
+                except Exception:
+                    voice = None
             response = AudioService.transcript_tts(
                 app_model=app_model,
-                text=args['text'],
-                end_user=end_user,
-                voice=args.get('voice'),
-                streaming=args['streaming']
+                message_id=message_id,
+                end_user=end_user.external_user_id,
+                voice=voice
             )
 
             return response

api/controllers/service_api/app/completion.py

@@ -17,7 +17,12 @@ from controllers.service_api.app.error import (
 from controllers.service_api.wraps import FetchUserArg, WhereisUserArg, validate_app_token
 from core.app.apps.base_app_queue_manager import AppQueueManager
 from core.app.entities.app_invoke_entities import InvokeFrom
-from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError
+from core.errors.error import (
+    AppInvokeQuotaExceededError,
+    ModelCurrentlyNotSupportError,
+    ProviderTokenNotInitError,
+    QuotaExceededError,
+)
 from core.model_runtime.errors.invoke import InvokeError
 from libs import helper
 from libs.helper import uuid_value
@@ -69,7 +74,7 @@ class CompletionApi(Resource):
             raise ProviderModelCurrentlyNotSupportError()
         except InvokeError as e:
             raise CompletionRequestError(e.description)
-        except ValueError as e:
+        except (ValueError, AppInvokeQuotaExceededError) as e:
             raise e
         except Exception as e:
             logging.exception("internal server error.")
@@ -132,7 +137,7 @@ class ChatApi(Resource):
             raise ProviderModelCurrentlyNotSupportError()
         except InvokeError as e:
             raise CompletionRequestError(e.description)
-        except ValueError as e:
+        except (ValueError, AppInvokeQuotaExceededError) as e:
             raise e
         except Exception as e:
             logging.exception("internal server error.")

api/controllers/service_api/app/workflow.py

@@ -14,7 +14,12 @@ from controllers.service_api.app.error import (
 from controllers.service_api.wraps import FetchUserArg, WhereisUserArg, validate_app_token
 from core.app.apps.base_app_queue_manager import AppQueueManager
 from core.app.entities.app_invoke_entities import InvokeFrom
-from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError
+from core.errors.error import (
+    AppInvokeQuotaExceededError,
+    ModelCurrentlyNotSupportError,
+    ProviderTokenNotInitError,
+    QuotaExceededError,
+)
 from core.model_runtime.errors.invoke import InvokeError
 from libs import helper
 from models.model import App, AppMode, EndUser
@@ -59,7 +64,7 @@ class WorkflowRunApi(Resource):
             raise ProviderModelCurrentlyNotSupportError()
         except InvokeError as e:
             raise CompletionRequestError(e.description)
-        except ValueError as e:
+        except (ValueError, AppInvokeQuotaExceededError) as e:
             raise e
         except Exception as e:
             logging.exception("internal server error.")

api/controllers/service_api/index.py

@@ -1,6 +1,6 @@
-from flask import current_app
 from flask_restful import Resource
 
+from configs import dify_config
 from controllers.service_api import api
 
 
@@ -9,7 +9,7 @@ class IndexApi(Resource):
         return {
             "welcome": "Dify OpenAPI",
             "api_version": "v1",
-            "server_version": current_app.config['CURRENT_VERSION']
+            "server_version": dify_config.CURRENT_VERSION,
         }
 
 

api/controllers/web/app.py

@@ -1,6 +1,6 @@
-from flask import current_app
 from flask_restful import fields, marshal_with
 
+from configs import dify_config
 from controllers.web import api
 from controllers.web.error import AppUnavailableError
 from controllers.web.wraps import WebApiResource
@@ -75,7 +75,7 @@ class AppParameterApi(WebApiResource):
                 "transfer_methods": ["remote_url", "local_file"]
             }}),
             'system_parameters': {
-                'image_file_size_limit': current_app.config.get('UPLOAD_IMAGE_FILE_SIZE_LIMIT')
+                'image_file_size_limit': dify_config.UPLOAD_IMAGE_FILE_SIZE_LIMIT
             }
         }
 

api/controllers/web/audio.py

@@ -19,7 +19,7 @@ from controllers.web.error import (
 from controllers.web.wraps import WebApiResource
 from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError
 from core.model_runtime.errors.invoke import InvokeError
-from models.model import App
+from models.model import App, AppMode
 from services.audio_service import AudioService
 from services.errors.audio import (
     AudioTooLargeServiceError,
@@ -69,16 +69,35 @@ class AudioApi(WebApiResource):
 
 class TextApi(WebApiResource):
     def post(self, app_model: App, end_user):
+        from flask_restful import reqparse
         try:
+            parser = reqparse.RequestParser()
+            parser.add_argument('message_id', type=str, required=False, location='json')
+            parser.add_argument('voice', type=str, location='json')
+            parser.add_argument('streaming', type=bool, location='json')
+            args = parser.parse_args()
+
+            message_id = args.get('message_id')
+            if (app_model.mode in [AppMode.ADVANCED_CHAT.value, AppMode.WORKFLOW.value]
+                    and app_model.workflow
+                    and app_model.workflow.features_dict):
+                text_to_speech = app_model.workflow.features_dict.get('text_to_speech')
+                voice = args.get('voice') if args.get('voice') else text_to_speech.get('voice')
+            else:
+                try:
+                    voice = args.get('voice') if args.get(
+                        'voice') else app_model.app_model_config.text_to_speech_dict.get('voice')
+                except Exception:
+                    voice = None
+
             response = AudioService.transcript_tts(
                 app_model=app_model,
-                text=request.form['text'],
+                message_id=message_id,
                 end_user=end_user.external_user_id,
-                voice=request.form['voice'] if request.form.get('voice') else None,
-                streaming=False
+                voice=voice
             )
 
-            return {'data': response.data.decode('latin1')}
+            return response
         except services.errors.app_model_config.AppModelConfigBrokenError:
             logging.exception("App model config broken.")
             raise AppUnavailableError()

api/controllers/web/site.py

@@ -1,8 +1,8 @@
 
-from flask import current_app
 from flask_restful import fields, marshal_with
 from werkzeug.exceptions import Forbidden
 
+from configs import dify_config
 from controllers.web import api
 from controllers.web.wraps import WebApiResource
 from extensions.ext_database import db
@@ -26,6 +26,8 @@ class AppSiteApi(WebApiResource):
 
     site_fields = {
         'title': fields.String,
+        'chat_color_theme': fields.String,
+        'chat_color_theme_inverted': fields.Boolean,
         'icon': fields.String,
         'icon_background': fields.String,
         'description': fields.String,
@@ -33,7 +35,8 @@ class AppSiteApi(WebApiResource):
         'privacy_policy': fields.String,
         'custom_disclaimer': fields.String,
         'default_language': fields.String,
-        'prompt_public': fields.Boolean
+        'prompt_public': fields.Boolean,
+        'show_workflow_steps': fields.Boolean,
     }
 
     app_fields = {
@@ -81,7 +84,7 @@ class AppSiteInfo:
         self.can_replace_logo = can_replace_logo
 
         if can_replace_logo:
-            base_url = current_app.config.get('FILES_URL')
+            base_url = dify_config.FILES_URL
             remove_webapp_brand = tenant.custom_config_dict.get('remove_webapp_brand', False)
             replace_webapp_logo = f'{base_url}/files/workspaces/{tenant.id}/webapp-logo' if tenant.custom_config_dict.get('replace_webapp_logo') else None
             self.custom_config = {

api/core/agent/base_agent_runner.py

@@ -32,7 +32,6 @@ from core.model_runtime.entities.model_entities import ModelFeature
 from core.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel
 from core.model_runtime.utils.encoders import jsonable_encoder
 from core.tools.entities.tool_entities import (
-    ToolInvokeMessage,
     ToolParameter,
     ToolRuntimeVariablePool,
 )
@@ -141,24 +140,6 @@ class BaseAgentRunner(AppRunner):
             app_generate_entity.app_config.prompt_template.simple_prompt_template = ''
 
         return app_generate_entity
-
-    def _convert_tool_response_to_str(self, tool_response: list[ToolInvokeMessage]) -> str:
-        """
-        Handle tool response
-        """
-        result = ''
-        for response in tool_response:
-            if response.type == ToolInvokeMessage.MessageType.TEXT:
-                result += response.message
-            elif response.type == ToolInvokeMessage.MessageType.LINK:
-                result += f"result link: {response.message}. please tell user to check it."
-            elif response.type == ToolInvokeMessage.MessageType.IMAGE_LINK or \
-                 response.type == ToolInvokeMessage.MessageType.IMAGE:
-                result += "image has been created and sent to user already, you do not need to create it, just tell the user to check it now."
-            else:
-                result += f"tool response: {response.message}."
-
-        return result
     
     def _convert_tool_to_prompt_message_tool(self, tool: AgentToolEntity) -> tuple[PromptMessageTool, Tool]:
         """