feat: 白名单中去除 maas 平台

Co-authored-by: 开坦克的贝塔 <k@aircode.io>
Co-authored-by: crazywoola <100913391+crazywoola@users.noreply.github.com>

.github/workflows/api-tests.yml

@@ -14,7 +14,6 @@ concurrency:
 
 jobs:
   test:
-    name: API Tests
     runs-on: ubuntu-latest
     strategy:
       matrix:
@@ -59,7 +58,7 @@ jobs:
       - name: Run Workflow
         run: dev/pytest/pytest_workflow.sh
 
-      - name: Set up Vector Stores (Weaviate, Qdrant, PGVector, Milvus, PgVecto-RS)
+      - name: Set up Vector Stores (Weaviate, Qdrant, PGVector, Milvus, PgVecto-RS, Chroma)
         uses: hoverkraft-tech/compose-action@v2.0.0
         with:
           compose-file: |
@@ -68,6 +67,7 @@ jobs:
             docker/docker-compose.milvus.yaml
             docker/docker-compose.pgvecto-rs.yaml
             docker/docker-compose.pgvector.yaml
+            docker/docker-compose.chroma.yaml
           services: |
             weaviate
             qdrant
@@ -76,6 +76,84 @@ jobs:
             milvus-standalone
             pgvecto-rs
             pgvector
+            chroma
 
       - name: Test Vector Stores
         run: dev/pytest/pytest_vdb.sh
+
+  test-in-poetry:
+    name: API Tests
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version:
+          - "3.10"
+          - "3.11"
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Install Poetry
+        uses: abatilo/actions-poetry@v3
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: 'poetry'
+          cache-dependency-path: |
+            api/pyproject.toml
+            api/poetry.lock
+
+      - name: Poetry check
+        run: |
+          poetry check -C api
+          poetry show -C api
+
+      - name: Install dependencies
+        run: poetry install -C api --with dev
+
+      - name: Run Unit tests
+        run: poetry run -C api bash dev/pytest/pytest_unit_tests.sh
+
+      - name: Run ModelRuntime
+        run: poetry run -C api bash dev/pytest/pytest_model_runtime.sh
+
+      - name: Run Tool
+        run: poetry run -C api bash dev/pytest/pytest_tools.sh
+
+      - name: Set up Sandbox
+        uses: hoverkraft-tech/compose-action@v2.0.0
+        with:
+          compose-file: |
+            docker/docker-compose.middleware.yaml
+          services: |
+            sandbox
+            ssrf_proxy
+
+      - name: Run Workflow
+        run: poetry run -C api bash dev/pytest/pytest_workflow.sh
+
+      - name: Set up Vector Stores (Weaviate, Qdrant, PGVector, Milvus, PgVecto-RS, Chroma)
+        uses: hoverkraft-tech/compose-action@v2.0.0
+        with:
+          compose-file: |
+            docker/docker-compose.middleware.yaml
+            docker/docker-compose.qdrant.yaml
+            docker/docker-compose.milvus.yaml
+            docker/docker-compose.pgvecto-rs.yaml
+            docker/docker-compose.pgvector.yaml
+            docker/docker-compose.chroma.yaml
+          services: |
+            weaviate
+            qdrant
+            etcd
+            minio
+            milvus-standalone
+            pgvecto-rs
+            pgvector
+            chroma
+
+      - name: Test Vector Stores
+        run: poetry run -C api bash dev/pytest/pytest_vdb.sh

.github/workflows/build-push.yml

@@ -17,7 +17,7 @@ env:
 jobs:
   build-and-push:
     runs-on: ubuntu-latest
-    if: github.event.pull_request.draft == false
+    if: github.repository == 'langgenius/dify'
     strategy:
       matrix:
         include:

.github/workflows/db-migration-test.yml

@@ -23,24 +23,29 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v4
 
+      - name: Install Poetry
+        uses: abatilo/actions-poetry@v3
+
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
-          cache: 'pip'
+          cache: 'poetry'
           cache-dependency-path: |
-            ./api/requirements.txt
+            api/pyproject.toml
+            api/poetry.lock
 
       - name: Install dependencies
-        run: pip install -r ./api/requirements.txt
+        run: poetry install -C api
 
-      - name: Set up Middleware
+      - name: Set up Middlewares
         uses: hoverkraft-tech/compose-action@v2.0.0
         with:
           compose-file: |
             docker/docker-compose.middleware.yaml
           services: |
             db
+            redis
 
       - name: Prepare configs
         run: |
@@ -50,4 +55,4 @@ jobs:
       - name: Run DB Migration
         run: |
           cd api
-          flask db upgrade
+          poetry run python -m flask upgrade-db

.github/workflows/style.yml

@@ -24,6 +24,9 @@ jobs:
         with:
           files: api/**
 
+      - name: Install Poetry
+        uses: abatilo/actions-poetry@v3
+
       - name: Set up Python
         uses: actions/setup-python@v5
         if: steps.changed-files.outputs.any_changed == 'true'
@@ -32,15 +35,15 @@ jobs:
 
       - name: Python dependencies
         if: steps.changed-files.outputs.any_changed == 'true'
-        run: pip install ruff dotenv-linter
+        run: poetry install -C api --only lint
 
       - name: Ruff check
         if: steps.changed-files.outputs.any_changed == 'true'
-        run: ruff check ./api
+        run: poetry run -C api ruff check ./api
 
       - name: Dotenv check
         if: steps.changed-files.outputs.any_changed == 'true'
-        run: dotenv-linter ./api/.env.example ./web/.env.example
+        run: poetry run -C api dotenv-linter ./api/.env.example ./web/.env.example
 
       - name: Lint hints
         if: failure()
@@ -97,6 +100,7 @@ jobs:
             **.yaml
             **.yml
             Dockerfile
+            dev/**
 
       - name: Super-linter
         uses: super-linter/super-linter/slim@v6

.gitignore

@@ -136,6 +136,7 @@ web/.vscode/settings.json
 # Intellij IDEA Files
 .idea/*
 !.idea/vcs.xml
+!.idea/icon.png
 .ideaDataSources/
 
 api/.env
@@ -149,6 +150,7 @@ docker/volumes/qdrant/*
 docker/volumes/etcd/*
 docker/volumes/minio/*
 docker/volumes/milvus/*
+docker/volumes/chroma/*
 
 sdks/python-client/build
 sdks/python-client/dist

CONTRIBUTING_JA.md

@@ -4,7 +4,7 @@ Dify にコントリビュートしたいとお考えなのですね。それは
 私たちは現状を鑑み、機敏かつ迅速に開発をする必要がありますが、同時にあなたのようなコントリビューターの方々に、可能な限りスムーズな貢献体験をしていただきたいと思っています。そのためにこのコントリビュートガイドを作成しました。
 コードベースやコントリビュータの方々と私たちがどのように仕事をしているのかに慣れていただき、楽しいパートにすぐに飛び込めるようにすることが目的です。
 
-このガイドは Dify そのものと同様に、継続的に改善されています。実際のプロジェクトに遅れをとることがあるかもしれませんが、ご理解をお願いします。
+このガイドは Dify そのものと同様に、継続的に改善されています。実際のプロジェクトに遅れをとることがあるかもしれませんが、ご理解のほどよろしくお願いいたします。
 
 ライセンスに関しては、私たちの短い[ライセンスおよびコントリビューター規約](./LICENSE)をお読みください。また、コミュニティは[行動規範](https://github.com/langgenius/.github/blob/main/CODE_OF_CONDUCT.md)を遵守しています。
 
@@ -14,7 +14,7 @@ Dify にコントリビュートしたいとお考えなのですね。それは
 
 ### 機能リクエスト
 
-* 新しい機能要望を出す場合は、提案する機能が何を実現するものなのかを説明し、可能な限り多くの文脈を含めてください。[@perzeusss](https://github.com/perzeuss)は、あなたの要望を書き出すのに役立つ [Feature Request Copilot](https://udify.app/chat/MK2kVSnw1gakVwMX) を作ってくれました。気軽に試してみてください。
+* 新しい機能要望を出す場合は、提案する機能が何を実現するものなのかを説明し、可能な限り多くのコンテキストを含めてください。[@perzeusss](https://github.com/perzeuss)は、あなたの要望を書き出すのに役立つ [Feature Request Copilot](https://udify.app/chat/MK2kVSnw1gakVwMX) を作ってくれました。気軽に試してみてください。
 
 * 既存の課題から 1 つ選びたい場合は、その下にコメントを書いてください。
 
@@ -54,7 +54,7 @@ Dify にコントリビュートしたいとお考えなのですね。それは
 
 ## インストール
 
-Dify を開発用にセットアップする手順は以下の通りです。
+以下の手順で 、Difyのセットアップをしてください。
 
 ### 1. このリポジトリをフォークする
 
@@ -120,7 +120,7 @@ Dify のバックエンドは[Flask](https://flask.palletsprojects.com/en/3.0.x/
 
 ### フロントエンド
 
-このウェブサイトは、Typescript の[Next.js](https://nextjs.org/)ボイラープレートでブートストラップされており、スタイリングには[Tailwind CSS](https://tailwindcss.com/)を使用しています。国際化には[React-i18next](https://react.i18next.com/)を使用しています。
+このウェブサイトは、Typescriptベースの[Next.js](https://nextjs.org/)テンプレートを使ってブートストラップされ、[Tailwind CSS](https://tailwindcss.com/)を使ってスタイリングされています。国際化には[React-i18next](https://react.i18next.com/)を使用しています。
 
 ```
 [web/]

README.md

@@ -36,6 +36,7 @@
   <a href="./README_FR.md"><img alt="README en Français" src="https://img.shields.io/badge/Français-d9d9d9"></a>
   <a href="./README_KL.md"><img alt="README tlhIngan Hol" src="https://img.shields.io/badge/Klingon-d9d9d9"></a>
   <a href="./README_KR.md"><img alt="README in Korean" src="https://img.shields.io/badge/한국어-d9d9d9"></a>
+  <a href="./README_AR.md"><img alt="README بالعربية" src="https://img.shields.io/badge/العربية-d9d9d9"></a>
 </p>
 
 
@@ -184,10 +185,11 @@ After running, you can access the Dify dashboard in your browser at [http://loca
 
 If you need to customize the configuration, please refer to the comments in our [docker-compose.yml](docker/docker-compose.yaml) file and manually set the environment configuration. After making the changes, please run `docker-compose up -d` again. You can see the full list of environment variables [here](https://docs.dify.ai/getting-started/install-self-hosted/environments).
 
-If you'd like to configure a highly-available setup, there are community-contributed [Helm Charts](https://helm.sh/) which allow Dify to be deployed on Kubernetes.
+If you'd like to configure a highly-available setup, there are community-contributed [Helm Charts](https://helm.sh/) and YAML files which allow Dify to be deployed on Kubernetes.
 
 - [Helm Chart by @LeoQuote](https://github.com/douban/charts/tree/master/charts/dify)
 - [Helm Chart by @BorisPolonsky](https://github.com/BorisPolonsky/dify-helm)
+- [YAML file by @Winson-030](https://github.com/Winson-030/dify-kubernetes)
 
 
 ## Contributing

README_AR.md

@@ -0,0 +1,226 @@
+![cover-v5-optimized](https://github.com/langgenius/dify/assets/13230914/f9e19af5-61ba-4119-b926-d10c4c06ebab)
+
+<p align="center">
+  <a href="https://cloud.dify.ai">Dify Cloud</a> ·
+  <a href="https://docs.dify.ai/getting-started/install-self-hosted">الاستضافة الذاتية</a> ·
+  <a href="https://docs.dify.ai">التوثيق</a> ·
+  <a href="https://cal.com/guchenhe/60-min-meeting">استفسارات الشركات</a>
+</p>
+
+<p align="center">
+    <a href="https://dify.ai" target="_blank">
+        <img alt="Static Badge" src="https://img.shields.io/badge/Product-F04438"></a>
+    <a href="https://dify.ai/pricing" target="_blank">
+        <img alt="Static Badge" src="https://img.shields.io/badge/free-pricing?logo=free&color=%20%23155EEF&label=pricing&labelColor=%20%23528bff"></a>
+    <a href="https://discord.gg/FngNHpbcY7" target="_blank">
+        <img src="https://img.shields.io/discord/1082486657678311454?logo=discord&labelColor=%20%235462eb&logoColor=%20%23f5f5f5&color=%20%235462eb"
+            alt="chat on Discord"></a>
+    <a href="https://twitter.com/intent/follow?screen_name=dify_ai" target="_blank">
+        <img src="https://img.shields.io/twitter/follow/dify_ai?logo=X&color=%20%23f5f5f5"
+            alt="follow on Twitter"></a>
+    <a href="https://hub.docker.com/u/langgenius" target="_blank">
+        <img alt="Docker Pulls" src="https://img.shields.io/docker/pulls/langgenius/dify-web?labelColor=%20%23FDB062&color=%20%23f79009"></a>
+    <a href="https://github.com/langgenius/dify/graphs/commit-activity" target="_blank">
+        <img alt="Commits last month" src="https://img.shields.io/github/commit-activity/m/langgenius/dify?labelColor=%20%2332b583&color=%20%2312b76a"></a>
+    <a href="https://github.com/langgenius/dify/" target="_blank">
+        <img alt="Issues closed" src="https://img.shields.io/github/issues-search?query=repo%3Alanggenius%2Fdify%20is%3Aclosed&label=issues%20closed&labelColor=%20%237d89b0&color=%20%235d6b98"></a>
+    <a href="https://github.com/langgenius/dify/discussions/" target="_blank">
+        <img alt="Discussion posts" src="https://img.shields.io/github/discussions/langgenius/dify?labelColor=%20%239b8afb&color=%20%237a5af8"></a>
+</p>
+
+<p align="center">
+  <a href="./README.md"><img alt="README in English" src="https://img.shields.io/badge/English-d9d9d9"></a>
+  <a href="./README_CN.md"><img alt="简体中文版自述文件" src="https://img.shields.io/badge/简体中文-d9d9d9"></a>
+  <a href="./README_JA.md"><img alt="日本語のREADME" src="https://img.shields.io/badge/日本語-d9d9d9"></a>
+  <a href="./README_ES.md"><img alt="README en Español" src="https://img.shields.io/badge/Español-d9d9d9"></a>
+  <a href="./README_FR.md"><img alt="README en Français" src="https://img.shields.io/badge/Français-d9d9d9"></a>
+  <a href="./README_KL.md"><img alt="README tlhIngan Hol" src="https://img.shields.io/badge/Klingon-d9d9d9"></a>
+  <a href="./README_KR.md"><img alt="README in Korean" src="https://img.shields.io/badge/한국어-d9d9d9"></a>
+  <a href="./README_AR.md"><img alt="README بالعربية" src="https://img.shields.io/badge/العربية-d9d9d9"></a>
+</p>
+
+<div style="text-align: right;">
+مشروع Dify هو منصة تطوير تطبيقات الذكاء الصناعي مفتوحة المصدر. تجمع واجهته البديهية بين سير العمل الذكي بالذكاء الاصطناعي وخط أنابيب RAG وقدرات الوكيل وإدارة النماذج وميزات الملاحظة وأكثر من ذلك، مما يتيح لك الانتقال بسرعة من المرحلة التجريبية إلى الإنتاج. إليك قائمة بالميزات الأساسية:
+</br> </br>
+
+**1. سير العمل**:  قم ببناء واختبار سير عمل الذكاء الاصطناعي القوي على قماش بصري، مستفيدًا من جميع الميزات التالية وأكثر.
+
+  https://github.com/langgenius/dify/assets/13230914/356df23e-1604-483d-80a6-9517ece318aa
+
+
+**2. الدعم الشامل للنماذج**: تكامل سلس مع مئات من LLMs الخاصة / مفتوحة المصدر من عشرات من موفري التحليل والحلول المستضافة ذاتيًا، مما يغطي GPT و Mistral و Llama3 وأي نماذج متوافقة مع واجهة OpenAI API. يمكن العثور على قائمة كاملة بمزودي النموذج المدعومين [هنا](https://docs.dify.ai/getting-started/readme/model-providers).
+
+![providers-v5](https://github.com/langgenius/dify/assets/13230914/5a17bdbe-097a-4100-8363-40255b70f6e3)
+
+**3. بيئة التطوير للأوامر**: واجهة بيئة التطوير المبتكرة لصياغة الأمر ومقارنة أداء النموذج، وإضافة ميزات إضافية مثل تحويل النص إلى كلام إلى تطبيق قائم على الدردشة.
+
+**4. خط أنابيب RAG**: قدرات RAG الواسعة التي تغطي كل شيء من استيعاب الوثائق إلى الاسترجاع، مع الدعم الفوري لاستخراج النص من ملفات PDF و PPT وتنسيقات الوثائق الشائعة الأخرى.
+
+**5. قدرات الوكيل**: يمكنك تعريف الوكلاء بناءً على أمر وظيفة LLM أو ReAct، وإضافة أدوات مدمجة أو مخصصة للوكيل. توفر Dify أكثر من 50 أداة مدمجة لوكلاء الذكاء الاصطناعي، مثل البحث في Google و DELL·E وStable Diffusion و WolframAlpha.
+
+**6. الـ LLMOps**: راقب وتحلل سجلات التطبيق والأداء على مر الزمن. يمكنك تحسين الأوامر والبيانات والنماذج باستمرار استنادًا إلى البيانات الإنتاجية والتعليقات.
+
+**7.الواجهة الخلفية (Backend) كخدمة**: تأتي جميع عروض Dify مع APIs مطابقة، حتى يمكنك دمج Dify بسهولة في منطق أعمالك الخاص.
+## مقارنة الميزات
+<table style="width: 100%;">
+  <tr>
+    <th align="center">الميزة</th>
+    <th align="center">Dify.AI</th>
+    <th align="center">LangChain</th>
+    <th align="center">Flowise</th>
+    <th align="center">OpenAI Assistants API</th>
+  </tr>
+  <tr>
+    <td align="center">نهج البرمجة</td>
+    <td align="center">موجّه لـ تطبيق + واجهة برمجة تطبيق (API)</td>
+    <td align="center">برمجة Python</td>
+    <td align="center">موجه لتطبيق</td>
+    <td align="center">واجهة برمجة تطبيق (API)</td>
+  </tr>
+  <tr>
+    <td align="center">LLMs المدعومة</td>
+    <td align="center">تنوع غني</td>
+    <td align="center">تنوع غني</td>
+    <td align="center">تنوع غني</td>
+    <td align="center">فقط OpenAI</td>
+  </tr>
+  <tr>
+    <td align="center">محرك RAG</td>
+    <td align="center">✅</td>
+    <td align="center">✅</td>
+    <td align="center">✅</td>
+    <td align="center">✅</td>
+  </tr>
+  <tr>
+    <td align="center">الوكيل</td>
+    <td align="center">✅</td>
+    <td align="center">✅</td>
+    <td align="center">❌</td>
+    <td align="center">✅</td>
+  </tr>
+  <tr>
+    <td align="center">سير العمل</td>
+    <td align="center">✅</td>
+    <td align="center">❌</td>
+    <td align="center">✅</td>
+    <td align="center">❌</td>
+  </tr>
+  <tr>
+    <td align="center">الملاحظة</td>
+    <td align="center">✅</td>
+    <td align="center">✅</td>
+    <td align="center">❌</td>
+    <td align="center">❌</td>
+  </tr>
+  <tr>
+    <td align="center">ميزات الشركات (SSO / مراقبة الوصول)</td>
+    <td align="center">✅</td>
+    <td align="center">❌</td>
+    <td align="center">❌</td>
+    <td align="center">❌</td>
+  </tr>
+  <tr>
+    <td align="center">نشر محلي</td>
+    <td align="center">✅</td>
+    <td align="center">✅</td>
+    <td align="center">✅</td>
+    <td align="center">❌</td>
+  </tr>
+</table>
+
+
+## استخدام Dify
+- **سحابة </br>**
+نحن نستضيف [خدمة Dify Cloud](https://dify.ai) لأي شخص لتجربتها بدون أي إعدادات. توفر كل قدرات النسخة التي تمت استضافتها ذاتيًا، وتتضمن 200 أمر GPT-4 مجانًا في خطة الصندوق الرملي.
+
+- **استضافة ذاتية لنسخة المجتمع Dify</br>**
+ابدأ سريعًا في تشغيل Dify في بيئتك باستخدام [دليل البدء السريع](#البدء السريع).
+استخدم [توثيقنا](https://docs.dify.ai) للمزيد من المراجع والتعليمات الأعمق.
+
+- **مشروع Dify للشركات / المؤسسات</br>**
+نحن نوفر ميزات إضافية مركزة على الشركات. [جدول اجتماع معنا](https://cal.com/guchenhe/30min) أو [أرسل لنا بريدًا إلكترونيًا](mailto:business@dify.ai?subject=[GitHub]Business%20License%20Inquiry) لمناقشة احتياجات الشركات. </br>
+> بالنسبة للشركات الناشئة والشركات الصغيرة التي تستخدم خدمات AWS، تحقق من [Dify Premium على AWS Marketplace](https://aws.amazon.com/marketplace/pp/prodview-t22mebxzwjhu6) ونشرها في شبكتك الخاصة على AWS VPC بنقرة واحدة. إنها عرض AMI بأسعار معقولة مع خيار إنشاء تطبيقات بشعار وعلامة تجارية مخصصة.
+## البقاء قدمًا
+
+قم بإضافة نجمة إلى Dify على GitHub وتلق تنبيهًا فوريًا بالإصدارات الجديدة.
+
+![نجمنا](https://github.com/langgenius/dify/assets/13230914/b823edc1-6388-4e25-ad45-2f6b187adbb4)
+## البداية السريعة
+> قبل تثبيت Dify، تأكد من أن جهازك يلبي الحد الأدنى من متطلبات النظام التالية:
+> 
+>- معالج >= 2 نواة
+>- ذاكرة وصول عشوائي (RAM) >= 4 جيجابايت
+
+</br>
+
+أسهل طريقة لبدء تشغيل خادم Dify هي تشغيل ملف [docker-compose.yml](docker/docker-compose.yaml) الخاص بنا. قبل تشغيل أمر التثبيت، تأكد من تثبيت [Docker](https://docs.docker.com/get-docker/) و [Docker Compose](https://docs.docker.com/compose/install/) على جهازك:
+
+```bash
+cd docker
+docker compose up -d
+```
+بعد التشغيل، يمكنك الوصول إلى لوحة تحكم Dify في متصفحك على [http://localhost/install](http://localhost/install) وبدء عملية التهيئة.
+
+> إذا كنت ترغب في المساهمة في Dify أو القيام بتطوير إضافي، فانظر إلى [دليلنا للنشر من الشفرة (code) المصدرية](https://docs.dify.ai/getting-started/install-self-hosted/local-source-code)
+
+## الخطوات التالية
+
+إذا كنت بحاجة إلى تخصيص التكوين، يرجى الرجوع إلى التعليقات في ملف [docker-compose.yml](docker/docker-compose.yaml) لدينا وتعيين التكوينات البيئية يدويًا. بعد إجراء التغييرات، يرجى تشغيل `docker-compose up -d` مرة أخرى. يمكنك رؤية قائمة كاملة بالمتغيرات البيئية [هنا](https://docs.dify.ai/getting-started/install-self-hosted/environments).
+
+يوجد مجتمع خاص بـ [Helm Charts](https://helm.sh/) وملفات YAML التي تسمح بتنفيذ Dify على Kubernetes للنظام من الإيجابيات العلوية.
+
+- [رسم بياني Helm من قبل @LeoQuote](https://github.com/douban/charts/tree/master/charts/dify)
+- [رسم بياني Helm من قبل @BorisPolonsky](https://github.com/BorisPolonsky/dify-helm)
+- [ملف YAML من قبل @Winson-030](https://github.com/Winson-030/dify-kubernetes)
+
+
+## المساهمة
+
+لأولئك الذين يرغبون في المساهمة، انظر إلى [دليل المساهمة](https://github.com/langgenius/dify/blob/main/CONTRIBUTING.md) لدينا. 
+في الوقت نفسه، يرجى النظر في دعم Dify عن طريق مشاركته على وسائل التواصل الاجتماعي وفي الفعاليات والمؤتمرات.
+
+
+> نحن نبحث عن مساهمين لمساعدة في ترجمة Dify إلى لغات أخرى غير اللغة الصينية المندرين أو الإنجليزية. إذا كنت مهتمًا بالمساعدة، يرجى الاطلاع على [README للترجمة](https://github.com/langgenius/dify/blob/main/web/i18n/README.md) لمزيد من المعلومات، واترك لنا تعليقًا في قناة `global-users` على [خادم المجتمع على Discord](https://discord.gg/8Tpq4AcN9c).
+
+**المساهمون**
+
+<a href="https://github.com/langgenius/dify/graphs/contributors">
+  <img src="https://contrib.rocks/image?repo=langgenius/dify" />
+</a>
+
+## المجتمع والاتصال
+* [مناقشة Github](https://github.com/langgenius/dify/discussions). الأفضل لـ: مشاركة التعليقات وطرح الأسئلة.
+* [المشكلات على GitHub](https://github.com/langgenius/dify/issues). الأفضل لـ: الأخطاء التي تواجهها في استخدام Dify.AI، واقتراحات الميزات. انظر [دليل المساهمة](https://github.com/langgenius/dify/blob/main/CONTRIBUTING.md).
+* [البريد الإلكتروني](mailto:support@dify.ai?subject=[GitHub]Questions%20About%20Dify). الأفضل لـ: الأسئلة التي تتعلق باستخدام Dify.AI.
+* [Discord](https://discord.gg/FngNHpbcY7). الأفضل لـ: مشاركة تطبيقاتك والترفيه مع المجتمع.
+* [تويتر](https://twitter.com/dify_ai). الأفضل لـ: مشاركة تطبيقاتك والترفيه مع المجتمع.
+
+أو، قم بجدولة اجتماع مباشرة مع أحد أعضاء الفريق:
+
+<table>
+  <tr>
+    <th>نقطة الاتصال</th>
+    <th>الغرض</th>
+  </tr>
+  <tr>
+    <td><a href='https://cal.com/guchenhe/15min' target='_blank'><img class="schedule-button" src='https://github.com/langgenius/dify/assets/13230914/9ebcd111-1205-4d71-83d5-948d70b809f5' alt='Git-Hub-README-Button-3x' style="width: 180px; height: auto; object-fit: contain;"/></a></td>
+    <td>استفسارات الأعمال واقتراحات حول المنتج</td>
+  </tr>
+  <tr>
+    <td><a href='https://cal.com/pinkbanana' target='_blank'><img class="schedule-button" src='https://github.com/langgenius/dify/assets/13230914/d1edd00a-d7e4-4513-be6c-e57038e143fd' alt='Git-Hub-README-Button-2x' style="width: 180px; height: auto; object-fit: contain;"/></a></td>
+    <td>المساهمات والمشكلات وطلبات الميزات</td>
+  </tr>
+</table>
+
+## تاريخ النجمة
+
+[![Star History Chart](https://api.star-history.com/svg?repos=langgenius/dify&type=Date)](https://star-history.com/#langgenius/dify&Date)
+
+
+## الكشف عن الأمان
+
+لحماية خصوصيتك، يرجى تجنب نشر مشكلات الأمان على GitHub. بدلاً من ذلك، أرسل أسئلتك إلى security@dify.ai وسنقدم لك إجابة أكثر تفصيلاً.
+
+## الرخصة
+
+هذا المستودع متاح تحت [رخصة البرنامج الحر Dify](LICENSE)، والتي تعتبر بشكل أساسي Apache 2.0 مع بعض القيود الإضافية.

README_CN.md

@@ -186,10 +186,11 @@ docker compose up -d
 
 #### 使用 Helm Chart 部署
 
-使用 [Helm Chart](https://helm.sh/) 版本，可以在 Kubernetes 上部署 Dify。
+使用 [Helm Chart](https://helm.sh/) 版本或者 YAML 文件，可以在 Kubernetes 上部署 Dify。
 
 - [Helm Chart by @LeoQuote](https://github.com/douban/charts/tree/master/charts/dify)
 - [Helm Chart by @BorisPolonsky](https://github.com/BorisPolonsky/dify-helm)
+- [YAML 文件 by @Winson-030](https://github.com/Winson-030/dify-kubernetes)
 
 ### 配置
 

README_ES.md

@@ -192,10 +192,11 @@ Si necesitas personalizar la configuración, consulta los comentarios en nuestro
 
 . Después de realizar los cambios, ejecuta `docker-compose up -d` nuevamente. Puedes ver la lista completa de variables de entorno [aquí](https://docs.dify.ai/getting-started/install-self-hosted/environments).
 
-Si deseas configurar una instalación altamente disponible, hay [Gráficos Helm](https://helm.sh/) contribuidos por la comunidad que permiten implementar Dify en Kubernetes.
+Si desea configurar una configuración de alta disponibilidad, la comunidad proporciona [Gráficos Helm](https://helm.sh/) y archivos YAML, a través de los cuales puede desplegar Dify en Kubernetes.
 
 - [Gráfico Helm por @LeoQuote](https://github.com/douban/charts/tree/master/charts/dify)
 - [Gráfico Helm por @BorisPolonsky](https://github.com/BorisPolonsky/dify-helm)
+- [Ficheros YAML por @Winson-030](https://github.com/Winson-030/dify-kubernetes)
 
 
 ## Contribuir

README_FR.md

@@ -192,10 +192,11 @@ Si vous devez personnaliser la configuration, veuillez
 
  vous référer aux commentaires dans notre fichier [docker-compose.yml](docker/docker-compose.yaml) et définir manuellement la configuration de l'environnement. Après avoir apporté les modifications, veuillez exécuter à nouveau `docker-compose up -d`. Vous pouvez voir la liste complète des variables d'environnement [ici](https://docs.dify.ai/getting-started/install-self-hosted/environments).
 
-Si vous souhaitez configurer une installation hautement disponible, il existe des [Helm Charts](https://helm.sh/) contribués par la communauté qui permettent de déployer Dify sur Kubernetes.
+Si vous souhaitez configurer une configuration haute disponibilité, la communauté fournit des [Helm Charts](https://helm.sh/) et des fichiers YAML, à travers lesquels vous pouvez déployer Dify sur Kubernetes.
 
 - [Helm Chart par @LeoQuote](https://github.com/douban/charts/tree/master/charts/dify)
 - [Helm Chart par @BorisPolonsky](https://github.com/BorisPolonsky/dify-helm)
+- [Fichier YAML par @Winson-030](https://github.com/Winson-030/dify-kubernetes)
 
 
 ## Contribuer

README_JA.md

@@ -2,9 +2,9 @@
 
 <p align="center">
   <a href="https://cloud.dify.ai">Dify Cloud</a> ·
-  <a href="https://docs.dify.ai/getting-started/install-self-hosted">セルフホスト</a> ·
+  <a href="https://docs.dify.ai/getting-started/install-self-hosted">セルフホスティング</a> ·
   <a href="https://docs.dify.ai">ドキュメント</a> ·
-  <a href="https://cal.com/guchenhe/dify-demo">デモのスケジュール</a>
+  <a href="https://cal.com/guchenhe/dify-demo">デモの予約</a>
 </p>
 
 <p align="center">
@@ -44,37 +44,37 @@
   <a href="https://trendshift.io/repositories/2152" target="_blank"><img src="https://trendshift.io/api/badge/repositories/2152" alt="langgenius%2Fdify | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
 </p>
 
-DifyはオープンソースのLLMアプリケーション開発プラットフォームです。直感的なインターフェースには、AIワークフロー、RAGパイプライン、エージェント機能、モデル管理、観測機能などが組み合わさっており、プロトタイプから本番までの移行を迅速に行うことができます。以下は、主要機能のリストです：
+DifyはオープンソースのLLMアプリケーション開発プラットフォームです。直感的なインターフェイスには、AIワークフロー、RAGパイプライン、エージェント機能、モデル管理、観測機能などが組み合わさっており、プロトタイプから生産まで迅速に進めることができます。以下の機能が含まれます：
 </br> </br>
 
 **1. ワークフロー**: 
-  ビジュアルキャンバス上で強力なAIワークフローを構築してテストし、以下の機能を活用してプロトタイプを超えることができます。
+  強力なAIワークフローをビジュアルキャンバス上で構築し、テストできます。すべての機能、および以下の機能を使用できます。
 
 
   https://github.com/langgenius/dify/assets/13230914/356df23e-1604-483d-80a6-9517ece318aa
 
 
 
-**2. 包括的なモデルサポート**: 
-  数百のプロプライエタリ/オープンソースのLLMと、数十の推論プロバイダーおよびセルフホスティングソリューションとのシームレスな統合を提供します。GPT、Mistral、Llama3、およびOpenAI API互換のモデルをカバーします。サポートされているモデルプロバイダーの完全なリストは[こちら](https://docs.dify.ai/getting-started/readme/model-providers)をご覧ください。
+**2. 総合的なモデルサポート**: 
+  数百ものプロプライエタリ/オープンソースのLLMと、数十もの推論プロバイダーおよびセルフホスティングソリューションとのシームレスな統合を提供します。GPT、Mistral、Llama3、OpenAI APIと互換性のあるすべてのモデルを統合されています。サポートされているモデルプロバイダーの完全なリストは[こちら](https://docs.dify.ai/getting-started/readme/model-providers)をご覧ください。
 
 ![providers-v5](https://github.com/langgenius/dify/assets/13230914/5a17bdbe-097a-4100-8363-40255b70f6e3)
 
 
 **3. プロンプトIDE**: 
-  チャットベースのアプリにテキスト読み上げなどの追加機能を追加するプロンプトを作成し、モデルのパフォーマンスを比較する直感的なインターフェース。
+  プロンプトの作成、モデルパフォーマンスの比較が行え、チャットベースのアプリに音声合成などの機能も追加できます。
 
 **4. RAGパイプライン**: 
-  文書の取り込みから取得までをカバーする幅広いRAG機能で、PDF、PPTなどの一般的なドキュメント形式からのテキスト抽出に対するアウトオブボックスのサポートを提供します。
+  ドキュメントの取り込みから検索までをカバーする広範なRAG機能ができます。ほかにもPDF、PPT、その他の一般的なドキュメントフォーマットからのテキスト抽出のサーポイントも提供します。
 
 **5. エージェント機能**: 
-  LLM関数呼び出しまたはReActに基づいてエージェントを定義し、エージェント向けの事前構築済みまたはカスタムのツールを追加できます。Difyには、Google検索、DELL·E、Stable Diffusion、WolframAlphaなどのAIエージェント用の50以上の組み込みツールが用意されています。
+  LLM Function CallingやReActに基づくエージェントの定義が可能で、AIエージェント用のプリビルトまたはカスタムツールを追加できます。Difyには、Google検索、DELL·E、Stable Diffusion、WolframAlphaなどのAIエージェント用の50以上の組み込みツールが提供します。
 
 **6. LLMOps**: 
-  アプリケーションログとパフォーマンスを時間の経過とともにモニタリングおよび分析します。本番データと注釈に基づいて、プロンプト、データセット、およびモデルを継続的に改善できます。
+  アプリケーションのログやパフォーマンスを監視と分析し、生産のデータと注釈に基づいて、プロンプト、データセット、モデルを継続的に改善できます。
 
 **7. Backend-as-a-Service**: 
-  Difyのすべての提供には、それに対応するAPIが付属しており、独自のビジネスロジックにDifyをシームレスに統合できます。
+  すべての機能はAPIを提供されており、Difyを自分のビジネスロジックに簡単に統合できます。
 
 
 ## 機能比較
@@ -95,9 +95,9 @@ DifyはオープンソースのLLMアプリケーション開発プラットフ
   </tr>
   <tr>
     <td align="center">サポートされているLLM</td>
-    <td align="center">バリエーション豊富</td>
-    <td align="center">バリエーション豊富</td>
-    <td align="center">バリエーション豊富</td>
+    <td align="center">バラエティ豊か</td>
+    <td align="center">バラエティ豊か</td>
+    <td align="center">バラエティ豊か</td>
     <td align="center">OpenAIのみ</td>
   </tr>
   <tr>
@@ -147,15 +147,15 @@ DifyはオープンソースのLLMアプリケーション開発プラットフ
 ## Difyの使用方法
 
 - **クラウド </br>**
-[こちら](https://dify.ai)のDify Cloudサービスを利用して、セットアップ不要で試すことができます。サンドボックスプランには、200回の無料のGPT-4呼び出しが含まれています。
+[こちら](https://dify.ai)のDify Cloudサービスを利用して、セットアップ不要で試すことができます。サンドボックスプランには、200回のGPT-4呼び出しが無料で含まれています。
 
 - **Dify Community Editionのセルフホスティング</br>**
-この[スターターガイド](#quick-start)を使用して、ローカル環境でDifyを簡単に実行できます。
-さらなる参考資料や詳細な手順については、[ドキュメント](https://docs.dify.ai)をご覧ください。
+この[スタートガイド](#quick-start)を使用して、ローカル環境でDifyを簡単に実行できます。
+詳しくは[ドキュメント](https://docs.dify.ai)をご覧ください。
 
-- **エンタープライズ/組織向けのDify</br>**
-追加のエンタープライズ向け機能を提供しています。[こちらからミーティングを予約](https://cal.com/guchenhe/30min)したり、[メールを送信](mailto:business@dify.ai?subject=[GitHub]Business%20License%20Inquiry)してエンタープライズのニーズについて相談してください。 </br>
-  > AWSを使用しているスタートアップや中小企業の場合は、[AWS Marketplace](https://aws.amazon.com/marketplace/pp/prodview-t22mebxzwjhu6)のDify Premiumをチェックして、ワンクリックで独自のAWS VPCにデプロイできます。カスタムロゴとブランディングでアプリを作成するオプションを備えた手頃な価格のAMIオファリングです。
+- **企業/組織向けのDify</br>**
+企業中心の機能を提供しています。[こちらからミーティングを予約](https://cal.com/guchenhe/30min)したり、[メールを送信](mailto:business@dify.ai?subject=[GitHub]Business%20License%20Inquiry)して企業のニーズについて相談してください。 </br>
+  > AWSを使用しているスタートアップ企業や中小企業の場合は、[AWS Marketplace](https://aws.amazon.com/marketplace/pp/prodview-t22mebxzwjhu6)のDify Premiumをチェックして、ワンクリックで自分のAWS VPCにデプロイできます。さらに、手頃な価格のAMIオファリングどして、ロゴやブランディングをカスタマイズしてアプリケーションを作成するオプションがあります。
 
 
 ## 最新の情報を入手
@@ -189,10 +189,11 @@ docker compose up -d
 
 環境設定をカスタマイズする場合は、[docker-compose.yml](docker/docker-compose.yaml)ファイル内のコメントを参照して、環境設定を手動で設定してください。変更を加えた後は、再び `docker-compose up -d` を実行してください。環境変数の完全なリストは[こちら](https://docs.dify.ai/getting-started/install-self-hosted/environments)をご覧ください。
 
-高可用性のセットアップを構成する場合は、コミュニティによって提供されている[Helm Charts](https://helm.sh/)があり、これによりKubernetes上にDifyを展開できます。
+高可用性設定を設定する必要がある場合、コミュニティは[Helm Charts](https://helm.sh/)とYAMLファイルにより、DifyをKubernetesにデプロイすることができます。
 
 - [Helm Chart by @LeoQuote](https://github.com/douban/charts/tree/master/charts/dify)
 - [Helm Chart by @BorisPolonsky](https://github.com/BorisPolonsky/dify-helm)
+- [YAML file by @Winson-030](https://github.com/Winson-030/dify-kubernetes)
 
 
 ## 貢献
@@ -212,7 +213,7 @@ docker compose up -d
 ## コミュニティ & お問い合わせ
 
 * [Github Discussion](https://github.com/langgenius/dify/discussions). 主に: フィードバックの共有や質問。
-* [GitHub Issues](https://github.com/langgenius/dify/issues). 主に: Dify.AIの使用中に遭遇したバグや機能提案。
+* [GitHub Issues](https://github.com/langgenius/dify/issues). 主に: Dify.AIを使用する際に発生するエラーや問題については、[貢献ガイド](CONTRIBUTING_JA.md)を参照してください
 * [Email](mailto:support@dify.ai?subject=[GitHub]Questions%20About%20Dify). 主に: Dify.AIの使用に関する質問。
 * [Discord](https://discord.gg/FngNHpbcY7). 主に: アプリケーションの共有やコミュニティとの交流。
 * [Twitter](https://twitter.com/dify_ai). 主に: アプリケーションの共有やコミュニティとの交流。

README_KL.md

@@ -190,11 +190,11 @@ After running, you can access the Dify dashboard in your browser at [http://loca
 
 If you need to customize the configuration, please refer to the comments in our [docker-compose.yml](docker/docker-compose.yaml) file and manually set the environment configuration. After making the changes, please run `docker-compose up -d` again. You can see the full list of environment variables [here](https://docs.dify.ai/getting-started/install-self-hosted/environments).
 
-If you'd like to configure a highly-available setup, there are community-contributed [Helm Charts](https://helm.sh/) which allow Dify to be deployed on Kubernetes.
+If you'd like to configure a highly-available setup, there are community-contributed [Helm Charts](https://helm.sh/) and YAML files which allow Dify to be deployed on Kubernetes.
 
 - [Helm Chart by @LeoQuote](https://github.com/douban/charts/tree/master/charts/dify)
 - [Helm Chart by @BorisPolonsky](https://github.com/BorisPolonsky/dify-helm)
-
+- [YAML file by @Winson-030](https://github.com/Winson-030/dify-kubernetes)
 
 ## Contributing
 

README_KR.md

@@ -184,11 +184,11 @@ docker compose up -d
 구성 커스터마이징이 필요한 경우, [docker-compose.yml](docker/docker-compose.yaml) 파일의 코멘트를 참조하여 환경 구성을 수동으로 설정하십시오. 변경 후 `docker-compose up -d` 를 다시 실행하십시오. 환경 변수의 전체 목록은 [여기](https://docs.dify.ai/getting-started/install-self-hosted/environments)에서 확인할 수 있습니다.
 
 
-고가용성 설정을 구성하려면 Dify를 Kubernetes에 배포할 수 있는 커뮤니티 제공 [Helm Charts](https://helm.sh/)가 있습니다.
+Dify를 Kubernetes에 배포하고 프리미엄 스케일링 설정을 구성했다는 커뮤니티가 제공하는 [Helm Charts](https://helm.sh/)와 YAML 파일이 존재합니다.
 
 - [Helm Chart by @LeoQuote](https://github.com/douban/charts/tree/master/charts/dify)
 - [Helm Chart by @BorisPolonsky](https://github.com/BorisPolonsky/dify-helm)
-
+- [YAML file by @Winson-030](https://github.com/Winson-030/dify-kubernetes)
 
 ## 기여
 

api/.env.example

@@ -42,6 +42,7 @@ DB_DATABASE=dify
 # storage type: local, s3, azure-blob
 STORAGE_TYPE=local
 STORAGE_LOCAL_PATH=storage
+S3_USE_AWS_MANAGED_IAM=false
 S3_ENDPOINT=https://your-bucket-name.storage.s3.clooudflare.com
 S3_BUCKET_NAME=your-bucket-name
 S3_ACCESS_KEY=your-access-key
@@ -64,6 +65,13 @@ ALIYUN_OSS_REGION=your-region
 GOOGLE_STORAGE_BUCKET_NAME=yout-bucket-name
 GOOGLE_STORAGE_SERVICE_ACCOUNT_JSON=your-google-service-account-json-base64-string
 
+# Tencent COS Storage configuration
+TENCENT_COS_BUCKET_NAME=your-bucket-name
+TENCENT_COS_SECRET_KEY=your-secret-key
+TENCENT_COS_SECRET_ID=your-secret-id
+TENCENT_COS_REGION=your-region
+TENCENT_COS_SCHEME=your-scheme
+
 # CORS configuration
 WEB_API_CORS_ALLOW_ORIGINS=http://127.0.0.1:3000,*
 CONSOLE_CORS_ALLOW_ORIGINS=http://127.0.0.1:3000,*
@@ -98,6 +106,15 @@ RELYT_USER=postgres
 RELYT_PASSWORD=postgres
 RELYT_DATABASE=postgres
 
+# Tencent configuration
+TENCENT_VECTOR_DB_URL=http://127.0.0.1
+TENCENT_VECTOR_DB_API_KEY=dify
+TENCENT_VECTOR_DB_TIMEOUT=30
+TENCENT_VECTOR_DB_USERNAME=dify
+TENCENT_VECTOR_DB_DATABASE=dify
+TENCENT_VECTOR_DB_SHARD=1
+TENCENT_VECTOR_DB_REPLICAS=2
+
 # PGVECTO_RS configuration
 PGVECTO_RS_HOST=localhost
 PGVECTO_RS_PORT=5431
@@ -112,6 +129,21 @@ PGVECTOR_USER=postgres
 PGVECTOR_PASSWORD=postgres
 PGVECTOR_DATABASE=postgres
 
+# Tidb Vector configuration
+TIDB_VECTOR_HOST=xxx.eu-central-1.xxx.aws.tidbcloud.com
+TIDB_VECTOR_PORT=4000
+TIDB_VECTOR_USER=xxx.root
+TIDB_VECTOR_PASSWORD=xxxxxx
+TIDB_VECTOR_DATABASE=dify
+
+# Chroma configuration
+CHROMA_HOST=127.0.0.1
+CHROMA_PORT=8000
+CHROMA_TENANT=default_tenant
+CHROMA_DATABASE=default_database
+CHROMA_AUTH_PROVIDER=chromadb.auth.token_authn.TokenAuthenticationServerProvider
+CHROMA_AUTH_CREDENTIALS=difyai123456
+
 # Upload configuration
 UPLOAD_FILE_SIZE_LIMIT=15
 UPLOAD_FILE_BATCH_LIMIT=5
@@ -188,3 +220,7 @@ INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH=1000
 WORKFLOW_MAX_EXECUTION_STEPS=500
 WORKFLOW_MAX_EXECUTION_TIME=1200
 WORKFLOW_CALL_MAX_DEPTH=5
+
+# App configuration
+APP_MAX_EXECUTION_TIME=1200
+

api/.vscode/launch.json

@@ -17,7 +17,8 @@
                 "FLASK_DEBUG": "1",
                 "GEVENT_SUPPORT": "True"
             },
-            "console": "integratedTerminal"
+            "console": "integratedTerminal",
+            "python": "${command:python.interpreterPath}"
         },
         {
             "name": "Python: Flask",
@@ -36,7 +37,8 @@
                 "--debug"
             ],
             "jinja": true,
-            "justMyCode": true
+            "justMyCode": true,
+            "python": "${command:python.interpreterPath}"
         }
     ]
 }

api/README.md

@@ -11,21 +11,118 @@
    docker-compose -f docker-compose.middleware.yaml -p dify up -d
    cd ../api
    ```
+
+2. Copy `.env.example` to `.env`
+3. Generate a `SECRET_KEY` in the `.env` file.
+
+   ```bash for Linux
+   sed -i "/^SECRET_KEY=/c\SECRET_KEY=$(openssl rand -base64 42)" .env
+   ```
+
+   ```bash for Mac
+   secret_key=$(openssl rand -base64 42)
+   sed -i '' "/^SECRET_KEY=/c\\
+   SECRET_KEY=${secret_key}" .env
+   ```
+
+4. Create environment.
+
+   Dify API service uses [Poetry](https://python-poetry.org/docs/) to manage dependencies. You can execute `poetry shell` to activate the environment.
+
+   > Using pip can be found [below](#usage-with-pip).
+
+5. Install dependencies
+
+=======
+   ```bash
+   poetry env use 3.10
+   poetry install
+   ```
+
+   In case of contributors missing to update dependencies for `pyproject.toml`, you can perform the following shell instead.
+
+   ```bash
+   poetry shell                                               # activate current environment
+   poetry add $(cat requirements.txt)           # install dependencies of production and update pyproject.toml
+   poetry add $(cat requirements-dev.txt) --group dev    # install dependencies of development and update pyproject.toml
+   ```
+
+6. Run migrate
+
+   Before the first launch, migrate the database to the latest version.
+
+   ```bash
+   poetry run python -m flask db upgrade
+   ```
+
+7. Start backend
+
+   ```bash
+   poetry run python -m flask run --host 0.0.0.0 --port=5001 --debug
+   ```
+
+8. Start Dify [web](../web) service.
+9. Setup your application by visiting `http://localhost:3000`...
+10. If you need to debug local async processing, please start the worker service.
+
+   ```bash
+   poetry run python -m celery -A app.celery worker -P gevent -c 1 --loglevel INFO -Q dataset,generation,mail
+   ```
+
+   The started celery app handles the async tasks, e.g. dataset importing and documents indexing.
+
+## Testing
+
+1. Install dependencies for both the backend and the test environment
+
+   ```bash
+   poetry install --with dev
+   ```
+
+2. Run the tests locally with mocked system environment variables in `tool.pytest_env` section in `pyproject.toml`
+
+   ```bash
+   cd ../
+   poetry run -C api bash dev/pytest/pytest_all_tests.sh
+   ```
+
+## Usage with pip
+
+> [!NOTE]  
+> In the next version, we will deprecate pip as the primary package management tool for dify api service, currently Poetry and pip coexist.
+
+1. Start the docker-compose stack
+
+   The backend require some middleware, including PostgreSQL, Redis, and Weaviate, which can be started together using `docker-compose`.
+
+   ```bash
+   cd ../docker
+   docker-compose -f docker-compose.middleware.yaml -p dify up -d
+   cd ../api
+   ```
+
 2. Copy `.env.example` to `.env`
 3. Generate a `SECRET_KEY` in the `.env` file.
 
    ```bash
    sed -i "/^SECRET_KEY=/c\SECRET_KEY=$(openssl rand -base64 42)" .env
    ```
-4. If you use Anaconda, create a new environment and activate it
+
+4. Create environment.
+
+   If you use Anaconda, create a new environment and activate it
+  
    ```bash
    conda create --name dify python=3.10
    conda activate dify
    ```
+
 5. Install dependencies
+
    ```bash
    pip install -r requirements.txt
    ```
+
 6. Run migrate
 
    Before the first launch, migrate the database to the latest version.
@@ -34,37 +131,17 @@
    flask db upgrade
    ```
 
-   ⚠️ If you encounter problems with jieba, for example
-
-   ```
-   > flask db upgrade
-   Error: While importing 'app', an ImportError was raised:
-   ```
-
-   Please run the following command instead.
-
-   ```
-   pip install -r requirements.txt --upgrade --force-reinstall
-   ```
-
 7. Start backend:
+
    ```bash
    flask run --host 0.0.0.0 --port=5001 --debug
    ```
-8. Setup your application by visiting http://localhost:5001/console/api/setup or other apis...
-9. If you need to debug local async processing, please start the worker service by running 
-`celery -A app.celery worker -P gevent -c 1 --loglevel INFO -Q dataset,generation,mail`.
-The started celery app handles the async tasks, e.g. dataset importing and documents indexing.
 
+8. Setup your application by visiting <http://localhost:5001/console/api/setup> or other apis...
+9. If you need to debug local async processing, please start the worker service.
 
-## Testing
-
-1. Install dependencies for both the backend and the test environment
    ```bash
-   pip install -r requirements.txt -r requirements-dev.txt
-   ``` 
-   
-2. Run the tests locally with mocked system environment variables in `tool.pytest_env` section in `pyproject.toml`
-   ```bash
-   dev/pytest/pytest_all_tests.sh
+   celery -A app.celery worker -P gevent -c 1 --loglevel INFO -Q dataset,generation,mail
    ```
+
+   The started celery app handles the async tasks, e.g. dataset importing and documents indexing.

api/commands.py

@@ -1,14 +1,19 @@
 import base64
 import json
+import logging
 import secrets
+from typing import Optional
 
 import click
 from flask import current_app
 from werkzeug.exceptions import NotFound
 
+from constants.languages import languages
 from core.rag.datasource.vdb.vector_factory import Vector
+from core.rag.datasource.vdb.vector_type import VectorType
 from core.rag.models.document import Document
 from extensions.ext_database import db
+from extensions.ext_redis import redis_client
 from libs.helper import email as email_validate
 from libs.password import hash_password, password_pattern, valid_password
 from libs.rsa import generate_key_pair
@@ -17,6 +22,7 @@ from models.dataset import Dataset, DatasetCollectionBinding, DocumentSegment
 from models.dataset import Document as DatasetDocument
 from models.model import Account, App, AppAnnotationSetting, AppMode, Conversation, MessageAnnotation
 from models.provider import Provider, ProviderModel
+from services.account_service import RegisterService, TenantService
 
 
 @click.command('reset-password', help='Reset the account password.')
@@ -57,7 +63,7 @@ def reset_password(email, new_password, password_confirm):
     account.password = base64_password_hashed
     account.password_salt = base64_salt
     db.session.commit()
-    click.echo(click.style('Congratulations!, password has been reset.', fg='green'))
+    click.echo(click.style('Congratulations! Password has been reset.', fg='green'))
 
 
 @click.command('reset-email', help='Reset the account email.')
@@ -263,15 +269,15 @@ def migrate_knowledge_vector_database():
                         skipped_count = skipped_count + 1
                         continue
                 collection_name = ''
-                if vector_type == "weaviate":
+                if vector_type == VectorType.WEAVIATE:
                     dataset_id = dataset.id
                     collection_name = Dataset.gen_collection_name_by_id(dataset_id)
                     index_struct_dict = {
-                        "type": 'weaviate',
+                        "type": VectorType.WEAVIATE,
                         "vector_store": {"class_prefix": collection_name}
                     }
                     dataset.index_struct = json.dumps(index_struct_dict)
-                elif vector_type == "qdrant":
+                elif vector_type == VectorType.QDRANT:
                     if dataset.collection_binding_id:
                         dataset_collection_binding = db.session.query(DatasetCollectionBinding). \
                             filter(DatasetCollectionBinding.id == dataset.collection_binding_id). \
@@ -284,20 +290,20 @@ def migrate_knowledge_vector_database():
                         dataset_id = dataset.id
                         collection_name = Dataset.gen_collection_name_by_id(dataset_id)
                     index_struct_dict = {
-                        "type": 'qdrant',
+                        "type": VectorType.QDRANT,
                         "vector_store": {"class_prefix": collection_name}
                     }
                     dataset.index_struct = json.dumps(index_struct_dict)
 
-                elif vector_type == "milvus":
+                elif vector_type == VectorType.MILVUS:
                     dataset_id = dataset.id
                     collection_name = Dataset.gen_collection_name_by_id(dataset_id)
                     index_struct_dict = {
-                        "type": 'milvus',
+                        "type": VectorType.MILVUS,
                         "vector_store": {"class_prefix": collection_name}
                     }
                     dataset.index_struct = json.dumps(index_struct_dict)
-                elif vector_type == "relyt":
+                elif vector_type == VectorType.RELYT:
                     dataset_id = dataset.id
                     collection_name = Dataset.gen_collection_name_by_id(dataset_id)
                     index_struct_dict = {
@@ -305,16 +311,24 @@ def migrate_knowledge_vector_database():
                         "vector_store": {"class_prefix": collection_name}
                     }
                     dataset.index_struct = json.dumps(index_struct_dict)
-                elif vector_type == "pgvector":
+                elif vector_type == VectorType.TENCENT:
                     dataset_id = dataset.id
                     collection_name = Dataset.gen_collection_name_by_id(dataset_id)
                     index_struct_dict = {
-                        "type": 'pgvector',
+                        "type": VectorType.TENCENT,
+                        "vector_store": {"class_prefix": collection_name}
+                    }
+                    dataset.index_struct = json.dumps(index_struct_dict)
+                elif vector_type == VectorType.PGVECTOR:
+                    dataset_id = dataset.id
+                    collection_name = Dataset.gen_collection_name_by_id(dataset_id)
+                    index_struct_dict = {
+                        "type": VectorType.PGVECTOR,
                         "vector_store": {"class_prefix": collection_name}
                     }
                     dataset.index_struct = json.dumps(index_struct_dict)
                 else:
-                    raise ValueError(f"Vector store {config.get('VECTOR_STORE')} is not supported.")
+                    raise ValueError(f"Vector store {vector_type} is not supported.")
 
                 vector = Vector(dataset)
                 click.echo(f"Start to migrate dataset {dataset.id}.")
@@ -501,6 +515,68 @@ def add_qdrant_doc_id_index(field: str):
                     fg='green'))
 
 
+@click.command('create-tenant', help='Create account and tenant.')
+@click.option('--email', prompt=True, help='The email address of the tenant account.')
+@click.option('--language', prompt=True, help='Account language, default: en-US.')
+def create_tenant(email: str, language: Optional[str] = None):
+    """
+    Create tenant account
+    """
+    if not email:
+        click.echo(click.style('Sorry, email is required.', fg='red'))
+        return
+
+    # Create account
+    email = email.strip()
+
+    if '@' not in email:
+        click.echo(click.style('Sorry, invalid email address.', fg='red'))
+        return
+
+    account_name = email.split('@')[0]
+
+    if language not in languages:
+        language = 'en-US'
+
+    # generate random password
+    new_password = secrets.token_urlsafe(16)
+
+    # register account
+    account = RegisterService.register(
+        email=email,
+        name=account_name,
+        password=new_password,
+        language=language
+    )
+
+    TenantService.create_owner_tenant_if_not_exist(account)
+
+    click.echo(click.style('Congratulations! Account and tenant created.\n'
+                           'Account: {}\nPassword: {}'.format(email, new_password), fg='green'))
+
+
+@click.command('upgrade-db', help='upgrade the database')
+def upgrade_db():
+    click.echo('Preparing database migration...')
+    lock = redis_client.lock(name='db_upgrade_lock', timeout=60)
+    if lock.acquire(blocking=False):
+        try:
+            click.echo(click.style('Start database migration.', fg='green'))
+
+            # run db migration
+            import flask_migrate
+            flask_migrate.upgrade()
+
+            click.echo(click.style('Database migration successful!', fg='green'))
+
+        except Exception as e:
+            logging.exception(f'Database migration failed, error: {e}')
+        finally:
+            lock.release()
+    else:
+        click.echo('Database migration skipped')
+
+
 def register_commands(app):
     app.cli.add_command(reset_password)
     app.cli.add_command(reset_email)
@@ -508,4 +584,5 @@ def register_commands(app):
     app.cli.add_command(vdb_migrate)
     app.cli.add_command(convert_to_agent_apps)
     app.cli.add_command(add_qdrant_doc_id_index)
-
+    app.cli.add_command(create_tenant)
+    app.cli.add_command(upgrade_db)

api/config.py

@@ -24,6 +24,7 @@ DEFAULTS = {
     'APP_WEB_URL': 'https://udify.app',
     'FILES_URL': '',
     'FILES_ACCESS_TIMEOUT': 300,
+    'S3_USE_AWS_MANAGED_IAM': 'False',
     'S3_ADDRESS_STYLE': 'auto',
     'STORAGE_TYPE': 'local',
     'STORAGE_LOCAL_PATH': 'storage',
@@ -85,6 +86,7 @@ DEFAULTS = {
     'WORKFLOW_MAX_EXECUTION_STEPS': 500,
     'WORKFLOW_MAX_EXECUTION_TIME': 1200,
     'WORKFLOW_CALL_MAX_DEPTH': 5,
+    'APP_MAX_EXECUTION_TIME': 1200,
 }
 
 
@@ -115,7 +117,7 @@ class Config:
         # ------------------------
         # General Configurations.
         # ------------------------
-        self.CURRENT_VERSION = "0.6.10"
+        self.CURRENT_VERSION = "0.6.11"
         self.COMMIT_SHA = get_env('COMMIT_SHA')
         self.EDITION = get_env('EDITION')
         self.DEPLOY_ENV = get_env('DEPLOY_ENV')
@@ -225,6 +227,7 @@ class Config:
         self.STORAGE_LOCAL_PATH = get_env('STORAGE_LOCAL_PATH')
 
         # S3 Storage settings
+        self.S3_USE_AWS_MANAGED_IAM = get_bool_env('S3_USE_AWS_MANAGED_IAM')
         self.S3_ENDPOINT = get_env('S3_ENDPOINT')
         self.S3_BUCKET_NAME = get_env('S3_BUCKET_NAME')
         self.S3_ACCESS_KEY = get_env('S3_ACCESS_KEY')
@@ -250,6 +253,13 @@ class Config:
         self.GOOGLE_STORAGE_BUCKET_NAME = get_env('GOOGLE_STORAGE_BUCKET_NAME')
         self.GOOGLE_STORAGE_SERVICE_ACCOUNT_JSON_BASE64 = get_env('GOOGLE_STORAGE_SERVICE_ACCOUNT_JSON_BASE64')
 
+        # Tencent Cos Storage settings
+        self.TENCENT_COS_BUCKET_NAME = get_env('TENCENT_COS_BUCKET_NAME')
+        self.TENCENT_COS_REGION = get_env('TENCENT_COS_REGION')
+        self.TENCENT_COS_SECRET_ID = get_env('TENCENT_COS_SECRET_ID')
+        self.TENCENT_COS_SECRET_KEY = get_env('TENCENT_COS_SECRET_KEY')
+        self.TENCENT_COS_SCHEME = get_env('TENCENT_COS_SCHEME')
+
         # ------------------------
         # Vector Store Configurations.
         # Currently, only support: qdrant, milvus, zilliz, weaviate, relyt, pgvector
@@ -285,6 +295,16 @@ class Config:
         self.RELYT_PASSWORD = get_env('RELYT_PASSWORD')
         self.RELYT_DATABASE = get_env('RELYT_DATABASE')
 
+
+        # tencent settings
+        self.TENCENT_VECTOR_DB_URL = get_env('TENCENT_VECTOR_DB_URL')
+        self.TENCENT_VECTOR_DB_API_KEY = get_env('TENCENT_VECTOR_DB_API_KEY')
+        self.TENCENT_VECTOR_DB_TIMEOUT = get_env('TENCENT_VECTOR_DB_TIMEOUT')
+        self.TENCENT_VECTOR_DB_USERNAME = get_env('TENCENT_VECTOR_DB_USERNAME')
+        self.TENCENT_VECTOR_DB_DATABASE = get_env('TENCENT_VECTOR_DB_DATABASE')
+        self.TENCENT_VECTOR_DB_SHARD = get_env('TENCENT_VECTOR_DB_SHARD')
+        self.TENCENT_VECTOR_DB_REPLICAS = get_env('TENCENT_VECTOR_DB_REPLICAS')
+
         # pgvecto rs settings
         self.PGVECTO_RS_HOST = get_env('PGVECTO_RS_HOST')
         self.PGVECTO_RS_PORT = get_env('PGVECTO_RS_PORT')
@@ -299,6 +319,21 @@ class Config:
         self.PGVECTOR_PASSWORD = get_env('PGVECTOR_PASSWORD')
         self.PGVECTOR_DATABASE = get_env('PGVECTOR_DATABASE')
 
+        # tidb-vector settings
+        self.TIDB_VECTOR_HOST = get_env('TIDB_VECTOR_HOST')
+        self.TIDB_VECTOR_PORT = get_env('TIDB_VECTOR_PORT')
+        self.TIDB_VECTOR_USER = get_env('TIDB_VECTOR_USER')
+        self.TIDB_VECTOR_PASSWORD = get_env('TIDB_VECTOR_PASSWORD')
+        self.TIDB_VECTOR_DATABASE = get_env('TIDB_VECTOR_DATABASE')
+
+        # chroma settings
+        self.CHROMA_HOST = get_env('CHROMA_HOST')
+        self.CHROMA_PORT = get_env('CHROMA_PORT')
+        self.CHROMA_TENANT = get_env('CHROMA_TENANT')
+        self.CHROMA_DATABASE = get_env('CHROMA_DATABASE')
+        self.CHROMA_AUTH_PROVIDER = get_env('CHROMA_AUTH_PROVIDER')
+        self.CHROMA_AUTH_CREDENTIALS = get_env('CHROMA_AUTH_CREDENTIALS')
+
         # ------------------------
         # Mail Configurations.
         # ------------------------
@@ -357,6 +392,7 @@ class Config:
         self.WORKFLOW_MAX_EXECUTION_STEPS = int(get_env('WORKFLOW_MAX_EXECUTION_STEPS'))
         self.WORKFLOW_MAX_EXECUTION_TIME = int(get_env('WORKFLOW_MAX_EXECUTION_TIME'))
         self.WORKFLOW_CALL_MAX_DEPTH = int(get_env('WORKFLOW_CALL_MAX_DEPTH'))
+        self.APP_MAX_EXECUTION_TIME = int(get_env('APP_MAX_EXECUTION_TIME'))
 
         # Moderation in app Configurations.
         self.OUTPUT_MODERATION_BUFFER_SIZE = int(get_env('OUTPUT_MODERATION_BUFFER_SIZE'))

api/constants/languages.py

@@ -1,6 +1,6 @@
 
 
-languages = ['en-US', 'zh-Hans', 'zh-Hant', 'pt-BR', 'es-ES', 'fr-FR', 'de-DE', 'ja-JP', 'ko-KR', 'ru-RU', 'it-IT', 'uk-UA', 'vi-VN', 'pl-PL']
+languages = ['en-US', 'zh-Hans', 'zh-Hant', 'pt-BR', 'es-ES', 'fr-FR', 'de-DE', 'ja-JP', 'ko-KR', 'ru-RU', 'it-IT', 'uk-UA', 'vi-VN', 'pl-PL', 'hi-IN']
 
 language_timezone_mapping = {
     'en-US': 'America/New_York',
@@ -18,6 +18,7 @@ language_timezone_mapping = {
     'vi-VN': 'Asia/Ho_Chi_Minh',
     'ro-RO': 'Europe/Bucharest',
     'pl-PL': 'Europe/Warsaw',
+    'hi-IN': 'Asia/Kolkata'
 }
 
 

api/controllers/console/__init__.py

@@ -29,13 +29,13 @@ from .app import (
 )
 
 # Import auth controllers
-from .auth import activate, data_source_oauth, login, oauth
+from .auth import activate, data_source_bearer_auth, data_source_oauth, login, oauth
 
 # Import billing controllers
 from .billing import billing
 
 # Import datasets controllers
-from .datasets import data_source, datasets, datasets_document, datasets_segments, file, hit_testing
+from .datasets import data_source, datasets, datasets_document, datasets_segments, file, hit_testing, website
 
 # Import explore controllers
 from .explore import (

api/controllers/console/app/app.py

@@ -68,8 +68,8 @@ class AppListApi(Resource):
         parser.add_argument('icon_background', type=str, location='json')
         args = parser.parse_args()
 
-        # The role of the current user in the ta table must be admin or owner
-        if not current_user.is_admin_or_owner:
+        # The role of the current user in the ta table must be admin, owner, or editor
+        if not current_user.is_editor:
             raise Forbidden()
 
         if 'mode' not in args or args['mode'] is None:
@@ -89,8 +89,8 @@ class AppImportApi(Resource):
     @cloud_edition_billing_resource_check('apps')
     def post(self):
         """Import app"""
-        # The role of the current user in the ta table must be admin or owner
-        if not current_user.is_admin_or_owner:
+        # The role of the current user in the ta table must be admin, owner, or editor
+        if not current_user.is_editor:
             raise Forbidden()
 
         parser = reqparse.RequestParser()
@@ -129,6 +129,10 @@ class AppApi(Resource):
     @marshal_with(app_detail_fields_with_site)
     def put(self, app_model):
         """Update app"""
+        # The role of the current user in the ta table must be admin, owner, or editor
+        if not current_user.is_editor:
+            raise Forbidden()
+        
         parser = reqparse.RequestParser()
         parser.add_argument('name', type=str, required=True, nullable=False, location='json')
         parser.add_argument('description', type=str, location='json')
@@ -147,7 +151,8 @@ class AppApi(Resource):
     @get_app_model
     def delete(self, app_model):
         """Delete app"""
-        if not current_user.is_admin_or_owner:
+        # The role of the current user in the ta table must be admin, owner, or editor
+        if not current_user.is_editor:
             raise Forbidden()
 
         app_service = AppService()
@@ -164,8 +169,8 @@ class AppCopyApi(Resource):
     @marshal_with(app_detail_fields_with_site)
     def post(self, app_model):
         """Copy app"""
-        # The role of the current user in the ta table must be admin or owner
-        if not current_user.is_admin_or_owner:
+        # The role of the current user in the ta table must be admin, owner, or editor
+        if not current_user.is_editor:
             raise Forbidden()
 
         parser = reqparse.RequestParser()
@@ -203,6 +208,10 @@ class AppNameApi(Resource):
     @get_app_model
     @marshal_with(app_detail_fields)
     def post(self, app_model):
+        # The role of the current user in the ta table must be admin, owner, or editor
+        if not current_user.is_editor:
+            raise Forbidden()
+        
         parser = reqparse.RequestParser()
         parser.add_argument('name', type=str, required=True, location='json')
         args = parser.parse_args()
@@ -220,6 +229,10 @@ class AppIconApi(Resource):
     @get_app_model
     @marshal_with(app_detail_fields)
     def post(self, app_model):
+        # The role of the current user in the ta table must be admin, owner, or editor
+        if not current_user.is_editor:
+            raise Forbidden()
+        
         parser = reqparse.RequestParser()
         parser.add_argument('icon', type=str, location='json')
         parser.add_argument('icon_background', type=str, location='json')
@@ -238,6 +251,10 @@ class AppSiteStatus(Resource):
     @get_app_model
     @marshal_with(app_detail_fields)
     def post(self, app_model):
+        # The role of the current user in the ta table must be admin, owner, or editor
+        if not current_user.is_editor:
+            raise Forbidden()
+        
         parser = reqparse.RequestParser()
         parser.add_argument('enable_site', type=bool, required=True, location='json')
         args = parser.parse_args()
@@ -255,6 +272,10 @@ class AppApiStatus(Resource):
     @get_app_model
     @marshal_with(app_detail_fields)
     def post(self, app_model):
+        # The role of the current user in the ta table must be admin or owner
+        if not current_user.is_admin_or_owner:
+            raise Forbidden()
+        
         parser = reqparse.RequestParser()
         parser.add_argument('enable_api', type=bool, required=True, location='json')
         args = parser.parse_args()

api/controllers/console/app/conversation.py

@@ -6,7 +6,7 @@ from flask_restful import Resource, marshal_with, reqparse
 from flask_restful.inputs import int_range
 from sqlalchemy import func, or_
 from sqlalchemy.orm import joinedload
-from werkzeug.exceptions import NotFound
+from werkzeug.exceptions import Forbidden, NotFound
 
 from controllers.console import api
 from controllers.console.app.wraps import get_app_model
@@ -33,6 +33,8 @@ class CompletionConversationApi(Resource):
     @get_app_model(mode=AppMode.COMPLETION)
     @marshal_with(conversation_pagination_fields)
     def get(self, app_model):
+        if not current_user.is_admin_or_owner:
+            raise Forbidden()
         parser = reqparse.RequestParser()
         parser.add_argument('keyword', type=str, location='args')
         parser.add_argument('start', type=datetime_string('%Y-%m-%d %H:%M'), location='args')
@@ -106,6 +108,8 @@ class CompletionConversationDetailApi(Resource):
     @get_app_model(mode=AppMode.COMPLETION)
     @marshal_with(conversation_message_detail_fields)
     def get(self, app_model, conversation_id):
+        if not current_user.is_admin_or_owner:
+            raise Forbidden()
         conversation_id = str(conversation_id)
 
         return _get_conversation(app_model, conversation_id)
@@ -115,6 +119,8 @@ class CompletionConversationDetailApi(Resource):
     @account_initialization_required
     @get_app_model(mode=[AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT])
     def delete(self, app_model, conversation_id):
+        if not current_user.is_admin_or_owner:
+            raise Forbidden()
         conversation_id = str(conversation_id)
 
         conversation = db.session.query(Conversation) \
@@ -137,6 +143,8 @@ class ChatConversationApi(Resource):
     @get_app_model(mode=[AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT])
     @marshal_with(conversation_with_summary_pagination_fields)
     def get(self, app_model):
+        if not current_user.is_admin_or_owner:
+            raise Forbidden()
         parser = reqparse.RequestParser()
         parser.add_argument('keyword', type=str, location='args')
         parser.add_argument('start', type=datetime_string('%Y-%m-%d %H:%M'), location='args')
@@ -225,6 +233,8 @@ class ChatConversationDetailApi(Resource):
     @get_app_model(mode=[AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT])
     @marshal_with(conversation_detail_fields)
     def get(self, app_model, conversation_id):
+        if not current_user.is_admin_or_owner:
+            raise Forbidden()
         conversation_id = str(conversation_id)
 
         return _get_conversation(app_model, conversation_id)
@@ -234,6 +244,8 @@ class ChatConversationDetailApi(Resource):
     @get_app_model(mode=[AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT])
     @account_initialization_required
     def delete(self, app_model, conversation_id):
+        if not current_user.is_admin_or_owner:
+            raise Forbidden()
         conversation_id = str(conversation_id)
 
         conversation = db.session.query(Conversation) \

api/controllers/console/app/site.py

@@ -40,8 +40,8 @@ class AppSite(Resource):
     def post(self, app_model):
         args = parse_app_site_args()
 
-        # The role of the current user in the ta table must be admin or owner
-        if not current_user.is_admin_or_owner:
+        # The role of the current user in the ta table must be editor, admin, or owner
+        if not current_user.is_editor:
             raise Forbidden()
 
         site = db.session.query(Site). \
@@ -65,13 +65,6 @@ class AppSite(Resource):
             if value is not None:
                 setattr(site, attr_name, value)
 
-                if attr_name == 'title':
-                    app_model.name = value
-                elif attr_name == 'icon':
-                    app_model.icon = value
-                elif attr_name == 'icon_background':
-                    app_model.icon_background = value
-
         db.session.commit()
 
         return site

api/controllers/console/app/workflow.py

@@ -3,7 +3,7 @@ import logging
 
 from flask import abort, request
 from flask_restful import Resource, marshal_with, reqparse
-from werkzeug.exceptions import InternalServerError, NotFound
+from werkzeug.exceptions import Forbidden, InternalServerError, NotFound
 
 import services
 from controllers.console import api
@@ -36,6 +36,10 @@ class DraftWorkflowApi(Resource):
         """
         Get draft workflow
         """
+        # The role of the current user in the ta table must be admin, owner, or editor
+        if not current_user.is_editor:
+            raise Forbidden()
+        
         # fetch draft workflow by app_model
         workflow_service = WorkflowService()
         workflow = workflow_service.get_draft_workflow(app_model=app_model)
@@ -54,6 +58,10 @@ class DraftWorkflowApi(Resource):
         """
         Sync draft workflow
         """
+        # The role of the current user in the ta table must be admin, owner, or editor
+        if not current_user.is_editor:
+            raise Forbidden()
+        
         content_type = request.headers.get('Content-Type')
 
         if 'application/json' in content_type:
@@ -110,6 +118,10 @@ class AdvancedChatDraftWorkflowRunApi(Resource):
         """
         Run draft workflow
         """
+        # The role of the current user in the ta table must be admin, owner, or editor
+        if not current_user.is_editor:
+            raise Forbidden()
+        
         parser = reqparse.RequestParser()
         parser.add_argument('inputs', type=dict, location='json')
         parser.add_argument('query', type=str, required=True, location='json', default='')
@@ -146,6 +158,10 @@ class AdvancedChatDraftRunIterationNodeApi(Resource):
         """
         Run draft workflow iteration node
         """
+        # The role of the current user in the ta table must be admin, owner, or editor
+        if not current_user.is_editor:
+            raise Forbidden()
+        
         parser = reqparse.RequestParser()
         parser.add_argument('inputs', type=dict, location='json')
         args = parser.parse_args()
@@ -179,6 +195,10 @@ class WorkflowDraftRunIterationNodeApi(Resource):
         """
         Run draft workflow iteration node
         """
+        # The role of the current user in the ta table must be admin, owner, or editor
+        if not current_user.is_editor:
+            raise Forbidden()
+        
         parser = reqparse.RequestParser()
         parser.add_argument('inputs', type=dict, location='json')
         args = parser.parse_args()
@@ -212,6 +232,10 @@ class DraftWorkflowRunApi(Resource):
         """
         Run draft workflow
         """
+        # The role of the current user in the ta table must be admin, owner, or editor
+        if not current_user.is_editor:
+            raise Forbidden()
+        
         parser = reqparse.RequestParser()
         parser.add_argument('inputs', type=dict, required=True, nullable=False, location='json')
         parser.add_argument('files', type=list, required=False, location='json')
@@ -243,6 +267,10 @@ class WorkflowTaskStopApi(Resource):
         """
         Stop workflow task
         """
+        # The role of the current user in the ta table must be admin, owner, or editor
+        if not current_user.is_editor:
+            raise Forbidden()
+        
         AppQueueManager.set_stop_flag(task_id, InvokeFrom.DEBUGGER, current_user.id)
 
         return {
@@ -260,6 +288,10 @@ class DraftWorkflowNodeRunApi(Resource):
         """
         Run draft workflow node
         """
+        # The role of the current user in the ta table must be admin, owner, or editor
+        if not current_user.is_editor:
+            raise Forbidden()
+        
         parser = reqparse.RequestParser()
         parser.add_argument('inputs', type=dict, required=True, nullable=False, location='json')
         args = parser.parse_args()
@@ -286,6 +318,10 @@ class PublishedWorkflowApi(Resource):
         """
         Get published workflow
         """
+        # The role of the current user in the ta table must be admin, owner, or editor
+        if not current_user.is_editor:
+            raise Forbidden()
+        
         # fetch published workflow by app_model
         workflow_service = WorkflowService()
         workflow = workflow_service.get_published_workflow(app_model=app_model)
@@ -301,6 +337,10 @@ class PublishedWorkflowApi(Resource):
         """
         Publish workflow
         """
+        # The role of the current user in the ta table must be admin, owner, or editor
+        if not current_user.is_editor:
+            raise Forbidden()
+        
         workflow_service = WorkflowService()
         workflow = workflow_service.publish_workflow(app_model=app_model, account=current_user)
 
@@ -319,6 +359,10 @@ class DefaultBlockConfigsApi(Resource):
         """
         Get default block config
         """
+        # The role of the current user in the ta table must be admin, owner, or editor
+        if not current_user.is_editor:
+            raise Forbidden()
+        
         # Get default block configs
         workflow_service = WorkflowService()
         return workflow_service.get_default_block_configs()
@@ -333,6 +377,10 @@ class DefaultBlockConfigApi(Resource):
         """
         Get default block config
         """
+        # The role of the current user in the ta table must be admin, owner, or editor
+        if not current_user.is_editor:
+            raise Forbidden()
+        
         parser = reqparse.RequestParser()
         parser.add_argument('q', type=str, location='args')
         args = parser.parse_args()
@@ -363,6 +411,10 @@ class ConvertToWorkflowApi(Resource):
         Convert expert mode of chatbot app to workflow mode
         Convert Completion App to Workflow App
         """
+        # The role of the current user in the ta table must be admin, owner, or editor
+        if not current_user.is_editor:
+            raise Forbidden()
+        
         if request.data:
             parser = reqparse.RequestParser()
             parser.add_argument('name', type=str, required=False, nullable=True, location='json')

api/controllers/console/auth/data_source_bearer_auth.py

@@ -0,0 +1,73 @@
+from flask_login import current_user
+from flask_restful import Resource, reqparse
+from werkzeug.exceptions import Forbidden
+
+from controllers.console import api
+from controllers.console.auth.error import ApiKeyAuthFailedError
+from libs.login import login_required
+from services.auth.api_key_auth_service import ApiKeyAuthService
+
+from ..setup import setup_required
+from ..wraps import account_initialization_required
+
+
+class ApiKeyAuthDataSource(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def get(self):
+        data_source_api_key_bindings = ApiKeyAuthService.get_provider_auth_list(current_user.current_tenant_id)
+        if data_source_api_key_bindings:
+            return {
+                'sources': [{
+                                'id': data_source_api_key_binding.id,
+                                'category': data_source_api_key_binding.category,
+                                'provider': data_source_api_key_binding.provider,
+                                'disabled': data_source_api_key_binding.disabled,
+                                'created_at': int(data_source_api_key_binding.created_at.timestamp()),
+                                'updated_at': int(data_source_api_key_binding.updated_at.timestamp()),
+                            }
+                            for data_source_api_key_binding in
+                             data_source_api_key_bindings]
+            }
+        return {'sources': []}
+
+
+class ApiKeyAuthDataSourceBinding(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def post(self):
+        # The role of the current user in the table must be admin or owner
+        if not current_user.is_admin_or_owner:
+            raise Forbidden()
+        parser = reqparse.RequestParser()
+        parser.add_argument('category', type=str, required=True, nullable=False, location='json')
+        parser.add_argument('provider', type=str, required=True, nullable=False, location='json')
+        parser.add_argument('credentials', type=dict, required=True, nullable=False, location='json')
+        args = parser.parse_args()
+        ApiKeyAuthService.validate_api_key_auth_args(args)
+        try:
+            ApiKeyAuthService.create_provider_auth(current_user.current_tenant_id, args)
+        except Exception as e:
+            raise ApiKeyAuthFailedError(str(e))
+        return {'result': 'success'}, 200
+
+
+class ApiKeyAuthDataSourceBindingDelete(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def delete(self, binding_id):
+        # The role of the current user in the table must be admin or owner
+        if not current_user.is_admin_or_owner:
+            raise Forbidden()
+
+        ApiKeyAuthService.delete_provider_auth(current_user.current_tenant_id, binding_id)
+
+        return {'result': 'success'}, 200
+
+
+api.add_resource(ApiKeyAuthDataSource, '/api-key-auth/data-source')
+api.add_resource(ApiKeyAuthDataSourceBinding, '/api-key-auth/data-source/binding')
+api.add_resource(ApiKeyAuthDataSourceBindingDelete, '/api-key-auth/data-source/<uuid:binding_id>')

api/controllers/console/auth/error.py

@@ -0,0 +1,7 @@
+from libs.exception import BaseHTTPException
+
+
+class ApiKeyAuthFailedError(BaseHTTPException):
+    error_code = 'auth_failed'
+    description = "{message}"
+    code = 500

api/controllers/console/datasets/data_source.py

@@ -16,7 +16,7 @@ from extensions.ext_database import db
 from fields.data_source_fields import integrate_list_fields, integrate_notion_info_list_fields
 from libs.login import login_required
 from models.dataset import Document
-from models.source import DataSourceBinding
+from models.source import DataSourceOauthBinding
 from services.dataset_service import DatasetService, DocumentService
 from tasks.document_indexing_sync_task import document_indexing_sync_task
 
@@ -29,9 +29,9 @@ class DataSourceApi(Resource):
     @marshal_with(integrate_list_fields)
     def get(self):
         # get workspace data source integrates
-        data_source_integrates = db.session.query(DataSourceBinding).filter(
-            DataSourceBinding.tenant_id == current_user.current_tenant_id,
-            DataSourceBinding.disabled == False
+        data_source_integrates = db.session.query(DataSourceOauthBinding).filter(
+            DataSourceOauthBinding.tenant_id == current_user.current_tenant_id,
+            DataSourceOauthBinding.disabled == False
         ).all()
 
         base_url = request.url_root.rstrip('/')
@@ -71,7 +71,7 @@ class DataSourceApi(Resource):
     def patch(self, binding_id, action):
         binding_id = str(binding_id)
         action = str(action)
-        data_source_binding = DataSourceBinding.query.filter_by(
+        data_source_binding = DataSourceOauthBinding.query.filter_by(
             id=binding_id
         ).first()
         if data_source_binding is None:
@@ -124,7 +124,7 @@ class DataSourceNotionListApi(Resource):
                     data_source_info = json.loads(document.data_source_info)
                     exist_page_ids.append(data_source_info['notion_page_id'])
         # get all authorized pages
-        data_source_bindings = DataSourceBinding.query.filter_by(
+        data_source_bindings = DataSourceOauthBinding.query.filter_by(
             tenant_id=current_user.current_tenant_id,
             provider='notion',
             disabled=False
@@ -163,12 +163,12 @@ class DataSourceNotionApi(Resource):
     def get(self, workspace_id, page_id, page_type):
         workspace_id = str(workspace_id)
         page_id = str(page_id)
-        data_source_binding = DataSourceBinding.query.filter(
+        data_source_binding = DataSourceOauthBinding.query.filter(
             db.and_(
-                DataSourceBinding.tenant_id == current_user.current_tenant_id,
-                DataSourceBinding.provider == 'notion',
-                DataSourceBinding.disabled == False,
-                DataSourceBinding.source_info['workspace_id'] == f'"{workspace_id}"'
+                DataSourceOauthBinding.tenant_id == current_user.current_tenant_id,
+                DataSourceOauthBinding.provider == 'notion',
+                DataSourceOauthBinding.disabled == False,
+                DataSourceOauthBinding.source_info['workspace_id'] == f'"{workspace_id}"'
             )
         ).first()
         if not data_source_binding:

api/controllers/console/datasets/datasets.py

@@ -8,13 +8,14 @@ import services
 from controllers.console import api
 from controllers.console.apikey import api_key_fields, api_key_list
 from controllers.console.app.error import ProviderNotInitializeError
-from controllers.console.datasets.error import DatasetNameDuplicateError
+from controllers.console.datasets.error import DatasetInUseError, DatasetNameDuplicateError
 from controllers.console.setup import setup_required
 from controllers.console.wraps import account_initialization_required
 from core.errors.error import LLMBadRequestError, ProviderTokenNotInitError
 from core.indexing_runner import IndexingRunner
 from core.model_runtime.entities.model_entities import ModelType
 from core.provider_manager import ProviderManager
+from core.rag.datasource.vdb.vector_type import VectorType
 from core.rag.extractor.entity.extract_setting import ExtractSetting
 from extensions.ext_database import db
 from fields.app_fields import related_app_list
@@ -106,8 +107,8 @@ class DatasetListApi(Resource):
                             help='Invalid indexing technique.')
         args = parser.parse_args()
 
-        # The role of the current user in the ta table must be admin or owner
-        if not current_user.is_admin_or_owner:
+        # The role of the current user in the ta table must be admin, owner, or editor
+        if not current_user.is_editor:
             raise Forbidden()
 
         try:
@@ -194,8 +195,8 @@ class DatasetApi(Resource):
         parser.add_argument('retrieval_model', type=dict, location='json', help='Invalid retrieval model.')
         args = parser.parse_args()
 
-        # The role of the current user in the ta table must be admin or owner
-        if not current_user.is_admin_or_owner:
+        # The role of the current user in the ta table must be admin, owner, or editor
+        if not current_user.is_editor:
             raise Forbidden()
 
         dataset = DatasetService.update_dataset(
@@ -212,14 +213,17 @@ class DatasetApi(Resource):
     def delete(self, dataset_id):
         dataset_id_str = str(dataset_id)
 
-        # The role of the current user in the ta table must be admin or owner
-        if not current_user.is_admin_or_owner:
+        # The role of the current user in the ta table must be admin, owner, or editor
+        if not current_user.is_editor:
             raise Forbidden()
 
-        if DatasetService.delete_dataset(dataset_id_str, current_user):
-            return {'result': 'success'}, 204
-        else:
-            raise NotFound("Dataset not found.")
+        try:
+            if DatasetService.delete_dataset(dataset_id_str, current_user):
+                return {'result': 'success'}, 204
+            else:
+                raise NotFound("Dataset not found.")
+        except services.errors.dataset.DatasetInUseError:
+            raise DatasetInUseError()
 
 
 class DatasetQueryApi(Resource):
@@ -311,6 +315,22 @@ class DatasetIndexingEstimateApi(Resource):
                         document_model=args['doc_form']
                     )
                     extract_settings.append(extract_setting)
+        elif args['info_list']['data_source_type'] == 'website_crawl':
+            website_info_list = args['info_list']['website_info_list']
+            for url in website_info_list['urls']:
+                extract_setting = ExtractSetting(
+                    datasource_type="website_crawl",
+                    website_info={
+                        "provider": website_info_list['provider'],
+                        "job_id": website_info_list['job_id'],
+                        "url": url,
+                        "tenant_id": current_user.current_tenant_id,
+                        "mode": 'crawl',
+                        "only_main_content": website_info_list['only_main_content']
+                    },
+                    document_model=args['doc_form']
+                )
+                extract_settings.append(extract_setting)
         else:
             raise ValueError('Data source type not support')
         indexing_runner = IndexingRunner()
@@ -476,20 +496,21 @@ class DatasetRetrievalSettingApi(Resource):
     @account_initialization_required
     def get(self):
         vector_type = current_app.config['VECTOR_STORE']
-        if vector_type in {"milvus", "relyt", "pgvector", "pgvecto_rs"}:
-            return {
-                'retrieval_method': [
-                    'semantic_search'
-                ]
-            }
-        elif vector_type in {"qdrant", "weaviate"}:
-            return {
-                'retrieval_method': [
-                    'semantic_search', 'full_text_search', 'hybrid_search'
-                ]
-            }
-        else:
-            raise ValueError("Unsupported vector db type.")
+        match vector_type:
+            case VectorType.MILVUS | VectorType.RELYT | VectorType.PGVECTOR | VectorType.TIDB_VECTOR | VectorType.CHROMA | VectorType.TENCENT:
+                return {
+                    'retrieval_method': [
+                        'semantic_search'
+                    ]
+                }
+            case VectorType.QDRANT | VectorType.WEAVIATE:
+                return {
+                    'retrieval_method': [
+                        'semantic_search', 'full_text_search', 'hybrid_search'
+                    ]
+                }
+            case _:
+                raise ValueError(f"Unsupported vector db type {vector_type}.")
 
 
 class DatasetRetrievalSettingMockApi(Resource):
@@ -497,20 +518,23 @@ class DatasetRetrievalSettingMockApi(Resource):
     @login_required
     @account_initialization_required
     def get(self, vector_type):
-        if vector_type in {'milvus', 'relyt', 'pgvector'}:
-            return {
-                'retrieval_method': [
-                    'semantic_search'
-                ]
-            }
-        elif vector_type in {'qdrant', 'weaviate'}:
-            return {
-                'retrieval_method': [
-                    'semantic_search', 'full_text_search', 'hybrid_search'
-                ]
-            }
-        else:
-            raise ValueError("Unsupported vector db type.")
+        match vector_type:
+            case VectorType.MILVUS | VectorType.RELYT | VectorType.PGVECTOR | VectorType.TIDB_VECTOR | VectorType.CHROMA | VectorType.TENCEN:
+                return {
+                    'retrieval_method': [
+                        'semantic_search'
+                    ]
+                }
+            case VectorType.QDRANT | VectorType.WEAVIATE:
+                return {
+                    'retrieval_method': [
+                        'semantic_search', 'full_text_search', 'hybrid_search'
+                    ]
+                }
+            case _:
+                raise ValueError(f"Unsupported vector db type {vector_type}.")
+
+
 
 class DatasetErrorDocs(Resource):
     @setup_required

api/controllers/console/datasets/datasets_document.py

@@ -226,8 +226,8 @@ class DatasetDocumentListApi(Resource):
         if not dataset:
             raise NotFound('Dataset not found.')
 
-        # The role of the current user in the ta table must be admin or owner
-        if not current_user.is_admin_or_owner:
+        # The role of the current user in the ta table must be admin, owner, or editor
+        if not current_user.is_editor:
             raise Forbidden()
 
         try:
@@ -278,8 +278,8 @@ class DatasetInitApi(Resource):
     @marshal_with(dataset_and_document_fields)
     @cloud_edition_billing_resource_check('vector_space')
     def post(self):
-        # The role of the current user in the ta table must be admin or owner
-        if not current_user.is_admin_or_owner:
+        # The role of the current user in the ta table must be admin, owner, or editor
+        if not current_user.is_editor:
             raise Forbidden()
 
         parser = reqparse.RequestParser()
@@ -465,6 +465,20 @@ class DocumentBatchIndexingEstimateApi(DocumentResource):
                     document_model=document.doc_form
                 )
                 extract_settings.append(extract_setting)
+            elif document.data_source_type == 'website_crawl':
+                extract_setting = ExtractSetting(
+                    datasource_type="website_crawl",
+                    website_info={
+                        "provider": data_source_info['provider'],
+                        "job_id": data_source_info['job_id'],
+                        "url": data_source_info['url'],
+                        "tenant_id": current_user.current_tenant_id,
+                        "mode": data_source_info['mode'],
+                        "only_main_content": data_source_info['only_main_content']
+                    },
+                    document_model=document.doc_form
+                )
+                extract_settings.append(extract_setting)
 
             else:
                 raise ValueError('Data source type not support')
@@ -632,8 +646,8 @@ class DocumentProcessingApi(DocumentResource):
         document_id = str(document_id)
         document = self.get_document(dataset_id, document_id)
 
-        # The role of the current user in the ta table must be admin or owner
-        if not current_user.is_admin_or_owner:
+        # The role of the current user in the ta table must be admin, owner, or editor
+        if not current_user.is_editor:
             raise Forbidden()
 
         if action == "pause":
@@ -696,8 +710,8 @@ class DocumentMetadataApi(DocumentResource):
         doc_type = req_data.get('doc_type')
         doc_metadata = req_data.get('doc_metadata')
 
-        # The role of the current user in the ta table must be admin or owner
-        if not current_user.is_admin_or_owner:
+        # The role of the current user in the ta table must be admin, owner, or editor
+        if not current_user.is_editor:
             raise Forbidden()
 
         if doc_type is None or doc_metadata is None:
@@ -743,8 +757,8 @@ class DocumentStatusApi(DocumentResource):
 
         document = self.get_document(dataset_id, document_id)
 
-        # The role of the current user in the ta table must be admin or owner
-        if not current_user.is_admin_or_owner:
+        # The role of the current user in the ta table must be admin, owner, or editor
+        if not current_user.is_editor:
             raise Forbidden()
 
         indexing_cache_key = 'document_{}_indexing'.format(document.id)
@@ -952,6 +966,33 @@ class DocumentRenameApi(DocumentResource):
         return document
 
 
+class WebsiteDocumentSyncApi(DocumentResource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def get(self, dataset_id, document_id):
+        """sync website document."""
+        dataset_id = str(dataset_id)
+        dataset = DatasetService.get_dataset(dataset_id)
+        if not dataset:
+            raise NotFound('Dataset not found.')
+        document_id = str(document_id)
+        document = DocumentService.get_document(dataset.id, document_id)
+        if not document:
+            raise NotFound('Document not found.')
+        if document.tenant_id != current_user.current_tenant_id:
+            raise Forbidden('No permission.')
+        if document.data_source_type != 'website_crawl':
+            raise ValueError('Document is not a website document.')
+        # 403 if document is archived
+        if DocumentService.check_archived(document):
+            raise ArchivedDocumentImmutableError()
+        # sync document
+        DocumentService.sync_website_document(dataset_id, document)
+
+        return {'result': 'success'}, 200
+
+
 api.add_resource(GetProcessRuleApi, '/datasets/process-rule')
 api.add_resource(DatasetDocumentListApi,
                  '/datasets/<uuid:dataset_id>/documents')
@@ -980,3 +1021,5 @@ api.add_resource(DocumentRecoverApi, '/datasets/<uuid:dataset_id>/documents/<uui
 api.add_resource(DocumentRetryApi, '/datasets/<uuid:dataset_id>/retry')
 api.add_resource(DocumentRenameApi,
                  '/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/rename')
+
+api.add_resource(WebsiteDocumentSyncApi, '/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/website-sync')

api/controllers/console/datasets/datasets_segments.py

@@ -126,8 +126,8 @@ class DatasetDocumentSegmentApi(Resource):
             raise NotFound('Dataset not found.')
         # check user's model setting
         DatasetService.check_dataset_model_setting(dataset)
-        # The role of the current user in the ta table must be admin or owner
-        if not current_user.is_admin_or_owner:
+        # The role of the current user in the ta table must be admin, owner, or editor
+        if not current_user.is_editor:
             raise Forbidden()
 
         try:
@@ -302,8 +302,8 @@ class DatasetDocumentSegmentUpdateApi(Resource):
         ).first()
         if not segment:
             raise NotFound('Segment not found.')
-        # The role of the current user in the ta table must be admin or owner
-        if not current_user.is_admin_or_owner:
+        # The role of the current user in the ta table must be admin, owner, or editor
+        if not current_user.is_editor:
             raise Forbidden()
         try:
             DatasetService.check_dataset_permission(dataset, current_user)

api/controllers/console/datasets/error.py

@@ -71,3 +71,15 @@ class InvalidMetadataError(BaseHTTPException):
     error_code = 'invalid_metadata'
     description = "The metadata content is incorrect. Please check and verify."
     code = 400
+
+
+class WebsiteCrawlError(BaseHTTPException):
+    error_code = 'crawl_failed'
+    description = "{message}"
+    code = 500
+
+
+class DatasetInUseError(BaseHTTPException):
+    error_code = 'dataset_in_use'
+    description = "The dataset is being used by some apps. Please remove the dataset from the apps before deleting it."
+    code = 409

api/controllers/console/datasets/website.py

@@ -0,0 +1,49 @@
+from flask_restful import Resource, reqparse
+
+from controllers.console import api
+from controllers.console.datasets.error import WebsiteCrawlError
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required
+from libs.login import login_required
+from services.website_service import WebsiteService
+
+
+class WebsiteCrawlApi(Resource):
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def post(self):
+        parser = reqparse.RequestParser()
+        parser.add_argument('provider', type=str, choices=['firecrawl'],
+                            required=True, nullable=True, location='json')
+        parser.add_argument('url', type=str, required=True, nullable=True, location='json')
+        parser.add_argument('options', type=dict, required=True, nullable=True, location='json')
+        args = parser.parse_args()
+        WebsiteService.document_create_args_validate(args)
+        # crawl url
+        try:
+            result = WebsiteService.crawl_url(args)
+        except Exception as e:
+            raise WebsiteCrawlError(str(e))
+        return result, 200
+
+
+class WebsiteCrawlStatusApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def get(self, job_id: str):
+        parser = reqparse.RequestParser()
+        parser.add_argument('provider', type=str, choices=['firecrawl'], required=True, location='args')
+        args = parser.parse_args()
+        # get crawl status
+        try:
+            result = WebsiteService.get_crawl_status(job_id, args['provider'])
+        except Exception as e:
+            raise WebsiteCrawlError(str(e))
+        return result, 200
+
+
+api.add_resource(WebsiteCrawlApi, '/website/crawl')
+api.add_resource(WebsiteCrawlStatusApi, '/website/crawl/status/<string:job_id>')

api/controllers/console/feature.py

@@ -16,12 +16,12 @@ class FeatureApi(Resource):
     @account_initialization_required
     @cloud_utm_record
     def get(self):
-        return FeatureService.get_features(current_user.current_tenant_id).dict()
+        return FeatureService.get_features(current_user.current_tenant_id).model_dump()
 
 
 class SystemFeatureApi(Resource):
     def get(self):
-        return FeatureService.get_system_features().dict()
+        return FeatureService.get_system_features().model_dump()
 
 
 api.add_resource(FeatureApi, '/features')

api/controllers/console/tag/tags.py

@@ -35,8 +35,8 @@ class TagListApi(Resource):
     @login_required
     @account_initialization_required
     def post(self):
-        # The role of the current user in the ta table must be admin or owner
-        if not current_user.is_admin_or_owner:
+        # The role of the current user in the ta table must be admin, owner, or editor
+        if not current_user.is_editor:
             raise Forbidden()
 
         parser = reqparse.RequestParser()
@@ -67,8 +67,8 @@ class TagUpdateDeleteApi(Resource):
     @account_initialization_required
     def patch(self, tag_id):
         tag_id = str(tag_id)
-        # The role of the current user in the ta table must be admin or owner
-        if not current_user.is_admin_or_owner:
+        # The role of the current user in the ta table must be admin, owner, or editor
+        if not current_user.is_editor:
             raise Forbidden()
 
         parser = reqparse.RequestParser()
@@ -94,8 +94,8 @@ class TagUpdateDeleteApi(Resource):
     @account_initialization_required
     def delete(self, tag_id):
         tag_id = str(tag_id)
-        # The role of the current user in the ta table must be admin or owner
-        if not current_user.is_admin_or_owner:
+        # The role of the current user in the ta table must be admin, owner, or editor
+        if not current_user.is_editor:
             raise Forbidden()
 
         TagService.delete_tag(tag_id)
@@ -109,8 +109,8 @@ class TagBindingCreateApi(Resource):
     @login_required
     @account_initialization_required
     def post(self):
-        # The role of the current user in the ta table must be admin or owner
-        if not current_user.is_admin_or_owner:
+        # The role of the current user in the ta table must be admin, owner, or editor
+        if not current_user.is_editor:
             raise Forbidden()
 
         parser = reqparse.RequestParser()
@@ -134,8 +134,8 @@ class TagBindingDeleteApi(Resource):
     @login_required
     @account_initialization_required
     def post(self):
-        # The role of the current user in the ta table must be admin or owner
-        if not current_user.is_admin_or_owner:
+        # The role of the current user in the ta table must be admin, owner, or editor
+        if not current_user.is_editor:
             raise Forbidden()
 
         parser = reqparse.RequestParser()

api/controllers/console/workspace/members.py

@@ -43,7 +43,7 @@ class MemberInviteEmailApi(Resource):
         invitee_emails = args['emails']
         invitee_role = args['role']
         interface_language = args['language']
-        if invitee_role not in [TenantAccountRole.ADMIN, TenantAccountRole.NORMAL]:
+        if not TenantAccountRole.is_non_owner_role(invitee_role):
             return {'code': 'invalid-role', 'message': 'Invalid role'}, 400
 
         inviter = current_user
@@ -114,7 +114,7 @@ class MemberUpdateRoleApi(Resource):
         args = parser.parse_args()
         new_role = args['role']
 
-        if new_role not in ['admin', 'normal', 'owner']:
+        if not TenantAccountRole.is_valid_role(new_role):
             return {'code': 'invalid-role', 'message': 'Invalid role'}, 400
 
         member = Account.query.get(str(member_id))

api/controllers/console/workspace/models.py

@@ -11,7 +11,6 @@ from core.model_runtime.entities.model_entities import ModelType
 from core.model_runtime.errors.validate import CredentialsValidateFailedError
 from core.model_runtime.utils.encoders import jsonable_encoder
 from libs.login import login_required
-from models.account import TenantAccountRole
 from services.model_load_balancing_service import ModelLoadBalancingService
 from services.model_provider_service import ModelProviderService
 
@@ -43,6 +42,9 @@ class DefaultModelApi(Resource):
     @login_required
     @account_initialization_required
     def post(self):
+        if not current_user.is_admin_or_owner:
+            raise Forbidden()
+        
         parser = reqparse.RequestParser()
         parser.add_argument('model_settings', type=list, required=True, nullable=False, location='json')
         args = parser.parse_args()
@@ -96,7 +98,7 @@ class ModelProviderModelApi(Resource):
     @login_required
     @account_initialization_required
     def post(self, provider: str):
-        if not TenantAccountRole.is_privileged_role(current_user.current_tenant.current_role):
+        if not current_user.is_admin_or_owner:
             raise Forbidden()
 
         tenant_id = current_user.current_tenant_id
@@ -162,7 +164,7 @@ class ModelProviderModelApi(Resource):
     @login_required
     @account_initialization_required
     def delete(self, provider: str):
-        if not TenantAccountRole.is_privileged_role(current_user.current_tenant.current_role):
+        if not current_user.is_admin_or_owner:
             raise Forbidden()
 
         tenant_id = current_user.current_tenant_id

api/controllers/service_api/dataset/dataset.py

@@ -1,9 +1,10 @@
 from flask import request
 from flask_restful import marshal, reqparse
+from werkzeug.exceptions import NotFound
 
 import services.dataset_service
 from controllers.service_api import api
-from controllers.service_api.dataset.error import DatasetNameDuplicateError
+from controllers.service_api.dataset.error import DatasetInUseError, DatasetNameDuplicateError
 from controllers.service_api.wraps import DatasetApiResource
 from core.model_runtime.entities.model_entities import ModelType
 from core.provider_manager import ProviderManager
@@ -19,10 +20,12 @@ def _validate_name(name):
     return name
 
 
-class DatasetApi(DatasetApiResource):
-    """Resource for get datasets."""
+class DatasetListApi(DatasetApiResource):
+    """Resource for datasets."""
 
     def get(self, tenant_id):
+        """Resource for getting datasets."""
+
         page = request.args.get('page', default=1, type=int)
         limit = request.args.get('limit', default=20, type=int)
         provider = request.args.get('provider', default="vendor")
@@ -65,9 +68,9 @@ class DatasetApi(DatasetApiResource):
         }
         return response, 200
 
-    """Resource for datasets."""
 
     def post(self, tenant_id):
+        """Resource for creating datasets."""
         parser = reqparse.RequestParser()
         parser.add_argument('name', nullable=False, required=True,
                             help='type is required. Name must be between 1 to 40 characters.',
@@ -89,6 +92,34 @@ class DatasetApi(DatasetApiResource):
 
         return marshal(dataset, dataset_detail_fields), 200
 
+class DatasetApi(DatasetApiResource):
+    """Resource for dataset."""
 
-api.add_resource(DatasetApi, '/datasets')
+    def delete(self, _, dataset_id):
+        """
+        Deletes a dataset given its ID.
 
+        Args:
+            dataset_id (UUID): The ID of the dataset to be deleted.
+
+        Returns:
+            dict: A dictionary with a key 'result' and a value 'success' 
+                  if the dataset was successfully deleted. Omitted in HTTP response.
+            int: HTTP status code 204 indicating that the operation was successful.
+
+        Raises:
+            NotFound: If the dataset with the given ID does not exist.
+        """
+
+        dataset_id_str = str(dataset_id)
+
+        try:
+            if DatasetService.delete_dataset(dataset_id_str, current_user):
+                return {'result': 'success'}, 204
+            else:
+                raise NotFound("Dataset not found.")
+        except services.errors.dataset.DatasetInUseError:
+            raise DatasetInUseError()
+
+api.add_resource(DatasetListApi, '/datasets')
+api.add_resource(DatasetApi, '/datasets/<uuid:dataset_id>')

api/controllers/service_api/dataset/error.py

@@ -71,3 +71,9 @@ class InvalidMetadataError(BaseHTTPException):
     error_code = 'invalid_metadata'
     description = "The metadata content is incorrect. Please check and verify."
     code = 400
+
+
+class DatasetInUseError(BaseHTTPException):
+    error_code = 'dataset_in_use'
+    description = "The dataset is being used by some apps. Please remove the dataset from the apps before deleting it."
+    code = 409

api/controllers/web/audio.py

@@ -74,7 +74,7 @@ class TextApi(WebApiResource):
                 app_model=app_model,
                 text=request.form['text'],
                 end_user=end_user.external_user_id,
-                voice=request.form['voice'] if request.form.get('voice') else app_model.app_model_config.text_to_speech_dict.get('voice'),
+                voice=request.form['voice'] if request.form.get('voice') else None,
                 streaming=False
             )
 

api/controllers/web/feature.py

@@ -6,7 +6,7 @@ from services.feature_service import FeatureService
 
 class SystemFeatureApi(Resource):
     def get(self):
-        return FeatureService.get_system_features().dict()
+        return FeatureService.get_system_features().model_dump()
 
 
 api.add_resource(SystemFeatureApi, '/system-features')

api/core/agent/base_agent_runner.py

@@ -528,4 +528,3 @@ class BaseAgentRunner(AppRunner):
                 return UserPromptMessage(content=prompt_message_contents)
         else:
             return UserPromptMessage(content=message.query)
-         

api/core/agent/cot_agent_runner.py

@@ -32,9 +32,9 @@ class CotAgentRunner(BaseAgentRunner, ABC):
     _prompt_messages_tools: list[PromptMessage] = None
 
     def run(self, message: Message,
-        query: str,
-        inputs: dict[str, str],
-    ) -> Union[Generator, LLMResult]:
+            query: str,
+            inputs: dict[str, str],
+            ) -> Union[Generator, LLMResult]:
         """
         Run Cot agent application
         """
@@ -43,16 +43,17 @@ class CotAgentRunner(BaseAgentRunner, ABC):
         self._init_react_state(query)
 
         # check model mode
-        if 'Observation' not in app_generate_entity.model_config.stop:
-            if app_generate_entity.model_config.provider not in self._ignore_observation_providers:
-                app_generate_entity.model_config.stop.append('Observation')
+        if 'Observation' not in app_generate_entity.model_conf.stop:
+            if app_generate_entity.model_conf.provider not in self._ignore_observation_providers:
+                app_generate_entity.model_conf.stop.append('Observation')
 
         app_config = self.app_config
 
         # init instruction
         inputs = inputs or {}
         instruction = app_config.prompt_template.simple_prompt_template
-        self._instruction = self._fill_in_inputs_from_external_data_tools(instruction, inputs)
+        self._instruction = self._fill_in_inputs_from_external_data_tools(
+            instruction, inputs)
 
         iteration_step = 1
         max_iteration_steps = min(app_config.agent.max_iteration, 5) + 1
@@ -60,8 +61,6 @@ class CotAgentRunner(BaseAgentRunner, ABC):
         # convert tools into ModelRuntime Tool format
         tool_instances, self._prompt_messages_tools = self._init_prompt_tools()
 
-        prompt_messages = self._organize_prompt_messages()
-        
         function_call_state = True
         llm_usage = {
             'usage': None
@@ -109,9 +108,9 @@ class CotAgentRunner(BaseAgentRunner, ABC):
             # invoke model
             chunks: Generator[LLMResultChunk, None, None] = model_instance.invoke_llm(
                 prompt_messages=prompt_messages,
-                model_parameters=app_generate_entity.model_config.parameters,
+                model_parameters=app_generate_entity.model_conf.parameters,
                 tools=[],
-                stop=app_generate_entity.model_config.stop,
+                stop=app_generate_entity.model_conf.stop,
                 stream=True,
                 user=self.user_id,
                 callbacks=[],
@@ -120,9 +119,10 @@ class CotAgentRunner(BaseAgentRunner, ABC):
             # check llm result
             if not chunks:
                 raise ValueError("failed to invoke llm")
-            
+
             usage_dict = {}
-            react_chunks = CotAgentOutputParser.handle_react_stream_output(chunks, usage_dict)
+            react_chunks = CotAgentOutputParser.handle_react_stream_output(
+                chunks, usage_dict)
             scratchpad = AgentScratchpadUnit(
                 agent_response='',
                 thought='',
@@ -141,8 +141,8 @@ class CotAgentRunner(BaseAgentRunner, ABC):
                 if isinstance(chunk, AgentScratchpadUnit.Action):
                     action = chunk
                     # detect action
-                    scratchpad.agent_response += json.dumps(chunk.dict())
-                    scratchpad.action_str = json.dumps(chunk.dict())
+                    scratchpad.agent_response += json.dumps(chunk.model_dump())
+                    scratchpad.action_str = json.dumps(chunk.model_dump())
                     scratchpad.action = action
                 else:
                     scratchpad.agent_response += chunk
@@ -160,15 +160,16 @@ class CotAgentRunner(BaseAgentRunner, ABC):
                         )
                     )
 
-            scratchpad.thought = scratchpad.thought.strip() or 'I am thinking about how to help you'
+            scratchpad.thought = scratchpad.thought.strip(
+            ) or 'I am thinking about how to help you'
             self._agent_scratchpad.append(scratchpad)
-            
+
             # get llm usage
             if 'usage' in usage_dict:
                 increase_usage(llm_usage, usage_dict['usage'])
             else:
                 usage_dict['usage'] = LLMUsage.empty_usage()
-            
+
             self.save_agent_thought(
                 agent_thought=agent_thought,
                 tool_name=scratchpad.action.action_name if scratchpad.action else '',
@@ -182,7 +183,7 @@ class CotAgentRunner(BaseAgentRunner, ABC):
                 messages_ids=[],
                 llm_usage=usage_dict['usage']
             )
-            
+
             if not scratchpad.is_final():
                 self.queue_manager.publish(QueueAgentThoughtEvent(
                     agent_thought_id=agent_thought.id
@@ -196,7 +197,8 @@ class CotAgentRunner(BaseAgentRunner, ABC):
                     # action is final answer, return final answer directly
                     try:
                         if isinstance(scratchpad.action.action_input, dict):
-                            final_answer = json.dumps(scratchpad.action.action_input)
+                            final_answer = json.dumps(
+                                scratchpad.action.action_input)
                         elif isinstance(scratchpad.action.action_input, str):
                             final_answer = scratchpad.action.action_input
                         else:
@@ -207,7 +209,7 @@ class CotAgentRunner(BaseAgentRunner, ABC):
                     function_call_state = True
                     # action is tool call, invoke tool
                     tool_invoke_response, tool_invoke_meta = self._handle_invoke_action(
-                        action=scratchpad.action, 
+                        action=scratchpad.action,
                         tool_instances=tool_instances,
                         message_file_ids=message_file_ids
                     )
@@ -217,10 +219,13 @@ class CotAgentRunner(BaseAgentRunner, ABC):
                     self.save_agent_thought(
                         agent_thought=agent_thought,
                         tool_name=scratchpad.action.action_name,
-                        tool_input={scratchpad.action.action_name: scratchpad.action.action_input},
+                        tool_input={
+                            scratchpad.action.action_name: scratchpad.action.action_input},
                         thought=scratchpad.thought,
-                        observation={scratchpad.action.action_name: tool_invoke_response},
-                        tool_invoke_meta={scratchpad.action.action_name: tool_invoke_meta.to_dict()},
+                        observation={
+                            scratchpad.action.action_name: tool_invoke_response},
+                        tool_invoke_meta={
+                            scratchpad.action.action_name: tool_invoke_meta.to_dict()},
                         answer=scratchpad.agent_response,
                         messages_ids=message_file_ids,
                         llm_usage=usage_dict['usage']
@@ -232,7 +237,8 @@ class CotAgentRunner(BaseAgentRunner, ABC):
 
                 # update prompt tool message
                 for prompt_tool in self._prompt_messages_tools:
-                    self.update_prompt_message_tool(tool_instances[prompt_tool.name], prompt_tool)
+                    self.update_prompt_message_tool(
+                        tool_instances[prompt_tool.name], prompt_tool)
 
             iteration_step += 1
 
@@ -251,12 +257,12 @@ class CotAgentRunner(BaseAgentRunner, ABC):
 
         # save agent thought
         self.save_agent_thought(
-            agent_thought=agent_thought, 
+            agent_thought=agent_thought,
             tool_name='',
             tool_input={},
             tool_invoke_meta={},
             thought=final_answer,
-            observation={}, 
+            observation={},
             answer=final_answer,
             messages_ids=[]
         )
@@ -269,11 +275,12 @@ class CotAgentRunner(BaseAgentRunner, ABC):
             message=AssistantPromptMessage(
                 content=final_answer
             ),
-            usage=llm_usage['usage'] if llm_usage['usage'] else LLMUsage.empty_usage(),
+            usage=llm_usage['usage'] if llm_usage['usage'] else LLMUsage.empty_usage(
+            ),
             system_fingerprint=''
         )), PublishFrom.APPLICATION_MANAGER)
 
-    def _handle_invoke_action(self, action: AgentScratchpadUnit.Action, 
+    def _handle_invoke_action(self, action: AgentScratchpadUnit.Action,
                               tool_instances: dict[str, Tool],
                               message_file_ids: list[str]) -> tuple[str, ToolInvokeMeta]:
         """
@@ -290,7 +297,7 @@ class CotAgentRunner(BaseAgentRunner, ABC):
         if not tool_instance:
             answer = f"there is not a tool named {tool_call_name}"
             return answer, ToolInvokeMeta.error_instance(answer)
-        
+
         if isinstance(tool_call_args, str):
             try:
                 tool_call_args = json.loads(tool_call_args)
@@ -311,7 +318,8 @@ class CotAgentRunner(BaseAgentRunner, ABC):
         # publish files
         for message_file, save_as in message_files:
             if save_as:
-                self.variables_pool.set_file(tool_name=tool_call_name, value=message_file.id, name=save_as)
+                self.variables_pool.set_file(
+                    tool_name=tool_call_name, value=message_file.id, name=save_as)
 
             # publish message file
             self.queue_manager.publish(QueueMessageFileEvent(
@@ -342,7 +350,7 @@ class CotAgentRunner(BaseAgentRunner, ABC):
                 continue
 
         return instruction
-    
+
     def _init_react_state(self, query) -> None:
         """
         init agent scratchpad
@@ -350,7 +358,7 @@ class CotAgentRunner(BaseAgentRunner, ABC):
         self._query = query
         self._agent_scratchpad = []
         self._historic_prompt_messages = self._organize_historic_prompt_messages()
-    
+
     @abstractmethod
     def _organize_prompt_messages(self) -> list[PromptMessage]:
         """
@@ -379,54 +387,54 @@ class CotAgentRunner(BaseAgentRunner, ABC):
             organize historic prompt messages
         """
         result: list[PromptMessage] = []
-        scratchpad: list[AgentScratchpadUnit] = []
+        scratchpads: list[AgentScratchpadUnit] = []
         current_scratchpad: AgentScratchpadUnit = None
 
-        self.history_prompt_messages = AgentHistoryPromptTransform(
-            model_config=self.model_config,
-            prompt_messages=current_session_messages or [],
-            history_messages=self.history_prompt_messages,
-            memory=self.memory
-        ).get_prompt()
-
         for message in self.history_prompt_messages:
             if isinstance(message, AssistantPromptMessage):
-                current_scratchpad = AgentScratchpadUnit(
-                    agent_response=message.content,
-                    thought=message.content or 'I am thinking about how to help you',
-                    action_str='',
-                    action=None,
-                    observation=None,
-                )
+                if not current_scratchpad:
+                    current_scratchpad = AgentScratchpadUnit(
+                        agent_response=message.content,
+                        thought=message.content or 'I am thinking about how to help you',
+                        action_str='',
+                        action=None,
+                        observation=None,
+                    )
+                    scratchpads.append(current_scratchpad)
                 if message.tool_calls:
                     try:
                         current_scratchpad.action = AgentScratchpadUnit.Action(
                             action_name=message.tool_calls[0].function.name,
-                            action_input=json.loads(message.tool_calls[0].function.arguments)
+                            action_input=json.loads(
+                                message.tool_calls[0].function.arguments)
                         )
                         current_scratchpad.action_str = json.dumps(
                             current_scratchpad.action.to_dict()
                         )
                     except:
                         pass
-                
-                scratchpad.append(current_scratchpad)
             elif isinstance(message, ToolPromptMessage):
                 if current_scratchpad:
                     current_scratchpad.observation = message.content
             elif isinstance(message, UserPromptMessage):
+                if scratchpads:
+                    result.append(AssistantPromptMessage(
+                        content=self._format_assistant_message(scratchpads)
+                    ))
+                    scratchpads = []
+                    current_scratchpad = None
+
                 result.append(message)
 
-                if scratchpad:
-                    result.append(AssistantPromptMessage(
-                        content=self._format_assistant_message(scratchpad)
-                    ))
-
-                scratchpad = []
-
-        if scratchpad:
+        if scratchpads:
             result.append(AssistantPromptMessage(
-                content=self._format_assistant_message(scratchpad)
+                content=self._format_assistant_message(scratchpads)
             ))
-        
-        return result
+
+        historic_prompts = AgentHistoryPromptTransform(
+            model_config=self.model_config,
+            prompt_messages=current_session_messages or [],
+            history_messages=result,
+            memory=self.memory
+        ).get_prompt()
+        return historic_prompts

api/core/agent/cot_chat_agent_runner.py

@@ -5,6 +5,7 @@ from core.model_runtime.entities.message_entities import (
     AssistantPromptMessage,
     PromptMessage,
     SystemPromptMessage,
+    TextPromptMessageContent,
     UserPromptMessage,
 )
 from core.model_runtime.utils.encoders import jsonable_encoder
@@ -25,6 +26,21 @@ class CotChatAgentRunner(CotAgentRunner):
 
         return SystemPromptMessage(content=system_prompt)
 
+    def _organize_user_query(self, query,  prompt_messages: list[PromptMessage] = None) -> list[PromptMessage]:
+        """
+        Organize user query
+        """
+        if self.files:
+            prompt_message_contents = [TextPromptMessageContent(data=query)]
+            for file_obj in self.files:
+                prompt_message_contents.append(file_obj.prompt_message_content)
+
+            prompt_messages.append(UserPromptMessage(content=prompt_message_contents))
+        else:
+            prompt_messages.append(UserPromptMessage(content=query))
+
+        return prompt_messages
+
     def _organize_prompt_messages(self) -> list[PromptMessage]:
         """
         Organize 
@@ -51,27 +67,27 @@ class CotChatAgentRunner(CotAgentRunner):
             assistant_messages = [assistant_message]
 
         # query messages
-        query_messages = UserPromptMessage(content=self._query)
+        query_messages = self._organize_user_query(self._query, [])
 
         if assistant_messages:
             # organize historic prompt messages
             historic_messages = self._organize_historic_prompt_messages([
                 system_message,
-                query_messages,
+                *query_messages,
                 *assistant_messages,
                 UserPromptMessage(content='continue')
-            ])            
+            ])
             messages = [
                 system_message,
                 *historic_messages,
-                query_messages,
+                *query_messages,
                 *assistant_messages,
                 UserPromptMessage(content='continue')
             ]
         else:
             # organize historic prompt messages
-            historic_messages = self._organize_historic_prompt_messages([system_message, query_messages])
-            messages = [system_message, *historic_messages, query_messages]
+            historic_messages = self._organize_historic_prompt_messages([system_message, *query_messages])
+            messages = [system_message, *historic_messages, *query_messages]
 
         # join all messages
-        return messages
+        return messages

api/core/agent/fc_agent_runner.py

@@ -84,9 +84,9 @@ class FunctionCallAgentRunner(BaseAgentRunner):
             # invoke model
             chunks: Union[Generator[LLMResultChunk, None, None], LLMResult] = model_instance.invoke_llm(
                 prompt_messages=prompt_messages,
-                model_parameters=app_generate_entity.model_config.parameters,
+                model_parameters=app_generate_entity.model_conf.parameters,
                 tools=prompt_messages_tools,
-                stop=app_generate_entity.model_config.stop,
+                stop=app_generate_entity.model_conf.stop,
                 stream=self.stream_tool_call,
                 user=self.user_id,
                 callbacks=[],

api/core/agent/output_parser/cot_output_parser.py

@@ -17,6 +17,10 @@ class CotAgentOutputParser:
                 action_name = None
                 action_input = None
 
+                # cohere always returns a list
+                if isinstance(action, list) and len(action) == 1:
+                    action = action[0]
+
                 for key, value in action.items():
                     if 'input' in key.lower():
                         action_input = value

api/core/app/apps/advanced_chat/generate_task_pipeline.py

@@ -107,8 +107,8 @@ class AdvancedChatAppGenerateTaskPipeline(BasedGenerateTaskPipeline, WorkflowCyc
             usage=LLMUsage.empty_usage()
         )
 
-        self._stream_generate_routes = self._get_stream_generate_routes()
         self._iteration_nested_relations = self._get_iteration_nested_relations(self._workflow.graph_dict)
+        self._stream_generate_routes = self._get_stream_generate_routes()
         self._conversation_name_generate_thread = None
 
     def process(self) -> Union[ChatbotAppBlockingResponse, Generator[ChatbotAppStreamResponse, None, None]]:
@@ -410,6 +410,18 @@ class AdvancedChatAppGenerateTaskPipeline(BasedGenerateTaskPipeline, WorkflowCyc
                 ingoing_edges.append(edge)
 
         if not ingoing_edges:
+            # check if it's the first node in the iteration
+            target_node = next((node for node in nodes if node.get('id') == target_node_id), None)
+            if not target_node:
+                return []
+            
+            node_iteration_id = target_node.get('data', {}).get('iteration_id')
+            # get iteration start node id
+            for node in nodes:
+                if node.get('id') == node_iteration_id:
+                    if node.get('data', {}).get('start_node_id') == target_node_id:
+                        return [target_node_id]
+                    
             return []
 
         start_node_ids = []
@@ -514,6 +526,7 @@ class AdvancedChatAppGenerateTaskPipeline(BasedGenerateTaskPipeline, WorkflowCyc
                 self._task_state.answer += route_chunk.text
                 yield self._message_to_stream_response(route_chunk.text, self._message.id)
             else:
+                value = None
                 route_chunk = cast(VarGenerateRouteChunk, route_chunk)
                 value_selector = route_chunk.value_selector
                 if not value_selector:
@@ -525,6 +538,20 @@ class AdvancedChatAppGenerateTaskPipeline(BasedGenerateTaskPipeline, WorkflowCyc
                 if route_chunk_node_id == 'sys':
                     # system variable
                     value = self._workflow_system_variables.get(SystemVariable.value_of(value_selector[1]))
+                elif route_chunk_node_id in self._iteration_nested_relations:
+                    # it's a iteration variable
+                    if not self._iteration_state or route_chunk_node_id not in self._iteration_state.current_iterations:
+                        continue
+                    iteration_state = self._iteration_state.current_iterations[route_chunk_node_id]
+                    iterator = iteration_state.inputs
+                    if not iterator:
+                        continue
+                    iterator_selector = iterator.get('iterator_selector', [])
+                    if value_selector[1] == 'index':
+                        value = iteration_state.current_index
+                    elif value_selector[1] == 'item':
+                        value = iterator_selector[iteration_state.current_index] if iteration_state.current_index < len(
+                            iterator_selector) else None
                 else:
                     # check chunk node id is before current node id or equal to current node id
                     if route_chunk_node_id not in self._task_state.ran_node_execution_infos:
@@ -554,7 +581,7 @@ class AdvancedChatAppGenerateTaskPipeline(BasedGenerateTaskPipeline, WorkflowCyc
                         else:
                             value = value.get(key)
 
-                if value:
+                if value is not None:
                     text = ''
                     if isinstance(value, str | int | float):
                         text = str(value)

api/core/app/apps/agent_chat/app_generator.py

@@ -107,7 +107,7 @@ class AgentChatAppGenerator(MessageBasedAppGenerator):
         application_generate_entity = AgentChatAppGenerateEntity(
             task_id=str(uuid.uuid4()),
             app_config=app_config,
-            model_config=ModelConfigConverter.convert(app_config),
+            model_conf=ModelConfigConverter.convert(app_config),
             conversation_id=conversation.id if conversation else None,
             inputs=conversation.inputs if conversation else self._get_cleaned_inputs(inputs, app_config),
             query=query,

api/core/app/apps/agent_chat/app_runner.py

@@ -58,7 +58,7 @@ class AgentChatAppRunner(AppRunner):
         # Not Include: memory, external data, dataset context
         self.get_pre_calculate_rest_tokens(
             app_record=app_record,
-            model_config=application_generate_entity.model_config,
+            model_config=application_generate_entity.model_conf,
             prompt_template_entity=app_config.prompt_template,
             inputs=inputs,
             files=files,
@@ -69,8 +69,8 @@ class AgentChatAppRunner(AppRunner):
         if application_generate_entity.conversation_id:
             # get memory of conversation (read-only)
             model_instance = ModelInstance(
-                provider_model_bundle=application_generate_entity.model_config.provider_model_bundle,
-                model=application_generate_entity.model_config.model
+                provider_model_bundle=application_generate_entity.model_conf.provider_model_bundle,
+                model=application_generate_entity.model_conf.model
             )
 
             memory = TokenBufferMemory(
@@ -83,7 +83,7 @@ class AgentChatAppRunner(AppRunner):
         #          memory(optional)
         prompt_messages, _ = self.organize_prompt_messages(
             app_record=app_record,
-            model_config=application_generate_entity.model_config,
+            model_config=application_generate_entity.model_conf,
             prompt_template_entity=app_config.prompt_template,
             inputs=inputs,
             files=files,
@@ -152,7 +152,7 @@ class AgentChatAppRunner(AppRunner):
         #          memory(optional), external data, dataset context(optional)
         prompt_messages, _ = self.organize_prompt_messages(
             app_record=app_record,
-            model_config=application_generate_entity.model_config,
+            model_config=application_generate_entity.model_conf,
             prompt_template_entity=app_config.prompt_template,
             inputs=inputs,
             files=files,
@@ -182,12 +182,12 @@ class AgentChatAppRunner(AppRunner):
 
         # init model instance
         model_instance = ModelInstance(
-            provider_model_bundle=application_generate_entity.model_config.provider_model_bundle,
-            model=application_generate_entity.model_config.model
+            provider_model_bundle=application_generate_entity.model_conf.provider_model_bundle,
+            model=application_generate_entity.model_conf.model
         )
         prompt_message, _ = self.organize_prompt_messages(
             app_record=app_record,
-            model_config=application_generate_entity.model_config,
+            model_config=application_generate_entity.model_conf,
             prompt_template_entity=app_config.prompt_template,
             inputs=inputs,
             files=files,
@@ -225,7 +225,7 @@ class AgentChatAppRunner(AppRunner):
             application_generate_entity=application_generate_entity,
             conversation=conversation,
             app_config=app_config,
-            model_config=application_generate_entity.model_config,
+            model_config=application_generate_entity.model_conf,
             config=agent_entity,
             queue_manager=queue_manager,
             message=message,

api/core/app/apps/base_app_queue_manager.py

@@ -5,6 +5,7 @@ from collections.abc import Generator
 from enum import Enum
 from typing import Any
 
+from flask import current_app
 from sqlalchemy.orm import DeclarativeMeta
 
 from core.app.entities.app_invoke_entities import InvokeFrom
@@ -46,8 +47,8 @@ class AppQueueManager:
         Listen to queue
         :return:
         """
-        # wait for 10 minutes to stop listen
-        listen_timeout = 600
+        # wait for APP_MAX_EXECUTION_TIME seconds to stop listen
+        listen_timeout = current_app.config.get("APP_MAX_EXECUTION_TIME")
         start_time = time.time()
         last_ping_time = 0
 
@@ -99,7 +100,7 @@ class AppQueueManager:
         :param pub_from:
         :return:
         """
-        self._check_for_sqlalchemy_models(event.dict())
+        self._check_for_sqlalchemy_models(event.model_dump())
         self._publish(event, pub_from)
 
     @abstractmethod

api/core/app/apps/base_app_runner.py

@@ -218,7 +218,7 @@ class AppRunner:
             index = 0
             for token in text:
                 chunk = LLMResultChunk(
-                    model=app_generate_entity.model_config.model,
+                    model=app_generate_entity.model_conf.model,
                     prompt_messages=prompt_messages,
                     delta=LLMResultChunkDelta(
                         index=index,
@@ -237,7 +237,7 @@ class AppRunner:
         queue_manager.publish(
             QueueMessageEndEvent(
                 llm_result=LLMResult(
-                    model=app_generate_entity.model_config.model,
+                    model=app_generate_entity.model_conf.model,
                     prompt_messages=prompt_messages,
                     message=AssistantPromptMessage(content=text),
                     usage=usage if usage else LLMUsage.empty_usage()

api/core/app/apps/chat/app_generator.py

@@ -104,7 +104,7 @@ class ChatAppGenerator(MessageBasedAppGenerator):
         application_generate_entity = ChatAppGenerateEntity(
             task_id=str(uuid.uuid4()),
             app_config=app_config,
-            model_config=ModelConfigConverter.convert(app_config),
+            model_conf=ModelConfigConverter.convert(app_config),
             conversation_id=conversation.id if conversation else None,
             inputs=conversation.inputs if conversation else self._get_cleaned_inputs(inputs, app_config),
             query=query,

api/core/app/apps/chat/app_runner.py

@@ -54,7 +54,7 @@ class ChatAppRunner(AppRunner):
         # Not Include: memory, external data, dataset context
         self.get_pre_calculate_rest_tokens(
             app_record=app_record,
-            model_config=application_generate_entity.model_config,
+            model_config=application_generate_entity.model_conf,
             prompt_template_entity=app_config.prompt_template,
             inputs=inputs,
             files=files,
@@ -65,8 +65,8 @@ class ChatAppRunner(AppRunner):
         if application_generate_entity.conversation_id:
             # get memory of conversation (read-only)
             model_instance = ModelInstance(
-                provider_model_bundle=application_generate_entity.model_config.provider_model_bundle,
-                model=application_generate_entity.model_config.model
+                provider_model_bundle=application_generate_entity.model_conf.provider_model_bundle,
+                model=application_generate_entity.model_conf.model
             )
 
             memory = TokenBufferMemory(
@@ -79,7 +79,7 @@ class ChatAppRunner(AppRunner):
         #          memory(optional)
         prompt_messages, stop = self.organize_prompt_messages(
             app_record=app_record,
-            model_config=application_generate_entity.model_config,
+            model_config=application_generate_entity.model_conf,
             prompt_template_entity=app_config.prompt_template,
             inputs=inputs,
             files=files,
@@ -159,7 +159,7 @@ class ChatAppRunner(AppRunner):
                 app_id=app_record.id,
                 user_id=application_generate_entity.user_id,
                 tenant_id=app_record.tenant_id,
-                model_config=application_generate_entity.model_config,
+                model_config=application_generate_entity.model_conf,
                 config=app_config.dataset,
                 query=query,
                 invoke_from=application_generate_entity.invoke_from,
@@ -173,7 +173,7 @@ class ChatAppRunner(AppRunner):
         #          memory(optional), external data, dataset context(optional)
         prompt_messages, stop = self.organize_prompt_messages(
             app_record=app_record,
-            model_config=application_generate_entity.model_config,
+            model_config=application_generate_entity.model_conf,
             prompt_template_entity=app_config.prompt_template,
             inputs=inputs,
             files=files,
@@ -194,21 +194,21 @@ class ChatAppRunner(AppRunner):
 
         # Re-calculate the max tokens if sum(prompt_token +  max_tokens) over model token limit
         self.recalc_llm_max_tokens(
-            model_config=application_generate_entity.model_config,
+            model_config=application_generate_entity.model_conf,
             prompt_messages=prompt_messages
         )
 
         # Invoke model
         model_instance = ModelInstance(
-            provider_model_bundle=application_generate_entity.model_config.provider_model_bundle,
-            model=application_generate_entity.model_config.model
+            provider_model_bundle=application_generate_entity.model_conf.provider_model_bundle,
+            model=application_generate_entity.model_conf.model
         )
 
         db.session.close()
 
         invoke_result = model_instance.invoke_llm(
             prompt_messages=prompt_messages,
-            model_parameters=application_generate_entity.model_config.parameters,
+            model_parameters=application_generate_entity.model_conf.parameters,
             stop=stop,
             stream=application_generate_entity.stream,
             user=application_generate_entity.user_id,

api/core/app/apps/completion/app_generator.py

@@ -98,7 +98,7 @@ class CompletionAppGenerator(MessageBasedAppGenerator):
         application_generate_entity = CompletionAppGenerateEntity(
             task_id=str(uuid.uuid4()),
             app_config=app_config,
-            model_config=ModelConfigConverter.convert(app_config),
+            model_conf=ModelConfigConverter.convert(app_config),
             inputs=self._get_cleaned_inputs(inputs, app_config),
             query=query,
             files=file_objs,
@@ -257,7 +257,7 @@ class CompletionAppGenerator(MessageBasedAppGenerator):
         application_generate_entity = CompletionAppGenerateEntity(
             task_id=str(uuid.uuid4()),
             app_config=app_config,
-            model_config=ModelConfigConverter.convert(app_config),
+            model_conf=ModelConfigConverter.convert(app_config),
             inputs=message.inputs,
             query=message.query,
             files=file_objs,

api/core/app/apps/completion/app_runner.py

@@ -50,7 +50,7 @@ class CompletionAppRunner(AppRunner):
         # Not Include: memory, external data, dataset context
         self.get_pre_calculate_rest_tokens(
             app_record=app_record,
-            model_config=application_generate_entity.model_config,
+            model_config=application_generate_entity.model_conf,
             prompt_template_entity=app_config.prompt_template,
             inputs=inputs,
             files=files,
@@ -61,7 +61,7 @@ class CompletionAppRunner(AppRunner):
         # Include: prompt template, inputs, query(optional), files(optional)
         prompt_messages, stop = self.organize_prompt_messages(
             app_record=app_record,
-            model_config=application_generate_entity.model_config,
+            model_config=application_generate_entity.model_conf,
             prompt_template_entity=app_config.prompt_template,
             inputs=inputs,
             files=files,
@@ -119,7 +119,7 @@ class CompletionAppRunner(AppRunner):
                 app_id=app_record.id,
                 user_id=application_generate_entity.user_id,
                 tenant_id=app_record.tenant_id,
-                model_config=application_generate_entity.model_config,
+                model_config=application_generate_entity.model_conf,
                 config=dataset_config,
                 query=query,
                 invoke_from=application_generate_entity.invoke_from,
@@ -132,7 +132,7 @@ class CompletionAppRunner(AppRunner):
         #          memory(optional), external data, dataset context(optional)
         prompt_messages, stop = self.organize_prompt_messages(
             app_record=app_record,
-            model_config=application_generate_entity.model_config,
+            model_config=application_generate_entity.model_conf,
             prompt_template_entity=app_config.prompt_template,
             inputs=inputs,
             files=files,
@@ -152,21 +152,21 @@ class CompletionAppRunner(AppRunner):
 
         # Re-calculate the max tokens if sum(prompt_token +  max_tokens) over model token limit
         self.recalc_llm_max_tokens(
-            model_config=application_generate_entity.model_config,
+            model_config=application_generate_entity.model_conf,
             prompt_messages=prompt_messages
         )
 
         # Invoke model
         model_instance = ModelInstance(
-            provider_model_bundle=application_generate_entity.model_config.provider_model_bundle,
-            model=application_generate_entity.model_config.model
+            provider_model_bundle=application_generate_entity.model_conf.provider_model_bundle,
+            model=application_generate_entity.model_conf.model
         )
 
         db.session.close()
 
         invoke_result = model_instance.invoke_llm(
             prompt_messages=prompt_messages,
-            model_parameters=application_generate_entity.model_config.parameters,
+            model_parameters=application_generate_entity.model_conf.parameters,
             stop=stop,
             stream=application_generate_entity.stream,
             user=application_generate_entity.user_id,

api/core/app/apps/message_based_app_generator.py

@@ -158,8 +158,8 @@ class MessageBasedAppGenerator(BaseAppGenerator):
             model_id = None
         else:
             app_model_config_id = app_config.app_model_config_id
-            model_provider = application_generate_entity.model_config.provider
-            model_id = application_generate_entity.model_config.model
+            model_provider = application_generate_entity.model_conf.provider
+            model_id = application_generate_entity.model_conf.model
             override_model_configs = None
             if app_config.app_model_config_from == EasyUIBasedAppModelConfigFrom.ARGS \
                     and app_config.app_mode in [AppMode.AGENT_CHAT, AppMode.CHAT, AppMode.COMPLETION]:

api/core/app/entities/app_invoke_entities.py

@@ -1,7 +1,7 @@
 from enum import Enum
 from typing import Any, Optional
 
-from pydantic import BaseModel
+from pydantic import BaseModel, ConfigDict
 
 from core.app.app_config.entities import AppConfig, EasyUIBasedAppConfig, WorkflowUIBasedAppConfig
 from core.entities.provider_configuration import ProviderModelBundle
@@ -62,6 +62,9 @@ class ModelConfigWithCredentialsEntity(BaseModel):
     parameters: dict[str, Any] = {}
     stop: list[str] = []
 
+    # pydantic configs
+    model_config = ConfigDict(protected_namespaces=())
+
 
 class AppGenerateEntity(BaseModel):
     """
@@ -93,10 +96,13 @@ class EasyUIBasedAppGenerateEntity(AppGenerateEntity):
     """
     # app config
     app_config: EasyUIBasedAppConfig
-    model_config: ModelConfigWithCredentialsEntity
+    model_conf: ModelConfigWithCredentialsEntity
 
     query: Optional[str] = None
 
+    # pydantic configs
+    model_config = ConfigDict(protected_namespaces=())
+
 
 class ChatAppGenerateEntity(EasyUIBasedAppGenerateEntity):
     """

api/core/app/entities/queue_entities.py

@@ -1,14 +1,14 @@
 from enum import Enum
 from typing import Any, Optional
 
-from pydantic import BaseModel, validator
+from pydantic import BaseModel, field_validator
 
 from core.model_runtime.entities.llm_entities import LLMResult, LLMResultChunk
 from core.workflow.entities.base_node_data_entities import BaseNodeData
 from core.workflow.entities.node_entities import NodeType
 
 
-class QueueEvent(Enum):
+class QueueEvent(str, Enum):
     """
     QueueEvent enum
     """
@@ -47,14 +47,14 @@ class QueueLLMChunkEvent(AppQueueEvent):
     """
     QueueLLMChunkEvent entity
     """
-    event = QueueEvent.LLM_CHUNK
+    event: QueueEvent = QueueEvent.LLM_CHUNK
     chunk: LLMResultChunk
 
 class QueueIterationStartEvent(AppQueueEvent):
     """
     QueueIterationStartEvent entity
     """
-    event = QueueEvent.ITERATION_START
+    event: QueueEvent = QueueEvent.ITERATION_START
     node_id: str
     node_type: NodeType
     node_data: BaseNodeData
@@ -68,16 +68,17 @@ class QueueIterationNextEvent(AppQueueEvent):
     """
     QueueIterationNextEvent entity
     """
-    event = QueueEvent.ITERATION_NEXT
+    event: QueueEvent = QueueEvent.ITERATION_NEXT
 
     index: int
     node_id: str
     node_type: NodeType
 
     node_run_index: int
-    output: Optional[Any] # output for the current iteration
+    output: Optional[Any] = None # output for the current iteration
 
-    @validator('output', pre=True, always=True)
+    @field_validator('output', mode='before')
+    @classmethod
     def set_output(cls, v):
         """
         Set output
@@ -92,7 +93,7 @@ class QueueIterationCompletedEvent(AppQueueEvent):
     """
     QueueIterationCompletedEvent entity
     """
-    event = QueueEvent.ITERATION_COMPLETED
+    event:QueueEvent = QueueEvent.ITERATION_COMPLETED
 
     node_id: str
     node_type: NodeType
@@ -104,7 +105,7 @@ class QueueTextChunkEvent(AppQueueEvent):
     """
     QueueTextChunkEvent entity
     """
-    event = QueueEvent.TEXT_CHUNK
+    event: QueueEvent = QueueEvent.TEXT_CHUNK
     text: str
     metadata: Optional[dict] = None
 
@@ -113,7 +114,7 @@ class QueueAgentMessageEvent(AppQueueEvent):
     """
     QueueMessageEvent entity
     """
-    event = QueueEvent.AGENT_MESSAGE
+    event: QueueEvent = QueueEvent.AGENT_MESSAGE
     chunk: LLMResultChunk
 
     
@@ -121,7 +122,7 @@ class QueueMessageReplaceEvent(AppQueueEvent):
     """
     QueueMessageReplaceEvent entity
     """
-    event = QueueEvent.MESSAGE_REPLACE
+    event: QueueEvent = QueueEvent.MESSAGE_REPLACE
     text: str
 
 
@@ -129,7 +130,7 @@ class QueueRetrieverResourcesEvent(AppQueueEvent):
     """
     QueueRetrieverResourcesEvent entity
     """
-    event = QueueEvent.RETRIEVER_RESOURCES
+    event: QueueEvent = QueueEvent.RETRIEVER_RESOURCES
     retriever_resources: list[dict]
 
 
@@ -137,7 +138,7 @@ class QueueAnnotationReplyEvent(AppQueueEvent):
     """
     QueueAnnotationReplyEvent entity
     """
-    event = QueueEvent.ANNOTATION_REPLY
+    event: QueueEvent = QueueEvent.ANNOTATION_REPLY
     message_annotation_id: str
 
 
@@ -145,7 +146,7 @@ class QueueMessageEndEvent(AppQueueEvent):
     """
     QueueMessageEndEvent entity
     """
-    event = QueueEvent.MESSAGE_END
+    event: QueueEvent = QueueEvent.MESSAGE_END
     llm_result: Optional[LLMResult] = None
 
 
@@ -153,28 +154,28 @@ class QueueAdvancedChatMessageEndEvent(AppQueueEvent):
     """
     QueueAdvancedChatMessageEndEvent entity
     """
-    event = QueueEvent.ADVANCED_CHAT_MESSAGE_END
+    event: QueueEvent = QueueEvent.ADVANCED_CHAT_MESSAGE_END
 
 
 class QueueWorkflowStartedEvent(AppQueueEvent):
     """
     QueueWorkflowStartedEvent entity
     """
-    event = QueueEvent.WORKFLOW_STARTED
+    event: QueueEvent = QueueEvent.WORKFLOW_STARTED
 
 
 class QueueWorkflowSucceededEvent(AppQueueEvent):
     """
     QueueWorkflowSucceededEvent entity
     """
-    event = QueueEvent.WORKFLOW_SUCCEEDED
+    event: QueueEvent = QueueEvent.WORKFLOW_SUCCEEDED
 
 
 class QueueWorkflowFailedEvent(AppQueueEvent):
     """
     QueueWorkflowFailedEvent entity
     """
-    event = QueueEvent.WORKFLOW_FAILED
+    event: QueueEvent = QueueEvent.WORKFLOW_FAILED
     error: str
 
 
@@ -182,7 +183,7 @@ class QueueNodeStartedEvent(AppQueueEvent):
     """
     QueueNodeStartedEvent entity
     """
-    event = QueueEvent.NODE_STARTED
+    event: QueueEvent = QueueEvent.NODE_STARTED
 
     node_id: str
     node_type: NodeType
@@ -195,7 +196,7 @@ class QueueNodeSucceededEvent(AppQueueEvent):
     """
     QueueNodeSucceededEvent entity
     """
-    event = QueueEvent.NODE_SUCCEEDED
+    event: QueueEvent = QueueEvent.NODE_SUCCEEDED
 
     node_id: str
     node_type: NodeType
@@ -213,7 +214,7 @@ class QueueNodeFailedEvent(AppQueueEvent):
     """
     QueueNodeFailedEvent entity
     """
-    event = QueueEvent.NODE_FAILED
+    event: QueueEvent = QueueEvent.NODE_FAILED
 
     node_id: str
     node_type: NodeType
@@ -230,7 +231,7 @@ class QueueAgentThoughtEvent(AppQueueEvent):
     """
     QueueAgentThoughtEvent entity
     """
-    event = QueueEvent.AGENT_THOUGHT
+    event: QueueEvent = QueueEvent.AGENT_THOUGHT
     agent_thought_id: str
 
 
@@ -238,7 +239,7 @@ class QueueMessageFileEvent(AppQueueEvent):
     """
     QueueAgentThoughtEvent entity
     """
-    event = QueueEvent.MESSAGE_FILE
+    event: QueueEvent = QueueEvent.MESSAGE_FILE
     message_file_id: str
 
 
@@ -246,15 +247,15 @@ class QueueErrorEvent(AppQueueEvent):
     """
     QueueErrorEvent entity
     """
-    event = QueueEvent.ERROR
-    error: Any
+    event: QueueEvent = QueueEvent.ERROR
+    error: Any = None
 
 
 class QueuePingEvent(AppQueueEvent):
     """
     QueuePingEvent entity
     """
-    event = QueueEvent.PING
+    event: QueueEvent = QueueEvent.PING
 
 
 class QueueStopEvent(AppQueueEvent):
@@ -270,7 +271,7 @@ class QueueStopEvent(AppQueueEvent):
         OUTPUT_MODERATION = "output-moderation"
         INPUT_MODERATION = "input-moderation"
 
-    event = QueueEvent.STOP
+    event: QueueEvent = QueueEvent.STOP
     stopped_by: StopBy
 
 

api/core/app/entities/task_entities.py

@@ -1,7 +1,7 @@
 from enum import Enum
 from typing import Any, Optional
 
-from pydantic import BaseModel
+from pydantic import BaseModel, ConfigDict
 
 from core.model_runtime.entities.llm_entities import LLMResult, LLMUsage
 from core.model_runtime.utils.encoders import jsonable_encoder
@@ -118,9 +118,7 @@ class ErrorStreamResponse(StreamResponse):
     """
     event: StreamEvent = StreamEvent.ERROR
     err: Exception
-
-    class Config:
-        arbitrary_types_allowed = True
+    model_config = ConfigDict(arbitrary_types_allowed=True)
 
 
 class MessageStreamResponse(StreamResponse):
@@ -360,7 +358,7 @@ class IterationNodeNextStreamResponse(StreamResponse):
         title: str
         index: int
         created_at: int
-        pre_iteration_output: Optional[Any]
+        pre_iteration_output: Optional[Any] = None
         extras: dict = {}
 
     event: StreamEvent = StreamEvent.ITERATION_NEXT
@@ -369,7 +367,7 @@ class IterationNodeNextStreamResponse(StreamResponse):
 
 class IterationNodeCompletedStreamResponse(StreamResponse):
     """
-    NodeStartStreamResponse entity
+    NodeCompletedStreamResponse entity
     """
     class Data(BaseModel):
         """
@@ -379,14 +377,15 @@ class IterationNodeCompletedStreamResponse(StreamResponse):
         node_id: str
         node_type: str
         title: str
-        outputs: Optional[dict]
+        outputs: Optional[dict] = None
         created_at: int
         extras: dict = None
         inputs: dict = None
         status: WorkflowNodeExecutionStatus
-        error: Optional[str]
+        error: Optional[str] = None
         elapsed_time: float
         total_tokens: int
+        execution_metadata: Optional[dict] = None
         finished_at: int
         steps: int
 
@@ -547,4 +546,4 @@ class WorkflowIterationState(BaseModel):
         total_tokens: int = 0
         node_data: BaseNodeData
 
-    current_iterations: dict[str, Data] = None
+    current_iterations: dict[str, Data] = None

api/core/app/features/hosting_moderation/hosting_moderation.py

@@ -16,7 +16,7 @@ class HostingModerationFeature:
         :param prompt_messages: prompt messages
         :return:
         """
-        model_config = application_generate_entity.model_config
+        model_config = application_generate_entity.model_conf
 
         text = ""
         for prompt_message in prompt_messages:

api/core/app/task_pipeline/easy_ui_based_generate_task_pipeline.py

@@ -85,7 +85,7 @@ class EasyUIBasedGenerateTaskPipeline(BasedGenerateTaskPipeline, MessageCycleMan
         :param stream: stream
         """
         super().__init__(application_generate_entity, queue_manager, user, stream)
-        self._model_config = application_generate_entity.model_config
+        self._model_config = application_generate_entity.model_conf
         self._conversation = conversation
         self._message = message
 

api/core/app/task_pipeline/workflow_iteration_cycle_manage.py

@@ -17,6 +17,7 @@ from core.app.entities.task_entities import (
 )
 from core.app.task_pipeline.workflow_cycle_state_manager import WorkflowCycleStateManager
 from core.workflow.entities.node_entities import NodeType
+from core.workflow.workflow_engine_manager import WorkflowEngineManager
 from extensions.ext_database import db
 from models.workflow import (
     WorkflowNodeExecution,
@@ -94,6 +95,9 @@ class WorkflowIterationCycleManage(WorkflowCycleStateManager):
                     error=None,
                     elapsed_time=time.perf_counter() - current_iteration.started_at,
                     total_tokens=current_iteration.total_tokens,
+                    execution_metadata={
+                        'total_tokens': current_iteration.total_tokens,
+                    },
                     finished_at=int(time.time()),
                     steps=current_iteration.current_index
                 )
@@ -205,7 +209,7 @@ class WorkflowIterationCycleManage(WorkflowCycleStateManager):
 
         db.session.close()
 
-    def _handle_iteration_completed(self, event: QueueIterationCompletedEvent) -> WorkflowNodeExecution:
+    def _handle_iteration_completed(self, event: QueueIterationCompletedEvent):
         if event.node_id not in self._iteration_state.current_iterations:
             return
         
@@ -215,9 +219,9 @@ class WorkflowIterationCycleManage(WorkflowCycleStateManager):
         ).first()
 
         workflow_node_execution.status = WorkflowNodeExecutionStatus.SUCCEEDED.value
-        workflow_node_execution.outputs = json.dumps(event.outputs) if event.outputs else None
+        workflow_node_execution.outputs = json.dumps(WorkflowEngineManager.handle_special_values(event.outputs)) if event.outputs else None
         workflow_node_execution.elapsed_time = time.perf_counter() - current_iteration.started_at
-        
+
         original_node_execution_metadata = workflow_node_execution.execution_metadata_dict
         if original_node_execution_metadata:
             original_node_execution_metadata['steps_boundary'] = current_iteration.iteration_steps_boundary
@@ -275,7 +279,10 @@ class WorkflowIterationCycleManage(WorkflowCycleStateManager):
                     error=error,
                     elapsed_time=time.perf_counter() - current_iteration.started_at,
                     total_tokens=current_iteration.total_tokens,
+                    execution_metadata={
+                        'total_tokens': current_iteration.total_tokens,
+                    },
                     finished_at=int(time.time()),
                     steps=current_iteration.current_index
                 )
-            )
+            )

api/core/callback_handler/agent_tool_callback_handler.py

@@ -29,7 +29,7 @@ def print_text(
 class DifyAgentCallbackHandler(BaseModel):
     """Callback Handler that prints to std out."""
     color: Optional[str] = ''
-    current_loop = 1
+    current_loop: int = 1
 
     def __init__(self, color: Optional[str] = None) -> None:
         super().__init__()

api/core/entities/message_entities.py

@@ -17,7 +17,7 @@ class PromptMessageFileType(enum.Enum):
 
 class PromptMessageFile(BaseModel):
     type: PromptMessageFileType
-    data: Any
+    data: Any = None
 
 
 class ImagePromptMessageFile(PromptMessageFile):

api/core/entities/model_entities.py

@@ -1,7 +1,7 @@
 from enum import Enum
 from typing import Optional
 
-from pydantic import BaseModel
+from pydantic import BaseModel, ConfigDict
 
 from core.model_runtime.entities.common_entities import I18nObject
 from core.model_runtime.entities.model_entities import ModelType, ProviderModel
@@ -77,3 +77,6 @@ class DefaultModelEntity(BaseModel):
     model: str
     model_type: ModelType
     provider: DefaultModelProviderEntity
+
+    # pydantic configs
+    model_config = ConfigDict(protected_namespaces=())

api/core/entities/provider_configuration.py

@@ -6,7 +6,7 @@ from collections.abc import Iterator
 from json import JSONDecodeError
 from typing import Optional
 
-from pydantic import BaseModel
+from pydantic import BaseModel, ConfigDict
 
 from core.entities.model_entities import ModelStatus, ModelWithProviderEntity, SimpleModelProviderEntity
 from core.entities.provider_entities import (
@@ -54,6 +54,9 @@ class ProviderConfiguration(BaseModel):
     custom_configuration: CustomConfiguration
     model_settings: list[ModelSettings]
 
+    # pydantic configs
+    model_config = ConfigDict(protected_namespaces=())
+
     def __init__(self, **data):
         super().__init__(**data)
 
@@ -1019,7 +1022,6 @@ class ProviderModelBundle(BaseModel):
     provider_instance: ModelProvider
     model_type_instance: AIModel
 
-    class Config:
-        """Configuration for this pydantic object."""
-
-        arbitrary_types_allowed = True
+    # pydantic configs
+    model_config = ConfigDict(arbitrary_types_allowed=True,
+                              protected_namespaces=())

api/core/entities/provider_entities.py

@@ -1,7 +1,7 @@
 from enum import Enum
 from typing import Optional
 
-from pydantic import BaseModel
+from pydantic import BaseModel, ConfigDict
 
 from core.model_runtime.entities.model_entities import ModelType
 from models.provider import ProviderQuotaType
@@ -27,6 +27,9 @@ class RestrictModel(BaseModel):
     base_model_name: Optional[str] = None
     model_type: ModelType
 
+    # pydantic configs
+    model_config = ConfigDict(protected_namespaces=())
+
 
 class QuotaConfiguration(BaseModel):
     """
@@ -65,6 +68,9 @@ class CustomModelConfiguration(BaseModel):
     model_type: ModelType
     credentials: dict
 
+    # pydantic configs
+    model_config = ConfigDict(protected_namespaces=())
+
 
 class CustomConfiguration(BaseModel):
     """
@@ -91,3 +97,6 @@ class ModelSettings(BaseModel):
     model_type: ModelType
     enabled: bool = True
     load_balancing_configs: list[ModelLoadBalancingConfiguration] = []
+
+    # pydantic configs
+    model_config = ConfigDict(protected_namespaces=())

api/core/extension/extensible.py

@@ -16,7 +16,7 @@ class ExtensionModule(enum.Enum):
 
 
 class ModuleExtension(BaseModel):
-    extension_class: Any
+    extension_class: Any = None
     name: str
     label: Optional[dict] = None
     form_schema: Optional[list] = None

api/core/helper/code_executor/code_executor.py

@@ -28,8 +28,8 @@ class CodeExecutionException(Exception):
 
 class CodeExecutionResponse(BaseModel):
     class Data(BaseModel):
-        stdout: Optional[str]
-        error: Optional[str]
+        stdout: Optional[str] = None
+        error: Optional[str] = None
 
     code: int
     message: str
@@ -88,7 +88,7 @@ class CodeExecutor:
         }
 
         if dependencies:
-            data['dependencies'] = [dependency.dict() for dependency in dependencies]
+            data['dependencies'] = [dependency.model_dump() for dependency in dependencies]
 
         try:
             response = post(str(url), json=data, headers=headers, timeout=CODE_EXECUTION_TIMEOUT)

api/core/helper/code_executor/code_node_provider.py

@@ -25,7 +25,7 @@ class CodeNodeProvider(BaseModel):
 
     @classmethod
     def get_default_available_packages(cls) -> list[dict]:
-        return [p.dict() for p in CodeExecutor.list_dependencies(cls.get_language())]
+        return [p.model_dump() for p in CodeExecutor.list_dependencies(cls.get_language())]
 
     @classmethod
     def get_default_config(cls) -> dict:

api/core/helper/code_executor/template_transformer.py

@@ -4,12 +4,10 @@ from abc import ABC, abstractmethod
 from base64 import b64encode
 from typing import Optional
 
-from pydantic import BaseModel
-
 from core.helper.code_executor.entities import CodeDependency
 
 
-class TemplateTransformer(ABC, BaseModel):
+class TemplateTransformer(ABC):
     _code_placeholder: str = '{{code}}'
     _inputs_placeholder: str = '{{inputs}}'
     _result_tag: str = '<<RESULT>>'

api/core/indexing_runner.py

@@ -339,7 +339,7 @@ class IndexingRunner:
     def _extract(self, index_processor: BaseIndexProcessor, dataset_document: DatasetDocument, process_rule: dict) \
             -> list[Document]:
         # load file
-        if dataset_document.data_source_type not in ["upload_file", "notion_import"]:
+        if dataset_document.data_source_type not in ["upload_file", "notion_import", "website_crawl"]:
             return []
 
         data_source_info = dataset_document.data_source_info_dict
@@ -375,6 +375,23 @@ class IndexingRunner:
                 document_model=dataset_document.doc_form
             )
             text_docs = index_processor.extract(extract_setting, process_rule_mode=process_rule['mode'])
+        elif dataset_document.data_source_type == 'website_crawl':
+            if (not data_source_info or 'provider' not in data_source_info
+                    or 'url' not in data_source_info or 'job_id' not in data_source_info):
+                raise ValueError("no website import info found")
+            extract_setting = ExtractSetting(
+                datasource_type="website_crawl",
+                website_info={
+                    "provider": data_source_info['provider'],
+                    "job_id": data_source_info['job_id'],
+                    "tenant_id": dataset_document.tenant_id,
+                    "url": data_source_info['url'],
+                    "mode": data_source_info['mode'],
+                    "only_main_content": data_source_info['only_main_content']
+                },
+                document_model=dataset_document.doc_form
+            )
+            text_docs = index_processor.extract(extract_setting, process_rule_mode=process_rule['mode'])
         # update document status to splitting
         self._update_document_index_status(
             document_id=dataset_document.id,
@@ -550,7 +567,7 @@ class IndexingRunner:
                 document_qa_list = self.format_split_text(response)
                 qa_documents = []
                 for result in document_qa_list:
-                    qa_document = Document(page_content=result['question'], metadata=document_node.metadata.copy())
+                    qa_document = Document(page_content=result['question'], metadata=document_node.metadata.model_copy())
                     doc_id = str(uuid.uuid4())
                     hash = helper.generate_text_hash(result['question'])
                     qa_document.metadata['answer'] = result['answer']

api/core/memory/token_buffer_memory.py

@@ -1,3 +1,5 @@
+from typing import Optional
+
 from core.app.app_config.features.file_upload.manager import FileUploadConfigManager
 from core.file.message_file_parser import MessageFileParser
 from core.model_manager import ModelInstance
@@ -19,7 +21,7 @@ class TokenBufferMemory:
         self.model_instance = model_instance
 
     def get_history_prompt_messages(self, max_token_limit: int = 2000,
-                                    message_limit: int = 10) -> list[PromptMessage]:
+                                    message_limit: Optional[int] = None) -> list[PromptMessage]:
         """
         Get history prompt messages.
         :param max_token_limit: max token limit
@@ -28,10 +30,15 @@ class TokenBufferMemory:
         app_record = self.conversation.app
 
         # fetch limited messages, and return reversed
-        messages = db.session.query(Message).filter(
+        query = db.session.query(Message).filter(
             Message.conversation_id == self.conversation.id,
             Message.answer != ''
-        ).order_by(Message.created_at.desc()).limit(message_limit).all()
+        ).order_by(Message.created_at.desc())
+
+        if message_limit and message_limit > 0:
+            messages = query.limit(message_limit).all()
+        else:
+            messages = query.all()
 
         messages = list(reversed(messages))
         message_file_parser = MessageFileParser(
@@ -93,7 +100,7 @@ class TokenBufferMemory:
     def get_history_prompt_text(self, human_prefix: str = "Human",
                                 ai_prefix: str = "Assistant",
                                 max_token_limit: int = 2000,
-                                message_limit: int = 10) -> str:
+                                message_limit: Optional[int] = None) -> str:
         """
         Get history prompt text.
         :param human_prefix: human prefix

api/core/model_manager.py

@@ -328,7 +328,7 @@ class ModelInstance:
             except Exception as e:
                 raise e
 
-    def get_tts_voices(self, language: str) -> list:
+    def get_tts_voices(self, language: Optional[str] = None) -> list:
         """
         Invoke large language tts model voices
 

api/core/model_runtime/docs/en_US/interfaces.md

@@ -336,7 +336,7 @@ Inherit the `__base.text2speech_model.Text2SpeechModel` base class and implement
 - Invoke Invocation
 
   ```python
-  def _invoke(elf, model: str, credentials: dict, content_text: str, streaming: bool, user: Optional[str] = None):
+  def _invoke(self, model: str, credentials: dict, content_text: str, streaming: bool, user: Optional[str] = None):
       """
       Invoke large language model
   

api/core/model_runtime/docs/zh_Hans/interfaces.md

@@ -376,7 +376,7 @@ class XinferenceProvider(Provider):
 - Invoke 调用
 
   ```python
-  def _invoke(elf, model: str, credentials: dict, content_text: str, streaming: bool, user: Optional[str] = None):
+  def _invoke(self, model: str, credentials: dict, content_text: str, streaming: bool, user: Optional[str] = None):
       """
       Invoke large language model
   

api/core/model_runtime/entities/message_entities.py

@@ -2,7 +2,7 @@ from abc import ABC
 from enum import Enum
 from typing import Optional
 
-from pydantic import BaseModel
+from pydantic import BaseModel, field_validator
 
 
 class PromptMessageRole(Enum):
@@ -123,6 +123,14 @@ class AssistantPromptMessage(PromptMessage):
         type: str
         function: ToolCallFunction
 
+        @field_validator('id', mode='before')
+        @classmethod
+        def transform_id_to_str(cls, value) -> str:
+            if not isinstance(value, str):
+                return str(value)
+            else:
+                return value
+
     role: PromptMessageRole = PromptMessageRole.ASSISTANT
     tool_calls: list[ToolCall] = []
 

api/core/model_runtime/entities/model_entities.py

@@ -2,7 +2,7 @@ from decimal import Decimal
 from enum import Enum
 from typing import Any, Optional
 
-from pydantic import BaseModel
+from pydantic import BaseModel, ConfigDict
 
 from core.model_runtime.entities.common_entities import I18nObject
 
@@ -148,9 +148,7 @@ class ProviderModel(BaseModel):
     fetch_from: FetchFrom
     model_properties: dict[ModelPropertyKey, Any]
     deprecated: bool = False
-
-    class Config:
-        protected_namespaces = ()
+    model_config = ConfigDict(protected_namespaces=())
 
 
 class ParameterRule(BaseModel):

api/core/model_runtime/entities/provider_entities.py

@@ -1,7 +1,7 @@
 from enum import Enum
 from typing import Optional
 
-from pydantic import BaseModel
+from pydantic import BaseModel, ConfigDict
 
 from core.model_runtime.entities.common_entities import I18nObject
 from core.model_runtime.entities.model_entities import AIModelEntity, ModelType, ProviderModel
@@ -122,8 +122,8 @@ class ProviderEntity(BaseModel):
     provider_credential_schema: Optional[ProviderCredentialSchema] = None
     model_credential_schema: Optional[ModelCredentialSchema] = None
 
-    class Config:
-        protected_namespaces = ()
+    # pydantic configs
+    model_config = ConfigDict(protected_namespaces=())
 
     def to_simple_provider(self) -> SimpleProviderEntity:
         """

api/core/model_runtime/model_providers/__base/ai_model.py

@@ -3,6 +3,8 @@ import os
 from abc import ABC, abstractmethod
 from typing import Optional
 
+from pydantic import ConfigDict
+
 from core.helper.position_helper import get_position_map, sort_by_position_map
 from core.model_runtime.entities.common_entities import I18nObject
 from core.model_runtime.entities.defaults import PARAMETER_RULE_TEMPLATE
@@ -28,6 +30,9 @@ class AIModel(ABC):
     model_schemas: list[AIModelEntity] = None
     started_at: float = 0
 
+    # pydantic configs
+    model_config = ConfigDict(protected_namespaces=())
+
     @abstractmethod
     def validate_credentials(self, model: str, credentials: dict) -> None:
         """

api/core/model_runtime/model_providers/__base/large_language_model.py

@@ -6,12 +6,15 @@ from abc import abstractmethod
 from collections.abc import Generator
 from typing import Optional, Union
 
+from pydantic import ConfigDict
+
 from core.model_runtime.callbacks.base_callback import Callback
 from core.model_runtime.callbacks.logging_callback import LoggingCallback
 from core.model_runtime.entities.llm_entities import LLMMode, LLMResult, LLMResultChunk, LLMResultChunkDelta, LLMUsage
 from core.model_runtime.entities.message_entities import (
     AssistantPromptMessage,
     PromptMessage,
+    PromptMessageContentType,
     PromptMessageTool,
     SystemPromptMessage,
     UserPromptMessage,
@@ -34,6 +37,9 @@ class LargeLanguageModel(AIModel):
     """
     model_type: ModelType = ModelType.LLM
 
+    # pydantic configs
+    model_config = ConfigDict(protected_namespaces=())
+
     def invoke(self, model: str, credentials: dict,
                prompt_messages: list[PromptMessage], model_parameters: Optional[dict] = None,
                tools: Optional[list[PromptMessageTool]] = None, stop: Optional[list[str]] = None,
@@ -200,8 +206,14 @@ if you are not sure about the structure.
             ))
 
         if len(prompt_messages) > 0 and isinstance(prompt_messages[-1], UserPromptMessage):
-            # add ```JSON\n to the last message
-            prompt_messages[-1].content += f"\n```{code_block}\n"
+            # add ```JSON\n to the last text message
+            if isinstance(prompt_messages[-1].content, str):
+                prompt_messages[-1].content += f"\n```{code_block}\n"
+            elif isinstance(prompt_messages[-1].content, list):
+                for i in range(len(prompt_messages[-1].content) - 1, -1, -1):
+                    if prompt_messages[-1].content[i].type == PromptMessageContentType.TEXT:
+                        prompt_messages[-1].content[i].data += f"\n```{code_block}\n"
+                        break
         else:
             # append a user message
             prompt_messages.append(UserPromptMessage(

api/core/model_runtime/model_providers/__base/moderation_model.py

@@ -2,6 +2,8 @@ import time
 from abc import abstractmethod
 from typing import Optional
 
+from pydantic import ConfigDict
+
 from core.model_runtime.entities.model_entities import ModelType
 from core.model_runtime.model_providers.__base.ai_model import AIModel
 
@@ -12,6 +14,9 @@ class ModerationModel(AIModel):
     """
     model_type: ModelType = ModelType.MODERATION
 
+    # pydantic configs
+    model_config = ConfigDict(protected_namespaces=())
+
     def invoke(self, model: str, credentials: dict,
                text: str, user: Optional[str] = None) \
             -> bool:

api/core/model_runtime/model_providers/__base/speech2text_model.py

@@ -2,6 +2,8 @@ import os
 from abc import abstractmethod
 from typing import IO, Optional
 
+from pydantic import ConfigDict
+
 from core.model_runtime.entities.model_entities import ModelType
 from core.model_runtime.model_providers.__base.ai_model import AIModel
 
@@ -12,6 +14,9 @@ class Speech2TextModel(AIModel):
     """
     model_type: ModelType = ModelType.SPEECH2TEXT
 
+    # pydantic configs
+    model_config = ConfigDict(protected_namespaces=())
+
     def invoke(self, model: str, credentials: dict,
                file: IO[bytes], user: Optional[str] = None) \
             -> str:

api/core/model_runtime/model_providers/__base/text2img_model.py

@@ -1,6 +1,8 @@
 from abc import abstractmethod
 from typing import IO, Optional
 
+from pydantic import ConfigDict
+
 from core.model_runtime.entities.model_entities import ModelType
 from core.model_runtime.model_providers.__base.ai_model import AIModel
 
@@ -11,6 +13,9 @@ class Text2ImageModel(AIModel):
     """
     model_type: ModelType = ModelType.TEXT2IMG
 
+    # pydantic configs
+    model_config = ConfigDict(protected_namespaces=())
+
     def invoke(self, model: str, credentials: dict, prompt: str, 
                model_parameters: dict, user: Optional[str] = None) \
             -> list[IO[bytes]]:

api/core/model_runtime/model_providers/__base/text_embedding_model.py

@@ -2,6 +2,8 @@ import time
 from abc import abstractmethod
 from typing import Optional
 
+from pydantic import ConfigDict
+
 from core.model_runtime.entities.model_entities import ModelPropertyKey, ModelType
 from core.model_runtime.entities.text_embedding_entities import TextEmbeddingResult
 from core.model_runtime.model_providers.__base.ai_model import AIModel
@@ -13,6 +15,9 @@ class TextEmbeddingModel(AIModel):
     """
     model_type: ModelType = ModelType.TEXT_EMBEDDING
 
+    # pydantic configs
+    model_config = ConfigDict(protected_namespaces=())
+
     def invoke(self, model: str, credentials: dict,
                texts: list[str], user: Optional[str] = None) \
             -> TextEmbeddingResult:

api/core/model_runtime/model_providers/__base/tts_model.py

@@ -4,6 +4,8 @@ import uuid
 from abc import abstractmethod
 from typing import Optional
 
+from pydantic import ConfigDict
+
 from core.model_runtime.entities.model_entities import ModelPropertyKey, ModelType
 from core.model_runtime.errors.invoke import InvokeBadRequestError
 from core.model_runtime.model_providers.__base.ai_model import AIModel
@@ -15,6 +17,9 @@ class TTSModel(AIModel):
     """
     model_type: ModelType = ModelType.TTS
 
+    # pydantic configs
+    model_config = ConfigDict(protected_namespaces=())
+
     def invoke(self, model: str, tenant_id: str, credentials: dict, content_text: str, voice: str, streaming: bool,
                user: Optional[str] = None):
         """

api/core/model_runtime/model_providers/_position.yaml

@@ -31,3 +31,5 @@
 - volcengine_maas
 - openai_api_compatible
 - deepseek
+- hunyuan
+- siliconflow

api/core/model_runtime/model_providers/azure_openai/_assets/icon_s_en.svg

@@ -1,5 +1,5 @@
 <svg width="21" height="22" viewBox="0 0 21 22" fill="none" xmlns="http://www.w3.org/2000/svg">
-<g id="Microsfot">
+<g id="Microsoft">
 <rect id="Rectangle 1010" y="0.5" width="10" height="10" fill="#EF4F21"/>
 <rect id="Rectangle 1012" y="11.5" width="10" height="10" fill="#03A4EE"/>
 <rect id="Rectangle 1011" x="11" y="0.5" width="10" height="10" fill="#7EB903"/>

api/core/model_runtime/model_providers/azure_openai/azure_openai.yaml

@@ -53,6 +53,15 @@ model_credential_schema:
       type: select
       required: true
       options:
+        - label:
+            en_US: 2024-05-01-preview
+          value: 2024-05-01-preview
+        - label:
+            en_US: 2024-04-01-preview
+          value: 2024-04-01-preview
+        - label:
+            en_US: 2024-03-01-preview
+          value: 2024-03-01-preview
         - label:
             en_US: 2024-02-15-preview
           value: 2024-02-15-preview

api/core/model_runtime/model_providers/baichuan/llm/baichuan_turbo.py

@@ -57,23 +57,23 @@ class BaichuanModel:
         }[model]
 
     def _handle_chat_generate_response(self, response) -> BaichuanMessage:
-            resp = response.json()
-            choices = resp.get('choices', [])
-            message = BaichuanMessage(content='', role='assistant')
-            for choice in choices:
-                message.content += choice['message']['content']
-                message.role = choice['message']['role']
-                if choice['finish_reason']:
-                    message.stop_reason = choice['finish_reason']
+        resp = response.json()
+        choices = resp.get('choices', [])
+        message = BaichuanMessage(content='', role='assistant')
+        for choice in choices:
+            message.content += choice['message']['content']
+            message.role = choice['message']['role']
+            if choice['finish_reason']:
+                message.stop_reason = choice['finish_reason']
 
-            if 'usage' in resp:
-                message.usage = {
-                    'prompt_tokens': resp['usage']['prompt_tokens'],
-                    'completion_tokens': resp['usage']['completion_tokens'],
-                    'total_tokens': resp['usage']['total_tokens'],
-                }
-            
-            return message
+        if 'usage' in resp:
+            message.usage = {
+                'prompt_tokens': resp['usage']['prompt_tokens'],
+                'completion_tokens': resp['usage']['completion_tokens'],
+                'total_tokens': resp['usage']['total_tokens'],
+            }
+
+        return message
     
     def _handle_chat_stream_generate_response(self, response) -> Generator:
         for line in response.iter_lines():

api/core/model_runtime/model_providers/baichuan/llm/llm.py

@@ -7,6 +7,7 @@ from core.model_runtime.entities.message_entities import (
     PromptMessage,
     PromptMessageTool,
     SystemPromptMessage,
+    ToolPromptMessage,
     UserPromptMessage,
 )
 from core.model_runtime.errors.invoke import (
@@ -32,20 +33,21 @@ from core.model_runtime.model_providers.baichuan.llm.baichuan_turbo_errors impor
 
 
 class BaichuanLarguageModel(LargeLanguageModel):
-    def _invoke(self, model: str, credentials: dict, 
-                prompt_messages: list[PromptMessage], model_parameters: dict, 
-                tools: list[PromptMessageTool] | None = None, stop: list[str] | None = None, 
+    def _invoke(self, model: str, credentials: dict,
+                prompt_messages: list[PromptMessage], model_parameters: dict,
+                tools: list[PromptMessageTool] | None = None, stop: list[str] | None = None,
                 stream: bool = True, user: str | None = None) \
             -> LLMResult | Generator:
         return self._generate(model=model, credentials=credentials, prompt_messages=prompt_messages,
-                                model_parameters=model_parameters, tools=tools, stop=stop, stream=stream, user=user)
+                              model_parameters=model_parameters, tools=tools, stop=stop, stream=stream, user=user)
 
     def get_num_tokens(self, model: str, credentials: dict, prompt_messages: list[PromptMessage],
                        tools: list[PromptMessageTool] | None = None) -> int:
         return self._num_tokens_from_messages(prompt_messages)
 
-    def _num_tokens_from_messages(self, messages: list[PromptMessage],) -> int:
+    def _num_tokens_from_messages(self, messages: list[PromptMessage], ) -> int:
         """Calculate num tokens for baichuan model"""
+
         def tokens(text: str):
             return BaichuanTokenizer._get_num_tokens(text)
 
@@ -85,9 +87,20 @@ class BaichuanLarguageModel(LargeLanguageModel):
         elif isinstance(message, SystemPromptMessage):
             message = cast(SystemPromptMessage, message)
             message_dict = {"role": "user", "content": message.content}
+        elif isinstance(message, ToolPromptMessage):
+            # copy from core/model_runtime/model_providers/anthropic/llm/llm.py
+            message = cast(ToolPromptMessage, message)
+            message_dict = {
+                "role": "user",
+                "content": [{
+                    "type": "tool_result",
+                    "tool_use_id": message.tool_call_id,
+                    "content": message.content
+                }]
+            }
         else:
             raise ValueError(f"Unknown message type {type(message)}")
-        
+
         return message_dict
 
     def validate_credentials(self, model: str, credentials: dict) -> None:
@@ -106,13 +119,13 @@ class BaichuanLarguageModel(LargeLanguageModel):
         except Exception as e:
             raise CredentialsValidateFailedError(f"Invalid API key: {e}")
 
-    def _generate(self, model: str, credentials: dict, prompt_messages: list[PromptMessage], 
-                 model_parameters: dict, tools: list[PromptMessageTool] | None = None, 
-                 stop: list[str] | None = None, stream: bool = True, user: str | None = None) \
+    def _generate(self, model: str, credentials: dict, prompt_messages: list[PromptMessage],
+                  model_parameters: dict, tools: list[PromptMessageTool] | None = None,
+                  stop: list[str] | None = None, stream: bool = True, user: str | None = None) \
             -> LLMResult | Generator:
         if tools is not None and len(tools) > 0:
             raise InvokeBadRequestError("Baichuan model doesn't support tools")
-        
+
         instance = BaichuanModel(
             api_key=credentials['api_key'],
             secret_key=credentials.get('secret_key', '')
@@ -129,11 +142,12 @@ class BaichuanLarguageModel(LargeLanguageModel):
         ]
 
         # invoke model
-        response = instance.generate(model=model, stream=stream, messages=messages, parameters=model_parameters, timeout=60)
+        response = instance.generate(model=model, stream=stream, messages=messages, parameters=model_parameters,
+                                     timeout=60)
 
         if stream:
             return self._handle_chat_generate_stream_response(model, prompt_messages, credentials, response)
-        
+
         return self._handle_chat_generate_response(model, prompt_messages, credentials, response)
 
     def _handle_chat_generate_response(self, model: str,
@@ -141,7 +155,9 @@ class BaichuanLarguageModel(LargeLanguageModel):
                                        credentials: dict,
                                        response: BaichuanMessage) -> LLMResult:
         # convert baichuan message to llm result
-        usage = self._calc_response_usage(model=model, credentials=credentials, prompt_tokens=response.usage['prompt_tokens'], completion_tokens=response.usage['completion_tokens'])
+        usage = self._calc_response_usage(model=model, credentials=credentials,
+                                          prompt_tokens=response.usage['prompt_tokens'],
+                                          completion_tokens=response.usage['completion_tokens'])
         return LLMResult(
             model=model,
             prompt_messages=prompt_messages,
@@ -158,7 +174,9 @@ class BaichuanLarguageModel(LargeLanguageModel):
                                               response: Generator[BaichuanMessage, None, None]) -> Generator:
         for message in response:
             if message.usage:
-                usage = self._calc_response_usage(model=model, credentials=credentials, prompt_tokens=message.usage['prompt_tokens'], completion_tokens=message.usage['completion_tokens'])
+                usage = self._calc_response_usage(model=model, credentials=credentials,
+                                                  prompt_tokens=message.usage['prompt_tokens'],
+                                                  completion_tokens=message.usage['completion_tokens'])
                 yield LLMResultChunk(
                     model=model,
                     prompt_messages=prompt_messages,

api/core/model_runtime/model_providers/bedrock/bedrock.yaml

@@ -21,16 +21,16 @@ configurate_methods:
 provider_credential_schema:
   credential_form_schemas:
     - variable: aws_access_key_id
-      required: true
+      required: false
       label:
-        en_US: Access Key
+        en_US: Access Key (If not provided, credentials are obtained from the running environment.)
         zh_Hans: Access Key
       type: secret-input
       placeholder:
         en_US: Enter your Access Key
         zh_Hans: 在此输入您的 Access Key
     - variable: aws_secret_access_key
-      required: true
+      required: false
       label:
         en_US: Secret Access Key
         zh_Hans: Secret Access Key

api/core/model_runtime/model_providers/bedrock/llm/_position.yaml

@@ -8,6 +8,8 @@
 - anthropic.claude-3-haiku-v1:0
 - cohere.command-light-text-v14
 - cohere.command-text-v14
+- cohere.command-r-plus-v1.0
+- cohere.command-r-v1.0
 - meta.llama3-8b-instruct-v1:0
 - meta.llama3-70b-instruct-v1:0
 - meta.llama2-13b-chat-v1

api/core/model_runtime/model_providers/bedrock/llm/cohere.command-r-plus-v1.0.yaml

@@ -0,0 +1,45 @@
+model: cohere.command-r-plus-v1:0
+label:
+  en_US: Command R+
+model_type: llm
+features:
+  #- multi-tool-call
+  - agent-thought
+  #- stream-tool-call
+model_properties:
+  mode: chat
+  context_size: 128000
+parameter_rules:
+  - name: temperature
+    use_template: temperature
+    max: 5.0
+  - name: p
+    use_template: top_p
+    default: 0.75
+    min: 0.01
+    max: 0.99
+  - name: k
+    label:
+      zh_Hans: 取样数量
+      en_US: Top k
+    type: int
+    help:
+      zh_Hans: 仅从每个后续标记的前 K 个选项中采样。
+      en_US: Only sample from the top K options for each subsequent token.
+    required: false
+    default: 0
+    min: 0
+    max: 500
+  - name: presence_penalty
+    use_template: presence_penalty
+  - name: frequency_penalty
+    use_template: frequency_penalty
+  - name: max_tokens
+    use_template: max_tokens
+    default: 1024
+    max: 4096
+pricing:
+  input: '3'
+  output: '15'
+  unit: '0.000001'
+  currency: USD

api/core/model_runtime/model_providers/bedrock/llm/cohere.command-r-v1.0.yaml

@@ -0,0 +1,45 @@
+model: cohere.command-r-v1:0
+label:
+  en_US: Command R
+model_type: llm
+features:
+  #- multi-tool-call
+  - agent-thought
+  #- stream-tool-call
+model_properties:
+  mode: chat
+  context_size: 128000
+parameter_rules:
+  - name: temperature
+    use_template: temperature
+    max: 5.0
+  - name: p
+    use_template: top_p
+    default: 0.75
+    min: 0.01
+    max: 0.99
+  - name: k
+    label:
+      zh_Hans: 取样数量
+      en_US: Top k
+    type: int
+    help:
+      zh_Hans: 仅从每个后续标记的前 K 个选项中采样。
+      en_US: Only sample from the top K options for each subsequent token.
+    required: false
+    default: 0
+    min: 0
+    max: 500
+  - name: presence_penalty
+    use_template: presence_penalty
+  - name: frequency_penalty
+    use_template: frequency_penalty
+  - name: max_tokens
+    use_template: max_tokens
+    default: 1024
+    max: 4096
+pricing:
+  input: '0.5'
+  output: '1.5'
+  unit: '0.000001'
+  currency: USD

api/core/model_runtime/model_providers/bedrock/llm/llm.py

@@ -25,6 +25,7 @@ from botocore.exceptions import (
     ServiceNotInRegionError,
     UnknownServiceError,
 )
+from cohere import ChatMessage
 
 from core.model_runtime.entities.llm_entities import LLMResult, LLMResultChunk, LLMResultChunkDelta, LLMUsage
 from core.model_runtime.entities.message_entities import (
@@ -48,6 +49,7 @@ from core.model_runtime.errors.invoke import (
 )
 from core.model_runtime.errors.validate import CredentialsValidateFailedError
 from core.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel
+from core.model_runtime.model_providers.cohere.llm.llm import CohereLargeLanguageModel
 
 logger = logging.getLogger(__name__)
 
@@ -75,8 +77,86 @@ class BedrockLargeLanguageModel(LargeLanguageModel):
         # invoke anthropic models via anthropic official SDK
         if "anthropic" in model:
             return self._generate_anthropic(model, credentials, prompt_messages, model_parameters, stop, stream, user)
+        # invoke Cohere models via boto3 client
+        if "cohere.command-r" in model:
+            return self._generate_cohere_chat(model, credentials, prompt_messages, model_parameters, stop, stream, user, tools)
         # invoke other models via boto3 client
         return self._generate(model, credentials, prompt_messages, model_parameters, stop, stream, user)
+    
+    def _generate_cohere_chat(
+            self, model: str, credentials: dict, prompt_messages: list[PromptMessage], model_parameters: dict,
+            stop: Optional[list[str]] = None, stream: bool = True, user: Optional[str] = None,
+            tools: Optional[list[PromptMessageTool]] = None,) -> Union[LLMResult, Generator]:
+        cohere_llm = CohereLargeLanguageModel()
+        client_config = Config(
+            region_name=credentials["aws_region"]
+        )
+
+        runtime_client = boto3.client(
+            service_name='bedrock-runtime',
+            config=client_config,
+            aws_access_key_id=credentials["aws_access_key_id"],
+            aws_secret_access_key=credentials["aws_secret_access_key"]
+        )
+
+        extra_model_kwargs = {}
+        if stop:
+            extra_model_kwargs['stop_sequences'] = stop
+
+        if tools:
+            tools = cohere_llm._convert_tools(tools)
+            model_parameters['tools'] = tools
+
+        message, chat_histories, tool_results \
+            = cohere_llm._convert_prompt_messages_to_message_and_chat_histories(prompt_messages)
+
+        if tool_results:
+            model_parameters['tool_results'] = tool_results
+
+        payload = {
+            **model_parameters,
+            "message": message,
+            "chat_history": chat_histories,
+        }
+
+        # need workaround for ai21 models which doesn't support streaming
+        if stream:
+            invoke = runtime_client.invoke_model_with_response_stream
+        else:
+            invoke = runtime_client.invoke_model
+
+        def serialize(obj):
+            if isinstance(obj, ChatMessage):
+                return obj.__dict__
+            raise TypeError(f"Type {type(obj)} not serializable")
+
+        try:
+            body_jsonstr=json.dumps(payload, default=serialize)
+            response = invoke(
+                modelId=model,
+                contentType="application/json",
+                accept="*/*",
+                body=body_jsonstr
+            )
+        except ClientError as ex:
+            error_code = ex.response['Error']['Code']
+            full_error_msg = f"{error_code}: {ex.response['Error']['Message']}"
+            raise self._map_client_to_invoke_error(error_code, full_error_msg)
+
+        except (EndpointConnectionError, NoRegionError, ServiceNotInRegionError) as ex:
+            raise InvokeConnectionError(str(ex))
+
+        except UnknownServiceError as ex:
+            raise InvokeServerUnavailableError(str(ex))
+
+        except Exception as ex:
+            raise InvokeError(str(ex))
+
+        if stream:
+            return self._handle_generate_stream_response(model, credentials, response, prompt_messages)
+
+        return self._handle_generate_response(model, credentials, response, prompt_messages)
+
 
     def _generate_anthropic(self, model: str, credentials: dict, prompt_messages: list[PromptMessage], model_parameters: dict,
                 stop: Optional[list[str]] = None, stream: bool = True, user: Optional[str] = None) -> Union[LLMResult, Generator]:
@@ -95,8 +175,8 @@ class BedrockLargeLanguageModel(LargeLanguageModel):
         # - https://docs.anthropic.com/claude/reference/claude-on-amazon-bedrock
         # - https://github.com/anthropics/anthropic-sdk-python
         client = AnthropicBedrock(
-            aws_access_key=credentials["aws_access_key_id"],
-            aws_secret_key=credentials["aws_secret_access_key"],
+            aws_access_key=credentials.get("aws_access_key_id", None),
+            aws_secret_key=credentials.get("aws_secret_access_key", None),
             aws_region=credentials["aws_region"],
         )
 
@@ -568,8 +648,8 @@ class BedrockLargeLanguageModel(LargeLanguageModel):
         runtime_client = boto3.client(
             service_name='bedrock-runtime',
             config=client_config,
-            aws_access_key_id=credentials["aws_access_key_id"],
-            aws_secret_access_key=credentials["aws_secret_access_key"]
+            aws_access_key_id=credentials.get("aws_access_key_id", None),
+            aws_secret_access_key=credentials.get("aws_secret_access_key", None)
         )
 
         model_prefix = model.split('.')[0]
@@ -826,4 +906,4 @@ class BedrockLargeLanguageModel(LargeLanguageModel):
         elif error_code == "ModelStreamErrorException":
             return InvokeConnectionError(error_msg)
 
-        return InvokeError(error_msg)
+        return InvokeError(error_msg)

api/core/model_runtime/model_providers/bedrock/text_embedding/text_embedding.py

@@ -49,8 +49,8 @@ class BedrockTextEmbeddingModel(TextEmbeddingModel):
         bedrock_runtime = boto3.client(
             service_name='bedrock-runtime',
             config=client_config,
-            aws_access_key_id=credentials["aws_access_key_id"],
-            aws_secret_access_key=credentials["aws_secret_access_key"]
+            aws_access_key_id=credentials.get("aws_access_key_id", None),
+            aws_secret_access_key=credentials.get("aws_secret_access_key", None)
         )
 
         embeddings = []
@@ -59,15 +59,15 @@ class BedrockTextEmbeddingModel(TextEmbeddingModel):
         model_prefix = model.split('.')[0]
          
         if model_prefix == "amazon" :
-           for text in texts:
-              body = {
+            for text in texts:
+                body = {
                  "inputText": text,
-              }
-              response_body = self._invoke_bedrock_embedding(model, bedrock_runtime, body)
-              embeddings.extend([response_body.get('embedding')])
-              token_usage += response_body.get('inputTextTokenCount')
-           logger.warning(f'Total Tokens: {token_usage}')
-           result = TextEmbeddingResult(
+                }
+                response_body = self._invoke_bedrock_embedding(model, bedrock_runtime, body)
+                embeddings.extend([response_body.get('embedding')])
+                token_usage += response_body.get('inputTextTokenCount')
+            logger.warning(f'Total Tokens: {token_usage}')
+            result = TextEmbeddingResult(
                 model=model,
                 embeddings=embeddings,
                 usage=self._calc_response_usage(
@@ -75,20 +75,20 @@ class BedrockTextEmbeddingModel(TextEmbeddingModel):
                     credentials=credentials,
                     tokens=token_usage
                 )
-           )
-           return result
-           
+            )
+            return result
+
         if model_prefix == "cohere" :
-           input_type = 'search_document' if len(texts) > 1 else 'search_query'
-           for text in texts:
-              body = {
+            input_type = 'search_document' if len(texts) > 1 else 'search_query'
+            for text in texts:
+                body = {
                  "texts": [text],
                  "input_type": input_type,
-              }
-              response_body = self._invoke_bedrock_embedding(model, bedrock_runtime, body)
-              embeddings.extend(response_body.get('embeddings'))
-              token_usage += len(text)
-           result = TextEmbeddingResult(
+                }
+                response_body = self._invoke_bedrock_embedding(model, bedrock_runtime, body)
+                embeddings.extend(response_body.get('embeddings'))
+                token_usage += len(text)
+            result = TextEmbeddingResult(
                 model=model,
                 embeddings=embeddings,
                 usage=self._calc_response_usage(
@@ -96,9 +96,9 @@ class BedrockTextEmbeddingModel(TextEmbeddingModel):
                     credentials=credentials,
                     tokens=token_usage
                 )
-           )
-           return result
-        
+            )
+            return result
+
         #others
         raise ValueError(f"Got unknown model prefix {model_prefix} when handling block response")
 
@@ -183,7 +183,7 @@ class BedrockTextEmbeddingModel(TextEmbeddingModel):
         )
 
         return usage
-    
+
     def _map_client_to_invoke_error(self, error_code: str, error_msg: str) -> type[InvokeError]:
         """
         Map client error to invoke error
@@ -212,9 +212,9 @@ class BedrockTextEmbeddingModel(TextEmbeddingModel):
         content_type = 'application/json'
         try:
             response = bedrock_runtime.invoke_model(
-                body=json.dumps(body), 
-                modelId=model, 
-                accept=accept, 
+                body=json.dumps(body),
+                modelId=model,
+                accept=accept,
                 contentType=content_type
             )
             response_body = json.loads(response.get('body').read().decode('utf-8'))

api/core/model_runtime/model_providers/deepseek/llm/deepseek-chat.yaml

@@ -23,7 +23,7 @@ parameter_rules:
     type: int
     default: 4096
     min: 1
-    max: 32000
+    max: 4096
     help:
       zh_Hans: 指定生成结果长度的上限。如果生成结果截断，可以调大该参数。
       en_US: Specifies the upper limit on the length of generated results. If the generated results are truncated, you can increase this parameter.