version to 0.5.4 (#2407)

Co-authored-by: Joel <iamjoel007@gmail.com>

.github/workflows/style.yml

@@ -10,9 +10,33 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  python-style:
+    name: Python Style
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+
+      - name: Python dependencies
+        run: pip install ruff
+
+      - name: Ruff check
+        run: ruff check ./api
+
+      - name: Lint hints
+        if: failure()
+        run: echo "Please run 'dev/reformat' to fix the fixable linting errors."
+
   test:
     name: ESLint and SuperLinter
     runs-on: ubuntu-latest
+    needs: python-style
 
     steps:
       - name: Checkout code

api/.env.example

@@ -120,4 +120,7 @@ HOSTED_ANTHROPIC_QUOTA_LIMIT=600000
 HOSTED_ANTHROPIC_PAID_ENABLED=false
 
 ETL_TYPE=dify
-UNSTRUCTURED_API_URL=
+UNSTRUCTURED_API_URL=
+
+SSRF_PROXY_HTTP_URL=
+SSRF_PROXY_HTTPS_URL=

api/app.py

@@ -19,18 +19,28 @@ import threading
 import time
 import warnings
 
-from commands import register_commands
-from config import CloudEditionConfig, Config
-from events import event_handlers
-from extensions import (ext_celery, ext_code_based_extension, ext_database, ext_hosting_provider, ext_login, ext_mail,
-                        ext_migrate, ext_redis, ext_sentry, ext_storage)
-from extensions.ext_database import db
-from extensions.ext_login import login_manager
 from flask import Flask, Response, request
 from flask_cors import CORS
+
+from commands import register_commands
+from config import CloudEditionConfig, Config
+from extensions import (
+    ext_celery,
+    ext_code_based_extension,
+    ext_database,
+    ext_hosting_provider,
+    ext_login,
+    ext_mail,
+    ext_migrate,
+    ext_redis,
+    ext_sentry,
+    ext_storage,
+)
+from extensions.ext_database import db
+from extensions.ext_login import login_manager
 from libs.passport import PassportService
+
 # DO NOT REMOVE BELOW
-from models import account, dataset, model, source, task, tool, web, tools
 from services.account_service import AccountService
 
 # DO NOT REMOVE ABOVE

api/commands.py

@@ -1,33 +1,22 @@
 import base64
-import datetime
 import json
-import math
-import random
 import secrets
-import string
-import threading
-import time
-import uuid
 
 import click
-import qdrant_client
-from constants.languages import user_input_form_template
+from flask import current_app
+from werkzeug.exceptions import NotFound
+
 from core.embedding.cached_embedding import CacheEmbedding
-from core.index.index import IndexBuilder
 from core.model_manager import ModelManager
 from core.model_runtime.entities.model_entities import ModelType
 from extensions.ext_database import db
-from flask import Flask, current_app
 from libs.helper import email as email_validate
 from libs.password import hash_password, password_pattern, valid_password
 from libs.rsa import generate_key_pair
-from models.account import InvitationCode, Tenant, TenantAccountJoin
-from models.dataset import Dataset, DatasetCollectionBinding, DatasetQuery, Document
-from models.model import Account, App, AppModelConfig, Message, MessageAnnotation, InstalledApp
-from models.provider import Provider, ProviderModel, ProviderQuotaType, ProviderType
-from qdrant_client.http.models import TextIndexParams, TextIndexType, TokenizerType
-from tqdm import tqdm
-from werkzeug.exceptions import NotFound
+from models.account import Tenant
+from models.dataset import Dataset
+from models.model import Account
+from models.provider import Provider, ProviderModel
 
 
 @click.command('reset-password', help='Reset the account password.')
@@ -35,15 +24,22 @@ from werkzeug.exceptions import NotFound
 @click.option('--new-password', prompt=True, help='the new password.')
 @click.option('--password-confirm', prompt=True, help='the new password confirm.')
 def reset_password(email, new_password, password_confirm):
+    """
+    Reset password of owner account
+    Only available in SELF_HOSTED mode
+    """
     if str(new_password).strip() != str(password_confirm).strip():
         click.echo(click.style('sorry. The two passwords do not match.', fg='red'))
         return
+
     account = db.session.query(Account). \
         filter(Account.email == email). \
         one_or_none()
+
     if not account:
         click.echo(click.style('sorry. the account: [{}] not exist .'.format(email), fg='red'))
         return
+
     try:
         valid_password(new_password)
     except:
@@ -69,15 +65,22 @@ def reset_password(email, new_password, password_confirm):
 @click.option('--new-email', prompt=True, help='the new email.')
 @click.option('--email-confirm', prompt=True, help='the new email confirm.')
 def reset_email(email, new_email, email_confirm):
+    """
+    Replace account email
+    :return:
+    """
     if str(new_email).strip() != str(email_confirm).strip():
         click.echo(click.style('Sorry, new email and confirm email do not match.', fg='red'))
         return
+
     account = db.session.query(Account). \
         filter(Account.email == email). \
         one_or_none()
+
     if not account:
         click.echo(click.style('sorry. the account: [{}] not exist .'.format(email), fg='red'))
         return
+
     try:
         email_validate(new_email)
     except:
@@ -97,6 +100,11 @@ def reset_email(email, new_email, email_confirm):
 @click.confirmation_option(prompt=click.style('Are you sure you want to reset encrypt key pair?'
                                               ' this operation cannot be rolled back!', fg='red'))
 def reset_encrypt_key_pair():
+    """
+    Reset the encrypted key pair of workspace for encrypt LLM credentials.
+    After the reset, all LLM credentials will become invalid, requiring re-entry.
+    Only support SELF_HOSTED mode.
+    """
     if current_app.config['EDITION'] != 'SELF_HOSTED':
         click.echo(click.style('Sorry, only support SELF_HOSTED mode.', fg='red'))
         return
@@ -116,201 +124,11 @@ def reset_encrypt_key_pair():
                            'the asymmetric key pair of workspace {} has been reset.'.format(tenant.id), fg='green'))
 
 
-@click.command('generate-invitation-codes', help='Generate invitation codes.')
-@click.option('--batch', help='The batch of invitation codes.')
-@click.option('--count', prompt=True, help='Invitation codes count.')
-def generate_invitation_codes(batch, count):
-    if not batch:
-        now = datetime.datetime.now()
-        batch = now.strftime('%Y%m%d%H%M%S')
-
-    if not count or int(count) <= 0:
-        click.echo(click.style('sorry. the count must be greater than 0.', fg='red'))
-        return
-
-    count = int(count)
-
-    click.echo('Start generate {} invitation codes for batch {}.'.format(count, batch))
-
-    codes = ''
-    for i in range(count):
-        code = generate_invitation_code()
-        invitation_code = InvitationCode(
-            code=code,
-            batch=batch
-        )
-        db.session.add(invitation_code)
-        click.echo(code)
-
-        codes += code + "\n"
-    db.session.commit()
-
-    filename = 'storage/invitation-codes-{}.txt'.format(batch)
-
-    with open(filename, 'w') as f:
-        f.write(codes)
-
-    click.echo(click.style(
-        'Congratulations! Generated {} invitation codes for batch {} and saved to the file \'{}\''.format(count, batch,
-                                                                                                          filename),
-        fg='green'))
-
-
-def generate_invitation_code():
-    code = generate_upper_string()
-    while db.session.query(InvitationCode).filter(InvitationCode.code == code).count() > 0:
-        code = generate_upper_string()
-
-    return code
-
-
-def generate_upper_string():
-    letters_digits = string.ascii_uppercase + string.digits
-    result = ""
-    for i in range(8):
-        result += random.choice(letters_digits)
-
-    return result
-
-
-@click.command('recreate-all-dataset-indexes', help='Recreate all dataset indexes.')
-def recreate_all_dataset_indexes():
-    click.echo(click.style('Start recreate all dataset indexes.', fg='green'))
-    recreate_count = 0
-
-    page = 1
-    while True:
-        try:
-            datasets = db.session.query(Dataset).filter(Dataset.indexing_technique == 'high_quality') \
-                .order_by(Dataset.created_at.desc()).paginate(page=page, per_page=50)
-        except NotFound:
-            break
-
-        page += 1
-        for dataset in datasets:
-            try:
-                click.echo('Recreating dataset index: {}'.format(dataset.id))
-                index = IndexBuilder.get_index(dataset, 'high_quality')
-                if index and index._is_origin():
-                    index.recreate_dataset(dataset)
-                    recreate_count += 1
-                else:
-                    click.echo('passed.')
-            except Exception as e:
-                click.echo(
-                    click.style('Recreate dataset index error: {} {}'.format(e.__class__.__name__, str(e)), fg='red'))
-                continue
-
-    click.echo(click.style('Congratulations! Recreate {} dataset indexes.'.format(recreate_count), fg='green'))
-
-
-@click.command('clean-unused-dataset-indexes', help='Clean unused dataset indexes.')
-def clean_unused_dataset_indexes():
-    click.echo(click.style('Start clean unused dataset indexes.', fg='green'))
-    clean_days = int(current_app.config.get('CLEAN_DAY_SETTING'))
-    start_at = time.perf_counter()
-    thirty_days_ago = datetime.datetime.now() - datetime.timedelta(days=clean_days)
-    page = 1
-    while True:
-        try:
-            datasets = db.session.query(Dataset).filter(Dataset.created_at < thirty_days_ago) \
-                .order_by(Dataset.created_at.desc()).paginate(page=page, per_page=50)
-        except NotFound:
-            break
-        page += 1
-        for dataset in datasets:
-            dataset_query = db.session.query(DatasetQuery).filter(
-                DatasetQuery.created_at > thirty_days_ago,
-                DatasetQuery.dataset_id == dataset.id
-            ).all()
-            if not dataset_query or len(dataset_query) == 0:
-                documents = db.session.query(Document).filter(
-                    Document.dataset_id == dataset.id,
-                    Document.indexing_status == 'completed',
-                    Document.enabled == True,
-                    Document.archived == False,
-                    Document.updated_at > thirty_days_ago
-                ).all()
-                if not documents or len(documents) == 0:
-                    try:
-                        # remove index
-                        vector_index = IndexBuilder.get_index(dataset, 'high_quality')
-                        kw_index = IndexBuilder.get_index(dataset, 'economy')
-                        # delete from vector index
-                        if vector_index:
-                            if dataset.collection_binding_id:
-                                vector_index.delete_by_group_id(dataset.id)
-                            else:
-                                if dataset.collection_binding_id:
-                                    vector_index.delete_by_group_id(dataset.id)
-                                else:
-                                    vector_index.delete()
-                        kw_index.delete()
-                        # update document
-                        update_params = {
-                            Document.enabled: False
-                        }
-
-                        Document.query.filter_by(dataset_id=dataset.id).update(update_params)
-                        db.session.commit()
-                        click.echo(click.style('Cleaned unused dataset {} from db success!'.format(dataset.id),
-                                               fg='green'))
-                    except Exception as e:
-                        click.echo(
-                            click.style('clean dataset index error: {} {}'.format(e.__class__.__name__, str(e)),
-                                        fg='red'))
-    end_at = time.perf_counter()
-    click.echo(click.style('Cleaned unused dataset from db success latency: {}'.format(end_at - start_at), fg='green'))
-
-
-@click.command('sync-anthropic-hosted-providers', help='Sync anthropic hosted providers.')
-def sync_anthropic_hosted_providers():
-    if not hosted_model_providers.anthropic:
-        click.echo(click.style('Anthropic hosted provider is not configured.', fg='red'))
-        return
-
-    click.echo(click.style('Start sync anthropic hosted providers.', fg='green'))
-    count = 0
-
-    new_quota_limit = hosted_model_providers.anthropic.quota_limit
-
-    page = 1
-    while True:
-        try:
-            providers = db.session.query(Provider).filter(
-                Provider.provider_name == 'anthropic',
-                Provider.provider_type == ProviderType.SYSTEM.value,
-                Provider.quota_type == ProviderQuotaType.TRIAL.value,
-                Provider.quota_limit != new_quota_limit
-            ).order_by(Provider.created_at.desc()).paginate(page=page, per_page=100)
-        except NotFound:
-            break
-
-        page += 1
-        for provider in providers:
-            try:
-                click.echo('Syncing tenant anthropic hosted provider: {}, origin: limit {}, used {}'
-                           .format(provider.tenant_id, provider.quota_limit, provider.quota_used))
-                original_quota_limit = provider.quota_limit
-                division = math.ceil(new_quota_limit / 1000)
-
-                provider.quota_limit = new_quota_limit if original_quota_limit == 1000 \
-                    else original_quota_limit * division
-                provider.quota_used = division * provider.quota_used
-                db.session.commit()
-
-                count += 1
-            except Exception as e:
-                click.echo(click.style(
-                    'Sync tenant anthropic hosted provider error: {} {}'.format(e.__class__.__name__, str(e)),
-                    fg='red'))
-                continue
-
-    click.echo(click.style('Congratulations! Synced {} anthropic hosted providers.'.format(count), fg='green'))
-
-
 @click.command('create-qdrant-indexes', help='Create qdrant indexes.')
 def create_qdrant_indexes():
+    """
+    Migrate other vector database datas to Qdrant.
+    """
     click.echo(click.style('Start create qdrant indexes.', fg='green'))
     create_count = 0
 
@@ -339,26 +157,7 @@ def create_qdrant_indexes():
 
                             )
                         except Exception:
-                            try:
-                                embedding_model = model_manager.get_default_model_instance(
-                                    tenant_id=dataset.tenant_id,
-                                    model_type=ModelType.TEXT_EMBEDDING,
-                                )
-                                dataset.embedding_model = embedding_model.model
-                                dataset.embedding_model_provider = embedding_model.provider
-                            except Exception:
-
-                                provider = Provider(
-                                    id='provider_id',
-                                    tenant_id=dataset.tenant_id,
-                                    provider_name='openai',
-                                    provider_type=ProviderType.SYSTEM.value,
-                                    encrypted_config=json.dumps({'openai_api_key': 'TEST'}),
-                                    is_valid=True,
-                                )
-                                model_provider = OpenAIProvider(provider=provider)
-                                embedding_model = OpenAIEmbedding(name="text-embedding-ada-002",
-                                                                  model_provider=model_provider)
+                            continue
                         embeddings = CacheEmbedding(embedding_model)
 
                         from core.index.vector_index.qdrant_vector_index import QdrantConfig, QdrantVectorIndex
@@ -393,380 +192,8 @@ def create_qdrant_indexes():
     click.echo(click.style('Congratulations! Create {} dataset indexes.'.format(create_count), fg='green'))
 
 
-@click.command('update-qdrant-indexes', help='Update qdrant indexes.')
-def update_qdrant_indexes():
-    click.echo(click.style('Start Update qdrant indexes.', fg='green'))
-    create_count = 0
-
-    page = 1
-    while True:
-        try:
-            datasets = db.session.query(Dataset).filter(Dataset.indexing_technique == 'high_quality') \
-                .order_by(Dataset.created_at.desc()).paginate(page=page, per_page=50)
-        except NotFound:
-            break
-
-        page += 1
-        for dataset in datasets:
-            if dataset.index_struct_dict:
-                if dataset.index_struct_dict['type'] != 'qdrant':
-                    try:
-                        click.echo('Update dataset qdrant index: {}'.format(dataset.id))
-                        try:
-                            embedding_model = ModelFactory.get_embedding_model(
-                                tenant_id=dataset.tenant_id,
-                                model_provider_name=dataset.embedding_model_provider,
-                                model_name=dataset.embedding_model
-                            )
-                        except Exception:
-                            provider = Provider(
-                                id='provider_id',
-                                tenant_id=dataset.tenant_id,
-                                provider_name='openai',
-                                provider_type=ProviderType.CUSTOM.value,
-                                encrypted_config=json.dumps({'openai_api_key': 'TEST'}),
-                                is_valid=True,
-                            )
-                            model_provider = OpenAIProvider(provider=provider)
-                            embedding_model = OpenAIEmbedding(name="text-embedding-ada-002",
-                                                              model_provider=model_provider)
-                        embeddings = CacheEmbedding(embedding_model)
-
-                        from core.index.vector_index.qdrant_vector_index import QdrantConfig, QdrantVectorIndex
-
-                        index = QdrantVectorIndex(
-                            dataset=dataset,
-                            config=QdrantConfig(
-                                endpoint=current_app.config.get('QDRANT_URL'),
-                                api_key=current_app.config.get('QDRANT_API_KEY'),
-                                root_path=current_app.root_path
-                            ),
-                            embeddings=embeddings
-                        )
-                        if index:
-                            index.update_qdrant_dataset(dataset)
-                            create_count += 1
-                        else:
-                            click.echo('passed.')
-                    except Exception as e:
-                        click.echo(
-                            click.style('Create dataset index error: {} {}'.format(e.__class__.__name__, str(e)),
-                                        fg='red'))
-                        continue
-
-    click.echo(click.style('Congratulations! Update {} dataset indexes.'.format(create_count), fg='green'))
-
-
-@click.command('normalization-collections', help='restore all collections in one')
-def normalization_collections():
-    click.echo(click.style('Start normalization collections.', fg='green'))
-    normalization_count = []
-    page = 1
-    while True:
-        try:
-            datasets = db.session.query(Dataset).filter(Dataset.indexing_technique == 'high_quality') \
-                .order_by(Dataset.created_at.desc()).paginate(page=page, per_page=100)
-        except NotFound:
-            break
-        datasets_result = datasets.items
-        page += 1
-        for i in range(0, len(datasets_result), 5):
-            threads = []
-            sub_datasets = datasets_result[i:i + 5]
-            for dataset in sub_datasets:
-                document_format_thread = threading.Thread(target=deal_dataset_vector, kwargs={
-                    'flask_app': current_app._get_current_object(),
-                    'dataset': dataset,
-                    'normalization_count': normalization_count
-                })
-                threads.append(document_format_thread)
-                document_format_thread.start()
-            for thread in threads:
-                thread.join()
-
-    click.echo(click.style('Congratulations! restore {} dataset indexes.'.format(len(normalization_count)), fg='green'))
-
-
-@click.command('add-qdrant-full-text-index', help='add qdrant full text index')
-def add_qdrant_full_text_index():
-    click.echo(click.style('Start add full text index.', fg='green'))
-    binds = db.session.query(DatasetCollectionBinding).all()
-    if binds and current_app.config['VECTOR_STORE'] == 'qdrant':
-        qdrant_url = current_app.config['QDRANT_URL']
-        qdrant_api_key = current_app.config['QDRANT_API_KEY']
-        client = qdrant_client.QdrantClient(
-            qdrant_url,
-            api_key=qdrant_api_key,  # For Qdrant Cloud, None for local instance
-        )
-        for bind in binds:
-            try:
-                text_index_params = TextIndexParams(
-                    type=TextIndexType.TEXT,
-                    tokenizer=TokenizerType.MULTILINGUAL,
-                    min_token_len=2,
-                    max_token_len=20,
-                    lowercase=True
-                )
-                client.create_payload_index(bind.collection_name, 'page_content',
-                                            field_schema=text_index_params)
-            except Exception as e:
-                click.echo(
-                    click.style('Create full text index error: {} {}'.format(e.__class__.__name__, str(e)),
-                                fg='red'))
-            click.echo(
-                click.style(
-                    'Congratulations! add collection {} full text index successful.'.format(bind.collection_name),
-                    fg='green'))
-
-
-def deal_dataset_vector(flask_app: Flask, dataset: Dataset, normalization_count: list):
-    with flask_app.app_context():
-        try:
-            click.echo('restore dataset index: {}'.format(dataset.id))
-            try:
-                embedding_model = ModelFactory.get_embedding_model(
-                    tenant_id=dataset.tenant_id,
-                    model_provider_name=dataset.embedding_model_provider,
-                    model_name=dataset.embedding_model
-                )
-            except Exception:
-                provider = Provider(
-                    id='provider_id',
-                    tenant_id=dataset.tenant_id,
-                    provider_name='openai',
-                    provider_type=ProviderType.CUSTOM.value,
-                    encrypted_config=json.dumps({'openai_api_key': 'TEST'}),
-                    is_valid=True,
-                )
-                model_provider = OpenAIProvider(provider=provider)
-                embedding_model = OpenAIEmbedding(name="text-embedding-ada-002",
-                                                  model_provider=model_provider)
-            embeddings = CacheEmbedding(embedding_model)
-            dataset_collection_binding = db.session.query(DatasetCollectionBinding). \
-                filter(DatasetCollectionBinding.provider_name == embedding_model.model_provider.provider_name,
-                       DatasetCollectionBinding.model_name == embedding_model.name). \
-                order_by(DatasetCollectionBinding.created_at). \
-                first()
-
-            if not dataset_collection_binding:
-                dataset_collection_binding = DatasetCollectionBinding(
-                    provider_name=embedding_model.model_provider.provider_name,
-                    model_name=embedding_model.name,
-                    collection_name="Vector_index_" + str(uuid.uuid4()).replace("-", "_") + '_Node'
-                )
-                db.session.add(dataset_collection_binding)
-                db.session.commit()
-
-            from core.index.vector_index.qdrant_vector_index import QdrantConfig, QdrantVectorIndex
-
-            index = QdrantVectorIndex(
-                dataset=dataset,
-                config=QdrantConfig(
-                    endpoint=current_app.config.get('QDRANT_URL'),
-                    api_key=current_app.config.get('QDRANT_API_KEY'),
-                    root_path=current_app.root_path
-                ),
-                embeddings=embeddings
-            )
-            if index:
-                # index.delete_by_group_id(dataset.id)
-                index.restore_dataset_in_one(dataset, dataset_collection_binding)
-            else:
-                click.echo('passed.')
-            normalization_count.append(1)
-        except Exception as e:
-            click.echo(
-                click.style('Create dataset index error: {} {}'.format(e.__class__.__name__, str(e)),
-                            fg='red'))
-
-
-@click.command('update_app_model_configs', help='Migrate data to support paragraph variable.')
-@click.option("--batch-size", default=500, help="Number of records to migrate in each batch.")
-def update_app_model_configs(batch_size):
-    pre_prompt_template = '{{default_input}}'
-
-    click.secho("Start migrate old data that the text generator can support paragraph variable.", fg='green')
-
-    total_records = db.session.query(AppModelConfig) \
-        .join(App, App.app_model_config_id == AppModelConfig.id) \
-        .filter(App.mode == 'completion') \
-        .count()
-
-    if total_records == 0:
-        click.secho("No data to migrate.", fg='green')
-        return
-
-    num_batches = (total_records + batch_size - 1) // batch_size
-
-    with tqdm(total=total_records, desc="Migrating Data") as pbar:
-        for i in range(num_batches):
-            offset = i * batch_size
-            limit = min(batch_size, total_records - offset)
-
-            click.secho(f"Fetching batch {i + 1}/{num_batches} from source database...", fg='green')
-
-            data_batch = db.session.query(AppModelConfig) \
-                .join(App, App.app_model_config_id == AppModelConfig.id) \
-                .filter(App.mode == 'completion') \
-                .order_by(App.created_at) \
-                .offset(offset).limit(limit).all()
-
-            if not data_batch:
-                click.secho("No more data to migrate.", fg='green')
-                break
-
-            try:
-                click.secho(f"Migrating {len(data_batch)} records...", fg='green')
-                for data in data_batch:
-                    # click.secho(f"Migrating data {data.id}, pre_prompt: {data.pre_prompt}, user_input_form: {data.user_input_form}", fg='green')
-
-                    if data.pre_prompt is None:
-                        data.pre_prompt = pre_prompt_template
-                    else:
-                        if pre_prompt_template in data.pre_prompt:
-                            continue
-                        data.pre_prompt += pre_prompt_template
-
-                    app_data = db.session.query(App) \
-                        .filter(App.id == data.app_id) \
-                        .one()
-
-                    account_data = db.session.query(Account) \
-                        .join(TenantAccountJoin, Account.id == TenantAccountJoin.account_id) \
-                        .filter(TenantAccountJoin.role == 'owner') \
-                        .filter(TenantAccountJoin.tenant_id == app_data.tenant_id) \
-                        .one_or_none()
-
-                    if not account_data:
-                        continue
-
-                    if data.user_input_form is None or data.user_input_form == 'null':
-                        data.user_input_form = json.dumps(user_input_form_template[account_data.interface_language])
-                    else:
-                        raw_json_data = json.loads(data.user_input_form)
-                        raw_json_data.append(user_input_form_template[account_data.interface_language][0])
-                        data.user_input_form = json.dumps(raw_json_data)
-
-                    # click.secho(f"Updated data {data.id}, pre_prompt: {data.pre_prompt}, user_input_form: {data.user_input_form}", fg='green')
-
-                db.session.commit()
-
-            except Exception as e:
-                click.secho(f"Error while migrating data: {e}, app_id: {data.app_id}, app_model_config_id: {data.id}",
-                            fg='red')
-                continue
-
-            click.secho(f"Successfully migrated batch {i + 1}/{num_batches}.", fg='green')
-
-            pbar.update(len(data_batch))
-
-
-@click.command('migrate_default_input_to_dataset_query_variable')
-@click.option("--batch-size", default=500, help="Number of records to migrate in each batch.")
-def migrate_default_input_to_dataset_query_variable(batch_size):
-    click.secho("Starting...", fg='green')
-
-    total_records = db.session.query(AppModelConfig) \
-        .join(App, App.app_model_config_id == AppModelConfig.id) \
-        .filter(App.mode == 'completion') \
-        .filter(AppModelConfig.dataset_query_variable == None) \
-        .count()
-
-    if total_records == 0:
-        click.secho("No data to migrate.", fg='green')
-        return
-
-    num_batches = (total_records + batch_size - 1) // batch_size
-
-    with tqdm(total=total_records, desc="Migrating Data") as pbar:
-        for i in range(num_batches):
-            offset = i * batch_size
-            limit = min(batch_size, total_records - offset)
-
-            click.secho(f"Fetching batch {i + 1}/{num_batches} from source database...", fg='green')
-
-            data_batch = db.session.query(AppModelConfig) \
-                .join(App, App.app_model_config_id == AppModelConfig.id) \
-                .filter(App.mode == 'completion') \
-                .filter(AppModelConfig.dataset_query_variable == None) \
-                .order_by(App.created_at) \
-                .offset(offset).limit(limit).all()
-
-            if not data_batch:
-                click.secho("No more data to migrate.", fg='green')
-                break
-
-            try:
-                click.secho(f"Migrating {len(data_batch)} records...", fg='green')
-                for data in data_batch:
-                    config = AppModelConfig.to_dict(data)
-
-                    tools = config["agent_mode"]["tools"]
-                    dataset_exists = "dataset" in str(tools)
-                    if not dataset_exists:
-                        continue
-
-                    user_input_form = config.get("user_input_form", [])
-                    for form in user_input_form:
-                        paragraph = form.get('paragraph')
-                        if paragraph \
-                                and paragraph.get('variable') == 'query':
-                            data.dataset_query_variable = 'query'
-                            break
-
-                        if paragraph \
-                                and paragraph.get('variable') == 'default_input':
-                            data.dataset_query_variable = 'default_input'
-                            break
-
-                db.session.commit()
-
-            except Exception as e:
-                click.secho(f"Error while migrating data: {e}, app_id: {data.app_id}, app_model_config_id: {data.id}",
-                            fg='red')
-                continue
-
-            click.secho(f"Successfully migrated batch {i + 1}/{num_batches}.", fg='green')
-
-            pbar.update(len(data_batch))
-
-
-@click.command('add-annotation-question-field-value', help='add annotation question value')
-def add_annotation_question_field_value():
-    click.echo(click.style('Start add annotation question value.', fg='green'))
-    message_annotations = db.session.query(MessageAnnotation).all()
-    message_annotation_deal_count = 0
-    if message_annotations:
-        for message_annotation in message_annotations:
-            try:
-                if message_annotation.message_id and not message_annotation.question:
-                    message = db.session.query(Message).filter(
-                        Message.id == message_annotation.message_id
-                    ).first()
-                    message_annotation.question = message.query
-                    db.session.add(message_annotation)
-                    db.session.commit()
-                    message_annotation_deal_count += 1
-            except Exception as e:
-                click.echo(
-                    click.style('Add annotation question value error: {} {}'.format(e.__class__.__name__, str(e)),
-                                fg='red'))
-            click.echo(
-                click.style(f'Congratulations! add annotation question value successful. Deal count {message_annotation_deal_count}', fg='green'))
-
-
 def register_commands(app):
     app.cli.add_command(reset_password)
     app.cli.add_command(reset_email)
-    app.cli.add_command(generate_invitation_codes)
     app.cli.add_command(reset_encrypt_key_pair)
-    app.cli.add_command(recreate_all_dataset_indexes)
-    app.cli.add_command(sync_anthropic_hosted_providers)
-    app.cli.add_command(clean_unused_dataset_indexes)
     app.cli.add_command(create_qdrant_indexes)
-    app.cli.add_command(update_qdrant_indexes)
-    app.cli.add_command(update_app_model_configs)
-    app.cli.add_command(normalization_collections)
-    app.cli.add_command(migrate_default_input_to_dataset_query_variable)
-    app.cli.add_command(add_qdrant_full_text_index)
-    app.cli.add_command(add_annotation_question_field_value)

api/config.py

@@ -40,17 +40,11 @@ DEFAULTS = {
     'HOSTED_OPENAI_QUOTA_LIMIT': 200,
     'HOSTED_OPENAI_TRIAL_ENABLED': 'False',
     'HOSTED_OPENAI_PAID_ENABLED': 'False',
-    'HOSTED_OPENAI_PAID_INCREASE_QUOTA': 1,
-    'HOSTED_OPENAI_PAID_MIN_QUANTITY': 1,
-    'HOSTED_OPENAI_PAID_MAX_QUANTITY': 1,
     'HOSTED_AZURE_OPENAI_ENABLED': 'False',
     'HOSTED_AZURE_OPENAI_QUOTA_LIMIT': 200,
     'HOSTED_ANTHROPIC_QUOTA_LIMIT': 600000,
     'HOSTED_ANTHROPIC_TRIAL_ENABLED': 'False',
     'HOSTED_ANTHROPIC_PAID_ENABLED': 'False',
-    'HOSTED_ANTHROPIC_PAID_INCREASE_QUOTA': 1,
-    'HOSTED_ANTHROPIC_PAID_MIN_QUANTITY': 1,
-    'HOSTED_ANTHROPIC_PAID_MAX_QUANTITY': 1,
     'HOSTED_MODERATION_ENABLED': 'False',
     'HOSTED_MODERATION_PROVIDERS': '',
     'CLEAN_DAY_SETTING': 30,
@@ -93,7 +87,7 @@ class Config:
         # ------------------------
         # General Configurations.
         # ------------------------
-        self.CURRENT_VERSION = "0.5.2"
+        self.CURRENT_VERSION = "0.5.4"
         self.COMMIT_SHA = get_env('COMMIT_SHA')
         self.EDITION = "SELF_HOSTED"
         self.DEPLOY_ENV = get_env('DEPLOY_ENV')
@@ -262,10 +256,6 @@ class Config:
         self.HOSTED_OPENAI_TRIAL_ENABLED = get_bool_env('HOSTED_OPENAI_TRIAL_ENABLED')
         self.HOSTED_OPENAI_QUOTA_LIMIT = int(get_env('HOSTED_OPENAI_QUOTA_LIMIT'))
         self.HOSTED_OPENAI_PAID_ENABLED = get_bool_env('HOSTED_OPENAI_PAID_ENABLED')
-        self.HOSTED_OPENAI_PAID_STRIPE_PRICE_ID = get_env('HOSTED_OPENAI_PAID_STRIPE_PRICE_ID')
-        self.HOSTED_OPENAI_PAID_INCREASE_QUOTA = int(get_env('HOSTED_OPENAI_PAID_INCREASE_QUOTA'))
-        self.HOSTED_OPENAI_PAID_MIN_QUANTITY = int(get_env('HOSTED_OPENAI_PAID_MIN_QUANTITY'))
-        self.HOSTED_OPENAI_PAID_MAX_QUANTITY = int(get_env('HOSTED_OPENAI_PAID_MAX_QUANTITY'))
 
         self.HOSTED_AZURE_OPENAI_ENABLED = get_bool_env('HOSTED_AZURE_OPENAI_ENABLED')
         self.HOSTED_AZURE_OPENAI_API_KEY = get_env('HOSTED_AZURE_OPENAI_API_KEY')
@@ -277,10 +267,6 @@ class Config:
         self.HOSTED_ANTHROPIC_TRIAL_ENABLED = get_bool_env('HOSTED_ANTHROPIC_TRIAL_ENABLED')
         self.HOSTED_ANTHROPIC_QUOTA_LIMIT = int(get_env('HOSTED_ANTHROPIC_QUOTA_LIMIT'))
         self.HOSTED_ANTHROPIC_PAID_ENABLED = get_bool_env('HOSTED_ANTHROPIC_PAID_ENABLED')
-        self.HOSTED_ANTHROPIC_PAID_STRIPE_PRICE_ID = get_env('HOSTED_ANTHROPIC_PAID_STRIPE_PRICE_ID')
-        self.HOSTED_ANTHROPIC_PAID_INCREASE_QUOTA = int(get_env('HOSTED_ANTHROPIC_PAID_INCREASE_QUOTA'))
-        self.HOSTED_ANTHROPIC_PAID_MIN_QUANTITY = int(get_env('HOSTED_ANTHROPIC_PAID_MIN_QUANTITY'))
-        self.HOSTED_ANTHROPIC_PAID_MAX_QUANTITY = int(get_env('HOSTED_ANTHROPIC_PAID_MAX_QUANTITY'))
 
         self.HOSTED_MINIMAX_ENABLED = get_bool_env('HOSTED_MINIMAX_ENABLED')
         self.HOSTED_SPARK_ENABLED = get_bool_env('HOSTED_SPARK_ENABLED')

api/constants/languages.py

@@ -1,5 +1,6 @@
 
 import json
+
 from models.model import AppModelConfig
 
 languages = ['en-US', 'zh-Hans', 'pt-BR', 'es-ES', 'fr-FR', 'de-DE', 'ja-JP', 'ko-KR', 'ru-RU', 'it-IT']

api/constants/model_template.py

@@ -1,7 +1,5 @@
 import json
 
-from models.model import App, AppModelConfig
-
 model_templates = {
     # completion default mode
     'completion_default': {

api/controllers/console/__init__.py

@@ -11,13 +11,11 @@ from .app import (advanced_prompt_template, annotation, app, audio, completion,
                   model_config, site, statistic)
 # Import auth controllers
 from .auth import activate, data_source_oauth, login, oauth
+# Import billing controllers
+from .billing import billing
 # Import datasets controllers
 from .datasets import data_source, datasets, datasets_document, datasets_segments, file, hit_testing
 # Import explore controllers
 from .explore import audio, completion, conversation, installed_app, message, parameter, recommended_app, saved_message
 # Import workspace controllers
 from .workspace import account, members, model_providers, models, tool_providers, workspace
-# Import billing controllers
-from .billing import billing
-# Import operation controllers
-from .operation import operation

api/controllers/console/admin.py

@@ -1,14 +1,15 @@
 import os
 from functools import wraps
 
+from flask import request
+from flask_restful import Resource, reqparse
+from werkzeug.exceptions import NotFound, Unauthorized
+
+from constants.languages import supported_language
 from controllers.console import api
 from controllers.console.wraps import only_edition_cloud
 from extensions.ext_database import db
-from flask import request
-from flask_restful import Resource, reqparse
-from constants.languages import supported_language
 from models.model import App, InstalledApp, RecommendedApp
-from werkzeug.exceptions import NotFound, Unauthorized
 
 
 def admin_required(view):

api/controllers/console/apikey.py

@@ -1,12 +1,13 @@
 import flask_restful
-from extensions.ext_database import db
 from flask_login import current_user
 from flask_restful import Resource, fields, marshal_with
+from werkzeug.exceptions import Forbidden
+
+from extensions.ext_database import db
 from libs.helper import TimestampField
 from libs.login import login_required
 from models.dataset import Dataset
 from models.model import ApiToken, App
-from werkzeug.exceptions import Forbidden
 
 from . import api
 from .setup import setup_required

api/controllers/console/app/advanced_prompt_template.py

@@ -1,7 +1,8 @@
+from flask_restful import Resource, reqparse
+
 from controllers.console import api
 from controllers.console.setup import setup_required
 from controllers.console.wraps import account_initialization_required
-from flask_restful import Resource, reqparse
 from libs.login import login_required
 from services.advanced_prompt_template_service import AdvancedPromptTemplateService
 

api/controllers/console/app/annotation.py

@@ -1,17 +1,20 @@
+from flask import request
+from flask_login import current_user
+from flask_restful import Resource, marshal, marshal_with, reqparse
+from werkzeug.exceptions import Forbidden
+
 from controllers.console import api
 from controllers.console.app.error import NoFileUploadedError
 from controllers.console.datasets.error import TooManyFilesError
 from controllers.console.setup import setup_required
 from controllers.console.wraps import account_initialization_required, cloud_edition_billing_resource_check
 from extensions.ext_redis import redis_client
-from fields.annotation_fields import (annotation_fields, annotation_hit_history_fields,
-                                      annotation_hit_history_list_fields, annotation_list_fields)
-from flask import request
-from flask_login import current_user
-from flask_restful import Resource, marshal, marshal_with, reqparse
+from fields.annotation_fields import (
+    annotation_fields,
+    annotation_hit_history_fields,
+)
 from libs.login import login_required
 from services.annotation_service import AppAnnotationService
-from werkzeug.exceptions import Forbidden
 
 
 class AnnotationReplyActionApi(Resource):

api/controllers/console/app/app.py

@@ -3,8 +3,12 @@ import json
 import logging
 from datetime import datetime
 
-from constants.model_template import model_templates
+from flask_login import current_user
+from flask_restful import Resource, abort, inputs, marshal_with, reqparse
+from werkzeug.exceptions import Forbidden
+
 from constants.languages import demo_model_templates, languages
+from constants.model_template import model_templates
 from controllers.console import api
 from controllers.console.app.error import AppNotFoundError, ProviderNotInitializeError
 from controllers.console.setup import setup_required
@@ -15,16 +19,16 @@ from core.model_runtime.entities.model_entities import ModelType
 from core.provider_manager import ProviderManager
 from events.app_event import app_was_created, app_was_deleted
 from extensions.ext_database import db
-from fields.app_fields import (app_detail_fields, app_detail_fields_with_site, app_pagination_fields,
-                               template_list_fields)
-from flask import current_app
-from flask_login import current_user
-from flask_restful import Resource, abort, inputs, marshal_with, reqparse
+from fields.app_fields import (
+    app_detail_fields,
+    app_detail_fields_with_site,
+    app_pagination_fields,
+    template_list_fields,
+)
 from libs.login import login_required
 from models.model import App, AppModelConfig, Site
-from models.tools import ApiToolProvider
 from services.app_model_config_service import AppModelConfigService
-from werkzeug.exceptions import Forbidden
+
 
 def _get_app(app_id, tenant_id):
     app = db.session.query(App).filter(App.id == app_id, App.tenant_id == tenant_id).first()
@@ -107,20 +111,33 @@ class AppListApi(Resource):
             # validate config
             model_config_dict = args['model_config']
 
-            # get model provider
-            model_manager = ModelManager()
-            model_instance = model_manager.get_default_model_instance(
-                tenant_id=current_user.current_tenant_id,
-                model_type=ModelType.LLM
+            # Get provider configurations
+            provider_manager = ProviderManager()
+            provider_configurations = provider_manager.get_configurations(current_user.current_tenant_id)
+
+            # get available models from provider_configurations
+            available_models = provider_configurations.get_models(
+                model_type=ModelType.LLM,
+                only_active=True
             )
 
-            if not model_instance:
-                raise ProviderNotInitializeError(
-                    f"No Default System Reasoning Model available. Please configure "
-                    f"in the Settings -> Model Provider.")
-            else:
-                model_config_dict["model"]["provider"] = model_instance.provider
-                model_config_dict["model"]["name"] = model_instance.model
+            # check if model is available
+            available_models_names = [f'{model.provider.provider}.{model.model}' for model in available_models]
+            provider_model = f"{model_config_dict['model']['provider']}.{model_config_dict['model']['name']}"
+            if provider_model not in available_models_names:
+                model_manager = ModelManager()
+                model_instance = model_manager.get_default_model_instance(
+                    tenant_id=current_user.current_tenant_id,
+                    model_type=ModelType.LLM
+                )
+
+                if not model_instance:
+                    raise ProviderNotInitializeError(
+                        f"No Default System Reasoning Model available. Please configure "
+                        f"in the Settings -> Model Provider.")
+                else:
+                    model_config_dict["model"]["provider"] = model_instance.provider
+                    model_config_dict["model"]["name"] = model_instance.model
 
             model_configuration = AppModelConfigService.validate_configuration(
                 tenant_id=current_user.current_tenant_id,

api/controllers/console/app/audio.py

@@ -1,24 +1,36 @@
 # -*- coding:utf-8 -*-
 import logging
 
+from flask import request
+from flask_restful import Resource
+from werkzeug.exceptions import InternalServerError
+
 import services
 from controllers.console import api
 from controllers.console.app import _get_app
-from controllers.console.app.error import (AppUnavailableError, AudioTooLargeError, CompletionRequestError,
-                                           NoAudioUploadedError, ProviderModelCurrentlyNotSupportError,
-                                           ProviderNotInitializeError, ProviderNotSupportSpeechToTextError,
-                                           ProviderQuotaExceededError, UnsupportedAudioTypeError)
+from controllers.console.app.error import (
+    AppUnavailableError,
+    AudioTooLargeError,
+    CompletionRequestError,
+    NoAudioUploadedError,
+    ProviderModelCurrentlyNotSupportError,
+    ProviderNotInitializeError,
+    ProviderNotSupportSpeechToTextError,
+    ProviderQuotaExceededError,
+    UnsupportedAudioTypeError,
+)
 from controllers.console.setup import setup_required
 from controllers.console.wraps import account_initialization_required
 from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError
 from core.model_runtime.errors.invoke import InvokeError
-from flask import request
-from flask_restful import Resource
 from libs.login import login_required
 from services.audio_service import AudioService
-from services.errors.audio import (AudioTooLargeServiceError, NoAudioUploadedServiceError,
-                                   ProviderNotSupportSpeechToTextServiceError, UnsupportedAudioTypeServiceError)
-from werkzeug.exceptions import InternalServerError
+from services.errors.audio import (
+    AudioTooLargeServiceError,
+    NoAudioUploadedServiceError,
+    ProviderNotSupportSpeechToTextServiceError,
+    UnsupportedAudioTypeServiceError,
+)
 
 
 class ChatMessageAudioApi(Resource):

api/controllers/console/app/completion.py

@@ -4,24 +4,30 @@ import logging
 from typing import Generator, Union
 
 import flask_login
+from flask import Response, stream_with_context
+from flask_restful import Resource, reqparse
+from werkzeug.exceptions import InternalServerError, NotFound
+
 import services
 from controllers.console import api
 from controllers.console.app import _get_app
-from controllers.console.app.error import (AppUnavailableError, CompletionRequestError, ConversationCompletedError,
-                                           ProviderModelCurrentlyNotSupportError, ProviderNotInitializeError,
-                                           ProviderQuotaExceededError)
+from controllers.console.app.error import (
+    AppUnavailableError,
+    CompletionRequestError,
+    ConversationCompletedError,
+    ProviderModelCurrentlyNotSupportError,
+    ProviderNotInitializeError,
+    ProviderQuotaExceededError,
+)
 from controllers.console.setup import setup_required
 from controllers.console.wraps import account_initialization_required
 from core.application_queue_manager import ApplicationQueueManager
 from core.entities.application_entities import InvokeFrom
 from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError
 from core.model_runtime.errors.invoke import InvokeError
-from flask import Response, stream_with_context
-from flask_restful import Resource, reqparse
 from libs.helper import uuid_value
 from libs.login import login_required
 from services.completion_service import CompletionService
-from werkzeug.exceptions import InternalServerError, NotFound
 
 
 # define completion message api for user

api/controllers/console/app/conversation.py

@@ -1,22 +1,27 @@
 from datetime import datetime
 
 import pytz
+from flask_login import current_user
+from flask_restful import Resource, marshal_with, reqparse
+from flask_restful.inputs import int_range
+from sqlalchemy import func, or_
+from sqlalchemy.orm import joinedload
+from werkzeug.exceptions import NotFound
+
 from controllers.console import api
 from controllers.console.app import _get_app
 from controllers.console.setup import setup_required
 from controllers.console.wraps import account_initialization_required
 from extensions.ext_database import db
-from fields.conversation_fields import (conversation_detail_fields, conversation_message_detail_fields,
-                                        conversation_pagination_fields, conversation_with_summary_pagination_fields)
-from flask_login import current_user
-from flask_restful import Resource, marshal_with, reqparse
-from flask_restful.inputs import int_range
+from fields.conversation_fields import (
+    conversation_detail_fields,
+    conversation_message_detail_fields,
+    conversation_pagination_fields,
+    conversation_with_summary_pagination_fields,
+)
 from libs.helper import datetime_string
 from libs.login import login_required
 from models.model import Conversation, Message, MessageAnnotation
-from sqlalchemy import func, or_
-from sqlalchemy.orm import joinedload
-from werkzeug.exceptions import NotFound
 
 
 class CompletionConversationApi(Resource):

api/controllers/console/app/generator.py

@@ -1,13 +1,18 @@
+from flask_login import current_user
+from flask_restful import Resource, reqparse
+
 from controllers.console import api
-from controllers.console.app.error import (CompletionRequestError, ProviderModelCurrentlyNotSupportError,
-                                           ProviderNotInitializeError, ProviderQuotaExceededError)
+from controllers.console.app.error import (
+    CompletionRequestError,
+    ProviderModelCurrentlyNotSupportError,
+    ProviderNotInitializeError,
+    ProviderQuotaExceededError,
+)
 from controllers.console.setup import setup_required
 from controllers.console.wraps import account_initialization_required
 from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError
 from core.generator.llm_generator import LLMGenerator
 from core.model_runtime.errors.invoke import InvokeError
-from flask_login import current_user
-from flask_restful import Resource, reqparse
 from libs.login import login_required
 
 

api/controllers/console/app/message.py

@@ -2,11 +2,21 @@ import json
 import logging
 from typing import Generator, Union
 
+from flask import Response, stream_with_context
+from flask_login import current_user
+from flask_restful import Resource, fields, marshal_with, reqparse
+from flask_restful.inputs import int_range
+from werkzeug.exceptions import Forbidden, InternalServerError, NotFound
+
 from controllers.console import api
 from controllers.console.app import _get_app
-from controllers.console.app.error import (AppMoreLikeThisDisabledError, CompletionRequestError,
-                                           ProviderModelCurrentlyNotSupportError, ProviderNotInitializeError,
-                                           ProviderQuotaExceededError)
+from controllers.console.app.error import (
+    AppMoreLikeThisDisabledError,
+    CompletionRequestError,
+    ProviderModelCurrentlyNotSupportError,
+    ProviderNotInitializeError,
+    ProviderQuotaExceededError,
+)
 from controllers.console.setup import setup_required
 from controllers.console.wraps import account_initialization_required, cloud_edition_billing_resource_check
 from core.entities.application_entities import InvokeFrom
@@ -14,10 +24,6 @@ from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotIni
 from core.model_runtime.errors.invoke import InvokeError
 from extensions.ext_database import db
 from fields.conversation_fields import annotation_fields, message_detail_fields
-from flask import Response, stream_with_context
-from flask_login import current_user
-from flask_restful import Resource, fields, marshal_with, reqparse
-from flask_restful.inputs import int_range
 from libs.helper import uuid_value
 from libs.infinite_scroll_pagination import InfiniteScrollPagination
 from libs.login import login_required
@@ -28,7 +34,6 @@ from services.errors.app import MoreLikeThisDisabledError
 from services.errors.conversation import ConversationNotExistsError
 from services.errors.message import MessageNotExistsError
 from services.message_service import MessageService
-from werkzeug.exceptions import Forbidden, InternalServerError, NotFound
 
 
 class ChatMessageListApi(Resource):

api/controllers/console/app/model_config.py

@@ -1,14 +1,15 @@
 # -*- coding:utf-8 -*-
 
+from flask import request
+from flask_login import current_user
+from flask_restful import Resource
+
 from controllers.console import api
 from controllers.console.app import _get_app
 from controllers.console.setup import setup_required
 from controllers.console.wraps import account_initialization_required
 from events.app_event import app_model_config_was_updated
 from extensions.ext_database import db
-from flask import request
-from flask_login import current_user
-from flask_restful import Resource
 from libs.login import login_required
 from models.model import AppModelConfig
 from services.app_model_config_service import AppModelConfigService

api/controllers/console/app/site.py

@@ -1,16 +1,17 @@
 # -*- coding:utf-8 -*-
+from flask_login import current_user
+from flask_restful import Resource, marshal_with, reqparse
+from werkzeug.exceptions import Forbidden, NotFound
+
+from constants.languages import supported_language
 from controllers.console import api
 from controllers.console.app import _get_app
 from controllers.console.setup import setup_required
 from controllers.console.wraps import account_initialization_required
 from extensions.ext_database import db
 from fields.app_fields import app_site_fields
-from flask_login import current_user
-from flask_restful import Resource, marshal_with, reqparse
-from constants.languages import supported_language
 from libs.login import login_required
 from models.model import Site
-from werkzeug.exceptions import Forbidden, NotFound
 
 
 def parse_app_site_args():

api/controllers/console/app/statistic.py

@@ -3,14 +3,15 @@ from datetime import datetime
 from decimal import Decimal
 
 import pytz
+from flask import jsonify
+from flask_login import current_user
+from flask_restful import Resource, reqparse
+
 from controllers.console import api
 from controllers.console.app import _get_app
 from controllers.console.setup import setup_required
 from controllers.console.wraps import account_initialization_required
 from extensions.ext_database import db
-from flask import jsonify
-from flask_login import current_user
-from flask_restful import Resource, reqparse
 from libs.helper import datetime_string
 from libs.login import login_required
 

api/controllers/console/auth/activate.py

@@ -2,14 +2,15 @@ import base64
 import secrets
 from datetime import datetime
 
+from flask_restful import Resource, reqparse
+
+from constants.languages import supported_language
 from controllers.console import api
 from controllers.console.error import AlreadyActivateError
 from extensions.ext_database import db
-from flask_restful import Resource, reqparse
 from libs.helper import email, str_len, timezone
-from constants.languages import supported_language
 from libs.password import hash_password, valid_password
-from models.account import AccountStatus, Tenant
+from models.account import AccountStatus
 from services.account_service import RegisterService
 
 

api/controllers/console/auth/data_source_oauth.py

@@ -1,13 +1,14 @@
 import logging
 
 import requests
-from controllers.console import api
 from flask import current_app, redirect, request
 from flask_login import current_user
 from flask_restful import Resource
+from werkzeug.exceptions import Forbidden
+
+from controllers.console import api
 from libs.login import login_required
 from libs.oauth_data_source import NotionOAuth
-from werkzeug.exceptions import Forbidden
 
 from ..setup import setup_required
 from ..wraps import account_initialization_required

api/controllers/console/auth/login.py

@@ -1,14 +1,14 @@
 # -*- coding:utf-8 -*-
-import flask
 import flask_login
+from flask import current_app, request
+from flask_restful import Resource, reqparse
+
 import services
 from controllers.console import api
 from controllers.console.setup import setup_required
-from flask import current_app, request
-from flask_restful import Resource, reqparse
 from libs.helper import email
 from libs.password import valid_password
-from services.account_service import AccountService, TenantService
+from services.account_service import AccountService
 
 
 class LoginApi(Resource):
@@ -30,11 +30,6 @@ class LoginApi(Resource):
         except services.errors.account.AccountLoginError:
             return {'code': 'unauthorized', 'message': 'Invalid email or password'}, 401
 
-        try:
-            TenantService.switch_tenant(account)
-        except Exception:
-            pass
-
         AccountService.update_last_login(account, request)
 
         # todo: return the user info
@@ -47,7 +42,6 @@ class LogoutApi(Resource):
 
     @setup_required
     def get(self):
-        flask.session.pop('workspace_id', None)
         flask_login.logout_user()
         return {'result': 'success'}
 

api/controllers/console/auth/oauth.py

@@ -3,10 +3,11 @@ from datetime import datetime
 from typing import Optional
 
 import requests
-from constants.languages import languages
-from extensions.ext_database import db
 from flask import current_app, redirect, request
 from flask_restful import Resource
+
+from constants.languages import languages
+from extensions.ext_database import db
 from libs.oauth import GitHubOAuth, GoogleOAuth, OAuthUserInfo
 from models.account import Account, AccountStatus
 from services.account_service import AccountService, RegisterService

api/controllers/console/billing/billing.py

@@ -1,8 +1,9 @@
+from flask_login import current_user
+from flask_restful import Resource, reqparse
+
 from controllers.console import api
 from controllers.console.setup import setup_required
 from controllers.console.wraps import account_initialization_required, only_edition_cloud
-from flask_login import current_user
-from flask_restful import Resource, reqparse
 from libs.login import login_required
 from services.billing_service import BillingService
 
@@ -20,7 +21,7 @@ class Subscription(Resource):
         parser.add_argument('interval', type=str, required=True, location='args', choices=['month', 'year'])
         args = parser.parse_args()
 
-        BillingService.is_tenant_owner(current_user)
+        BillingService.is_tenant_owner_or_admin(current_user)
 
         return BillingService.get_subscription(args['plan'],
                                                args['interval'],
@@ -35,8 +36,8 @@ class Invoices(Resource):
     @account_initialization_required
     @only_edition_cloud
     def get(self):
-        BillingService.is_tenant_owner(current_user)
-        return BillingService.get_invoices(current_user.email)
+        BillingService.is_tenant_owner_or_admin(current_user)
+        return BillingService.get_invoices(current_user.email, current_user.current_tenant_id)
 
 
 api.add_resource(Subscription, '/billing/subscription')

api/controllers/console/datasets/data_source.py

@@ -1,6 +1,11 @@
 import datetime
 import json
 
+from flask import request
+from flask_login import current_user
+from flask_restful import Resource, marshal_with, reqparse
+from werkzeug.exceptions import NotFound
+
 from controllers.console import api
 from controllers.console.setup import setup_required
 from controllers.console.wraps import account_initialization_required
@@ -8,15 +13,11 @@ from core.data_loader.loader.notion import NotionLoader
 from core.indexing_runner import IndexingRunner
 from extensions.ext_database import db
 from fields.data_source_fields import integrate_list_fields, integrate_notion_info_list_fields
-from flask import request
-from flask_login import current_user
-from flask_restful import Resource, marshal_with, reqparse
 from libs.login import login_required
 from models.dataset import Document
 from models.source import DataSourceBinding
 from services.dataset_service import DatasetService, DocumentService
 from tasks.document_indexing_sync_task import document_indexing_sync_task
-from werkzeug.exceptions import NotFound
 
 
 class DataSourceApi(Resource):

api/controllers/console/datasets/datasets.py

@@ -1,5 +1,10 @@
 # -*- coding:utf-8 -*-
 import flask_restful
+from flask import current_app, request
+from flask_login import current_user
+from flask_restful import Resource, marshal, marshal_with, reqparse
+from werkzeug.exceptions import Forbidden, NotFound
+
 import services
 from controllers.console import api
 from controllers.console.apikey import api_key_fields, api_key_list
@@ -15,14 +20,10 @@ from extensions.ext_database import db
 from fields.app_fields import related_app_list
 from fields.dataset_fields import dataset_detail_fields, dataset_query_detail_fields
 from fields.document_fields import document_status_fields
-from flask import current_app, request
-from flask_login import current_user
-from flask_restful import Resource, marshal, marshal_with, reqparse
 from libs.login import login_required
 from models.dataset import Dataset, Document, DocumentSegment
 from models.model import ApiToken, UploadFile
 from services.dataset_service import DatasetService, DocumentService
-from werkzeug.exceptions import Forbidden, NotFound
 
 
 def _validate_name(name):

api/controllers/console/datasets/datasets_document.py

@@ -2,35 +2,52 @@
 from datetime import datetime
 from typing import List
 
+from flask import request
+from flask_login import current_user
+from flask_restful import Resource, fields, marshal, marshal_with, reqparse
+from sqlalchemy import asc, desc
+from werkzeug.exceptions import Forbidden, NotFound
+
 import services
 from controllers.console import api
-from controllers.console.app.error import (ProviderModelCurrentlyNotSupportError, ProviderNotInitializeError,
-                                           ProviderQuotaExceededError)
-from controllers.console.datasets.error import (ArchivedDocumentImmutableError, DocumentAlreadyFinishedError,
-                                                DocumentIndexingError, InvalidActionError, InvalidMetadataError)
+from controllers.console.app.error import (
+    ProviderModelCurrentlyNotSupportError,
+    ProviderNotInitializeError,
+    ProviderQuotaExceededError,
+)
+from controllers.console.datasets.error import (
+    ArchivedDocumentImmutableError,
+    DocumentAlreadyFinishedError,
+    DocumentIndexingError,
+    InvalidActionError,
+    InvalidMetadataError,
+)
 from controllers.console.setup import setup_required
 from controllers.console.wraps import account_initialization_required, cloud_edition_billing_resource_check
-from core.errors.error import (LLMBadRequestError, ModelCurrentlyNotSupportError, ProviderTokenNotInitError,
-                               QuotaExceededError)
+from core.errors.error import (
+    LLMBadRequestError,
+    ModelCurrentlyNotSupportError,
+    ProviderTokenNotInitError,
+    QuotaExceededError,
+)
 from core.indexing_runner import IndexingRunner
 from core.model_manager import ModelManager
 from core.model_runtime.entities.model_entities import ModelType
 from core.model_runtime.errors.invoke import InvokeAuthorizationError
 from extensions.ext_database import db
 from extensions.ext_redis import redis_client
-from fields.document_fields import (dataset_and_document_fields, document_fields, document_status_fields,
-                                    document_with_segments_fields)
-from flask import request
-from flask_login import current_user
-from flask_restful import Resource, fields, marshal, marshal_with, reqparse
+from fields.document_fields import (
+    dataset_and_document_fields,
+    document_fields,
+    document_status_fields,
+    document_with_segments_fields,
+)
 from libs.login import login_required
 from models.dataset import Dataset, DatasetProcessRule, Document, DocumentSegment
 from models.model import UploadFile
 from services.dataset_service import DatasetService, DocumentService
-from sqlalchemy import asc, desc
 from tasks.add_document_to_index_task import add_document_to_index_task
 from tasks.remove_document_from_index_task import remove_document_from_index_task
-from werkzeug.exceptions import Forbidden, NotFound
 
 
 class DocumentResource(Resource):

api/controllers/console/datasets/datasets_segments.py

@@ -3,6 +3,11 @@ import uuid
 from datetime import datetime
 
 import pandas as pd
+from flask import request
+from flask_login import current_user
+from flask_restful import Resource, marshal, reqparse
+from werkzeug.exceptions import Forbidden, NotFound
+
 import services
 from controllers.console import api
 from controllers.console.app.error import ProviderNotInitializeError
@@ -15,16 +20,12 @@ from core.model_runtime.entities.model_entities import ModelType
 from extensions.ext_database import db
 from extensions.ext_redis import redis_client
 from fields.segment_fields import segment_fields
-from flask import request
-from flask_login import current_user
-from flask_restful import Resource, marshal, reqparse
 from libs.login import login_required
 from models.dataset import DocumentSegment
 from services.dataset_service import DatasetService, DocumentService, SegmentService
 from tasks.batch_create_segment_to_index_task import batch_create_segment_to_index_task
 from tasks.disable_segment_from_index_task import disable_segment_from_index_task
 from tasks.enable_segment_to_index_task import enable_segment_to_index_task
-from werkzeug.exceptions import Forbidden, NotFound
 
 
 class DatasetDocumentSegmentListApi(Resource):

api/controllers/console/datasets/file.py

@@ -1,15 +1,20 @@
-import services
-from controllers.console import api
-from controllers.console.datasets.error import (FileTooLargeError, NoFileUploadedError, TooManyFilesError,
-                                                UnsupportedFileTypeError)
-from controllers.console.setup import setup_required
-from controllers.console.wraps import account_initialization_required
-from fields.file_fields import file_fields, upload_config_fields
 from flask import current_app, request
 from flask_login import current_user
 from flask_restful import Resource, marshal_with
+
+import services
+from controllers.console import api
+from controllers.console.datasets.error import (
+    FileTooLargeError,
+    NoFileUploadedError,
+    TooManyFilesError,
+    UnsupportedFileTypeError,
+)
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required
+from fields.file_fields import file_fields, upload_config_fields
 from libs.login import login_required
-from services.file_service import FileService, ALLOWED_EXTENSIONS, UNSTRUSTURED_ALLOWED_EXTENSIONS
+from services.file_service import ALLOWED_EXTENSIONS, UNSTRUSTURED_ALLOWED_EXTENSIONS, FileService
 
 PREVIEW_WORDS_LIMIT = 3000
 

api/controllers/console/datasets/hit_testing.py

@@ -1,22 +1,31 @@
 import logging
 
+from flask_login import current_user
+from flask_restful import Resource, marshal, reqparse
+from werkzeug.exceptions import Forbidden, InternalServerError, NotFound
+
 import services
 from controllers.console import api
-from controllers.console.app.error import (CompletionRequestError, ProviderModelCurrentlyNotSupportError,
-                                           ProviderNotInitializeError, ProviderQuotaExceededError)
+from controllers.console.app.error import (
+    CompletionRequestError,
+    ProviderModelCurrentlyNotSupportError,
+    ProviderNotInitializeError,
+    ProviderQuotaExceededError,
+)
 from controllers.console.datasets.error import DatasetNotInitializedError, HighQualityDatasetOnlyError
 from controllers.console.setup import setup_required
 from controllers.console.wraps import account_initialization_required
-from core.errors.error import (LLMBadRequestError, ModelCurrentlyNotSupportError, ProviderTokenNotInitError,
-                               QuotaExceededError)
+from core.errors.error import (
+    LLMBadRequestError,
+    ModelCurrentlyNotSupportError,
+    ProviderTokenNotInitError,
+    QuotaExceededError,
+)
 from core.model_runtime.errors.invoke import InvokeError
 from fields.hit_testing_fields import hit_testing_record_fields
-from flask_login import current_user
-from flask_restful import Resource, marshal, reqparse
 from libs.login import login_required
 from services.dataset_service import DatasetService
 from services.hit_testing_service import HitTestingService
-from werkzeug.exceptions import Forbidden, InternalServerError, NotFound
 
 
 class HitTestingApi(Resource):

api/controllers/console/error.py

@@ -13,6 +13,16 @@ class NotSetupError(BaseHTTPException):
                   "Please proceed with the initialization and installation process first."
     code = 401
 
+class NotInitValidateError(BaseHTTPException):
+    error_code = 'not_init_validated'
+    description = "Init validation has not been completed yet. " \
+                  "Please proceed with the init validation process first."
+    code = 401
+
+class InitValidateFailedError(BaseHTTPException):
+    error_code = 'init_validate_failed'
+    description = "Init validation failed. Please check the password and try again."
+    code = 401
 
 class AccountNotLinkTenantError(BaseHTTPException):
     error_code = 'account_not_link_tenant'

api/controllers/console/explore/audio.py

@@ -1,21 +1,33 @@
 # -*- coding:utf-8 -*-
 import logging
 
+from flask import request
+from werkzeug.exceptions import InternalServerError
+
 import services
 from controllers.console import api
-from controllers.console.app.error import (AppUnavailableError, AudioTooLargeError, CompletionRequestError,
-                                           NoAudioUploadedError, ProviderModelCurrentlyNotSupportError,
-                                           ProviderNotInitializeError, ProviderNotSupportSpeechToTextError,
-                                           ProviderQuotaExceededError, UnsupportedAudioTypeError)
+from controllers.console.app.error import (
+    AppUnavailableError,
+    AudioTooLargeError,
+    CompletionRequestError,
+    NoAudioUploadedError,
+    ProviderModelCurrentlyNotSupportError,
+    ProviderNotInitializeError,
+    ProviderNotSupportSpeechToTextError,
+    ProviderQuotaExceededError,
+    UnsupportedAudioTypeError,
+)
 from controllers.console.explore.wraps import InstalledAppResource
 from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError
 from core.model_runtime.errors.invoke import InvokeError
-from flask import request
 from models.model import AppModelConfig
 from services.audio_service import AudioService
-from services.errors.audio import (AudioTooLargeServiceError, NoAudioUploadedServiceError,
-                                   ProviderNotSupportSpeechToTextServiceError, UnsupportedAudioTypeServiceError)
-from werkzeug.exceptions import InternalServerError
+from services.errors.audio import (
+    AudioTooLargeServiceError,
+    NoAudioUploadedServiceError,
+    ProviderNotSupportSpeechToTextServiceError,
+    UnsupportedAudioTypeServiceError,
+)
 
 
 class ChatAudioApi(InstalledAppResource):

api/controllers/console/explore/completion.py

@@ -4,11 +4,21 @@ import logging
 from datetime import datetime
 from typing import Generator, Union
 
+from flask import Response, stream_with_context
+from flask_login import current_user
+from flask_restful import reqparse
+from werkzeug.exceptions import InternalServerError, NotFound
+
 import services
 from controllers.console import api
-from controllers.console.app.error import (AppUnavailableError, CompletionRequestError, ConversationCompletedError,
-                                           ProviderModelCurrentlyNotSupportError, ProviderNotInitializeError,
-                                           ProviderQuotaExceededError)
+from controllers.console.app.error import (
+    AppUnavailableError,
+    CompletionRequestError,
+    ConversationCompletedError,
+    ProviderModelCurrentlyNotSupportError,
+    ProviderNotInitializeError,
+    ProviderQuotaExceededError,
+)
 from controllers.console.explore.error import NotChatAppError, NotCompletionAppError
 from controllers.console.explore.wraps import InstalledAppResource
 from core.application_queue_manager import ApplicationQueueManager
@@ -16,12 +26,8 @@ from core.entities.application_entities import InvokeFrom
 from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError
 from core.model_runtime.errors.invoke import InvokeError
 from extensions.ext_database import db
-from flask import Response, stream_with_context
-from flask_login import current_user
-from flask_restful import reqparse
 from libs.helper import uuid_value
 from services.completion_service import CompletionService
-from werkzeug.exceptions import InternalServerError, NotFound
 
 
 # define completion api for user

api/controllers/console/explore/conversation.py

@@ -1,16 +1,17 @@
 # -*- coding:utf-8 -*-
+from flask_login import current_user
+from flask_restful import marshal_with, reqparse
+from flask_restful.inputs import int_range
+from werkzeug.exceptions import NotFound
+
 from controllers.console import api
 from controllers.console.explore.error import NotChatAppError
 from controllers.console.explore.wraps import InstalledAppResource
 from fields.conversation_fields import conversation_infinite_scroll_pagination_fields, simple_conversation_fields
-from flask_login import current_user
-from flask_restful import fields, marshal_with, reqparse
-from flask_restful.inputs import int_range
-from libs.helper import TimestampField, uuid_value
+from libs.helper import uuid_value
 from services.conversation_service import ConversationService
 from services.errors.conversation import ConversationNotExistsError, LastConversationNotExistsError
 from services.web_conversation_service import WebConversationService
-from werkzeug.exceptions import NotFound
 
 
 class ConversationListApi(InstalledAppResource):

api/controllers/console/explore/installed_app.py

@@ -1,18 +1,19 @@
 # -*- coding:utf-8 -*-
 from datetime import datetime
 
+from flask_login import current_user
+from flask_restful import Resource, inputs, marshal_with, reqparse
+from sqlalchemy import and_
+from werkzeug.exceptions import BadRequest, Forbidden, NotFound
+
 from controllers.console import api
 from controllers.console.explore.wraps import InstalledAppResource
 from controllers.console.wraps import account_initialization_required, cloud_edition_billing_resource_check
 from extensions.ext_database import db
 from fields.installed_app_fields import installed_app_list_fields
-from flask_login import current_user
-from flask_restful import Resource, inputs, marshal_with, reqparse
 from libs.login import login_required
 from models.model import App, InstalledApp, RecommendedApp
 from services.account_service import TenantService
-from sqlalchemy import and_
-from werkzeug.exceptions import BadRequest, Forbidden, NotFound
 
 
 class InstalledAppsListApi(Resource):

api/controllers/console/explore/message.py

@@ -3,29 +3,37 @@ import json
 import logging
 from typing import Generator, Union
 
+from flask import Response, stream_with_context
+from flask_login import current_user
+from flask_restful import marshal_with, reqparse
+from flask_restful.inputs import int_range
+from werkzeug.exceptions import InternalServerError, NotFound
+
 import services
 from controllers.console import api
-from controllers.console.app.error import (AppMoreLikeThisDisabledError, CompletionRequestError,
-                                           ProviderModelCurrentlyNotSupportError, ProviderNotInitializeError,
-                                           ProviderQuotaExceededError)
-from controllers.console.explore.error import (AppSuggestedQuestionsAfterAnswerDisabledError, NotChatAppError,
-                                               NotCompletionAppError)
+from controllers.console.app.error import (
+    AppMoreLikeThisDisabledError,
+    CompletionRequestError,
+    ProviderModelCurrentlyNotSupportError,
+    ProviderNotInitializeError,
+    ProviderQuotaExceededError,
+)
+from controllers.console.explore.error import (
+    AppSuggestedQuestionsAfterAnswerDisabledError,
+    NotChatAppError,
+    NotCompletionAppError,
+)
 from controllers.console.explore.wraps import InstalledAppResource
 from core.entities.application_entities import InvokeFrom
 from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError
 from core.model_runtime.errors.invoke import InvokeError
 from fields.message_fields import message_infinite_scroll_pagination_fields
-from flask import Response, stream_with_context
-from flask_login import current_user
-from flask_restful import marshal_with, reqparse, fields
-from flask_restful.inputs import int_range
-from libs.helper import uuid_value, TimestampField
+from libs.helper import uuid_value
 from services.completion_service import CompletionService
 from services.errors.app import MoreLikeThisDisabledError
 from services.errors.conversation import ConversationNotExistsError
 from services.errors.message import MessageNotExistsError, SuggestedQuestionsAfterAnswerDisabledError
 from services.message_service import MessageService
-from werkzeug.exceptions import InternalServerError, NotFound
 
 
 class MessageListApi(InstalledAppResource):

api/controllers/console/explore/parameter.py

@@ -1,14 +1,15 @@
 # -*- coding:utf-8 -*-
 import json
 
-from controllers.console import api
-from controllers.console.explore.wraps import InstalledAppResource
 from flask import current_app
 from flask_restful import fields, marshal_with
-from models.model import InstalledApp, AppModelConfig
+
+from controllers.console import api
+from controllers.console.explore.wraps import InstalledAppResource
+from extensions.ext_database import db
+from models.model import AppModelConfig, InstalledApp
 from models.tools import ApiToolProvider
 
-from extensions.ext_database import db
 
 class AppParameterApi(InstalledAppResource):
     """Resource for app variables."""

api/controllers/console/explore/recommended_app.py

@@ -1,15 +1,16 @@
 # -*- coding:utf-8 -*-
+from flask_login import current_user
+from flask_restful import Resource, fields, marshal_with
+from sqlalchemy import and_
+
+from constants.languages import languages
 from controllers.console import api
 from controllers.console.app.error import AppNotFoundError
 from controllers.console.wraps import account_initialization_required
 from extensions.ext_database import db
-from flask_login import current_user
-from flask_restful import Resource, fields, marshal_with
 from libs.login import login_required
 from models.model import App, InstalledApp, RecommendedApp
 from services.account_service import TenantService
-from sqlalchemy import and_
-from constants.languages import languages
 
 app_fields = {
     'id': fields.String,

api/controllers/console/explore/saved_message.py

@@ -1,14 +1,15 @@
+from flask_login import current_user
+from flask_restful import fields, marshal_with, reqparse
+from flask_restful.inputs import int_range
+from werkzeug.exceptions import NotFound
+
 from controllers.console import api
 from controllers.console.explore.error import NotCompletionAppError
 from controllers.console.explore.wraps import InstalledAppResource
 from fields.conversation_fields import message_file_fields
-from flask_login import current_user
-from flask_restful import fields, marshal_with, reqparse
-from flask_restful.inputs import int_range
 from libs.helper import TimestampField, uuid_value
 from services.errors.message import MessageNotExistsError
 from services.saved_message_service import SavedMessageService
-from werkzeug.exceptions import NotFound
 
 feedback_fields = {
     'rating': fields.String

api/controllers/console/explore/wraps.py

@@ -1,12 +1,13 @@
 from functools import wraps
 
+from flask_login import current_user
+from flask_restful import Resource
+from werkzeug.exceptions import NotFound
+
 from controllers.console.wraps import account_initialization_required
 from extensions.ext_database import db
-from flask_login import current_user
-from flask_restful import Resource
 from libs.login import login_required
 from models.model import InstalledApp
-from werkzeug.exceptions import NotFound
 
 
 def installed_app_required(view=None):

api/controllers/console/extension.py

@@ -1,9 +1,10 @@
+from flask_login import current_user
+from flask_restful import Resource, marshal_with, reqparse
+
 from controllers.console import api
 from controllers.console.setup import setup_required
 from controllers.console.wraps import account_initialization_required
 from fields.api_based_extension_fields import api_based_extension_fields
-from flask_login import current_user
-from flask_restful import Resource, marshal_with, reqparse
 from libs.login import login_required
 from models.api_based_extension import APIBasedExtension
 from services.api_based_extension_service import APIBasedExtensionService

api/controllers/console/feature.py

@@ -1,12 +1,15 @@
 from flask_login import current_user
 from flask_restful import Resource
+
 from services.feature_service import FeatureService
 
 from . import api
+from .wraps import cloud_utm_record
 
 
 class FeatureApi(Resource):
 
+    @cloud_utm_record
     def get(self):
         return FeatureService.get_features(current_user.current_tenant_id).dict()
 

api/controllers/console/init_validate.py

@@ -0,0 +1,49 @@
+import os
+
+from flask import current_app, session
+from flask_restful import Resource, reqparse
+
+from libs.helper import str_len
+from models.model import DifySetup
+from services.account_service import TenantService
+
+from . import api
+from .error import AlreadySetupError, InitValidateFailedError
+from .wraps import only_edition_self_hosted
+
+
+class InitValidateAPI(Resource):
+
+    def get(self):
+        init_status = get_init_validate_status()
+        if init_status:
+            return { 'status': 'finished' }
+        return {'status': 'not_started' }
+
+    @only_edition_self_hosted
+    def post(self):
+        # is tenant created
+        tenant_count = TenantService.get_tenant_count()
+        if tenant_count > 0:
+            raise AlreadySetupError()
+
+        parser = reqparse.RequestParser()
+        parser.add_argument('password', type=str_len(30),
+                            required=True, location='json')
+        input_password = parser.parse_args()['password']
+
+        if input_password != os.environ.get('INIT_PASSWORD'):
+            session['is_init_validated'] = False
+            raise InitValidateFailedError()
+            
+        session['is_init_validated'] = True
+        return {'result': 'success'}, 201
+
+def get_init_validate_status():
+    if current_app.config['EDITION'] == 'SELF_HOSTED':
+        if os.environ.get('INIT_PASSWORD'):
+            return session.get('is_init_validated') or DifySetup.query.first()
+    
+    return True
+
+api.add_resource(InitValidateAPI, '/init')

api/controllers/console/operation/operation.py

@@ -1,30 +0,0 @@
-from flask_login import current_user
-from flask_restful import Resource, reqparse
-
-from controllers.console import api
-from controllers.console.setup import setup_required
-from controllers.console.wraps import account_initialization_required, only_edition_cloud
-from libs.login import login_required
-from services.operation_service import OperationService
-
-
-class TenantUtm(Resource):
-
-    @setup_required
-    @login_required
-    @account_initialization_required
-    @only_edition_cloud
-    def post(self):
-
-        parser = reqparse.RequestParser()
-        parser.add_argument('utm_source', type=str, required=True)
-        parser.add_argument('utm_medium', type=str, required=True)
-        parser.add_argument('utm_campaign', type=str, required=False, default='')
-        parser.add_argument('utm_content', type=str, required=False, default='')
-        parser.add_argument('utm_term', type=str, required=False, default='')
-        args = parser.parse_args()
-
-        return OperationService.record_utm(current_user.current_tenant_id, args)
-
-
-api.add_resource(TenantUtm, '/operation/utm')

api/controllers/console/setup.py

@@ -1,16 +1,18 @@
 # -*- coding:utf-8 -*-
 from functools import wraps
 
-from extensions.ext_database import db
 from flask import current_app, request
 from flask_restful import Resource, reqparse
+
+from extensions.ext_database import db
 from libs.helper import email, str_len
 from libs.password import valid_password
 from models.model import DifySetup
 from services.account_service import AccountService, RegisterService, TenantService
 
 from . import api
-from .error import AlreadySetupError, NotSetupError
+from .error import AlreadySetupError, NotInitValidateError, NotSetupError
+from .init_validate import get_init_validate_status
 from .wraps import only_edition_self_hosted
 
 
@@ -24,7 +26,7 @@ class SetupApi(Resource):
                     'step': 'finished',
                     'setup_at': setup_status.setup_at.isoformat()
                 }
-            return {'step': 'not_start'}
+            return {'step': 'not_started'}
         return {'step': 'finished'}
 
     @only_edition_self_hosted
@@ -37,6 +39,9 @@ class SetupApi(Resource):
         tenant_count = TenantService.get_tenant_count()
         if tenant_count > 0:
             raise AlreadySetupError()
+    
+        if not get_init_validate_status():
+            raise NotInitValidateError()
 
         parser = reqparse.RequestParser()
         parser.add_argument('email', type=email,
@@ -71,7 +76,10 @@ def setup_required(view):
     @wraps(view)
     def decorated(*args, **kwargs):
         # check setup
-        if not get_setup_status():
+        if not get_init_validate_status():
+            raise NotInitValidateError()
+        
+        elif not get_setup_status():
             raise NotSetupError()
 
         return view(*args, **kwargs)

api/controllers/console/version.py

@@ -6,7 +6,6 @@ import logging
 import requests
 from flask import current_app
 from flask_restful import Resource, reqparse
-from werkzeug.exceptions import InternalServerError
 
 from . import api
 

api/controllers/console/workspace/account.py

@@ -2,17 +2,22 @@
 from datetime import datetime
 
 import pytz
-from controllers.console import api
-from controllers.console.setup import setup_required
-from controllers.console.workspace.error import (AccountAlreadyInitedError, CurrentPasswordIncorrectError,
-                                                 InvalidInvitationCodeError, RepeatPasswordNotMatchError)
-from controllers.console.wraps import account_initialization_required
-from extensions.ext_database import db
 from flask import current_app, request
 from flask_login import current_user
 from flask_restful import Resource, fields, marshal_with, reqparse
-from libs.helper import TimestampField, timezone
+
 from constants.languages import supported_language
+from controllers.console import api
+from controllers.console.setup import setup_required
+from controllers.console.workspace.error import (
+    AccountAlreadyInitedError,
+    CurrentPasswordIncorrectError,
+    InvalidInvitationCodeError,
+    RepeatPasswordNotMatchError,
+)
+from controllers.console.wraps import account_initialization_required
+from extensions.ext_database import db
+from libs.helper import TimestampField, timezone
 from libs.login import login_required
 from models.account import AccountIntegrate, InvitationCode
 from services.account_service import AccountService

api/controllers/console/workspace/members.py

@@ -52,10 +52,12 @@ class MemberInviteEmailApi(Resource):
         parser = reqparse.RequestParser()
         parser.add_argument('emails', type=str, required=True, location='json', action='append')
         parser.add_argument('role', type=str, required=True, default='admin', location='json')
+        parser.add_argument('language', type=str, required=False, location='json')
         args = parser.parse_args()
 
         invitee_emails = args['emails']
         invitee_role = args['role']
+        interface_language = args['language']
         if invitee_role not in ['admin', 'normal']:
             return {'code': 'invalid-role', 'message': 'Invalid role'}, 400
 
@@ -64,8 +66,7 @@ class MemberInviteEmailApi(Resource):
         console_web_url = current_app.config.get("CONSOLE_WEB_URL")
         for invitee_email in invitee_emails:
             try:
-                token = RegisterService.invite_new_member(inviter.current_tenant, invitee_email, role=invitee_role,
-                                                          inviter=inviter)
+                token = RegisterService.invite_new_member(inviter.current_tenant, invitee_email, interface_language, role=invitee_role, inviter=inviter)
                 invitation_results.append({
                     'status': 'success',
                     'email': invitee_email,

api/controllers/console/workspace/model_providers.py

@@ -1,18 +1,19 @@
 import io
 
+from flask import send_file
+from flask_login import current_user
+from flask_restful import Resource, reqparse
+from werkzeug.exceptions import Forbidden
+
 from controllers.console import api
 from controllers.console.setup import setup_required
 from controllers.console.wraps import account_initialization_required
 from core.model_runtime.entities.model_entities import ModelType
 from core.model_runtime.errors.validate import CredentialsValidateFailedError
 from core.model_runtime.utils.encoders import jsonable_encoder
-from flask import send_file
-from flask_login import current_user
-from flask_restful import Resource, reqparse
 from libs.login import login_required
 from services.billing_service import BillingService
 from services.model_provider_service import ModelProviderService
-from werkzeug.exceptions import Forbidden
 
 
 class ModelProviderListApi(Resource):
@@ -186,10 +187,11 @@ class ModelProviderPaymentCheckoutUrlApi(Resource):
     def get(self, provider: str):
         if provider != 'anthropic':
             raise ValueError(f'provider name {provider} is invalid')
-
+        BillingService.is_tenant_owner_or_admin(current_user)
         data = BillingService.get_model_provider_payment_link(provider_name=provider,
                                                               tenant_id=current_user.current_tenant_id,
-                                                              account_id=current_user.id)
+                                                              account_id=current_user.id,
+                                                              prefilled_email=current_user.email)
         return data
 
 

api/controllers/console/workspace/models.py

@@ -1,16 +1,17 @@
 import logging
 
+from flask_login import current_user
+from flask_restful import Resource, reqparse
+from werkzeug.exceptions import Forbidden
+
 from controllers.console import api
 from controllers.console.setup import setup_required
 from controllers.console.wraps import account_initialization_required
 from core.model_runtime.entities.model_entities import ModelType
 from core.model_runtime.errors.validate import CredentialsValidateFailedError
 from core.model_runtime.utils.encoders import jsonable_encoder
-from flask_login import current_user
-from flask_restful import Resource, reqparse
 from libs.login import login_required
 from services.model_provider_service import ModelProviderService
-from werkzeug.exceptions import Forbidden
 
 
 class DefaultModelApi(Resource):

api/controllers/console/workspace/tool_providers.py

@@ -1,18 +1,16 @@
-import json
+import io
 
-from libs.login import login_required
+from flask import send_file
 from flask_login import current_user
 from flask_restful import Resource, reqparse
-from flask import send_file
 from werkzeug.exceptions import Forbidden
 
 from controllers.console import api
 from controllers.console.setup import setup_required
 from controllers.console.wraps import account_initialization_required
-
+from libs.login import login_required
 from services.tools_manage_service import ToolManageService
 
-import io
 
 class ToolProviderListApi(Resource):
     @setup_required
@@ -171,8 +169,8 @@ class ToolApiProviderUpdateApi(Resource):
         parser.add_argument('schema', type=str, required=True, nullable=False, location='json')
         parser.add_argument('provider', type=str, required=True, nullable=False, location='json')
         parser.add_argument('original_provider', type=str, required=True, nullable=False, location='json')
-        parser.add_argument('icon', type=str, required=True, nullable=False, location='json')
-        parser.add_argument('privacy_policy', type=str, required=True, nullable=False, location='json')
+        parser.add_argument('icon', type=dict, required=True, nullable=False, location='json')
+        parser.add_argument('privacy_policy', type=str, required=True, nullable=True, location='json')
 
         args = parser.parse_args()
 

api/controllers/console/workspace/workspace.py

@@ -1,18 +1,23 @@
 # -*- coding:utf-8 -*-
 import logging
 
+from flask import request
+from flask_login import current_user
+from flask_restful import Resource, fields, inputs, marshal, marshal_with, reqparse
+
 import services
 from controllers.console import api
 from controllers.console.admin import admin_required
-from controllers.console.datasets.error import (FileTooLargeError, NoFileUploadedError, TooManyFilesError,
-                                                UnsupportedFileTypeError)
+from controllers.console.datasets.error import (
+    FileTooLargeError,
+    NoFileUploadedError,
+    TooManyFilesError,
+    UnsupportedFileTypeError,
+)
 from controllers.console.error import AccountNotLinkTenantError
 from controllers.console.setup import setup_required
 from controllers.console.wraps import account_initialization_required, cloud_edition_billing_resource_check
 from extensions.ext_database import db
-from flask import request
-from flask_login import current_user
-from flask_restful import Resource, fields, inputs, marshal, marshal_with, reqparse
 from libs.helper import TimestampField
 from libs.login import login_required
 from models.account import Tenant

api/controllers/console/wraps.py

@@ -1,10 +1,13 @@
 # -*- coding:utf-8 -*-
+import json
 from functools import wraps
 
-from controllers.console.workspace.error import AccountNotInitializedError
-from flask import abort, current_app
+from flask import abort, current_app, request
 from flask_login import current_user
+
+from controllers.console.workspace.error import AccountNotInitializedError
 from services.feature_service import FeatureService
+from services.operation_service import OperationService
 
 
 def account_initialization_required(view):
@@ -73,3 +76,20 @@ def cloud_edition_billing_resource_check(resource: str,
         return decorated
     return interceptor
 
+
+def cloud_utm_record(view):
+    @wraps(view)
+    def decorated(*args, **kwargs):
+        try:
+            features = FeatureService.get_features(current_user.current_tenant_id)
+
+            if features.billing.enabled:
+                utm_info = request.cookies.get('utm_info')
+
+                if utm_info:
+                    utm_info = json.loads(utm_info)
+                    OperationService.record_utm(current_user.current_tenant_id, utm_info)
+        except Exception as e:
+            pass
+        return view(*args, **kwargs)
+    return decorated

api/controllers/files/__init__.py

@@ -6,5 +6,4 @@ bp = Blueprint('files', __name__)
 api = ExternalApi(bp)
 
 
-from . import image_preview
-from . import tool_files
+from . import image_preview, tool_files

api/controllers/files/image_preview.py

@@ -1,11 +1,12 @@
-import services
-from controllers.files import api
 from flask import Response, request
 from flask_restful import Resource
+from werkzeug.exceptions import NotFound
+
+import services
+from controllers.files import api
 from libs.exception import BaseHTTPException
 from services.account_service import TenantService
 from services.file_service import FileService
-from werkzeug.exceptions import NotFound
 
 
 class ImagePreviewApi(Resource):

api/controllers/files/tool_files.py

@@ -1,10 +1,11 @@
-from controllers.files import api
 from flask import Response
 from flask_restful import Resource, reqparse
-from libs.exception import BaseHTTPException
-from werkzeug.exceptions import NotFound, Forbidden
+from werkzeug.exceptions import Forbidden, NotFound
 
+from controllers.files import api
 from core.tools.tool_file_manager import ToolFileManager
+from libs.exception import BaseHTTPException
+
 
 class ToolFilePreviewApi(Resource):
     def get(self, file_id, extension):

api/controllers/service_api/app/app.py

@@ -1,14 +1,14 @@
 # -*- coding:utf-8 -*-
-from controllers.service_api import api
-from controllers.service_api.wraps import AppApiResource
-from flask import current_app
-from flask_restful import fields, marshal_with
-from models.model import App, AppModelConfig
-from models.tools import ApiToolProvider
-
 import json
 
+from flask import current_app
+from flask_restful import fields, marshal_with
+
+from controllers.service_api import api
+from controllers.service_api.wraps import AppApiResource
 from extensions.ext_database import db
+from models.model import App, AppModelConfig
+from models.tools import ApiToolProvider
 
 
 class AppParameterApi(AppApiResource):

api/controllers/service_api/app/audio.py

@@ -1,21 +1,33 @@
 import logging
 
+from flask import request
+from flask_restful import reqparse
+from werkzeug.exceptions import InternalServerError
+
 import services
 from controllers.service_api import api
-from controllers.service_api.app.error import (AppUnavailableError, AudioTooLargeError, CompletionRequestError,
-                                               NoAudioUploadedError, ProviderModelCurrentlyNotSupportError,
-                                               ProviderNotInitializeError, ProviderNotSupportSpeechToTextError,
-                                               ProviderQuotaExceededError, UnsupportedAudioTypeError)
+from controllers.service_api.app.error import (
+    AppUnavailableError,
+    AudioTooLargeError,
+    CompletionRequestError,
+    NoAudioUploadedError,
+    ProviderModelCurrentlyNotSupportError,
+    ProviderNotInitializeError,
+    ProviderNotSupportSpeechToTextError,
+    ProviderQuotaExceededError,
+    UnsupportedAudioTypeError,
+)
 from controllers.service_api.wraps import AppApiResource
 from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError
 from core.model_runtime.errors.invoke import InvokeError
-from flask import request
-from flask_restful import reqparse
 from models.model import App, AppModelConfig
 from services.audio_service import AudioService
-from services.errors.audio import (AudioTooLargeServiceError, NoAudioUploadedServiceError,
-                                   ProviderNotSupportSpeechToTextServiceError, UnsupportedAudioTypeServiceError)
-from werkzeug.exceptions import InternalServerError
+from services.errors.audio import (
+    AudioTooLargeServiceError,
+    NoAudioUploadedServiceError,
+    ProviderNotSupportSpeechToTextServiceError,
+    UnsupportedAudioTypeServiceError,
+)
 
 
 class AudioApi(AppApiResource):

api/controllers/service_api/app/completion.py

@@ -2,22 +2,29 @@ import json
 import logging
 from typing import Generator, Union
 
+from flask import Response, stream_with_context
+from flask_restful import reqparse
+from werkzeug.exceptions import InternalServerError, NotFound
+
 import services
 from controllers.service_api import api
 from controllers.service_api.app import create_or_update_end_user_for_user_id
-from controllers.service_api.app.error import (AppUnavailableError, CompletionRequestError, ConversationCompletedError,
-                                               NotChatAppError, ProviderModelCurrentlyNotSupportError,
-                                               ProviderNotInitializeError, ProviderQuotaExceededError)
+from controllers.service_api.app.error import (
+    AppUnavailableError,
+    CompletionRequestError,
+    ConversationCompletedError,
+    NotChatAppError,
+    ProviderModelCurrentlyNotSupportError,
+    ProviderNotInitializeError,
+    ProviderQuotaExceededError,
+)
 from controllers.service_api.wraps import AppApiResource
 from core.application_queue_manager import ApplicationQueueManager
 from core.entities.application_entities import InvokeFrom
 from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError
 from core.model_runtime.errors.invoke import InvokeError
-from flask import Response, stream_with_context, request
-from flask_restful import reqparse
 from libs.helper import uuid_value
 from services.completion_service import CompletionService
-from werkzeug.exceptions import InternalServerError, NotFound
 
 
 class CompletionApi(AppApiResource):
@@ -75,18 +82,22 @@ class CompletionApi(AppApiResource):
 
 
 class CompletionStopApi(AppApiResource):
-    def post(self, app_model, _, task_id):
+    def post(self, app_model, end_user, task_id):
         if app_model.mode != 'completion':
             raise AppUnavailableError()
 
-        parser = reqparse.RequestParser()
-        parser.add_argument('user', required=True, nullable=False, type=str, location='json')
+        if end_user is None:
+            parser = reqparse.RequestParser()
+            parser.add_argument('user', required=True, nullable=False, type=str, location='json')
+            args = parser.parse_args()
 
-        args = parser.parse_args()
+            user = args.get('user')
+            if user is not None:
+                end_user = create_or_update_end_user_for_user_id(app_model, user)
+            else:
+                raise ValueError("arg user muse be input.")
 
-        end_user_id = args.get('user')
-
-        ApplicationQueueManager.set_stop_flag(task_id, InvokeFrom.SERVICE_API, end_user_id)
+        ApplicationQueueManager.set_stop_flag(task_id, InvokeFrom.SERVICE_API, end_user.id)
 
         return {'result': 'success'}, 200
 
@@ -146,13 +157,22 @@ class ChatApi(AppApiResource):
 
 
 class ChatStopApi(AppApiResource):
-    def post(self, app_model, _, task_id):
+    def post(self, app_model, end_user, task_id):
         if app_model.mode != 'chat':
             raise NotChatAppError()
 
-        end_user_id = request.get_json().get('user')
+        if end_user is None:
+            parser = reqparse.RequestParser()
+            parser.add_argument('user', required=True, nullable=False, type=str, location='json')
+            args = parser.parse_args()
 
-        ApplicationQueueManager.set_stop_flag(task_id, InvokeFrom.SERVICE_API, end_user_id)
+            user = args.get('user')
+            if user is not None:
+                end_user = create_or_update_end_user_for_user_id(app_model, user)
+            else:
+                raise ValueError("arg user muse be input.")
+
+        ApplicationQueueManager.set_stop_flag(task_id, InvokeFrom.SERVICE_API, end_user.id)
 
         return {'result': 'success'}, 200
 

api/controllers/service_api/app/conversation.py

@@ -1,16 +1,17 @@
 # -*- coding:utf-8 -*-
+from flask import request
+from flask_restful import marshal_with, reqparse
+from flask_restful.inputs import int_range
+from werkzeug.exceptions import NotFound
+
 import services
 from controllers.service_api import api
 from controllers.service_api.app import create_or_update_end_user_for_user_id
 from controllers.service_api.app.error import NotChatAppError
 from controllers.service_api.wraps import AppApiResource
 from fields.conversation_fields import conversation_infinite_scroll_pagination_fields, simple_conversation_fields
-from flask import request
-from flask_restful import fields, marshal_with, reqparse
-from flask_restful.inputs import int_range
-from libs.helper import TimestampField, uuid_value
+from libs.helper import uuid_value
 from services.conversation_service import ConversationService
-from werkzeug.exceptions import NotFound
 
 
 class ConversationApi(AppApiResource):

api/controllers/service_api/app/file.py

@@ -1,12 +1,17 @@
+from flask import request
+from flask_restful import marshal_with
+
 import services
 from controllers.service_api import api
 from controllers.service_api.app import create_or_update_end_user_for_user_id
-from controllers.service_api.app.error import (FileTooLargeError, NoFileUploadedError, TooManyFilesError,
-                                               UnsupportedFileTypeError)
+from controllers.service_api.app.error import (
+    FileTooLargeError,
+    NoFileUploadedError,
+    TooManyFilesError,
+    UnsupportedFileTypeError,
+)
 from controllers.service_api.wraps import AppApiResource
 from fields.file_fields import file_fields
-from flask import request
-from flask_restful import marshal_with
 from services.file_service import FileService
 
 

api/controllers/service_api/app/message.py

@@ -1,4 +1,8 @@
 # -*- coding:utf-8 -*-
+from flask_restful import fields, marshal_with, reqparse
+from flask_restful.inputs import int_range
+from werkzeug.exceptions import NotFound
+
 import services
 from controllers.service_api import api
 from controllers.service_api.app import create_or_update_end_user_for_user_id
@@ -6,12 +10,9 @@ from controllers.service_api.app.error import NotChatAppError
 from controllers.service_api.wraps import AppApiResource
 from extensions.ext_database import db
 from fields.conversation_fields import message_file_fields
-from flask_restful import fields, marshal_with, reqparse
-from flask_restful.inputs import int_range
 from libs.helper import TimestampField, uuid_value
 from models.model import EndUser, Message
 from services.message_service import MessageService
-from werkzeug.exceptions import NotFound
 
 
 class MessageListApi(AppApiResource):

api/controllers/service_api/dataset/dataset.py

@@ -1,4 +1,6 @@
-from models.dataset import Dataset
+from flask import request
+from flask_restful import marshal, reqparse
+
 import services.dataset_service
 from controllers.service_api import api
 from controllers.service_api.dataset.error import DatasetNameDuplicateError
@@ -6,9 +8,8 @@ from controllers.service_api.wraps import DatasetApiResource
 from core.model_runtime.entities.model_entities import ModelType
 from core.provider_manager import ProviderManager
 from fields.dataset_fields import dataset_detail_fields
-from flask import request
-from flask_restful import marshal, reqparse
 from libs.login import current_user
+from models.dataset import Dataset
 from services.dataset_service import DatasetService
 
 

api/controllers/service_api/dataset/document.py

@@ -1,23 +1,28 @@
 import json
 
+from flask import request
+from flask_login import current_user
+from flask_restful import marshal, reqparse
+from sqlalchemy import desc
+from werkzeug.exceptions import NotFound
+
 import services.dataset_service
 from controllers.service_api import api
 from controllers.service_api.app.error import ProviderNotInitializeError
-from controllers.service_api.dataset.error import (ArchivedDocumentImmutableError, DocumentIndexingError,
-                                                   NoFileUploadedError, TooManyFilesError)
+from controllers.service_api.dataset.error import (
+    ArchivedDocumentImmutableError,
+    DocumentIndexingError,
+    NoFileUploadedError,
+    TooManyFilesError,
+)
 from controllers.service_api.wraps import DatasetApiResource, cloud_edition_billing_resource_check
 from core.errors.error import ProviderTokenNotInitError
 from extensions.ext_database import db
 from fields.document_fields import document_fields, document_status_fields
-from flask import request
-from flask_login import current_user
-from flask_restful import marshal, reqparse
 from libs.login import current_user
 from models.dataset import Dataset, Document, DocumentSegment
 from services.dataset_service import DocumentService
 from services.file_service import FileService
-from sqlalchemy import desc
-from werkzeug.exceptions import NotFound
 
 
 class DocumentAddByTextApi(DatasetApiResource):

api/controllers/service_api/dataset/segment.py

@@ -1,3 +1,7 @@
+from flask_login import current_user
+from flask_restful import marshal, reqparse
+from werkzeug.exceptions import NotFound
+
 from controllers.service_api import api
 from controllers.service_api.app.error import ProviderNotInitializeError
 from controllers.service_api.wraps import DatasetApiResource, cloud_edition_billing_resource_check
@@ -6,11 +10,8 @@ from core.model_manager import ModelManager
 from core.model_runtime.entities.model_entities import ModelType
 from extensions.ext_database import db
 from fields.segment_fields import segment_fields
-from flask_login import current_user
-from flask_restful import marshal, reqparse
 from models.dataset import Dataset, DocumentSegment
 from services.dataset_service import DatasetService, DocumentService, SegmentService
-from werkzeug.exceptions import NotFound
 
 
 class SegmentApi(DatasetApiResource):

api/controllers/service_api/wraps.py

@@ -2,15 +2,16 @@
 from datetime import datetime
 from functools import wraps
 
-from extensions.ext_database import db
 from flask import current_app, request
 from flask_login import user_logged_in
 from flask_restful import Resource
+from werkzeug.exceptions import NotFound, Unauthorized
+
+from extensions.ext_database import db
 from libs.login import _get_user
 from models.account import Account, Tenant, TenantAccountJoin
 from models.model import ApiToken, App
 from services.feature_service import FeatureService
-from werkzeug.exceptions import NotFound, Unauthorized
 
 
 def validate_app_token(view=None):

api/controllers/web/app.py

@@ -1,15 +1,15 @@
 # -*- coding:utf-8 -*-
-from controllers.web import api
-from controllers.web.wraps import WebApiResource
+import json
+
 from flask import current_app
 from flask_restful import fields, marshal_with
+
+from controllers.web import api
+from controllers.web.wraps import WebApiResource
+from extensions.ext_database import db
 from models.model import App, AppModelConfig
 from models.tools import ApiToolProvider
 
-from extensions.ext_database import db
-
-import json
-
 
 class AppParameterApi(WebApiResource):
     """Resource for app variables."""

api/controllers/web/audio.py

@@ -1,21 +1,33 @@
 # -*- coding:utf-8 -*-
 import logging
 
+from flask import request
+from werkzeug.exceptions import InternalServerError
+
 import services
 from controllers.web import api
-from controllers.web.error import (AppUnavailableError, AudioTooLargeError, CompletionRequestError,
-                                   NoAudioUploadedError, ProviderModelCurrentlyNotSupportError,
-                                   ProviderNotInitializeError, ProviderNotSupportSpeechToTextError,
-                                   ProviderQuotaExceededError, UnsupportedAudioTypeError)
+from controllers.web.error import (
+    AppUnavailableError,
+    AudioTooLargeError,
+    CompletionRequestError,
+    NoAudioUploadedError,
+    ProviderModelCurrentlyNotSupportError,
+    ProviderNotInitializeError,
+    ProviderNotSupportSpeechToTextError,
+    ProviderQuotaExceededError,
+    UnsupportedAudioTypeError,
+)
 from controllers.web.wraps import WebApiResource
 from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError
 from core.model_runtime.errors.invoke import InvokeError
-from flask import request
 from models.model import App, AppModelConfig
 from services.audio_service import AudioService
-from services.errors.audio import (AudioTooLargeServiceError, NoAudioUploadedServiceError,
-                                   ProviderNotSupportSpeechToTextServiceError, UnsupportedAudioTypeServiceError)
-from werkzeug.exceptions import InternalServerError
+from services.errors.audio import (
+    AudioTooLargeServiceError,
+    NoAudioUploadedServiceError,
+    ProviderNotSupportSpeechToTextServiceError,
+    UnsupportedAudioTypeServiceError,
+)
 
 
 class AudioApi(WebApiResource):

api/controllers/web/completion.py

@@ -3,21 +3,29 @@ import json
 import logging
 from typing import Generator, Union
 
+from flask import Response, stream_with_context
+from flask_restful import reqparse
+from werkzeug.exceptions import InternalServerError, NotFound
+
 import services
 from controllers.web import api
-from controllers.web.error import (AppUnavailableError, CompletionRequestError, ConversationCompletedError,
-                                   NotChatAppError, NotCompletionAppError, ProviderModelCurrentlyNotSupportError,
-                                   ProviderNotInitializeError, ProviderQuotaExceededError)
+from controllers.web.error import (
+    AppUnavailableError,
+    CompletionRequestError,
+    ConversationCompletedError,
+    NotChatAppError,
+    NotCompletionAppError,
+    ProviderModelCurrentlyNotSupportError,
+    ProviderNotInitializeError,
+    ProviderQuotaExceededError,
+)
 from controllers.web.wraps import WebApiResource
 from core.application_queue_manager import ApplicationQueueManager
 from core.entities.application_entities import InvokeFrom
 from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError
 from core.model_runtime.errors.invoke import InvokeError
-from flask import Response, stream_with_context
-from flask_restful import reqparse
 from libs.helper import uuid_value
 from services.completion_service import CompletionService
-from werkzeug.exceptions import InternalServerError, NotFound
 
 
 # define completion api for user

api/controllers/web/conversation.py

@@ -1,15 +1,16 @@
 # -*- coding:utf-8 -*-
+from flask_restful import marshal_with, reqparse
+from flask_restful.inputs import int_range
+from werkzeug.exceptions import NotFound
+
 from controllers.web import api
 from controllers.web.error import NotChatAppError
 from controllers.web.wraps import WebApiResource
 from fields.conversation_fields import conversation_infinite_scroll_pagination_fields, simple_conversation_fields
-from flask_restful import fields, marshal_with, reqparse
-from flask_restful.inputs import int_range
-from libs.helper import TimestampField, uuid_value
+from libs.helper import uuid_value
 from services.conversation_service import ConversationService
 from services.errors.conversation import ConversationNotExistsError, LastConversationNotExistsError
 from services.web_conversation_service import WebConversationService
-from werkzeug.exceptions import NotFound
 
 
 class ConversationListApi(WebApiResource):

api/controllers/web/file.py

@@ -1,10 +1,11 @@
+from flask import request
+from flask_restful import marshal_with
+
 import services
 from controllers.web import api
 from controllers.web.error import FileTooLargeError, NoFileUploadedError, TooManyFilesError, UnsupportedFileTypeError
 from controllers.web.wraps import WebApiResource
 from fields.file_fields import file_fields
-from flask import request
-from flask_restful import marshal_with
 from services.file_service import FileService
 
 

api/controllers/web/message.py

@@ -3,28 +3,35 @@ import json
 import logging
 from typing import Generator, Union
 
+from flask import Response, stream_with_context
+from flask_restful import fields, marshal_with, reqparse
+from flask_restful.inputs import int_range
+from werkzeug.exceptions import InternalServerError, NotFound
+
 import services
 from controllers.web import api
-from controllers.web.error import (AppMoreLikeThisDisabledError, AppSuggestedQuestionsAfterAnswerDisabledError,
-                                   CompletionRequestError, NotChatAppError, NotCompletionAppError,
-                                   ProviderModelCurrentlyNotSupportError, ProviderNotInitializeError,
-                                   ProviderQuotaExceededError)
+from controllers.web.error import (
+    AppMoreLikeThisDisabledError,
+    AppSuggestedQuestionsAfterAnswerDisabledError,
+    CompletionRequestError,
+    NotChatAppError,
+    NotCompletionAppError,
+    ProviderModelCurrentlyNotSupportError,
+    ProviderNotInitializeError,
+    ProviderQuotaExceededError,
+)
 from controllers.web.wraps import WebApiResource
 from core.entities.application_entities import InvokeFrom
 from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError
 from core.model_runtime.errors.invoke import InvokeError
 from fields.conversation_fields import message_file_fields
 from fields.message_fields import agent_thought_fields
-from flask import Response, stream_with_context
-from flask_restful import fields, marshal_with, reqparse
-from flask_restful.inputs import int_range
 from libs.helper import TimestampField, uuid_value
 from services.completion_service import CompletionService
 from services.errors.app import MoreLikeThisDisabledError
 from services.errors.conversation import ConversationNotExistsError
 from services.errors.message import MessageNotExistsError, SuggestedQuestionsAfterAnswerDisabledError
 from services.message_service import MessageService
-from werkzeug.exceptions import InternalServerError, NotFound
 
 
 class MessageListApi(WebApiResource):

api/controllers/web/passport.py

@@ -1,13 +1,14 @@
 # -*- coding:utf-8 -*-
 import uuid
 
-from controllers.web import api
-from extensions.ext_database import db
 from flask import request
 from flask_restful import Resource
+from werkzeug.exceptions import NotFound, Unauthorized
+
+from controllers.web import api
+from extensions.ext_database import db
 from libs.passport import PassportService
 from models.model import App, EndUser, Site
-from werkzeug.exceptions import NotFound, Unauthorized
 
 
 class PassportResource(Resource):

api/controllers/web/saved_message.py

@@ -1,13 +1,14 @@
+from flask_restful import fields, marshal_with, reqparse
+from flask_restful.inputs import int_range
+from werkzeug.exceptions import NotFound
+
 from controllers.web import api
 from controllers.web.error import NotCompletionAppError
 from controllers.web.wraps import WebApiResource
 from fields.conversation_fields import message_file_fields
-from flask_restful import fields, marshal_with, reqparse
-from flask_restful.inputs import int_range
 from libs.helper import TimestampField, uuid_value
 from services.errors.message import MessageNotExistsError
 from services.saved_message_service import SavedMessageService
-from werkzeug.exceptions import NotFound
 
 feedback_fields = {
     'rating': fields.String

api/controllers/web/site.py

@@ -1,14 +1,14 @@
 # -*- coding:utf-8 -*-
-import os
+
+from flask import current_app
+from flask_restful import fields, marshal_with
+from werkzeug.exceptions import Forbidden
 
 from controllers.web import api
 from controllers.web.wraps import WebApiResource
 from extensions.ext_database import db
-from flask import current_app
-from flask_restful import fields, marshal_with
 from models.model import Site
 from services.feature_service import FeatureService
-from werkzeug.exceptions import Forbidden
 
 
 class AppSiteApi(WebApiResource):

api/controllers/web/wraps.py

@@ -1,12 +1,13 @@
 # -*- coding:utf-8 -*-
 from functools import wraps
 
-from extensions.ext_database import db
 from flask import request
 from flask_restful import Resource
+from werkzeug.exceptions import NotFound, Unauthorized
+
+from extensions.ext_database import db
 from libs.passport import PassportService
 from models.model import App, EndUser, Site
-from werkzeug.exceptions import NotFound, Unauthorized
 
 
 def validate_jwt_token(view=None):

api/core/agent/agent/calc_token_mixin.py

@@ -1,11 +1,9 @@
 from typing import List, cast
 
 from core.entities.application_entities import ModelConfigEntity
-from core.entities.message_entities import lc_messages_to_prompt_messages
 from core.model_runtime.entities.message_entities import PromptMessage
 from core.model_runtime.entities.model_entities import ModelPropertyKey
 from core.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel
-from langchain.schema import BaseMessage
 
 
 class CalcTokenMixin:

api/core/agent/agent/multi_dataset_router_agent.py

@@ -1,10 +1,5 @@
-from typing import Any, List, Optional, Sequence, Tuple, Union, cast
+from typing import Any, List, Optional, Sequence, Tuple, Union
 
-from core.entities.application_entities import ModelConfigEntity
-from core.entities.message_entities import lc_messages_to_prompt_messages
-from core.model_manager import ModelInstance
-from core.model_runtime.entities.message_entities import PromptMessageTool
-from core.third_party.langchain.llms.fake import FakeLLM
 from langchain.agents import BaseSingleActionAgent, OpenAIFunctionsAgent
 from langchain.agents.openai_functions_agent.base import _format_intermediate_steps, _parse_ai_message
 from langchain.callbacks.base import BaseCallbackManager
@@ -14,6 +9,12 @@ from langchain.schema import AgentAction, AgentFinish, AIMessage, SystemMessage
 from langchain.tools import BaseTool
 from pydantic import root_validator
 
+from core.entities.application_entities import ModelConfigEntity
+from core.entities.message_entities import lc_messages_to_prompt_messages
+from core.model_manager import ModelInstance
+from core.model_runtime.entities.message_entities import PromptMessageTool
+from core.third_party.langchain.llms.fake import FakeLLM
+
 
 class MultiDatasetRouterAgent(OpenAIFunctionsAgent):
     """

api/core/agent/agent/openai_function_call.py

@@ -1,4 +1,23 @@
-from typing import Any, List, Optional, Sequence, Tuple, Union, cast
+from typing import Any, List, Optional, Sequence, Tuple, Union
+
+from langchain.agents import BaseSingleActionAgent, OpenAIFunctionsAgent
+from langchain.agents.openai_functions_agent.base import _format_intermediate_steps, _parse_ai_message
+from langchain.callbacks.base import BaseCallbackManager
+from langchain.callbacks.manager import Callbacks
+from langchain.chat_models.openai import _convert_message_to_dict, _import_tiktoken
+from langchain.memory.prompt import SUMMARY_PROMPT
+from langchain.prompts.chat import BaseMessagePromptTemplate
+from langchain.schema import (
+    AgentAction,
+    AgentFinish,
+    AIMessage,
+    BaseMessage,
+    HumanMessage,
+    SystemMessage,
+    get_buffer_string,
+)
+from langchain.tools import BaseTool
+from pydantic import root_validator
 
 from core.agent.agent.agent_llm_callback import AgentLLMCallback
 from core.agent.agent.calc_token_mixin import CalcTokenMixin, ExceededLLMTokensLimitError
@@ -7,19 +26,7 @@ from core.entities.application_entities import ModelConfigEntity
 from core.entities.message_entities import lc_messages_to_prompt_messages
 from core.model_manager import ModelInstance
 from core.model_runtime.entities.message_entities import PromptMessage, PromptMessageTool
-from core.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel
 from core.third_party.langchain.llms.fake import FakeLLM
-from langchain.agents import BaseSingleActionAgent, OpenAIFunctionsAgent
-from langchain.agents.openai_functions_agent.base import _format_intermediate_steps, _parse_ai_message
-from langchain.callbacks.base import BaseCallbackManager
-from langchain.callbacks.manager import Callbacks
-from langchain.chat_models.openai import _convert_message_to_dict, _import_tiktoken
-from langchain.memory.prompt import SUMMARY_PROMPT
-from langchain.prompts.chat import BaseMessagePromptTemplate
-from langchain.schema import (AgentAction, AgentFinish, AIMessage, BaseMessage, HumanMessage, SystemMessage,
-                              get_buffer_string)
-from langchain.tools import BaseTool
-from pydantic import root_validator
 
 
 class AutoSummarizingOpenAIFunctionCallAgent(OpenAIFunctionsAgent, CalcTokenMixin):

api/core/agent/agent/structed_multi_dataset_router_agent.py

@@ -1,8 +1,6 @@
 import re
 from typing import Any, List, Optional, Sequence, Tuple, Union, cast
 
-from core.chain.llm_chain import LLMChain
-from core.entities.application_entities import ModelConfigEntity
 from langchain import BasePromptTemplate, PromptTemplate
 from langchain.agents import Agent, AgentOutputParser, StructuredChatAgent
 from langchain.agents.structured_chat.base import HUMAN_MESSAGE_TEMPLATE
@@ -13,6 +11,9 @@ from langchain.prompts import ChatPromptTemplate, HumanMessagePromptTemplate, Sy
 from langchain.schema import AgentAction, AgentFinish, OutputParserException
 from langchain.tools import BaseTool
 
+from core.chain.llm_chain import LLMChain
+from core.entities.application_entities import ModelConfigEntity
+
 FORMAT_INSTRUCTIONS = """Use a json blob to specify a tool by providing an action key (tool name) and an action_input key (tool input).
 The nouns in the format of "Thought", "Action", "Action Input", "Final Answer" must be expressed in English.
 Valid "action" values: "Final Answer" or {tool_names}

api/core/agent/agent/structured_chat.py

@@ -1,11 +1,6 @@
 import re
 from typing import Any, List, Optional, Sequence, Tuple, Union, cast
 
-from core.agent.agent.agent_llm_callback import AgentLLMCallback
-from core.agent.agent.calc_token_mixin import CalcTokenMixin, ExceededLLMTokensLimitError
-from core.chain.llm_chain import LLMChain
-from core.entities.application_entities import ModelConfigEntity
-from core.entities.message_entities import lc_messages_to_prompt_messages
 from langchain import BasePromptTemplate, PromptTemplate
 from langchain.agents import Agent, AgentOutputParser, StructuredChatAgent
 from langchain.agents.structured_chat.base import HUMAN_MESSAGE_TEMPLATE
@@ -14,10 +9,23 @@ from langchain.callbacks.base import BaseCallbackManager
 from langchain.callbacks.manager import Callbacks
 from langchain.memory.prompt import SUMMARY_PROMPT
 from langchain.prompts import ChatPromptTemplate, HumanMessagePromptTemplate, SystemMessagePromptTemplate
-from langchain.schema import (AgentAction, AgentFinish, AIMessage, BaseMessage, HumanMessage, OutputParserException,
-                              get_buffer_string)
+from langchain.schema import (
+    AgentAction,
+    AgentFinish,
+    AIMessage,
+    BaseMessage,
+    HumanMessage,
+    OutputParserException,
+    get_buffer_string,
+)
 from langchain.tools import BaseTool
 
+from core.agent.agent.agent_llm_callback import AgentLLMCallback
+from core.agent.agent.calc_token_mixin import CalcTokenMixin, ExceededLLMTokensLimitError
+from core.chain.llm_chain import LLMChain
+from core.entities.application_entities import ModelConfigEntity
+from core.entities.message_entities import lc_messages_to_prompt_messages
+
 FORMAT_INSTRUCTIONS = """Use a json blob to specify a tool by providing an action key (tool name) and an action_input key (tool input).
 The nouns in the format of "Thought", "Action", "Action Input", "Final Answer" must be expressed in English.
 Valid "action" values: "Final Answer" or {tool_names}

api/core/agent/agent_executor.py

@@ -2,6 +2,12 @@ import enum
 import logging
 from typing import Optional, Union
 
+from langchain.agents import AgentExecutor as LCAgentExecutor
+from langchain.agents import BaseMultiActionAgent, BaseSingleActionAgent
+from langchain.callbacks.manager import Callbacks
+from langchain.tools import BaseTool
+from pydantic import BaseModel, Extra
+
 from core.agent.agent.agent_llm_callback import AgentLLMCallback
 from core.agent.agent.multi_dataset_router_agent import MultiDatasetRouterAgent
 from core.agent.agent.openai_function_call import AutoSummarizingOpenAIFunctionCallAgent
@@ -15,11 +21,6 @@ from core.memory.token_buffer_memory import TokenBufferMemory
 from core.model_runtime.errors.invoke import InvokeError
 from core.tools.tool.dataset_retriever.dataset_multi_retriever_tool import DatasetMultiRetrieverTool
 from core.tools.tool.dataset_retriever.dataset_retriever_tool import DatasetRetrieverTool
-from langchain.agents import AgentExecutor as LCAgentExecutor
-from langchain.agents import BaseMultiActionAgent, BaseSingleActionAgent
-from langchain.callbacks.manager import Callbacks
-from langchain.tools import BaseTool
-from pydantic import BaseModel, Extra
 
 
 class PlanningStrategy(str, enum.Enum):

api/core/app_runner/app_runner.py

@@ -2,8 +2,18 @@ import time
 from typing import Generator, List, Optional, Tuple, Union, cast
 
 from core.application_queue_manager import ApplicationQueueManager, PublishFrom
-from core.entities.application_entities import AppOrchestrationConfigEntity, ModelConfigEntity, \
-    PromptTemplateEntity, ExternalDataVariableEntity, ApplicationGenerateEntity, InvokeFrom
+from core.entities.application_entities import (
+    ApplicationGenerateEntity,
+    AppOrchestrationConfigEntity,
+    ExternalDataVariableEntity,
+    InvokeFrom,
+    ModelConfigEntity,
+    PromptTemplateEntity,
+)
+from core.features.annotation_reply import AnnotationReplyFeature
+from core.features.external_data_fetch import ExternalDataFetchFeature
+from core.features.hosting_moderation import HostingModerationFeature
+from core.features.moderation import ModerationFeature
 from core.file.file_obj import FileObj
 from core.memory.token_buffer_memory import TokenBufferMemory
 from core.model_runtime.entities.llm_entities import LLMResult, LLMResultChunk, LLMResultChunkDelta, LLMUsage
@@ -11,12 +21,9 @@ from core.model_runtime.entities.message_entities import AssistantPromptMessage,
 from core.model_runtime.entities.model_entities import ModelPropertyKey
 from core.model_runtime.errors.invoke import InvokeBadRequestError
 from core.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel
-from core.features.hosting_moderation import HostingModerationFeature
-from core.features.moderation import ModerationFeature
-from core.features.external_data_fetch import ExternalDataFetchFeature
-from core.features.annotation_reply import AnnotationReplyFeature
 from core.prompt.prompt_transform import PromptTransform
-from models.model import App, MessageAnnotation, Message
+from models.model import App, Message, MessageAnnotation
+
 
 class AppRunner:
     def get_pre_calculate_rest_tokens(self, app_record: App,

api/core/app_runner/assistant_app_runner.py

@@ -3,19 +3,19 @@ import logging
 from typing import cast
 
 from core.app_runner.app_runner import AppRunner
+from core.application_queue_manager import ApplicationQueueManager, PublishFrom
+from core.entities.application_entities import AgentEntity, ApplicationGenerateEntity, ModelConfigEntity
 from core.features.assistant_cot_runner import AssistantCotApplicationRunner
 from core.features.assistant_fc_runner import AssistantFunctionCallApplicationRunner
-from core.entities.application_entities import ApplicationGenerateEntity, ModelConfigEntity, \
-    AgentEntity
-from core.application_queue_manager import ApplicationQueueManager, PublishFrom
 from core.memory.token_buffer_memory import TokenBufferMemory
 from core.model_manager import ModelInstance
 from core.model_runtime.entities.llm_entities import LLMUsage
+from core.model_runtime.entities.model_entities import ModelFeature
 from core.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel
 from core.moderation.base import ModerationException
 from core.tools.entities.tool_entities import ToolRuntimeVariablePool
 from extensions.ext_database import db
-from models.model import Conversation, Message, App, MessageChain, MessageAgentThought
+from models.model import App, Conversation, Message, MessageAgentThought, MessageChain
 from models.tools import ToolConversationVariables
 
 logger = logging.getLogger(__name__)
@@ -169,7 +169,7 @@ class AssistantApplicationRunner(AppRunner):
         # load tool variables
         tool_conversation_variables = self._load_tool_variables(conversation_id=conversation.id,
                                                    user_id=application_generate_entity.user_id,
-                                                   tanent_id=application_generate_entity.tenant_id)
+                                                   tenant_id=application_generate_entity.tenant_id)
 
         # convert db variables to tool variables
         tool_variables = self._convert_db_variables_to_tool_variables(tool_conversation_variables)
@@ -194,6 +194,13 @@ class AssistantApplicationRunner(AppRunner):
             memory=memory,
         )
 
+        # change function call strategy based on LLM model
+        llm_model = cast(LargeLanguageModel, model_instance.model_type_instance)
+        model_schema = llm_model.get_model_schema(model_instance.model, model_instance.credentials)
+
+        if set([ModelFeature.MULTI_TOOL_CALL, ModelFeature.TOOL_CALL]).intersection(model_schema.features or []):
+            agent_entity.strategy = AgentEntity.Strategy.FUNCTION_CALLING
+
         # start agent runner
         if agent_entity.strategy == AgentEntity.Strategy.CHAIN_OF_THOUGHT:
             assistant_cot_runner = AssistantCotApplicationRunner(
@@ -209,12 +216,13 @@ class AssistantApplicationRunner(AppRunner):
                 prompt_messages=prompt_message,
                 variables_pool=tool_variables,
                 db_variables=tool_conversation_variables,
+                model_instance=model_instance
             )
             invoke_result = assistant_cot_runner.run(
-                model_instance=model_instance,
                 conversation=conversation,
                 message=message,
                 query=query,
+                inputs=inputs,
             )
         elif agent_entity.strategy == AgentEntity.Strategy.FUNCTION_CALLING:
             assistant_fc_runner = AssistantFunctionCallApplicationRunner(
@@ -229,10 +237,10 @@ class AssistantApplicationRunner(AppRunner):
                 memory=memory,
                 prompt_messages=prompt_message,
                 variables_pool=tool_variables,
-                db_variables=tool_conversation_variables
+                db_variables=tool_conversation_variables,
+                model_instance=model_instance
             )
             invoke_result = assistant_fc_runner.run(
-                model_instance=model_instance,
                 conversation=conversation,
                 message=message,
                 query=query,
@@ -246,13 +254,13 @@ class AssistantApplicationRunner(AppRunner):
             agent=True
         )
 
-    def _load_tool_variables(self, conversation_id: str, user_id: str, tanent_id: str) -> ToolConversationVariables:
+    def _load_tool_variables(self, conversation_id: str, user_id: str, tenant_id: str) -> ToolConversationVariables:
         """
         load tool variables from database
         """
         tool_variables: ToolConversationVariables = db.session.query(ToolConversationVariables).filter(
             ToolConversationVariables.conversation_id == conversation_id,
-            ToolConversationVariables.tenant_id == tanent_id
+            ToolConversationVariables.tenant_id == tenant_id
         ).first()
 
         if tool_variables:
@@ -263,7 +271,7 @@ class AssistantApplicationRunner(AppRunner):
             tool_variables = ToolConversationVariables(
                 conversation_id=conversation_id,
                 user_id=user_id,
-                tenant_id=tanent_id,
+                tenant_id=tenant_id,
                 variables_str='[]',
             )
             db.session.add(tool_variables)

api/core/app_runner/basic_app_runner.py

@@ -4,8 +4,7 @@ from typing import Optional
 from core.app_runner.app_runner import AppRunner
 from core.application_queue_manager import ApplicationQueueManager, PublishFrom
 from core.callback_handler.index_tool_callback_handler import DatasetIndexToolCallbackHandler
-from core.entities.application_entities import (ApplicationGenerateEntity, DatasetEntity,
-                                                 InvokeFrom, ModelConfigEntity)
+from core.entities.application_entities import ApplicationGenerateEntity, DatasetEntity, InvokeFrom, ModelConfigEntity
 from core.features.dataset_retrieval import DatasetRetrievalFeature
 from core.memory.token_buffer_memory import TokenBufferMemory
 from core.model_manager import ModelInstance

api/core/app_runner/generate_task_pipeline.py

@@ -3,28 +3,42 @@ import logging
 import time
 from typing import Generator, Optional, Union, cast
 
+from pydantic import BaseModel
+
 from core.app_runner.moderation_handler import ModerationRule, OutputModerationHandler
 from core.application_queue_manager import ApplicationQueueManager, PublishFrom
 from core.entities.application_entities import ApplicationGenerateEntity, InvokeFrom
-from core.entities.queue_entities import (AnnotationReplyEvent, QueueAgentThoughtEvent, QueueErrorEvent,
-                                          QueueMessageEndEvent, QueueMessageEvent, QueueMessageReplaceEvent,
-                                          QueuePingEvent, QueueRetrieverResourcesEvent, QueueStopEvent,
-                                          QueueMessageFileEvent, QueueAgentMessageEvent)
-from core.errors.error import ProviderTokenNotInitError, QuotaExceededError, ModelCurrentlyNotSupportError
+from core.entities.queue_entities import (
+    AnnotationReplyEvent,
+    QueueAgentMessageEvent,
+    QueueAgentThoughtEvent,
+    QueueErrorEvent,
+    QueueMessageEndEvent,
+    QueueMessageEvent,
+    QueueMessageFileEvent,
+    QueueMessageReplaceEvent,
+    QueuePingEvent,
+    QueueRetrieverResourcesEvent,
+    QueueStopEvent,
+)
+from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError
 from core.model_runtime.entities.llm_entities import LLMResult, LLMResultChunk, LLMResultChunkDelta, LLMUsage
-from core.model_runtime.entities.message_entities import (AssistantPromptMessage, ImagePromptMessageContent,
-                                                          PromptMessage, PromptMessageContentType, PromptMessageRole,
-                                                          TextPromptMessageContent)
+from core.model_runtime.entities.message_entities import (
+    AssistantPromptMessage,
+    ImagePromptMessageContent,
+    PromptMessage,
+    PromptMessageContentType,
+    PromptMessageRole,
+    TextPromptMessageContent,
+)
 from core.model_runtime.errors.invoke import InvokeAuthorizationError, InvokeError
 from core.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel
-from core.tools.tool_file_manager import ToolFileManager
-from core.tools.tool_manager import ToolManager
 from core.model_runtime.utils.encoders import jsonable_encoder
 from core.prompt.prompt_template import PromptTemplateParser
+from core.tools.tool_file_manager import ToolFileManager
 from events.message_event import message_was_created
 from extensions.ext_database import db
 from models.model import Conversation, Message, MessageAgentThought, MessageFile
-from pydantic import BaseModel
 from services.annotation_service import AppAnnotationService
 
 logger = logging.getLogger(__name__)
@@ -463,44 +477,34 @@ class GenerateTaskPipeline:
         :param e: exception
         :return:
         """
-        if isinstance(e, ValueError):
-            data = {
-                'code': 'invalid_param',
-                'message': str(e),
-                'status': 400
-            }
-        elif isinstance(e, ProviderTokenNotInitError):
-            data = {
-                'code': 'provider_not_initialize',
-                'message': e.description,
-                'status': 400
-            }
-        elif isinstance(e, QuotaExceededError):
-            data = {
+        error_responses = {
+            ValueError: {'code': 'invalid_param', 'status': 400},
+            ProviderTokenNotInitError: {'code': 'provider_not_initialize', 'status': 400},
+            QuotaExceededError: {
                 'code': 'provider_quota_exceeded',
                 'message': "Your quota for Dify Hosted Model Provider has been exhausted. "
-                           "Please go to Settings -> Model Provider to complete your own provider credentials.",
+                       "Please go to Settings -> Model Provider to complete your own provider credentials.",
                 'status': 400
-            }
-        elif isinstance(e, ModelCurrentlyNotSupportError):
-            data = {
-                'code': 'model_currently_not_support',
-                'message': e.description,
-                'status': 400
-            }
-        elif isinstance(e, InvokeError):
-            data = {
-                'code': 'completion_request_error',
-                'message': e.description,
-                'status': 400
-            }
+            },
+            ModelCurrentlyNotSupportError: {'code': 'model_currently_not_support', 'status': 400},
+            InvokeError: {'code': 'completion_request_error', 'status': 400}
+        }
+
+        # Determine the response based on the type of exception
+        data = None
+        for k, v in error_responses.items():
+            if isinstance(e, k):
+                data = v
+
+        if data:
+            data.setdefault('message', getattr(e, 'description', str(e)))
         else:
             logging.error(e)
             data = {
-                'code': 'internal_server_error',
+                'code': 'internal_server_error', 
                 'message': 'Internal Server Error, please contact support.',
                 'status': 500
-            }
+                }
 
         return {
             'event': 'error',

api/core/app_runner/moderation_handler.py

@@ -3,11 +3,12 @@ import threading
 import time
 from typing import Any, Dict, Optional
 
+from flask import Flask, current_app
+from pydantic import BaseModel
+
 from core.application_queue_manager import PublishFrom
 from core.moderation.base import ModerationAction, ModerationOutputsResult
 from core.moderation.factory import ModerationFactory
-from flask import Flask, current_app
-from pydantic import BaseModel
 
 logger = logging.getLogger(__name__)
 

api/core/application_manager.py

@@ -4,16 +4,30 @@ import threading
 import uuid
 from typing import Any, Generator, Optional, Tuple, Union, cast
 
+from flask import Flask, current_app
+from pydantic import ValidationError
+
 from core.app_runner.assistant_app_runner import AssistantApplicationRunner
 from core.app_runner.basic_app_runner import BasicApplicationRunner
 from core.app_runner.generate_task_pipeline import GenerateTaskPipeline
 from core.application_queue_manager import ApplicationQueueManager, ConversationTaskStoppedException, PublishFrom
-from core.entities.application_entities import (AdvancedChatPromptTemplateEntity,
-                                                AdvancedCompletionPromptTemplateEntity, AgentEntity, AgentToolEntity,
-                                                ApplicationGenerateEntity, AppOrchestrationConfigEntity, DatasetEntity,
-                                                DatasetRetrieveConfigEntity, ExternalDataVariableEntity,
-                                                FileUploadEntity, InvokeFrom, ModelConfigEntity, PromptTemplateEntity,
-                                                SensitiveWordAvoidanceEntity, AgentPromptEntity)
+from core.entities.application_entities import (
+    AdvancedChatPromptTemplateEntity,
+    AdvancedCompletionPromptTemplateEntity,
+    AgentEntity,
+    AgentPromptEntity,
+    AgentToolEntity,
+    ApplicationGenerateEntity,
+    AppOrchestrationConfigEntity,
+    DatasetEntity,
+    DatasetRetrieveConfigEntity,
+    ExternalDataVariableEntity,
+    FileUploadEntity,
+    InvokeFrom,
+    ModelConfigEntity,
+    PromptTemplateEntity,
+    SensitiveWordAvoidanceEntity,
+)
 from core.entities.model_entities import ModelStatus
 from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError
 from core.file.file_obj import FileObj
@@ -25,10 +39,8 @@ from core.prompt.prompt_template import PromptTemplateParser
 from core.provider_manager import ProviderManager
 from core.tools.prompt.template import REACT_PROMPT_TEMPLATES
 from extensions.ext_database import db
-from flask import Flask, current_app
 from models.account import Account
 from models.model import App, Conversation, EndUser, Message, MessageFile
-from pydantic import ValidationError
 
 logger = logging.getLogger(__name__)
 

api/core/application_queue_manager.py

@@ -3,15 +3,27 @@ import time
 from enum import Enum
 from typing import Any, Generator
 
+from sqlalchemy.orm import DeclarativeMeta
+
 from core.entities.application_entities import InvokeFrom
-from core.entities.queue_entities import (AnnotationReplyEvent, AppQueueEvent, QueueAgentThoughtEvent, QueueErrorEvent,
-                                          QueueMessage, QueueMessageEndEvent, QueueMessageEvent,
-                                          QueueMessageReplaceEvent, QueuePingEvent, QueueRetrieverResourcesEvent,
-                                          QueueStopEvent, QueueMessageFileEvent, QueueAgentMessageEvent)
+from core.entities.queue_entities import (
+    AnnotationReplyEvent,
+    AppQueueEvent,
+    QueueAgentMessageEvent,
+    QueueAgentThoughtEvent,
+    QueueErrorEvent,
+    QueueMessage,
+    QueueMessageEndEvent,
+    QueueMessageEvent,
+    QueueMessageFileEvent,
+    QueueMessageReplaceEvent,
+    QueuePingEvent,
+    QueueRetrieverResourcesEvent,
+    QueueStopEvent,
+)
 from core.model_runtime.entities.llm_entities import LLMResult, LLMResultChunk
 from extensions.ext_redis import redis_client
 from models.model import MessageAgentThought, MessageFile
-from sqlalchemy.orm import DeclarativeMeta
 
 
 class PublishFrom(Enum):

api/core/callback_handler/agent_loop_gather_callback_handler.py

@@ -3,6 +3,10 @@ import logging
 import time
 from typing import Any, Dict, List, Optional, Union, cast
 
+from langchain.agents import openai_functions_agent, openai_functions_multi_agent
+from langchain.callbacks.base import BaseCallbackHandler
+from langchain.schema import AgentAction, AgentFinish, BaseMessage, LLMResult
+
 from core.application_queue_manager import ApplicationQueueManager, PublishFrom
 from core.callback_handler.entity.agent_loop import AgentLoop
 from core.entities.application_entities import ModelConfigEntity
@@ -10,9 +14,6 @@ from core.model_runtime.entities.llm_entities import LLMResult as RuntimeLLMResu
 from core.model_runtime.entities.message_entities import AssistantPromptMessage, PromptMessage, UserPromptMessage
 from core.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel
 from extensions.ext_database import db
-from langchain.agents import openai_functions_agent, openai_functions_multi_agent
-from langchain.callbacks.base import BaseCallbackHandler
-from langchain.schema import AgentAction, AgentFinish, BaseMessage, ChatGeneration, LLMResult
 from models.model import Message, MessageAgentThought, MessageChain
 
 

api/core/callback_handler/agent_tool_callback_handler.py

@@ -1,9 +1,10 @@
 import os
 from typing import Any, Dict, Optional, Union
-from pydantic import BaseModel
 
 from langchain.callbacks.base import BaseCallbackHandler
 from langchain.input import print_text
+from pydantic import BaseModel
+
 
 class DifyAgentCallbackHandler(BaseCallbackHandler, BaseModel):
     """Callback Handler that prints to std out."""

api/core/callback_handler/index_tool_callback_handler.py

@@ -1,9 +1,10 @@
-from typing import List, Union
+from typing import List
+
+from langchain.schema import Document
 
 from core.application_queue_manager import ApplicationQueueManager, PublishFrom
 from core.entities.application_entities import InvokeFrom
 from extensions.ext_database import db
-from langchain.schema import Document
 from models.dataset import DatasetQuery, DocumentSegment
 from models.model import DatasetRetrieverResource
 

api/core/chain/llm_chain.py

@@ -1,14 +1,15 @@
 from typing import Any, Dict, List, Optional
 
+from langchain import LLMChain as LCLLMChain
+from langchain.callbacks.manager import CallbackManagerForChainRun
+from langchain.schema import Generation, LLMResult
+from langchain.schema.language_model import BaseLanguageModel
+
 from core.agent.agent.agent_llm_callback import AgentLLMCallback
 from core.entities.application_entities import ModelConfigEntity
 from core.entities.message_entities import lc_messages_to_prompt_messages
 from core.model_manager import ModelInstance
 from core.third_party.langchain.llms.fake import FakeLLM
-from langchain import LLMChain as LCLLMChain
-from langchain.callbacks.manager import CallbackManagerForChainRun
-from langchain.schema import Generation, LLMResult
-from langchain.schema.language_model import BaseLanguageModel
 
 
 class LLMChain(LCLLMChain):

api/core/data_loader/file_extractor.py

@@ -3,6 +3,10 @@ from pathlib import Path
 from typing import List, Optional, Union
 
 import requests
+from flask import current_app
+from langchain.document_loaders import Docx2txtLoader, TextLoader
+from langchain.schema import Document
+
 from core.data_loader.loader.csv_loader import CSVLoader
 from core.data_loader.loader.excel import ExcelLoader
 from core.data_loader.loader.html import HTMLLoader
@@ -16,9 +20,6 @@ from core.data_loader.loader.unstructured.unstructured_pptx import UnstructuredP
 from core.data_loader.loader.unstructured.unstructured_text import UnstructuredTextLoader
 from core.data_loader.loader.unstructured.unstructured_xml import UnstructuredXmlLoader
 from extensions.ext_storage import storage
-from flask import current_app
-from langchain.document_loaders import Docx2txtLoader, TextLoader
-from langchain.schema import Document
 from models.model import UploadFile
 
 SUPPORT_URL_CONTENT_TYPES = ['application/pdf', 'text/plain']

api/core/data_loader/loader/excel.py

@@ -1,4 +1,3 @@
-import json
 import logging
 from typing import List
 

api/core/data_loader/loader/notion.py

@@ -3,10 +3,11 @@ import logging
 from typing import Any, Dict, List, Optional
 
 import requests
-from extensions.ext_database import db
 from flask import current_app
 from langchain.document_loaders.base import BaseLoader
 from langchain.schema import Document
+
+from extensions.ext_database import db
 from models.dataset import Document as DocumentModel
 from models.source import DataSourceBinding
 

api/core/data_loader/loader/pdf.py

@@ -1,10 +1,11 @@
 import logging
 from typing import List, Optional
 
-from extensions.ext_storage import storage
 from langchain.document_loaders import PyPDFium2Loader
 from langchain.document_loaders.base import BaseLoader
 from langchain.schema import Document
+
+from extensions.ext_storage import storage
 from models.model import UploadFile
 
 logger = logging.getLogger(__name__)

api/core/data_loader/loader/unstructured/unstructured_ppt.py

@@ -1,5 +1,6 @@
 import logging
 from typing import List
+
 from langchain.document_loaders.base import BaseLoader
 from langchain.schema import Document