Merge remote-tracking branch 'origin/main' into feat/rag-2

# Conflicts:
#	.github/workflows/build-push.yml
#	web/app/components/workflow/nodes/_base/components/workflow-panel/index.tsx

api/services/tools/api_tools_manage_service.py

@@ -18,7 +18,7 @@ from core.tools.entities.tool_entities import (
 )
 from core.tools.tool_label_manager import ToolLabelManager
 from core.tools.tool_manager import ToolManager
-from core.tools.utils.configuration import ProviderConfigEncrypter
+from core.tools.utils.encryption import create_tool_provider_encrypter
 from core.tools.utils.parser import ApiBasedToolSchemaParser
 from extensions.ext_database import db
 from models.tools import ApiToolProvider
@@ -164,15 +164,11 @@ class ApiToolManageService:
         provider_controller.load_bundled_tools(tool_bundles)
 
         # encrypt credentials
-        tool_configuration = ProviderConfigEncrypter(
+        encrypter, _ = create_tool_provider_encrypter(
             tenant_id=tenant_id,
-            config=list(provider_controller.get_credentials_schema()),
-            provider_type=provider_controller.provider_type.value,
-            provider_identity=provider_controller.entity.identity.name,
+            controller=provider_controller,
         )
-
-        encrypted_credentials = tool_configuration.encrypt(credentials)
-        db_provider.credentials_str = json.dumps(encrypted_credentials)
+        db_provider.credentials_str = json.dumps(encrypter.encrypt(credentials))
 
         db.session.add(db_provider)
         db.session.commit()
@@ -297,28 +293,26 @@ class ApiToolManageService:
         provider_controller.load_bundled_tools(tool_bundles)
 
         # get original credentials if exists
-        tool_configuration = ProviderConfigEncrypter(
+        encrypter, cache = create_tool_provider_encrypter(
             tenant_id=tenant_id,
-            config=list(provider_controller.get_credentials_schema()),
-            provider_type=provider_controller.provider_type.value,
-            provider_identity=provider_controller.entity.identity.name,
+            controller=provider_controller,
         )
 
-        original_credentials = tool_configuration.decrypt(provider.credentials)
-        masked_credentials = tool_configuration.mask_tool_credentials(original_credentials)
+        original_credentials = encrypter.decrypt(provider.credentials)
+        masked_credentials = encrypter.mask_tool_credentials(original_credentials)
         # check if the credential has changed, save the original credential
         for name, value in credentials.items():
             if name in masked_credentials and value == masked_credentials[name]:
                 credentials[name] = original_credentials[name]
 
-        credentials = tool_configuration.encrypt(credentials)
+        credentials = encrypter.encrypt(credentials)
         provider.credentials_str = json.dumps(credentials)
 
         db.session.add(provider)
         db.session.commit()
 
         # delete cache
-        tool_configuration.delete_tool_credentials_cache()
+        cache.delete()
 
         # update labels
         ToolLabelManager.update_tool_labels(provider_controller, labels)
@@ -416,15 +410,13 @@ class ApiToolManageService:
 
         # decrypt credentials
         if db_provider.id:
-            tool_configuration = ProviderConfigEncrypter(
+            encrypter, _ = create_tool_provider_encrypter(
                 tenant_id=tenant_id,
-                config=list(provider_controller.get_credentials_schema()),
-                provider_type=provider_controller.provider_type.value,
-                provider_identity=provider_controller.entity.identity.name,
+                controller=provider_controller,
             )
-            decrypted_credentials = tool_configuration.decrypt(credentials)
+            decrypted_credentials = encrypter.decrypt(credentials)
             # check if the credential has changed, save the original credential
-            masked_credentials = tool_configuration.mask_tool_credentials(decrypted_credentials)
+            masked_credentials = encrypter.mask_tool_credentials(decrypted_credentials)
             for name, value in credentials.items():
                 if name in masked_credentials and value == masked_credentials[name]:
                     credentials[name] = decrypted_credentials[name]
@@ -446,7 +438,7 @@ class ApiToolManageService:
         return {"result": result or "empty response"}
 
     @staticmethod
-    def list_api_tools(user_id: str, tenant_id: str) -> list[ToolProviderApiEntity]:
+    def list_api_tools(tenant_id: str) -> list[ToolProviderApiEntity]:
         """
         list api tools
         """
@@ -474,7 +466,7 @@ class ApiToolManageService:
             for tool in tools or []:
                 user_provider.tools.append(
                     ToolTransformService.convert_tool_entity_to_api_entity(
-                        tenant_id=tenant_id, tool=tool, credentials=user_provider.original_credentials, labels=labels
+                        tenant_id=tenant_id, tool=tool, labels=labels
                     )
                 )
 

api/services/tools/builtin_tools_manage_service.py

@@ -1,28 +1,84 @@
 import json
 import logging
+import re
+from collections.abc import Mapping
 from pathlib import Path
+from typing import Any, Optional
 
 from sqlalchemy.orm import Session
 
 from configs import dify_config
+from constants import HIDDEN_VALUE, UNKNOWN_VALUE
 from core.helper.position_helper import is_filtered
-from core.model_runtime.utils.encoders import jsonable_encoder
+from core.helper.provider_cache import NoOpProviderCredentialCache, ToolProviderCredentialsCache
 from core.plugin.entities.plugin import ToolProviderID
-from core.plugin.impl.exc import PluginDaemonClientSideError
+from core.tools.builtin_tool.provider import BuiltinToolProviderController
 from core.tools.builtin_tool.providers._positions import BuiltinToolProviderSort
-from core.tools.entities.api_entities import ToolApiEntity, ToolProviderApiEntity
-from core.tools.errors import ToolNotFoundError, ToolProviderCredentialValidationError, ToolProviderNotFoundError
+from core.tools.entities.api_entities import (
+    ToolApiEntity,
+    ToolProviderApiEntity,
+    ToolProviderCredentialApiEntity,
+    ToolProviderCredentialInfoApiEntity,
+)
+from core.tools.entities.tool_entities import CredentialType
+from core.tools.errors import ToolProviderNotFoundError
+from core.tools.plugin_tool.provider import PluginToolProviderController
 from core.tools.tool_label_manager import ToolLabelManager
 from core.tools.tool_manager import ToolManager
-from core.tools.utils.configuration import ProviderConfigEncrypter
+from core.tools.utils.encryption import create_provider_encrypter
+from core.tools.utils.system_oauth_encryption import decrypt_system_oauth_params
 from extensions.ext_database import db
-from models.tools import BuiltinToolProvider
+from extensions.ext_redis import redis_client
+from models.tools import BuiltinToolProvider, ToolOAuthSystemClient, ToolOAuthTenantClient
+from services.plugin.plugin_service import PluginService
 from services.tools.tools_transform_service import ToolTransformService
 
 logger = logging.getLogger(__name__)
 
 
 class BuiltinToolManageService:
+    __MAX_BUILTIN_TOOL_PROVIDER_COUNT__ = 100
+
+    @staticmethod
+    def delete_custom_oauth_client_params(tenant_id: str, provider: str):
+        """
+        delete custom oauth client params
+        """
+        tool_provider = ToolProviderID(provider)
+        with Session(db.engine) as session:
+            session.query(ToolOAuthTenantClient).filter_by(
+                tenant_id=tenant_id,
+                provider=tool_provider.provider_name,
+                plugin_id=tool_provider.plugin_id,
+            ).delete()
+            session.commit()
+        return {"result": "success"}
+
+    @staticmethod
+    def get_builtin_tool_provider_oauth_client_schema(tenant_id: str, provider_name: str):
+        """
+        get builtin tool provider oauth client schema
+        """
+        provider = ToolManager.get_builtin_provider(provider_name, tenant_id)
+        verified = not isinstance(provider, PluginToolProviderController) or PluginService.is_plugin_verified(
+            tenant_id, provider.plugin_unique_identifier
+        )
+
+        is_oauth_custom_client_enabled = BuiltinToolManageService.is_oauth_custom_client_enabled(
+            tenant_id, provider_name
+        )
+        is_system_oauth_params_exists = verified and BuiltinToolManageService.is_oauth_system_client_exists(
+            provider_name
+        )
+        result = {
+            "schema": provider.get_oauth_client_schema(),
+            "is_oauth_custom_client_enabled": is_oauth_custom_client_enabled,
+            "is_system_oauth_params_exists": is_system_oauth_params_exists,
+            "client_params": BuiltinToolManageService.get_custom_oauth_client_params(tenant_id, provider_name),
+            "redirect_uri": f"{dify_config.CONSOLE_API_URL}/console/api/oauth/plugin/{provider_name}/tool/callback",
+        }
+        return result
+
     @staticmethod
     def list_builtin_tool_provider_tools(tenant_id: str, provider: str) -> list[ToolApiEntity]:
         """
@@ -36,27 +92,11 @@ class BuiltinToolManageService:
         provider_controller = ToolManager.get_builtin_provider(provider, tenant_id)
         tools = provider_controller.get_tools()
 
-        tool_provider_configurations = ProviderConfigEncrypter(
-            tenant_id=tenant_id,
-            config=[x.to_basic_provider_config() for x in provider_controller.get_credentials_schema()],
-            provider_type=provider_controller.provider_type.value,
-            provider_identity=provider_controller.entity.identity.name,
-        )
-        # check if user has added the provider
-        builtin_provider = BuiltinToolManageService._fetch_builtin_provider(provider, tenant_id)
-
-        credentials = {}
-        if builtin_provider is not None:
-            # get credentials
-            credentials = builtin_provider.credentials
-            credentials = tool_provider_configurations.decrypt(credentials)
-
         result: list[ToolApiEntity] = []
         for tool in tools or []:
             result.append(
                 ToolTransformService.convert_tool_entity_to_api_entity(
                     tool=tool,
-                    credentials=credentials,
                     tenant_id=tenant_id,
                     labels=ToolLabelManager.get_tool_labels(provider_controller),
                 )
@@ -65,25 +105,15 @@ class BuiltinToolManageService:
         return result
 
     @staticmethod
-    def get_builtin_tool_provider_info(user_id: str, tenant_id: str, provider: str):
+    def get_builtin_tool_provider_info(tenant_id: str, provider: str):
         """
         get builtin tool provider info
         """
         provider_controller = ToolManager.get_builtin_provider(provider, tenant_id)
-        tool_provider_configurations = ProviderConfigEncrypter(
-            tenant_id=tenant_id,
-            config=[x.to_basic_provider_config() for x in provider_controller.get_credentials_schema()],
-            provider_type=provider_controller.provider_type.value,
-            provider_identity=provider_controller.entity.identity.name,
-        )
         # check if user has added the provider
-        builtin_provider = BuiltinToolManageService._fetch_builtin_provider(provider, tenant_id)
-
-        credentials = {}
-        if builtin_provider is not None:
-            # get credentials
-            credentials = builtin_provider.credentials
-            credentials = tool_provider_configurations.decrypt(credentials)
+        builtin_provider = BuiltinToolManageService.get_builtin_provider(provider, tenant_id)
+        if builtin_provider is None:
+            raise ValueError(f"you have not added provider {provider}")
 
         entity = ToolTransformService.builtin_provider_to_user_provider(
             provider_controller=provider_controller,
@@ -92,128 +122,407 @@ class BuiltinToolManageService:
         )
 
         entity.original_credentials = {}
-
         return entity
 
     @staticmethod
-    def list_builtin_provider_credentials_schema(provider_name: str, tenant_id: str):
+    def list_builtin_provider_credentials_schema(provider_name: str, credential_type: CredentialType, tenant_id: str):
         """
         list builtin provider credentials schema
 
+        :param credential_type: credential type
         :param provider_name: the name of the provider
         :param tenant_id: the id of the tenant
         :return: the list of tool providers
         """
         provider = ToolManager.get_builtin_provider(provider_name, tenant_id)
-        return jsonable_encoder(provider.get_credentials_schema())
+        return provider.get_credentials_schema_by_type(credential_type)
 
     @staticmethod
     def update_builtin_tool_provider(
-        session: Session, user_id: str, tenant_id: str, provider_name: str, credentials: dict
+        user_id: str,
+        tenant_id: str,
+        provider: str,
+        credential_id: str,
+        credentials: dict | None = None,
+        name: str | None = None,
     ):
         """
         update builtin tool provider
         """
-        # get if the provider exists
-        provider = BuiltinToolManageService._fetch_builtin_provider(provider_name, tenant_id)
-
-        try:
-            # get provider
-            provider_controller = ToolManager.get_builtin_provider(provider_name, tenant_id)
-            if not provider_controller.need_credentials:
-                raise ValueError(f"provider {provider_name} does not need credentials")
-            tool_configuration = ProviderConfigEncrypter(
-                tenant_id=tenant_id,
-                config=[x.to_basic_provider_config() for x in provider_controller.get_credentials_schema()],
-                provider_type=provider_controller.provider_type.value,
-                provider_identity=provider_controller.entity.identity.name,
+        with Session(db.engine) as session:
+            # get if the provider exists
+            db_provider = (
+                session.query(BuiltinToolProvider)
+                .filter(
+                    BuiltinToolProvider.tenant_id == tenant_id,
+                    BuiltinToolProvider.id == credential_id,
+                )
+                .first()
             )
+            if db_provider is None:
+                raise ValueError(f"you have not added provider {provider}")
 
-            # get original credentials if exists
-            if provider is not None:
-                original_credentials = tool_configuration.decrypt(provider.credentials)
-                masked_credentials = tool_configuration.mask_tool_credentials(original_credentials)
-                # check if the credential has changed, save the original credential
-                for name, value in credentials.items():
-                    if name in masked_credentials and value == masked_credentials[name]:
-                        credentials[name] = original_credentials[name]
-            # validate credentials
-            provider_controller.validate_credentials(user_id, credentials)
-            # encrypt credentials
-            credentials = tool_configuration.encrypt(credentials)
-        except (
-            PluginDaemonClientSideError,
-            ToolProviderNotFoundError,
-            ToolNotFoundError,
-            ToolProviderCredentialValidationError,
-        ) as e:
-            raise ValueError(str(e))
+            try:
+                if CredentialType.of(db_provider.credential_type).is_editable() and credentials:
+                    provider_controller = ToolManager.get_builtin_provider(provider, tenant_id)
+                    if not provider_controller.need_credentials:
+                        raise ValueError(f"provider {provider} does not need credentials")
 
-        if provider is None:
-            # create provider
-            provider = BuiltinToolProvider(
-                tenant_id=tenant_id,
-                user_id=user_id,
-                provider=provider_name,
-                encrypted_credentials=json.dumps(credentials),
-            )
+                    encrypter, cache = BuiltinToolManageService.create_tool_encrypter(
+                        tenant_id, db_provider, provider, provider_controller
+                    )
 
-            db.session.add(provider)
-        else:
-            provider.encrypted_credentials = json.dumps(credentials)
+                    original_credentials = encrypter.decrypt(db_provider.credentials)
+                    new_credentials: dict = {
+                        key: value if value != HIDDEN_VALUE else original_credentials.get(key, UNKNOWN_VALUE)
+                        for key, value in credentials.items()
+                    }
 
-            # delete cache
-            tool_configuration.delete_tool_credentials_cache()
+                    if CredentialType.of(db_provider.credential_type).is_validate_allowed():
+                        provider_controller.validate_credentials(user_id, new_credentials)
 
-        db.session.commit()
+                    # encrypt credentials
+                    db_provider.encrypted_credentials = json.dumps(encrypter.encrypt(new_credentials))
+
+                    cache.delete()
+
+                # update name if provided
+                if name and name != db_provider.name:
+                    # check if the name is already used
+                    if (
+                        session.query(BuiltinToolProvider)
+                        .filter_by(tenant_id=tenant_id, provider=provider, name=name)
+                        .count()
+                        > 0
+                    ):
+                        raise ValueError(f"the credential name '{name}' is already used")
+
+                    db_provider.name = name
+
+                session.commit()
+            except Exception as e:
+                session.rollback()
+                raise ValueError(str(e))
         return {"result": "success"}
 
     @staticmethod
-    def get_builtin_tool_provider_credentials(tenant_id: str, provider_name: str):
+    def add_builtin_tool_provider(
+        user_id: str,
+        api_type: CredentialType,
+        tenant_id: str,
+        provider: str,
+        credentials: dict,
+        name: str | None = None,
+    ):
+        """
+        add builtin tool provider
+        """
+        try:
+            with Session(db.engine) as session:
+                lock = f"builtin_tool_provider_create_lock:{tenant_id}_{provider}"
+                with redis_client.lock(lock, timeout=20):
+                    provider_controller = ToolManager.get_builtin_provider(provider, tenant_id)
+                    if not provider_controller.need_credentials:
+                        raise ValueError(f"provider {provider} does not need credentials")
+
+                    provider_count = (
+                        session.query(BuiltinToolProvider).filter_by(tenant_id=tenant_id, provider=provider).count()
+                    )
+
+                    # check if the provider count is reached the limit
+                    if provider_count >= BuiltinToolManageService.__MAX_BUILTIN_TOOL_PROVIDER_COUNT__:
+                        raise ValueError(f"you have reached the maximum number of providers for {provider}")
+
+                    # validate credentials if allowed
+                    if CredentialType.of(api_type).is_validate_allowed():
+                        provider_controller.validate_credentials(user_id, credentials)
+
+                    # generate name if not provided
+                    if name is None or name == "":
+                        name = BuiltinToolManageService.generate_builtin_tool_provider_name(
+                            session=session, tenant_id=tenant_id, provider=provider, credential_type=api_type
+                        )
+                    else:
+                        # check if the name is already used
+                        if (
+                            session.query(BuiltinToolProvider)
+                            .filter_by(tenant_id=tenant_id, provider=provider, name=name)
+                            .count()
+                            > 0
+                        ):
+                            raise ValueError(f"the credential name '{name}' is already used")
+
+                    # create encrypter
+                    encrypter, _ = create_provider_encrypter(
+                        tenant_id=tenant_id,
+                        config=[
+                            x.to_basic_provider_config()
+                            for x in provider_controller.get_credentials_schema_by_type(api_type)
+                        ],
+                        cache=NoOpProviderCredentialCache(),
+                    )
+
+                    db_provider = BuiltinToolProvider(
+                        tenant_id=tenant_id,
+                        user_id=user_id,
+                        provider=provider,
+                        encrypted_credentials=json.dumps(encrypter.encrypt(credentials)),
+                        credential_type=api_type.value,
+                        name=name,
+                    )
+
+                    session.add(db_provider)
+                    session.commit()
+        except Exception as e:
+            session.rollback()
+            raise ValueError(str(e))
+        return {"result": "success"}
+
+    @staticmethod
+    def create_tool_encrypter(
+        tenant_id: str,
+        db_provider: BuiltinToolProvider,
+        provider: str,
+        provider_controller: BuiltinToolProviderController,
+    ):
+        encrypter, cache = create_provider_encrypter(
+            tenant_id=tenant_id,
+            config=[
+                x.to_basic_provider_config()
+                for x in provider_controller.get_credentials_schema_by_type(db_provider.credential_type)
+            ],
+            cache=ToolProviderCredentialsCache(tenant_id=tenant_id, provider=provider, credential_id=db_provider.id),
+        )
+        return encrypter, cache
+
+    @staticmethod
+    def generate_builtin_tool_provider_name(
+        session: Session, tenant_id: str, provider: str, credential_type: CredentialType
+    ) -> str:
+        try:
+            db_providers = (
+                session.query(BuiltinToolProvider)
+                .filter_by(
+                    tenant_id=tenant_id,
+                    provider=provider,
+                    credential_type=credential_type.value,
+                )
+                .order_by(BuiltinToolProvider.created_at.desc())
+                .all()
+            )
+
+            # Get the default name pattern
+            default_pattern = f"{credential_type.get_name()}"
+
+            # Find all names that match the default pattern: "{default_pattern} {number}"
+            pattern = rf"^{re.escape(default_pattern)}\s+(\d+)$"
+            numbers = []
+
+            for db_provider in db_providers:
+                if db_provider.name:
+                    match = re.match(pattern, db_provider.name.strip())
+                    if match:
+                        numbers.append(int(match.group(1)))
+
+            # If no default pattern names found, start with 1
+            if not numbers:
+                return f"{default_pattern} 1"
+
+            # Find the next number
+            max_number = max(numbers)
+            return f"{default_pattern} {max_number + 1}"
+        except Exception as e:
+            logger.warning(f"Error generating next provider name for {provider}: {str(e)}")
+            # fallback
+            return f"{credential_type.get_name()} 1"
+
+    @staticmethod
+    def get_builtin_tool_provider_credentials(
+        tenant_id: str, provider_name: str
+    ) -> list[ToolProviderCredentialApiEntity]:
         """
         get builtin tool provider credentials
         """
-        provider_obj = BuiltinToolManageService._fetch_builtin_provider(provider_name, tenant_id)
+        with db.session.no_autoflush:
+            providers = (
+                db.session.query(BuiltinToolProvider)
+                .filter_by(tenant_id=tenant_id, provider=provider_name)
+                .order_by(BuiltinToolProvider.is_default.desc(), BuiltinToolProvider.created_at.asc())
+                .all()
+            )
 
-        if provider_obj is None:
-            return {}
+            if len(providers) == 0:
+                return []
 
-        provider_controller = ToolManager.get_builtin_provider(provider_obj.provider, tenant_id)
-        tool_configuration = ProviderConfigEncrypter(
-            tenant_id=tenant_id,
-            config=[x.to_basic_provider_config() for x in provider_controller.get_credentials_schema()],
-            provider_type=provider_controller.provider_type.value,
-            provider_identity=provider_controller.entity.identity.name,
-        )
-        credentials = tool_configuration.decrypt(provider_obj.credentials)
-        credentials = tool_configuration.mask_tool_credentials(credentials)
-        return credentials
+            default_provider = providers[0]
+            default_provider.is_default = True
+            provider_controller = ToolManager.get_builtin_provider(default_provider.provider, tenant_id)
+
+            credentials: list[ToolProviderCredentialApiEntity] = []
+            encrypters = {}
+            for provider in providers:
+                credential_type = provider.credential_type
+                if credential_type not in encrypters:
+                    encrypters[credential_type] = BuiltinToolManageService.create_tool_encrypter(
+                        tenant_id, provider, provider.provider, provider_controller
+                    )[0]
+                encrypter = encrypters[credential_type]
+                decrypt_credential = encrypter.mask_tool_credentials(encrypter.decrypt(provider.credentials))
+                credential_entity = ToolTransformService.convert_builtin_provider_to_credential_entity(
+                    provider=provider,
+                    credentials=decrypt_credential,
+                )
+                credentials.append(credential_entity)
+            return credentials
 
     @staticmethod
-    def delete_builtin_tool_provider(user_id: str, tenant_id: str, provider_name: str):
+    def get_builtin_tool_provider_credential_info(tenant_id: str, provider: str) -> ToolProviderCredentialInfoApiEntity:
+        """
+        get builtin tool provider credential info
+        """
+        provider_controller = ToolManager.get_builtin_provider(provider, tenant_id)
+        supported_credential_types = provider_controller.get_supported_credential_types()
+        credentials = BuiltinToolManageService.get_builtin_tool_provider_credentials(tenant_id, provider)
+        credential_info = ToolProviderCredentialInfoApiEntity(
+            supported_credential_types=supported_credential_types,
+            is_oauth_custom_client_enabled=BuiltinToolManageService.is_oauth_custom_client_enabled(tenant_id, provider),
+            credentials=credentials,
+        )
+
+        return credential_info
+
+    @staticmethod
+    def delete_builtin_tool_provider(tenant_id: str, provider: str, credential_id: str):
         """
         delete tool provider
         """
-        provider_obj = BuiltinToolManageService._fetch_builtin_provider(provider_name, tenant_id)
+        with Session(db.engine) as session:
+            db_provider = (
+                session.query(BuiltinToolProvider)
+                .filter(
+                    BuiltinToolProvider.tenant_id == tenant_id,
+                    BuiltinToolProvider.id == credential_id,
+                )
+                .first()
+            )
 
-        if provider_obj is None:
-            raise ValueError(f"you have not added provider {provider_name}")
+            if db_provider is None:
+                raise ValueError(f"you have not added provider {provider}")
 
-        db.session.delete(provider_obj)
-        db.session.commit()
+            session.delete(db_provider)
+            session.commit()
 
-        # delete cache
-        provider_controller = ToolManager.get_builtin_provider(provider_name, tenant_id)
-        tool_configuration = ProviderConfigEncrypter(
-            tenant_id=tenant_id,
-            config=[x.to_basic_provider_config() for x in provider_controller.get_credentials_schema()],
-            provider_type=provider_controller.provider_type.value,
-            provider_identity=provider_controller.entity.identity.name,
-        )
-        tool_configuration.delete_tool_credentials_cache()
+            # delete cache
+            provider_controller = ToolManager.get_builtin_provider(provider, tenant_id)
+            _, cache = BuiltinToolManageService.create_tool_encrypter(
+                tenant_id, db_provider, provider, provider_controller
+            )
+            cache.delete()
 
         return {"result": "success"}
 
+    @staticmethod
+    def set_default_provider(tenant_id: str, user_id: str, provider: str, id: str):
+        """
+        set default provider
+        """
+        with Session(db.engine) as session:
+            # get provider
+            target_provider = session.query(BuiltinToolProvider).filter_by(id=id).first()
+            if target_provider is None:
+                raise ValueError("provider not found")
+
+            # clear default provider
+            session.query(BuiltinToolProvider).filter_by(
+                tenant_id=tenant_id, user_id=user_id, provider=provider, is_default=True
+            ).update({"is_default": False})
+
+            # set new default provider
+            target_provider.is_default = True
+            session.commit()
+        return {"result": "success"}
+
+    @staticmethod
+    def is_oauth_system_client_exists(provider_name: str) -> bool:
+        """
+        check if oauth system client exists
+        """
+        tool_provider = ToolProviderID(provider_name)
+        with Session(db.engine).no_autoflush as session:
+            system_client: ToolOAuthSystemClient | None = (
+                session.query(ToolOAuthSystemClient)
+                .filter_by(plugin_id=tool_provider.plugin_id, provider=tool_provider.provider_name)
+                .first()
+            )
+            return system_client is not None
+
+    @staticmethod
+    def is_oauth_custom_client_enabled(tenant_id: str, provider: str) -> bool:
+        """
+        check if oauth custom client is enabled
+        """
+        tool_provider = ToolProviderID(provider)
+        with Session(db.engine).no_autoflush as session:
+            user_client: ToolOAuthTenantClient | None = (
+                session.query(ToolOAuthTenantClient)
+                .filter_by(
+                    tenant_id=tenant_id,
+                    provider=tool_provider.provider_name,
+                    plugin_id=tool_provider.plugin_id,
+                    enabled=True,
+                )
+                .first()
+            )
+            return user_client is not None and user_client.enabled
+
+    @staticmethod
+    def get_oauth_client(tenant_id: str, provider: str) -> Mapping[str, Any] | None:
+        """
+        get builtin tool provider
+        """
+        tool_provider = ToolProviderID(provider)
+        provider_controller = ToolManager.get_builtin_provider(provider, tenant_id)
+        encrypter, _ = create_provider_encrypter(
+            tenant_id=tenant_id,
+            config=[x.to_basic_provider_config() for x in provider_controller.get_oauth_client_schema()],
+            cache=NoOpProviderCredentialCache(),
+        )
+        with Session(db.engine).no_autoflush as session:
+            user_client: ToolOAuthTenantClient | None = (
+                session.query(ToolOAuthTenantClient)
+                .filter_by(
+                    tenant_id=tenant_id,
+                    provider=tool_provider.provider_name,
+                    plugin_id=tool_provider.plugin_id,
+                    enabled=True,
+                )
+                .first()
+            )
+            oauth_params: Mapping[str, Any] | None = None
+            if user_client:
+                oauth_params = encrypter.decrypt(user_client.oauth_params)
+                return oauth_params
+
+            # only verified provider can use custom oauth client
+            is_verified = not isinstance(provider, PluginToolProviderController) or PluginService.is_plugin_verified(
+                tenant_id, provider.plugin_unique_identifier
+            )
+            if not is_verified:
+                return oauth_params
+
+            system_client: ToolOAuthSystemClient | None = (
+                session.query(ToolOAuthSystemClient)
+                .filter_by(plugin_id=tool_provider.plugin_id, provider=tool_provider.provider_name)
+                .first()
+            )
+            if system_client:
+                try:
+                    oauth_params = decrypt_system_oauth_params(system_client.encrypted_oauth_params)
+                except Exception as e:
+                    raise ValueError(f"Error decrypting system oauth params: {e}")
+
+            return oauth_params
+
     @staticmethod
     def get_builtin_tool_provider_icon(provider: str):
         """
@@ -234,9 +543,7 @@ class BuiltinToolManageService:
 
         with db.session.no_autoflush:
             # get all user added providers
-            db_providers: list[BuiltinToolProvider] = (
-                db.session.query(BuiltinToolProvider).filter(BuiltinToolProvider.tenant_id == tenant_id).all() or []
-            )
+            db_providers: list[BuiltinToolProvider] = ToolManager.list_default_builtin_providers(tenant_id)
 
             # rewrite db_providers
             for db_provider in db_providers:
@@ -275,7 +582,6 @@ class BuiltinToolManageService:
                             ToolTransformService.convert_tool_entity_to_api_entity(
                                 tenant_id=tenant_id,
                                 tool=tool,
-                                credentials=user_builtin_provider.original_credentials,
                                 labels=ToolLabelManager.get_tool_labels(provider_controller),
                             )
                         )
@@ -287,43 +593,153 @@ class BuiltinToolManageService:
         return BuiltinToolProviderSort.sort(result)
 
     @staticmethod
-    def _fetch_builtin_provider(provider_name: str, tenant_id: str) -> BuiltinToolProvider | None:
-        try:
-            full_provider_name = provider_name
-            provider_id_entity = ToolProviderID(provider_name)
-            provider_name = provider_id_entity.provider_name
-            if provider_id_entity.organization != "langgenius":
-                provider_obj = (
-                    db.session.query(BuiltinToolProvider)
-                    .filter(
-                        BuiltinToolProvider.tenant_id == tenant_id,
-                        BuiltinToolProvider.provider == full_provider_name,
+    def get_builtin_provider(provider_name: str, tenant_id: str) -> Optional[BuiltinToolProvider]:
+        """
+        This method is used to fetch the builtin provider from the database
+        1.if the default provider exists, return the default provider
+        2.if the default provider does not exist, return the oldest provider
+        """
+        with Session(db.engine) as session:
+            try:
+                full_provider_name = provider_name
+                provider_id_entity = ToolProviderID(provider_name)
+                provider_name = provider_id_entity.provider_name
+
+                if provider_id_entity.organization != "langgenius":
+                    provider = (
+                        session.query(BuiltinToolProvider)
+                        .filter(
+                            BuiltinToolProvider.tenant_id == tenant_id,
+                            BuiltinToolProvider.provider == full_provider_name,
+                        )
+                        .order_by(
+                            BuiltinToolProvider.is_default.desc(),  # default=True first
+                            BuiltinToolProvider.created_at.asc(),  # oldest first
+                        )
+                        .first()
                     )
-                    .first()
-                )
-            else:
-                provider_obj = (
-                    db.session.query(BuiltinToolProvider)
-                    .filter(
-                        BuiltinToolProvider.tenant_id == tenant_id,
-                        (BuiltinToolProvider.provider == provider_name)
-                        | (BuiltinToolProvider.provider == full_provider_name),
+                else:
+                    provider = (
+                        session.query(BuiltinToolProvider)
+                        .filter(
+                            BuiltinToolProvider.tenant_id == tenant_id,
+                            (BuiltinToolProvider.provider == provider_name)
+                            | (BuiltinToolProvider.provider == full_provider_name),
+                        )
+                        .order_by(
+                            BuiltinToolProvider.is_default.desc(),  # default=True first
+                            BuiltinToolProvider.created_at.asc(),  # oldest first
+                        )
+                        .first()
+                    )
+
+                if provider is None:
+                    return None
+
+                provider.provider = ToolProviderID(provider.provider).to_string()
+                return provider
+            except Exception:
+                # it's an old provider without organization
+                return (
+                    session.query(BuiltinToolProvider)
+                    .filter(BuiltinToolProvider.tenant_id == tenant_id, BuiltinToolProvider.provider == provider_name)
+                    .order_by(
+                        BuiltinToolProvider.is_default.desc(),  # default=True first
+                        BuiltinToolProvider.created_at.asc(),  # oldest first
                     )
                     .first()
                 )
 
-            if provider_obj is None:
-                return None
+    @staticmethod
+    def save_custom_oauth_client_params(
+        tenant_id: str,
+        provider: str,
+        client_params: Optional[dict] = None,
+        enable_oauth_custom_client: Optional[bool] = None,
+    ):
+        """
+        setup oauth custom client
+        """
+        if client_params is None and enable_oauth_custom_client is None:
+            return {"result": "success"}
 
-            provider_obj.provider = ToolProviderID(provider_obj.provider).to_string()
-            return provider_obj
-        except Exception:
-            # it's an old provider without organization
-            return (
-                db.session.query(BuiltinToolProvider)
-                .filter(
-                    BuiltinToolProvider.tenant_id == tenant_id,
-                    (BuiltinToolProvider.provider == provider_name),
+        tool_provider = ToolProviderID(provider)
+        provider_controller = ToolManager.get_builtin_provider(provider, tenant_id)
+        if not provider_controller:
+            raise ToolProviderNotFoundError(f"Provider {provider} not found")
+
+        if not isinstance(provider_controller, (BuiltinToolProviderController, PluginToolProviderController)):
+            raise ValueError(f"Provider {provider} is not a builtin or plugin provider")
+
+        with Session(db.engine) as session:
+            custom_client_params = (
+                session.query(ToolOAuthTenantClient)
+                .filter_by(
+                    tenant_id=tenant_id,
+                    plugin_id=tool_provider.plugin_id,
+                    provider=tool_provider.provider_name,
                 )
                 .first()
             )
+
+            # if the record does not exist, create a basic record
+            if custom_client_params is None:
+                custom_client_params = ToolOAuthTenantClient(
+                    tenant_id=tenant_id,
+                    plugin_id=tool_provider.plugin_id,
+                    provider=tool_provider.provider_name,
+                )
+                session.add(custom_client_params)
+
+            if client_params is not None:
+                encrypter, _ = create_provider_encrypter(
+                    tenant_id=tenant_id,
+                    config=[x.to_basic_provider_config() for x in provider_controller.get_oauth_client_schema()],
+                    cache=NoOpProviderCredentialCache(),
+                )
+                original_params = encrypter.decrypt(custom_client_params.oauth_params)
+                new_params: dict = {
+                    key: value if value != HIDDEN_VALUE else original_params.get(key, UNKNOWN_VALUE)
+                    for key, value in client_params.items()
+                }
+                custom_client_params.encrypted_oauth_params = json.dumps(encrypter.encrypt(new_params))
+
+            if enable_oauth_custom_client is not None:
+                custom_client_params.enabled = enable_oauth_custom_client
+
+            session.commit()
+        return {"result": "success"}
+
+    @staticmethod
+    def get_custom_oauth_client_params(tenant_id: str, provider: str):
+        """
+        get custom oauth client params
+        """
+        with Session(db.engine) as session:
+            tool_provider = ToolProviderID(provider)
+            custom_oauth_client_params: ToolOAuthTenantClient | None = (
+                session.query(ToolOAuthTenantClient)
+                .filter_by(
+                    tenant_id=tenant_id,
+                    plugin_id=tool_provider.plugin_id,
+                    provider=tool_provider.provider_name,
+                )
+                .first()
+            )
+            if custom_oauth_client_params is None:
+                return {}
+
+            provider_controller = ToolManager.get_builtin_provider(provider, tenant_id)
+            if not provider_controller:
+                raise ToolProviderNotFoundError(f"Provider {provider} not found")
+
+            if not isinstance(provider_controller, BuiltinToolProviderController):
+                raise ValueError(f"Provider {provider} is not a builtin or plugin provider")
+
+            encrypter, _ = create_provider_encrypter(
+                tenant_id=tenant_id,
+                config=[x.to_basic_provider_config() for x in provider_controller.get_oauth_client_schema()],
+                cache=NoOpProviderCredentialCache(),
+            )
+
+            return encrypter.mask_tool_credentials(encrypter.decrypt(custom_oauth_client_params.oauth_params))

api/services/tools/mcp_tools_mange_service.py

@@ -7,13 +7,14 @@ from sqlalchemy import or_
 from sqlalchemy.exc import IntegrityError
 
 from core.helper import encrypter
+from core.helper.provider_cache import NoOpProviderCredentialCache
 from core.mcp.error import MCPAuthError, MCPError
 from core.mcp.mcp_client import MCPClient
 from core.tools.entities.api_entities import ToolProviderApiEntity
 from core.tools.entities.common_entities import I18nObject
 from core.tools.entities.tool_entities import ToolProviderType
 from core.tools.mcp_tool.provider import MCPToolProviderController
-from core.tools.utils.configuration import ProviderConfigEncrypter
+from core.tools.utils.encryption import ProviderConfigEncrypter
 from extensions.ext_database import db
 from models.tools import MCPToolProvider
 from services.tools.tools_transform_service import ToolTransformService
@@ -69,6 +70,7 @@ class MCPToolManageService:
                     MCPToolProvider.server_url_hash == server_url_hash,
                     MCPToolProvider.server_identifier == server_identifier,
                 ),
+                MCPToolProvider.tenant_id == tenant_id,
             )
             .first()
         )
@@ -197,8 +199,7 @@ class MCPToolManageService:
         tool_configuration = ProviderConfigEncrypter(
             tenant_id=mcp_provider.tenant_id,
             config=list(provider_controller.get_credentials_schema()),
-            provider_type=provider_controller.provider_type.value,
-            provider_identity=provider_controller.provider_id,
+            provider_config_cache=NoOpProviderCredentialCache(),
         )
         credentials = tool_configuration.encrypt(credentials)
         mcp_provider.updated_at = datetime.now()

api/services/tools/tools_transform_service.py

@@ -5,22 +5,24 @@ from typing import Any, Optional, Union, cast
 from yarl import URL
 
 from configs import dify_config
+from core.helper.provider_cache import ToolProviderCredentialsCache
 from core.mcp.types import Tool as MCPTool
 from core.plugin.entities.plugin_daemon import PluginDatasourceProviderEntity
 from core.tools.__base.tool import Tool
 from core.tools.__base.tool_runtime import ToolRuntime
 from core.tools.builtin_tool.provider import BuiltinToolProviderController
 from core.tools.custom_tool.provider import ApiToolProviderController
-from core.tools.entities.api_entities import ToolApiEntity, ToolProviderApiEntity
+from core.tools.entities.api_entities import ToolApiEntity, ToolProviderApiEntity, ToolProviderCredentialApiEntity
 from core.tools.entities.common_entities import I18nObject
 from core.tools.entities.tool_bundle import ApiToolBundle
 from core.tools.entities.tool_entities import (
     ApiProviderAuthType,
+    CredentialType,
     ToolParameter,
     ToolProviderType,
 )
 from core.tools.plugin_tool.provider import PluginToolProviderController
-from core.tools.utils.configuration import ProviderConfigEncrypter
+from core.tools.utils.encryption import create_provider_encrypter, create_tool_provider_encrypter
 from core.tools.workflow_as_tool.provider import WorkflowToolProviderController
 from core.tools.workflow_as_tool.tool import WorkflowTool
 from models.tools import ApiToolProvider, BuiltinToolProvider, MCPToolProvider, WorkflowToolProvider
@@ -132,7 +134,12 @@ class ToolTransformService:
             result.plugin_unique_identifier = provider_controller.plugin_unique_identifier
 
         # get credentials schema
-        schema = {x.to_basic_provider_config().name: x for x in provider_controller.get_credentials_schema()}
+        schema = {
+            x.to_basic_provider_config().name: x
+            for x in provider_controller.get_credentials_schema_by_type(
+                CredentialType.of(db_provider.credential_type) if db_provider else CredentialType.API_KEY
+            )
+        }
 
         for name, value in schema.items():
             if result.masked_credentials:
@@ -149,15 +156,23 @@ class ToolTransformService:
                 credentials = db_provider.credentials
 
                 # init tool configuration
-                tool_configuration = ProviderConfigEncrypter(
+                encrypter, _ = create_provider_encrypter(
                     tenant_id=db_provider.tenant_id,
-                    config=[x.to_basic_provider_config() for x in provider_controller.get_credentials_schema()],
-                    provider_type=provider_controller.provider_type.value,
-                    provider_identity=provider_controller.entity.identity.name,
+                    config=[
+                        x.to_basic_provider_config()
+                        for x in provider_controller.get_credentials_schema_by_type(
+                            CredentialType.of(db_provider.credential_type)
+                        )
+                    ],
+                    cache=ToolProviderCredentialsCache(
+                        tenant_id=db_provider.tenant_id,
+                        provider=db_provider.provider,
+                        credential_id=db_provider.id,
+                    ),
                 )
                 # decrypt the credentials and mask the credentials
-                decrypted_credentials = tool_configuration.decrypt(data=credentials)
-                masked_credentials = tool_configuration.mask_tool_credentials(data=decrypted_credentials)
+                decrypted_credentials = encrypter.decrypt(data=credentials)
+                masked_credentials = encrypter.mask_tool_credentials(data=decrypted_credentials)
 
                 result.masked_credentials = masked_credentials
                 result.original_credentials = decrypted_credentials
@@ -300,16 +315,14 @@ class ToolTransformService:
 
         if decrypt_credentials:
             # init tool configuration
-            tool_configuration = ProviderConfigEncrypter(
+            encrypter, _ = create_tool_provider_encrypter(
                 tenant_id=db_provider.tenant_id,
-                config=[x.to_basic_provider_config() for x in provider_controller.get_credentials_schema()],
-                provider_type=provider_controller.provider_type.value,
-                provider_identity=provider_controller.entity.identity.name,
+                controller=provider_controller,
             )
 
             # decrypt the credentials and mask the credentials
-            decrypted_credentials = tool_configuration.decrypt(data=credentials)
-            masked_credentials = tool_configuration.mask_tool_credentials(data=decrypted_credentials)
+            decrypted_credentials = encrypter.decrypt(data=credentials)
+            masked_credentials = encrypter.mask_tool_credentials(data=decrypted_credentials)
 
             result.masked_credentials = masked_credentials
 
@@ -319,7 +332,6 @@ class ToolTransformService:
     def convert_tool_entity_to_api_entity(
         tool: Union[ApiToolBundle, WorkflowTool, Tool],
         tenant_id: str,
-        credentials: dict | None = None,
         labels: list[str] | None = None,
     ) -> ToolApiEntity:
         """
@@ -329,7 +341,7 @@ class ToolTransformService:
             # fork tool runtime
             tool = tool.fork_tool_runtime(
                 runtime=ToolRuntime(
-                    credentials=credentials or {},
+                    credentials={},
                     tenant_id=tenant_id,
                 )
             )
@@ -370,6 +382,19 @@ class ToolTransformService:
                 labels=labels or [],
             )
 
+    @staticmethod
+    def convert_builtin_provider_to_credential_entity(
+        provider: BuiltinToolProvider, credentials: dict
+    ) -> ToolProviderCredentialApiEntity:
+        return ToolProviderCredentialApiEntity(
+            id=provider.id,
+            name=provider.name,
+            provider=provider.provider,
+            credential_type=CredentialType.of(provider.credential_type),
+            is_default=provider.is_default,
+            credentials=credentials,
+        )
+
     @staticmethod
     def convert_mcp_schema_to_parameter(schema: dict) -> list["ToolParameter"]:
         """