feat: bump version to 0.3.29 (#1462)

Co-authored-by: takatost <takatost@gmail.com>

README.md

@@ -37,7 +37,6 @@ https://github.com/langgenius/dify/assets/100913391/f6e658d5-31b3-4c16-a0af-9e19
 
 
 We provide the following free resources for registered Dify cloud users (sign up at [dify.ai](https://dify.ai)):
-* 600,000 free Claude model tokens to build Claude-powered apps
 * 200 free OpenAI queries to build OpenAI-based apps
 
   

README_CN.md

@@ -36,7 +36,6 @@ https://github.com/langgenius/dify/assets/100913391/f6e658d5-31b3-4c16-a0af-9e19
 
 
 我们为所有注册云端版的用户免费提供以下资源（登录 [dify.ai](https://cloud.dify.ai) 即可使用）：
-* 60 万 Tokens Claude 模型的消息调用额度，用于创建基于 Claude 模型的 AI 应用
 * 200 次 OpenAI 模型的消息调用额度，用于创建基于 OpenAI 模型的 AI 应用
 * 300 万 讯飞星火大模型 Token 的调用额度，用于创建基于讯飞星火大模型的 AI 应用
 * 100 万 MiniMax Token 的调用额度，用于创建基于 MiniMax 模型的 AI 应用

api/.vscode/launch.json

@@ -10,7 +10,7 @@
             "request": "launch",
             "module": "flask",
             "env": {
-                "FLASK_APP": "api/app.py",
+                "FLASK_APP": "app.py",
                 "FLASK_DEBUG": "1",
                 "GEVENT_SUPPORT": "True"
             },

api/app.py

@@ -19,7 +19,7 @@ from flask_cors import CORS
 
 from core.model_providers.providers import hosted
 from extensions import ext_celery, ext_sentry, ext_redis, ext_login, ext_migrate, \
-    ext_database, ext_storage, ext_mail, ext_stripe
+    ext_database, ext_storage, ext_mail, ext_stripe, ext_code_based_extension
 from extensions.ext_database import db
 from extensions.ext_login import login_manager
 
@@ -79,6 +79,7 @@ def create_app(test_config=None) -> Flask:
 def initialize_extensions(app):
     # Since the application instance is now created, pass it to each Flask
     # extension instance to bind it to the Flask application instance (app)
+    ext_code_based_extension.init()
     ext_database.init_app(app)
     ext_migrate.init(app, db)
     ext_redis.init_app(app)

api/config.py

@@ -57,6 +57,7 @@ DEFAULTS = {
     'CLEAN_DAY_SETTING': 30,
     'UPLOAD_FILE_SIZE_LIMIT': 15,
     'UPLOAD_FILE_BATCH_LIMIT': 5,
+    'OUTPUT_MODERATION_BUFFER_SIZE': 300
 }
 
 
@@ -92,7 +93,7 @@ class Config:
         self.CONSOLE_URL = get_env('CONSOLE_URL')
         self.API_URL = get_env('API_URL')
         self.APP_URL = get_env('APP_URL')
-        self.CURRENT_VERSION = "0.3.27"
+        self.CURRENT_VERSION = "0.3.29"
         self.COMMIT_SHA = get_env('COMMIT_SHA')
         self.EDITION = "SELF_HOSTED"
         self.DEPLOY_ENV = get_env('DEPLOY_ENV')
@@ -228,6 +229,9 @@ class Config:
         self.UPLOAD_FILE_SIZE_LIMIT = int(get_env('UPLOAD_FILE_SIZE_LIMIT'))
         self.UPLOAD_FILE_BATCH_LIMIT = int(get_env('UPLOAD_FILE_BATCH_LIMIT'))
 
+        # moderation settings
+        self.OUTPUT_MODERATION_BUFFER_SIZE = int(get_env('OUTPUT_MODERATION_BUFFER_SIZE'))
+
 
 class CloudEditionConfig(Config):
 

api/controllers/console/__init__.py

@@ -6,7 +6,7 @@ bp = Blueprint('console', __name__, url_prefix='/console/api')
 api = ExternalApi(bp)
 
 # Import other controllers
-from . import setup, version, apikey, admin
+from . import extension, setup, version, apikey, admin
 
 # Import app controllers
 from .app import advanced_prompt_template, app, site, completion, model_config, statistic, conversation, message, generator, audio

api/controllers/console/app/advanced_prompt_template.py

@@ -20,7 +20,6 @@ class AdvancedPromptTemplateList(Resource):
         parser.add_argument('model_name', type=str, required=True, location='args')
         args = parser.parse_args()
 
-        service = AdvancedPromptTemplateService()
-        return service.get_prompt(args)
+        return AdvancedPromptTemplateService.get_prompt(args)
 
 api.add_resource(AdvancedPromptTemplateList, '/app/prompt-templates')

api/controllers/console/app/message.py

@@ -295,8 +295,8 @@ class MessageSuggestedQuestionApi(Resource):
         try:
             questions = MessageService.get_suggested_questions_after_answer(
                 app_model=app_model,
-                user=current_user,
                 message_id=message_id,
+                user=current_user,
                 check_enabled=False
             )
         except MessageNotExistsError:

api/controllers/console/explore/parameter.py

@@ -27,6 +27,7 @@ class AppParameterApi(InstalledAppResource):
         'retriever_resource': fields.Raw,
         'more_like_this': fields.Raw,
         'user_input_form': fields.Raw,
+        'sensitive_word_avoidance': fields.Raw
     }
 
     @marshal_with(parameters_fields)
@@ -42,7 +43,8 @@ class AppParameterApi(InstalledAppResource):
             'speech_to_text': app_model_config.speech_to_text_dict,
             'retriever_resource': app_model_config.retriever_resource_dict,
             'more_like_this': app_model_config.more_like_this_dict,
-            'user_input_form': app_model_config.user_input_form_list
+            'user_input_form': app_model_config.user_input_form_list,
+            'sensitive_word_avoidance': app_model_config.sensitive_word_avoidance_dict
         }
 
 

api/controllers/console/extension.py

@@ -0,0 +1,114 @@
+from flask_restful import Resource, reqparse, marshal_with
+from flask_login import current_user
+
+from controllers.console import api
+from controllers.console.setup import setup_required
+from controllers.console.wraps import account_initialization_required
+from libs.login import login_required
+from models.api_based_extension import APIBasedExtension
+from fields.api_based_extension_fields import api_based_extension_fields
+from services.code_based_extension_service import CodeBasedExtensionService
+from services.api_based_extension_service import APIBasedExtensionService
+
+
+class CodeBasedExtensionAPI(Resource):
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def get(self):
+        parser = reqparse.RequestParser()
+        parser.add_argument('module', type=str, required=True, location='args')
+        args = parser.parse_args()
+
+        return {
+            'module': args['module'],
+            'data': CodeBasedExtensionService.get_code_based_extension(args['module'])
+        }
+
+
+class APIBasedExtensionAPI(Resource):
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @marshal_with(api_based_extension_fields)
+    def get(self):
+        tenant_id = current_user.current_tenant_id
+        return APIBasedExtensionService.get_all_by_tenant_id(tenant_id)
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @marshal_with(api_based_extension_fields)
+    def post(self):
+        parser = reqparse.RequestParser()
+        parser.add_argument('name', type=str, required=True, location='json')
+        parser.add_argument('api_endpoint', type=str, required=True, location='json')
+        parser.add_argument('api_key', type=str, required=True, location='json')
+        args = parser.parse_args()
+
+        extension_data = APIBasedExtension(
+            tenant_id=current_user.current_tenant_id,
+            name=args['name'],
+            api_endpoint=args['api_endpoint'],
+            api_key=args['api_key']
+        )
+
+        return APIBasedExtensionService.save(extension_data)
+
+
+class APIBasedExtensionDetailAPI(Resource):
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @marshal_with(api_based_extension_fields)
+    def get(self, id):
+        api_based_extension_id = str(id)
+        tenant_id = current_user.current_tenant_id
+
+        return APIBasedExtensionService.get_with_tenant_id(tenant_id, api_based_extension_id)
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @marshal_with(api_based_extension_fields)
+    def post(self, id):
+        api_based_extension_id = str(id)
+        tenant_id = current_user.current_tenant_id
+
+        extension_data_from_db = APIBasedExtensionService.get_with_tenant_id(tenant_id, api_based_extension_id)
+
+        parser = reqparse.RequestParser()
+        parser.add_argument('name', type=str, required=True, location='json')
+        parser.add_argument('api_endpoint', type=str, required=True, location='json')
+        parser.add_argument('api_key', type=str, required=True, location='json')
+        args = parser.parse_args()
+
+        extension_data_from_db.name = args['name']
+        extension_data_from_db.api_endpoint = args['api_endpoint']
+
+        if args['api_key'] != '[__HIDDEN__]':
+            extension_data_from_db.api_key = args['api_key']
+
+        return APIBasedExtensionService.save(extension_data_from_db)
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def delete(self, id):
+        api_based_extension_id = str(id)
+        tenant_id = current_user.current_tenant_id
+
+        extension_data_from_db = APIBasedExtensionService.get_with_tenant_id(tenant_id, api_based_extension_id)
+
+        APIBasedExtensionService.delete(extension_data_from_db)
+
+        return {'result': 'success'}
+
+
+api.add_resource(CodeBasedExtensionAPI, '/code-based-extension')
+
+api.add_resource(APIBasedExtensionAPI, '/api-based-extension')
+api.add_resource(APIBasedExtensionDetailAPI, '/api-based-extension/<uuid:id>')

api/controllers/service_api/app/app.py

@@ -28,6 +28,7 @@ class AppParameterApi(AppApiResource):
         'retriever_resource': fields.Raw,
         'more_like_this': fields.Raw,
         'user_input_form': fields.Raw,
+        'sensitive_word_avoidance': fields.Raw
     }
 
     @marshal_with(parameters_fields)
@@ -42,7 +43,8 @@ class AppParameterApi(AppApiResource):
             'speech_to_text': app_model_config.speech_to_text_dict,
             'retriever_resource': app_model_config.retriever_resource_dict,
             'more_like_this': app_model_config.more_like_this_dict,
-            'user_input_form': app_model_config.user_input_form_list
+            'user_input_form': app_model_config.user_input_form_list,
+            'sensitive_word_avoidance': app_model_config.sensitive_word_avoidance_dict
         }
 
 

api/controllers/service_api/app/completion.py

@@ -183,4 +183,3 @@ api.add_resource(CompletionApi, '/completion-messages')
 api.add_resource(CompletionStopApi, '/completion-messages/<string:task_id>/stop')
 api.add_resource(ChatApi, '/chat-messages')
 api.add_resource(ChatStopApi, '/chat-messages/<string:task_id>/stop')
-

api/controllers/service_api/app/conversation.py

@@ -54,6 +54,7 @@ class ConversationDetailApi(AppApiResource):
             raise NotFound("Conversation Not Exists.")
         return {"result": "success"}, 204
 
+
 class ConversationRenameApi(AppApiResource):
 
     @marshal_with(simple_conversation_fields)

api/controllers/service_api/app/message.py

@@ -10,6 +10,8 @@ from controllers.service_api.app.error import NotChatAppError
 from controllers.service_api.wraps import AppApiResource
 from libs.helper import TimestampField, uuid_value
 from services.message_service import MessageService
+from extensions.ext_database import db
+from models.model import Message, EndUser
 
 
 class MessageListApi(AppApiResource):
@@ -96,5 +98,38 @@ class MessageFeedbackApi(AppApiResource):
         return {'result': 'success'}
 
 
+class MessageSuggestedApi(AppApiResource):
+    def get(self, app_model, end_user, message_id):
+        message_id = str(message_id)
+        if app_model.mode != 'chat':
+            raise NotChatAppError()
+        try:
+            message = db.session.query(Message).filter(
+                Message.id == message_id,
+                Message.app_id == app_model.id,
+            ).first()
+
+            if end_user is None and message.from_end_user_id is not None:
+                user = db.session.query(EndUser) \
+                    .filter(
+                        EndUser.tenant_id == app_model.tenant_id,
+                        EndUser.id == message.from_end_user_id,
+                        EndUser.type == 'service_api'
+                    ).first()
+            else:
+                user = end_user
+            questions = MessageService.get_suggested_questions_after_answer(
+                app_model=app_model,
+                user=user,
+                message_id=message_id,
+                check_enabled=False
+            )
+        except services.errors.message.MessageNotExistsError:
+            raise NotFound("Message Not Exists.")
+
+        return {'result': 'success', 'data': questions}
+
+
 api.add_resource(MessageListApi, '/messages')
 api.add_resource(MessageFeedbackApi, '/messages/<uuid:message_id>/feedbacks')
+api.add_resource(MessageSuggestedApi, '/messages/<uuid:message_id>/suggested')

api/controllers/web/app.py

@@ -27,6 +27,7 @@ class AppParameterApi(WebApiResource):
         'retriever_resource': fields.Raw,
         'more_like_this': fields.Raw,
         'user_input_form': fields.Raw,
+        'sensitive_word_avoidance': fields.Raw
     }
 
     @marshal_with(parameters_fields)
@@ -41,7 +42,8 @@ class AppParameterApi(WebApiResource):
             'speech_to_text': app_model_config.speech_to_text_dict,
             'retriever_resource': app_model_config.retriever_resource_dict,
             'more_like_this': app_model_config.more_like_this_dict,
-            'user_input_form': app_model_config.user_input_form_list
+            'user_input_form': app_model_config.user_input_form_list,
+            'sensitive_word_avoidance': app_model_config.sensitive_word_avoidance_dict
         }
 
 

api/controllers/web/completion.py

@@ -139,7 +139,7 @@ class ChatStopApi(WebApiResource):
         return {'result': 'success'}, 200
 
 
-def compact_response(response: Union[dict | Generator]) -> Response:
+def compact_response(response: Union[dict, Generator]) -> Response:
     if isinstance(response, dict):
         return Response(response=json.dumps(response), status=200, mimetype='application/json')
     else:

api/core/__init__.py

@@ -0,0 +1 @@
+import core.moderation.base

api/core/agent/agent/multi_dataset_router_agent.py

@@ -76,7 +76,7 @@ class MultiDatasetRouterAgent(OpenAIFunctionsAgent):
             agent_decision = self.real_plan(intermediate_steps, callbacks, **kwargs)
             if isinstance(agent_decision, AgentAction):
                 tool_inputs = agent_decision.tool_input
-                if isinstance(tool_inputs, dict) and 'query' in tool_inputs:
+                if isinstance(tool_inputs, dict) and 'query' in tool_inputs and 'chat_history' not in kwargs:
                     tool_inputs['query'] = kwargs['input']
                     agent_decision.tool_input = tool_inputs
             else:

api/core/agent/agent/structed_multi_dataset_router_agent.py

@@ -1,7 +1,7 @@
 import re
 from typing import List, Tuple, Any, Union, Sequence, Optional, cast
 
-from langchain import BasePromptTemplate
+from langchain import BasePromptTemplate, PromptTemplate
 from langchain.agents import StructuredChatAgent, AgentOutputParser, Agent
 from langchain.agents.structured_chat.base import HUMAN_MESSAGE_TEMPLATE
 from langchain.callbacks.base import BaseCallbackManager
@@ -12,6 +12,7 @@ from langchain.tools import BaseTool
 from langchain.agents.structured_chat.prompt import PREFIX, SUFFIX
 
 from core.chain.llm_chain import LLMChain
+from core.model_providers.models.entity.model_params import ModelMode
 from core.model_providers.models.llm.base import BaseLLM
 from core.tool.dataset_retriever_tool import DatasetRetrieverTool
 
@@ -92,6 +93,10 @@ class StructuredMultiDatasetRouterAgent(StructuredChatAgent):
             rst = tool.run(tool_input={'query': kwargs['input']})
             return AgentFinish(return_values={"output": rst}, log=rst)
 
+        if intermediate_steps:
+            _, observation = intermediate_steps[-1]
+            return AgentFinish(return_values={"output": observation}, log=observation)
+
         full_inputs = self.get_full_inputs(intermediate_steps, **kwargs)
 
         try:
@@ -107,6 +112,8 @@ class StructuredMultiDatasetRouterAgent(StructuredChatAgent):
                 if isinstance(tool_inputs, dict) and 'query' in tool_inputs:
                     tool_inputs['query'] = kwargs['input']
                     agent_decision.tool_input = tool_inputs
+                elif isinstance(tool_inputs, str):
+                    agent_decision.tool_input = kwargs['input']
             else:
                 agent_decision.return_values['output'] = ''
             return agent_decision
@@ -143,6 +150,61 @@ class StructuredMultiDatasetRouterAgent(StructuredChatAgent):
         ]
         return ChatPromptTemplate(input_variables=input_variables, messages=messages)
 
+    @classmethod
+    def create_completion_prompt(
+            cls,
+            tools: Sequence[BaseTool],
+            prefix: str = PREFIX,
+            format_instructions: str = FORMAT_INSTRUCTIONS,
+            input_variables: Optional[List[str]] = None,
+    ) -> PromptTemplate:
+        """Create prompt in the style of the zero shot agent.
+
+        Args:
+            tools: List of tools the agent will have access to, used to format the
+                prompt.
+            prefix: String to put before the list of tools.
+            input_variables: List of input variables the final prompt will expect.
+
+        Returns:
+            A PromptTemplate with the template assembled from the pieces here.
+        """
+        suffix = """Begin! Reminder to ALWAYS respond with a valid json blob of a single action. Use tools if necessary. Respond directly if appropriate. Format is Action:```$JSON_BLOB```then Observation:.
+Question: {input}
+Thought: {agent_scratchpad}
+"""
+
+        tool_strings = "\n".join([f"{tool.name}: {tool.description}" for tool in tools])
+        tool_names = ", ".join([tool.name for tool in tools])
+        format_instructions = format_instructions.format(tool_names=tool_names)
+        template = "\n\n".join([prefix, tool_strings, format_instructions, suffix])
+        if input_variables is None:
+            input_variables = ["input", "agent_scratchpad"]
+        return PromptTemplate(template=template, input_variables=input_variables)
+
+    def _construct_scratchpad(
+        self, intermediate_steps: List[Tuple[AgentAction, str]]
+    ) -> str:
+        agent_scratchpad = ""
+        for action, observation in intermediate_steps:
+            agent_scratchpad += action.log
+            agent_scratchpad += f"\n{self.observation_prefix}{observation}\n{self.llm_prefix}"
+
+        if not isinstance(agent_scratchpad, str):
+            raise ValueError("agent_scratchpad should be of type string.")
+        if agent_scratchpad:
+            llm_chain = cast(LLMChain, self.llm_chain)
+            if llm_chain.model_instance.model_mode == ModelMode.CHAT:
+                return (
+                    f"This was your previous work "
+                    f"(but I haven't seen any of it! I only see what "
+                    f"you return as final answer):\n{agent_scratchpad}"
+                )
+            else:
+                return agent_scratchpad
+        else:
+            return agent_scratchpad
+
     @classmethod
     def from_llm_and_tools(
             cls,
@@ -160,15 +222,23 @@ class StructuredMultiDatasetRouterAgent(StructuredChatAgent):
     ) -> Agent:
         """Construct an agent from an LLM and tools."""
         cls._validate_tools(tools)
-        prompt = cls.create_prompt(
-            tools,
-            prefix=prefix,
-            suffix=suffix,
-            human_message_template=human_message_template,
-            format_instructions=format_instructions,
-            input_variables=input_variables,
-            memory_prompts=memory_prompts,
-        )
+        if model_instance.model_mode == ModelMode.CHAT:
+            prompt = cls.create_prompt(
+                tools,
+                prefix=prefix,
+                suffix=suffix,
+                human_message_template=human_message_template,
+                format_instructions=format_instructions,
+                input_variables=input_variables,
+                memory_prompts=memory_prompts,
+            )
+        else:
+            prompt = cls.create_completion_prompt(
+                tools,
+                prefix=prefix,
+                format_instructions=format_instructions,
+                input_variables=input_variables
+            )
         llm_chain = LLMChain(
             model_instance=model_instance,
             prompt=prompt,

api/core/agent/agent/structured_chat.py

@@ -1,7 +1,7 @@
 import re
-from typing import List, Tuple, Any, Union, Sequence, Optional
+from typing import List, Tuple, Any, Union, Sequence, Optional, cast
 
-from langchain import BasePromptTemplate
+from langchain import BasePromptTemplate, PromptTemplate
 from langchain.agents import StructuredChatAgent, AgentOutputParser, Agent
 from langchain.agents.structured_chat.base import HUMAN_MESSAGE_TEMPLATE
 from langchain.callbacks.base import BaseCallbackManager
@@ -15,6 +15,7 @@ from langchain.agents.structured_chat.prompt import PREFIX, SUFFIX
 
 from core.agent.agent.calc_token_mixin import CalcTokenMixin, ExceededLLMTokensLimitError
 from core.chain.llm_chain import LLMChain
+from core.model_providers.models.entity.model_params import ModelMode
 from core.model_providers.models.llm.base import BaseLLM
 
 FORMAT_INSTRUCTIONS = """Use a json blob to specify a tool by providing an action key (tool name) and an action_input key (tool input).
@@ -184,6 +185,61 @@ class AutoSummarizingStructuredChatAgent(StructuredChatAgent, CalcTokenMixin):
         ]
         return ChatPromptTemplate(input_variables=input_variables, messages=messages)
 
+    @classmethod
+    def create_completion_prompt(
+            cls,
+            tools: Sequence[BaseTool],
+            prefix: str = PREFIX,
+            format_instructions: str = FORMAT_INSTRUCTIONS,
+            input_variables: Optional[List[str]] = None,
+    ) -> PromptTemplate:
+        """Create prompt in the style of the zero shot agent.
+
+        Args:
+            tools: List of tools the agent will have access to, used to format the
+                prompt.
+            prefix: String to put before the list of tools.
+            input_variables: List of input variables the final prompt will expect.
+
+        Returns:
+            A PromptTemplate with the template assembled from the pieces here.
+        """
+        suffix = """Begin! Reminder to ALWAYS respond with a valid json blob of a single action. Use tools if necessary. Respond directly if appropriate. Format is Action:```$JSON_BLOB```then Observation:.
+Question: {input}
+Thought: {agent_scratchpad}
+"""
+
+        tool_strings = "\n".join([f"{tool.name}: {tool.description}" for tool in tools])
+        tool_names = ", ".join([tool.name for tool in tools])
+        format_instructions = format_instructions.format(tool_names=tool_names)
+        template = "\n\n".join([prefix, tool_strings, format_instructions, suffix])
+        if input_variables is None:
+            input_variables = ["input", "agent_scratchpad"]
+        return PromptTemplate(template=template, input_variables=input_variables)
+
+    def _construct_scratchpad(
+        self, intermediate_steps: List[Tuple[AgentAction, str]]
+    ) -> str:
+        agent_scratchpad = ""
+        for action, observation in intermediate_steps:
+            agent_scratchpad += action.log
+            agent_scratchpad += f"\n{self.observation_prefix}{observation}\n{self.llm_prefix}"
+
+        if not isinstance(agent_scratchpad, str):
+            raise ValueError("agent_scratchpad should be of type string.")
+        if agent_scratchpad:
+            llm_chain = cast(LLMChain, self.llm_chain)
+            if llm_chain.model_instance.model_mode == ModelMode.CHAT:
+                return (
+                    f"This was your previous work "
+                    f"(but I haven't seen any of it! I only see what "
+                    f"you return as final answer):\n{agent_scratchpad}"
+                )
+            else:
+                return agent_scratchpad
+        else:
+            return agent_scratchpad
+
     @classmethod
     def from_llm_and_tools(
             cls,
@@ -201,15 +257,23 @@ class AutoSummarizingStructuredChatAgent(StructuredChatAgent, CalcTokenMixin):
     ) -> Agent:
         """Construct an agent from an LLM and tools."""
         cls._validate_tools(tools)
-        prompt = cls.create_prompt(
-            tools,
-            prefix=prefix,
-            suffix=suffix,
-            human_message_template=human_message_template,
-            format_instructions=format_instructions,
-            input_variables=input_variables,
-            memory_prompts=memory_prompts,
-        )
+        if model_instance.model_mode == ModelMode.CHAT:
+            prompt = cls.create_prompt(
+                tools,
+                prefix=prefix,
+                suffix=suffix,
+                human_message_template=human_message_template,
+                format_instructions=format_instructions,
+                input_variables=input_variables,
+                memory_prompts=memory_prompts,
+            )
+        else:
+            prompt = cls.create_completion_prompt(
+                tools,
+                prefix=prefix,
+                format_instructions=format_instructions,
+                input_variables=input_variables,
+            )
         llm_chain = LLMChain(
             model_instance=model_instance,
             prompt=prompt,

api/core/callback_handler/llm_callback_handler.py

@@ -1,13 +1,25 @@
 import logging
-from typing import Any, Dict, List, Union
+import threading
+import time
+from typing import Any, Dict, List, Union, Optional
 
+from flask import Flask, current_app
 from langchain.callbacks.base import BaseCallbackHandler
 from langchain.schema import LLMResult, BaseMessage
+from pydantic import BaseModel
 
 from core.callback_handler.entity.llm_message import LLMMessage
-from core.conversation_message_task import ConversationMessageTask, ConversationTaskStoppedException
+from core.conversation_message_task import ConversationMessageTask, ConversationTaskStoppedException, \
+    ConversationTaskInterruptException
 from core.model_providers.models.entity.message import to_prompt_messages, PromptMessage
 from core.model_providers.models.llm.base import BaseLLM
+from core.moderation.base import ModerationOutputsResult, ModerationAction
+from core.moderation.factory import ModerationFactory
+
+
+class ModerationRule(BaseModel):
+    type: str
+    config: Dict[str, Any]
 
 
 class LLMCallbackHandler(BaseCallbackHandler):
@@ -20,6 +32,24 @@ class LLMCallbackHandler(BaseCallbackHandler):
         self.start_at = None
         self.conversation_message_task = conversation_message_task
 
+        self.output_moderation_handler = None
+        self.init_output_moderation()
+
+    def init_output_moderation(self):
+        app_model_config = self.conversation_message_task.app_model_config
+        sensitive_word_avoidance_dict = app_model_config.sensitive_word_avoidance_dict
+
+        if sensitive_word_avoidance_dict and sensitive_word_avoidance_dict.get("enabled"):
+            self.output_moderation_handler = OutputModerationHandler(
+                tenant_id=self.conversation_message_task.tenant_id,
+                app_id=self.conversation_message_task.app.id,
+                rule=ModerationRule(
+                    type=sensitive_word_avoidance_dict.get("type"),
+                    config=sensitive_word_avoidance_dict.get("config")
+                ),
+                on_message_replace_func=self.conversation_message_task.on_message_replace
+            )
+
     @property
     def always_verbose(self) -> bool:
         """Whether to call verbose callbacks even if verbose is False."""
@@ -59,10 +89,19 @@ class LLMCallbackHandler(BaseCallbackHandler):
         self.llm_message.prompt_tokens = self.model_instance.get_num_tokens([PromptMessage(content=prompts[0])])
 
     def on_llm_end(self, response: LLMResult, **kwargs: Any) -> None:
-        if not self.conversation_message_task.streaming:
-            self.conversation_message_task.append_message_text(response.generations[0][0].text)
+        if self.output_moderation_handler:
+            self.output_moderation_handler.stop_thread()
+
+            self.llm_message.completion = self.output_moderation_handler.moderation_completion(
+                completion=response.generations[0][0].text,
+                public_event=True if self.conversation_message_task.streaming else False
+            )
+        else:
             self.llm_message.completion = response.generations[0][0].text
 
+        if not self.conversation_message_task.streaming:
+            self.conversation_message_task.append_message_text(self.llm_message.completion)
+
         if response.llm_output and 'token_usage' in response.llm_output:
             if 'prompt_tokens' in response.llm_output['token_usage']:
                 self.llm_message.prompt_tokens = response.llm_output['token_usage']['prompt_tokens']
@@ -79,23 +118,161 @@ class LLMCallbackHandler(BaseCallbackHandler):
         self.conversation_message_task.save_message(self.llm_message)
 
     def on_llm_new_token(self, token: str, **kwargs: Any) -> None:
-        try:
-            self.conversation_message_task.append_message_text(token)
-        except ConversationTaskStoppedException as ex:
+        if self.output_moderation_handler and self.output_moderation_handler.should_direct_output():
+            # stop subscribe new token when output moderation should direct output
+            ex = ConversationTaskInterruptException()
             self.on_llm_error(error=ex)
             raise ex
 
-        self.llm_message.completion += token
+        try:
+            self.conversation_message_task.append_message_text(token)
+            self.llm_message.completion += token
+
+            if self.output_moderation_handler:
+                self.output_moderation_handler.append_new_token(token)
+        except ConversationTaskStoppedException as ex:
+            self.on_llm_error(error=ex)
+            raise ex
 
     def on_llm_error(
             self, error: Union[Exception, KeyboardInterrupt], **kwargs: Any
     ) -> None:
         """Do nothing."""
+        if self.output_moderation_handler:
+            self.output_moderation_handler.stop_thread()
+
         if isinstance(error, ConversationTaskStoppedException):
             if self.conversation_message_task.streaming:
                 self.llm_message.completion_tokens = self.model_instance.get_num_tokens(
                     [PromptMessage(content=self.llm_message.completion)]
                 )
                 self.conversation_message_task.save_message(llm_message=self.llm_message, by_stopped=True)
+        if isinstance(error, ConversationTaskInterruptException):
+            self.llm_message.completion = self.output_moderation_handler.get_final_output()
+            self.llm_message.completion_tokens = self.model_instance.get_num_tokens(
+                [PromptMessage(content=self.llm_message.completion)]
+            )
+            self.conversation_message_task.save_message(llm_message=self.llm_message)
         else:
             logging.debug("on_llm_error: %s", error)
+
+
+class OutputModerationHandler(BaseModel):
+    DEFAULT_BUFFER_SIZE: int = 300
+
+    tenant_id: str
+    app_id: str
+
+    rule: ModerationRule
+    on_message_replace_func: Any
+
+    thread: Optional[threading.Thread] = None
+    thread_running: bool = True
+    buffer: str = ''
+    is_final_chunk: bool = False
+    final_output: Optional[str] = None
+
+    class Config:
+        arbitrary_types_allowed = True
+
+    def should_direct_output(self):
+        return self.final_output is not None
+
+    def get_final_output(self):
+        return self.final_output
+
+    def append_new_token(self, token: str):
+        self.buffer += token
+
+        if not self.thread:
+            self.thread = self.start_thread()
+
+    def moderation_completion(self, completion: str, public_event: bool = False) -> str:
+        self.buffer = completion
+        self.is_final_chunk = True
+
+        result = self.moderation(
+            tenant_id=self.tenant_id,
+            app_id=self.app_id,
+            moderation_buffer=completion
+        )
+
+        if not result or not result.flagged:
+            return completion
+
+        if result.action == ModerationAction.DIRECT_OUTPUT:
+            final_output = result.preset_response
+        else:
+            final_output = result.text
+
+        if public_event:
+            self.on_message_replace_func(final_output)
+
+        return final_output
+
+    def start_thread(self) -> threading.Thread:
+        buffer_size = int(current_app.config.get('MODERATION_BUFFER_SIZE', self.DEFAULT_BUFFER_SIZE))
+        thread = threading.Thread(target=self.worker, kwargs={
+            'flask_app': current_app._get_current_object(),
+            'buffer_size': buffer_size if buffer_size > 0 else self.DEFAULT_BUFFER_SIZE
+        })
+
+        thread.start()
+
+        return thread
+
+    def stop_thread(self):
+        if self.thread and self.thread.is_alive():
+            self.thread_running = False
+
+    def worker(self, flask_app: Flask, buffer_size: int):
+        with flask_app.app_context():
+            current_length = 0
+            while self.thread_running:
+                moderation_buffer = self.buffer
+                buffer_length = len(moderation_buffer)
+                if not self.is_final_chunk:
+                    chunk_length = buffer_length - current_length
+                    if 0 <= chunk_length < buffer_size:
+                        time.sleep(1)
+                        continue
+
+                current_length = buffer_length
+
+                result = self.moderation(
+                    tenant_id=self.tenant_id,
+                    app_id=self.app_id,
+                    moderation_buffer=moderation_buffer
+                )
+
+                if not result or not result.flagged:
+                    continue
+
+                if result.action == ModerationAction.DIRECT_OUTPUT:
+                    final_output = result.preset_response
+                    self.final_output = final_output
+                else:
+                    final_output = result.text + self.buffer[len(moderation_buffer):]
+
+                # trigger replace event
+                if self.thread_running:
+                    self.on_message_replace_func(final_output)
+
+                if result.action == ModerationAction.DIRECT_OUTPUT:
+                    break
+
+    def moderation(self, tenant_id: str, app_id: str, moderation_buffer: str) -> Optional[ModerationOutputsResult]:
+        try:
+            moderation_factory = ModerationFactory(
+                name=self.rule.type,
+                app_id=app_id,
+                tenant_id=tenant_id,
+                config=self.rule.config
+            )
+
+            result: ModerationOutputsResult = moderation_factory.moderation_for_outputs(moderation_buffer)
+            return result
+        except Exception as e:
+            logging.error("Moderation Output error: %s", e)
+
+        return None

api/core/chain/sensitive_word_avoidance_chain.py

@@ -1,92 +0,0 @@
-import enum
-import logging
-from typing import List, Dict, Optional, Any
-
-from langchain.callbacks.manager import CallbackManagerForChainRun
-from langchain.chains.base import Chain
-from pydantic import BaseModel
-
-from core.model_providers.error import LLMBadRequestError
-from core.model_providers.model_factory import ModelFactory
-from core.model_providers.models.llm.base import BaseLLM
-from core.model_providers.models.moderation import openai_moderation
-
-
-class SensitiveWordAvoidanceRule(BaseModel):
-    class Type(enum.Enum):
-        MODERATION = "moderation"
-        KEYWORDS = "keywords"
-
-    type: Type
-    canned_response: str = 'Your content violates our usage policy. Please revise and try again.'
-    extra_params: dict = {}
-
-
-class SensitiveWordAvoidanceChain(Chain):
-    input_key: str = "input"  #: :meta private:
-    output_key: str = "output"  #: :meta private:
-
-    model_instance: BaseLLM
-    sensitive_word_avoidance_rule: SensitiveWordAvoidanceRule
-
-    @property
-    def _chain_type(self) -> str:
-        return "sensitive_word_avoidance_chain"
-
-    @property
-    def input_keys(self) -> List[str]:
-        """Expect input key.
-
-        :meta private:
-        """
-        return [self.input_key]
-
-    @property
-    def output_keys(self) -> List[str]:
-        """Return output key.
-
-        :meta private:
-        """
-        return [self.output_key]
-
-    def _check_sensitive_word(self, text: str) -> bool:
-        for word in self.sensitive_word_avoidance_rule.extra_params.get('sensitive_words', []):
-            if word in text:
-                return False
-        return True
-
-    def _check_moderation(self, text: str) -> bool:
-        moderation_model_instance = ModelFactory.get_moderation_model(
-            tenant_id=self.model_instance.model_provider.provider.tenant_id,
-            model_provider_name='openai',
-            model_name=openai_moderation.DEFAULT_MODEL
-        )
-
-        try:
-            return moderation_model_instance.run(text=text)
-        except Exception as ex:
-            logging.exception(ex)
-            raise LLMBadRequestError('Rate limit exceeded, please try again later.')
-
-    def _call(
-            self,
-            inputs: Dict[str, Any],
-            run_manager: Optional[CallbackManagerForChainRun] = None,
-    ) -> Dict[str, Any]:
-        text = inputs[self.input_key]
-
-        if self.sensitive_word_avoidance_rule.type == SensitiveWordAvoidanceRule.Type.KEYWORDS:
-            result = self._check_sensitive_word(text)
-        else:
-            result = self._check_moderation(text)
-
-        if not result:
-            raise SensitiveWordAvoidanceError(self.sensitive_word_avoidance_rule.canned_response)
-
-        return {self.output_key: text}
-
-
-class SensitiveWordAvoidanceError(Exception):
-    def __init__(self, message):
-        super().__init__(message)
-        self.message = message

api/core/completion.py

@@ -1,13 +1,18 @@
+import concurrent
+import json
 import logging
-from typing import Optional, List, Union
+from concurrent.futures import ThreadPoolExecutor
+from typing import Optional, List, Union, Tuple
 
+from flask import current_app, Flask
 from requests.exceptions import ChunkedEncodingError
 
 from core.agent.agent_executor import AgentExecuteResult, PlanningStrategy
 from core.callback_handler.main_chain_gather_callback_handler import MainChainGatherCallbackHandler
 from core.callback_handler.llm_callback_handler import LLMCallbackHandler
-from core.chain.sensitive_word_avoidance_chain import SensitiveWordAvoidanceError
-from core.conversation_message_task import ConversationMessageTask, ConversationTaskStoppedException
+from core.conversation_message_task import ConversationMessageTask, ConversationTaskStoppedException, \
+    ConversationTaskInterruptException
+from core.external_data_tool.factory import ExternalDataToolFactory
 from core.model_providers.error import LLMBadRequestError
 from core.memory.read_only_conversation_token_db_buffer_shared_memory import \
     ReadOnlyConversationTokenDBBufferSharedMemory
@@ -16,7 +21,10 @@ from core.model_providers.models.entity.message import PromptMessage
 from core.model_providers.models.llm.base import BaseLLM
 from core.orchestrator_rule_parser import OrchestratorRuleParser
 from core.prompt.prompt_template import PromptTemplateParser
+from core.prompt.prompt_transform import PromptTransform
 from models.model import App, AppModelConfig, Account, Conversation, EndUser
+from core.moderation.base import ModerationException, ModerationAction
+from core.moderation.factory import ModerationFactory
 
 
 class Completion:
@@ -75,26 +83,35 @@ class Completion:
         )
 
         try:
-            # parse sensitive_word_avoidance_chain
             chain_callback = MainChainGatherCallbackHandler(conversation_message_task)
-            sensitive_word_avoidance_chain = orchestrator_rule_parser.to_sensitive_word_avoidance_chain(
-                final_model_instance, [chain_callback])
-            if sensitive_word_avoidance_chain:
-                try:
-                    query = sensitive_word_avoidance_chain.run(query)
-                except SensitiveWordAvoidanceError as ex:
-                    cls.run_final_llm(
-                        model_instance=final_model_instance,
-                        mode=app.mode,
-                        app_model_config=app_model_config,
-                        query=query,
-                        inputs=inputs,
-                        agent_execute_result=None,
-                        conversation_message_task=conversation_message_task,
-                        memory=memory,
-                        fake_response=ex.message
-                    )
-                    return
+
+            try:
+                # process sensitive_word_avoidance
+                inputs, query = cls.moderation_for_inputs(app.id, app.tenant_id, app_model_config, inputs, query)
+            except ModerationException as e:
+                cls.run_final_llm(
+                    model_instance=final_model_instance,
+                    mode=app.mode,
+                    app_model_config=app_model_config,
+                    query=query,
+                    inputs=inputs,
+                    agent_execute_result=None,
+                    conversation_message_task=conversation_message_task,
+                    memory=memory,
+                    fake_response=str(e)
+                )
+                return
+
+            # fill in variable inputs from external data tools if exists
+            external_data_tools = app_model_config.external_data_tools_list
+            if external_data_tools:
+                inputs = cls.fill_in_inputs_from_external_data_tools(
+                    tenant_id=app.tenant_id,
+                    app_id=app.id,
+                    external_data_tools=external_data_tools,
+                    inputs=inputs,
+                    query=query
+                )
 
             # get agent executor
             agent_executor = orchestrator_rule_parser.to_agent_executor(
@@ -134,19 +151,110 @@ class Completion:
                 memory=memory,
                 fake_response=fake_response
             )
-        except ConversationTaskStoppedException:
+        except (ConversationTaskInterruptException, ConversationTaskStoppedException):
             return
         except ChunkedEncodingError as e:
             # Interrupt by LLM (like OpenAI), handle it.
             logging.warning(f'ChunkedEncodingError: {e}')
             conversation_message_task.end()
             return
-        
+
+    @classmethod
+    def moderation_for_inputs(cls, app_id: str, tenant_id: str, app_model_config: AppModelConfig, inputs: dict, query: str):
+        if not app_model_config.sensitive_word_avoidance_dict['enabled']:
+            return inputs, query
+
+        type = app_model_config.sensitive_word_avoidance_dict['type']
+
+        moderation = ModerationFactory(type, app_id, tenant_id, app_model_config.sensitive_word_avoidance_dict['config'])
+        moderation_result = moderation.moderation_for_inputs(inputs, query)
+
+        if not moderation_result.flagged:
+            return inputs, query
+
+        if moderation_result.action == ModerationAction.DIRECT_OUTPUT:
+            raise ModerationException(moderation_result.preset_response)
+        elif moderation_result.action == ModerationAction.OVERRIDED:
+            inputs = moderation_result.inputs
+            query = moderation_result.query
+
+        return inputs, query
+
+    @classmethod
+    def fill_in_inputs_from_external_data_tools(cls, tenant_id: str, app_id: str, external_data_tools: list[dict],
+                                                inputs: dict, query: str) -> dict:
+        """
+        Fill in variable inputs from external data tools if exists.
+
+        :param tenant_id: workspace id
+        :param app_id: app id
+        :param external_data_tools: external data tools configs
+        :param inputs: the inputs
+        :param query: the query
+        :return: the filled inputs
+        """
+        # Group tools by type and config
+        grouped_tools = {}
+        for tool in external_data_tools:
+            if not tool.get("enabled"):
+                continue
+
+            tool_key = (tool.get("type"), json.dumps(tool.get("config"), sort_keys=True))
+            grouped_tools.setdefault(tool_key, []).append(tool)
+
+        results = {}
+        with ThreadPoolExecutor() as executor:
+            futures = {}
+            for tools in grouped_tools.values():
+                # Only query the first tool in each group
+                first_tool = tools[0]
+                future = executor.submit(
+                    cls.query_external_data_tool, current_app._get_current_object(), tenant_id, app_id, first_tool,
+                    inputs, query
+                )
+                for tool in tools:
+                    futures[future] = tool
+
+            for future in concurrent.futures.as_completed(futures):
+                tool_key, result = future.result()
+                if tool_key in grouped_tools:
+                    for tool in grouped_tools[tool_key]:
+                        results[tool['variable']] = result
+
+        inputs.update(results)
+        return inputs
+
+    @classmethod
+    def query_external_data_tool(cls, flask_app: Flask, tenant_id: str, app_id: str, external_data_tool: dict,
+                                 inputs: dict, query: str) -> Tuple[Optional[str], Optional[str]]:
+        with flask_app.app_context():
+            tool_variable = external_data_tool.get("variable")
+            tool_type = external_data_tool.get("type")
+            tool_config = external_data_tool.get("config")
+
+            external_data_tool_factory = ExternalDataToolFactory(
+                name=tool_type,
+                tenant_id=tenant_id,
+                app_id=app_id,
+                variable=tool_variable,
+                config=tool_config
+            )
+
+            # query external data tool
+            result = external_data_tool_factory.query(
+                inputs=inputs,
+                query=query
+            )
+
+            tool_key = (external_data_tool.get("type"), json.dumps(external_data_tool.get("config"), sort_keys=True))
+
+            return tool_key, result
+
     @classmethod
     def get_query_for_agent(cls, app: App, app_model_config: AppModelConfig, query: str, inputs: dict) -> str:
         if app.mode != 'completion':
             return query
-        
+
         return inputs.get(app_model_config.dataset_query_variable, "")
 
     @classmethod
@@ -156,24 +264,28 @@ class Completion:
                       conversation_message_task: ConversationMessageTask,
                       memory: Optional[ReadOnlyConversationTokenDBBufferSharedMemory],
                       fake_response: Optional[str]):
+        prompt_transform = PromptTransform()
+
         # get llm prompt
         if app_model_config.prompt_type == 'simple':
-            prompt_messages, stop_words = model_instance.get_prompt(
+            prompt_messages, stop_words = prompt_transform.get_prompt(
                 mode=mode,
                 pre_prompt=app_model_config.pre_prompt,
                 inputs=inputs,
                 query=query,
                 context=agent_execute_result.output if agent_execute_result else None,
-                memory=memory
+                memory=memory,
+                model_instance=model_instance
             )
         else:
-            prompt_messages = model_instance.get_advanced_prompt(
+            prompt_messages = prompt_transform.get_advanced_prompt(
                 app_mode=mode,
                 app_model_config=app_model_config,
                 inputs=inputs,
                 query=query,
                 context=agent_execute_result.output if agent_execute_result else None,
-                memory=memory
+                memory=memory,
+                model_instance=model_instance
             )
 
             model_config = app_model_config.model_dict
@@ -238,15 +350,30 @@ class Completion:
         if max_tokens is None:
             max_tokens = 0
 
+        prompt_transform = PromptTransform()
+        prompt_messages = []
+
         # get prompt without memory and context
-        prompt_messages, _ = model_instance.get_prompt(
-            mode=mode,
-            pre_prompt=app_model_config.pre_prompt,
-            inputs=inputs,
-            query=query,
-            context=None,
-            memory=None
-        )
+        if app_model_config.prompt_type == 'simple':
+            prompt_messages, _ = prompt_transform.get_prompt(
+                mode=mode,
+                pre_prompt=app_model_config.pre_prompt,
+                inputs=inputs,
+                query=query,
+                context=None,
+                memory=None,
+                model_instance=model_instance
+            )
+        else:
+            prompt_messages = prompt_transform.get_advanced_prompt(
+                app_mode=mode,
+                app_model_config=app_model_config,
+                inputs=inputs,
+                query=query,
+                context=None,
+                memory=None,
+                model_instance=model_instance
+            )
 
         prompt_tokens = model_instance.get_num_tokens(prompt_messages)
         rest_tokens = model_limited_tokens - max_tokens - prompt_tokens

api/core/conversation_message_task.py

@@ -290,6 +290,10 @@ class ConversationMessageTask:
                 db.session.commit()
             self.retriever_resource = resource
 
+    def on_message_replace(self, text: str):
+        if text is not None:
+            self._pub_handler.pub_message_replace(text)
+
     def message_end(self):
         self._pub_handler.pub_message_end(self.retriever_resource)
 
@@ -342,6 +346,24 @@ class PubHandler:
             self.pub_end()
             raise ConversationTaskStoppedException()
 
+    def pub_message_replace(self, text: str):
+        content = {
+            'event': 'message_replace',
+            'data': {
+                'task_id': self._task_id,
+                'message_id': str(self._message.id),
+                'text': text,
+                'mode': self._conversation.mode,
+                'conversation_id': str(self._conversation.id)
+            }
+        }
+
+        redis_client.publish(self._channel, json.dumps(content))
+
+        if self._is_stopped():
+            self.pub_end()
+            raise ConversationTaskStoppedException()
+
     def pub_chain(self, message_chain: MessageChain):
         if self._chain_pub:
             content = {
@@ -443,3 +465,7 @@ class PubHandler:
 
 class ConversationTaskStoppedException(Exception):
     pass
+
+
+class ConversationTaskInterruptException(Exception):
+    pass

api/core/extension/api_based_extension_requestor.py

@@ -0,0 +1,62 @@
+import os
+
+import requests
+
+from models.api_based_extension import APIBasedExtensionPoint
+
+
+class APIBasedExtensionRequestor:
+    timeout: (int, int) = (5, 60)
+    """timeout for request connect and read"""
+
+    def __init__(self, api_endpoint: str, api_key: str) -> None:
+        self.api_endpoint = api_endpoint
+        self.api_key = api_key
+
+    def request(self, point: APIBasedExtensionPoint, params: dict) -> dict:
+        """
+        Request the api.
+
+        :param point: the api point
+        :param params: the request params
+        :return: the response json
+        """
+        headers = {
+            "Content-Type": "application/json",
+            "Authorization": "Bearer {}".format(self.api_key)
+        }
+
+        url = self.api_endpoint
+
+        try:
+            # proxy support for security
+            proxies = None
+            if os.environ.get("API_BASED_EXTENSION_HTTP_PROXY") and os.environ.get("API_BASED_EXTENSION_HTTPS_PROXY"):
+                proxies = {
+                    'http': os.environ.get("API_BASED_EXTENSION_HTTP_PROXY"),
+                    'https': os.environ.get("API_BASED_EXTENSION_HTTPS_PROXY"),
+                }
+
+            response = requests.request(
+                method='POST',
+                url=url,
+                json={
+                    'point': point.value,
+                    'params': params
+                },
+                headers=headers,
+                timeout=self.timeout,
+                proxies=proxies
+            )
+        except requests.exceptions.Timeout:
+            raise ValueError("request timeout")
+        except requests.exceptions.ConnectionError:
+            raise ValueError("request connection error")
+
+        if response.status_code != 200:
+            raise ValueError("request error, status_code: {}, content: {}".format(
+                response.status_code,
+                response.text[:100]
+            ))
+
+        return response.json()

api/core/extension/extensible.py

@@ -0,0 +1,111 @@
+import enum
+import importlib.util
+import json
+import logging
+import os
+from collections import OrderedDict
+from typing import Any, Optional
+
+from pydantic import BaseModel
+
+
+class ExtensionModule(enum.Enum):
+    MODERATION = 'moderation'
+    EXTERNAL_DATA_TOOL = 'external_data_tool'
+
+
+class ModuleExtension(BaseModel):
+    extension_class: Any
+    name: str
+    label: Optional[dict] = None
+    form_schema: Optional[list] = None
+    builtin: bool = True
+    position: Optional[int] = None
+
+
+class Extensible:
+    module: ExtensionModule
+
+    name: str
+    tenant_id: str
+    config: Optional[dict] = None
+
+    def __init__(self, tenant_id: str, config: Optional[dict] = None) -> None:
+        self.tenant_id = tenant_id
+        self.config = config
+
+    @classmethod
+    def scan_extensions(cls):
+        extensions = {}
+
+        # get the path of the current class
+        current_path = os.path.abspath(cls.__module__.replace(".", os.path.sep) + '.py')
+        current_dir_path = os.path.dirname(current_path)
+
+        # traverse subdirectories
+        for subdir_name in os.listdir(current_dir_path):
+            if subdir_name.startswith('__'):
+                continue
+
+            subdir_path = os.path.join(current_dir_path, subdir_name)
+            extension_name = subdir_name
+            if os.path.isdir(subdir_path):
+                file_names = os.listdir(subdir_path)
+
+                # is builtin extension, builtin extension
+                # in the front-end page and business logic, there are special treatments.
+                builtin = False
+                position = None
+                if '__builtin__' in file_names:
+                    builtin = True
+
+                    builtin_file_path = os.path.join(subdir_path, '__builtin__')
+                    if os.path.exists(builtin_file_path):
+                        with open(builtin_file_path, 'r') as f:
+                            position = int(f.read().strip())
+
+                if (extension_name + '.py') not in file_names:
+                    logging.warning(f"Missing {extension_name}.py file in {subdir_path}, Skip.")
+                    continue
+
+                # Dynamic loading {subdir_name}.py file and find the subclass of Extensible
+                py_path = os.path.join(subdir_path, extension_name + '.py')
+                spec = importlib.util.spec_from_file_location(extension_name, py_path)
+                mod = importlib.util.module_from_spec(spec)
+                spec.loader.exec_module(mod)
+
+                extension_class = None
+                for name, obj in vars(mod).items():
+                    if isinstance(obj, type) and issubclass(obj, cls) and obj != cls:
+                        extension_class = obj
+                        break
+
+                if not extension_class:
+                    logging.warning(f"Missing subclass of {cls.__name__} in {py_path}, Skip.")
+                    continue
+
+                json_data = {}
+                if not builtin:
+                    if 'schema.json' not in file_names:
+                        logging.warning(f"Missing schema.json file in {subdir_path}, Skip.")
+                        continue
+
+                    json_path = os.path.join(subdir_path, 'schema.json')
+                    json_data = {}
+                    if os.path.exists(json_path):
+                        with open(json_path, 'r') as f:
+                            json_data = json.load(f)
+
+                extensions[extension_name] = ModuleExtension(
+                    extension_class=extension_class,
+                    name=extension_name,
+                    label=json_data.get('label'),
+                    form_schema=json_data.get('form_schema'),
+                    builtin=builtin,
+                    position=position
+                )
+
+        sorted_items = sorted(extensions.items(), key=lambda x: (x[1].position is None, x[1].position))
+        sorted_extensions = OrderedDict(sorted_items)
+
+        return sorted_extensions

api/core/extension/extension.py

@@ -0,0 +1,47 @@
+from core.extension.extensible import ModuleExtension, ExtensionModule
+from core.external_data_tool.base import ExternalDataTool
+from core.moderation.base import Moderation
+
+
+class Extension:
+    __module_extensions: dict[str, dict[str, ModuleExtension]] = {}
+
+    module_classes = {
+        ExtensionModule.MODERATION: Moderation,
+        ExtensionModule.EXTERNAL_DATA_TOOL: ExternalDataTool
+    }
+
+    def init(self):
+        for module, module_class in self.module_classes.items():
+            self.__module_extensions[module.value] = module_class.scan_extensions()
+
+    def module_extensions(self, module: str) -> list[ModuleExtension]:
+        module_extensions = self.__module_extensions.get(module)
+
+        if not module_extensions:
+            raise ValueError(f"Extension Module {module} not found")
+
+        return list(module_extensions.values())
+
+    def module_extension(self, module: ExtensionModule, extension_name: str) -> ModuleExtension:
+        module_extensions = self.__module_extensions.get(module.value)
+
+        if not module_extensions:
+            raise ValueError(f"Extension Module {module} not found")
+
+        module_extension = module_extensions.get(extension_name)
+
+        if not module_extension:
+            raise ValueError(f"Extension {extension_name} not found")
+
+        return module_extension
+
+    def extension_class(self, module: ExtensionModule, extension_name: str) -> type:
+        module_extension = self.module_extension(module, extension_name)
+        return module_extension.extension_class
+
+    def validate_form_schema(self, module: ExtensionModule, extension_name: str, config: dict) -> None:
+        module_extension = self.module_extension(module, extension_name)
+        form_schema = module_extension.form_schema
+
+        # TODO validate form_schema

api/core/external_data_tool/api/__builtin__

@@ -0,0 +1 @@
+1

api/core/external_data_tool/api/api.py

@@ -0,0 +1,92 @@
+from typing import Optional
+
+from core.extension.api_based_extension_requestor import APIBasedExtensionRequestor
+from core.external_data_tool.base import ExternalDataTool
+from core.helper import encrypter
+from extensions.ext_database import db
+from models.api_based_extension import APIBasedExtension, APIBasedExtensionPoint
+
+
+class ApiExternalDataTool(ExternalDataTool):
+    """
+    The api external data tool.
+    """
+
+    name: str = "api"
+    """the unique name of external data tool"""
+
+    @classmethod
+    def validate_config(cls, tenant_id: str, config: dict) -> None:
+        """
+        Validate the incoming form config data.
+
+        :param tenant_id: the id of workspace
+        :param config: the form config data
+        :return:
+        """
+        # own validation logic
+        api_based_extension_id = config.get("api_based_extension_id")
+        if not api_based_extension_id:
+            raise ValueError("api_based_extension_id is required")
+
+        # get api_based_extension
+        api_based_extension = db.session.query(APIBasedExtension).filter(
+            APIBasedExtension.tenant_id == tenant_id,
+            APIBasedExtension.id == api_based_extension_id
+        ).first()
+
+        if not api_based_extension:
+            raise ValueError("api_based_extension_id is invalid")
+
+    def query(self, inputs: dict, query: Optional[str] = None) -> str:
+        """
+        Query the external data tool.
+
+        :param inputs: user inputs
+        :param query: the query of chat app
+        :return: the tool query result
+        """
+        # get params from config
+        api_based_extension_id = self.config.get("api_based_extension_id")
+
+        # get api_based_extension
+        api_based_extension = db.session.query(APIBasedExtension).filter(
+            APIBasedExtension.tenant_id == self.tenant_id,
+            APIBasedExtension.id == api_based_extension_id
+        ).first()
+
+        if not api_based_extension:
+            raise ValueError("[External data tool] API query failed, variable: {}, "
+                             "error: api_based_extension_id is invalid"
+                             .format(self.config.get('variable')))
+
+        # decrypt api_key
+        api_key = encrypter.decrypt_token(
+            tenant_id=self.tenant_id,
+            token=api_based_extension.api_key
+        )
+
+        try:
+            # request api
+            requestor = APIBasedExtensionRequestor(
+                api_endpoint=api_based_extension.api_endpoint,
+                api_key=api_key
+            )
+        except Exception as e:
+            raise ValueError("[External data tool] API query failed, variable: {}, error: {}".format(
+                self.config.get('variable'),
+                e
+            ))
+
+        response_json = requestor.request(point=APIBasedExtensionPoint.APP_EXTERNAL_DATA_TOOL_QUERY, params={
+            'app_id': self.app_id,
+            'tool_variable': self.variable,
+            'inputs': inputs,
+            'query': query
+        })
+
+        if 'result' not in response_json:
+            raise ValueError("[External data tool] API query failed, variable: {}, error: result not found in response"
+                             .format(self.config.get('variable')))
+
+        return response_json['result']

api/core/external_data_tool/base.py

@@ -0,0 +1,45 @@
+from abc import abstractmethod, ABC
+from typing import Optional
+
+from core.extension.extensible import Extensible, ExtensionModule
+
+
+class ExternalDataTool(Extensible, ABC):
+    """
+    The base class of external data tool.
+    """
+
+    module: ExtensionModule = ExtensionModule.EXTERNAL_DATA_TOOL
+
+    app_id: str
+    """the id of app"""
+    variable: str
+    """the tool variable name of app tool"""
+
+    def __init__(self, tenant_id: str, app_id: str, variable: str, config: Optional[dict] = None) -> None:
+        super().__init__(tenant_id, config)
+        self.app_id = app_id
+        self.variable = variable
+
+    @classmethod
+    @abstractmethod
+    def validate_config(cls, tenant_id: str, config: dict) -> None:
+        """
+        Validate the incoming form config data.
+
+        :param tenant_id: the id of workspace
+        :param config: the form config data
+        :return:
+        """
+        raise NotImplementedError
+
+    @abstractmethod
+    def query(self, inputs: dict, query: Optional[str] = None) -> str:
+        """
+        Query the external data tool.
+
+        :param inputs: user inputs
+        :param query: the query of chat app
+        :return: the tool query result
+        """
+        raise NotImplementedError

api/core/external_data_tool/factory.py

@@ -0,0 +1,40 @@
+from typing import Optional
+
+from core.extension.extensible import ExtensionModule
+from extensions.ext_code_based_extension import code_based_extension
+
+
+class ExternalDataToolFactory:
+
+    def __init__(self, name: str, tenant_id: str, app_id: str, variable: str, config: dict) -> None:
+        extension_class = code_based_extension.extension_class(ExtensionModule.EXTERNAL_DATA_TOOL, name)
+        self.__extension_instance = extension_class(
+            tenant_id=tenant_id,
+            app_id=app_id,
+            variable=variable,
+            config=config
+        )
+
+    @classmethod
+    def validate_config(cls, name: str, tenant_id: str, config: dict) -> None:
+        """
+        Validate the incoming form config data.
+
+        :param name: the name of external data tool
+        :param tenant_id: the id of workspace
+        :param config: the form config data
+        :return:
+        """
+        code_based_extension.validate_form_schema(ExtensionModule.EXTERNAL_DATA_TOOL, name, config)
+        extension_class = code_based_extension.extension_class(ExtensionModule.EXTERNAL_DATA_TOOL, name)
+        extension_class.validate_config(tenant_id, config)
+
+    def query(self, inputs: dict, query: Optional[str] = None) -> str:
+        """
+        Query the external data tool.
+
+        :param inputs: user inputs
+        :param query: the query of chat app
+        :return: the tool query result
+        """
+        return self.__extension_instance.query(inputs, query)

api/core/model_providers/model_provider_factory.py

@@ -211,6 +211,9 @@ class ModelProviderFactory:
                                 Provider.quota_type == ProviderQuotaType.TRIAL.value
                             ).first()
 
+                        if provider.quota_limit == 0:
+                            return None
+
                         return provider
 
             no_system_provider = True

api/core/model_providers/models/embedding/xinference_embedding.py

@@ -1,8 +1,7 @@
-from core.third_party.langchain.embeddings.xinference_embedding import XinferenceEmbedding as XinferenceEmbeddings
-
 from core.model_providers.error import LLMBadRequestError
 from core.model_providers.providers.base import BaseModelProvider
 from core.model_providers.models.embedding.base import BaseEmbedding
+from core.third_party.langchain.embeddings.xinference_embedding import XinferenceEmbeddings
 
 
 class XinferenceEmbedding(BaseEmbedding):

api/core/model_providers/models/llm/baichuan_model.py

@@ -37,12 +37,6 @@ class BaichuanModel(BaseLLM):
         prompts = self._get_prompt_from_messages(messages)
         return self._client.generate([prompts], stop, callbacks)
 
-    def prompt_file_name(self, mode: str) -> str:
-        if mode == 'completion':
-            return 'baichuan_completion'
-        else:
-            return 'baichuan_chat'
-
     def get_num_tokens(self, messages: List[PromptMessage]) -> int:
         """
         get num tokens of prompt messages.

api/core/model_providers/models/llm/base.py

@@ -1,28 +1,18 @@
-import json
-import os
-import re
-import time
 from abc import abstractmethod
-from typing import List, Optional, Any, Union, Tuple
+from typing import List, Optional, Any, Union
 import decimal
+import logging
 
 from langchain.callbacks.manager import Callbacks
-from langchain.memory.chat_memory import BaseChatMemory
-from langchain.schema import LLMResult, SystemMessage, AIMessage, HumanMessage, BaseMessage, ChatGeneration
+from langchain.schema import LLMResult, BaseMessage, ChatGeneration
 
 from core.callback_handler.std_out_callback_handler import DifyStreamingStdOutCallbackHandler, DifyStdOutCallbackHandler
 from core.helper import moderation
 from core.model_providers.models.base import BaseProviderModel
-from core.model_providers.models.entity.message import PromptMessage, MessageType, LLMRunResult, to_prompt_messages, \
-    to_lc_messages
+from core.model_providers.models.entity.message import PromptMessage, MessageType, LLMRunResult, to_lc_messages
 from core.model_providers.models.entity.model_params import ModelType, ModelKwargs, ModelMode, ModelKwargsRules
 from core.model_providers.providers.base import BaseModelProvider
-from core.prompt.prompt_builder import PromptBuilder
-from core.prompt.prompt_template import PromptTemplateParser
 from core.third_party.langchain.llms.fake import FakeLLM
-import logging
-
-from extensions.ext_database import db
 
 logger = logging.getLogger(__name__)
 
@@ -320,206 +310,8 @@ class BaseLLM(BaseProviderModel):
     def support_streaming(self):
         return False
 
-    def get_prompt(self, mode: str,
-                   pre_prompt: str, inputs: dict,
-                   query: str,
-                   context: Optional[str],
-                   memory: Optional[BaseChatMemory]) -> \
-            Tuple[List[PromptMessage], Optional[List[str]]]:
-        prompt_rules = self._read_prompt_rules_from_file(self.prompt_file_name(mode))
-        prompt, stops = self._get_prompt_and_stop(prompt_rules, pre_prompt, inputs, query, context, memory)
-        return [PromptMessage(content=prompt)], stops
-
-    def get_advanced_prompt(self, app_mode: str,
-                   app_model_config: str, inputs: dict,
-                   query: str,
-                   context: Optional[str],
-                   memory: Optional[BaseChatMemory]) -> List[PromptMessage]:
-
-        model_mode = app_model_config.model_dict['mode']
-        conversation_histories_role = {}
-
-        raw_prompt_list = []
-        prompt_messages = []
-
-        if app_mode == 'chat' and model_mode == ModelMode.COMPLETION.value:
-            prompt_text = app_model_config.completion_prompt_config_dict['prompt']['text']
-            raw_prompt_list = [{
-                'role': MessageType.USER.value,
-                'text': prompt_text
-            }]
-            conversation_histories_role = app_model_config.completion_prompt_config_dict['conversation_histories_role']
-        elif app_mode == 'chat' and model_mode == ModelMode.CHAT.value:
-            raw_prompt_list = app_model_config.chat_prompt_config_dict['prompt']
-        elif app_mode == 'completion' and model_mode == ModelMode.CHAT.value:
-            raw_prompt_list = app_model_config.chat_prompt_config_dict['prompt']
-        elif app_mode == 'completion' and model_mode == ModelMode.COMPLETION.value:
-            prompt_text = app_model_config.completion_prompt_config_dict['prompt']['text']
-            raw_prompt_list = [{
-                'role': MessageType.USER.value,
-                'text': prompt_text
-            }]
-        else:
-            raise Exception("app_mode or model_mode not support")
-
-        for prompt_item in raw_prompt_list:
-            prompt = prompt_item['text']
-
-            # set prompt template variables
-            prompt_template = PromptTemplateParser(template=prompt)
-            prompt_inputs = {k: inputs[k] for k in prompt_template.variable_keys if k in inputs}
-
-            if '#context#' in prompt:
-                if context:
-                    prompt_inputs['#context#'] = context
-                else:
-                    prompt_inputs['#context#'] = ''
-
-            if '#query#' in prompt:
-                if query:
-                    prompt_inputs['#query#'] = query
-                else:
-                    prompt_inputs['#query#'] = ''
-
-            if '#histories#' in prompt:
-                if memory and app_mode == 'chat' and model_mode == ModelMode.COMPLETION.value:
-                    memory.human_prefix = conversation_histories_role['user_prefix']
-                    memory.ai_prefix = conversation_histories_role['assistant_prefix']
-                    histories = self._get_history_messages_from_memory(memory, 2000)
-                    prompt_inputs['#histories#'] = histories
-                else:
-                    prompt_inputs['#histories#'] = ''
-
-            prompt = prompt_template.format(
-                prompt_inputs
-            )
-
-            prompt = re.sub(r'<\|.*?\|>', '', prompt)
-
-            prompt_messages.append(PromptMessage(type = MessageType(prompt_item['role']) ,content=prompt))
-
-        if memory and app_mode == 'chat' and model_mode == ModelMode.CHAT.value:
-            memory.human_prefix = MessageType.USER.value
-            memory.ai_prefix = MessageType.ASSISTANT.value
-            histories = self._get_history_messages_list_from_memory(memory, 2000)
-            prompt_messages.extend(histories)
-
-        if app_mode == 'chat' and model_mode == ModelMode.CHAT.value:
-            prompt_messages.append(PromptMessage(type = MessageType.USER ,content=query))
-
-        return prompt_messages
-
-    def prompt_file_name(self, mode: str) -> str:
-        if mode == 'completion':
-            return 'common_completion'
-        else:
-            return 'common_chat'
-
-    def _get_prompt_and_stop(self, prompt_rules: dict, pre_prompt: str, inputs: dict,
-                             query: str,
-                             context: Optional[str],
-                             memory: Optional[BaseChatMemory]) -> Tuple[str, Optional[list]]:
-        context_prompt_content = ''
-        if context and 'context_prompt' in prompt_rules:
-            prompt_template = PromptTemplateParser(template=prompt_rules['context_prompt'])
-            context_prompt_content = prompt_template.format(
-                {'context': context}
-            )
-
-        pre_prompt_content = ''
-        if pre_prompt:
-            prompt_template = PromptTemplateParser(template=pre_prompt)
-            prompt_inputs = {k: inputs[k] for k in prompt_template.variable_keys if k in inputs}
-            pre_prompt_content = prompt_template.format(
-                prompt_inputs
-            )
-
-        prompt = ''
-        for order in prompt_rules['system_prompt_orders']:
-            if order == 'context_prompt':
-                prompt += context_prompt_content
-            elif order == 'pre_prompt':
-                prompt += pre_prompt_content
-
-        query_prompt = prompt_rules['query_prompt'] if 'query_prompt' in prompt_rules else '{{query}}'
-
-        if memory and 'histories_prompt' in prompt_rules:
-            # append chat histories
-            tmp_human_message = PromptBuilder.to_human_message(
-                prompt_content=prompt + query_prompt,
-                inputs={
-                    'query': query
-                }
-            )
-
-            if self.model_rules.max_tokens.max:
-                curr_message_tokens = self.get_num_tokens(to_prompt_messages([tmp_human_message]))
-                max_tokens = self.model_kwargs.max_tokens
-                rest_tokens = self.model_rules.max_tokens.max - max_tokens - curr_message_tokens
-                rest_tokens = max(rest_tokens, 0)
-            else:
-                rest_tokens = 2000
-
-            memory.human_prefix = prompt_rules['human_prefix'] if 'human_prefix' in prompt_rules else 'Human'
-            memory.ai_prefix = prompt_rules['assistant_prefix'] if 'assistant_prefix' in prompt_rules else 'Assistant'
-
-            histories = self._get_history_messages_from_memory(memory, rest_tokens)
-            prompt_template = PromptTemplateParser(template=prompt_rules['histories_prompt'])
-            histories_prompt_content = prompt_template.format({'histories': histories})
-
-            prompt = ''
-            for order in prompt_rules['system_prompt_orders']:
-                if order == 'context_prompt':
-                    prompt += context_prompt_content
-                elif order == 'pre_prompt':
-                    prompt += (pre_prompt_content + '\n') if pre_prompt_content else ''
-                elif order == 'histories_prompt':
-                    prompt += histories_prompt_content
-
-        prompt_template = PromptTemplateParser(template=query_prompt)
-        query_prompt_content = prompt_template.format({'query': query})
-
-        prompt += query_prompt_content
-
-        prompt = re.sub(r'<\|.*?\|>', '', prompt)
-
-        stops = prompt_rules.get('stops')
-        if stops is not None and len(stops) == 0:
-            stops = None
-
-        return prompt, stops
-
-    def _read_prompt_rules_from_file(self, prompt_name: str) -> dict:
-        # Get the absolute path of the subdirectory
-        prompt_path = os.path.join(
-            os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__))))),
-            'prompt/generate_prompts')
-
-        json_file_path = os.path.join(prompt_path, f'{prompt_name}.json')
-        # Open the JSON file and read its content
-        with open(json_file_path, 'r') as json_file:
-            return json.load(json_file)
-
-    def _get_history_messages_from_memory(self, memory: BaseChatMemory,
-                                          max_token_limit: int) -> str:
-        """Get memory messages."""
-        memory.max_token_limit = max_token_limit
-        memory_key = memory.memory_variables[0]
-        external_context = memory.load_memory_variables({})
-        return external_context[memory_key]
-
-    def _get_history_messages_list_from_memory(self, memory: BaseChatMemory,
-                                          max_token_limit: int) -> List[PromptMessage]:
-        """Get memory messages."""
-        memory.max_token_limit = max_token_limit
-        memory.return_messages = True
-        memory_key = memory.memory_variables[0]
-        external_context = memory.load_memory_variables({})
-        memory.return_messages = False
-        return to_prompt_messages(external_context[memory_key])
-
     def _get_prompt_from_messages(self, messages: List[PromptMessage],
-                                  model_mode: Optional[ModelMode] = None) -> Union[str | List[BaseMessage]]:
+                                  model_mode: Optional[ModelMode] = None) -> Union[str , List[BaseMessage]]:
         if not model_mode:
             model_mode = self.model_mode
 

api/core/model_providers/models/llm/huggingface_hub_model.py

@@ -66,15 +66,6 @@ class HuggingfaceHubModel(BaseLLM):
         prompts = self._get_prompt_from_messages(messages)
         return self._client.get_num_tokens(prompts)
 
-    def prompt_file_name(self, mode: str) -> str:
-        if 'baichuan' in self.name.lower():
-            if mode == 'completion':
-                return 'baichuan_completion'
-            else:
-                return 'baichuan_chat'
-        else:
-            return super().prompt_file_name(mode)
-
     def _set_model_kwargs(self, model_kwargs: ModelKwargs):
         provider_model_kwargs = self._to_model_kwargs_input(self.model_rules, model_kwargs)
         self.client.model_kwargs = provider_model_kwargs

api/core/model_providers/models/llm/openai_model.py

@@ -33,7 +33,7 @@ MODEL_MAX_TOKENS = {
     'gpt-4': 8192,
     'gpt-4-32k': 32768,
     'gpt-3.5-turbo': 4096,
-    'gpt-3.5-turbo-instruct': 8192,
+    'gpt-3.5-turbo-instruct': 4097,
     'gpt-3.5-turbo-16k': 16384,
     'text-davinci-003': 4097,
 }

api/core/model_providers/models/llm/openllm_model.py

@@ -49,15 +49,6 @@ class OpenLLMModel(BaseLLM):
         prompts = self._get_prompt_from_messages(messages)
         return max(self._client.get_num_tokens(prompts), 0)
 
-    def prompt_file_name(self, mode: str) -> str:
-        if 'baichuan' in self.name.lower():
-            if mode == 'completion':
-                return 'baichuan_completion'
-            else:
-                return 'baichuan_chat'
-        else:
-            return super().prompt_file_name(mode)
-
     def _set_model_kwargs(self, model_kwargs: ModelKwargs):
         pass
 

api/core/model_providers/models/llm/wenxin_model.py

@@ -6,17 +6,16 @@ from langchain.schema import LLMResult
 
 from core.model_providers.error import LLMBadRequestError
 from core.model_providers.models.llm.base import BaseLLM
-from core.model_providers.models.entity.message import PromptMessage, MessageType
+from core.model_providers.models.entity.message import PromptMessage
 from core.model_providers.models.entity.model_params import ModelMode, ModelKwargs
 from core.third_party.langchain.llms.wenxin import Wenxin
 
 
 class WenxinModel(BaseLLM):
-    model_mode: ModelMode = ModelMode.COMPLETION
+    model_mode: ModelMode = ModelMode.CHAT
 
     def _init_client(self) -> Any:
         provider_model_kwargs = self._to_model_kwargs_input(self.model_rules, self.model_kwargs)
-        # TODO load price_config from configs(db)
         return Wenxin(
             model=self.name,
             streaming=self.streaming,
@@ -38,7 +37,13 @@ class WenxinModel(BaseLLM):
         :return:
         """
         prompts = self._get_prompt_from_messages(messages)
-        return self._client.generate([prompts], stop, callbacks)
+
+        generate_kwargs = {'stop': stop, 'callbacks': callbacks, 'messages': [prompts]}
+
+        if 'functions' in kwargs:
+            generate_kwargs['functions'] = kwargs['functions']
+
+        return self._client.generate(**generate_kwargs)
 
     def get_num_tokens(self, messages: List[PromptMessage]) -> int:
         """
@@ -48,7 +53,7 @@ class WenxinModel(BaseLLM):
         :return:
         """
         prompts = self._get_prompt_from_messages(messages)
-        return max(self._client.get_num_tokens(prompts), 0)
+        return max(self._client.get_num_tokens_from_messages(prompts), 0)
 
     def _set_model_kwargs(self, model_kwargs: ModelKwargs):
         provider_model_kwargs = self._to_model_kwargs_input(self.model_rules, model_kwargs)
@@ -58,3 +63,7 @@ class WenxinModel(BaseLLM):
 
     def handle_exceptions(self, ex: Exception) -> Exception:
         return LLMBadRequestError(f"Wenxin: {str(ex)}")
+
+    @property
+    def support_streaming(self):
+        return True

api/core/model_providers/models/llm/xinference_model.py

@@ -59,15 +59,6 @@ class XinferenceModel(BaseLLM):
         prompts = self._get_prompt_from_messages(messages)
         return max(self._client.get_num_tokens(prompts), 0)
 
-    def prompt_file_name(self, mode: str) -> str:
-        if 'baichuan' in self.name.lower():
-            if mode == 'completion':
-                return 'baichuan_completion'
-            else:
-                return 'baichuan_chat'
-        else:
-            return super().prompt_file_name(mode)
-
     def _set_model_kwargs(self, model_kwargs: ModelKwargs):
         pass
 

api/core/model_providers/models/llm/zhipuai_model.py

@@ -16,6 +16,7 @@ class ZhipuAIModel(BaseLLM):
     def _init_client(self) -> Any:
         provider_model_kwargs = self._to_model_kwargs_input(self.model_rules, self.model_kwargs)
         return ZhipuAIChatLLM(
+            model=self.name,
             streaming=self.streaming,
             callbacks=self.callbacks,
             **self.credentials,

api/core/model_providers/providers/anthropic_provider.py

@@ -172,7 +172,7 @@ class AnthropicProvider(BaseModelProvider):
 
     def should_deduct_quota(self):
         if hosted_model_providers.anthropic and \
-                hosted_model_providers.anthropic.quota_limit and hosted_model_providers.anthropic.quota_limit > 0:
+                hosted_model_providers.anthropic.quota_limit and hosted_model_providers.anthropic.quota_limit > -1:
             return True
 
         return False

api/core/model_providers/providers/azure_openai_provider.py

@@ -329,7 +329,7 @@ class AzureOpenAIProvider(BaseModelProvider):
 
     def should_deduct_quota(self):
         if hosted_model_providers.azure_openai \
-                and hosted_model_providers.azure_openai.quota_limit and hosted_model_providers.azure_openai.quota_limit > 0:
+                and hosted_model_providers.azure_openai.quota_limit and hosted_model_providers.azure_openai.quota_limit > -1:
             return True
 
         return False

api/core/model_providers/providers/hosted.py

@@ -11,7 +11,7 @@ class HostedOpenAI(BaseModel):
     api_organization: str = None
     api_key: str
     quota_limit: int = 0
-    """Quota limit for the openai hosted model. 0 means unlimited."""
+    """Quota limit for the openai hosted model. -1 means unlimited."""
     paid_enabled: bool = False
     paid_stripe_price_id: str = None
     paid_increase_quota: int = 1
@@ -21,14 +21,14 @@ class HostedAzureOpenAI(BaseModel):
     api_base: str
     api_key: str
     quota_limit: int = 0
-    """Quota limit for the azure openai hosted model. 0 means unlimited."""
+    """Quota limit for the azure openai hosted model. -1 means unlimited."""
 
 
 class HostedAnthropic(BaseModel):
     api_base: str = None
     api_key: str
     quota_limit: int = 0
-    """Quota limit for the anthropic hosted model. 0 means unlimited."""
+    """Quota limit for the anthropic hosted model. -1 means unlimited."""
     paid_enabled: bool = False
     paid_stripe_price_id: str = None
     paid_increase_quota: int = 1000000

api/core/model_providers/providers/openai_provider.py

@@ -144,7 +144,7 @@ class OpenAIProvider(BaseModelProvider):
             'gpt-4': 8192,
             'gpt-4-32k': 32768,
             'gpt-3.5-turbo': 4096,
-            'gpt-3.5-turbo-instruct': 8192,
+            'gpt-3.5-turbo-instruct': 4097,
             'gpt-3.5-turbo-16k': 16384,
             'text-davinci-003': 4097,
         }
@@ -250,7 +250,7 @@ class OpenAIProvider(BaseModelProvider):
 
     def should_deduct_quota(self):
         if hosted_model_providers.openai \
-                and hosted_model_providers.openai.quota_limit and hosted_model_providers.openai.quota_limit > 0:
+                and hosted_model_providers.openai.quota_limit and hosted_model_providers.openai.quota_limit > -1:
             return True
 
         return False

api/core/model_providers/providers/spark_provider.py

@@ -28,14 +28,19 @@ class SparkProvider(BaseModelProvider):
         if model_type == ModelType.TEXT_GENERATION:
             return [
                 {
-                    'id': 'spark',
-                    'name': 'Spark V1.5',
+                    'id': 'spark-v3',
+                    'name': 'Spark V3.0',
                     'mode': ModelMode.CHAT.value,
                 },
                 {
                     'id': 'spark-v2',
                     'name': 'Spark V2.0',
                     'mode': ModelMode.CHAT.value,
+                },
+                {
+                    'id': 'spark',
+                    'name': 'Spark V1.5',
+                    'mode': ModelMode.CHAT.value,
                 }
             ]
         else:
@@ -96,7 +101,7 @@ class SparkProvider(BaseModelProvider):
 
         try:
             chat_llm = ChatSpark(
-                model_name='spark-v2',
+                model_name='spark-v3',
                 max_tokens=10,
                 temperature=0.01,
                 **credential_kwargs
@@ -110,10 +115,10 @@ class SparkProvider(BaseModelProvider):
 
             chat_llm(messages)
         except SparkError as ex:
-            # try spark v1.5 if v2.1 failed
+            # try spark v2.1 if v3.1 failed
             try:
                 chat_llm = ChatSpark(
-                    model_name='spark',
+                    model_name='spark-v2',
                     max_tokens=10,
                     temperature=0.01,
                     **credential_kwargs
@@ -127,10 +132,27 @@ class SparkProvider(BaseModelProvider):
 
                 chat_llm(messages)
             except SparkError as ex:
-                raise CredentialsValidateFailedError(str(ex))
-            except Exception as ex:
-                logging.exception('Spark config validation failed')
-                raise ex
+                # try spark v1.5 if v2.1 failed
+                try:
+                    chat_llm = ChatSpark(
+                        model_name='spark',
+                        max_tokens=10,
+                        temperature=0.01,
+                        **credential_kwargs
+                    )
+
+                    messages = [
+                        HumanMessage(
+                            content="ping"
+                        )
+                    ]
+
+                    chat_llm(messages)
+                except SparkError as ex:
+                    raise CredentialsValidateFailedError(str(ex))
+                except Exception as ex:
+                    logging.exception('Spark config validation failed')
+                    raise ex
         except Exception as ex:
             logging.exception('Spark config validation failed')
             raise ex

api/core/model_providers/providers/wenxin_provider.py

@@ -2,6 +2,8 @@ import json
 from json import JSONDecodeError
 from typing import Type
 
+from langchain.schema import HumanMessage
+
 from core.helper import encrypter
 from core.model_providers.models.base import BaseProviderModel
 from core.model_providers.models.entity.model_params import ModelKwargsRules, KwargRule, ModelType, ModelMode
@@ -23,20 +25,25 @@ class WenxinProvider(BaseModelProvider):
     def _get_fixed_model_list(self, model_type: ModelType) -> list[dict]:
         if model_type == ModelType.TEXT_GENERATION:
             return [
+                {
+                    'id': 'ernie-bot-4',
+                    'name': 'ERNIE-Bot-4',
+                    'mode': ModelMode.CHAT.value,
+                },
                 {
                     'id': 'ernie-bot',
                     'name': 'ERNIE-Bot',
-                    'mode': ModelMode.COMPLETION.value,
+                    'mode': ModelMode.CHAT.value,
                 },
                 {
                     'id': 'ernie-bot-turbo',
                     'name': 'ERNIE-Bot-turbo',
-                    'mode': ModelMode.COMPLETION.value,
+                    'mode': ModelMode.CHAT.value,
                 },
                 {
                     'id': 'bloomz-7b',
                     'name': 'BLOOMZ-7B',
-                    'mode': ModelMode.COMPLETION.value,
+                    'mode': ModelMode.CHAT.value,
                 }
             ]
         else:
@@ -68,11 +75,12 @@ class WenxinProvider(BaseModelProvider):
         :return:
         """
         model_max_tokens = {
+            'ernie-bot-4': 4800,
             'ernie-bot': 4800,
             'ernie-bot-turbo': 11200,
         }
 
-        if model_name in ['ernie-bot', 'ernie-bot-turbo']:
+        if model_name in ['ernie-bot-4', 'ernie-bot', 'ernie-bot-turbo']:
             return ModelKwargsRules(
                 temperature=KwargRule[float](min=0.01, max=1, default=0.95, precision=2),
                 top_p=KwargRule[float](min=0.01, max=1, default=0.8, precision=2),
@@ -111,7 +119,7 @@ class WenxinProvider(BaseModelProvider):
                 **credential_kwargs
             )
 
-            llm("ping")
+            llm([HumanMessage(content='ping')])
         except Exception as ex:
             raise CredentialsValidateFailedError(str(ex))
 

api/core/model_providers/providers/xinference_provider.py

@@ -2,7 +2,6 @@ import json
 from typing import Type
 
 import requests
-from langchain.embeddings import XinferenceEmbeddings
 
 from core.helper import encrypter
 from core.model_providers.models.embedding.xinference_embedding import XinferenceEmbedding
@@ -11,6 +10,7 @@ from core.model_providers.models.llm.xinference_model import XinferenceModel
 from core.model_providers.providers.base import BaseModelProvider, CredentialsValidateFailedError
 
 from core.model_providers.models.base import BaseProviderModel
+from core.third_party.langchain.embeddings.xinference_embedding import XinferenceEmbeddings
 from core.third_party.langchain.llms.xinference_llm import XinferenceLLM
 from models.provider import ProviderType
 

api/core/model_providers/providers/zhipuai_provider.py

@@ -26,6 +26,11 @@ class ZhipuAIProvider(BaseModelProvider):
     def _get_fixed_model_list(self, model_type: ModelType) -> list[dict]:
         if model_type == ModelType.TEXT_GENERATION:
             return [
+                {
+                    'id': 'chatglm_turbo',
+                    'name': 'chatglm_turbo',
+                    'mode': ModelMode.CHAT.value,
+                },
                 {
                     'id': 'chatglm_pro',
                     'name': 'chatglm_pro',

api/core/model_providers/rules/anthropic.json

@@ -9,7 +9,7 @@
             "trial"
         ],
         "quota_unit": "tokens",
-        "quota_limit": 600000
+        "quota_limit": 0
     },
     "model_flexibility": "fixed",
     "price_config": {

api/core/model_providers/rules/spark.json

@@ -22,6 +22,12 @@
             "completion": "0.36",
             "unit": "0.0001",
             "currency": "RMB"
+        },
+        "spark-v3": {
+            "prompt": "0.36",
+            "completion": "0.36",
+            "unit": "0.0001",
+            "currency": "RMB"
         }
     }
 }

api/core/model_providers/rules/wenxin.json

@@ -5,6 +5,12 @@
     "system_config": null,
     "model_flexibility": "fixed",
     "price_config": {
+        "ernie-bot-4": {
+            "prompt": "0",
+            "completion": "0",
+            "unit": "0.001",
+            "currency": "RMB"
+        },
         "ernie-bot": {
             "prompt": "0.012",
             "completion": "0.012",

api/core/model_providers/rules/zhipuai.json

@@ -11,6 +11,12 @@
     },
     "model_flexibility": "fixed",
     "price_config": {
+        "chatglm_turbo": {
+            "prompt": "0.005",
+            "completion": "0.005",
+            "unit": "0.001",
+            "currency": "RMB"
+        },
         "chatglm_pro": {
             "prompt": "0.01",
             "completion": "0.01",

api/core/moderation/api/__builtin__

@@ -0,0 +1 @@
+3

api/core/moderation/api/api.py

@@ -0,0 +1,88 @@
+from pydantic import BaseModel
+
+from core.moderation.base import Moderation, ModerationInputsResult, ModerationOutputsResult, ModerationAction
+from core.extension.api_based_extension_requestor import APIBasedExtensionRequestor, APIBasedExtensionPoint
+from core.helper.encrypter import decrypt_token
+from extensions.ext_database import db
+from models.api_based_extension import APIBasedExtension
+
+
+class ModerationInputParams(BaseModel):
+    app_id: str = ""
+    inputs: dict = {}
+    query: str = ""
+
+
+class ModerationOutputParams(BaseModel):
+    app_id: str = ""
+    text: str
+
+
+class ApiModeration(Moderation):
+    name: str = "api"
+
+    @classmethod
+    def validate_config(cls, tenant_id: str, config: dict) -> None:
+        """
+        Validate the incoming form config data.
+
+        :param tenant_id: the id of workspace
+        :param config: the form config data
+        :return:
+        """
+        cls._validate_inputs_and_outputs_config(config, False)
+
+        api_based_extension_id = config.get("api_based_extension_id")
+        if not api_based_extension_id:
+            raise ValueError("api_based_extension_id is required")
+
+        extension = cls._get_api_based_extension(tenant_id, api_based_extension_id)
+        if not extension:
+            raise ValueError("API-based Extension not found. Please check it again.")
+
+    def moderation_for_inputs(self, inputs: dict, query: str = "") -> ModerationInputsResult:
+        flagged = False
+        preset_response = ""
+
+        if self.config['inputs_config']['enabled']:
+            params = ModerationInputParams(
+                app_id=self.app_id,
+                inputs=inputs,
+                query=query
+            )
+
+            result = self._get_config_by_requestor(APIBasedExtensionPoint.APP_MODERATION_INPUT, params.dict())
+            return ModerationInputsResult(**result)
+
+        return ModerationInputsResult(flagged=flagged, action=ModerationAction.DIRECT_OUTPUT, preset_response=preset_response)
+
+    def moderation_for_outputs(self, text: str) -> ModerationOutputsResult:
+        flagged = False
+        preset_response = ""
+
+        if self.config['outputs_config']['enabled']:
+            params = ModerationOutputParams(
+                app_id=self.app_id,
+                text=text
+            )
+
+            result = self._get_config_by_requestor(APIBasedExtensionPoint.APP_MODERATION_OUTPUT, params.dict())
+            return ModerationOutputsResult(**result)
+
+        return ModerationOutputsResult(flagged=flagged, action=ModerationAction.DIRECT_OUTPUT, preset_response=preset_response)
+
+    def _get_config_by_requestor(self, extension_point: APIBasedExtensionPoint, params: dict) -> dict:
+        extension = self._get_api_based_extension(self.tenant_id, self.config.get("api_based_extension_id"))
+        requestor = APIBasedExtensionRequestor(extension.api_endpoint, decrypt_token(self.tenant_id, extension.api_key))
+
+        result = requestor.request(extension_point, params)
+        return result
+
+    @staticmethod
+    def _get_api_based_extension(tenant_id: str, api_based_extension_id: str) -> APIBasedExtension:
+        extension = db.session.query(APIBasedExtension).filter(
+            APIBasedExtension.tenant_id == tenant_id,
+            APIBasedExtension.id == api_based_extension_id
+        ).first()
+
+        return extension

api/core/moderation/base.py

@@ -0,0 +1,113 @@
+from abc import ABC, abstractmethod
+from typing import Optional
+from pydantic import BaseModel
+from enum import Enum
+
+from core.extension.extensible import Extensible, ExtensionModule
+
+
+class ModerationAction(Enum):
+    DIRECT_OUTPUT = 'direct_output'
+    OVERRIDED = 'overrided'
+
+
+class ModerationInputsResult(BaseModel):
+    flagged: bool = False
+    action: ModerationAction
+    preset_response: str = ""
+    inputs: dict = {}
+    query: str = ""
+
+
+class ModerationOutputsResult(BaseModel):
+    flagged: bool = False
+    action: ModerationAction
+    preset_response: str = ""
+    text: str = ""
+
+
+class Moderation(Extensible, ABC):
+    """
+    The base class of moderation.
+    """
+    module: ExtensionModule = ExtensionModule.MODERATION
+
+    def __init__(self, app_id: str, tenant_id: str, config: Optional[dict] = None) -> None:
+        super().__init__(tenant_id, config)
+        self.app_id = app_id
+
+    @classmethod
+    @abstractmethod
+    def validate_config(cls, tenant_id: str, config: dict) -> None:
+        """
+        Validate the incoming form config data.
+
+        :param tenant_id: the id of workspace
+        :param config: the form config data
+        :return:
+        """
+        raise NotImplementedError
+
+    @abstractmethod
+    def moderation_for_inputs(self, inputs: dict, query: str = "") -> ModerationInputsResult:
+        """
+        Moderation for inputs.
+        After the user inputs, this method will be called to perform sensitive content review
+        on the user inputs and return the processed results.
+
+        :param inputs: user inputs
+        :param query: query string (required in chat app)
+        :return:
+        """
+        raise NotImplementedError
+
+    @abstractmethod
+    def moderation_for_outputs(self, text: str) -> ModerationOutputsResult:
+        """
+        Moderation for outputs.
+        When LLM outputs content, the front end will pass the output content (may be segmented)
+        to this method for sensitive content review, and the output content will be shielded if the review fails.
+
+        :param text: LLM output content
+        :return:
+        """
+        raise NotImplementedError
+
+    @classmethod
+    def _validate_inputs_and_outputs_config(self, config: dict, is_preset_response_required: bool) -> None:
+        # inputs_config
+        inputs_config = config.get("inputs_config")
+        if not isinstance(inputs_config, dict):
+            raise ValueError("inputs_config must be a dict")
+
+        # outputs_config
+        outputs_config = config.get("outputs_config")
+        if not isinstance(outputs_config, dict):
+            raise ValueError("outputs_config must be a dict")
+
+        inputs_config_enabled = inputs_config.get("enabled")
+        outputs_config_enabled = outputs_config.get("enabled")
+        if not inputs_config_enabled and not outputs_config_enabled:
+            raise ValueError("At least one of inputs_config or outputs_config must be enabled")
+
+        # preset_response
+        if not is_preset_response_required:
+            return
+
+        if inputs_config_enabled:
+            if not inputs_config.get("preset_response"):
+                raise ValueError("inputs_config.preset_response is required")
+
+            if len(inputs_config.get("preset_response")) > 100:
+                raise ValueError("inputs_config.preset_response must be less than 100 characters")
+
+        if outputs_config_enabled:
+            if not outputs_config.get("preset_response"):
+                raise ValueError("outputs_config.preset_response is required")
+
+            if len(outputs_config.get("preset_response")) > 100:
+                raise ValueError("outputs_config.preset_response must be less than 100 characters")
+
+
+class ModerationException(Exception):
+    pass

api/core/moderation/factory.py

@@ -0,0 +1,48 @@
+from core.extension.extensible import ExtensionModule
+from core.moderation.base import Moderation, ModerationInputsResult, ModerationOutputsResult
+from extensions.ext_code_based_extension import code_based_extension
+
+
+class ModerationFactory:
+    __extension_instance: Moderation
+
+    def __init__(self, name: str, app_id: str, tenant_id: str, config: dict) -> None:
+        extension_class = code_based_extension.extension_class(ExtensionModule.MODERATION, name)
+        self.__extension_instance = extension_class(app_id, tenant_id, config)
+
+    @classmethod
+    def validate_config(cls, name: str, tenant_id: str, config: dict) -> None:
+        """
+        Validate the incoming form config data.
+
+        :param name: the name of extension
+        :param tenant_id: the id of workspace
+        :param config: the form config data
+        :return:
+        """
+        code_based_extension.validate_form_schema(ExtensionModule.MODERATION, name, config)
+        extension_class = code_based_extension.extension_class(ExtensionModule.MODERATION, name)
+        extension_class.validate_config(tenant_id, config)
+
+    def moderation_for_inputs(self, inputs: dict, query: str = "") -> ModerationInputsResult:
+        """
+        Moderation for inputs.
+        After the user inputs, this method will be called to perform sensitive content review
+        on the user inputs and return the processed results.
+
+        :param inputs: user inputs
+        :param query: query string (required in chat app)
+        :return:
+        """
+        return self.__extension_instance.moderation_for_inputs(inputs, query)
+
+    def moderation_for_outputs(self, text: str) -> ModerationOutputsResult:
+        """
+        Moderation for outputs.
+        When LLM outputs content, the front end will pass the output content (may be segmented)
+        to this method for sensitive content review, and the output content will be shielded if the review fails.
+
+        :param text: LLM output content
+        :return:
+        """
+        return self.__extension_instance.moderation_for_outputs(text)

api/core/moderation/keywords/__builtin__

@@ -0,0 +1 @@
+2

api/core/moderation/keywords/keywords.py

@@ -0,0 +1,60 @@
+from core.moderation.base import Moderation, ModerationInputsResult, ModerationOutputsResult, ModerationAction
+
+
+class KeywordsModeration(Moderation):
+    name: str = "keywords"
+
+    @classmethod
+    def validate_config(cls, tenant_id: str, config: dict) -> None:
+        """
+        Validate the incoming form config data.
+
+        :param tenant_id: the id of workspace
+        :param config: the form config data
+        :return:
+        """
+        cls._validate_inputs_and_outputs_config(config, True)
+
+        if not config.get("keywords"):
+            raise ValueError("keywords is required")
+
+        if len(config.get("keywords")) > 1000:
+            raise ValueError("keywords length must be less than 1000")
+
+    def moderation_for_inputs(self, inputs: dict, query: str = "") -> ModerationInputsResult:
+        flagged = False
+        preset_response = ""
+
+        if self.config['inputs_config']['enabled']:
+            preset_response = self.config['inputs_config']['preset_response']
+
+            if query:
+                inputs['query__'] = query
+            keywords_list = self.config['keywords'].split('\n')
+            flagged = self._is_violated(inputs, keywords_list)
+
+        return ModerationInputsResult(flagged=flagged, action=ModerationAction.DIRECT_OUTPUT, preset_response=preset_response)
+
+    def moderation_for_outputs(self, text: str) -> ModerationOutputsResult:
+        flagged = False
+        preset_response = ""
+
+        if self.config['outputs_config']['enabled']:
+            keywords_list = self.config['keywords'].split('\n')
+            flagged = self._is_violated({'text': text}, keywords_list)
+            preset_response = self.config['outputs_config']['preset_response']
+
+        return ModerationOutputsResult(flagged=flagged, action=ModerationAction.DIRECT_OUTPUT, preset_response=preset_response)
+
+    def _is_violated(self, inputs: dict, keywords_list: list) -> bool:
+        for value in inputs.values():
+            if self._check_keywords_in_value(keywords_list, value):
+                return True
+
+        return False
+
+    def _check_keywords_in_value(self, keywords_list, value):
+        for keyword in keywords_list:
+            if keyword.lower() in value.lower():
+                return True
+        return False

api/core/moderation/openai_moderation/__builtin__

@@ -0,0 +1 @@
+1

api/core/moderation/openai_moderation/openai_moderation.py

@@ -0,0 +1,46 @@
+from core.moderation.base import Moderation, ModerationInputsResult, ModerationOutputsResult, ModerationAction
+from core.model_providers.model_factory import ModelFactory
+
+
+class OpenAIModeration(Moderation):
+    name: str = "openai_moderation"
+
+    @classmethod
+    def validate_config(cls, tenant_id: str, config: dict) -> None:
+        """
+        Validate the incoming form config data.
+
+        :param tenant_id: the id of workspace
+        :param config: the form config data
+        :return:
+        """
+        cls._validate_inputs_and_outputs_config(config, True)
+
+    def moderation_for_inputs(self, inputs: dict, query: str = "") -> ModerationInputsResult:
+        flagged = False
+        preset_response = ""
+
+        if self.config['inputs_config']['enabled']:
+            preset_response = self.config['inputs_config']['preset_response']
+
+            if query:
+                inputs['query__'] = query
+            flagged = self._is_violated(inputs)
+
+        return ModerationInputsResult(flagged=flagged, action=ModerationAction.DIRECT_OUTPUT, preset_response=preset_response)
+
+    def moderation_for_outputs(self, text: str) -> ModerationOutputsResult:
+        flagged = False
+        preset_response = ""
+
+        if self.config['outputs_config']['enabled']:
+            flagged = self._is_violated({'text': text})
+            preset_response = self.config['outputs_config']['preset_response']
+
+        return ModerationOutputsResult(flagged=flagged, action=ModerationAction.DIRECT_OUTPUT, preset_response=preset_response)
+
+    def _is_violated(self, inputs: dict):
+        text = '\n'.join(inputs.values())
+        openai_moderation = ModelFactory.get_moderation_model(self.tenant_id, "openai", "moderation")
+        is_not_invalid = openai_moderation.run(text)
+        return not is_not_invalid

api/core/orchestrator_rule_parser.py

@@ -11,7 +11,6 @@ from core.callback_handler.agent_loop_gather_callback_handler import AgentLoopGa
 from core.callback_handler.dataset_tool_callback_handler import DatasetToolCallbackHandler
 from core.callback_handler.main_chain_gather_callback_handler import MainChainGatherCallbackHandler
 from core.callback_handler.std_out_callback_handler import DifyStdOutCallbackHandler
-from core.chain.sensitive_word_avoidance_chain import SensitiveWordAvoidanceChain, SensitiveWordAvoidanceRule
 from core.conversation_message_task import ConversationMessageTask
 from core.model_providers.error import ProviderTokenNotInitError
 from core.model_providers.model_factory import ModelFactory
@@ -125,52 +124,6 @@ class OrchestratorRuleParser:
 
         return chain
 
-    def to_sensitive_word_avoidance_chain(self, model_instance: BaseLLM, callbacks: Callbacks = None, **kwargs) \
-            -> Optional[SensitiveWordAvoidanceChain]:
-        """
-        Convert app sensitive word avoidance config to chain
-
-        :param model_instance: model instance
-        :param callbacks: callbacks for the chain
-        :param kwargs:
-        :return:
-        """
-        sensitive_word_avoidance_rule = None
-
-        if self.app_model_config.sensitive_word_avoidance_dict:
-            sensitive_word_avoidance_config = self.app_model_config.sensitive_word_avoidance_dict
-            if sensitive_word_avoidance_config.get("enabled", False):
-                if sensitive_word_avoidance_config.get('type') == 'moderation':
-                    sensitive_word_avoidance_rule = SensitiveWordAvoidanceRule(
-                        type=SensitiveWordAvoidanceRule.Type.MODERATION,
-                        canned_response=sensitive_word_avoidance_config.get("canned_response")
-                        if sensitive_word_avoidance_config.get("canned_response")
-                        else 'Your content violates our usage policy. Please revise and try again.',
-                    )
-                else:
-                    sensitive_words = sensitive_word_avoidance_config.get("words", "")
-                    if sensitive_words:
-                        sensitive_word_avoidance_rule = SensitiveWordAvoidanceRule(
-                            type=SensitiveWordAvoidanceRule.Type.KEYWORDS,
-                            canned_response=sensitive_word_avoidance_config.get("canned_response")
-                            if sensitive_word_avoidance_config.get("canned_response")
-                            else 'Your content violates our usage policy. Please revise and try again.',
-                            extra_params={
-                                'sensitive_words': sensitive_words.split(','),
-                            }
-                        )
-
-        if sensitive_word_avoidance_rule:
-            return SensitiveWordAvoidanceChain(
-                model_instance=model_instance,
-                sensitive_word_avoidance_rule=sensitive_word_avoidance_rule,
-                output_key="sensitive_word_avoidance_output",
-                callbacks=callbacks,
-                **kwargs
-            )
-
-        return None
-
     def to_tools(self, tool_configs: list, callbacks: Callbacks = None, **kwargs) -> list[BaseTool]:
         """
         Convert app agent tool configs to tools

api/core/prompt/advanced_prompt_templates.py

@@ -11,7 +11,8 @@ CHAT_APP_COMPLETION_PROMPT_CONFIG = {
             "user_prefix": "Human",
             "assistant_prefix": "Assistant"
         }
-    }
+    },
+    "stop": ["Human:"]
 }
 
 CHAT_APP_CHAT_PROMPT_CONFIG = { 
@@ -37,7 +38,8 @@ COMPLETION_APP_COMPLETION_PROMPT_CONFIG = {
         "prompt": {
             "text": "{{#pre_prompt#}}"
         }
-    }
+    },
+    "stop": ["Human:"]
 }
 
 BAICHUAN_CHAT_APP_COMPLETION_PROMPT_CONFIG = {
@@ -49,7 +51,8 @@ BAICHUAN_CHAT_APP_COMPLETION_PROMPT_CONFIG = {
             "user_prefix": "用户",
             "assistant_prefix": "助手"
         }
-    }
+    },
+    "stop": ["用户:"]
 }
 
 BAICHUAN_CHAT_APP_CHAT_PROMPT_CONFIG = { 
@@ -75,5 +78,6 @@ BAICHUAN_COMPLETION_APP_COMPLETION_PROMPT_CONFIG = {
         "prompt": {
             "text": "{{#pre_prompt#}}"
         }
-    }
+    },
+    "stop": ["用户:"]
 }

api/core/prompt/prompt_transform.py

@@ -0,0 +1,344 @@
+import json
+import os
+import re
+import enum
+from typing import List, Optional, Tuple
+
+from langchain.memory.chat_memory import BaseChatMemory
+from langchain.schema import BaseMessage
+
+from core.model_providers.models.entity.model_params import ModelMode
+from core.model_providers.models.entity.message import PromptMessage, MessageType, to_prompt_messages
+from core.model_providers.models.llm.base import BaseLLM
+from core.model_providers.models.llm.baichuan_model import BaichuanModel
+from core.model_providers.models.llm.huggingface_hub_model import HuggingfaceHubModel
+from core.model_providers.models.llm.openllm_model import OpenLLMModel
+from core.model_providers.models.llm.xinference_model import XinferenceModel
+from core.prompt.prompt_builder import PromptBuilder
+from core.prompt.prompt_template import PromptTemplateParser
+
+class AppMode(enum.Enum):
+    COMPLETION = 'completion'
+    CHAT = 'chat'
+
+class PromptTransform:
+    def get_prompt(self, mode: str,
+                   pre_prompt: str, inputs: dict,
+                   query: str,
+                   context: Optional[str],
+                   memory: Optional[BaseChatMemory],
+                   model_instance: BaseLLM) -> \
+            Tuple[List[PromptMessage], Optional[List[str]]]:
+        prompt_rules = self._read_prompt_rules_from_file(self._prompt_file_name(mode, model_instance))
+        prompt, stops = self._get_prompt_and_stop(prompt_rules, pre_prompt, inputs, query, context, memory, model_instance)
+        return [PromptMessage(content=prompt)], stops
+
+    def get_advanced_prompt(self, 
+            app_mode: str,
+            app_model_config: str, 
+            inputs: dict,
+            query: str,
+            context: Optional[str],
+            memory: Optional[BaseChatMemory],
+            model_instance: BaseLLM) -> List[PromptMessage]:
+        
+        model_mode = app_model_config.model_dict['mode']
+
+        app_mode_enum = AppMode(app_mode)
+        model_mode_enum = ModelMode(model_mode)
+
+        prompt_messages = []
+
+        if app_mode_enum == AppMode.CHAT:
+            if model_mode_enum == ModelMode.COMPLETION:
+                prompt_messages = self._get_chat_app_completion_model_prompt_messages(app_model_config, inputs, query, context, memory, model_instance)
+            elif model_mode_enum == ModelMode.CHAT:
+                prompt_messages =  self._get_chat_app_chat_model_prompt_messages(app_model_config, inputs, query, context, memory, model_instance)
+        elif app_mode_enum == AppMode.COMPLETION:
+            if model_mode_enum == ModelMode.CHAT:
+                prompt_messages =  self._get_completion_app_chat_model_prompt_messages(app_model_config, inputs, context)
+            elif model_mode_enum == ModelMode.COMPLETION:
+                prompt_messages =  self._get_completion_app_completion_model_prompt_messages(app_model_config, inputs, context)
+            
+        return prompt_messages
+
+    def _get_history_messages_from_memory(self, memory: BaseChatMemory,
+                                          max_token_limit: int) -> str:
+        """Get memory messages."""
+        memory.max_token_limit = max_token_limit
+        memory_key = memory.memory_variables[0]
+        external_context = memory.load_memory_variables({})
+        return external_context[memory_key]
+
+    def _get_history_messages_list_from_memory(self, memory: BaseChatMemory,
+                                          max_token_limit: int) -> List[PromptMessage]:
+        """Get memory messages."""
+        memory.max_token_limit = max_token_limit
+        memory.return_messages = True
+        memory_key = memory.memory_variables[0]
+        external_context = memory.load_memory_variables({})
+        memory.return_messages = False
+        return to_prompt_messages(external_context[memory_key])
+    
+    def _prompt_file_name(self, mode: str, model_instance: BaseLLM) -> str:
+        # baichuan
+        if isinstance(model_instance, BaichuanModel):
+            return self._prompt_file_name_for_baichuan(mode)
+
+        baichuan_model_hosted_platforms = (HuggingfaceHubModel, OpenLLMModel, XinferenceModel)
+        if isinstance(model_instance, baichuan_model_hosted_platforms) and 'baichuan' in model_instance.name.lower():
+            return self._prompt_file_name_for_baichuan(mode)
+
+        # common
+        if mode == 'completion':
+            return 'common_completion'
+        else:
+            return 'common_chat'
+        
+    def _prompt_file_name_for_baichuan(self, mode: str) -> str:
+        if mode == 'completion':
+            return 'baichuan_completion'
+        else:
+            return 'baichuan_chat'
+    
+    def _read_prompt_rules_from_file(self, prompt_name: str) -> dict:
+        # Get the absolute path of the subdirectory
+        prompt_path = os.path.join(
+            os.path.dirname(os.path.realpath(__file__)),
+            'generate_prompts')
+
+        json_file_path = os.path.join(prompt_path, f'{prompt_name}.json')
+        # Open the JSON file and read its content
+        with open(json_file_path, 'r') as json_file:
+            return json.load(json_file)
+        
+    def _get_prompt_and_stop(self, prompt_rules: dict, pre_prompt: str, inputs: dict,
+                             query: str,
+                             context: Optional[str],
+                             memory: Optional[BaseChatMemory],
+                             model_instance: BaseLLM) -> Tuple[str, Optional[list]]:
+        context_prompt_content = ''
+        if context and 'context_prompt' in prompt_rules:
+            prompt_template = PromptTemplateParser(template=prompt_rules['context_prompt'])
+            context_prompt_content = prompt_template.format(
+                {'context': context}
+            )
+
+        pre_prompt_content = ''
+        if pre_prompt:
+            prompt_template = PromptTemplateParser(template=pre_prompt)
+            prompt_inputs = {k: inputs[k] for k in prompt_template.variable_keys if k in inputs}
+            pre_prompt_content = prompt_template.format(
+                prompt_inputs
+            )
+
+        prompt = ''
+        for order in prompt_rules['system_prompt_orders']:
+            if order == 'context_prompt':
+                prompt += context_prompt_content
+            elif order == 'pre_prompt':
+                prompt += pre_prompt_content
+
+        query_prompt = prompt_rules['query_prompt'] if 'query_prompt' in prompt_rules else '{{query}}'
+
+        if memory and 'histories_prompt' in prompt_rules:
+            # append chat histories
+            tmp_human_message = PromptBuilder.to_human_message(
+                prompt_content=prompt + query_prompt,
+                inputs={
+                    'query': query
+                }
+            )
+
+            rest_tokens = self._calculate_rest_token(tmp_human_message, model_instance)
+
+            memory.human_prefix = prompt_rules['human_prefix'] if 'human_prefix' in prompt_rules else 'Human'
+            memory.ai_prefix = prompt_rules['assistant_prefix'] if 'assistant_prefix' in prompt_rules else 'Assistant'
+
+            histories = self._get_history_messages_from_memory(memory, rest_tokens)
+            prompt_template = PromptTemplateParser(template=prompt_rules['histories_prompt'])
+            histories_prompt_content = prompt_template.format({'histories': histories})
+
+            prompt = ''
+            for order in prompt_rules['system_prompt_orders']:
+                if order == 'context_prompt':
+                    prompt += context_prompt_content
+                elif order == 'pre_prompt':
+                    prompt += (pre_prompt_content + '\n') if pre_prompt_content else ''
+                elif order == 'histories_prompt':
+                    prompt += histories_prompt_content
+
+        prompt_template = PromptTemplateParser(template=query_prompt)
+        query_prompt_content = prompt_template.format({'query': query})
+
+        prompt += query_prompt_content
+
+        prompt = re.sub(r'<\|.*?\|>', '', prompt)
+
+        stops = prompt_rules.get('stops')
+        if stops is not None and len(stops) == 0:
+            stops = None
+
+        return prompt, stops
+    
+    def _set_context_variable(self, context: str, prompt_template: PromptTemplateParser, prompt_inputs: dict) -> None:
+        if '#context#' in prompt_template.variable_keys:
+            if context:
+                prompt_inputs['#context#'] = context    
+            else:
+                prompt_inputs['#context#'] = ''
+
+    def _set_query_variable(self, query: str, prompt_template: PromptTemplateParser, prompt_inputs: dict) -> None:
+        if '#query#' in prompt_template.variable_keys:
+            if query:
+                prompt_inputs['#query#'] = query
+            else:
+                prompt_inputs['#query#'] = ''
+
+    def _set_histories_variable(self, memory: BaseChatMemory, raw_prompt: str, conversation_histories_role: dict, 
+                                prompt_template: PromptTemplateParser, prompt_inputs: dict, model_instance: BaseLLM) -> None:
+        if '#histories#' in prompt_template.variable_keys:
+            if memory:
+                tmp_human_message = PromptBuilder.to_human_message(
+                    prompt_content=raw_prompt,
+                    inputs={ '#histories#': '', **prompt_inputs }
+                )
+
+                rest_tokens = self._calculate_rest_token(tmp_human_message, model_instance)
+                
+                memory.human_prefix = conversation_histories_role['user_prefix']
+                memory.ai_prefix = conversation_histories_role['assistant_prefix']
+                histories = self._get_history_messages_from_memory(memory, rest_tokens)
+                prompt_inputs['#histories#'] = histories
+            else:
+                prompt_inputs['#histories#'] = ''
+
+    def _append_chat_histories(self, memory: BaseChatMemory, prompt_messages: list[PromptMessage], model_instance: BaseLLM) -> None:
+        if memory:
+            rest_tokens = self._calculate_rest_token(prompt_messages, model_instance)
+
+            memory.human_prefix = MessageType.USER.value
+            memory.ai_prefix = MessageType.ASSISTANT.value
+            histories = self._get_history_messages_list_from_memory(memory, rest_tokens)
+            prompt_messages.extend(histories)
+
+    def _calculate_rest_token(self, prompt_messages: BaseMessage, model_instance: BaseLLM) -> int:
+        rest_tokens = 2000
+
+        if model_instance.model_rules.max_tokens.max:
+            curr_message_tokens = model_instance.get_num_tokens(to_prompt_messages(prompt_messages))
+            max_tokens = model_instance.model_kwargs.max_tokens
+            rest_tokens = model_instance.model_rules.max_tokens.max - max_tokens - curr_message_tokens
+            rest_tokens = max(rest_tokens, 0)
+
+        return rest_tokens
+
+    def _format_prompt(self, prompt_template: PromptTemplateParser, prompt_inputs: dict) -> str:
+        prompt = prompt_template.format(
+            prompt_inputs
+        )
+
+        prompt = re.sub(r'<\|.*?\|>', '', prompt)
+        return prompt
+
+    def _get_chat_app_completion_model_prompt_messages(self,
+            app_model_config: str,
+            inputs: dict,
+            query: str,
+            context: Optional[str],
+            memory: Optional[BaseChatMemory],
+            model_instance: BaseLLM) -> List[PromptMessage]:
+        
+        raw_prompt = app_model_config.completion_prompt_config_dict['prompt']['text']
+        conversation_histories_role = app_model_config.completion_prompt_config_dict['conversation_histories_role']
+
+        prompt_messages = []
+        prompt = ''
+        
+        prompt_template = PromptTemplateParser(template=raw_prompt)
+        prompt_inputs = {k: inputs[k] for k in prompt_template.variable_keys if k in inputs}
+
+        self._set_context_variable(context, prompt_template, prompt_inputs)
+
+        self._set_query_variable(query, prompt_template, prompt_inputs)
+
+        self._set_histories_variable(memory, raw_prompt, conversation_histories_role, prompt_template, prompt_inputs, model_instance)
+
+        prompt = self._format_prompt(prompt_template, prompt_inputs)
+
+        prompt_messages.append(PromptMessage(type = MessageType(MessageType.USER) ,content=prompt))
+
+        return prompt_messages
+
+    def _get_chat_app_chat_model_prompt_messages(self,
+            app_model_config: str,
+            inputs: dict,
+            query: str,
+            context: Optional[str],
+            memory: Optional[BaseChatMemory],
+            model_instance: BaseLLM) -> List[PromptMessage]:
+        raw_prompt_list = app_model_config.chat_prompt_config_dict['prompt']
+
+        prompt_messages = []
+
+        for prompt_item in raw_prompt_list:
+            raw_prompt = prompt_item['text']
+            prompt = ''
+
+            prompt_template = PromptTemplateParser(template=raw_prompt)
+            prompt_inputs = {k: inputs[k] for k in prompt_template.variable_keys if k in inputs}
+
+            self._set_context_variable(context, prompt_template, prompt_inputs)
+
+            prompt = self._format_prompt(prompt_template, prompt_inputs)
+
+            prompt_messages.append(PromptMessage(type = MessageType(prompt_item['role']) ,content=prompt))
+        
+        self._append_chat_histories(memory, prompt_messages, model_instance)
+
+        prompt_messages.append(PromptMessage(type = MessageType.USER ,content=query))
+
+        return prompt_messages
+
+    def _get_completion_app_completion_model_prompt_messages(self,
+                   app_model_config: str,
+                   inputs: dict,
+                   context: Optional[str]) -> List[PromptMessage]:
+        raw_prompt = app_model_config.completion_prompt_config_dict['prompt']['text']
+
+        prompt_messages = []
+        prompt = ''
+        
+        prompt_template = PromptTemplateParser(template=raw_prompt)
+        prompt_inputs = {k: inputs[k] for k in prompt_template.variable_keys if k in inputs}
+
+        self._set_context_variable(context, prompt_template, prompt_inputs)
+
+        prompt = self._format_prompt(prompt_template, prompt_inputs)
+
+        prompt_messages.append(PromptMessage(type = MessageType(MessageType.USER) ,content=prompt))
+
+        return prompt_messages
+
+    def _get_completion_app_chat_model_prompt_messages(self,
+                   app_model_config: str,
+                   inputs: dict,
+                   context: Optional[str]) -> List[PromptMessage]:
+        raw_prompt_list = app_model_config.chat_prompt_config_dict['prompt']
+
+        prompt_messages = []
+
+        for prompt_item in raw_prompt_list:
+            raw_prompt = prompt_item['text']
+            prompt = ''
+
+            prompt_template = PromptTemplateParser(template=raw_prompt)
+            prompt_inputs = {k: inputs[k] for k in prompt_template.variable_keys if k in inputs}
+
+            self._set_context_variable(context, prompt_template, prompt_inputs)
+
+            prompt = self._format_prompt(prompt_template, prompt_inputs)
+
+            prompt_messages.append(PromptMessage(type = MessageType(prompt_item['role']) ,content=prompt))
+        
+        return prompt_messages

api/core/third_party/langchain/embeddings/xinference_embedding.py

@@ -1,21 +1,54 @@
-from typing import List
+from typing import List, Optional, Any
 
 import numpy as np
-from langchain.embeddings import XinferenceEmbeddings
+from langchain.embeddings.base import Embeddings
+from xinference_client.client.restful.restful_client import Client
 
 
-class XinferenceEmbedding(XinferenceEmbeddings):
+class XinferenceEmbeddings(Embeddings):
+    client: Any
+    server_url: Optional[str]
+    """URL of the xinference server"""
+    model_uid: Optional[str]
+    """UID of the launched model"""
+
+    def __init__(
+            self, server_url: Optional[str] = None, model_uid: Optional[str] = None
+    ):
+
+        super().__init__()
+
+        if server_url is None:
+            raise ValueError("Please provide server URL")
+
+        if model_uid is None:
+            raise ValueError("Please provide the model UID")
+
+        self.server_url = server_url
+
+        self.model_uid = model_uid
+
+        self.client = Client(server_url)
 
     def embed_documents(self, texts: List[str]) -> List[List[float]]:
-        vectors = super().embed_documents(texts)
+        model = self.client.get_model(self.model_uid)
 
+        embeddings = [
+            model.create_embedding(text)["data"][0]["embedding"] for text in texts
+        ]
+        vectors = [list(map(float, e)) for e in embeddings]
         normalized_vectors = [(vector / np.linalg.norm(vector)).tolist() for vector in vectors]
 
         return normalized_vectors
 
     def embed_query(self, text: str) -> List[float]:
-        vector = super().embed_query(text)
+        model = self.client.get_model(self.model_uid)
 
+        embedding_res = model.create_embedding(text)
+
+        embedding = embedding_res["data"][0]["embedding"]
+
+        vector = list(map(float, embedding))
         normalized_vector = (vector / np.linalg.norm(vector)).tolist()
 
         return normalized_vector

api/core/third_party/langchain/llms/minimax_llm.py

@@ -211,6 +211,16 @@ class MinimaxChatLLM(BaseChatModel):
 
                 token = token.lstrip("data:").strip()
                 data = json.loads(token)
+
+                if "base_resp" in data and data["base_resp"]["status_code"] > 0:
+                    raise ValueError(
+                        f"API {data['base_resp']['status_code']}"
+                        f" error: {data['base_resp']['status_msg']}"
+                    )
+
+                if not data['choices']:
+                    continue
+
                 content = data['choices'][0]['delta']
 
                 chunk_kwargs = {

api/core/third_party/langchain/llms/wenxin.py

@@ -8,12 +8,15 @@ from typing import (
     Any,
     Dict,
     List,
-    Optional, Iterator,
+    Optional, Iterator, Tuple,
 )
 
 import requests
+from langchain.chat_models.base import BaseChatModel
 from langchain.llms.utils import enforce_stop_tokens
-from langchain.schema.output import GenerationChunk
+from langchain.schema import BaseMessage, ChatMessage, HumanMessage, AIMessage, SystemMessage
+from langchain.schema.messages import AIMessageChunk
+from langchain.schema.output import GenerationChunk, ChatResult, ChatGenerationChunk, ChatGeneration
 from pydantic import BaseModel, Extra, Field, PrivateAttr, root_validator
 
 from langchain.callbacks.manager import (
@@ -61,6 +64,7 @@ class _WenxinEndpointClient(BaseModel):
             raise ValueError(f"Wenxin Model name is required")
 
         model_url_map = {
+            'ernie-bot-4': 'completions_pro',
             'ernie-bot': 'completions',
             'ernie-bot-turbo': 'eb-instant',
             'bloomz-7b': 'bloomz_7b1',
@@ -70,6 +74,7 @@ class _WenxinEndpointClient(BaseModel):
 
         access_token = self.get_access_token()
         api_url = f"{self.base_url}{model_url_map[request['model']]}?access_token={access_token}"
+        del request['model']
 
         headers = {"Content-Type": "application/json"}
         response = requests.post(api_url,
@@ -86,22 +91,21 @@ class _WenxinEndpointClient(BaseModel):
                     f"Wenxin API {json_response['error_code']}"
                     f" error: {json_response['error_msg']}"
                 )
-            return json_response["result"]
+            return json_response
         else:
             return response
 
 
-class Wenxin(LLM):
-    """Wrapper around Wenxin large language models.
-    To use, you should have the environment variable
-    ``WENXIN_API_KEY`` and ``WENXIN_SECRET_KEY`` set with your API key,
-    or pass them as a named parameter to the constructor.
-    Example:
-     .. code-block:: python
-         from langchain.llms.wenxin import Wenxin
-         wenxin = Wenxin(model="<model_name>", api_key="my-api-key",
-          secret_key="my-group-id")
-    """
+class Wenxin(BaseChatModel):
+    """Wrapper around Wenxin large language models."""
+
+    @property
+    def lc_secrets(self) -> Dict[str, str]:
+        return {"api_key": "API_KEY", "secret_key": "SECRET_KEY"}
+
+    @property
+    def lc_serializable(self) -> bool:
+        return True
 
     _client: _WenxinEndpointClient = PrivateAttr()
     model: str = "ernie-bot"
@@ -161,64 +165,89 @@ class Wenxin(LLM):
             secret_key=self.secret_key,
         )
 
-    def _call(
+    def _convert_message_to_dict(self, message: BaseMessage) -> dict:
+        if isinstance(message, ChatMessage):
+            message_dict = {"role": message.role, "content": message.content}
+        elif isinstance(message, HumanMessage):
+            message_dict = {"role": "user", "content": message.content}
+        elif isinstance(message, AIMessage):
+            message_dict = {"role": "assistant", "content": message.content}
+        elif isinstance(message, SystemMessage):
+            message_dict = {"role": "system", "content": message.content}
+        else:
+            raise ValueError(f"Got unknown type {message}")
+        return message_dict
+
+    def _create_message_dicts(
+        self, messages: List[BaseMessage]
+    ) -> Tuple[List[Dict[str, Any]], str]:
+        dict_messages = []
+        system = None
+        for m in messages:
+            message = self._convert_message_to_dict(m)
+            if message['role'] == 'system':
+                if not system:
+                    system = message['content']
+                else:
+                    system += f"\n{message['content']}"
+                continue
+
+            if dict_messages:
+                previous_message = dict_messages[-1]
+                if previous_message['role'] == message['role']:
+                    dict_messages[-1]['content'] += f"\n{message['content']}"
+                else:
+                    dict_messages.append(message)
+            else:
+                dict_messages.append(message)
+
+        return dict_messages, system
+
+    def _generate(
         self,
-        prompt: str,
+        messages: List[BaseMessage],
         stop: Optional[List[str]] = None,
         run_manager: Optional[CallbackManagerForLLMRun] = None,
         **kwargs: Any,
-    ) -> str:
-        r"""Call out to Wenxin's completion endpoint to chat
-        Args:
-            prompt: The prompt to pass into the model.
-        Returns:
-            The string generated by the model.
-        Example:
-            .. code-block:: python
-                response = wenxin("Tell me a joke.")
-        """
+    ) -> ChatResult:
         if self.streaming:
-            completion = ""
+            generation: Optional[ChatGenerationChunk] = None
+            llm_output: Optional[Dict] = None
             for chunk in self._stream(
-                prompt=prompt, stop=stop, run_manager=run_manager, **kwargs
+                    messages=messages, stop=stop, run_manager=run_manager, **kwargs
             ):
-                completion += chunk.text
+                if chunk.generation_info is not None \
+                        and 'token_usage' in chunk.generation_info:
+                    llm_output = {"token_usage": chunk.generation_info['token_usage'], "model_name": self.model}
+
+                if generation is None:
+                    generation = chunk
+                else:
+                    generation += chunk
+            assert generation is not None
+            return ChatResult(generations=[generation], llm_output=llm_output)
         else:
+            message_dicts, system = self._create_message_dicts(messages)
             request = self._default_params
-            request["messages"] = [{"role": "user", "content": prompt}]
+            request["messages"] = message_dicts
+            if system:
+                request["system"] = system
             request.update(kwargs)
-            completion = self._client.post(request)
-
-        if stop is not None:
-            completion = enforce_stop_tokens(completion, stop)
-
-        return completion
+            response = self._client.post(request)
+            return self._create_chat_result(response)
 
     def _stream(
-        self,
-        prompt: str,
-        stop: Optional[List[str]] = None,
-        run_manager: Optional[CallbackManagerForLLMRun] = None,
-        **kwargs: Any,
-    ) -> Iterator[GenerationChunk]:
-        r"""Call wenxin completion_stream and return the resulting generator.
-
-        Args:
-            prompt: The prompt to pass into the model.
-            stop: Optional list of stop words to use when generating.
-        Returns:
-            A generator representing the stream of tokens from Wenxin.
-        Example:
-            .. code-block:: python
-
-                prompt = "Write a poem about a stream."
-                prompt = f"\n\nHuman: {prompt}\n\nAssistant:"
-                generator = wenxin.stream(prompt)
-                for token in generator:
-                    yield token
-        """
+            self,
+            messages: List[BaseMessage],
+            stop: Optional[List[str]] = None,
+            run_manager: Optional[CallbackManagerForLLMRun] = None,
+            **kwargs: Any,
+    ) -> Iterator[ChatGenerationChunk]:
+        message_dicts, system = self._create_message_dicts(messages)
         request = self._default_params
-        request["messages"] = [{"role": "user", "content": prompt}]
+        request["messages"] = message_dicts
+        if system:
+            request["system"] = system
         request.update(kwargs)
 
         for token in self._client.post(request).iter_lines():
@@ -228,12 +257,18 @@ class Wenxin(LLM):
                 if token.startswith('data:'):
                     completion = json.loads(token[5:])
 
-                    yield GenerationChunk(text=completion['result'])
-                    if run_manager:
-                        run_manager.on_llm_new_token(completion['result'])
+                    chunk_dict = {
+                        'message': AIMessageChunk(content=completion['result']),
+                    }
 
                     if completion['is_end']:
-                        break
+                        token_usage = completion['usage']
+                        token_usage['completion_tokens'] = token_usage['total_tokens'] - token_usage['prompt_tokens']
+                        chunk_dict['generation_info'] = dict({'token_usage': token_usage})
+
+                    yield ChatGenerationChunk(**chunk_dict)
+                    if run_manager:
+                        run_manager.on_llm_new_token(completion['result'])
                 else:
                     try:
                         json_response = json.loads(token)
@@ -245,3 +280,40 @@ class Wenxin(LLM):
                         f" error: {json_response['error_msg']}, "
                         f"please confirm if the model you have chosen is already paid for."
                     )
+
+    def _create_chat_result(self, response: Dict[str, Any]) -> ChatResult:
+        generations = [ChatGeneration(
+            message=AIMessage(content=response['result']),
+        )]
+        token_usage = response.get("usage")
+        token_usage['completion_tokens'] = token_usage['total_tokens'] - token_usage['prompt_tokens']
+
+        llm_output = {"token_usage": token_usage, "model_name": self.model}
+        return ChatResult(generations=generations, llm_output=llm_output)
+
+    def get_num_tokens_from_messages(self, messages: List[BaseMessage]) -> int:
+        """Get the number of tokens in the messages.
+
+        Useful for checking if an input will fit in a model's context window.
+
+        Args:
+            messages: The message inputs to tokenize.
+
+        Returns:
+            The sum of the number of tokens across the messages.
+        """
+        return sum([self.get_num_tokens(m.content) for m in messages])
+
+    def _combine_llm_outputs(self, llm_outputs: List[Optional[dict]]) -> dict:
+        overall_token_usage: dict = {}
+        for output in llm_outputs:
+            if output is None:
+                # Happens in streaming
+                continue
+            token_usage = output["token_usage"]
+            for k, v in token_usage.items():
+                if k in overall_token_usage:
+                    overall_token_usage[k] += v
+                else:
+                    overall_token_usage[k] = v
+        return {"token_usage": overall_token_usage, "model_name": self.model}

api/core/third_party/langchain/llms/xinference_llm.py

@@ -1,16 +1,53 @@
-from typing import Optional, List, Any, Union, Generator
+from typing import Optional, List, Any, Union, Generator, Mapping
 
 from langchain.callbacks.manager import CallbackManagerForLLMRun
-from langchain.llms import Xinference
+from langchain.llms.base import LLM
 from langchain.llms.utils import enforce_stop_tokens
-from xinference.client import (
+from xinference_client.client.restful.restful_client import (
     RESTfulChatglmCppChatModelHandle,
     RESTfulChatModelHandle,
-    RESTfulGenerateModelHandle,
+    RESTfulGenerateModelHandle, Client,
 )
 
 
-class XinferenceLLM(Xinference):
+class XinferenceLLM(LLM):
+    client: Any
+    server_url: Optional[str]
+    """URL of the xinference server"""
+    model_uid: Optional[str]
+    """UID of the launched model"""
+
+    def __init__(
+            self, server_url: Optional[str] = None, model_uid: Optional[str] = None
+    ):
+        super().__init__(
+            **{
+                "server_url": server_url,
+                "model_uid": model_uid,
+            }
+        )
+
+        if self.server_url is None:
+            raise ValueError("Please provide server URL")
+
+        if self.model_uid is None:
+            raise ValueError("Please provide the model UID")
+
+        self.client = Client(server_url)
+
+    @property
+    def _llm_type(self) -> str:
+        """Return type of llm."""
+        return "xinference"
+
+    @property
+    def _identifying_params(self) -> Mapping[str, Any]:
+        """Get the identifying parameters."""
+        return {
+            **{"server_url": self.server_url},
+            **{"model_uid": self.model_uid},
+        }
+
     def _call(
         self,
         prompt: str,

api/core/third_party/langchain/llms/zhipuai_llm.py

@@ -96,7 +96,7 @@ class ZhipuAIChatLLM(BaseChatModel):
         return True
 
     client: Any = None  #: :meta private:
-    model: str = "chatglm_lite"
+    model: str = "chatglm_turbo"
     """Model name to use."""
     temperature: float = 0.95
     """A non-negative float that tunes the degree of randomness in generation."""

api/core/third_party/spark/spark_llm.py

@@ -19,9 +19,25 @@ class SparkLLMClient:
     def __init__(self, model_name: str, app_id: str, api_key: str, api_secret: str, api_domain: Optional[str] = None):
 
         domain = 'spark-api.xf-yun.com' if not api_domain else api_domain
-        api_version = 'v2.1' if model_name == 'spark-v2' else 'v1.1'
 
-        self.chat_domain = 'generalv2' if model_name == 'spark-v2' else 'general'
+        model_api_configs = {
+            'spark': {
+                'version': 'v1.1',
+                'chat_domain': 'general'
+            },
+            'spark-v2': {
+                'version': 'v2.1',
+                'chat_domain': 'generalv2'
+            },
+            'spark-v3': {
+                'version': 'v3.1',
+                'chat_domain': 'generalv3'
+            }
+        }
+
+        api_version = model_api_configs[model_name]['version']
+
+        self.chat_domain = model_api_configs[model_name]['chat_domain']
         self.api_base = f"wss://{domain}/{api_version}/chat"
         self.app_id = app_id
         self.ws_url = self.create_url(

api/extensions/ext_code_based_extension.py

@@ -0,0 +1,8 @@
+from core.extension.extension import Extension
+
+
+def init():
+    code_based_extension.init()
+
+
+code_based_extension = Extension()

api/fields/api_based_extension_fields.py

@@ -0,0 +1,17 @@
+from flask_restful import fields
+
+from libs.helper import TimestampField
+
+
+class HiddenAPIKey(fields.Raw):
+    def output(self, key, obj):
+        return obj.api_key[:3] + '***' + obj.api_key[-3:]
+
+
+api_based_extension_fields = {
+    'id': fields.String,
+    'name': fields.String,
+    'api_endpoint': fields.String,
+    'api_key': HiddenAPIKey,
+    'created_at': TimestampField
+}

api/fields/app_fields.py

@@ -23,6 +23,7 @@ model_config_fields = {
     'retriever_resource': fields.Raw(attribute='retriever_resource_dict'),
     'more_like_this': fields.Raw(attribute='more_like_this_dict'),
     'sensitive_word_avoidance': fields.Raw(attribute='sensitive_word_avoidance_dict'),
+    'external_data_tools': fields.Raw(attribute='external_data_tools_list'),
     'model': fields.Raw(attribute='model_dict'),
     'user_input_form': fields.Raw(attribute='user_input_form_list'),
     'dataset_query_variable': fields.String,

api/migrations/versions/968fff4c0ab9_add_api_based_extension.py

@@ -0,0 +1,45 @@
+"""add_api_based_extension
+
+Revision ID: 968fff4c0ab9
+Revises: b3a09c049e8e
+Create Date: 2023-10-27 13:05:58.901858
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = '968fff4c0ab9'
+down_revision = 'b3a09c049e8e'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+
+    op.create_table('api_based_extensions',
+    sa.Column('id', postgresql.UUID(), server_default=sa.text('uuid_generate_v4()'), nullable=False),
+    sa.Column('tenant_id', postgresql.UUID(), nullable=False),
+    sa.Column('name', sa.String(length=255), nullable=False),
+    sa.Column('api_endpoint', sa.String(length=255), nullable=False),
+    sa.Column('api_key', sa.Text(), nullable=False),
+    sa.Column('created_at', sa.DateTime(), server_default=sa.text('CURRENT_TIMESTAMP(0)'), nullable=False),
+    sa.PrimaryKeyConstraint('id', name='api_based_extension_pkey')
+    )
+    with op.batch_alter_table('api_based_extensions', schema=None) as batch_op:
+        batch_op.create_index('api_based_extension_tenant_idx', ['tenant_id'], unique=False)
+
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+
+    with op.batch_alter_table('api_based_extensions', schema=None) as batch_op:
+        batch_op.drop_index('api_based_extension_tenant_idx')
+
+    op.drop_table('api_based_extensions')
+
+    # ### end Alembic commands ###

api/migrations/versions/a9836e3baeee_add_external_data_tools_in_app_model_.py

@@ -0,0 +1,32 @@
+"""add external_data_tools in app model config
+
+Revision ID: a9836e3baeee
+Revises: 968fff4c0ab9
+Create Date: 2023-11-02 04:04:57.609485
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = 'a9836e3baeee'
+down_revision = '968fff4c0ab9'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('app_model_configs', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('external_data_tools', sa.Text(), nullable=True))
+
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('app_model_configs', schema=None) as batch_op:
+        batch_op.drop_column('external_data_tools')
+
+    # ### end Alembic commands ###

api/models/api_based_extension.py

@@ -0,0 +1,27 @@
+import enum
+
+from sqlalchemy.dialects.postgresql import UUID
+
+from extensions.ext_database import db
+
+
+class APIBasedExtensionPoint(enum.Enum):
+    APP_EXTERNAL_DATA_TOOL_QUERY = 'app.external_data_tool.query'
+    PING = 'ping'
+    APP_MODERATION_INPUT = 'app.moderation.input'
+    APP_MODERATION_OUTPUT = 'app.moderation.output'
+
+
+class APIBasedExtension(db.Model):
+    __tablename__ = 'api_based_extensions'
+    __table_args__ = (
+        db.PrimaryKeyConstraint('id', name='api_based_extension_pkey'),
+        db.Index('api_based_extension_tenant_idx', 'tenant_id'),
+    )
+
+    id = db.Column(UUID, server_default=db.text('uuid_generate_v4()'))
+    tenant_id = db.Column(UUID, nullable=False)
+    name = db.Column(db.String(255), nullable=False)
+    api_endpoint = db.Column(db.String(255), nullable=False)
+    api_key = db.Column(db.Text, nullable=False)
+    created_at = db.Column(db.DateTime, nullable=False, server_default=db.text('CURRENT_TIMESTAMP(0)'))

api/models/model.py

@@ -97,6 +97,7 @@ class AppModelConfig(db.Model):
     chat_prompt_config = db.Column(db.Text)
     completion_prompt_config = db.Column(db.Text)
     dataset_configs = db.Column(db.Text)
+    external_data_tools = db.Column(db.Text)
 
     @property
     def app(self):
@@ -133,7 +134,12 @@ class AppModelConfig(db.Model):
     @property
     def sensitive_word_avoidance_dict(self) -> dict:
         return json.loads(self.sensitive_word_avoidance) if self.sensitive_word_avoidance \
-            else {"enabled": False, "words": [], "canned_response": []}
+            else {"enabled": False, "type": "", "configs": []}
+
+    @property
+    def external_data_tools_list(self) -> list[dict]:
+        return json.loads(self.external_data_tools) if self.external_data_tools \
+            else []
 
     @property
     def user_input_form_list(self) -> dict:
@@ -167,6 +173,7 @@ class AppModelConfig(db.Model):
             "retriever_resource": self.retriever_resource_dict,
             "more_like_this": self.more_like_this_dict,
             "sensitive_word_avoidance": self.sensitive_word_avoidance_dict,
+            "external_data_tools": self.external_data_tools_list,
             "model": self.model_dict,
             "user_input_form": self.user_input_form_list,
             "dataset_query_variable": self.dataset_query_variable,
@@ -190,6 +197,7 @@ class AppModelConfig(db.Model):
         self.more_like_this = json.dumps(model_config['more_like_this'])
         self.sensitive_word_avoidance = json.dumps(model_config['sensitive_word_avoidance']) \
             if model_config.get('sensitive_word_avoidance') else None
+        self.external_data_tools = json.dumps(model_config['external_data_tools'])
         self.model = json.dumps(model_config['model'])
         self.user_input_form = json.dumps(model_config['user_input_form'])
         self.dataset_query_variable = model_config.get('dataset_query_variable')
@@ -219,6 +227,7 @@ class AppModelConfig(db.Model):
             speech_to_text=self.speech_to_text,
             more_like_this=self.more_like_this,
             sensitive_word_avoidance=self.sensitive_word_avoidance,
+            external_data_tools=self.external_data_tools,
             model=self.model,
             user_input_form=self.user_input_form,
             dataset_query_variable=self.dataset_query_variable,
@@ -332,41 +341,16 @@ class Conversation(db.Model):
             override_model_configs = json.loads(self.override_model_configs)
 
             if 'model' in override_model_configs:
-                model_config['model'] = override_model_configs['model']
-                model_config['pre_prompt'] = override_model_configs['pre_prompt']
-                model_config['agent_mode'] = override_model_configs['agent_mode']
-                model_config['opening_statement'] = override_model_configs['opening_statement']
-                model_config['suggested_questions'] = override_model_configs['suggested_questions']
-                model_config['suggested_questions_after_answer'] = override_model_configs[
-                    'suggested_questions_after_answer'] \
-                    if 'suggested_questions_after_answer' in override_model_configs else {"enabled": False}
-                model_config['speech_to_text'] = override_model_configs[
-                    'speech_to_text'] \
-                    if 'speech_to_text' in override_model_configs else {"enabled": False}
-                model_config['more_like_this'] = override_model_configs['more_like_this'] \
-                    if 'more_like_this' in override_model_configs else {"enabled": False}
-                model_config['sensitive_word_avoidance'] = override_model_configs['sensitive_word_avoidance'] \
-                    if 'sensitive_word_avoidance' in override_model_configs \
-                    else {"enabled": False, "words": [], "canned_response": []}
-                model_config['user_input_form'] = override_model_configs['user_input_form']
+                app_model_config = AppModelConfig()
+                app_model_config = app_model_config.from_model_config_dict(override_model_configs)
+                model_config = app_model_config.to_dict()
             else:
                 model_config['configs'] = override_model_configs
         else:
             app_model_config = db.session.query(AppModelConfig).filter(
                 AppModelConfig.id == self.app_model_config_id).first()
 
-            model_config['configs'] = app_model_config.configs
-            model_config['model'] = app_model_config.model_dict
-            model_config['pre_prompt'] = app_model_config.pre_prompt
-            model_config['agent_mode'] = app_model_config.agent_mode_dict
-            model_config['opening_statement'] = app_model_config.opening_statement
-            model_config['suggested_questions'] = app_model_config.suggested_questions_list
-            model_config['suggested_questions_after_answer'] = app_model_config.suggested_questions_after_answer_dict
-            model_config['speech_to_text'] = app_model_config.speech_to_text_dict
-            model_config['retriever_resource'] = app_model_config.retriever_resource_dict
-            model_config['more_like_this'] = app_model_config.more_like_this_dict
-            model_config['sensitive_word_avoidance'] = app_model_config.sensitive_word_avoidance_dict
-            model_config['user_input_form'] = app_model_config.user_input_form_list
+            model_config = app_model_config.to_dict()
 
         model_config['model_id'] = self.model_id
         model_config['provider'] = self.model_provider

api/requirements.txt

@@ -49,7 +49,7 @@ huggingface_hub~=0.16.4
 transformers~=4.31.0
 stripe~=5.5.0
 pandas==1.5.3
-xinference==0.5.2
+xinference-client~=0.5.4
 safetensors==0.3.2
 zhipuai==1.0.7
 werkzeug==2.3.7

api/services/account_service.py

@@ -6,7 +6,7 @@ import secrets
 import uuid
 from datetime import datetime, timedelta
 from hashlib import sha256
-from typing import Optional
+from typing import Optional, Dict, Any
 
 from werkzeug.exceptions import Forbidden, Unauthorized
 from flask import session, current_app
@@ -24,7 +24,8 @@ from libs.passport import PassportService
 from models.account import *
 from tasks.mail_invite_member_task import send_invite_member_mail_task
 
-def _create_tenant_for_account(account):
+
+def _create_tenant_for_account(account) -> Tenant:
     tenant = TenantService.create_tenant(f"{account.name}'s Workspace")
 
     TenantService.create_tenant_member(tenant, account, role='owner')
@@ -505,7 +506,7 @@ class RegisterService:
             redis_client.delete(cls._get_invitation_token_key(token))
 
     @classmethod
-    def get_invitation_if_token_valid(cls, workspace_id: str, email: str, token: str) -> Optional[Account]:
+    def get_invitation_if_token_valid(cls, workspace_id: str, email: str, token: str) -> Optional[Dict[str, Any]]:
         invitation_data = cls._get_invitation_by_token(token, workspace_id, email)
         if not invitation_data:
             return None
@@ -539,7 +540,7 @@ class RegisterService:
                 }
 
     @classmethod
-    def _get_invitation_by_token(cls, token: str, workspace_id: str, email: str) -> Optional[str]:
+    def _get_invitation_by_token(cls, token: str, workspace_id: str, email: str) -> Optional[Dict[str, str]]:
         if workspace_id is not None and email is not None:
             email_hash = sha256(email.encode()).hexdigest()
             cache_key = f'member_invite_token:{workspace_id}, {email_hash}:{token}'

api/services/advanced_prompt_template_service.py

@@ -1,56 +1,65 @@
 
 import copy
 
+from core.model_providers.models.entity.model_params import ModelMode
+from core.prompt.prompt_transform import AppMode
 from core.prompt.advanced_prompt_templates import CHAT_APP_COMPLETION_PROMPT_CONFIG, CHAT_APP_CHAT_PROMPT_CONFIG, COMPLETION_APP_CHAT_PROMPT_CONFIG, COMPLETION_APP_COMPLETION_PROMPT_CONFIG, \
     BAICHUAN_CHAT_APP_COMPLETION_PROMPT_CONFIG, BAICHUAN_CHAT_APP_CHAT_PROMPT_CONFIG, BAICHUAN_COMPLETION_APP_COMPLETION_PROMPT_CONFIG, BAICHUAN_COMPLETION_APP_CHAT_PROMPT_CONFIG, CONTEXT, BAICHUAN_CONTEXT
 
 class AdvancedPromptTemplateService:
 
-    def get_prompt(self, args: dict) -> dict:
+    @classmethod
+    def get_prompt(cls, args: dict) -> dict:
         app_mode = args['app_mode']
         model_mode = args['model_mode']
         model_name = args['model_name']
         has_context = args['has_context']
 
-        if 'baichuan' in model_name:
-            return self.get_baichuan_prompt(app_mode, model_mode, has_context)
+        if 'baichuan' in model_name.lower():
+            return cls.get_baichuan_prompt(app_mode, model_mode, has_context)
         else:
-            return self.get_common_prompt(app_mode, model_mode, has_context)
+            return cls.get_common_prompt(app_mode, model_mode, has_context)
 
-    def get_common_prompt(self, app_mode: str, model_mode:str, has_context: bool) -> dict:
-        if app_mode == 'chat':
-            if model_mode == 'completion':
-                return self.get_completion_prompt(copy.deepcopy(CHAT_APP_COMPLETION_PROMPT_CONFIG), has_context, CONTEXT)
-            elif model_mode == 'chat':
-                return self.get_chat_prompt(copy.deepcopy(CHAT_APP_CHAT_PROMPT_CONFIG), has_context, CONTEXT)
-        elif app_mode == 'completion':
-            if model_mode == 'completion':
-                return self.get_completion_prompt(copy.deepcopy(COMPLETION_APP_COMPLETION_PROMPT_CONFIG), has_context, CONTEXT)
-            elif model_mode == 'chat':
-                return self.get_chat_prompt(copy.deepcopy(COMPLETION_APP_CHAT_PROMPT_CONFIG), has_context, CONTEXT)
+    @classmethod
+    def get_common_prompt(cls, app_mode: str, model_mode:str, has_context: str) -> dict:
+        context_prompt = copy.deepcopy(CONTEXT)
+
+        if app_mode == AppMode.CHAT.value:
+            if model_mode == ModelMode.COMPLETION.value:
+                return cls.get_completion_prompt(copy.deepcopy(CHAT_APP_COMPLETION_PROMPT_CONFIG), has_context, context_prompt)
+            elif model_mode == ModelMode.CHAT.value:
+                return cls.get_chat_prompt(copy.deepcopy(CHAT_APP_CHAT_PROMPT_CONFIG), has_context, context_prompt)
+        elif app_mode == AppMode.COMPLETION.value:
+            if model_mode == ModelMode.COMPLETION.value:
+                return cls.get_completion_prompt(copy.deepcopy(COMPLETION_APP_COMPLETION_PROMPT_CONFIG), has_context, context_prompt)
+            elif model_mode == ModelMode.CHAT.value:
+                return cls.get_chat_prompt(copy.deepcopy(COMPLETION_APP_CHAT_PROMPT_CONFIG), has_context, context_prompt)
             
-    def get_completion_prompt(self, prompt_template: str, has_context: bool, context: str) -> dict:
+    @classmethod
+    def get_completion_prompt(cls, prompt_template: dict, has_context: str, context: str) -> dict:
         if has_context == 'true':
             prompt_template['completion_prompt_config']['prompt']['text'] = context + prompt_template['completion_prompt_config']['prompt']['text']
         
         return prompt_template
 
-
-    def get_chat_prompt(self, prompt_template: str, has_context: bool, context: str) -> dict:
+    @classmethod
+    def get_chat_prompt(cls, prompt_template: dict, has_context: str, context: str) -> dict:
         if has_context == 'true':
             prompt_template['chat_prompt_config']['prompt'][0]['text'] = context + prompt_template['chat_prompt_config']['prompt'][0]['text']
         
         return prompt_template
 
+    @classmethod
+    def get_baichuan_prompt(cls, app_mode: str, model_mode:str, has_context: str) -> dict:
+        baichuan_context_prompt = copy.deepcopy(BAICHUAN_CONTEXT)
 
-    def get_baichuan_prompt(self, app_mode: str, model_mode:str, has_context: bool) -> dict:
-        if app_mode == 'chat':
-            if model_mode == 'completion':
-                return self.get_completion_prompt(copy.deepcopy(BAICHUAN_CHAT_APP_COMPLETION_PROMPT_CONFIG), has_context, BAICHUAN_CONTEXT)
-            elif model_mode == 'chat':
-                return self.get_chat_prompt(copy.deepcopy(BAICHUAN_CHAT_APP_CHAT_PROMPT_CONFIG), has_context, BAICHUAN_CONTEXT)
-        elif app_mode == 'completion':
-            if model_mode == 'completion':
-                return self.get_completion_prompt(copy.deepcopy(BAICHUAN_COMPLETION_APP_COMPLETION_PROMPT_CONFIG), has_context, BAICHUAN_CONTEXT)
-            elif model_mode == 'chat':
-                return self.get_chat_prompt(copy.deepcopy(BAICHUAN_COMPLETION_APP_CHAT_PROMPT_CONFIG), has_context, BAICHUAN_CONTEXT)
+        if app_mode == AppMode.CHAT.value:
+            if model_mode == ModelMode.COMPLETION.value:
+                return cls.get_completion_prompt(copy.deepcopy(BAICHUAN_CHAT_APP_COMPLETION_PROMPT_CONFIG), has_context, baichuan_context_prompt)
+            elif model_mode == ModelMode.CHAT.value:
+                return cls.get_chat_prompt(copy.deepcopy(BAICHUAN_CHAT_APP_CHAT_PROMPT_CONFIG), has_context, baichuan_context_prompt)
+        elif app_mode == AppMode.COMPLETION.value:
+            if model_mode == ModelMode.COMPLETION.value:
+                return cls.get_completion_prompt(copy.deepcopy(BAICHUAN_COMPLETION_APP_COMPLETION_PROMPT_CONFIG), has_context, baichuan_context_prompt)
+            elif model_mode == ModelMode.CHAT.value:
+                return cls.get_chat_prompt(copy.deepcopy(BAICHUAN_COMPLETION_APP_CHAT_PROMPT_CONFIG), has_context, baichuan_context_prompt)

api/services/api_based_extension_service.py

@@ -0,0 +1,98 @@
+from extensions.ext_database import db
+from models.api_based_extension import APIBasedExtension, APIBasedExtensionPoint
+from core.helper.encrypter import encrypt_token, decrypt_token
+from core.extension.api_based_extension_requestor import APIBasedExtensionRequestor
+
+
+class APIBasedExtensionService:
+
+    @staticmethod
+    def get_all_by_tenant_id(tenant_id: str) -> list[APIBasedExtension]:
+        extension_list = db.session.query(APIBasedExtension) \
+                    .filter_by(tenant_id=tenant_id) \
+                    .order_by(APIBasedExtension.created_at.desc()) \
+                    .all()
+
+        for extension in extension_list:
+            extension.api_key = decrypt_token(extension.tenant_id, extension.api_key)
+
+        return extension_list
+
+    @classmethod
+    def save(cls, extension_data: APIBasedExtension) -> APIBasedExtension:
+        cls._validation(extension_data)
+
+        extension_data.api_key = encrypt_token(extension_data.tenant_id, extension_data.api_key)
+
+        db.session.add(extension_data)
+        db.session.commit()
+        return extension_data
+
+    @staticmethod
+    def delete(extension_data: APIBasedExtension) -> None:
+        db.session.delete(extension_data)
+        db.session.commit()
+
+    @staticmethod
+    def get_with_tenant_id(tenant_id: str, api_based_extension_id: str) -> APIBasedExtension:
+        extension = db.session.query(APIBasedExtension) \
+            .filter_by(tenant_id=tenant_id) \
+            .filter_by(id=api_based_extension_id) \
+            .first()
+
+        if not extension:
+            raise ValueError("API based extension is not found")
+
+        extension.api_key = decrypt_token(extension.tenant_id, extension.api_key)
+
+        return extension
+
+    @classmethod
+    def _validation(cls, extension_data: APIBasedExtension) -> None:
+        # name
+        if not extension_data.name:
+            raise ValueError("name must not be empty")
+
+        if not extension_data.id:
+            # case one: check new data, name must be unique
+            is_name_existed = db.session.query(APIBasedExtension) \
+                .filter_by(tenant_id=extension_data.tenant_id) \
+                .filter_by(name=extension_data.name) \
+                .first()
+
+            if is_name_existed:
+                raise ValueError("name must be unique, it is already existed")
+        else:
+            # case two: check existing data, name must be unique
+            is_name_existed = db.session.query(APIBasedExtension) \
+                .filter_by(tenant_id=extension_data.tenant_id) \
+                .filter_by(name=extension_data.name) \
+                .filter(APIBasedExtension.id != extension_data.id) \
+                .first()
+
+            if is_name_existed:
+                raise ValueError("name must be unique, it is already existed")
+
+        # api_endpoint
+        if not extension_data.api_endpoint:
+            raise ValueError("api_endpoint must not be empty")
+
+        # api_key
+        if not extension_data.api_key:
+            raise ValueError("api_key must not be empty")
+
+        if len(extension_data.api_key) < 5:
+            raise ValueError("api_key must be at least 5 characters")
+
+        # check endpoint
+        cls._ping_connection(extension_data)
+
+    @staticmethod
+    def _ping_connection(extension_data: APIBasedExtension) -> None:
+        try:
+            client = APIBasedExtensionRequestor(extension_data.api_endpoint, extension_data.api_key)
+            resp = client.request(point=APIBasedExtensionPoint.PING, params={})
+            if resp.get('result') != 'pong':
+                raise ValueError(resp)
+        except Exception as e:
+            raise ValueError("connection error: {}".format(e))

api/services/app_model_config_service.py

@@ -1,6 +1,9 @@
 import re
 import uuid
 
+from core.external_data_tool.factory import ExternalDataToolFactory
+from core.moderation.factory import ModerationFactory
+from core.prompt.prompt_transform import AppMode
 from core.agent.agent_executor import PlanningStrategy
 from core.model_providers.model_provider_factory import ModelProviderFactory
 from core.model_providers.models.entity.model_params import ModelType, ModelMode
@@ -12,8 +15,8 @@ SUPPORT_TOOLS = ["dataset", "google_search", "web_reader", "wikipedia", "current
 
 
 class AppModelConfigService:
-    @staticmethod
-    def is_dataset_exists(account: Account, dataset_id: str) -> bool:
+    @classmethod
+    def is_dataset_exists(cls, account: Account, dataset_id: str) -> bool:
         # verify if the dataset ID exists
         dataset = DatasetService.get_dataset(dataset_id)
 
@@ -25,8 +28,8 @@ class AppModelConfigService:
 
         return True
 
-    @staticmethod
-    def validate_model_completion_params(cp: dict, model_name: str) -> dict:
+    @classmethod
+    def validate_model_completion_params(cls, cp: dict, model_name: str) -> dict:
         # 6. model.completion_params
         if not isinstance(cp, dict):
             raise ValueError("model.completion_params must be of object type")
@@ -57,6 +60,9 @@ class AppModelConfigService:
         elif not isinstance(cp["stop"], list):
             raise ValueError("stop in model.completion_params must be of list type")
 
+        if len(cp["stop"]) > 4:
+            raise ValueError("stop sequences must be less than 4")
+
         # Filter out extra parameters
         filtered_cp = {
             "max_tokens": cp["max_tokens"],
@@ -69,8 +75,8 @@ class AppModelConfigService:
 
         return filtered_cp
 
-    @staticmethod
-    def validate_configuration(tenant_id: str, account: Account, config: dict, mode: str) -> dict:
+    @classmethod
+    def validate_configuration(cls, tenant_id: str, account: Account, config: dict, mode: str) -> dict:
         # opening_statement
         if 'opening_statement' not in config or not config["opening_statement"]:
             config["opening_statement"] = ""
@@ -149,33 +155,6 @@ class AppModelConfigService:
         if not isinstance(config["more_like_this"]["enabled"], bool):
             raise ValueError("enabled in more_like_this must be of boolean type")
 
-        # sensitive_word_avoidance
-        if 'sensitive_word_avoidance' not in config or not config["sensitive_word_avoidance"]:
-            config["sensitive_word_avoidance"] = {
-                "enabled": False
-            }
-
-        if not isinstance(config["sensitive_word_avoidance"], dict):
-            raise ValueError("sensitive_word_avoidance must be of dict type")
-
-        if "enabled" not in config["sensitive_word_avoidance"] or not config["sensitive_word_avoidance"]["enabled"]:
-            config["sensitive_word_avoidance"]["enabled"] = False
-
-        if not isinstance(config["sensitive_word_avoidance"]["enabled"], bool):
-            raise ValueError("enabled in sensitive_word_avoidance must be of boolean type")
-
-        if "words" not in config["sensitive_word_avoidance"] or not config["sensitive_word_avoidance"]["words"]:
-            config["sensitive_word_avoidance"]["words"] = ""
-
-        if not isinstance(config["sensitive_word_avoidance"]["words"], str):
-            raise ValueError("words in sensitive_word_avoidance must be of string type")
-
-        if "canned_response" not in config["sensitive_word_avoidance"] or not config["sensitive_word_avoidance"]["canned_response"]:
-            config["sensitive_word_avoidance"]["canned_response"] = ""
-
-        if not isinstance(config["sensitive_word_avoidance"]["canned_response"], str):
-            raise ValueError("canned_response in sensitive_word_avoidance must be of string type")
-
         # model
         if 'model' not in config:
             raise ValueError("model is required")
@@ -200,7 +179,7 @@ class AppModelConfigService:
         model_ids = [m['id'] for m in model_list]
         if config["model"]["name"] not in model_ids:
             raise ValueError("model.name must be in the specified model list")
-        
+
         # model.mode
         if 'mode' not in config['model'] or not config['model']["mode"]:
             config['model']["mode"] = ""
@@ -209,7 +188,7 @@ class AppModelConfigService:
         if 'completion_params' not in config["model"]:
             raise ValueError("model.completion_params is required")
 
-        config["model"]["completion_params"] = AppModelConfigService.validate_model_completion_params(
+        config["model"]["completion_params"] = cls.validate_model_completion_params(
             config["model"]["completion_params"],
             config["model"]["name"]
         )
@@ -271,11 +250,6 @@ class AppModelConfigService:
         if not isinstance(config["pre_prompt"], str):
             raise ValueError("pre_prompt must be of string type")
 
-        template_vars = re.findall(r"\{\{(\w+)\}\}", config["pre_prompt"])
-        for var in template_vars:
-            if var not in variables:
-                raise ValueError("Template variables in pre_prompt must be defined in user_input_form")
-
         # agent_mode
         if "agent_mode" not in config or not config["agent_mode"]:
             config["agent_mode"] = {
@@ -326,14 +300,20 @@ class AppModelConfigService:
                 except ValueError:
                     raise ValueError("id in dataset must be of UUID type")
 
-                if not AppModelConfigService.is_dataset_exists(account, tool_item["id"]):
+                if not cls.is_dataset_exists(account, tool_item["id"]):
                     raise ValueError("Dataset ID does not exist, please check your permission.")
-        
+
         # dataset_query_variable
-        AppModelConfigService.is_dataset_query_variable_valid(config, mode)
+        cls.is_dataset_query_variable_valid(config, mode)
 
         # advanced prompt validation
-        AppModelConfigService.is_advanced_prompt_valid(config, mode)
+        cls.is_advanced_prompt_valid(config, mode)
+
+        # external data tools validation
+        cls.is_external_data_tools_valid(tenant_id, config)
+
+        # moderation validation
+        cls.is_moderation_valid(tenant_id, config)
 
         # Filter out extra parameters
         filtered_config = {
@@ -344,6 +324,7 @@ class AppModelConfigService:
             "retriever_resource": config["retriever_resource"],
             "more_like_this": config["more_like_this"],
             "sensitive_word_avoidance": config["sensitive_word_avoidance"],
+            "external_data_tools": config["external_data_tools"],
             "model": {
                 "provider": config["model"]["provider"],
                 "name": config["model"]["name"],
@@ -361,32 +342,86 @@ class AppModelConfigService:
         }
 
         return filtered_config
-    
-    @staticmethod
-    def is_dataset_query_variable_valid(config: dict, mode: str) -> None:
+
+    @classmethod
+    def is_moderation_valid(cls, tenant_id: str, config: dict):
+        if 'sensitive_word_avoidance' not in config or not config["sensitive_word_avoidance"]:
+            config["sensitive_word_avoidance"] = {
+                "enabled": False
+            }
+
+        if not isinstance(config["sensitive_word_avoidance"], dict):
+            raise ValueError("sensitive_word_avoidance must be of dict type")
+
+        if "enabled" not in config["sensitive_word_avoidance"] or not config["sensitive_word_avoidance"]["enabled"]:
+            config["sensitive_word_avoidance"]["enabled"] = False
+
+        if not config["sensitive_word_avoidance"]["enabled"]:
+            return
+
+        if "type" not in config["sensitive_word_avoidance"] or not config["sensitive_word_avoidance"]["type"]:
+            raise ValueError("sensitive_word_avoidance.type is required")
+
+        type = config["sensitive_word_avoidance"]["type"]
+        config = config["sensitive_word_avoidance"]["config"]
+
+        ModerationFactory.validate_config(
+            name=type,
+            tenant_id=tenant_id,
+            config=config
+        )
+
+    @classmethod
+    def is_external_data_tools_valid(cls, tenant_id: str, config: dict):
+        if 'external_data_tools' not in config or not config["external_data_tools"]:
+            config["external_data_tools"] = []
+
+        if not isinstance(config["external_data_tools"], list):
+            raise ValueError("external_data_tools must be of list type")
+
+        for tool in config["external_data_tools"]:
+            if "enabled" not in tool or not tool["enabled"]:
+                tool["enabled"] = False
+
+            if not tool["enabled"]:
+                continue
+
+            if "type" not in tool or not tool["type"]:
+                raise ValueError("external_data_tools[].type is required")
+
+            type = tool["type"]
+            config = tool["config"]
+
+            ExternalDataToolFactory.validate_config(
+                name=type,
+                tenant_id=tenant_id,
+                config=config
+            )
+
+    @classmethod
+    def is_dataset_query_variable_valid(cls, config: dict, mode: str) -> None:
         # Only check when mode is completion
         if mode != 'completion':
             return
-        
+
         agent_mode = config.get("agent_mode", {})
         tools = agent_mode.get("tools", [])
         dataset_exists = "dataset" in str(tools)
-        
+
         dataset_query_variable = config.get("dataset_query_variable")
 
         if dataset_exists and not dataset_query_variable:
             raise ValueError("Dataset query variable is required when dataset is exist")
-        
 
-    @staticmethod
-    def is_advanced_prompt_valid(config: dict, app_mode: str) -> None:
+    @classmethod
+    def is_advanced_prompt_valid(cls, config: dict, app_mode: str) -> None:
         # prompt_type
         if 'prompt_type' not in config or not config["prompt_type"]:
             config["prompt_type"] = "simple"
 
         if config['prompt_type'] not in ['simple', 'advanced']:
             raise ValueError("prompt_type must be in ['simple', 'advanced']")
-        
+
         # chat_prompt_config
         if 'chat_prompt_config' not in config or not config["chat_prompt_config"]:
             config["chat_prompt_config"] = {}
@@ -400,7 +435,7 @@ class AppModelConfigService:
 
         if not isinstance(config["completion_prompt_config"], dict):
             raise ValueError("completion_prompt_config must be of object type")
-        
+
         # dataset_configs
         if 'dataset_configs' not in config or not config["dataset_configs"]:
             config["dataset_configs"] = {"top_k": 2, "score_threshold": {"enable": False}}
@@ -411,11 +446,11 @@ class AppModelConfigService:
         if config['prompt_type'] == 'advanced':
             if not config['chat_prompt_config'] and not config['completion_prompt_config']:
                 raise ValueError("chat_prompt_config or completion_prompt_config is required when prompt_type is advanced")
-            
+
             if config['model']["mode"] not in ['chat', 'completion']:
                 raise ValueError("model.mode must be in ['chat', 'completion'] when prompt_type is advanced")
-            
-            if app_mode == 'chat' and config['model']["mode"] == ModelMode.COMPLETION.value:
+
+            if app_mode == AppMode.CHAT.value and config['model']["mode"] == ModelMode.COMPLETION.value:
                 user_prefix = config['completion_prompt_config']['conversation_histories_role']['user_prefix']
                 assistant_prefix = config['completion_prompt_config']['conversation_histories_role']['assistant_prefix']
 
@@ -424,3 +459,9 @@ class AppModelConfigService:
 
                 if not assistant_prefix:
                     config['completion_prompt_config']['conversation_histories_role']['assistant_prefix'] = 'Assistant'
+
+            if config['model']["mode"] == ModelMode.CHAT.value:
+                prompt_list = config['chat_prompt_config']['prompt']
+
+                if len(prompt_list) > 10:
+                    raise ValueError("prompt messages must be less than 10")

api/services/code_based_extension_service.py

@@ -0,0 +1,13 @@
+from extensions.ext_code_based_extension import code_based_extension
+
+
+class CodeBasedExtensionService:
+
+    @staticmethod
+    def get_code_based_extension(module: str) -> list[dict]:
+        module_extensions = code_based_extension.module_extensions(module)
+        return [{
+            'name': module_extension.name,
+            'label': module_extension.label,
+            'form_schema': module_extension.form_schema
+        } for module_extension in module_extensions if not module_extension.builtin]

api/services/completion_service.py

@@ -10,7 +10,8 @@ from redis.client import PubSub
 from sqlalchemy import and_
 
 from core.completion import Completion
-from core.conversation_message_task import PubHandler, ConversationTaskStoppedException
+from core.conversation_message_task import PubHandler, ConversationTaskStoppedException, \
+    ConversationTaskInterruptException
 from core.model_providers.error import LLMBadRequestError, LLMAPIConnectionError, LLMAPIUnavailableError, \
     LLMRateLimitError, \
     LLMAuthorizationError, ProviderTokenNotInitError, QuotaExceededError, ModelCurrentlyNotSupportError
@@ -28,9 +29,9 @@ from services.errors.message import MessageNotExistsError
 class CompletionService:
 
     @classmethod
-    def completion(cls, app_model: App, user: Union[Account | EndUser], args: Any,
+    def completion(cls, app_model: App, user: Union[Account, EndUser], args: Any,
                    from_source: str, streaming: bool = True,
-                   is_model_config_override: bool = False) -> Union[dict | Generator]:
+                   is_model_config_override: bool = False) -> Union[dict, Generator]:
         # is streaming mode
         inputs = args['inputs']
         query = args['query']
@@ -142,7 +143,7 @@ class CompletionService:
             'flask_app': current_app._get_current_object(),
             'generate_task_id': generate_task_id,
             'detached_app_model': app_model,
-            'app_model_config': app_model_config,
+            'app_model_config': app_model_config.copy(),
             'query': query,
             'inputs': inputs,
             'detached_user': user,
@@ -199,9 +200,9 @@ class CompletionService:
                     is_override=is_model_config_override,
                     retriever_from=retriever_from
                 )
-            except ConversationTaskStoppedException:
+            except (ConversationTaskInterruptException, ConversationTaskStoppedException):
                 pass
-            except (LLMBadRequestError, LLMAPIConnectionError, LLMAPIUnavailableError,
+            except (ValueError, LLMBadRequestError, LLMAPIConnectionError, LLMAPIUnavailableError,
                     LLMRateLimitError, ProviderTokenNotInitError, QuotaExceededError,
                     ModelCurrentlyNotSupportError) as e:
                 PubHandler.pub_error(user, generate_task_id, e)
@@ -234,7 +235,7 @@ class CompletionService:
                     PubHandler.stop(user, generate_task_id)
                     try:
                         pubsub.close()
-                    except:
+                    except Exception:
                         pass
 
         countdown_thread = threading.Thread(target=close_pubsub)
@@ -243,9 +244,9 @@ class CompletionService:
         return countdown_thread
 
     @classmethod
-    def generate_more_like_this(cls, app_model: App, user: Union[Account | EndUser],
+    def generate_more_like_this(cls, app_model: App, user: Union[Account, EndUser],
                                 message_id: str, streaming: bool = True,
-                                retriever_from: str = 'dev') -> Union[dict | Generator]:
+                                retriever_from: str = 'dev') -> Union[dict, Generator]:
         if not user:
             raise ValueError('user cannot be None')
 
@@ -284,7 +285,7 @@ class CompletionService:
             'flask_app': current_app._get_current_object(),
             'generate_task_id': generate_task_id,
             'detached_app_model': app_model,
-            'app_model_config': app_model_config,
+            'app_model_config': app_model_config.copy(),
             'query': message.query,
             'inputs': message.inputs,
             'detached_user': user,
@@ -341,7 +342,7 @@ class CompletionService:
         return filtered_inputs
 
     @classmethod
-    def compact_response(cls, pubsub: PubSub, streaming: bool = False) -> Union[dict | Generator]:
+    def compact_response(cls, pubsub: PubSub, streaming: bool = False) -> Union[dict, Generator]:
         generate_channel = list(pubsub.channels.keys())[0].decode('utf-8')
         if not streaming:
             try:
@@ -386,6 +387,8 @@ class CompletionService:
                                 break
                             if event == 'message':
                                 yield "data: " + json.dumps(cls.get_message_response_data(result.get('data'))) + "\n\n"
+                            elif event == 'message_replace':
+                                yield "data: " + json.dumps(cls.get_message_replace_response_data(result.get('data'))) + "\n\n"
                             elif event == 'chain':
                                 yield "data: " + json.dumps(cls.get_chain_response_data(result.get('data'))) + "\n\n"
                             elif event == 'agent_thought':
@@ -427,6 +430,21 @@ class CompletionService:
 
         return response_data
 
+    @classmethod
+    def get_message_replace_response_data(cls, data: dict):
+        response_data = {
+            'event': 'message_replace',
+            'task_id': data.get('task_id'),
+            'id': data.get('message_id'),
+            'answer': data.get('text'),
+            'created_at': int(time.time())
+        }
+
+        if data.get('mode') == 'chat':
+            response_data['conversation_id'] = data.get('conversation_id')
+
+        return response_data
+
     @classmethod
     def get_blocking_message_response_data(cls, data: dict):
         message = data.get('message')
@@ -508,6 +526,7 @@ class CompletionService:
 
         # handle errors
         llm_errors = {
+            'ValueError': LLMBadRequestError,
             'LLMBadRequestError': LLMBadRequestError,
             'LLMAPIConnectionError': LLMAPIConnectionError,
             'LLMAPIUnavailableError': LLMAPIUnavailableError,

api/services/moderation_service.py

@@ -0,0 +1,20 @@
+from models.model import AppModelConfig, App
+from core.moderation.factory import ModerationFactory, ModerationOutputsResult
+from extensions.ext_database import db
+
+
+class ModerationService:
+
+    def moderation_for_outputs(self, app_id: str, app_model: App, text: str) -> ModerationOutputsResult:
+        app_model_config: AppModelConfig = None
+
+        app_model_config = db.session.query(AppModelConfig).filter(AppModelConfig.id == app_model.app_model_config_id).first()
+
+        if not app_model_config:
+            raise ValueError("app model config not found")
+
+        name = app_model_config.sensitive_word_avoidance_dict['type']
+        config = app_model_config.sensitive_word_avoidance_dict['config']
+
+        moderation = ModerationFactory(name, app_id, app_model.tenant_id, config)
+        return moderation.moderation_for_outputs(text)

api/tests/integration_tests/models/llm/test_wenxin_model.py

@@ -56,9 +56,8 @@ def test_run(mock_decrypt, mocker):
     mocker.patch('core.model_providers.providers.base.BaseModelProvider.update_last_used', return_value=None)
 
     model = get_mock_model('ernie-bot')
-    messages = [PromptMessage(content='Human: 1 + 1=? \nAssistant: Integer answer is:')]
+    messages = [PromptMessage(type=MessageType.USER, content='Human: 1 + 1=? \nAssistant: Integer answer is:')]
     rst = model.run(
-        messages,
-        stop=['\nHuman:'],
+        messages
     )
     assert len(rst.content) > 0

api/tests/unit_tests/model_providers/test_wenxin_provider.py

@@ -2,6 +2,8 @@ import pytest
 from unittest.mock import patch
 import json
 
+from langchain.schema import AIMessage, ChatGeneration, ChatResult
+
 from core.model_providers.providers.base import CredentialsValidateFailedError
 from core.model_providers.providers.wenxin_provider import WenxinProvider
 from models.provider import ProviderType, Provider
@@ -24,7 +26,8 @@ def decrypt_side_effect(tenant_id, encrypted_key):
 
 
 def test_is_provider_credentials_valid_or_raise_valid(mocker):
-    mocker.patch('core.third_party.langchain.llms.wenxin.Wenxin._call', return_value="abc")
+    mocker.patch('core.third_party.langchain.llms.wenxin.Wenxin._generate',
+                 return_value=ChatResult(generations=[ChatGeneration(message=AIMessage(content='abc'))]))
 
     MODEL_PROVIDER_CLASS.is_provider_credentials_valid_or_raise(VALIDATE_CREDENTIAL)
 

docker/docker-compose.yaml

@@ -2,7 +2,7 @@ version: '3.1'
 services:
   # API service
   api:
-    image: langgenius/dify-api:0.3.27
+    image: langgenius/dify-api:0.3.29
     restart: always
     environment:
       # Startup mode, 'api' starts the API server.
@@ -19,7 +19,7 @@ services:
       # different from api or web app domain.
       # example: http://cloud.dify.ai
       CONSOLE_API_URL: ''
-      # The URL for Service API endpoints，refers to the base URL of the current API service if api domain is
+      # The URL for Service API endpoints, refers to the base URL of the current API service if api domain is
       # different from console domain.
       # example: http://api.dify.ai
       SERVICE_API_URL: ''
@@ -98,7 +98,7 @@ services:
       # The milvus password.
       MILVUS_PASSWORD: Milvus
       # The milvus tls switch.
-      MILVUS_SECURE: false
+      MILVUS_SECURE: 'false'
       # Mail configuration, support: resend
       MAIL_TYPE: ''
       # default send from email address, if not specified
@@ -124,7 +124,7 @@ services:
   # worker service
   # The Celery worker for processing the queue.
   worker:
-    image: langgenius/dify-api:0.3.27
+    image: langgenius/dify-api:0.3.29
     restart: always
     environment:
       # Startup mode, 'worker' starts the Celery worker for processing the queue.
@@ -176,7 +176,7 @@ services:
       # The milvus password.
       MILVUS_PASSWORD: Milvus
       # The milvus tls switch.
-      MILVUS_SECURE: false
+      MILVUS_SECURE: 'false'
       # Mail configuration, support: resend
       MAIL_TYPE: ''
       # default send from email address, if not specified
@@ -192,7 +192,7 @@ services:
 
   # Frontend web application.
   web:
-    image: langgenius/dify-web:0.3.27
+    image: langgenius/dify-web:0.3.29
     restart: always
     environment:
       EDITION: SELF_HOSTED

third-party/chrome plug-in/README_CN.md

@@ -0,0 +1,35 @@
+## Chrome Dify ChatBot插件
+
+### 方式1：Chrome插件商店 * [点击访问](https://chrome.google.com/webstore/detail/dify-chatbot/ceehdapohffmjmkdcifjofadiaoeggaf/related?hl=zh-CN&authuser=0) *
+ 
+### 方式2：本地开发者模式加载
+
+- 进入Chrome浏览器管理扩展程序，可直接访问 [chrome://extensions/](chrome://extensions/)
+- 选择开启 “开发者模式”，并点击 “加载已解压的扩展程序”
+
+![img-1.png](images/img-1.png)
+
+- 然后打开插件源文件所在根目录
+  - third-party
+    - chrome plug-in
+      - content.js          浮动按钮JS脚本
+      - favicon.png         插件图标
+      - manifest.json       插件描述文件
+      - options.css         插件配置页面样式文件
+      - options.html        插件配置静态HTML页面
+      - options.js          插件配置JS脚本
+
+### 插件导入完成后，后续配置无差异
+- 初始化设置Dify 应用配置，分别输入Dify根域名和应用Token，Token可以在Dify应用嵌入中获取，如图：
+
+![img-2.png](images/img-2.png)
+![img-3.png](images/img-3.png)
+
+- 点击保存，确认提示配置成功即可
+
+![img-4.png](images/img-4.png)
+
+- 保险起见重启浏览器确保所有分页刷新成功
+- Chrome打开任意页面均可正常加载DIfy机器人浮动栏，后续如需更换机器人只需要变更Token即可
+
+![img-5.png](images/img-5.png)

third-party/chrome plug-in/content.js

@@ -0,0 +1,170 @@
+var storage = chrome.storage.sync;
+chrome.storage.sync.get(['baseUrl', 'token'], function(result) {
+  window.difyChatbotConfig = { 
+    baseUrl: result.baseUrl,
+    token: result.token
+  };
+});
+
+document.body.onload = embedChatbot;
+
+async function embedChatbot() {
+  const difyChatbotConfig = window.difyChatbotConfig;
+  if (!difyChatbotConfig || !difyChatbotConfig.token) {
+    console.warn('difyChatbotConfig is empty or token is not provided');
+    return;
+  }
+  const baseUrl = difyChatbotConfig.baseUrl
+  const openIcon = `<svg
+            id="openIcon"
+            width="24"
+            height="24"
+            viewBox="0 0 24 24"
+            fill="none"
+            xmlns="http://www.w3.org/2000/svg"
+          >
+            <path
+              fill-rule="evenodd"
+              clip-rule="evenodd"
+              d="M7.7586 2L16.2412 2C17.0462 1.99999 17.7105 1.99998 18.2517 2.04419C18.8138 2.09012 19.3305 2.18868 19.8159 2.43598C20.5685 2.81947 21.1804 3.43139 21.5639 4.18404C21.8112 4.66937 21.9098 5.18608 21.9557 5.74818C21.9999 6.28937 21.9999 6.95373 21.9999 7.7587L22 14.1376C22.0004 14.933 22.0007 15.5236 21.8636 16.0353C21.4937 17.4156 20.4155 18.4938 19.0352 18.8637C18.7277 18.9461 18.3917 18.9789 17.9999 18.9918L17.9999 20.371C18 20.6062 18 20.846 17.9822 21.0425C17.9651 21.2305 17.9199 21.5852 17.6722 21.8955C17.3872 22.2525 16.9551 22.4602 16.4983 22.4597C16.1013 22.4593 15.7961 22.273 15.6386 22.1689C15.474 22.06 15.2868 21.9102 15.1031 21.7632L12.69 19.8327C12.1714 19.4178 12.0174 19.3007 11.8575 19.219C11.697 19.137 11.5262 19.0771 11.3496 19.0408C11.1737 19.0047 10.9803 19 10.3162 19H7.75858C6.95362 19 6.28927 19 5.74808 18.9558C5.18598 18.9099 4.66928 18.8113 4.18394 18.564C3.43129 18.1805 2.81937 17.5686 2.43588 16.816C2.18859 16.3306 2.09002 15.8139 2.0441 15.2518C1.99988 14.7106 1.99989 14.0463 1.9999 13.2413V7.75868C1.99989 6.95372 1.99988 6.28936 2.0441 5.74818C2.09002 5.18608 2.18859 4.66937 2.43588 4.18404C2.81937 3.43139 3.43129 2.81947 4.18394 2.43598C4.66928 2.18868 5.18598 2.09012 5.74808 2.04419C6.28927 1.99998 6.95364 1.99999 7.7586 2ZM10.5073 7.5C10.5073 6.67157 9.83575 6 9.00732 6C8.1789 6 7.50732 6.67157 7.50732 7.5C7.50732 8.32843 8.1789 9 9.00732 9C9.83575 9 10.5073 8.32843 10.5073 7.5ZM16.6073 11.7001C16.1669 11.3697 15.5426 11.4577 15.2105 11.8959C15.1488 11.9746 15.081 12.0486 15.0119 12.1207C14.8646 12.2744 14.6432 12.4829 14.3566 12.6913C13.7796 13.111 12.9818 13.5001 12.0073 13.5001C11.0328 13.5001 10.235 13.111 9.65799 12.6913C9.37138 12.4829 9.15004 12.2744 9.00274 12.1207C8.93366 12.0486 8.86581 11.9745 8.80418 11.8959C8.472 11.4577 7.84775 11.3697 7.40732 11.7001C6.96549 12.0314 6.87595 12.6582 7.20732 13.1001C7.20479 13.0968 7.21072 13.1043 7.22094 13.1171C7.24532 13.1478 7.29407 13.2091 7.31068 13.2289C7.36932 13.2987 7.45232 13.3934 7.55877 13.5045C7.77084 13.7258 8.08075 14.0172 8.48165 14.3088C9.27958 14.8891 10.4818 15.5001 12.0073 15.5001C13.5328 15.5001 14.735 14.8891 15.533 14.3088C15.9339 14.0172 16.2438 13.7258 16.4559 13.5045C16.5623 13.3934 16.6453 13.2987 16.704 13.2289C16.7333 13.1939 16.7567 13.165 16.7739 13.1432C17.1193 12.6969 17.0729 12.0493 16.6073 11.7001ZM15.0073 6C15.8358 6 16.5073 6.67157 16.5073 7.5C16.5073 8.32843 15.8358 9 15.0073 9C14.1789 9 13.5073 8.32843 13.5073 7.5C13.5073 6.67157 14.1789 6 15.0073 6Z"
+              fill="white"
+            />
+          </svg>`;
+  const closeIcon = `<svg
+          id="closeIcon"
+          width="24"
+          height="24"
+          viewBox="0 0 24 24"
+          fill="none"
+          xmlns="http://www.w3.org/2000/svg"
+        >
+          <path
+            d="M18 18L6 6M6 18L18 6"
+            stroke="white"
+            stroke-width="2"
+            stroke-linecap="round"
+            stroke-linejoin="round"
+          />
+        </svg>`;
+
+  // create iframe
+  function createIframe() {
+    const iframe = document.createElement('iframe');
+    iframe.allow = "fullscreen;microphone"
+    iframe.title = "dify chatbot bubble window"
+    iframe.id = 'dify-chatbot-bubble-window'
+    iframe.src = `${baseUrl}/chat/${difyChatbotConfig.token}`
+    iframe.style.cssText = 'border: none; position: fixed; flex-direction: column; justify-content: space-between; box-shadow: rgba(150, 150, 150, 0.2) 0px 10px 30px 0px, rgba(150, 150, 150, 0.2) 0px 0px 0px 1px; bottom: 6.7rem; right: 1rem; width: 30rem; height: 48rem; border-radius: 0.75rem; display: flex; z-index: 2147483647; overflow: hidden; left: unset; background-color: #F3F4F6;'
+    document.body.appendChild(iframe);
+  }
+
+  /**
+   * rem to px
+   * @param {*} rem ：30rem
+   */
+  function handleRemToPx(rem) {
+    if (!rem) return;
+    let pxValue = 0;
+    try {
+      const regex = /\d+/;
+      // extract the numeric part and convert it to a numeric type
+      const remValue = parseInt(regex.exec(rem)[0], 10);
+      const rootFontSize = parseFloat(
+        window.getComputedStyle(document.documentElement).fontSize
+      );
+      pxValue = remValue * rootFontSize;
+    } catch (error) {
+      console.error(error);
+    }
+    return pxValue;
+  }
+
+  /**
+   * support element drag
+   * @param {*} targetButton entry element
+   */
+  function handleElementDrag(targetButton) {
+    // define a variable to hold the mouse position
+    let mouseX = 0,
+      mouseY = 0,
+      offsetX = 0,
+      offsetY = 0;
+
+    // Listen for mouse press events, get mouse position and element position
+    targetButton.addEventListener("mousedown", function (event) {
+      // calculate mouse position
+      mouseX = event.clientX;
+      mouseY = event.clientY;
+
+      // calculate element position
+      const rect = targetButton.getBoundingClientRect();
+      offsetX = mouseX - rect.left;
+      offsetY = mouseY - rect.top;
+
+      // listen for mouse movement events
+      document.addEventListener("mousemove", onMouseMove);
+    });
+
+    // listen for mouse lift events and stop listening for mouse move events
+    document.addEventListener("mouseup", function () {
+      document.removeEventListener("mousemove", onMouseMove);
+    });
+
+    // the mouse moves the event handler to update the element position
+    function onMouseMove(event) {
+      // calculate element position
+      let newX = event.clientX - offsetX,
+        newY = event.clientY - offsetY;
+
+      // 计算视线边界
+      const viewportWidth = window.innerWidth,
+        viewportHeight = window.innerHeight;
+
+      const maxX = viewportWidth - targetButton.offsetWidth,
+        maxY = viewportHeight - targetButton.offsetHeight;
+
+      // application limitation
+      newX = Math.max(12, Math.min(newX, maxX));
+      newY = Math.max(12, Math.min(newY, maxY));
+
+      // update element position
+      targetButton.style.left = newX + "px";
+      targetButton.style.top = newY + "px";
+    }
+  }
+
+  const targetButton = document.getElementById("dify-chatbot-bubble-button");
+
+  if (!targetButton) {
+    // create button
+    const containerDiv = document.createElement("div");
+    containerDiv.id = 'dify-chatbot-bubble-button';
+    containerDiv.style.cssText = `position: fixed; bottom: 3rem; right: 1rem; width: 50px; height: 50px; border-radius: 25px; background-color: #155EEF; box-shadow: rgba(0, 0, 0, 0.2) 0px 4px 8px 0px; cursor: move; z-index: 2147483647; transition: all 0.2s ease-in-out 0s; left: unset; transform: scale(1); :hover {transform: scale(1.1);}`;
+    const displayDiv = document.createElement('div');
+    displayDiv.style.cssText = "display: flex; align-items: center; justify-content: center; width: 100%; height: 100%; z-index: 2147483647;";
+    displayDiv.innerHTML = openIcon;
+    containerDiv.appendChild(displayDiv);
+    document.body.appendChild(containerDiv);
+    handleElementDrag(containerDiv);
+
+    // add click event to control iframe display
+    containerDiv.addEventListener('click', function () {
+      const targetIframe = document.getElementById('dify-chatbot-bubble-window');
+      if (!targetIframe) {
+        createIframe();
+        displayDiv.innerHTML = closeIcon;
+        return;
+      }
+      if (targetIframe.style.display === "none") {
+        targetIframe.style.display = "block";
+        displayDiv.innerHTML = closeIcon;
+      } else {
+        targetIframe.style.display = "none";
+        displayDiv.innerHTML = openIcon;
+      }
+    });
+  } else {
+    // add any drag and drop to the floating icon
+    handleElementDrag(targetButton);
+  }
+}