version to 0.5.2 (#2230)

Co-authored-by: jyong <jyong@dify.ai>

api/config.py

@@ -93,7 +93,7 @@ class Config:
         # ------------------------
         # General Configurations.
         # ------------------------
-        self.CURRENT_VERSION = "0.5.1"
+        self.CURRENT_VERSION = "0.5.2"
         self.COMMIT_SHA = get_env('COMMIT_SHA')
         self.EDITION = "SELF_HOSTED"
         self.DEPLOY_ENV = get_env('DEPLOY_ENV')

api/controllers/console/apikey.py

@@ -61,9 +61,7 @@ class BaseApiKeyListResource(Resource):
         resource_id = str(resource_id)
         _get_resource(resource_id, current_user.current_tenant_id,
                       self.resource_model)
-
-        # The role of the current user in the ta table must be admin or owner
-        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+        if not current_user.is_admin_or_owner:
             raise Forbidden()
 
         current_key_count = db.session.query(ApiToken). \
@@ -102,7 +100,7 @@ class BaseApiKeyResource(Resource):
                       self.resource_model)
 
         # The role of the current user in the ta table must be admin or owner
-        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+        if not current_user.is_admin_or_owner:
             raise Forbidden()
 
         key = db.session.query(ApiToken). \

api/controllers/console/app/annotation.py

@@ -21,7 +21,7 @@ class AnnotationReplyActionApi(Resource):
     @cloud_edition_billing_resource_check('annotation')
     def post(self, app_id, action):
         # The role of the current user in the ta table must be admin or owner
-        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+        if not current_user.is_admin_or_owner:
             raise Forbidden()
 
         app_id = str(app_id)
@@ -45,7 +45,7 @@ class AppAnnotationSettingDetailApi(Resource):
     @account_initialization_required
     def get(self, app_id):
         # The role of the current user in the ta table must be admin or owner
-        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+        if not current_user.is_admin_or_owner:
             raise Forbidden()
 
         app_id = str(app_id)
@@ -59,7 +59,7 @@ class AppAnnotationSettingUpdateApi(Resource):
     @account_initialization_required
     def post(self, app_id, annotation_setting_id):
         # The role of the current user in the ta table must be admin or owner
-        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+        if not current_user.is_admin_or_owner:
             raise Forbidden()
 
         app_id = str(app_id)
@@ -80,7 +80,7 @@ class AnnotationReplyActionStatusApi(Resource):
     @cloud_edition_billing_resource_check('annotation')
     def get(self, app_id, job_id, action):
         # The role of the current user in the ta table must be admin or owner
-        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+        if not current_user.is_admin_or_owner:
             raise Forbidden()
 
         job_id = str(job_id)
@@ -108,7 +108,7 @@ class AnnotationListApi(Resource):
     @account_initialization_required
     def get(self, app_id):
         # The role of the current user in the ta table must be admin or owner
-        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+        if not current_user.is_admin_or_owner:
             raise Forbidden()
 
         page = request.args.get('page', default=1, type=int)
@@ -133,7 +133,7 @@ class AnnotationExportApi(Resource):
     @account_initialization_required
     def get(self, app_id):
         # The role of the current user in the ta table must be admin or owner
-        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+        if not current_user.is_admin_or_owner:
             raise Forbidden()
 
         app_id = str(app_id)
@@ -152,7 +152,7 @@ class AnnotationCreateApi(Resource):
     @marshal_with(annotation_fields)
     def post(self, app_id):
         # The role of the current user in the ta table must be admin or owner
-        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+        if not current_user.is_admin_or_owner:
             raise Forbidden()
 
         app_id = str(app_id)
@@ -172,7 +172,7 @@ class AnnotationUpdateDeleteApi(Resource):
     @marshal_with(annotation_fields)
     def post(self, app_id, annotation_id):
         # The role of the current user in the ta table must be admin or owner
-        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+        if not current_user.is_admin_or_owner:
             raise Forbidden()
 
         app_id = str(app_id)
@@ -189,7 +189,7 @@ class AnnotationUpdateDeleteApi(Resource):
     @account_initialization_required
     def delete(self, app_id, annotation_id):
         # The role of the current user in the ta table must be admin or owner
-        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+        if not current_user.is_admin_or_owner:
             raise Forbidden()
 
         app_id = str(app_id)
@@ -205,7 +205,7 @@ class AnnotationBatchImportApi(Resource):
     @cloud_edition_billing_resource_check('annotation')
     def post(self, app_id):
         # The role of the current user in the ta table must be admin or owner
-        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+        if not current_user.is_admin_or_owner:
             raise Forbidden()
 
         app_id = str(app_id)
@@ -230,7 +230,7 @@ class AnnotationBatchImportStatusApi(Resource):
     @cloud_edition_billing_resource_check('annotation')
     def get(self, app_id, job_id):
         # The role of the current user in the ta table must be admin or owner
-        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+        if not current_user.is_admin_or_owner:
             raise Forbidden()
 
         job_id = str(job_id)
@@ -257,7 +257,7 @@ class AnnotationHitHistoryListApi(Resource):
     @account_initialization_required
     def get(self, app_id, annotation_id):
         # The role of the current user in the table must be admin or owner
-        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+        if not current_user.is_admin_or_owner:
             raise Forbidden()
 
         page = request.args.get('page', default=1, type=int)

api/controllers/console/app/app.py

@@ -88,7 +88,7 @@ class AppListApi(Resource):
         args = parser.parse_args()
 
         # The role of the current user in the ta table must be admin or owner
-        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+        if not current_user.is_admin_or_owner:
             raise Forbidden()
 
         try:
@@ -237,7 +237,7 @@ class AppApi(Resource):
         """Delete app"""
         app_id = str(app_id)
 
-        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+        if not current_user.is_admin_or_owner:
             raise Forbidden()
 
         app = _get_app(app_id, current_user.current_tenant_id)

api/controllers/console/app/message.py

@@ -157,7 +157,7 @@ class MessageAnnotationApi(Resource):
     @marshal_with(annotation_fields)
     def post(self, app_id):
         # The role of the current user in the ta table must be admin or owner
-        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+        if not current_user.is_admin_or_owner:
             raise Forbidden()
 
         app_id = str(app_id)

api/controllers/console/app/site.py

@@ -42,7 +42,7 @@ class AppSite(Resource):
         app_model = _get_app(app_id)
 
         # The role of the current user in the ta table must be admin or owner
-        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+        if not current_user.is_admin_or_owner:
             raise Forbidden()
 
         site = db.session.query(Site). \
@@ -88,7 +88,7 @@ class AppSiteAccessTokenReset(Resource):
         app_model = _get_app(app_id)
 
         # The role of the current user in the ta table must be admin or owner
-        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+        if not current_user.is_admin_or_owner:
             raise Forbidden()
 
         site = db.session.query(Site).filter(Site.app_id == app_model.id).first()

api/controllers/console/auth/data_source_oauth.py

@@ -30,7 +30,7 @@ def get_oauth_providers():
 class OAuthDataSource(Resource):
     def get(self, provider: str):
         # The role of the current user in the table must be admin or owner
-        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+        if not current_user.is_admin_or_owner:
             raise Forbidden()
         OAUTH_DATASOURCE_PROVIDERS = get_oauth_providers()
         with current_app.app_context():

api/controllers/console/datasets/datasets.py

@@ -103,7 +103,7 @@ class DatasetListApi(Resource):
         args = parser.parse_args()
 
         # The role of the current user in the ta table must be admin or owner
-        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+        if not current_user.is_admin_or_owner:
             raise Forbidden()
 
         try:
@@ -187,7 +187,7 @@ class DatasetApi(Resource):
         args = parser.parse_args()
 
         # The role of the current user in the ta table must be admin or owner
-        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+        if not current_user.is_admin_or_owner:
             raise Forbidden()
 
         dataset = DatasetService.update_dataset(
@@ -205,7 +205,7 @@ class DatasetApi(Resource):
         dataset_id_str = str(dataset_id)
 
         # The role of the current user in the ta table must be admin or owner
-        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+        if not current_user.is_admin_or_owner:
             raise Forbidden()
 
         if DatasetService.delete_dataset(dataset_id_str, current_user):
@@ -391,7 +391,7 @@ class DatasetApiKeyApi(Resource):
     @marshal_with(api_key_fields)
     def post(self):
         # The role of the current user in the ta table must be admin or owner
-        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+        if not current_user.is_admin_or_owner:
             raise Forbidden()
 
         current_key_count = db.session.query(ApiToken). \
@@ -425,7 +425,7 @@ class DatasetApiDeleteApi(Resource):
         api_key_id = str(api_key_id)
 
         # The role of the current user in the ta table must be admin or owner
-        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+        if not current_user.is_admin_or_owner:
             raise Forbidden()
 
         key = db.session.query(ApiToken). \

api/controllers/console/datasets/datasets_document.py

@@ -204,7 +204,7 @@ class DatasetDocumentListApi(Resource):
             raise NotFound('Dataset not found.')
 
         # The role of the current user in the ta table must be admin or owner
-        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+        if not current_user.is_admin_or_owner:
             raise Forbidden()
 
         try:
@@ -256,7 +256,7 @@ class DatasetInitApi(Resource):
     @cloud_edition_billing_resource_check('vector_space')
     def post(self):
         # The role of the current user in the ta table must be admin or owner
-        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+        if not current_user.is_admin_or_owner:
             raise Forbidden()
 
         parser = reqparse.RequestParser()
@@ -599,7 +599,7 @@ class DocumentProcessingApi(DocumentResource):
         document = self.get_document(dataset_id, document_id)
 
         # The role of the current user in the ta table must be admin or owner
-        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+        if not current_user.is_admin_or_owner:
             raise Forbidden()
 
         if action == "pause":
@@ -663,7 +663,7 @@ class DocumentMetadataApi(DocumentResource):
         doc_metadata = req_data.get('doc_metadata')
 
         # The role of the current user in the ta table must be admin or owner
-        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+        if not current_user.is_admin_or_owner:
             raise Forbidden()
 
         if doc_type is None or doc_metadata is None:
@@ -710,7 +710,7 @@ class DocumentStatusApi(DocumentResource):
         document = self.get_document(dataset_id, document_id)
 
         # The role of the current user in the ta table must be admin or owner
-        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+        if not current_user.is_admin_or_owner:
             raise Forbidden()
 
         indexing_cache_key = 'document_{}_indexing'.format(document.id)

api/controllers/console/datasets/datasets_segments.py

@@ -123,7 +123,7 @@ class DatasetDocumentSegmentApi(Resource):
         # check user's model setting
         DatasetService.check_dataset_model_setting(dataset)
         # The role of the current user in the ta table must be admin or owner
-        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+        if not current_user.is_admin_or_owner:
             raise Forbidden()
 
         try:
@@ -219,7 +219,7 @@ class DatasetDocumentSegmentAddApi(Resource):
         if not document:
             raise NotFound('Document not found.')
         # The role of the current user in the ta table must be admin or owner
-        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+        if not current_user.is_admin_or_owner:
             raise Forbidden()
         # check embedding model setting
         if dataset.indexing_technique == 'high_quality':
@@ -298,7 +298,7 @@ class DatasetDocumentSegmentUpdateApi(Resource):
         if not segment:
             raise NotFound('Segment not found.')
         # The role of the current user in the ta table must be admin or owner
-        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+        if not current_user.is_admin_or_owner:
             raise Forbidden()
         try:
             DatasetService.check_dataset_permission(dataset, current_user)
@@ -342,7 +342,7 @@ class DatasetDocumentSegmentUpdateApi(Resource):
         if not segment:
             raise NotFound('Segment not found.')
         # The role of the current user in the ta table must be admin or owner
-        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+        if not current_user.is_admin_or_owner:
             raise Forbidden()
         try:
             DatasetService.check_dataset_permission(dataset, current_user)

api/controllers/console/workspace/model_providers.py

@@ -98,7 +98,7 @@ class ModelProviderApi(Resource):
     @login_required
     @account_initialization_required
     def post(self, provider: str):
-        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+        if not current_user.is_admin_or_owner:
             raise Forbidden()
 
         parser = reqparse.RequestParser()
@@ -122,7 +122,7 @@ class ModelProviderApi(Resource):
     @login_required
     @account_initialization_required
     def delete(self, provider: str):
-        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+        if not current_user.is_admin_or_owner:
             raise Forbidden()
 
         model_provider_service = ModelProviderService()
@@ -159,7 +159,7 @@ class PreferredProviderTypeUpdateApi(Resource):
     @login_required
     @account_initialization_required
     def post(self, provider: str):
-        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+        if not current_user.is_admin_or_owner:
             raise Forbidden()
 
         tenant_id = current_user.current_tenant_id

api/controllers/console/workspace/tool_providers.py

@@ -43,7 +43,7 @@ class ToolBuiltinProviderDeleteApi(Resource):
     @login_required
     @account_initialization_required
     def post(self, provider):
-        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+        if not current_user.is_admin_or_owner:
             raise Forbidden()
         
         user_id = current_user.id
@@ -60,7 +60,7 @@ class ToolBuiltinProviderUpdateApi(Resource):
     @login_required
     @account_initialization_required
     def post(self, provider):
-        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+        if not current_user.is_admin_or_owner:
             raise Forbidden()
         
         user_id = current_user.id
@@ -90,7 +90,7 @@ class ToolApiProviderAddApi(Resource):
     @login_required
     @account_initialization_required
     def post(self):
-        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+        if not current_user.is_admin_or_owner:
             raise Forbidden()
         
         user_id = current_user.id
@@ -159,7 +159,7 @@ class ToolApiProviderUpdateApi(Resource):
     @login_required
     @account_initialization_required
     def post(self):
-        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+        if not current_user.is_admin_or_owner:
             raise Forbidden()
         
         user_id = current_user.id
@@ -193,7 +193,7 @@ class ToolApiProviderDeleteApi(Resource):
     @login_required
     @account_initialization_required
     def post(self):
-        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+        if not current_user.is_admin_or_owner:
             raise Forbidden()
         
         user_id = current_user.id

api/controllers/service_api/wraps.py

@@ -76,7 +76,7 @@ def validate_dataset_token(view=None):
                 .filter(Tenant.id == api_token.tenant_id) \
                 .filter(TenantAccountJoin.tenant_id == Tenant.id) \
                 .filter(TenantAccountJoin.role.in_(['owner'])) \
-                .one_or_none()
+                .one_or_none() # TODO: only owner information is required, so only one is returned.
             if tenant_account_join:
                 tenant, ta = tenant_account_join
                 account = Account.query.filter_by(id=ta.account_id).first()
@@ -86,9 +86,9 @@ def validate_dataset_token(view=None):
                     current_app.login_manager._update_request_context_with_user(account)
                     user_logged_in.send(current_app._get_current_object(), user=_get_user())
                 else:
-                    raise Unauthorized("Tenant owner account is not exist.")
+                    raise Unauthorized("Tenant owner account does not exist.")
             else:
-                raise Unauthorized("Tenant is not exist.")
+                raise Unauthorized("Tenant does not exist.")
             return view(api_token.tenant_id, *args, **kwargs)
         return decorated
 

api/core/indexing_runner.py

@@ -562,7 +562,7 @@ class IndexingRunner:
 
             character_splitter = FixedRecursiveCharacterTextSplitter.from_encoder(
                 chunk_size=segmentation["max_tokens"],
-                chunk_overlap=0,
+                chunk_overlap=segmentation.get('chunk_overlap', 0),
                 fixed_separator=separator,
                 separators=["\n\n", "。", ".", " ", ""],
                 embedding_model_instance=embedding_model_instance
@@ -571,7 +571,7 @@ class IndexingRunner:
             # Automatic segmentation
             character_splitter = EnhanceRecursiveCharacterTextSplitter.from_encoder(
                 chunk_size=DatasetProcessRule.AUTOMATIC_RULES['segmentation']['max_tokens'],
-                chunk_overlap=0,
+                chunk_overlap=DatasetProcessRule.AUTOMATIC_RULES['segmentation']['chunk_overlap'],
                 separators=["\n\n", "。", ".", " ", ""],
                 embedding_model_instance=embedding_model_instance
             )

api/core/model_runtime/model_providers/openai/llm/_position.yaml

@@ -1,6 +1,8 @@
 - gpt-4
+- gpt-4-turbo-preview
 - gpt-4-32k
 - gpt-4-1106-preview
+- gpt-4-0125-preview
 - gpt-4-vision-preview
 - gpt-3.5-turbo
 - gpt-3.5-turbo-16k

api/core/model_runtime/model_providers/openai/llm/gpt-4-0125-preview.yaml

@@ -0,0 +1,58 @@
+model: gpt-4-0125-preview
+label:
+  zh_Hans: gpt-4-0125-preview
+  en_US: gpt-4-0125-preview
+model_type: llm
+features:
+  - multi-tool-call
+  - agent-thought
+model_properties:
+  mode: chat
+  context_size: 128000
+parameter_rules:
+  - name: temperature
+    use_template: temperature
+  - name: top_p
+    use_template: top_p
+  - name: presence_penalty
+    use_template: presence_penalty
+  - name: frequency_penalty
+    use_template: frequency_penalty
+  - name: max_tokens
+    use_template: max_tokens
+    default: 512
+    min: 1
+    max: 4096
+  - name: seed
+    label:
+      zh_Hans: 种子
+      en_US: Seed
+    type: int
+    help:
+      zh_Hans: 如果指定，模型将尽最大努力进行确定性采样，使得重复的具有相同种子和参数的请求应该返回相同的结果。不能保证确定性，您应该参考 system_fingerprint
+        响应参数来监视变化。
+      en_US: If specified, model will make a best effort to sample deterministically,
+        such that repeated requests with the same seed and parameters should return
+        the same result. Determinism is not guaranteed, and you should refer to the
+        system_fingerprint response parameter to monitor changes in the backend.
+    required: false
+    precision: 2
+    min: 0
+    max: 1
+  - name: response_format
+    label:
+      zh_Hans: 回复格式
+      en_US: response_format
+    type: string
+    help:
+      zh_Hans: 指定模型必须输出的格式
+      en_US: specifying the format that the model must output
+    required: false
+    options:
+      - text
+      - json_object
+pricing:
+  input: '0.01'
+  output: '0.03'
+  unit: '0.001'
+  currency: USD

api/core/model_runtime/model_providers/openai/llm/gpt-4-turbo-preview.yaml

@@ -0,0 +1,58 @@
+model: gpt-4-turbo-preview
+label:
+  zh_Hans: gpt-4-turbo-preview
+  en_US: gpt-4-turbo-preview
+model_type: llm
+features:
+  - multi-tool-call
+  - agent-thought
+model_properties:
+  mode: chat
+  context_size: 128000
+parameter_rules:
+  - name: temperature
+    use_template: temperature
+  - name: top_p
+    use_template: top_p
+  - name: presence_penalty
+    use_template: presence_penalty
+  - name: frequency_penalty
+    use_template: frequency_penalty
+  - name: max_tokens
+    use_template: max_tokens
+    default: 512
+    min: 1
+    max: 4096
+  - name: seed
+    label:
+      zh_Hans: 种子
+      en_US: Seed
+    type: int
+    help:
+      zh_Hans: 如果指定，模型将尽最大努力进行确定性采样，使得重复的具有相同种子和参数的请求应该返回相同的结果。不能保证确定性，您应该参考 system_fingerprint
+        响应参数来监视变化。
+      en_US: If specified, model will make a best effort to sample deterministically,
+        such that repeated requests with the same seed and parameters should return
+        the same result. Determinism is not guaranteed, and you should refer to the
+        system_fingerprint response parameter to monitor changes in the backend.
+    required: false
+    precision: 2
+    min: 0
+    max: 1
+  - name: response_format
+    label:
+      zh_Hans: 回复格式
+      en_US: response_format
+    type: string
+    help:
+      zh_Hans: 指定模型必须输出的格式
+      en_US: specifying the format that the model must output
+    required: false
+    options:
+      - text
+      - json_object
+pricing:
+  input: '0.01'
+  output: '0.03'
+  unit: '0.001'
+  currency: USD

api/core/model_runtime/model_providers/openai/text_embedding/text-embedding-3-large.yaml

@@ -0,0 +1,9 @@
+model: text-embedding-3-large
+model_type: text-embedding
+model_properties:
+  context_size: 8191
+  max_chunks: 32
+pricing:
+  input: '0.00013'
+  unit: '0.001'
+  currency: USD

api/core/model_runtime/model_providers/openai/text_embedding/text-embedding-3-small.yaml

@@ -0,0 +1,9 @@
+model: text-embedding-3-small
+model_type: text-embedding
+model_properties:
+  context_size: 8191
+  max_chunks: 32
+pricing:
+  input: '0.00002'
+  unit: '0.001'
+  currency: USD

api/models/account.py

@@ -101,7 +101,10 @@ class Account(UserMixin, db.Model):
         return db.session.query(ai).filter(
             ai.account_id == self.id
         ).all()
-
+    # check current_user.current_tenant.current_role in ['admin', 'owner']
+    @property
+    def is_admin_or_owner(self):
+        return self._current_tenant.current_role in ['admin', 'owner']
 
 class Tenant(db.Model):
     __tablename__ = 'tenants'

api/models/dataset.py

@@ -134,7 +134,8 @@ class DatasetProcessRule(db.Model):
         ],
         'segmentation': {
             'delimiter': '\n',
-            'max_tokens': 1000
+            'max_tokens': 500,
+            'chunk_overlap': 50
         }
     }
 

api/services/dataset_service.py

@@ -241,7 +241,8 @@ class DocumentService:
             ],
             'segmentation': {
                 'delimiter': '\n',
-                'max_tokens': 500
+                'max_tokens': 500,
+                'chunk_overlap': 50
             }
         }
     }

docker/docker-compose.yaml

@@ -2,7 +2,7 @@ version: '3.1'
 services:
   # API service
   api:
-    image: langgenius/dify-api:0.5.1
+    image: langgenius/dify-api:0.5.2
     restart: always
     environment:
       # Startup mode, 'api' starts the API server.
@@ -127,7 +127,7 @@ services:
   # worker service
   # The Celery worker for processing the queue.
   worker:
-    image: langgenius/dify-api:0.5.1
+    image: langgenius/dify-api:0.5.2
     restart: always
     environment:
       # Startup mode, 'worker' starts the Celery worker for processing the queue.
@@ -198,7 +198,7 @@ services:
 
   # Frontend web application.
   web:
-    image: langgenius/dify-web:0.5.1
+    image: langgenius/dify-web:0.5.2
     restart: always
     environment:
       EDITION: SELF_HOSTED

web/app/(commonLayout)/apps/AppCard.tsx

@@ -20,6 +20,7 @@ import type { HtmlContentProps } from '@/app/components/base/popover'
 import CustomPopover from '@/app/components/base/popover'
 import Divider from '@/app/components/base/divider'
 import { asyncRunSafe } from '@/utils'
+import { useProviderContext } from '@/context/provider-context'
 
 export type AppCardProps = {
   app: App
@@ -30,6 +31,7 @@ const AppCard = ({ app, onRefresh }: AppCardProps) => {
   const { t } = useTranslation()
   const { notify } = useContext(ToastContext)
   const { isCurrentWorkspaceManager } = useAppContext()
+  const { onPlanInfoChanged } = useProviderContext()
   const { push } = useRouter()
 
   const mutateApps = useContextSelector(
@@ -51,6 +53,7 @@ const AppCard = ({ app, onRefresh }: AppCardProps) => {
       if (onRefresh)
         onRefresh()
       mutateApps()
+      onPlanInfoChanged()
     }
     catch (e: any) {
       notify({

web/app/(commonLayout)/apps/NewAppCard.tsx

@@ -5,6 +5,7 @@ import classNames from 'classnames'
 import { useTranslation } from 'react-i18next'
 import style from '../list.module.css'
 import NewAppDialog from './NewAppDialog'
+import { useProviderContext } from '@/context/provider-context'
 
 export type CreateAppCardProps = {
   onSuccess?: () => void
@@ -12,6 +13,8 @@ export type CreateAppCardProps = {
 
 const CreateAppCard = forwardRef<HTMLAnchorElement, CreateAppCardProps>(({ onSuccess }, ref) => {
   const { t } = useTranslation()
+  const { onPlanInfoChanged } = useProviderContext()
+
   const [showNewAppDialog, setShowNewAppDialog] = useState(false)
   return (
     <a ref={ref} className={classNames(style.listItem, style.newItemCard)} onClick={() => setShowNewAppDialog(true)}>
@@ -24,7 +27,12 @@ const CreateAppCard = forwardRef<HTMLAnchorElement, CreateAppCardProps>(({ onSuc
         </div>
       </div>
       {/* <div className='text-xs text-gray-500'>{t('app.createFromConfigFile')}</div> */}
-      <NewAppDialog show={showNewAppDialog} onSuccess={onSuccess} onClose={() => setShowNewAppDialog(false)} />
+      <NewAppDialog show={showNewAppDialog} onSuccess={
+        () => {
+          onPlanInfoChanged()
+          if (onSuccess)
+            onSuccess()
+        }} onClose={() => setShowNewAppDialog(false)} />
     </a>
   )
 })

web/app/components/base/icons/assets/public/other/Icon-3-dots.svg

@@ -0,0 +1,3 @@
+<svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M5 6.5V5M8.93934 7.56066L10 6.5M10.0103 11.5H11.5103" stroke="#667085" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"/>
+</svg>

web/app/components/base/icons/src/public/other/Icon3Dots.json

@@ -0,0 +1,29 @@
+{
+	"icon": {
+		"type": "element",
+		"isRootNode": true,
+		"name": "svg",
+		"attributes": {
+			"width": "16",
+			"height": "16",
+			"viewBox": "0 0 16 16",
+			"fill": "none",
+			"xmlns": "http://www.w3.org/2000/svg"
+		},
+		"children": [
+			{
+				"type": "element",
+				"name": "path",
+				"attributes": {
+					"d": "M5 6.5V5M8.93934 7.56066L10 6.5M10.0103 11.5H11.5103",
+					"stroke": "#667085",
+					"stroke-width": "2",
+					"stroke-linecap": "round",
+					"stroke-linejoin": "round"
+				},
+				"children": []
+			}
+		]
+	},
+	"name": "Icon3Dots"
+}

web/app/components/base/icons/src/public/other/Icon3Dots.tsx

@@ -0,0 +1,16 @@
+// GENERATE BY script
+// DON NOT EDIT IT MANUALLY
+
+import * as React from 'react'
+import data from './Icon3Dots.json'
+import IconBase from '@/app/components/base/icons/IconBase'
+import type { IconBaseProps, IconData } from '@/app/components/base/icons/IconBase'
+
+const Icon = React.forwardRef<React.MutableRefObject<SVGElement>, Omit<IconBaseProps, 'data'>>((
+  props,
+  ref,
+) => <IconBase {...props} ref={ref} data={data as IconData} />)
+
+Icon.displayName = 'Icon3Dots'
+
+export default Icon

web/app/components/base/icons/src/public/other/index.ts

@@ -1 +1,2 @@
+export { default as Icon3Dots } from './Icon3Dots'
 export { default as DefaultToolIcon } from './DefaultToolIcon'

web/app/components/datasets/create/step-two/index.module.css

@@ -18,7 +18,7 @@
 }
 
 .form .label {
-  @apply pt-6 pb-2;
+  @apply pt-6 pb-2 flex items-center;
   font-weight: 500;
   font-size: 16px;
   line-height: 24px;

web/app/components/datasets/create/step-two/index.tsx

@@ -33,13 +33,14 @@ import { DataSourceType, DocForm } from '@/models/datasets'
 import NotionIcon from '@/app/components/base/notion-icon'
 import Switch from '@/app/components/base/switch'
 import { MessageChatSquare } from '@/app/components/base/icons/src/public/common'
-import { XClose } from '@/app/components/base/icons/src/vender/line/general'
+import { HelpCircle, XClose } from '@/app/components/base/icons/src/vender/line/general'
 import { useDatasetDetailContext } from '@/context/dataset-detail'
 import I18n from '@/context/i18n'
 import { IS_CE_EDITION } from '@/config'
 import { RETRIEVE_METHOD } from '@/types/app'
 import useBreakpoints, { MediaType } from '@/hooks/use-breakpoints'
 import Tooltip from '@/app/components/base/tooltip'
+import TooltipPlus from '@/app/components/base/tooltip-plus'
 import { useModelListAndDefaultModelAndCurrentProviderAndModel } from '@/app/components/header/account-setting/model-provider-page/hooks'
 import { LanguagesSupportedUnderscore, getModelRuntimeSupported } from '@/utils/language'
 
@@ -99,7 +100,8 @@ const StepTwo = ({
   const [previewScrolled, setPreviewScrolled] = useState(false)
   const [segmentationType, setSegmentationType] = useState<SegmentType>(SegmentType.AUTO)
   const [segmentIdentifier, setSegmentIdentifier] = useState('\\n')
-  const [max, setMax] = useState(1000)
+  const [max, setMax] = useState(500)
+  const [overlap, setOverlap] = useState(50)
   const [rules, setRules] = useState<PreProcessingRule[]>([])
   const [defaultConfig, setDefaultConfig] = useState<Rules>()
   const hasSetIndexType = !!indexingType
@@ -171,6 +173,7 @@ const StepTwo = ({
     if (defaultConfig) {
       setSegmentIdentifier((defaultConfig.segmentation.separator === '\n' ? '\\n' : defaultConfig.segmentation.separator) || '\\n')
       setMax(defaultConfig.segmentation.max_tokens)
+      setOverlap(defaultConfig.segmentation.chunk_overlap)
       setRules(defaultConfig.pre_processing_rules)
     }
   }
@@ -207,6 +210,7 @@ const StepTwo = ({
         segmentation: {
           separator: segmentIdentifier === '\\n' ? '\n' : segmentIdentifier,
           max_tokens: max,
+          chunk_overlap: overlap,
         },
       }
       processRule.rules = ruleObj
@@ -275,6 +279,10 @@ const StepTwo = ({
   } = useModelListAndDefaultModelAndCurrentProviderAndModel(3)
   const getCreationParams = () => {
     let params
+    if (segmentationType === SegmentType.CUSTOM && overlap > max) {
+      Toast.notify({ type: 'error', message: t('datasetCreation.stepTwo.overlapCheck') })
+      return
+    }
     if (isSetting) {
       params = {
         original_document_id: documentDetail?.id,
@@ -337,6 +345,7 @@ const StepTwo = ({
       const separator = res.rules.segmentation.separator
       setSegmentIdentifier((separator === '\n' ? '\\n' : separator) || '\\n')
       setMax(res.rules.segmentation.max_tokens)
+      setOverlap(res.rules.segmentation.chunk_overlap)
       setRules(res.rules.pre_processing_rules)
       setDefaultConfig(res.rules)
     }
@@ -350,8 +359,10 @@ const StepTwo = ({
       const rules = documentDetail.dataset_process_rule.rules
       const separator = rules.segmentation.separator
       const max = rules.segmentation.max_tokens
+      const overlap = rules.segmentation.chunk_overlap
       setSegmentIdentifier((separator === '\n' ? '\\n' : separator) || '\\n')
       setMax(max)
+      setOverlap(overlap)
       setRules(rules.pre_processing_rules)
       setDefaultConfig(rules)
     }
@@ -569,13 +580,35 @@ const StepTwo = ({
                       <input
                         type="number"
                         className={s.input}
-                        placeholder={t('datasetCreation.stepTwo.separatorPlaceholder') || ''}
+                        placeholder={t('datasetCreation.stepTwo.maxLength') || ''}
                         value={max}
                         min={1}
                         onChange={e => setMax(parseInt(e.target.value.replace(/^0+/, ''), 10))}
                       />
                     </div>
                   </div>
+                  <div className={s.formRow}>
+                    <div className='w-full'>
+                      <div className={s.label}>
+                        {t('datasetCreation.stepTwo.overlap')}
+                        <TooltipPlus popupContent={
+                          <div className='max-w-[200px]'>
+                            {t('datasetCreation.stepTwo.overlapTip')}
+                          </div>
+                        }>
+                          <HelpCircle className='ml-1 w-3.5 h-3.5 text-gray-400' />
+                        </TooltipPlus>
+                      </div>
+                      <input
+                        type="number"
+                        className={s.input}
+                        placeholder={t('datasetCreation.stepTwo.overlap') || ''}
+                        value={overlap}
+                        min={1}
+                        onChange={e => setOverlap(parseInt(e.target.value.replace(/^0+/, ''), 10))}
+                      />
+                    </div>
+                  </div>
                   <div className={s.formRow}>
                     <div className='w-full flex flex-col gap-1'>
                       <div className={s.label}>{t('datasetCreation.stepTwo.rules')}</div>

web/app/components/header/maintenance-notice.tsx

@@ -7,7 +7,9 @@ import { NOTICE_I18N } from '@/utils/language'
 const MaintenanceNotice = () => {
   const { locale } = useContext(I18n)
   const [showNotice, setShowNotice] = useState(localStorage.getItem('hide-maintenance-notice') !== '1')
-
+  const handleJumpNotice = () => {
+    window.open(NOTICE_I18N.href, '_blank')
+  }
   const handleCloseNotice = () => {
     localStorage.setItem('hide-maintenance-notice', '1')
     setShowNotice(false)
@@ -22,8 +24,8 @@ const MaintenanceNotice = () => {
   return (
     <div className='shrink-0 flex items-center px-4 h-[38px] bg-[#FFFAEB] border-b border-[0.5px] border-b-[#FEF0C7] z-20'>
       <div className='shrink-0 flex items-center mr-2 px-2 h-[22px] bg-[#F79009] text-white text-[11px] font-medium rounded-xl'>{titleByLocale[locale]}</div>
-      <div className='grow text-xs font-medium text-gray-700'>{descByLocale[locale]}</div>
-      <X className='shrink-0 w-4 h-4 text-gray-500 cursor-pointer' onClick={handleCloseNotice} />
+      <div className='grow text-xs font-medium text-gray-700 cursor-pointer' onClick={handleJumpNotice}>{descByLocale[locale]}</div>
+      <X className='shrink-0 w-4 h-4 text-gray-500 cursor-pointer' onClick={handleCloseNotice}/>
     </div>
   )
 }

web/app/components/tools/edit-custom-collection-modal/index.tsx

@@ -205,7 +205,7 @@ const EditCustomCollectionModal: FC<Props> = ({
                           <td className="p-2 pl-3">{item.server_url ? new URL(item.server_url).pathname : ''}</td>
                           <td className="p-2 pl-3 w-[62px]">
                             <Button
-                              className='!h-6 !px-2 text-xs font-medium text-gray-700'
+                              className='!h-6 !px-2 text-xs font-medium text-gray-700 whitespace-nowrap'
                               onClick={() => {
                                 setCurrTool(item)
                                 setIsShowTestApi(true)

web/app/components/tools/index.tsx

@@ -15,6 +15,7 @@ import ToolList from './tool-list'
 import EditCustomToolModal from './edit-custom-collection-modal'
 import NoCustomTool from './info/no-custom-tool'
 import NoSearchRes from './info/no-search-res'
+import NoCustomToolPlaceholder from './no-custom-tool-placeholder'
 import TabSlider from '@/app/components/base/tab-slider'
 import { createCustomCollection, fetchCollectionList as doFetchCollectionList, fetchBuiltInToolList, fetchCustomToolList } from '@/service/tools'
 import type { AgentTool } from '@/types/app'
@@ -216,6 +217,10 @@ const Tools: FC<Props> = ({
                 isLoading={isDetailLoading}
               />
             )}
+
+            {collectionType === CollectionType.custom && hasNoCustomCollection && (
+              <NoCustomToolPlaceholder />
+            )}
           </div>
         </div>
       </div>

web/app/components/tools/info/no-custom-tool.tsx

@@ -2,8 +2,8 @@
 import type { FC } from 'react'
 import React from 'react'
 import { useTranslation } from 'react-i18next'
+import { Icon3Dots } from '../../base/icons/src/public/other'
 import { Tools } from '@/app/components/base/icons/src/public/header-nav/tools'
-
 type Props = {
   onCreateTool: () => void
 }
@@ -20,7 +20,7 @@ const NoCustomTool: FC<Props> = ({
       </div>
       <div className='mt-2'>
         <div className='leading-5 text-sm font-medium text-gray-500'>
-          {t('tools.noCustomTool.title')}
+          {t('tools.noCustomTool.title')}<Icon3Dots className='inline relative -top-3 -left-1.5' />
         </div>
         <div className='mt-1 leading-[18px] text-xs font-normal text-gray-500'>
           {t('tools.noCustomTool.content')}

web/app/components/tools/no-custom-tool-placeholder.tsx

@@ -0,0 +1,26 @@
+'use client'
+import type { FC } from 'react'
+import React from 'react'
+import { useTranslation } from 'react-i18next'
+import { BookOpen01 } from '../base/icons/src/vender/line/education'
+import { Icon3Dots } from '../base/icons/src/public/other'
+
+const NoCustomToolPlaceHolder: FC = () => {
+  const { t } = useTranslation()
+
+  return (
+    <div className='h-full flex items-center justify-center'>
+      <div className='p-6 rounded-xl bg-gray-50'>
+        <div className='inline-flex p-2 border border-gray-200 rounded-md'>
+          <BookOpen01 className='w-4 h-4 text-primary-600' />
+        </div>
+        <div className='mt-3 leading-6 text-base font-medium text-gray-700'>
+          {t('tools.noCustomTool.title')}
+          <Icon3Dots className='inline relative -top-3 -left-1.5' />
+        </div>
+        <div className='mt-2 leading-5 text-sm font-normal text-gray-700'>{t('tools.noCustomTool.content')}</div>
+      </div>
+    </div>
+  )
+}
+export default React.memo(NoCustomToolPlaceHolder)

web/context/provider-context.tsx

@@ -34,32 +34,34 @@ const ProviderContext = createContext<{
   }
   isFetchedPlan: boolean
   enableBilling: boolean
+  onPlanInfoChanged: () => void
   enableReplaceWebAppLogo: boolean
 }>({
-  modelProviders: [],
-  textGenerationModelList: [],
-  agentThoughtModelList: [],
-  supportRetrievalMethods: [],
-  hasSettedApiKey: true,
-  plan: {
-    type: Plan.sandbox,
-    usage: {
-      vectorSpace: 32,
-      buildApps: 12,
-      teamMembers: 1,
-      annotatedResponse: 1,
-    },
-    total: {
-      vectorSpace: 200,
-      buildApps: 50,
-      teamMembers: 1,
-      annotatedResponse: 10,
-    },
-  },
-  isFetchedPlan: false,
-  enableBilling: false,
-  enableReplaceWebAppLogo: false,
-})
+      modelProviders: [],
+      textGenerationModelList: [],
+      agentThoughtModelList: [],
+      supportRetrievalMethods: [],
+      hasSettedApiKey: true,
+      plan: {
+        type: Plan.sandbox,
+        usage: {
+          vectorSpace: 32,
+          buildApps: 12,
+          teamMembers: 1,
+          annotatedResponse: 1,
+        },
+        total: {
+          vectorSpace: 200,
+          buildApps: 50,
+          teamMembers: 1,
+          annotatedResponse: 10,
+        },
+      },
+      isFetchedPlan: false,
+      enableBilling: false,
+      onPlanInfoChanged: () => { },
+      enableReplaceWebAppLogo: false,
+    })
 
 export const useProviderContext = () => useContext(ProviderContext)
 
@@ -98,7 +100,6 @@ export const ProviderContextProvider = ({
   const [isFetchedPlan, setIsFetchedPlan] = useState(false)
   const [enableBilling, setEnableBilling] = useState(true)
   const [enableReplaceWebAppLogo, setEnableReplaceWebAppLogo] = useState(false)
-
   const handleOperateUtm = () => {
     let utm
     try {
@@ -116,18 +117,19 @@ export const ProviderContextProvider = ({
     if (utm.utm_source || utm.utm_medium || utm.utm_campaign || utm.utm_content || utm.utm_term)
       operationUtm({ url: '/operation/utm', body: utm })
   }
+  const fetchPlan = async () => {
+    const data = await fetchCurrentPlanInfo()
+    const enabled = data.billing.enabled
+    setEnableBilling(enabled)
+    setEnableReplaceWebAppLogo(data.can_replace_logo)
+    if (enabled) {
+      setPlan(parseCurrentPlan(data))
+      handleOperateUtm()
+      setIsFetchedPlan(true)
+    }
+  }
   useEffect(() => {
-    (async () => {
-      const data = await fetchCurrentPlanInfo()
-      const enabled = data.billing.enabled
-      setEnableBilling(enabled)
-      setEnableReplaceWebAppLogo(data.can_replace_logo)
-      if (enabled) {
-        setPlan(parseCurrentPlan(data))
-        handleOperateUtm()
-        setIsFetchedPlan(true)
-      }
-    })()
+    fetchPlan()
   }, [])
 
   return (
@@ -140,6 +142,7 @@ export const ProviderContextProvider = ({
       plan,
       isFetchedPlan,
       enableBilling,
+      onPlanInfoChanged: fetchPlan,
       enableReplaceWebAppLogo,
     }}>
       {children}

web/i18n/lang/common.en.ts

@@ -249,8 +249,8 @@ const translation = {
       tip: 'Set the default model for speech-to-text input in conversation.',
     },
     ttsModel: {
-      key: 'Speech-to-Text Model',
-      tip: 'Set the default model for speech-to-text input in conversation.',
+      key: 'Text-to-Speech Model',
+      tip: 'Set the default model for text-to-speech input in conversation.',
     },
     rerankModel: {
       key: 'Rerank Model',

web/i18n/lang/dataset-creation.en.ts

@@ -59,6 +59,9 @@ const translation = {
     separator: 'Segment identifier',
     separatorPlaceholder: 'For example, newline (\\\\n) or special separator (such as "***")',
     maxLength: 'Maximum chunk length',
+    overlap: 'Chunk overlap',
+    overlapTip: 'Setting the chunk overlap can maintain the semantic relevance between them, enhancing the retrieve effect. It is recommended to set 10%-25% of the maximum chunk size.',
+    overlapCheck: 'chunk overlap should not bigger than maximun chunk length',
     rules: 'Text preprocessing rules',
     removeExtraSpaces: 'Replace consecutive spaces, newlines and tabs',
     removeUrlEmails: 'Delete all URLs and email addresses',

web/i18n/lang/dataset-creation.pt.ts

@@ -59,6 +59,9 @@ const translation = {
     separator: 'Identificador de segmento',
     separatorPlaceholder: 'Por exemplo, nova linha (\\\\n) ou separador especial (como "***")',
     maxLength: 'Comprimento máximo do fragmento',
+    overlap: 'Sobreposição de blocos',
+    overlapTip: 'Configurar a sobreposição de blocos pode manter a relevância semântica entre eles, melhorando o efeito de recuperação. É recomendado definir de 10% a 25% do tamanho máximo do bloco.',
+    overlapCheck: 'a sobreposição de blocos não deve ser maior que o comprimento máximo do bloco',
     rules: 'Regras de pré-processamento de texto',
     removeExtraSpaces: 'Substituir espaços consecutivos, quebras de linha e tabulações',
     removeUrlEmails: 'Excluir todos os URLs e endereços de e-mail',

web/i18n/lang/dataset-creation.zh.ts

@@ -59,6 +59,9 @@ const translation = {
     separator: '分段标识符',
     separatorPlaceholder: '例如换行符（\n）或特定的分隔符（如 "***"）',
     maxLength: '分段最大长度',
+    overlap: '分段重叠长度',
+    overlapTip: '设置分段之间的重叠长度可以保留分段之间的语义关系，提升召回效果。建议设置为最大分段长度的10%-25%',
+    overlapCheck: '分段重叠长度不能大于分段最大长度',
     rules: '文本预处理规则',
     removeExtraSpaces: '替换掉连续的空格、换行符和制表符',
     removeUrlEmails: '删除所有 URL 和电子邮件地址',

web/i18n/lang/tools.en.ts

@@ -86,7 +86,7 @@ const translation = {
     infoAndSetting: 'Info & Settings',
   },
   noCustomTool: {
-    title: 'No custom tools',
+    title: 'No custom tools!',
     content: 'Add and manage your custom tools here for building AI apps.',
     createTool: 'Create Tool',
   },

web/i18n/lang/tools.pt.ts

@@ -86,7 +86,7 @@ const translation = {
     infoAndSetting: 'Informações e Configurações',
   },
   noCustomTool: {
-    title: 'Nenhuma ferramenta personalizada',
+    title: 'Nenhuma ferramenta personalizada!',
     content: 'Você não possui ferramentas personalizadas. ',
     createTool: 'Criar Ferramenta',
   },

web/i18n/lang/tools.zh.ts

@@ -78,7 +78,7 @@ const translation = {
     infoAndSetting: '信息和设置',
   },
   noCustomTool: {
-    title: '没有自定义工具',
+    title: '没有自定义工具!',
     content: '在此统一添加和管理你的自定义工具，方便构建应用时使用。',
     createTool: '创建工具',
   },

web/models/datasets.ts

@@ -108,6 +108,7 @@ export type PreProcessingRule = {
 export type Segmentation = {
   separator: string
   max_tokens: number
+  chunk_overlap: number
 }
 
 export const DocumentIndexingStatusList = [

web/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dify-web",
-  "version": "0.5.1",
+  "version": "0.5.2",
   "private": true,
   "scripts": {
     "dev": "next dev",

web/utils/language.ts

@@ -102,4 +102,5 @@ export const NOTICE_I18N = {
     'ja-JP': 'Our system will be unavailable from 19:00 to 24:00 UTC on August 28 for an upgrade. For questions, kindly contact our support team (support@dify.ai). We value your patience.',
     'ko-KR': 'Our system will be unavailable from 19:00 to 24:00 UTC on August 28 for an upgrade. For questions, kindly contact our support team (support@dify.ai). We value your patience.',
   },
+  href: '#',
 }