youngkingdom/dify
1
0
Fork 0
mirror of https://github.com/langgenius/dify.git synced 2026-05-03 17:08:03 +08:00
Code Issues Packages Projects Releases Wiki Activity

allow to config max segmentation tokens length for RAG document using environment variable (#4375)

Browse Source
This commit is contained in:
Rain Chen
2024-05-20 13:20:27 +08:00
committed by GitHub
parent b5204111da
commit c255a20d7c
5 changed files with 21 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
api/core/rag/index_processor/index_processor_base.py
View File

@ -2,6 +2,8 @@
from abc import ABC, abstractmethod
from typing import Optional
from flask import current_app
from core.model_manager import ModelInstance
from core.rag.extractor.entity.extract_setting import ExtractSetting
from core.rag.models.document import Document
@ -43,8 +45,9 @@ class BaseIndexProcessor(ABC):
# The user-defined segmentation rule
rules = processing_rule['rules']
segmentation = rules["segmentation"]
if segmentation["max_tokens"] < 50 or segmentation["max_tokens"] > 1000:
raise ValueError("Custom segment length should be between 50 and 1000.")
max_segmentation_tokens_length = int(current_app.config['INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH'])
if segmentation["max_tokens"] < 50 or segmentation["max_tokens"] > max_segmentation_tokens_length:
raise ValueError(f"Custom segment length should be between 50 and {max_segmentation_tokens_length}.")
separator = segmentation["separator"]
if separator: