mirror of
https://github.com/langgenius/dify.git
synced 2026-05-04 09:28:04 +08:00
Plugin/merge main to plugin/beta 20250122 (#12962)
Signed-off-by: yihong0618 <zouzou0208@gmail.com> Signed-off-by: -LAN- <laipz8200@outlook.com> Co-authored-by: kurokobo <kuro664@gmail.com> Co-authored-by: Hiroshi Fujita <fujita-h@users.noreply.github.com> Co-authored-by: NFish <douxc512@gmail.com> Co-authored-by: Gen Sato <52241300+halogen22@users.noreply.github.com> Co-authored-by: eux <euxuuu@gmail.com> Co-authored-by: huangzhuo1949 <167434202+huangzhuo1949@users.noreply.github.com> Co-authored-by: huangzhuo <huangzhuo1@xiaomi.com> Co-authored-by: lotsik <lotsik@mail.ru> Co-authored-by: crazywoola <100913391+crazywoola@users.noreply.github.com> Co-authored-by: Wu Tianwei <30284043+WTW0313@users.noreply.github.com> Co-authored-by: nite-knite <nkCoding@gmail.com> Co-authored-by: Jyong <76649700+JohnJyong@users.noreply.github.com> Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: gakkiyomi <gakkiyomi@aliyun.com> Co-authored-by: CN-P5 <heibai2006@gmail.com> Co-authored-by: CN-P5 <heibai2006@qq.com> Co-authored-by: Chuehnone <1897025+chuehnone@users.noreply.github.com> Co-authored-by: yihong <zouzou0208@gmail.com> Co-authored-by: Kevin9703 <51311316+Kevin9703@users.noreply.github.com> Co-authored-by: -LAN- <laipz8200@outlook.com> Co-authored-by: Boris Feld <lothiraldan@gmail.com> Co-authored-by: mbo <himabo@gmail.com> Co-authored-by: mabo <mabo@aeyes.ai> Co-authored-by: Warren Chen <warren.chen830@gmail.com> Co-authored-by: KVOJJJin <jzongcode@gmail.com> Co-authored-by: JzoNgKVO <27049666+JzoNgKVO@users.noreply.github.com> Co-authored-by: jiandanfeng <chenjh3@wangsu.com> Co-authored-by: zhu-an <70234959+xhdd123321@users.noreply.github.com> Co-authored-by: zhaoqingyu.1075 <zhaoqingyu.1075@bytedance.com> Co-authored-by: 海狸大師 <86974027+yenslife@users.noreply.github.com> Co-authored-by: Xu Song <xusong.vip@gmail.com> Co-authored-by: rayshaw001 <396301947@163.com> Co-authored-by: Ding Jiatong <dingjiatong@gmail.com> Co-authored-by: Bowen Liang <liangbowen@gf.com.cn> Co-authored-by: JasonVV <jasonwangiii@outlook.com> Co-authored-by: le0zh <newlight@qq.com> Co-authored-by: zhuxinliang <zhuxinliang@didiglobal.com> Co-authored-by: k-zaku <zaku99@outlook.jp> Co-authored-by: Joel <iamjoel007@gmail.com> Co-authored-by: luckylhb90 <luckylhb90@gmail.com> Co-authored-by: hobo.l <hobo.l@binance.com> Co-authored-by: jiangbo721 <365065261@qq.com> Co-authored-by: 刘江波 <jiangbo721@163.com> Co-authored-by: Shun Miyazawa <34241526+miya@users.noreply.github.com> Co-authored-by: EricPan <30651140+Egfly@users.noreply.github.com> Co-authored-by: crazywoola <427733928@qq.com> Co-authored-by: zxhlyh <jasonapring2015@outlook.com> Co-authored-by: sino <sino2322@gmail.com> Co-authored-by: Jhvcc <37662342+Jhvcc@users.noreply.github.com> Co-authored-by: lowell <lowell.hu@zkteco.in>
This commit is contained in:
Yeuoly
GitHub
@ -37,7 +37,7 @@ import Button from '@/app/components/base/button'
|
||||
import FloatRightContainer from '@/app/components/base/float-right-container'
|
||||
import RetrievalMethodConfig from '@/app/components/datasets/common/retrieval-method-config'
|
||||
import EconomicalRetrievalMethodConfig from '@/app/components/datasets/common/economical-retrieval-method-config'
|
||||
import { type RetrievalConfig } from '@/types/app'
|
||||
import type { RetrievalConfig } from '@/types/app'
|
||||
import { isReRankModelSelected } from '@/app/components/datasets/common/check-rerank-model'
|
||||
import Toast from '@/app/components/base/toast'
|
||||
import type { NotionPage } from '@/models/common'
|
||||
@ -98,6 +98,7 @@ export enum IndexingType {
|
||||
const DEFAULT_SEGMENT_IDENTIFIER = '\\n\\n'
|
||||
const DEFAULT_MAXIMUM_CHUNK_LENGTH = 500
|
||||
const DEFAULT_OVERLAP = 50
|
||||
const MAXIMUM_CHUNK_TOKEN_LENGTH = Number.parseInt(globalThis.document?.body?.getAttribute('data-public-indexing-max-segmentation-tokens-length') || '4000', 10)
|
||||
|
||||
type ParentChildConfig = {
|
||||
chunkForContext: ParentMode
|
||||
@ -163,7 +164,7 @@ const StepTwo = ({
|
||||
doSetSegmentIdentifier(value ? escape(value) : (canEmpty ? '' : DEFAULT_SEGMENT_IDENTIFIER))
|
||||
}, [])
|
||||
const [maxChunkLength, setMaxChunkLength] = useState(DEFAULT_MAXIMUM_CHUNK_LENGTH) // default chunk length
|
||||
const [limitMaxChunkLength, setLimitMaxChunkLength] = useState(4000)
|
||||
const [limitMaxChunkLength, setLimitMaxChunkLength] = useState(MAXIMUM_CHUNK_TOKEN_LENGTH)
|
||||
const [overlap, setOverlap] = useState(DEFAULT_OVERLAP)
|
||||
const [rules, setRules] = useState<PreProcessingRule[]>([])
|
||||
const [defaultConfig, setDefaultConfig] = useState<Rules>()
|
||||
@ -205,7 +206,7 @@ const StepTwo = ({
|
||||
if (value === ChunkingMode.parentChild && indexType === IndexingType.ECONOMICAL)
|
||||
setIndexType(IndexingType.QUALIFIED)
|
||||
setDocForm(value)
|
||||
// eslint-disable-next-line @typescript-eslint/no-use-before-define
|
||||
// eslint-disable-next-line ts/no-use-before-define
|
||||
currentEstimateMutation.reset()
|
||||
}
|
||||
|
||||
@ -342,8 +343,8 @@ const StepTwo = ({
|
||||
}
|
||||
|
||||
const updatePreview = () => {
|
||||
if (segmentationType === ProcessMode.general && maxChunkLength > 4000) {
|
||||
Toast.notify({ type: 'error', message: t('datasetCreation.stepTwo.maxLengthCheck') })
|
||||
if (segmentationType === ProcessMode.general && maxChunkLength > MAXIMUM_CHUNK_TOKEN_LENGTH) {
|
||||
Toast.notify({ type: 'error', message: t('datasetCreation.stepTwo.maxLengthCheck', { limit: MAXIMUM_CHUNK_TOKEN_LENGTH }) })
|
||||
return
|
||||
}
|
||||
fetchEstimate()
|
||||
@ -393,7 +394,7 @@ const StepTwo = ({
|
||||
score_threshold_enabled: false,
|
||||
score_threshold: 0.5,
|
||||
})
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [rerankDefaultModel, isRerankDefaultModelValid])
|
||||
|
||||
const getCreationParams = () => {
|
||||
@ -575,6 +576,8 @@ const StepTwo = ({
|
||||
const economyDomRef = useRef<HTMLDivElement>(null)
|
||||
const isHoveringEconomy = useHover(economyDomRef)
|
||||
|
||||
const isModelAndRetrievalConfigDisabled = !!datasetId && !!currentDataset?.data_source_type
|
||||
|
||||
return (
|
||||
<div className='flex w-full h-full'>
|
||||
<div className={cn('relative h-full w-1/2 py-6 overflow-y-auto', isMobile ? 'px-4' : 'px-12')}>
|
||||
@ -931,15 +934,15 @@ const StepTwo = ({
|
||||
<div className='mt-5'>
|
||||
<div className={cn('system-md-semibold mb-1', datasetId && 'flex justify-between items-center')}>{t('datasetSettings.form.embeddingModel')}</div>
|
||||
<ModelSelector
|
||||
readonly={!!datasetId}
|
||||
triggerClassName={datasetId ? 'opacity-50' : ''}
|
||||
readonly={isModelAndRetrievalConfigDisabled}
|
||||
triggerClassName={isModelAndRetrievalConfigDisabled ? 'opacity-50' : ''}
|
||||
defaultModel={embeddingModel}
|
||||
modelList={embeddingModelList}
|
||||
onSelect={(model: DefaultModel) => {
|
||||
setEmbeddingModel(model)
|
||||
}}
|
||||
/>
|
||||
{!!datasetId && (
|
||||
{isModelAndRetrievalConfigDisabled && (
|
||||
<div className='mt-2 system-xs-medium'>
|
||||
{t('datasetCreation.stepTwo.indexSettingTip')}
|
||||
<Link className='text-text-accent' href={`/datasets/${datasetId}/settings`}>{t('datasetCreation.stepTwo.datasetSettingLink')}</Link>
|
||||
@ -950,7 +953,7 @@ const StepTwo = ({
|
||||
<Divider className='my-5' />
|
||||
{/* Retrieval Method Config */}
|
||||
<div>
|
||||
{!datasetId
|
||||
{!isModelAndRetrievalConfigDisabled
|
||||
? (
|
||||
<div className={'mb-1'}>
|
||||
<div className='system-md-semibold mb-0.5'>{t('datasetSettings.form.retrievalSetting.title')}</div>
|
||||
@ -971,14 +974,14 @@ const StepTwo = ({
|
||||
getIndexing_technique() === IndexingType.QUALIFIED
|
||||
? (
|
||||
<RetrievalMethodConfig
|
||||
disabled={!!datasetId}
|
||||
disabled={isModelAndRetrievalConfigDisabled}
|
||||
value={retrievalConfig}
|
||||
onChange={setRetrievalConfig}
|
||||
/>
|
||||
)
|
||||
: (
|
||||
<EconomicalRetrievalMethodConfig
|
||||
disabled={!!datasetId}
|
||||
disabled={isModelAndRetrievalConfigDisabled}
|
||||
value={retrievalConfig}
|
||||
onChange={setRetrievalConfig}
|
||||
/>
|
||||
@ -999,7 +1002,7 @@ const StepTwo = ({
|
||||
)
|
||||
: (
|
||||
<div className='flex items-center mt-8 py-2'>
|
||||
{!datasetId && <Button loading={isCreating} variant='primary' onClick={createHandle}>{t('datasetCreation.stepTwo.save')}</Button>}
|
||||
<Button loading={isCreating} variant='primary' onClick={createHandle}>{t('datasetCreation.stepTwo.save')}</Button>
|
||||
<Button className='ml-2' onClick={onCancel}>{t('datasetCreation.stepTwo.cancel')}</Button>
|
||||
</div>
|
||||
)}
|
||||
@ -1070,10 +1073,9 @@ const StepTwo = ({
|
||||
}
|
||||
{
|
||||
currentDocForm !== ChunkingMode.qa
|
||||
&& <Badge text={t(
|
||||
'datasetCreation.stepTwo.previewChunkCount', {
|
||||
count: estimate?.total_segments || 0,
|
||||
}) as string}
|
||||
&& <Badge text={t('datasetCreation.stepTwo.previewChunkCount', {
|
||||
count: estimate?.total_segments || 0,
|
||||
}) as string}
|
||||
/>
|
||||
}
|
||||
</div>
|
||||
|
||||
@ -39,6 +39,8 @@ export const DelimiterInput: FC<InputProps & { tooltip?: string }> = (props) =>
|
||||
}
|
||||
|
||||
export const MaxLengthInput: FC<InputNumberProps> = (props) => {
|
||||
const maxValue = parseInt(globalThis.document?.body?.getAttribute('data-public-indexing-max-segmentation-tokens-length') || '4000', 10)
|
||||
|
||||
const { t } = useTranslation()
|
||||
return <FormField label={<div className='system-sm-semibold mb-1'>
|
||||
{t('datasetCreation.stepTwo.maxLength')}
|
||||
@ -46,8 +48,8 @@ export const MaxLengthInput: FC<InputNumberProps> = (props) => {
|
||||
<InputNumber
|
||||
type="number"
|
||||
className='h-9'
|
||||
placeholder={'≤ 4000'}
|
||||
max={4000}
|
||||
placeholder={`≤ ${maxValue}`}
|
||||
max={maxValue}
|
||||
min={1}
|
||||
{...props}
|
||||
/>
|
||||
|
||||
Reference in New Issue
Block a user