mirror of
https://github.com/langgenius/dify.git
synced 2026-05-04 01:18:05 +08:00
feat: summary index
This commit is contained in:
zxhlyh
@ -2,7 +2,7 @@
|
||||
import type { FC, PropsWithChildren } from 'react'
|
||||
import type { DefaultModel } from '@/app/components/header/account-setting/model-provider-page/declarations'
|
||||
import type { NotionPage } from '@/models/common'
|
||||
import type { CrawlOptions, CrawlResultItem, CreateDocumentReq, createDocumentResponse, CustomFile, DocumentItem, FullDocumentDetail, ParentMode, PreProcessingRule, ProcessRule, Rules } from '@/models/datasets'
|
||||
import type { CrawlOptions, CrawlResultItem, CreateDocumentReq, createDocumentResponse, CustomFile, DocumentItem, FullDocumentDetail, ParentMode, PreProcessingRule, ProcessRule, Rules, SummaryIndexSetting as SummaryIndexSettingType } from '@/models/datasets'
|
||||
import type { RetrievalConfig } from '@/types/app'
|
||||
import {
|
||||
RiAlertFill,
|
||||
@ -12,7 +12,7 @@ import {
|
||||
import { noop } from 'es-toolkit/function'
|
||||
import Image from 'next/image'
|
||||
import Link from 'next/link'
|
||||
import { useCallback, useEffect, useMemo, useState } from 'react'
|
||||
import { useCallback, useEffect, useMemo, useRef, useState } from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import { trackEvent } from '@/app/components/base/amplitude'
|
||||
import Badge from '@/app/components/base/badge'
|
||||
@ -29,8 +29,8 @@ import Toast from '@/app/components/base/toast'
|
||||
import Tooltip from '@/app/components/base/tooltip'
|
||||
import { isReRankModelSelected } from '@/app/components/datasets/common/check-rerank-model'
|
||||
import EconomicalRetrievalMethodConfig from '@/app/components/datasets/common/economical-retrieval-method-config'
|
||||
import RetrievalMethodConfig from '@/app/components/datasets/common/retrieval-method-config'
|
||||
|
||||
import RetrievalMethodConfig from '@/app/components/datasets/common/retrieval-method-config'
|
||||
import { ModelTypeEnum } from '@/app/components/header/account-setting/model-provider-page/declarations'
|
||||
import { useDefaultModel, useModelList, useModelListAndDefaultModelAndCurrentProviderAndModel } from '@/app/components/header/account-setting/model-provider-page/hooks'
|
||||
import ModelSelector from '@/app/components/header/account-setting/model-provider-page/model-selector'
|
||||
@ -51,6 +51,7 @@ import { PreviewSlice } from '../../formatted-text/flavours/preview-slice'
|
||||
import { FormattedText } from '../../formatted-text/formatted'
|
||||
import PreviewContainer from '../../preview/container'
|
||||
import { PreviewHeader } from '../../preview/header'
|
||||
import SummaryIndexSetting from '../../settings/summary-index-setting'
|
||||
import { checkShowMultiModalTip } from '../../settings/utils'
|
||||
import FileList from '../assets/file-list-3-fill.svg'
|
||||
import Note from '../assets/note-mod.svg'
|
||||
@ -174,6 +175,18 @@ const StepTwo = ({
|
||||
const [limitMaxChunkLength, setLimitMaxChunkLength] = useState(MAXIMUM_CHUNK_TOKEN_LENGTH)
|
||||
const [overlap, setOverlap] = useState(DEFAULT_OVERLAP)
|
||||
const [rules, setRules] = useState<PreProcessingRule[]>([])
|
||||
const [summaryIndexSetting, setSummaryIndexSetting] = useState<SummaryIndexSettingType | undefined>(currentDataset?.summary_index_setting)
|
||||
const summaryIndexSettingRef = useRef<SummaryIndexSettingType | undefined>(currentDataset?.summary_index_setting)
|
||||
const handleSummaryIndexSettingChange = useCallback((payload?: SummaryIndexSettingType, reset?: boolean) => {
|
||||
if (reset) {
|
||||
setSummaryIndexSetting(payload)
|
||||
summaryIndexSettingRef.current = payload
|
||||
}
|
||||
else {
|
||||
setSummaryIndexSetting({ ...summaryIndexSettingRef.current, ...payload })
|
||||
summaryIndexSettingRef.current = { ...summaryIndexSettingRef.current, ...payload }
|
||||
}
|
||||
}, [setSummaryIndexSetting, summaryIndexSettingRef])
|
||||
const [defaultConfig, setDefaultConfig] = useState<Rules>()
|
||||
const hasSetIndexType = !!indexingType
|
||||
const [indexType, setIndexType] = useState<IndexingType>(() => {
|
||||
@ -250,6 +263,7 @@ const StepTwo = ({
|
||||
},
|
||||
},
|
||||
mode: 'hierarchical',
|
||||
summary_index_setting: summaryIndexSetting,
|
||||
} as ProcessRule
|
||||
}
|
||||
return {
|
||||
@ -262,6 +276,7 @@ const StepTwo = ({
|
||||
},
|
||||
}, // api will check this. It will be removed after api refactored.
|
||||
mode: segmentationType,
|
||||
summary_index_setting: summaryIndexSetting,
|
||||
} as ProcessRule
|
||||
}
|
||||
|
||||
@ -352,6 +367,7 @@ const StepTwo = ({
|
||||
setMaxChunkLength(defaultConfig.segmentation.max_tokens)
|
||||
setOverlap(defaultConfig.segmentation.chunk_overlap!)
|
||||
setRules(defaultConfig.pre_processing_rules)
|
||||
handleSummaryIndexSettingChange(currentDataset?.summary_index_setting, true)
|
||||
}
|
||||
setParentChildConfig(defaultParentChildConfig)
|
||||
}
|
||||
@ -492,6 +508,7 @@ const StepTwo = ({
|
||||
setMaxChunkLength(data.rules.segmentation.max_tokens)
|
||||
setOverlap(data.rules.segmentation.chunk_overlap!)
|
||||
setRules(data.rules.pre_processing_rules)
|
||||
handleSummaryIndexSettingChange(data.summary_index_setting, true)
|
||||
setDefaultConfig(data.rules)
|
||||
setLimitMaxChunkLength(data.limits.indexing_max_segmentation_tokens_length)
|
||||
},
|
||||
@ -509,6 +526,7 @@ const StepTwo = ({
|
||||
setMaxChunkLength(max)
|
||||
setOverlap(overlap!)
|
||||
setRules(rules.pre_processing_rules)
|
||||
handleSummaryIndexSettingChange(documentDetail.summary_index_setting)
|
||||
setDefaultConfig(rules)
|
||||
|
||||
if (isHierarchicalDocument) {
|
||||
@ -728,6 +746,13 @@ const StepTwo = ({
|
||||
)}
|
||||
</>
|
||||
)}
|
||||
<div className="mt-4">
|
||||
<SummaryIndexSetting
|
||||
entry="create-document"
|
||||
summaryIndexSetting={summaryIndexSetting}
|
||||
onSummaryIndexSettingChange={handleSummaryIndexSettingChange}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
@ -879,6 +904,13 @@ const StepTwo = ({
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
<div className="mt-4">
|
||||
<SummaryIndexSetting
|
||||
entry="create-document"
|
||||
summaryIndexSetting={summaryIndexSetting}
|
||||
onSummaryIndexSettingChange={handleSummaryIndexSettingChange}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</OptionCard>
|
||||
|
||||
@ -2,7 +2,7 @@
|
||||
import type { AppIconSelection } from '@/app/components/base/app-icon-picker'
|
||||
import type { DefaultModel } from '@/app/components/header/account-setting/model-provider-page/declarations'
|
||||
import type { Member } from '@/models/common'
|
||||
import type { IconInfo } from '@/models/datasets'
|
||||
import type { IconInfo, SummaryIndexSetting as SummaryIndexSettingType } from '@/models/datasets'
|
||||
import type { AppIconType, RetrievalConfig } from '@/types/app'
|
||||
import { RiAlertFill } from '@remixicon/react'
|
||||
import { useCallback, useEffect, useMemo, useRef, useState } from 'react'
|
||||
@ -33,6 +33,7 @@ import RetrievalSettings from '../../external-knowledge-base/create/RetrievalSet
|
||||
import ChunkStructure from '../chunk-structure'
|
||||
import IndexMethod from '../index-method'
|
||||
import PermissionSelector from '../permission-selector'
|
||||
import SummaryIndexSetting from '../summary-index-setting'
|
||||
import { checkShowMultiModalTip } from '../utils'
|
||||
|
||||
const rowClass = 'flex gap-x-1'
|
||||
@ -76,6 +77,12 @@ const Form = () => {
|
||||
model: '',
|
||||
},
|
||||
)
|
||||
const [summaryIndexSetting, setSummaryIndexSetting] = useState(currentDataset?.summary_index_setting)
|
||||
const summaryIndexSettingRef = useRef(currentDataset?.summary_index_setting)
|
||||
const handleSummaryIndexSettingChange = useCallback((payload: SummaryIndexSettingType) => {
|
||||
setSummaryIndexSetting({ ...summaryIndexSettingRef.current, ...payload })
|
||||
summaryIndexSettingRef.current = { ...summaryIndexSettingRef.current, ...payload }
|
||||
}, [])
|
||||
const { data: rerankModelList } = useModelList(ModelTypeEnum.rerank)
|
||||
const { data: embeddingModelList } = useModelList(ModelTypeEnum.textEmbedding)
|
||||
const { data: membersData } = useMembers()
|
||||
@ -167,6 +174,7 @@ const Form = () => {
|
||||
},
|
||||
}),
|
||||
keyword_number: keywordNumber,
|
||||
summary_index_setting: summaryIndexSetting,
|
||||
},
|
||||
} as any
|
||||
if (permission === DatasetPermission.partialMembers) {
|
||||
@ -348,6 +356,21 @@ const Form = () => {
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
{
|
||||
indexMethod === IndexingType.QUALIFIED && (
|
||||
<>
|
||||
<Divider
|
||||
type="horizontal"
|
||||
className="my-1 h-px bg-divider-subtle"
|
||||
/>
|
||||
<SummaryIndexSetting
|
||||
entry="dataset-settings"
|
||||
summaryIndexSetting={summaryIndexSetting}
|
||||
onSummaryIndexSettingChange={handleSummaryIndexSettingChange}
|
||||
/>
|
||||
</>
|
||||
)
|
||||
}
|
||||
{/* Retrieval Method Config */}
|
||||
{currentDataset?.provider === 'external'
|
||||
? (
|
||||
|
||||
228
web/app/components/datasets/settings/summary-index-setting.tsx
Normal file
228
web/app/components/datasets/settings/summary-index-setting.tsx
Normal file
@ -0,0 +1,228 @@
|
||||
import type { ChangeEvent } from 'react'
|
||||
import type { DefaultModel } from '@/app/components/header/account-setting/model-provider-page/declarations'
|
||||
import type { SummaryIndexSetting as SummaryIndexSettingType } from '@/models/datasets'
|
||||
import {
|
||||
memo,
|
||||
useCallback,
|
||||
useMemo,
|
||||
} from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import Switch from '@/app/components/base/switch'
|
||||
import Textarea from '@/app/components/base/textarea'
|
||||
import Tooltip from '@/app/components/base/tooltip'
|
||||
import { ModelTypeEnum } from '@/app/components/header/account-setting/model-provider-page/declarations'
|
||||
import { useModelList } from '@/app/components/header/account-setting/model-provider-page/hooks'
|
||||
import ModelSelector from '@/app/components/header/account-setting/model-provider-page/model-selector'
|
||||
|
||||
type SummaryIndexSettingProps = {
|
||||
entry?: 'knowledge-base' | 'dataset-settings' | 'create-document'
|
||||
summaryIndexSetting?: SummaryIndexSettingType
|
||||
onSummaryIndexSettingChange?: (payload: SummaryIndexSettingType) => void
|
||||
readonly?: boolean
|
||||
}
|
||||
const SummaryIndexSetting = ({
|
||||
entry = 'knowledge-base',
|
||||
summaryIndexSetting,
|
||||
onSummaryIndexSettingChange,
|
||||
readonly = false,
|
||||
}: SummaryIndexSettingProps) => {
|
||||
const { t } = useTranslation()
|
||||
const {
|
||||
data: textGenerationModelList,
|
||||
} = useModelList(ModelTypeEnum.textGeneration)
|
||||
const summaryIndexModelConfig = useMemo(() => {
|
||||
if (!summaryIndexSetting?.model_name || !summaryIndexSetting?.model_provider_name)
|
||||
return undefined
|
||||
|
||||
return {
|
||||
providerName: summaryIndexSetting?.model_provider_name,
|
||||
modelName: summaryIndexSetting?.model_name,
|
||||
}
|
||||
}, [summaryIndexSetting?.model_name, summaryIndexSetting?.model_provider_name])
|
||||
|
||||
const handleSummaryIndexEnableChange = useCallback((value: boolean) => {
|
||||
onSummaryIndexSettingChange?.({
|
||||
enable: value,
|
||||
})
|
||||
}, [onSummaryIndexSettingChange])
|
||||
|
||||
const handleSummaryIndexModelChange = useCallback((model: DefaultModel) => {
|
||||
onSummaryIndexSettingChange?.({
|
||||
model_provider_name: model.provider,
|
||||
model_name: model.model,
|
||||
})
|
||||
}, [onSummaryIndexSettingChange])
|
||||
|
||||
const handleSummaryIndexPromptChange = useCallback((e: ChangeEvent<HTMLTextAreaElement>) => {
|
||||
onSummaryIndexSettingChange?.({
|
||||
summary_prompt: e.target.value,
|
||||
})
|
||||
}, [onSummaryIndexSettingChange])
|
||||
|
||||
if (entry === 'knowledge-base') {
|
||||
return (
|
||||
<div>
|
||||
<div className="flex h-6 items-center justify-between">
|
||||
<div className="system-sm-semibold-uppercase flex items-center text-text-secondary">
|
||||
{t('form.summaryAutoGen', { ns: 'datasetSettings' })}
|
||||
<Tooltip
|
||||
triggerClassName="ml-1 h-4 w-4 shrink-0"
|
||||
popupContent={t('form.summaryAutoGenTip', { ns: 'datasetSettings' })}
|
||||
>
|
||||
</Tooltip>
|
||||
</div>
|
||||
<Switch
|
||||
defaultValue={summaryIndexSetting?.enable ?? false}
|
||||
onChange={handleSummaryIndexEnableChange}
|
||||
size="md"
|
||||
/>
|
||||
</div>
|
||||
{
|
||||
summaryIndexSetting?.enable && (
|
||||
<div>
|
||||
<div className="system-xs-medium-uppercase mb-1.5 mt-2 flex h-6 items-center text-text-tertiary">
|
||||
{t('form.summaryModel', { ns: 'datasetSettings' })}
|
||||
</div>
|
||||
<ModelSelector
|
||||
defaultModel={summaryIndexModelConfig && { provider: summaryIndexModelConfig.providerName, model: summaryIndexModelConfig.modelName }}
|
||||
modelList={textGenerationModelList}
|
||||
onSelect={handleSummaryIndexModelChange}
|
||||
readonly={readonly}
|
||||
showDeprecatedWarnIcon
|
||||
/>
|
||||
<div className="system-xs-medium-uppercase mt-3 flex h-6 items-center text-text-tertiary">
|
||||
{t('form.summaryInstructions', { ns: 'datasetSettings' })}
|
||||
</div>
|
||||
<Textarea
|
||||
value={summaryIndexSetting?.summary_prompt ?? ''}
|
||||
onChange={handleSummaryIndexPromptChange}
|
||||
disabled={readonly}
|
||||
placeholder={t('form.summaryInstructionsPlaceholder', { ns: 'datasetSettings' })}
|
||||
/>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
if (entry === 'dataset-settings') {
|
||||
return (
|
||||
<div className="space-y-4">
|
||||
<div className="flex gap-x-1">
|
||||
<div className="flex h-7 w-[180px] shrink-0 items-center pt-1">
|
||||
<div className="system-sm-semibold text-text-secondary">
|
||||
{t('form.summaryAutoGen', { ns: 'datasetSettings' })}
|
||||
</div>
|
||||
</div>
|
||||
<div className="py-1.5">
|
||||
<div className="system-sm-semibold flex items-center text-text-secondary">
|
||||
<Switch
|
||||
className="mr-2"
|
||||
defaultValue={summaryIndexSetting?.enable ?? false}
|
||||
onChange={handleSummaryIndexEnableChange}
|
||||
size="md"
|
||||
/>
|
||||
{
|
||||
summaryIndexSetting?.enable ? t('list.status.enabled', { ns: 'datasetDocuments' }) : t('list.status.disabled', { ns: 'datasetDocuments' })
|
||||
}
|
||||
</div>
|
||||
<div className="system-sm-regular mt-2 text-text-tertiary">
|
||||
{
|
||||
summaryIndexSetting?.enable && t('form.summaryAutoGenTip', { ns: 'datasetSettings' })
|
||||
}
|
||||
{
|
||||
!summaryIndexSetting?.enable && t('form.summaryAutoGenEnableTip', { ns: 'datasetSettings' })
|
||||
}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
{
|
||||
summaryIndexSetting?.enable && (
|
||||
<>
|
||||
<div className="flex gap-x-1">
|
||||
<div className="flex h-7 w-[180px] shrink-0 items-center pt-1">
|
||||
<div className="system-sm-medium text-text-tertiary">
|
||||
{t('form.summaryModel', { ns: 'datasetSettings' })}
|
||||
</div>
|
||||
</div>
|
||||
<div className="grow">
|
||||
<ModelSelector
|
||||
defaultModel={summaryIndexModelConfig && { provider: summaryIndexModelConfig.providerName, model: summaryIndexModelConfig.modelName }}
|
||||
modelList={textGenerationModelList}
|
||||
onSelect={handleSummaryIndexModelChange}
|
||||
readonly={readonly}
|
||||
showDeprecatedWarnIcon
|
||||
triggerClassName="h-8"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
<div className="flex">
|
||||
<div className="flex h-7 w-[180px] shrink-0 items-center pt-1">
|
||||
<div className="system-sm-medium text-text-tertiary">
|
||||
{t('form.summaryInstructions', { ns: 'datasetSettings' })}
|
||||
</div>
|
||||
</div>
|
||||
<div className="grow">
|
||||
<Textarea
|
||||
value={summaryIndexSetting?.summary_prompt ?? ''}
|
||||
onChange={handleSummaryIndexPromptChange}
|
||||
disabled={readonly}
|
||||
placeholder={t('form.summaryInstructionsPlaceholder', { ns: 'datasetSettings' })}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
</>
|
||||
)
|
||||
}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="space-y-3">
|
||||
<div className="flex h-6 items-center">
|
||||
<Switch
|
||||
className="mr-2"
|
||||
defaultValue={summaryIndexSetting?.enable ?? false}
|
||||
onChange={handleSummaryIndexEnableChange}
|
||||
size="md"
|
||||
/>
|
||||
<div className="system-sm-semibold text-text-secondary">
|
||||
{t('form.summaryAutoGen', { ns: 'datasetSettings' })}
|
||||
</div>
|
||||
</div>
|
||||
{
|
||||
summaryIndexSetting?.enable && (
|
||||
<>
|
||||
<div>
|
||||
<div className="system-sm-medium mb-1.5 flex h-6 items-center text-text-secondary">
|
||||
{t('form.summaryModel', { ns: 'datasetSettings' })}
|
||||
</div>
|
||||
<ModelSelector
|
||||
defaultModel={summaryIndexModelConfig && { provider: summaryIndexModelConfig.providerName, model: summaryIndexModelConfig.modelName }}
|
||||
modelList={textGenerationModelList}
|
||||
onSelect={handleSummaryIndexModelChange}
|
||||
readonly={readonly}
|
||||
showDeprecatedWarnIcon
|
||||
triggerClassName="h-8"
|
||||
/>
|
||||
</div>
|
||||
<div>
|
||||
<div className="system-sm-medium mb-1.5 flex h-6 items-center text-text-secondary">
|
||||
{t('form.summaryInstructions', { ns: 'datasetSettings' })}
|
||||
</div>
|
||||
<Textarea
|
||||
value={summaryIndexSetting?.summary_prompt ?? ''}
|
||||
onChange={handleSummaryIndexPromptChange}
|
||||
disabled={readonly}
|
||||
placeholder={t('form.summaryInstructionsPlaceholder', { ns: 'datasetSettings' })}
|
||||
/>
|
||||
</div>
|
||||
</>
|
||||
)
|
||||
}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
export default memo(SummaryIndexSetting)
|
||||
Reference in New Issue
Block a user