feat: summary index

web/app/components/datasets/create/step-two/index.tsx

@@ -2,7 +2,7 @@
 import type { FC, PropsWithChildren } from 'react'
 import type { DefaultModel } from '@/app/components/header/account-setting/model-provider-page/declarations'
 import type { NotionPage } from '@/models/common'
-import type { CrawlOptions, CrawlResultItem, CreateDocumentReq, createDocumentResponse, CustomFile, DocumentItem, FullDocumentDetail, ParentMode, PreProcessingRule, ProcessRule, Rules } from '@/models/datasets'
+import type { CrawlOptions, CrawlResultItem, CreateDocumentReq, createDocumentResponse, CustomFile, DocumentItem, FullDocumentDetail, ParentMode, PreProcessingRule, ProcessRule, Rules, SummaryIndexSetting as SummaryIndexSettingType } from '@/models/datasets'
 import type { RetrievalConfig } from '@/types/app'
 import {
   RiAlertFill,
@@ -12,7 +12,7 @@ import {
 import { noop } from 'es-toolkit/function'
 import Image from 'next/image'
 import Link from 'next/link'
-import { useCallback, useEffect, useMemo, useState } from 'react'
+import { useCallback, useEffect, useMemo, useRef, useState } from 'react'
 import { useTranslation } from 'react-i18next'
 import { trackEvent } from '@/app/components/base/amplitude'
 import Badge from '@/app/components/base/badge'
@@ -29,8 +29,8 @@ import Toast from '@/app/components/base/toast'
 import Tooltip from '@/app/components/base/tooltip'
 import { isReRankModelSelected } from '@/app/components/datasets/common/check-rerank-model'
 import EconomicalRetrievalMethodConfig from '@/app/components/datasets/common/economical-retrieval-method-config'
-import RetrievalMethodConfig from '@/app/components/datasets/common/retrieval-method-config'
 
+import RetrievalMethodConfig from '@/app/components/datasets/common/retrieval-method-config'
 import { ModelTypeEnum } from '@/app/components/header/account-setting/model-provider-page/declarations'
 import { useDefaultModel, useModelList, useModelListAndDefaultModelAndCurrentProviderAndModel } from '@/app/components/header/account-setting/model-provider-page/hooks'
 import ModelSelector from '@/app/components/header/account-setting/model-provider-page/model-selector'
@@ -51,6 +51,7 @@ import { PreviewSlice } from '../../formatted-text/flavours/preview-slice'
 import { FormattedText } from '../../formatted-text/formatted'
 import PreviewContainer from '../../preview/container'
 import { PreviewHeader } from '../../preview/header'
+import SummaryIndexSetting from '../../settings/summary-index-setting'
 import { checkShowMultiModalTip } from '../../settings/utils'
 import FileList from '../assets/file-list-3-fill.svg'
 import Note from '../assets/note-mod.svg'
@@ -174,6 +175,18 @@ const StepTwo = ({
   const [limitMaxChunkLength, setLimitMaxChunkLength] = useState(MAXIMUM_CHUNK_TOKEN_LENGTH)
   const [overlap, setOverlap] = useState(DEFAULT_OVERLAP)
   const [rules, setRules] = useState<PreProcessingRule[]>([])
+  const [summaryIndexSetting, setSummaryIndexSetting] = useState<SummaryIndexSettingType | undefined>(currentDataset?.summary_index_setting)
+  const summaryIndexSettingRef = useRef<SummaryIndexSettingType | undefined>(currentDataset?.summary_index_setting)
+  const handleSummaryIndexSettingChange = useCallback((payload?: SummaryIndexSettingType, reset?: boolean) => {
+    if (reset) {
+      setSummaryIndexSetting(payload)
+      summaryIndexSettingRef.current = payload
+    }
+    else {
+      setSummaryIndexSetting({ ...summaryIndexSettingRef.current, ...payload })
+      summaryIndexSettingRef.current = { ...summaryIndexSettingRef.current, ...payload }
+    }
+  }, [setSummaryIndexSetting, summaryIndexSettingRef])
   const [defaultConfig, setDefaultConfig] = useState<Rules>()
   const hasSetIndexType = !!indexingType
   const [indexType, setIndexType] = useState<IndexingType>(() => {
@@ -250,6 +263,7 @@ const StepTwo = ({
           },
         },
         mode: 'hierarchical',
+        summary_index_setting: summaryIndexSetting,
       } as ProcessRule
     }
     return {
@@ -262,6 +276,7 @@ const StepTwo = ({
         },
       }, // api will check this. It will be removed after api refactored.
       mode: segmentationType,
+      summary_index_setting: summaryIndexSetting,
     } as ProcessRule
   }
 
@@ -352,6 +367,7 @@ const StepTwo = ({
       setMaxChunkLength(defaultConfig.segmentation.max_tokens)
       setOverlap(defaultConfig.segmentation.chunk_overlap!)
       setRules(defaultConfig.pre_processing_rules)
+      handleSummaryIndexSettingChange(currentDataset?.summary_index_setting, true)
     }
     setParentChildConfig(defaultParentChildConfig)
   }
@@ -492,6 +508,7 @@ const StepTwo = ({
       setMaxChunkLength(data.rules.segmentation.max_tokens)
       setOverlap(data.rules.segmentation.chunk_overlap!)
       setRules(data.rules.pre_processing_rules)
+      handleSummaryIndexSettingChange(data.summary_index_setting, true)
       setDefaultConfig(data.rules)
       setLimitMaxChunkLength(data.limits.indexing_max_segmentation_tokens_length)
     },
@@ -509,6 +526,7 @@ const StepTwo = ({
       setMaxChunkLength(max)
       setOverlap(overlap!)
       setRules(rules.pre_processing_rules)
+      handleSummaryIndexSettingChange(documentDetail.summary_index_setting)
       setDefaultConfig(rules)
 
       if (isHierarchicalDocument) {
@@ -728,6 +746,13 @@ const StepTwo = ({
                       )}
                     </>
                   )}
+                  <div className="mt-4">
+                    <SummaryIndexSetting
+                      entry="create-document"
+                      summaryIndexSetting={summaryIndexSetting}
+                      onSummaryIndexSettingChange={handleSummaryIndexSettingChange}
+                    />
+                  </div>
                 </div>
               </div>
             </div>
@@ -879,6 +904,13 @@ const StepTwo = ({
                       </div>
                     ))}
                   </div>
+                  <div className="mt-4">
+                    <SummaryIndexSetting
+                      entry="create-document"
+                      summaryIndexSetting={summaryIndexSetting}
+                      onSummaryIndexSettingChange={handleSummaryIndexSettingChange}
+                    />
+                  </div>
                 </div>
               </div>
             </OptionCard>

web/app/components/datasets/settings/form/index.tsx

@@ -2,7 +2,7 @@
 import type { AppIconSelection } from '@/app/components/base/app-icon-picker'
 import type { DefaultModel } from '@/app/components/header/account-setting/model-provider-page/declarations'
 import type { Member } from '@/models/common'
-import type { IconInfo } from '@/models/datasets'
+import type { IconInfo, SummaryIndexSetting as SummaryIndexSettingType } from '@/models/datasets'
 import type { AppIconType, RetrievalConfig } from '@/types/app'
 import { RiAlertFill } from '@remixicon/react'
 import { useCallback, useEffect, useMemo, useRef, useState } from 'react'
@@ -33,6 +33,7 @@ import RetrievalSettings from '../../external-knowledge-base/create/RetrievalSet
 import ChunkStructure from '../chunk-structure'
 import IndexMethod from '../index-method'
 import PermissionSelector from '../permission-selector'
+import SummaryIndexSetting from '../summary-index-setting'
 import { checkShowMultiModalTip } from '../utils'
 
 const rowClass = 'flex gap-x-1'
@@ -76,6 +77,12 @@ const Form = () => {
           model: '',
         },
   )
+  const [summaryIndexSetting, setSummaryIndexSetting] = useState(currentDataset?.summary_index_setting)
+  const summaryIndexSettingRef = useRef(currentDataset?.summary_index_setting)
+  const handleSummaryIndexSettingChange = useCallback((payload: SummaryIndexSettingType) => {
+    setSummaryIndexSetting({ ...summaryIndexSettingRef.current, ...payload })
+    summaryIndexSettingRef.current = { ...summaryIndexSettingRef.current, ...payload }
+  }, [])
   const { data: rerankModelList } = useModelList(ModelTypeEnum.rerank)
   const { data: embeddingModelList } = useModelList(ModelTypeEnum.textEmbedding)
   const { data: membersData } = useMembers()
@@ -167,6 +174,7 @@ const Form = () => {
             },
           }),
           keyword_number: keywordNumber,
+          summary_index_setting: summaryIndexSetting,
         },
       } as any
       if (permission === DatasetPermission.partialMembers) {
@@ -348,6 +356,21 @@ const Form = () => {
           </div>
         </div>
       )}
+      {
+        indexMethod === IndexingType.QUALIFIED && (
+          <>
+            <Divider
+              type="horizontal"
+              className="my-1 h-px bg-divider-subtle"
+            />
+            <SummaryIndexSetting
+              entry="dataset-settings"
+              summaryIndexSetting={summaryIndexSetting}
+              onSummaryIndexSettingChange={handleSummaryIndexSettingChange}
+            />
+          </>
+        )
+      }
       {/* Retrieval Method Config */}
       {currentDataset?.provider === 'external'
         ? (

web/app/components/datasets/settings/summary-index-setting.tsx

@@ -0,0 +1,228 @@
+import type { ChangeEvent } from 'react'
+import type { DefaultModel } from '@/app/components/header/account-setting/model-provider-page/declarations'
+import type { SummaryIndexSetting as SummaryIndexSettingType } from '@/models/datasets'
+import {
+  memo,
+  useCallback,
+  useMemo,
+} from 'react'
+import { useTranslation } from 'react-i18next'
+import Switch from '@/app/components/base/switch'
+import Textarea from '@/app/components/base/textarea'
+import Tooltip from '@/app/components/base/tooltip'
+import { ModelTypeEnum } from '@/app/components/header/account-setting/model-provider-page/declarations'
+import { useModelList } from '@/app/components/header/account-setting/model-provider-page/hooks'
+import ModelSelector from '@/app/components/header/account-setting/model-provider-page/model-selector'
+
+type SummaryIndexSettingProps = {
+  entry?: 'knowledge-base' | 'dataset-settings' | 'create-document'
+  summaryIndexSetting?: SummaryIndexSettingType
+  onSummaryIndexSettingChange?: (payload: SummaryIndexSettingType) => void
+  readonly?: boolean
+}
+const SummaryIndexSetting = ({
+  entry = 'knowledge-base',
+  summaryIndexSetting,
+  onSummaryIndexSettingChange,
+  readonly = false,
+}: SummaryIndexSettingProps) => {
+  const { t } = useTranslation()
+  const {
+    data: textGenerationModelList,
+  } = useModelList(ModelTypeEnum.textGeneration)
+  const summaryIndexModelConfig = useMemo(() => {
+    if (!summaryIndexSetting?.model_name || !summaryIndexSetting?.model_provider_name)
+      return undefined
+
+    return {
+      providerName: summaryIndexSetting?.model_provider_name,
+      modelName: summaryIndexSetting?.model_name,
+    }
+  }, [summaryIndexSetting?.model_name, summaryIndexSetting?.model_provider_name])
+
+  const handleSummaryIndexEnableChange = useCallback((value: boolean) => {
+    onSummaryIndexSettingChange?.({
+      enable: value,
+    })
+  }, [onSummaryIndexSettingChange])
+
+  const handleSummaryIndexModelChange = useCallback((model: DefaultModel) => {
+    onSummaryIndexSettingChange?.({
+      model_provider_name: model.provider,
+      model_name: model.model,
+    })
+  }, [onSummaryIndexSettingChange])
+
+  const handleSummaryIndexPromptChange = useCallback((e: ChangeEvent<HTMLTextAreaElement>) => {
+    onSummaryIndexSettingChange?.({
+      summary_prompt: e.target.value,
+    })
+  }, [onSummaryIndexSettingChange])
+
+  if (entry === 'knowledge-base') {
+    return (
+      <div>
+        <div className="flex h-6 items-center justify-between">
+          <div className="system-sm-semibold-uppercase flex items-center text-text-secondary">
+            {t('form.summaryAutoGen', { ns: 'datasetSettings' })}
+            <Tooltip
+              triggerClassName="ml-1 h-4 w-4 shrink-0"
+              popupContent={t('form.summaryAutoGenTip', { ns: 'datasetSettings' })}
+            >
+            </Tooltip>
+          </div>
+          <Switch
+            defaultValue={summaryIndexSetting?.enable ?? false}
+            onChange={handleSummaryIndexEnableChange}
+            size="md"
+          />
+        </div>
+        {
+          summaryIndexSetting?.enable && (
+            <div>
+              <div className="system-xs-medium-uppercase mb-1.5 mt-2 flex h-6 items-center text-text-tertiary">
+                {t('form.summaryModel', { ns: 'datasetSettings' })}
+              </div>
+              <ModelSelector
+                defaultModel={summaryIndexModelConfig && { provider: summaryIndexModelConfig.providerName, model: summaryIndexModelConfig.modelName }}
+                modelList={textGenerationModelList}
+                onSelect={handleSummaryIndexModelChange}
+                readonly={readonly}
+                showDeprecatedWarnIcon
+              />
+              <div className="system-xs-medium-uppercase mt-3 flex h-6 items-center text-text-tertiary">
+                {t('form.summaryInstructions', { ns: 'datasetSettings' })}
+              </div>
+              <Textarea
+                value={summaryIndexSetting?.summary_prompt ?? ''}
+                onChange={handleSummaryIndexPromptChange}
+                disabled={readonly}
+                placeholder={t('form.summaryInstructionsPlaceholder', { ns: 'datasetSettings' })}
+              />
+            </div>
+          )
+        }
+      </div>
+    )
+  }
+
+  if (entry === 'dataset-settings') {
+    return (
+      <div className="space-y-4">
+        <div className="flex gap-x-1">
+          <div className="flex h-7 w-[180px] shrink-0 items-center pt-1">
+            <div className="system-sm-semibold text-text-secondary">
+              {t('form.summaryAutoGen', { ns: 'datasetSettings' })}
+            </div>
+          </div>
+          <div className="py-1.5">
+            <div className="system-sm-semibold flex items-center text-text-secondary">
+              <Switch
+                className="mr-2"
+                defaultValue={summaryIndexSetting?.enable ?? false}
+                onChange={handleSummaryIndexEnableChange}
+                size="md"
+              />
+              {
+                summaryIndexSetting?.enable ? t('list.status.enabled', { ns: 'datasetDocuments' }) : t('list.status.disabled', { ns: 'datasetDocuments' })
+              }
+            </div>
+            <div className="system-sm-regular mt-2 text-text-tertiary">
+              {
+                summaryIndexSetting?.enable && t('form.summaryAutoGenTip', { ns: 'datasetSettings' })
+              }
+              {
+                !summaryIndexSetting?.enable && t('form.summaryAutoGenEnableTip', { ns: 'datasetSettings' })
+              }
+            </div>
+          </div>
+        </div>
+        {
+          summaryIndexSetting?.enable && (
+            <>
+              <div className="flex gap-x-1">
+                <div className="flex h-7 w-[180px] shrink-0 items-center pt-1">
+                  <div className="system-sm-medium text-text-tertiary">
+                    {t('form.summaryModel', { ns: 'datasetSettings' })}
+                  </div>
+                </div>
+                <div className="grow">
+                  <ModelSelector
+                    defaultModel={summaryIndexModelConfig && { provider: summaryIndexModelConfig.providerName, model: summaryIndexModelConfig.modelName }}
+                    modelList={textGenerationModelList}
+                    onSelect={handleSummaryIndexModelChange}
+                    readonly={readonly}
+                    showDeprecatedWarnIcon
+                    triggerClassName="h-8"
+                  />
+                </div>
+              </div>
+              <div className="flex">
+                <div className="flex h-7 w-[180px] shrink-0 items-center pt-1">
+                  <div className="system-sm-medium text-text-tertiary">
+                    {t('form.summaryInstructions', { ns: 'datasetSettings' })}
+                  </div>
+                </div>
+                <div className="grow">
+                  <Textarea
+                    value={summaryIndexSetting?.summary_prompt ?? ''}
+                    onChange={handleSummaryIndexPromptChange}
+                    disabled={readonly}
+                    placeholder={t('form.summaryInstructionsPlaceholder', { ns: 'datasetSettings' })}
+                  />
+                </div>
+              </div>
+            </>
+          )
+        }
+      </div>
+    )
+  }
+
+  return (
+    <div className="space-y-3">
+      <div className="flex h-6 items-center">
+        <Switch
+          className="mr-2"
+          defaultValue={summaryIndexSetting?.enable ?? false}
+          onChange={handleSummaryIndexEnableChange}
+          size="md"
+        />
+        <div className="system-sm-semibold text-text-secondary">
+          {t('form.summaryAutoGen', { ns: 'datasetSettings' })}
+        </div>
+      </div>
+      {
+        summaryIndexSetting?.enable && (
+          <>
+            <div>
+              <div className="system-sm-medium mb-1.5 flex h-6 items-center text-text-secondary">
+                {t('form.summaryModel', { ns: 'datasetSettings' })}
+              </div>
+              <ModelSelector
+                defaultModel={summaryIndexModelConfig && { provider: summaryIndexModelConfig.providerName, model: summaryIndexModelConfig.modelName }}
+                modelList={textGenerationModelList}
+                onSelect={handleSummaryIndexModelChange}
+                readonly={readonly}
+                showDeprecatedWarnIcon
+                triggerClassName="h-8"
+              />
+            </div>
+            <div>
+              <div className="system-sm-medium mb-1.5 flex h-6 items-center text-text-secondary">
+                {t('form.summaryInstructions', { ns: 'datasetSettings' })}
+              </div>
+              <Textarea
+                value={summaryIndexSetting?.summary_prompt ?? ''}
+                onChange={handleSummaryIndexPromptChange}
+                disabled={readonly}
+                placeholder={t('form.summaryInstructionsPlaceholder', { ns: 'datasetSettings' })}
+              />
+            </div>
+          </>
+        )
+      }
+    </div>
+  )
+}
+export default memo(SummaryIndexSetting)

web/app/components/workflow/nodes/knowledge-base/hooks/use-config.ts

@@ -1,6 +1,7 @@
 import type {
   KnowledgeBaseNodeType,
   RerankingModel,
+  SummaryIndexSetting,
 } from '../types'
 import type { ValueSelector } from '@/app/components/workflow/types'
 import { produce } from 'immer'
@@ -246,6 +247,16 @@ export const useConfig = (id: string) => {
     })
   }, [handleNodeDataUpdate])
 
+  const handleSummaryIndexSettingChange = useCallback((summaryIndexSetting: SummaryIndexSetting) => {
+    const nodeData = getNodeData()
+    handleNodeDataUpdate({
+      summary_index_setting: {
+        ...nodeData?.data.summary_index_setting,
+        ...summaryIndexSetting,
+      },
+    })
+  }, [handleNodeDataUpdate, getNodeData])
+
   return {
     handleChunkStructureChange,
     handleIndexMethodChange,
@@ -260,5 +271,6 @@ export const useConfig = (id: string) => {
     handleScoreThresholdChange,
     handleScoreThresholdEnabledChange,
     handleInputVariableChange,
+    handleSummaryIndexSettingChange,
   }
 }

web/app/components/workflow/nodes/knowledge-base/panel.tsx

@@ -7,6 +7,7 @@ import {
   useMemo,
 } from 'react'
 import { useTranslation } from 'react-i18next'
+import SummaryIndexSetting from '@/app/components/datasets/settings/summary-index-setting'
 import { checkShowMultiModalTip } from '@/app/components/datasets/settings/utils'
 import { ModelTypeEnum } from '@/app/components/header/account-setting/model-provider-page/declarations'
 import { useModelList } from '@/app/components/header/account-setting/model-provider-page/hooks'
@@ -51,6 +52,7 @@ const Panel: FC<NodePanelProps<KnowledgeBaseNodeType>> = ({
     handleScoreThresholdChange,
     handleScoreThresholdEnabledChange,
     handleInputVariableChange,
+    handleSummaryIndexSettingChange,
   } = useConfig(id)
 
   const filterVar = useCallback((variable: Var) => {
@@ -167,6 +169,20 @@ const Panel: FC<NodePanelProps<KnowledgeBaseNodeType>> = ({
                 <div className="pt-1">
                   <Split className="h-[1px]" />
                 </div>
+                {
+                  data.indexing_technique === IndexMethodEnum.QUALIFIED && (
+                    <>
+                      <SummaryIndexSetting
+                        summaryIndexSetting={data.summary_index_setting}
+                        onSummaryIndexSettingChange={handleSummaryIndexSettingChange}
+                        readonly={nodesReadOnly}
+                      />
+                      <div className="pt-1">
+                        <Split className="h-[1px]" />
+                      </div>
+                    </>
+                  )
+                }
                 <RetrievalSetting
                   indexMethod={data.indexing_technique}
                   searchMethod={data.retrieval_model.search_method}

web/app/components/workflow/nodes/knowledge-base/types.ts

@@ -42,6 +42,12 @@ export type RetrievalSetting = {
   score_threshold: number
   reranking_mode?: RerankingModeEnum
 }
+export type SummaryIndexSetting = {
+  enable?: boolean
+  model_name?: string
+  model_provider_name?: string
+  summary_prompt?: string
+}
 export type KnowledgeBaseNodeType = CommonNodeType & {
   index_chunk_variable_selector: string[]
   chunk_structure?: ChunkStructureEnum
@@ -52,4 +58,5 @@ export type KnowledgeBaseNodeType = CommonNodeType & {
   retrieval_model: RetrievalSetting
   _embeddingModelList?: Model[]
   _rerankModelList?: Model[]
+  summary_index_setting?: SummaryIndexSetting
 }

web/i18n/en-US/dataset-settings.json

@@ -39,6 +39,12 @@
   "form.retrievalSettings": "Retrieval Settings",
   "form.save": "Save",
   "form.searchModel": "Search model",
+  "form.summaryAutoGen": "Summary Auto-Gen",
+  "form.summaryAutoGenEnableTip": "Once enabled, summaries will be generated automatically for newly added documents. Existing documents can still be summarized manually.",
+  "form.summaryAutoGenTip": "Summaries are automatically generated for newly added documents. Existing documents can still be summarized manually.",
+  "form.summaryInstructions": "Instructions",
+  "form.summaryInstructionsPlaceholder": "Describe the rules or style for auto-generated summaries…",
+  "form.summaryModel": "Summary Model",
   "form.upgradeHighQualityTip": "Once upgrading to High Quality mode, reverting to Economical mode is not available",
   "title": "Knowledge settings"
 }

web/i18n/zh-Hans/dataset-settings.json

@@ -39,6 +39,12 @@
   "form.retrievalSettings": "检索设置",
   "form.save": "保存",
   "form.searchModel": "搜索模型",
+  "form.summaryAutoGen": "摘要自动生成",
+  "form.summaryAutoGenEnableTip": "启用后，将自动为新添加的文档生成摘要。已有的文档仍可以手动摘要。",
+  "form.summaryAutoGenTip": "将自动为新添加的文档生成摘要。已有的文档仍可以手动摘要。",
+  "form.summaryInstructions": "指令",
+  "form.summaryInstructionsPlaceholder": "描述自动生成摘要的规则或风格…",
+  "form.summaryModel": "摘要模型",
   "form.upgradeHighQualityTip": "一旦升级为高质量模式，将无法切换回经济模式。",
   "title": "知识库设置"
 }

web/models/datasets.ts

@@ -42,6 +42,13 @@ export type IconInfo = {
   icon_url?: string
 }
 
+export type SummaryIndexSetting = {
+  enable?: boolean
+  model_name?: string
+  model_provider_name?: string
+  summary_prompt?: string
+}
+
 export type DataSet = {
   id: string
   name: string
@@ -88,6 +95,7 @@ export type DataSet = {
   runtime_mode: 'rag_pipeline' | 'general'
   enable_api: boolean // Indicates if the service API is enabled
   is_multimodal: boolean // Indicates if the dataset supports multimodal
+  summary_index_setting?: SummaryIndexSetting
 }
 
 export type ExternalAPIItem = {
@@ -262,6 +270,7 @@ export type ProcessRuleResponse = {
   mode: ProcessMode
   rules: Rules
   limits: Limits
+  summary_index_setting?: SummaryIndexSetting
 }
 
 export type Rules = {
@@ -467,6 +476,7 @@ export type NotionPage = {
 export type ProcessRule = {
   mode: ProcessMode
   rules: Rules
+  summary_index_setting?: SummaryIndexSetting
 }
 
 export type createDocumentResponse = {