refactor: update variable naming for consistency and improve data source handling in pipeline components

2026-04-25 13:16:16 +08:00 · 2025-05-20 11:42:22 +08:00
parent 314a2f9be8
commit 14a9052d60
11 changed files with 183 additions and 111 deletions
--- a/web/app/components/rag-pipeline/components/input-field/editor/index.tsx
+++ b/web/app/components/rag-pipeline/components/input-field/editor/index.tsx
@ -48,7 +48,7 @@ const InputFieldEditor = ({
      options,
      placeholder,
      unit,
-      default: defaultValue,
+      default_value: defaultValue,
      allowed_file_upload_methods: allowedFileUploadMethods,
      allowed_file_types: allowedTypesAndExtensions.allowedFileTypes,
      allowed_file_extensions: allowedTypesAndExtensions.allowedFileExtensions,
--- a/web/app/components/rag-pipeline/components/input-field/editor/utils.ts
+++ b/web/app/components/rag-pipeline/components/input-field/editor/utils.ts
@ -14,7 +14,7 @@ export const convertToInputFieldFormData = (data?: InputVar): FormData => {
    label,
    variable,
    max_length,
-    'default': defaultValue,
+    default_value,
    required,
    tooltips,
    options,
@ -30,7 +30,7 @@ export const convertToInputFieldFormData = (data?: InputVar): FormData => {
    label,
    variable,
    maxLength: max_length,
-    default: defaultValue,
+    default: default_value,
    required,
    tooltips,
    options,
--- a/web/app/components/rag-pipeline/components/panel/test-run/data-source-options/index.tsx
+++ b/web/app/components/rag-pipeline/components/panel/test-run/data-source-options/index.tsx
@ -1,4 +1,4 @@
-import { useCallback } from 'react'
+import { useCallback, useEffect } from 'react'
 import { useDataSourceOptions } from '../hooks'
 import OptionCard from './option-card'
 import { File, Watercrawl } from '@/app/components/base/icons/src/public/knowledge'
@ -9,9 +9,8 @@ import { DataSourceProvider } from '@/models/common'
 import type { Datasource } from '../types'

 type DataSourceOptionsProps = {
-  dataSources: Datasource[]
  dataSourceNodeId: string
-  onSelect: (option: string) => void
+  onSelect: (option: Datasource) => void
 }

 const DATA_SOURCE_ICONS = {
@ -23,15 +22,23 @@ const DATA_SOURCE_ICONS = {
 }

 const DataSourceOptions = ({
-  dataSources,
  dataSourceNodeId,
  onSelect,
 }: DataSourceOptionsProps) => {
-  const options = useDataSourceOptions(dataSources)
+  const { dataSources, options } = useDataSourceOptions()

  const handelSelect = useCallback((value: string) => {
-    onSelect(value)
-  }, [onSelect])
+    const selectedOption = dataSources.find(option => option.nodeId === value)
+    if (!selectedOption)
+      return
+    onSelect(selectedOption)
+  }, [dataSources, onSelect])
+
+  useEffect(() => {
+    if (options.length > 0)
+      handelSelect(options[0].value)
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [])

  return (
    <div className='grid w-full grid-cols-4 gap-1'>
--- a/web/app/components/rag-pipeline/components/panel/test-run/document-processing/hooks.ts
+++ b/web/app/components/rag-pipeline/components/panel/test-run/document-processing/hooks.ts
@ -1,21 +1,24 @@
 import { useMemo } from 'react'
 import { BaseFieldType } from '@/app/components/base/form/form-scenarios/base/types'
 import { useStore } from '@/app/components/workflow/store'
-import { InputVarType } from '@/app/components/workflow/types'
 import { usePipelineProcessingParams } from '@/service/use-pipeline'
+import { PipelineInputVarType } from '@/models/pipeline'

-type PartialInputVarType = InputVarType.textInput | InputVarType.number | InputVarType.select | InputVarType.checkbox
+type PartialInputVarType = PipelineInputVarType.textInput | PipelineInputVarType.number | PipelineInputVarType.select | PipelineInputVarType.checkbox

 const VAR_TYPE_MAP: Record<PartialInputVarType, BaseFieldType> = {
-  [InputVarType.textInput]: BaseFieldType.textInput,
-  [InputVarType.number]: BaseFieldType.numberInput,
-  [InputVarType.select]: BaseFieldType.select,
-  [InputVarType.checkbox]: BaseFieldType.checkbox,
+  [PipelineInputVarType.textInput]: BaseFieldType.textInput,
+  [PipelineInputVarType.number]: BaseFieldType.numberInput,
+  [PipelineInputVarType.select]: BaseFieldType.select,
+  [PipelineInputVarType.checkbox]: BaseFieldType.checkbox,
 }

-export const useConfigurations = () => {
+export const useConfigurations = (datasourceNodeId: string) => {
  const pipelineId = useStore(state => state.pipelineId)
-  const { data: paramsConfig } = usePipelineProcessingParams(pipelineId!)
+  const { data: paramsConfig } = usePipelineProcessingParams({
+    pipeline_id: pipelineId!,
+    node_id: datasourceNodeId,
+  })

  const initialData = useMemo(() => {
    const variables = paramsConfig?.variables || []
--- a/web/app/components/rag-pipeline/components/panel/test-run/document-processing/index.tsx
+++ b/web/app/components/rag-pipeline/components/panel/test-run/document-processing/index.tsx
@ -6,15 +6,17 @@ import type { FormType } from '@/app/components/base/form'
 import { useCallback } from 'react'

 type DocumentProcessingProps = {
+  dataSourceNodeId: string
  onProcess: (data: Record<string, any>) => void
  onBack: () => void
 }

 const DocumentProcessing = ({
+  dataSourceNodeId,
  onProcess,
  onBack,
 }: DocumentProcessingProps) => {
-  const { initialData, configurations } = useConfigurations()
+  const { initialData, configurations } = useConfigurations(dataSourceNodeId)
  const schema = generateZodSchema(configurations)

  const renderCustomActions = useCallback((form: FormType) => (
--- a/web/app/components/rag-pipeline/components/panel/test-run/hooks.ts
+++ b/web/app/components/rag-pipeline/components/panel/test-run/hooks.ts
@ -3,6 +3,10 @@ import type { DataSourceOption, Datasource } from './types'
 import { TestRunStep } from './types'
 import { DataSourceType } from '@/models/datasets'
 import { DataSourceProvider } from '@/models/common'
+import { useNodes } from 'reactflow'
+import { BlockEnum } from '@/app/components/workflow/types'
+import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types'
+import { useMemo } from 'react'

 export const useTestRunSteps = () => {
  const { t } = useTranslation()
@ -19,45 +23,78 @@ export const useTestRunSteps = () => {
  return steps
 }

-export const useDataSourceOptions = (dataSources: Datasource[]) => {
+export const useDataSourceOptions = () => {
  const { t } = useTranslation()
-  const options: DataSourceOption[] = []
-  dataSources.forEach((source) => {
-    if (source.type === DataSourceType.FILE) {
-      options.push({
-        label: t('datasetPipeline.testRun.dataSource.localFiles'),
-        value: source.nodeId,
-        type: DataSourceType.FILE,
-      })
-    }
-    if (source.type === DataSourceType.NOTION) {
-      options.push({
-        label: 'Notion',
-        value: source.nodeId,
-        type: DataSourceType.NOTION,
-      })
-    }
-    if (source.type === DataSourceProvider.fireCrawl) {
-      options.push({
-        label: 'Firecrawl',
-        value: source.nodeId,
-        type: DataSourceProvider.fireCrawl,
-      })
-    }
-    if (source.type === DataSourceProvider.jinaReader) {
-      options.push({
-        label: 'Jina Reader',
-        value: source.nodeId,
-        type: DataSourceProvider.jinaReader,
-      })
-    }
-    if (source.type === DataSourceProvider.waterCrawl) {
-      options.push({
-        label: 'Water Crawl',
-        value: source.nodeId,
-        type: DataSourceProvider.waterCrawl,
-      })
-    }
-  })
-  return options
+  const nodes = useNodes<DataSourceNodeType>()
+  const dataSources: Datasource[] = useMemo(() => {
+    const dataSourceNodes = nodes.filter(node => node.data.type === BlockEnum.DataSource)
+    return dataSourceNodes.map((node) => {
+      let type: DataSourceType | DataSourceProvider = DataSourceType.FILE
+      switch (node.data.tool_name) {
+        case 'file_upload':
+          type = DataSourceType.FILE
+          break
+        case 'search_notion':
+          type = DataSourceType.NOTION
+          break
+        case 'firecrawl':
+          type = DataSourceProvider.fireCrawl
+          break
+        case 'jina_reader':
+          type = DataSourceProvider.jinaReader
+          break
+        case 'water_crawl':
+          type = DataSourceProvider.waterCrawl
+          break
+      }
+      return {
+        nodeId: node.id,
+        type,
+        config: {},
+      }
+    })
+  }, [nodes])
+
+  const options = useMemo(() => {
+    const options: DataSourceOption[] = []
+    dataSources.forEach((source) => {
+      if (source.type === DataSourceType.FILE) {
+        options.push({
+          label: t('datasetPipeline.testRun.dataSource.localFiles'),
+          value: source.nodeId,
+          type: DataSourceType.FILE,
+        })
+      }
+      if (source.type === DataSourceType.NOTION) {
+        options.push({
+          label: 'Notion',
+          value: source.nodeId,
+          type: DataSourceType.NOTION,
+        })
+      }
+      if (source.type === DataSourceProvider.fireCrawl) {
+        options.push({
+          label: 'Firecrawl',
+          value: source.nodeId,
+          type: DataSourceProvider.fireCrawl,
+        })
+      }
+      if (source.type === DataSourceProvider.jinaReader) {
+        options.push({
+          label: 'Jina Reader',
+          value: source.nodeId,
+          type: DataSourceProvider.jinaReader,
+        })
+      }
+      if (source.type === DataSourceProvider.waterCrawl) {
+        options.push({
+          label: 'Water Crawl',
+          value: source.nodeId,
+          type: DataSourceProvider.waterCrawl,
+        })
+      }
+    })
+    return options
+  }, [dataSources, t])
+  return { dataSources, options }
 }
--- a/web/app/components/rag-pipeline/components/panel/test-run/index.tsx
+++ b/web/app/components/rag-pipeline/components/panel/test-run/index.tsx
@ -26,11 +26,7 @@ const TestRunPanel = () => {
  const { t } = useTranslation()
  const setShowDebugAndPreviewPanel = useWorkflowStoreWithSelector(state => state.setShowDebugAndPreviewPanel)
  const [currentStep, setCurrentStep] = useState(1)
-  const [datasource, setDatasource] = useState<Datasource>({
-    nodeId: '1',
-    type: DataSourceType.FILE,
-    config: {},
-  })
+  const [datasource, setDatasource] = useState<Datasource>()
  const [fileList, setFiles] = useState<FileItem[]>([])
  const [notionPages, setNotionPages] = useState<NotionPage[]>([])
  const [websitePages, setWebsitePages] = useState<CrawlResultItem[]>([])
@ -41,28 +37,6 @@ const TestRunPanel = () => {
  const enableBilling = useProviderContextSelector(state => state.enableBilling)

  const steps = useTestRunSteps()
-  // TODO: replace with real data sources from API
-  const dataSources = useMemo(() => [{
-    nodeId: '1',
-    type: DataSourceType.FILE,
-    config: {},
-  }, {
-    nodeId: '2',
-    type: DataSourceType.NOTION,
-    config: {},
-  }, {
-    nodeId: '3',
-    type: DataSourceProvider.fireCrawl,
-    config: {},
-  }, {
-    nodeId: '4',
-    type: DataSourceProvider.jinaReader,
-    config: {},
-  }, {
-    nodeId: '5',
-    type: DataSourceProvider.waterCrawl,
-    config: {},
-  }], [])

  const allFileLoaded = (fileList.length > 0 && fileList.every(file => file.file.id))
  const isVectorSpaceFull = plan.usage.vectorSpace >= plan.total.vectorSpace
@ -77,6 +51,7 @@ const TestRunPanel = () => {
  }, [fileList, isShowVectorSpaceFull])

  const nextBtnDisabled = useMemo(() => {
+    if (!datasource) return false
    if (datasource.type === DataSourceType.FILE)
      return nextDisabled
    if (datasource.type === DataSourceType.NOTION)
@ -92,13 +67,6 @@ const TestRunPanel = () => {
    setShowDebugAndPreviewPanel(false)
  }

-  const handleDataSourceSelect = useCallback((option: string) => {
-    const dataSource = dataSources.find(dataSource => dataSource.nodeId === option)
-    if (!dataSource)
-      return
-    setDatasource(dataSource)
-  }, [dataSources])
-
  const updateFile = (fileItem: FileItem, progress: number, list: FileItem[]) => {
    const newList = produce(list, (draft) => {
      const targetIndex = draft.findIndex(file => file.fileID === fileItem.fileID)
@ -129,6 +97,8 @@ const TestRunPanel = () => {
  const { handleRun } = usePipelineRun()

  const handleProcess = useCallback((data: Record<string, any>) => {
+    if (!datasource)
+      return
    const datasourceInfo: Record<string, any> = {}
    if (datasource.type === DataSourceType.FILE)
      datasourceInfo.fileId = fileList.map(file => file.fileID)
@ -176,11 +146,10 @@ const TestRunPanel = () => {
            <>
              <div className='flex flex-col gap-y-4 px-4 py-2'>
                <DataSourceOptions
-                  dataSources={dataSources}
-                  dataSourceNodeId={datasource.nodeId}
-                  onSelect={handleDataSourceSelect}
+                  dataSourceNodeId={datasource?.nodeId || ''}
+                  onSelect={setDatasource}
                />
-                {datasource.type === DataSourceType.FILE && (
+                {datasource?.type === DataSourceType.FILE && (
                  <LocalFile
                    files={fileList}
                    updateFile={updateFile}
@ -188,13 +157,13 @@ const TestRunPanel = () => {
                    notSupportBatchUpload={notSupportBatchUpload}
                  />
                )}
-                {datasource.type === DataSourceType.NOTION && (
+                {datasource?.type === DataSourceType.NOTION && (
                  <Notion
                    notionPages={notionPages}
                    updateNotionPages={updateNotionPages}
                  />
                )}
-                {datasource.type === DataSourceProvider.fireCrawl && (
+                {datasource?.type === DataSourceProvider.fireCrawl && (
                  <Firecrawl
                    checkedCrawlResult={websitePages}
                    onCheckedCrawlResultChange={setWebsitePages}
@ -203,7 +172,7 @@ const TestRunPanel = () => {
                    onCrawlOptionsChange={setCrawlOptions}
                  />
                )}
-                {datasource.type === DataSourceProvider.jinaReader && (
+                {datasource?.type === DataSourceProvider.jinaReader && (
                  <JinaReader
                    checkedCrawlResult={websitePages}
                    onCheckedCrawlResultChange={setWebsitePages}
@ -212,7 +181,7 @@ const TestRunPanel = () => {
                    onCrawlOptionsChange={setCrawlOptions}
                  />
                )}
-                {datasource.type === DataSourceProvider.waterCrawl && (
+                {datasource?.type === DataSourceProvider.waterCrawl && (
                  <WaterCrawl
                    checkedCrawlResult={websitePages}
                    onCheckedCrawlResultChange={setWebsitePages}
@ -232,6 +201,7 @@ const TestRunPanel = () => {
        {
          currentStep === 2 && (
            <DocumentProcessing
+              dataSourceNodeId={datasource?.nodeId || ''}
              onProcess={handleProcess}
              onBack={handleBackStep}
            />
--- a/web/app/components/rag-pipeline/hooks/use-pipeline-init.ts
+++ b/web/app/components/rag-pipeline/hooks/use-pipeline-init.ts
@ -81,7 +81,7 @@ export const usePipelineInit = () => {

  useEffect(() => {
    handleGetInitialWorkflowData()
-  // eslint-disable-next-line react-hooks/exhaustive-deps
+    // eslint-disable-next-line react-hooks/exhaustive-deps
  }, [])

  return {
--- a/web/app/components/workflow/nodes/data-source/types.ts
+++ b/web/app/components/workflow/nodes/data-source/types.ts
@ -1,3 +1,6 @@
 import type { CommonNodeType } from '@/app/components/workflow/types'
+import type { RAGPipelineVariables } from '@/models/pipeline'

-export type DataSourceNodeType = CommonNodeType
+export type DataSourceNodeType = CommonNodeType & {
+  variables: RAGPipelineVariables
+}