feat: implement document creation pipeline with multi-step wizard and datasource management (#30843)

Co-authored-by: CodingOnStar <hanxujiang@dify.ai>

web/app/components/datasets/documents/create-from-pipeline/steps/index.ts

@@ -0,0 +1,3 @@
+export { default as StepOneContent } from './step-one-content'
+export { default as StepThreeContent } from './step-three-content'
+export { default as StepTwoContent } from './step-two-content'

web/app/components/datasets/documents/create-from-pipeline/steps/preview-panel.tsx

@@ -0,0 +1,112 @@
+'use client'
+import type { Datasource } from '@/app/components/rag-pipeline/components/panel/test-run/types'
+import type { NotionPage } from '@/models/common'
+import type { CrawlResultItem, CustomFile, DocumentItem, FileIndexingEstimateResponse, FileItem } from '@/models/datasets'
+import type { DatasourceType, OnlineDriveFile } from '@/models/pipeline'
+import { memo } from 'react'
+import ChunkPreview from '../preview/chunk-preview'
+import FilePreview from '../preview/file-preview'
+import OnlineDocumentPreview from '../preview/online-document-preview'
+import WebsitePreview from '../preview/web-preview'
+
+type StepOnePreviewProps = {
+  datasource: Datasource | undefined
+  currentLocalFile: CustomFile | undefined
+  currentDocument: (NotionPage & { workspace_id: string }) | undefined
+  currentWebsite: CrawlResultItem | undefined
+  hidePreviewLocalFile: () => void
+  hidePreviewOnlineDocument: () => void
+  hideWebsitePreview: () => void
+}
+
+export const StepOnePreview = memo(({
+  datasource,
+  currentLocalFile,
+  currentDocument,
+  currentWebsite,
+  hidePreviewLocalFile,
+  hidePreviewOnlineDocument,
+  hideWebsitePreview,
+}: StepOnePreviewProps) => {
+  return (
+    <div className="h-full min-w-0 flex-1">
+      <div className="flex h-full flex-col pl-2 pt-2">
+        {currentLocalFile && (
+          <FilePreview
+            file={currentLocalFile}
+            hidePreview={hidePreviewLocalFile}
+          />
+        )}
+        {currentDocument && (
+          <OnlineDocumentPreview
+            datasourceNodeId={datasource!.nodeId}
+            currentPage={currentDocument}
+            hidePreview={hidePreviewOnlineDocument}
+          />
+        )}
+        {currentWebsite && (
+          <WebsitePreview
+            currentWebsite={currentWebsite}
+            hidePreview={hideWebsitePreview}
+          />
+        )}
+      </div>
+    </div>
+  )
+})
+StepOnePreview.displayName = 'StepOnePreview'
+
+type StepTwoPreviewProps = {
+  datasourceType: string | undefined
+  localFileList: FileItem[]
+  onlineDocuments: (NotionPage & { workspace_id: string })[]
+  websitePages: CrawlResultItem[]
+  selectedOnlineDriveFileList: OnlineDriveFile[]
+  isIdle: boolean
+  isPendingPreview: boolean
+  estimateData: FileIndexingEstimateResponse | undefined
+  onPreview: () => void
+  handlePreviewFileChange: (file: DocumentItem) => void
+  handlePreviewOnlineDocumentChange: (page: NotionPage) => void
+  handlePreviewWebsitePageChange: (website: CrawlResultItem) => void
+  handlePreviewOnlineDriveFileChange: (file: OnlineDriveFile) => void
+}
+
+export const StepTwoPreview = memo(({
+  datasourceType,
+  localFileList,
+  onlineDocuments,
+  websitePages,
+  selectedOnlineDriveFileList,
+  isIdle,
+  isPendingPreview,
+  estimateData,
+  onPreview,
+  handlePreviewFileChange,
+  handlePreviewOnlineDocumentChange,
+  handlePreviewWebsitePageChange,
+  handlePreviewOnlineDriveFileChange,
+}: StepTwoPreviewProps) => {
+  return (
+    <div className="h-full min-w-0 flex-1">
+      <div className="flex h-full flex-col pl-2 pt-2">
+        <ChunkPreview
+          dataSourceType={datasourceType as DatasourceType}
+          localFiles={localFileList.map(file => file.file)}
+          onlineDocuments={onlineDocuments}
+          websitePages={websitePages}
+          onlineDriveFiles={selectedOnlineDriveFileList}
+          isIdle={isIdle}
+          isPending={isPendingPreview}
+          estimateData={estimateData}
+          onPreview={onPreview}
+          handlePreviewFileChange={handlePreviewFileChange}
+          handlePreviewOnlineDocumentChange={handlePreviewOnlineDocumentChange}
+          handlePreviewWebsitePageChange={handlePreviewWebsitePageChange}
+          handlePreviewOnlineDriveFileChange={handlePreviewOnlineDriveFileChange}
+        />
+      </div>
+    </div>
+  )
+})
+StepTwoPreview.displayName = 'StepTwoPreview'

web/app/components/datasets/documents/create-from-pipeline/steps/step-one-content.tsx

@@ -0,0 +1,110 @@
+'use client'
+import type { Datasource } from '@/app/components/rag-pipeline/components/panel/test-run/types'
+import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types'
+import type { Node } from '@/app/components/workflow/types'
+import { memo } from 'react'
+import Divider from '@/app/components/base/divider'
+import VectorSpaceFull from '@/app/components/billing/vector-space-full'
+import LocalFile from '@/app/components/datasets/documents/create-from-pipeline/data-source/local-file'
+import OnlineDocuments from '@/app/components/datasets/documents/create-from-pipeline/data-source/online-documents'
+import OnlineDrive from '@/app/components/datasets/documents/create-from-pipeline/data-source/online-drive'
+import WebsiteCrawl from '@/app/components/datasets/documents/create-from-pipeline/data-source/website-crawl'
+import { DatasourceType } from '@/models/pipeline'
+import UpgradeCard from '../../../create/step-one/upgrade-card'
+import Actions from '../actions'
+import DataSourceOptions from '../data-source-options'
+
+type StepOneContentProps = {
+  datasource: Datasource | undefined
+  datasourceType: string | undefined
+  pipelineNodes: Node<DataSourceNodeType>[]
+  supportBatchUpload: boolean
+  localFileListLength: number
+  isShowVectorSpaceFull: boolean
+  showSelect: boolean
+  totalOptions: number | undefined
+  selectedOptions: number | undefined
+  tip: string
+  nextBtnDisabled: boolean
+  onSelectDataSource: (dataSource: Datasource) => void
+  onCredentialChange: (credentialId: string) => void
+  onSelectAll: () => void
+  onNextStep: () => void
+}
+
+const StepOneContent = ({
+  datasource,
+  datasourceType,
+  pipelineNodes,
+  supportBatchUpload,
+  localFileListLength,
+  isShowVectorSpaceFull,
+  showSelect,
+  totalOptions,
+  selectedOptions,
+  tip,
+  nextBtnDisabled,
+  onSelectDataSource,
+  onCredentialChange,
+  onSelectAll,
+  onNextStep,
+}: StepOneContentProps) => {
+  const showUpgradeCard = !supportBatchUpload
+    && datasourceType === DatasourceType.localFile
+    && localFileListLength > 0
+
+  return (
+    <div className="flex flex-col gap-y-5 pt-4">
+      <DataSourceOptions
+        datasourceNodeId={datasource?.nodeId || ''}
+        onSelect={onSelectDataSource}
+        pipelineNodes={pipelineNodes}
+      />
+      {datasourceType === DatasourceType.localFile && (
+        <LocalFile
+          allowedExtensions={datasource!.nodeData.fileExtensions || []}
+          supportBatchUpload={supportBatchUpload}
+        />
+      )}
+      {datasourceType === DatasourceType.onlineDocument && (
+        <OnlineDocuments
+          nodeId={datasource!.nodeId}
+          nodeData={datasource!.nodeData}
+          onCredentialChange={onCredentialChange}
+        />
+      )}
+      {datasourceType === DatasourceType.websiteCrawl && (
+        <WebsiteCrawl
+          nodeId={datasource!.nodeId}
+          nodeData={datasource!.nodeData}
+          onCredentialChange={onCredentialChange}
+        />
+      )}
+      {datasourceType === DatasourceType.onlineDrive && (
+        <OnlineDrive
+          nodeId={datasource!.nodeId}
+          nodeData={datasource!.nodeData}
+          onCredentialChange={onCredentialChange}
+        />
+      )}
+      {isShowVectorSpaceFull && <VectorSpaceFull />}
+      <Actions
+        showSelect={showSelect}
+        totalOptions={totalOptions}
+        selectedOptions={selectedOptions}
+        onSelectAll={onSelectAll}
+        disabled={nextBtnDisabled}
+        handleNextStep={onNextStep}
+        tip={tip}
+      />
+      {showUpgradeCard && (
+        <>
+          <Divider type="horizontal" className="my-4 h-px bg-divider-subtle" />
+          <UpgradeCard />
+        </>
+      )}
+    </div>
+  )
+}
+
+export default memo(StepOneContent)

web/app/components/datasets/documents/create-from-pipeline/steps/step-three-content.tsx

@@ -0,0 +1,23 @@
+'use client'
+import type { InitialDocumentDetail } from '@/models/pipeline'
+import { memo } from 'react'
+import Processing from '../processing'
+
+type StepThreeContentProps = {
+  batchId: string
+  documents: InitialDocumentDetail[]
+}
+
+const StepThreeContent = ({
+  batchId,
+  documents,
+}: StepThreeContentProps) => {
+  return (
+    <Processing
+      batchId={batchId}
+      documents={documents}
+    />
+  )
+}
+
+export default memo(StepThreeContent)

web/app/components/datasets/documents/create-from-pipeline/steps/step-two-content.tsx

@@ -0,0 +1,38 @@
+'use client'
+import type { RefObject } from 'react'
+import { memo } from 'react'
+import ProcessDocuments from '../process-documents'
+
+type StepTwoContentProps = {
+  formRef: RefObject<{ submit: () => void } | null>
+  dataSourceNodeId: string
+  isRunning: boolean
+  onProcess: () => void
+  onPreview: () => void
+  onSubmit: (data: Record<string, unknown>) => void
+  onBack: () => void
+}
+
+const StepTwoContent = ({
+  formRef,
+  dataSourceNodeId,
+  isRunning,
+  onProcess,
+  onPreview,
+  onSubmit,
+  onBack,
+}: StepTwoContentProps) => {
+  return (
+    <ProcessDocuments
+      ref={formRef}
+      dataSourceNodeId={dataSourceNodeId}
+      isRunning={isRunning}
+      onProcess={onProcess}
+      onPreview={onPreview}
+      onSubmit={onSubmit}
+      onBack={onBack}
+    />
+  )
+}
+
+export default memo(StepTwoContent)