refactor: refactor online documents handling and update related components

web/app/components/datasets/documents/create-from-pipeline/hooks.ts

@@ -119,31 +119,31 @@ export const useLocalFile = () => {
   }
 }
 
-export const useNotionsPages = () => {
-  const [notionPages, setNotionPages] = useState<NotionPage[]>([])
-  const [currentNotionPage, setCurrentNotionPage] = useState<NotionPage | undefined>()
+export const useOnlineDocuments = () => {
+  const [onlineDocuments, setOnlineDocuments] = useState<NotionPage[]>([])
+  const [currentDocuments, setCurrentDocuments] = useState<NotionPage | undefined>()
 
-  const previewNotionPage = useRef<NotionPage>(notionPages[0])
+  const previewOnlineDocument = useRef<NotionPage>(onlineDocuments[0])
 
-  const updateNotionPages = (value: NotionPage[]) => {
-    setNotionPages(value)
+  const updateOnlineDocuments = (value: NotionPage[]) => {
+    setOnlineDocuments(value)
   }
 
   const updateCurrentPage = useCallback((page: NotionPage) => {
-    setCurrentNotionPage(page)
+    setCurrentDocuments(page)
   }, [])
 
-  const hideNotionPagePreview = useCallback(() => {
-    setCurrentNotionPage(undefined)
+  const hideOnlineDocumentPreview = useCallback(() => {
+    setCurrentDocuments(undefined)
   }, [])
 
   return {
-    notionPages,
-    previewNotionPage,
-    updateNotionPages,
-    currentNotionPage,
+    onlineDocuments,
+    previewOnlineDocument,
+    updateOnlineDocuments,
+    currentDocuments,
     updateCurrentPage,
-    hideNotionPagePreview,
+    hideOnlineDocumentPreview,
   }
 }
 

web/app/components/datasets/documents/create-from-pipeline/index.tsx

@@ -5,7 +5,7 @@ import type { CrawlResultItem, DocumentItem, CustomFile as File, FileIndexingEst
 import LocalFile from '@/app/components/rag-pipeline/components/panel/test-run/data-source/local-file'
 import { useProviderContextSelector } from '@/context/provider-context'
 import type { NotionPage } from '@/models/common'
-import Notion from '@/app/components/rag-pipeline/components/panel/test-run/data-source/notion'
+import OnlineDocuments from '@/app/components/rag-pipeline/components/panel/test-run/data-source/online-documents'
 import VectorSpaceFull from '@/app/components/billing/vector-space-full'
 import WebsiteCrawl from '@/app/components/rag-pipeline/components/panel/test-run/data-source/website-crawl'
 import Actions from './data-source/actions'
@@ -26,7 +26,7 @@ import Processing from './processing'
 import type { InitialDocumentDetail, PublishedPipelineRunPreviewResponse, PublishedPipelineRunResponse } from '@/models/pipeline'
 import { DatasourceType } from '@/models/pipeline'
 import { TransferMethod } from '@/types/app'
-import { useAddDocumentsSteps, useLocalFile, useNotionsPages, useWebsiteCrawl } from './hooks'
+import { useAddDocumentsSteps, useLocalFile, useOnlineDocuments, useWebsiteCrawl } from './hooks'
 
 const CreateFormPipeline = () => {
   const { t } = useTranslation()
@@ -63,13 +63,13 @@ const CreateFormPipeline = () => {
     hideFilePreview,
   } = useLocalFile()
   const {
-    notionPages,
-    previewNotionPage,
-    updateNotionPages,
-    currentNotionPage,
+    onlineDocuments,
+    previewOnlineDocument,
+    updateOnlineDocuments,
+    currentDocuments,
     updateCurrentPage,
-    hideNotionPagePreview,
-  } = useNotionsPages()
+    hideOnlineDocumentPreview,
+  } = useOnlineDocuments()
   const {
     websitePages,
     websiteCrawlJobId,
@@ -90,11 +90,11 @@ const CreateFormPipeline = () => {
     if (datasource.type === DatasourceType.localFile)
       return isShowVectorSpaceFull || !fileList.length || fileList.some(file => !file.file.id)
     if (datasource.type === DatasourceType.onlineDocument)
-      return isShowVectorSpaceFull || !notionPages.length
+      return isShowVectorSpaceFull || !onlineDocuments.length
     if (datasource.type === DatasourceType.websiteCrawl)
       return isShowVectorSpaceFull || !websitePages.length
     return false
-  }, [datasource, isShowVectorSpaceFull, fileList, notionPages.length, websitePages.length])
+  }, [datasource, isShowVectorSpaceFull, fileList, onlineDocuments.length, websitePages.length])
 
   const { mutateAsync: runPublishedPipeline, isIdle, isPending } = useRunPublishedPipeline()
 
@@ -117,7 +117,7 @@ const CreateFormPipeline = () => {
       datasourceInfoList.push(documentInfo)
     }
     if (datasource.type === DatasourceType.onlineDocument) {
-      const { workspace_id, ...rest } = previewNotionPage.current
+      const { workspace_id, ...rest } = previewOnlineDocument.current
       const documentInfo = {
         workspace_id,
         page: rest,
@@ -143,7 +143,7 @@ const CreateFormPipeline = () => {
         setEstimateData((res as PublishedPipelineRunPreviewResponse).data.outputs)
       },
     })
-  }, [datasource, pipelineId, previewFile, previewNotionPage, previewWebsitePage, runPublishedPipeline, websiteCrawlJobId])
+  }, [datasource, pipelineId, previewFile, previewOnlineDocument, previewWebsitePage, runPublishedPipeline, websiteCrawlJobId])
 
   const handleProcess = useCallback(async (data: Record<string, any>) => {
     if (!datasource)
@@ -166,7 +166,7 @@ const CreateFormPipeline = () => {
       })
     }
     if (datasource.type === DatasourceType.onlineDocument) {
-      notionPages.forEach((page) => {
+      onlineDocuments.forEach((page) => {
         const { workspace_id, ...rest } = page
         const documentInfo = {
           workspace_id,
@@ -196,7 +196,7 @@ const CreateFormPipeline = () => {
         handleNextStep()
       },
     })
-  }, [datasource, fileList, handleNextStep, notionPages, pipelineId, runPublishedPipeline, websiteCrawlJobId, websitePages])
+  }, [datasource, fileList, handleNextStep, onlineDocuments, pipelineId, runPublishedPipeline, websiteCrawlJobId, websitePages])
 
   const onClickProcess = useCallback(() => {
     isPreview.current = false
@@ -217,10 +217,10 @@ const CreateFormPipeline = () => {
     onClickPreview()
   }, [onClickPreview, previewFile])
 
-  const handlePreviewNotionPageChange = useCallback((page: NotionPage) => {
-    previewNotionPage.current = page
+  const handlePreviewOnlineDocumentChange = useCallback((page: NotionPage) => {
+    previewOnlineDocument.current = page
     onClickPreview()
-  }, [onClickPreview, previewNotionPage])
+  }, [onClickPreview, previewOnlineDocument])
 
   const handlePreviewWebsiteChange = useCallback((website: CrawlResultItem) => {
     previewWebsitePage.current = website
@@ -263,15 +263,15 @@ const CreateFormPipeline = () => {
                   />
                 )}
                 {datasource?.type === DatasourceType.onlineDocument && (
-                  <Notion
+                  <OnlineDocuments
                     nodeId={datasource?.nodeId || ''}
                     headerInfo={{
                       title: datasource.description,
                       docTitle: datasource.docTitle || '',
                       docLink: datasource.docLink || '',
                     }}
-                    notionPages={notionPages}
-                    updateNotionPages={updateNotionPages}
+                    onlineDocuments={onlineDocuments}
+                    updateOnlineDocuments={updateOnlineDocuments}
                     canPreview
                     onPreview={updateCurrentPage}
                   />
@@ -327,7 +327,7 @@ const CreateFormPipeline = () => {
         currentStep === 1 && (
           <div className='flex h-full w-[752px] shrink-0 pl-2 pt-2'>
             {currentFile && <FilePreview file={currentFile} hidePreview={hideFilePreview} />}
-            {currentNotionPage && <NotionPagePreview currentPage={currentNotionPage} hidePreview={hideNotionPagePreview} />}
+            {currentDocuments && <NotionPagePreview currentPage={currentDocuments} hidePreview={hideOnlineDocumentPreview} />}
             {currentWebsite && <WebsitePreview payload={currentWebsite} hidePreview={hideWebsitePreview} />}
           </div>
         )
@@ -339,14 +339,14 @@ const CreateFormPipeline = () => {
               <ChunkPreview
                 datasource={datasource!}
                 files={fileList.map(file => file.file)}
-                notionPages={notionPages}
+                onlineDocuments={onlineDocuments}
                 websitePages={websitePages}
                 isIdle={isIdle && isPreview.current}
                 isPending={isPending && isPreview.current}
                 estimateData={estimateData}
                 onPreview={onClickPreview}
                 handlePreviewFileChange={handlePreviewFileChange}
-                handlePreviewNotionPageChange={handlePreviewNotionPageChange}
+                handlePreviewOnlineDocumentChange={handlePreviewOnlineDocumentChange}
                 handlePreviewWebsitePageChange={handlePreviewWebsiteChange}
               />
             )}

web/app/components/datasets/documents/create-from-pipeline/preview/chunk-preview.tsx

@@ -20,35 +20,35 @@ import { DatasourceType } from '@/models/pipeline'
 type ChunkPreviewProps = {
   datasource: Datasource
   files: CustomFile[]
-  notionPages: NotionPage[]
+  onlineDocuments: NotionPage[]
   websitePages: CrawlResultItem[]
   isIdle: boolean
   isPending: boolean
   estimateData: FileIndexingEstimateResponse | undefined
   onPreview: () => void
   handlePreviewFileChange: (file: DocumentItem) => void
-  handlePreviewNotionPageChange: (page: NotionPage) => void
+  handlePreviewOnlineDocumentChange: (page: NotionPage) => void
   handlePreviewWebsitePageChange: (page: CrawlResultItem) => void
 }
 
 const ChunkPreview = ({
   datasource,
   files,
-  notionPages,
+  onlineDocuments,
   websitePages,
   isIdle,
   isPending,
   estimateData,
   onPreview,
   handlePreviewFileChange,
-  handlePreviewNotionPageChange,
+  handlePreviewOnlineDocumentChange,
   handlePreviewWebsitePageChange,
 }: ChunkPreviewProps) => {
   const { t } = useTranslation()
   const currentDocForm = useDatasetDetailContextWithSelector(s => s.dataset?.doc_form)
 
   const [previewFile, setPreviewFile] = useState<DocumentItem>(files[0] as DocumentItem)
-  const [previewNotionPage, setPreviewNotionPage] = useState<NotionPage>(notionPages[0])
+  const [previewOnlineDocument, setPreviewOnlineDocument] = useState<NotionPage>(onlineDocuments[0])
   const [previewWebsitePage, setPreviewWebsitePage] = useState<CrawlResultItem>(websitePages[0])
 
   const dataSourceType = datasource?.type
@@ -72,20 +72,20 @@ const ChunkPreview = ({
           {dataSourceType === DatasourceType.onlineDocument
             && <PreviewDocumentPicker
               files={
-                notionPages.map(page => ({
+                onlineDocuments.map(page => ({
                   id: page.page_id,
                   name: page.page_name,
                   extension: 'md',
                 }))
               }
               onChange={(selected) => {
-                const selectedPage = notionPages.find(page => page.page_id === selected.id)
-                setPreviewNotionPage(selectedPage!)
-                handlePreviewNotionPageChange(selectedPage!)
+                const selectedPage = onlineDocuments.find(page => page.page_id === selected.id)
+                setPreviewOnlineDocument(selectedPage!)
+                handlePreviewOnlineDocumentChange(selectedPage!)
               }}
               value={{
-                id: previewNotionPage?.page_id || '',
-                name: previewNotionPage?.page_name || '',
+                id: previewOnlineDocument?.page_id || '',
+                name: previewOnlineDocument?.page_name || '',
                 extension: 'md',
               }}
             />