mirror of
https://github.com/langgenius/dify.git
synced 2026-05-04 01:18:05 +08:00
refactor: Refactor useOnlineDocument hook
This commit is contained in:
twwu
@ -0,0 +1,339 @@
|
||||
'use client'
|
||||
import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import { useContext } from 'use-context-selector'
|
||||
import { RiDeleteBinLine, RiErrorWarningFill, RiUploadCloud2Line } from '@remixicon/react'
|
||||
import DocumentFileIcon from '@/app/components/datasets/common/document-file-icon'
|
||||
import cn from '@/utils/classnames'
|
||||
import type { CustomFile as File, FileItem } from '@/models/datasets'
|
||||
import { ToastContext } from '@/app/components/base/toast'
|
||||
import SimplePieChart from '@/app/components/base/simple-pie-chart'
|
||||
import { upload } from '@/service/base'
|
||||
import I18n from '@/context/i18n'
|
||||
import { LanguagesSupported } from '@/i18n/language'
|
||||
import { IS_CE_EDITION } from '@/config'
|
||||
import { Theme } from '@/types/app'
|
||||
import useTheme from '@/hooks/use-theme'
|
||||
import { useFileUploadConfig } from '@/service/use-common'
|
||||
|
||||
const FILES_NUMBER_LIMIT = 20
|
||||
|
||||
export type LocalFileProps = {
|
||||
fileList: FileItem[]
|
||||
allowedExtensions: string[]
|
||||
prepareFileList: (files: FileItem[]) => void
|
||||
onFileUpdate: (fileItem: FileItem, progress: number, list: FileItem[]) => void
|
||||
onFileListUpdate?: (files: FileItem[]) => void
|
||||
onPreview?: (file: File) => void
|
||||
notSupportBatchUpload?: boolean
|
||||
}
|
||||
|
||||
const LocalFile = ({
|
||||
fileList,
|
||||
allowedExtensions,
|
||||
prepareFileList,
|
||||
onFileUpdate,
|
||||
onFileListUpdate,
|
||||
onPreview,
|
||||
notSupportBatchUpload,
|
||||
}: LocalFileProps) => {
|
||||
const { t } = useTranslation()
|
||||
const { notify } = useContext(ToastContext)
|
||||
const { locale } = useContext(I18n)
|
||||
const [dragging, setDragging] = useState(false)
|
||||
const dropRef = useRef<HTMLDivElement>(null)
|
||||
const dragRef = useRef<HTMLDivElement>(null)
|
||||
const fileUploader = useRef<HTMLInputElement>(null)
|
||||
const hideUpload = notSupportBatchUpload && fileList.length > 0
|
||||
|
||||
const { data: fileUploadConfigResponse } = useFileUploadConfig()
|
||||
const supportTypesShowNames = useMemo(() => {
|
||||
const extensionMap: { [key: string]: string } = {
|
||||
md: 'markdown',
|
||||
pptx: 'pptx',
|
||||
htm: 'html',
|
||||
xlsx: 'xlsx',
|
||||
docx: 'docx',
|
||||
}
|
||||
|
||||
return allowedExtensions
|
||||
.map(item => extensionMap[item] || item) // map to standardized extension
|
||||
.map(item => item.toLowerCase()) // convert to lower case
|
||||
.filter((item, index, self) => self.indexOf(item) === index) // remove duplicates
|
||||
.map(item => item.toUpperCase()) // convert to upper case
|
||||
.join(locale !== LanguagesSupported[1] ? ', ' : '、 ')
|
||||
}, [locale, allowedExtensions])
|
||||
const ACCEPTS = allowedExtensions.map((ext: string) => `.${ext}`)
|
||||
const fileUploadConfig = useMemo(() => fileUploadConfigResponse ?? {
|
||||
file_size_limit: 15,
|
||||
batch_count_limit: 5,
|
||||
}, [fileUploadConfigResponse])
|
||||
|
||||
const fileListRef = useRef<FileItem[]>([])
|
||||
|
||||
// utils
|
||||
const getFileType = (currentFile: File) => {
|
||||
if (!currentFile)
|
||||
return ''
|
||||
|
||||
const arr = currentFile.name.split('.')
|
||||
return arr[arr.length - 1]
|
||||
}
|
||||
|
||||
const getFileSize = (size: number) => {
|
||||
if (size / 1024 < 10)
|
||||
return `${(size / 1024).toFixed(2)}KB`
|
||||
|
||||
return `${(size / 1024 / 1024).toFixed(2)}MB`
|
||||
}
|
||||
|
||||
const isValid = useCallback((file: File) => {
|
||||
const { size } = file
|
||||
const ext = `.${getFileType(file)}`
|
||||
const isValidType = ACCEPTS.includes(ext.toLowerCase())
|
||||
if (!isValidType)
|
||||
notify({ type: 'error', message: t('datasetCreation.stepOne.uploader.validation.typeError') })
|
||||
|
||||
const isValidSize = size <= fileUploadConfig.file_size_limit * 1024 * 1024
|
||||
if (!isValidSize)
|
||||
notify({ type: 'error', message: t('datasetCreation.stepOne.uploader.validation.size', { size: fileUploadConfig.file_size_limit }) })
|
||||
|
||||
return isValidType && isValidSize
|
||||
}, [fileUploadConfig, notify, t, ACCEPTS])
|
||||
|
||||
const fileUpload = useCallback(async (fileItem: FileItem): Promise<FileItem> => {
|
||||
const formData = new FormData()
|
||||
formData.append('file', fileItem.file)
|
||||
const onProgress = (e: ProgressEvent) => {
|
||||
if (e.lengthComputable) {
|
||||
const percent = Math.floor(e.loaded / e.total * 100)
|
||||
onFileUpdate(fileItem, percent, fileListRef.current)
|
||||
}
|
||||
}
|
||||
|
||||
return upload({
|
||||
xhr: new XMLHttpRequest(),
|
||||
data: formData,
|
||||
onprogress: onProgress,
|
||||
}, false, undefined, '?source=datasets')
|
||||
.then((res: File) => {
|
||||
const completeFile = {
|
||||
fileID: fileItem.fileID,
|
||||
file: res,
|
||||
progress: -1,
|
||||
}
|
||||
const index = fileListRef.current.findIndex(item => item.fileID === fileItem.fileID)
|
||||
fileListRef.current[index] = completeFile
|
||||
onFileUpdate(completeFile, 100, fileListRef.current)
|
||||
return Promise.resolve({ ...completeFile })
|
||||
})
|
||||
.catch((e) => {
|
||||
notify({ type: 'error', message: e?.response?.code === 'forbidden' ? e?.response?.message : t('datasetCreation.stepOne.uploader.failed') })
|
||||
onFileUpdate(fileItem, -2, fileListRef.current)
|
||||
return Promise.resolve({ ...fileItem })
|
||||
})
|
||||
.finally()
|
||||
}, [fileListRef, notify, onFileUpdate, t])
|
||||
|
||||
const uploadBatchFiles = useCallback((bFiles: FileItem[]) => {
|
||||
bFiles.forEach(bf => (bf.progress = 0))
|
||||
return Promise.all(bFiles.map(fileUpload))
|
||||
}, [fileUpload])
|
||||
|
||||
const uploadMultipleFiles = useCallback(async (files: FileItem[]) => {
|
||||
const batchCountLimit = fileUploadConfig.batch_count_limit
|
||||
const length = files.length
|
||||
let start = 0
|
||||
let end = 0
|
||||
|
||||
while (start < length) {
|
||||
if (start + batchCountLimit > length)
|
||||
end = length
|
||||
else
|
||||
end = start + batchCountLimit
|
||||
const bFiles = files.slice(start, end)
|
||||
await uploadBatchFiles(bFiles)
|
||||
start = end
|
||||
}
|
||||
}, [fileUploadConfig, uploadBatchFiles])
|
||||
|
||||
const initialUpload = useCallback((files: File[]) => {
|
||||
if (!files.length)
|
||||
return false
|
||||
|
||||
if (files.length + fileList.length > FILES_NUMBER_LIMIT && !IS_CE_EDITION) {
|
||||
notify({ type: 'error', message: t('datasetCreation.stepOne.uploader.validation.filesNumber', { filesNumber: FILES_NUMBER_LIMIT }) })
|
||||
return false
|
||||
}
|
||||
|
||||
const preparedFiles = files.map((file, index) => ({
|
||||
fileID: `file${index}-${Date.now()}`,
|
||||
file,
|
||||
progress: -1,
|
||||
}))
|
||||
const newFiles = [...fileListRef.current, ...preparedFiles]
|
||||
prepareFileList(newFiles)
|
||||
fileListRef.current = newFiles
|
||||
uploadMultipleFiles(preparedFiles)
|
||||
}, [prepareFileList, uploadMultipleFiles, notify, t, fileList])
|
||||
|
||||
const handleDragEnter = (e: DragEvent) => {
|
||||
e.preventDefault()
|
||||
e.stopPropagation()
|
||||
e.target !== dragRef.current && setDragging(true)
|
||||
}
|
||||
const handleDragOver = (e: DragEvent) => {
|
||||
e.preventDefault()
|
||||
e.stopPropagation()
|
||||
}
|
||||
const handleDragLeave = (e: DragEvent) => {
|
||||
e.preventDefault()
|
||||
e.stopPropagation()
|
||||
e.target === dragRef.current && setDragging(false)
|
||||
}
|
||||
|
||||
const handleDrop = useCallback((e: DragEvent) => {
|
||||
e.preventDefault()
|
||||
e.stopPropagation()
|
||||
setDragging(false)
|
||||
if (!e.dataTransfer)
|
||||
return
|
||||
|
||||
let files = [...e.dataTransfer.files] as File[]
|
||||
if (notSupportBatchUpload)
|
||||
files = files.slice(0, 1)
|
||||
|
||||
const validFiles = files.filter(isValid)
|
||||
initialUpload(validFiles)
|
||||
}, [initialUpload, isValid, notSupportBatchUpload])
|
||||
|
||||
const selectHandle = () => {
|
||||
if (fileUploader.current)
|
||||
fileUploader.current.click()
|
||||
}
|
||||
|
||||
const removeFile = (fileID: string) => {
|
||||
if (fileUploader.current)
|
||||
fileUploader.current.value = ''
|
||||
|
||||
fileListRef.current = fileListRef.current.filter(item => item.fileID !== fileID)
|
||||
onFileListUpdate?.([...fileListRef.current])
|
||||
}
|
||||
const fileChangeHandle = useCallback((e: React.ChangeEvent<HTMLInputElement>) => {
|
||||
const files = [...(e.target.files ?? [])] as File[]
|
||||
initialUpload(files.filter(isValid))
|
||||
}, [isValid, initialUpload])
|
||||
|
||||
const { theme } = useTheme()
|
||||
const chartColor = useMemo(() => theme === Theme.dark ? '#5289ff' : '#296dff', [theme])
|
||||
|
||||
useEffect(() => {
|
||||
const dropElement = dropRef.current
|
||||
dropElement?.addEventListener('dragenter', handleDragEnter)
|
||||
dropElement?.addEventListener('dragover', handleDragOver)
|
||||
dropElement?.addEventListener('dragleave', handleDragLeave)
|
||||
dropElement?.addEventListener('drop', handleDrop)
|
||||
return () => {
|
||||
dropElement?.removeEventListener('dragenter', handleDragEnter)
|
||||
dropElement?.removeEventListener('dragover', handleDragOver)
|
||||
dropElement?.removeEventListener('dragleave', handleDragLeave)
|
||||
dropElement?.removeEventListener('drop', handleDrop)
|
||||
}
|
||||
}, [handleDrop])
|
||||
|
||||
return (
|
||||
<div className='flex flex-col'>
|
||||
{!hideUpload && (
|
||||
<input
|
||||
ref={fileUploader}
|
||||
id='fileUploader'
|
||||
className='hidden'
|
||||
type='file'
|
||||
multiple={!notSupportBatchUpload}
|
||||
accept={ACCEPTS.join(',')}
|
||||
onChange={fileChangeHandle}
|
||||
/>
|
||||
)}
|
||||
{!hideUpload && (
|
||||
<div
|
||||
ref={dropRef}
|
||||
className={cn(
|
||||
'relative box-border flex min-h-20 flex-col items-center justify-center gap-1 rounded-xl border border-dashed border-components-dropzone-border bg-components-dropzone-bg px-4 py-3 text-xs leading-4 text-text-tertiary',
|
||||
dragging && 'border-components-dropzone-border-accent bg-components-dropzone-bg-accent',
|
||||
)}>
|
||||
<div className='flex min-h-5 items-center justify-center text-sm leading-4 text-text-secondary'>
|
||||
<RiUploadCloud2Line className='mr-2 size-5' />
|
||||
|
||||
<span>
|
||||
{t('datasetCreation.stepOne.uploader.button')}
|
||||
{allowedExtensions.length > 0 && (
|
||||
<label className='ml-1 cursor-pointer text-text-accent' onClick={selectHandle}>{t('datasetCreation.stepOne.uploader.browse')}</label>
|
||||
)}
|
||||
</span>
|
||||
</div>
|
||||
<div>{t('datasetCreation.stepOne.uploader.tip', {
|
||||
size: fileUploadConfig.file_size_limit,
|
||||
supportTypes: supportTypesShowNames,
|
||||
batchCount: fileUploadConfig.batch_count_limit,
|
||||
})}</div>
|
||||
{dragging && <div ref={dragRef} className='absolute left-0 top-0 h-full w-full' />}
|
||||
</div>
|
||||
)}
|
||||
{fileList.length > 0 && (
|
||||
<div className='mt-1 flex flex-col gap-y-1'>
|
||||
{fileList.map((fileItem, index) => {
|
||||
const isUploading = fileItem.progress >= 0 && fileItem.progress < 100
|
||||
const isError = fileItem.progress === -2
|
||||
return (
|
||||
<div
|
||||
key={`${fileItem.fileID}-${index}`}
|
||||
onClick={() => fileItem.file?.id && onPreview?.(fileItem.file)}
|
||||
className={cn(
|
||||
'flex h-12 items-center rounded-lg border border-components-panel-border bg-components-panel-on-panel-item-bg shadow-xs shadow-shadow-shadow-4',
|
||||
isError && 'border-state-destructive-border bg-state-destructive-hover',
|
||||
)}
|
||||
>
|
||||
<div className='flex w-12 shrink-0 items-center justify-center'>
|
||||
<DocumentFileIcon
|
||||
size='lg'
|
||||
className='shrink-0'
|
||||
name={fileItem.file.name}
|
||||
extension={getFileType(fileItem.file)}
|
||||
/>
|
||||
</div>
|
||||
<div className='flex shrink grow flex-col gap-0.5'>
|
||||
<div className='flex w-full'>
|
||||
<div className='w-0 grow truncate text-xs text-text-secondary'>{fileItem.file.name}</div>
|
||||
</div>
|
||||
<div className='w-full truncate text-2xs leading-3 text-text-tertiary'>
|
||||
<span className='uppercase'>{getFileType(fileItem.file)}</span>
|
||||
<span className='px-1 text-text-quaternary'>·</span>
|
||||
<span>{getFileSize(fileItem.file.size)}</span>
|
||||
</div>
|
||||
</div>
|
||||
<div className='flex w-16 shrink-0 items-center justify-end gap-1 pr-3'>
|
||||
{isUploading && (
|
||||
<SimplePieChart percentage={fileItem.progress} stroke={chartColor} fill={chartColor} animationDuration={0} />
|
||||
)}
|
||||
{
|
||||
isError && (
|
||||
<RiErrorWarningFill className='size-4 text-text-destructive' />
|
||||
)
|
||||
}
|
||||
<span className='flex h-6 w-6 cursor-pointer items-center justify-center' onClick={(e) => {
|
||||
e.stopPropagation()
|
||||
removeFile(fileItem.fileID)
|
||||
}}>
|
||||
<RiDeleteBinLine className='size-4 text-text-tertiary' />
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
})}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
export default LocalFile
|
||||
@ -0,0 +1,158 @@
|
||||
import { useCallback, useEffect, useMemo } from 'react'
|
||||
import WorkspaceSelector from '@/app/components/base/notion-page-selector/workspace-selector'
|
||||
import SearchInput from '@/app/components/base/notion-page-selector/search-input'
|
||||
import PageSelector from './page-selector'
|
||||
import type { DataSourceNotionPageMap, DataSourceNotionWorkspace, NotionPage } from '@/models/common'
|
||||
import Header from '@/app/components/datasets/create/website/base/header'
|
||||
import { useDatasetDetailContextWithSelector } from '@/context/dataset-detail'
|
||||
import { DatasourceType } from '@/models/pipeline'
|
||||
import { ssePost } from '@/service/base'
|
||||
import Toast from '@/app/components/base/toast'
|
||||
import type { DataSourceNodeCompletedResponse } from '@/types/pipeline'
|
||||
import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types'
|
||||
|
||||
type OnlineDocumentsProps = {
|
||||
onSelect: (selectedPages: NotionPage[]) => void
|
||||
previewPageId?: string
|
||||
onPreview?: (selectedPage: NotionPage) => void
|
||||
isInPipeline?: boolean
|
||||
nodeId: string
|
||||
nodeData: DataSourceNodeType
|
||||
documentsData: DataSourceNotionWorkspace[]
|
||||
setDocumentsData: (documentsData: DataSourceNotionWorkspace[]) => void
|
||||
searchValue: string
|
||||
setSearchValue: (value: string) => void
|
||||
currentWorkspaceId: string
|
||||
setCurrentWorkspaceId: (workspaceId: string) => void
|
||||
PagesMapAndSelectedPagesId: DataSourceNotionPageMap
|
||||
selectedPagesId: Set<string>
|
||||
setSelectedPagesId: (selectedPagesId: Set<string>) => void
|
||||
}
|
||||
|
||||
const OnlineDocuments = ({
|
||||
onSelect,
|
||||
previewPageId,
|
||||
onPreview,
|
||||
isInPipeline = false,
|
||||
nodeId,
|
||||
nodeData,
|
||||
documentsData,
|
||||
setDocumentsData,
|
||||
searchValue,
|
||||
setSearchValue,
|
||||
currentWorkspaceId,
|
||||
setCurrentWorkspaceId,
|
||||
PagesMapAndSelectedPagesId,
|
||||
selectedPagesId,
|
||||
setSelectedPagesId,
|
||||
}: OnlineDocumentsProps) => {
|
||||
const pipelineId = useDatasetDetailContextWithSelector(s => s.dataset?.pipeline_id)
|
||||
|
||||
const datasourceNodeRunURL = !isInPipeline
|
||||
? `/rag/pipelines/${pipelineId}/workflows/published/datasource/nodes/${nodeId}/run`
|
||||
: `/rag/pipelines/${pipelineId}/workflows/draft/datasource/nodes/${nodeId}/run`
|
||||
|
||||
const getOnlineDocuments = useCallback(async () => {
|
||||
ssePost(
|
||||
datasourceNodeRunURL,
|
||||
{
|
||||
body: {
|
||||
inputs: {},
|
||||
datasource_type: DatasourceType.onlineDocument,
|
||||
},
|
||||
},
|
||||
{
|
||||
onDataSourceNodeCompleted: (documentsData: DataSourceNodeCompletedResponse) => {
|
||||
setDocumentsData(documentsData.data as DataSourceNotionWorkspace[])
|
||||
setCurrentWorkspaceId(documentsData.data[0].workspace_id)
|
||||
},
|
||||
onError: (message: string) => {
|
||||
Toast.notify({
|
||||
type: 'error',
|
||||
message,
|
||||
})
|
||||
},
|
||||
},
|
||||
)
|
||||
}, [datasourceNodeRunURL, setCurrentWorkspaceId, setDocumentsData])
|
||||
|
||||
useEffect(() => {
|
||||
if (!documentsData.length)
|
||||
getOnlineDocuments()
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [])
|
||||
|
||||
const currentWorkspace = documentsData.find(workspace => workspace.workspace_id === currentWorkspaceId)
|
||||
|
||||
const handleSearchValueChange = useCallback((value: string) => {
|
||||
setSearchValue(value)
|
||||
}, [setSearchValue])
|
||||
|
||||
const handleSelectWorkspace = useCallback((workspaceId: string) => {
|
||||
setCurrentWorkspaceId(workspaceId)
|
||||
}, [setCurrentWorkspaceId])
|
||||
|
||||
const handleSelectPages = useCallback((newSelectedPagesId: Set<string>) => {
|
||||
const selectedPages = Array.from(newSelectedPagesId).map(pageId => PagesMapAndSelectedPagesId[pageId])
|
||||
|
||||
setSelectedPagesId(new Set(Array.from(newSelectedPagesId)))
|
||||
onSelect(selectedPages)
|
||||
}, [setSelectedPagesId, onSelect, PagesMapAndSelectedPagesId])
|
||||
|
||||
const handlePreviewPage = useCallback((previewPageId: string) => {
|
||||
if (onPreview)
|
||||
onPreview(PagesMapAndSelectedPagesId[previewPageId])
|
||||
}, [PagesMapAndSelectedPagesId, onPreview])
|
||||
|
||||
const headerInfo = useMemo(() => {
|
||||
return {
|
||||
title: nodeData.title,
|
||||
docTitle: 'How to use?',
|
||||
docLink: 'https://docs.dify.ai',
|
||||
}
|
||||
}, [nodeData])
|
||||
|
||||
if (!documentsData?.length)
|
||||
return null
|
||||
|
||||
return (
|
||||
<div className='flex flex-col gap-y-2'>
|
||||
<Header
|
||||
isInPipeline={isInPipeline}
|
||||
{...headerInfo}
|
||||
/>
|
||||
<div className='rounded-xl border border-components-panel-border bg-background-default-subtle'>
|
||||
<div className='flex h-12 items-center gap-x-2 rounded-t-xl border-b border-b-divider-regular bg-components-panel-bg p-2'>
|
||||
<div className='flex grow items-center gap-x-1'>
|
||||
<WorkspaceSelector
|
||||
value={currentWorkspaceId}
|
||||
items={documentsData}
|
||||
onSelect={handleSelectWorkspace}
|
||||
/>
|
||||
</div>
|
||||
<SearchInput
|
||||
value={searchValue}
|
||||
onChange={handleSearchValueChange}
|
||||
/>
|
||||
</div>
|
||||
<div className='overflow-hidden rounded-b-xl'>
|
||||
<PageSelector
|
||||
checkedIds={selectedPagesId}
|
||||
disabledValue={new Set()}
|
||||
searchValue={searchValue}
|
||||
list={currentWorkspace?.pages || []}
|
||||
pagesMap={PagesMapAndSelectedPagesId}
|
||||
onSelect={handleSelectPages}
|
||||
canPreview={!isInPipeline}
|
||||
previewPageId={previewPageId}
|
||||
onPreview={handlePreviewPage}
|
||||
isMultipleChoice={!isInPipeline}
|
||||
currentWorkspaceId={currentWorkspaceId}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
export default OnlineDocuments
|
||||
@ -0,0 +1,194 @@
|
||||
import { useCallback, useEffect, useMemo, useState } from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import { FixedSizeList as List } from 'react-window'
|
||||
import type { DataSourceNotionPage, DataSourceNotionPageMap } from '@/models/common'
|
||||
import Item from './item'
|
||||
import { recursivePushInParentDescendants } from './utils'
|
||||
|
||||
type PageSelectorProps = {
|
||||
checkedIds: Set<string>
|
||||
disabledValue: Set<string>
|
||||
searchValue: string
|
||||
pagesMap: DataSourceNotionPageMap
|
||||
list: DataSourceNotionPage[]
|
||||
onSelect: (selectedPagesId: Set<string>) => void
|
||||
canPreview?: boolean
|
||||
previewPageId?: string
|
||||
onPreview?: (selectedPageId: string) => void
|
||||
isMultipleChoice?: boolean
|
||||
currentWorkspaceId: string
|
||||
}
|
||||
|
||||
export type NotionPageTreeItem = {
|
||||
children: Set<string>
|
||||
descendants: Set<string>
|
||||
depth: number
|
||||
ancestors: string[]
|
||||
} & DataSourceNotionPage
|
||||
|
||||
export type NotionPageTreeMap = Record<string, NotionPageTreeItem>
|
||||
|
||||
type NotionPageItem = {
|
||||
expand: boolean
|
||||
depth: number
|
||||
} & DataSourceNotionPage
|
||||
|
||||
const PageSelector = ({
|
||||
checkedIds,
|
||||
disabledValue,
|
||||
searchValue,
|
||||
pagesMap,
|
||||
list,
|
||||
onSelect,
|
||||
canPreview = true,
|
||||
previewPageId,
|
||||
onPreview,
|
||||
isMultipleChoice = true,
|
||||
currentWorkspaceId,
|
||||
}: PageSelectorProps) => {
|
||||
const { t } = useTranslation()
|
||||
const [dataList, setDataList] = useState<NotionPageItem[]>([])
|
||||
const [localPreviewPageId, setLocalPreviewPageId] = useState('')
|
||||
|
||||
useEffect(() => {
|
||||
setDataList(list.filter(item => item.parent_id === 'root' || !pagesMap[item.parent_id]).map((item) => {
|
||||
return {
|
||||
...item,
|
||||
expand: false,
|
||||
depth: 0,
|
||||
}
|
||||
}))
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [currentWorkspaceId])
|
||||
|
||||
const searchDataList = list.filter((item) => {
|
||||
return item.page_name.includes(searchValue)
|
||||
}).map((item) => {
|
||||
return {
|
||||
...item,
|
||||
expand: false,
|
||||
depth: 0,
|
||||
}
|
||||
})
|
||||
const currentDataList = searchValue ? searchDataList : dataList
|
||||
const currentPreviewPageId = previewPageId === undefined ? localPreviewPageId : previewPageId
|
||||
|
||||
const listMapWithChildrenAndDescendants = useMemo(() => {
|
||||
return list.reduce((prev: NotionPageTreeMap, next: DataSourceNotionPage) => {
|
||||
const pageId = next.page_id
|
||||
if (!prev[pageId])
|
||||
prev[pageId] = { ...next, children: new Set(), descendants: new Set(), depth: 0, ancestors: [] }
|
||||
|
||||
recursivePushInParentDescendants(pagesMap, prev, prev[pageId], prev[pageId])
|
||||
return prev
|
||||
}, {})
|
||||
}, [list, pagesMap])
|
||||
|
||||
const handleToggle = useCallback((index: number) => {
|
||||
const current = dataList[index]
|
||||
const pageId = current.page_id
|
||||
const currentWithChildrenAndDescendants = listMapWithChildrenAndDescendants[pageId]
|
||||
const descendantsIds = Array.from(currentWithChildrenAndDescendants.descendants)
|
||||
const childrenIds = Array.from(currentWithChildrenAndDescendants.children)
|
||||
let newDataList = []
|
||||
|
||||
if (current.expand) {
|
||||
current.expand = false
|
||||
|
||||
newDataList = [...dataList.filter(item => !descendantsIds.includes(item.page_id))]
|
||||
}
|
||||
else {
|
||||
current.expand = true
|
||||
|
||||
newDataList = [
|
||||
...dataList.slice(0, index + 1),
|
||||
...childrenIds.map(item => ({
|
||||
...pagesMap[item],
|
||||
expand: false,
|
||||
depth: listMapWithChildrenAndDescendants[item].depth,
|
||||
})),
|
||||
...dataList.slice(index + 1),
|
||||
]
|
||||
}
|
||||
setDataList(newDataList)
|
||||
}, [dataList, listMapWithChildrenAndDescendants, pagesMap])
|
||||
|
||||
const handleCheck = useCallback((index: number) => {
|
||||
const copyValue = new Set([...checkedIds])
|
||||
const current = currentDataList[index]
|
||||
const pageId = current.page_id
|
||||
const currentWithChildrenAndDescendants = listMapWithChildrenAndDescendants[pageId]
|
||||
|
||||
if (copyValue.has(pageId)) {
|
||||
if (!searchValue && isMultipleChoice) {
|
||||
for (const item of currentWithChildrenAndDescendants.descendants)
|
||||
copyValue.delete(item)
|
||||
}
|
||||
|
||||
copyValue.delete(pageId)
|
||||
}
|
||||
else {
|
||||
if (!searchValue && isMultipleChoice) {
|
||||
for (const item of currentWithChildrenAndDescendants.descendants)
|
||||
copyValue.add(item)
|
||||
}
|
||||
// Single choice mode, clear previous selection
|
||||
if (!isMultipleChoice && copyValue.size > 0) {
|
||||
copyValue.clear()
|
||||
copyValue.add(pageId)
|
||||
}
|
||||
else {
|
||||
copyValue.add(pageId)
|
||||
}
|
||||
}
|
||||
|
||||
onSelect(new Set([...copyValue]))
|
||||
}, [currentDataList, isMultipleChoice, listMapWithChildrenAndDescendants, onSelect, searchValue, checkedIds])
|
||||
|
||||
const handlePreview = useCallback((index: number) => {
|
||||
const current = currentDataList[index]
|
||||
const pageId = current.page_id
|
||||
|
||||
setLocalPreviewPageId(pageId)
|
||||
|
||||
if (onPreview)
|
||||
onPreview(pageId)
|
||||
}, [currentDataList, onPreview])
|
||||
|
||||
if (!currentDataList.length) {
|
||||
return (
|
||||
<div className='flex h-[296px] items-center justify-center text-[13px] text-text-tertiary'>
|
||||
{t('common.dataSource.notion.selector.noSearchResult')}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
return (
|
||||
<List
|
||||
className='py-2'
|
||||
height={296}
|
||||
itemCount={currentDataList.length}
|
||||
itemSize={28}
|
||||
width='100%'
|
||||
itemKey={(index, data) => data.dataList[index].page_id}
|
||||
itemData={{
|
||||
dataList: currentDataList,
|
||||
handleToggle,
|
||||
checkedIds,
|
||||
disabledCheckedIds: disabledValue,
|
||||
handleCheck,
|
||||
canPreview,
|
||||
handlePreview,
|
||||
listMapWithChildrenAndDescendants,
|
||||
searchValue,
|
||||
previewPageId: currentPreviewPageId,
|
||||
pagesMap,
|
||||
isMultipleChoice,
|
||||
}}
|
||||
>
|
||||
{Item}
|
||||
</List>
|
||||
)
|
||||
}
|
||||
|
||||
export default PageSelector
|
||||
@ -0,0 +1,149 @@
|
||||
import React from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import { areEqual } from 'react-window'
|
||||
import type { ListChildComponentProps } from 'react-window'
|
||||
import { RiArrowDownSLine, RiArrowRightSLine } from '@remixicon/react'
|
||||
import Checkbox from '@/app/components/base/checkbox'
|
||||
import NotionIcon from '@/app/components/base/notion-icon'
|
||||
import cn from '@/utils/classnames'
|
||||
import type { DataSourceNotionPage, DataSourceNotionPageMap } from '@/models/common'
|
||||
import Radio from '@/app/components/base/radio/ui'
|
||||
|
||||
type NotionPageTreeItem = {
|
||||
children: Set<string>
|
||||
descendants: Set<string>
|
||||
depth: number
|
||||
ancestors: string[]
|
||||
} & DataSourceNotionPage
|
||||
|
||||
type NotionPageTreeMap = Record<string, NotionPageTreeItem>
|
||||
|
||||
type NotionPageItem = {
|
||||
expand: boolean
|
||||
depth: number
|
||||
} & DataSourceNotionPage
|
||||
|
||||
const Item = ({ index, style, data }: ListChildComponentProps<{
|
||||
dataList: NotionPageItem[]
|
||||
handleToggle: (index: number) => void
|
||||
checkedIds: Set<string>
|
||||
disabledCheckedIds: Set<string>
|
||||
handleCheck: (index: number) => void
|
||||
canPreview?: boolean
|
||||
handlePreview: (index: number) => void
|
||||
listMapWithChildrenAndDescendants: NotionPageTreeMap
|
||||
searchValue: string
|
||||
previewPageId: string
|
||||
pagesMap: DataSourceNotionPageMap
|
||||
isMultipleChoice?: boolean
|
||||
}>) => {
|
||||
const { t } = useTranslation()
|
||||
const {
|
||||
dataList,
|
||||
handleToggle,
|
||||
checkedIds,
|
||||
disabledCheckedIds,
|
||||
handleCheck,
|
||||
canPreview,
|
||||
handlePreview,
|
||||
listMapWithChildrenAndDescendants,
|
||||
searchValue,
|
||||
previewPageId,
|
||||
pagesMap,
|
||||
isMultipleChoice,
|
||||
} = data
|
||||
const current = dataList[index]
|
||||
const currentWithChildrenAndDescendants = listMapWithChildrenAndDescendants[current.page_id]
|
||||
const hasChild = currentWithChildrenAndDescendants.descendants.size > 0
|
||||
const ancestors = currentWithChildrenAndDescendants.ancestors
|
||||
const breadCrumbs = ancestors.length ? [...ancestors, current.page_name] : [current.page_name]
|
||||
const disabled = disabledCheckedIds.has(current.page_id)
|
||||
|
||||
const renderArrow = () => {
|
||||
if (hasChild) {
|
||||
return (
|
||||
<div
|
||||
className='mr-1 flex h-5 w-5 shrink-0 items-center justify-center rounded-md hover:bg-components-button-ghost-bg-hover'
|
||||
style={{ marginLeft: current.depth * 8 }}
|
||||
onClick={() => handleToggle(index)}
|
||||
>
|
||||
{
|
||||
current.expand
|
||||
? <RiArrowDownSLine className='h-4 w-4 text-text-tertiary' />
|
||||
: <RiArrowRightSLine className='h-4 w-4 text-text-tertiary' />
|
||||
}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
if (current.parent_id === 'root' || !pagesMap[current.parent_id]) {
|
||||
return (
|
||||
<div></div>
|
||||
)
|
||||
}
|
||||
return (
|
||||
<div className='mr-1 h-5 w-5 shrink-0' style={{ marginLeft: current.depth * 8 }} />
|
||||
)
|
||||
}
|
||||
|
||||
return (
|
||||
<div
|
||||
className={cn('group flex cursor-pointer items-center rounded-md pl-2 pr-[2px] hover:bg-state-base-hover',
|
||||
previewPageId === current.page_id && 'bg-state-base-hover')}
|
||||
style={{ ...style, top: style.top as number + 8, left: 8, right: 8, width: 'calc(100% - 16px)' }}
|
||||
>
|
||||
{isMultipleChoice ? (
|
||||
<Checkbox
|
||||
className='mr-2 shrink-0'
|
||||
checked={checkedIds.has(current.page_id)}
|
||||
disabled={disabled}
|
||||
onCheck={() => {
|
||||
handleCheck(index)
|
||||
}}
|
||||
/>) : (
|
||||
<Radio
|
||||
className='mr-2 shrink-0'
|
||||
isChecked={checkedIds.has(current.page_id)}
|
||||
disabled={disabled}
|
||||
onCheck={() => {
|
||||
handleCheck(index)
|
||||
}}
|
||||
/>
|
||||
)}
|
||||
{!searchValue && renderArrow()}
|
||||
<NotionIcon
|
||||
className='mr-1 shrink-0'
|
||||
type='page'
|
||||
src={current.page_icon}
|
||||
/>
|
||||
<div
|
||||
className='grow truncate text-[13px] font-medium leading-4 text-text-secondary'
|
||||
title={current.page_name}
|
||||
>
|
||||
{current.page_name}
|
||||
</div>
|
||||
{
|
||||
canPreview && (
|
||||
<div
|
||||
className='ml-1 hidden h-6 shrink-0 cursor-pointer items-center rounded-md border-[0.5px] border-components-button-secondary-border bg-components-button-secondary-bg px-2 text-xs
|
||||
font-medium leading-4 text-components-button-secondary-text shadow-xs shadow-shadow-shadow-3 backdrop-blur-[10px]
|
||||
hover:border-components-button-secondary-border-hover hover:bg-components-button-secondary-bg-hover group-hover:flex'
|
||||
onClick={() => handlePreview(index)}>
|
||||
{t('common.dataSource.notion.selector.preview')}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
{
|
||||
searchValue && (
|
||||
<div
|
||||
className='ml-1 max-w-[120px] shrink-0 truncate text-xs text-text-quaternary'
|
||||
title={breadCrumbs.join(' / ')}
|
||||
>
|
||||
{breadCrumbs.join(' / ')}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
export default React.memo(Item, areEqual)
|
||||
@ -0,0 +1,39 @@
|
||||
import type { DataSourceNotionPageMap } from '@/models/common'
|
||||
import type { NotionPageTreeItem, NotionPageTreeMap } from './index'
|
||||
|
||||
export const recursivePushInParentDescendants = (
|
||||
pagesMap: DataSourceNotionPageMap,
|
||||
listTreeMap: NotionPageTreeMap,
|
||||
current: NotionPageTreeItem,
|
||||
leafItem: NotionPageTreeItem,
|
||||
) => {
|
||||
const parentId = current.parent_id
|
||||
const pageId = current.page_id
|
||||
|
||||
if (!parentId || !pageId)
|
||||
return
|
||||
|
||||
if (parentId !== 'root' && pagesMap[parentId]) {
|
||||
if (!listTreeMap[parentId]) {
|
||||
const children = new Set([pageId])
|
||||
const descendants = new Set([pageId, leafItem.page_id])
|
||||
listTreeMap[parentId] = {
|
||||
...pagesMap[parentId],
|
||||
children,
|
||||
descendants,
|
||||
depth: 0,
|
||||
ancestors: [],
|
||||
}
|
||||
}
|
||||
else {
|
||||
listTreeMap[parentId].children.add(pageId)
|
||||
listTreeMap[parentId].descendants.add(pageId)
|
||||
listTreeMap[parentId].descendants.add(leafItem.page_id)
|
||||
}
|
||||
leafItem.depth++
|
||||
leafItem.ancestors.unshift(listTreeMap[parentId].page_name)
|
||||
|
||||
if (listTreeMap[parentId].parent_id !== 'root')
|
||||
recursivePushInParentDescendants(pagesMap, listTreeMap, listTreeMap[parentId], leafItem)
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,50 @@
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import Button from '@/app/components/base/button'
|
||||
import { Icon3Dots } from '@/app/components/base/icons/src/vender/line/others'
|
||||
import BlockIcon from '@/app/components/workflow/block-icon'
|
||||
import { useToolIcon } from '@/app/components/workflow/hooks'
|
||||
import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types'
|
||||
import { BlockEnum } from '@/app/components/workflow/types'
|
||||
|
||||
type ConnectProps = {
|
||||
nodeData: DataSourceNodeType
|
||||
onSetting: () => void
|
||||
}
|
||||
|
||||
const Connect = ({
|
||||
nodeData,
|
||||
onSetting,
|
||||
}: ConnectProps) => {
|
||||
const { t } = useTranslation()
|
||||
const toolIcon = useToolIcon(nodeData)
|
||||
|
||||
return (
|
||||
<div className='flex flex-col items-start gap-y-2 rounded-xl bg-workflow-process-bg p-6'>
|
||||
<div className='flex size-12 items-center justify-center rounded-[10px] border-[0.5px] border-components-card-border bg-components-card-bg p-1 shadow-lg shadow-shadow-shadow-5'>
|
||||
<BlockIcon
|
||||
type={BlockEnum.DataSource}
|
||||
toolIcon={toolIcon}
|
||||
size='md'
|
||||
/>
|
||||
</div>
|
||||
<div className='flex flex-col gap-y-1'>
|
||||
<div className='flex flex-col gap-y-1 pb-3 pt-1'>
|
||||
<div className='system-md-semibold text-text-secondary'>
|
||||
<span className='relative'>
|
||||
{t('datasetPipeline.onlineDrive.notConnected', { name: nodeData.title })}
|
||||
<Icon3Dots className='absolute -right-2.5 -top-1.5 size-4 text-text-secondary' />
|
||||
</span>
|
||||
</div>
|
||||
<div className='system-sm-regular text-text-tertiary'>
|
||||
{t('datasetPipeline.onlineDrive.notConnectedTip', { name: nodeData.title })}
|
||||
</div>
|
||||
</div>
|
||||
<Button className='w-fit' variant='primary' onClick={onSetting}>
|
||||
{t('datasetCreation.stepOne.connect')}
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
export default Connect
|
||||
@ -0,0 +1,33 @@
|
||||
import React from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
|
||||
type BreadcrumbsProps = {
|
||||
prefix: string[]
|
||||
keywords: string
|
||||
searchResultsLength: number
|
||||
}
|
||||
|
||||
const Breadcrumbs = ({
|
||||
prefix,
|
||||
keywords,
|
||||
searchResultsLength,
|
||||
}: BreadcrumbsProps) => {
|
||||
const { t } = useTranslation()
|
||||
const isRoot = prefix.length === 0
|
||||
const isSearching = !!keywords
|
||||
|
||||
return (
|
||||
<div className='flex grow items-center py-1'>
|
||||
{isRoot && (
|
||||
<div className='system-sm-medium text-test-secondary px-[5px] py-1'>
|
||||
{t('datasetPipeline.onlineDrive.breadcrumbs.allBuckets')}
|
||||
</div>
|
||||
)}
|
||||
{!isRoot && (
|
||||
<div></div>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
export default React.memo(Breadcrumbs)
|
||||
@ -0,0 +1,45 @@
|
||||
import React from 'react'
|
||||
import Breadcrumbs from './breadcrumbs'
|
||||
import Input from '@/app/components/base/input'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
|
||||
type HeaderProps = {
|
||||
prefix: string[]
|
||||
inputValue: string
|
||||
keywords: string
|
||||
searchResultsLength: number
|
||||
handleInputChange: React.ChangeEventHandler<HTMLInputElement>
|
||||
handleResetKeywords: () => void
|
||||
}
|
||||
|
||||
const Header = ({
|
||||
prefix,
|
||||
inputValue,
|
||||
keywords,
|
||||
searchResultsLength,
|
||||
handleInputChange,
|
||||
handleResetKeywords,
|
||||
}: HeaderProps) => {
|
||||
const { t } = useTranslation()
|
||||
|
||||
return (
|
||||
<div className='flex items-center gap-x-2 bg-components-panel-bg p-1 pl-3'>
|
||||
<Breadcrumbs
|
||||
prefix={prefix}
|
||||
keywords={keywords}
|
||||
searchResultsLength={searchResultsLength}
|
||||
/>
|
||||
<Input
|
||||
value={inputValue}
|
||||
onChange={handleInputChange}
|
||||
onClear={handleResetKeywords}
|
||||
placeholder={t('datasetPipeline.onlineDrive.breadcrumbs.searchPlaceholder')}
|
||||
showLeftIcon
|
||||
showClearIcon
|
||||
wrapperClassName='w-[200px] h-8 shrink-0'
|
||||
/>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
export default React.memo(Header)
|
||||
@ -0,0 +1,66 @@
|
||||
import type { OnlineDriveFile } from '@/models/pipeline'
|
||||
import Header from './header'
|
||||
import List from './list'
|
||||
import { useState } from 'react'
|
||||
import { useDebounceFn } from 'ahooks'
|
||||
|
||||
type FileListProps = {
|
||||
fileList: OnlineDriveFile[]
|
||||
selectedFileList: string[]
|
||||
prefix: string[]
|
||||
keywords: string
|
||||
resetKeywords: () => void
|
||||
updateKeywords: (keywords: string) => void
|
||||
searchResultsLength: number
|
||||
}
|
||||
|
||||
const FileList = ({
|
||||
fileList,
|
||||
selectedFileList,
|
||||
prefix,
|
||||
keywords,
|
||||
resetKeywords,
|
||||
updateKeywords,
|
||||
searchResultsLength,
|
||||
}: FileListProps) => {
|
||||
const [inputValue, setInputValue] = useState(keywords)
|
||||
|
||||
const { run: updateKeywordsWithDebounce } = useDebounceFn(
|
||||
(keywords: string) => {
|
||||
updateKeywords(keywords)
|
||||
},
|
||||
{ wait: 500 },
|
||||
)
|
||||
|
||||
const handleInputChange = (e: React.ChangeEvent<HTMLInputElement>) => {
|
||||
const keywords = e.target.value
|
||||
setInputValue(keywords)
|
||||
updateKeywordsWithDebounce(keywords)
|
||||
}
|
||||
|
||||
const handleResetKeywords = () => {
|
||||
setInputValue('')
|
||||
resetKeywords()
|
||||
}
|
||||
|
||||
return (
|
||||
<div className='flex h-[400px] flex-col overflow-hidden rounded-xl border border-components-panel-border bg-components-panel-bg shadow-xs shadow-shadow-shadow-3'>
|
||||
<Header
|
||||
prefix={prefix}
|
||||
inputValue={inputValue}
|
||||
keywords={keywords}
|
||||
handleInputChange={handleInputChange}
|
||||
searchResultsLength={searchResultsLength}
|
||||
handleResetKeywords={handleResetKeywords}
|
||||
/>
|
||||
<List
|
||||
fileList={fileList}
|
||||
selectedFileList={selectedFileList}
|
||||
keywords={keywords}
|
||||
handleResetKeywords={handleResetKeywords}
|
||||
/>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
export default FileList
|
||||
@ -0,0 +1,14 @@
|
||||
import React from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
|
||||
const EmptyFolder = () => {
|
||||
const { t } = useTranslation()
|
||||
|
||||
return (
|
||||
<div className='flex size-full items-center justify-center rounded-[10px] bg-background-section px-1 py-1.5'>
|
||||
<span className='system-xs-regular text-text-tertiary'>{t('datasetPipeline.onlineDrive.emptyFolder')}</span>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
export default React.memo(EmptyFolder)
|
||||
@ -0,0 +1,35 @@
|
||||
import Button from '@/app/components/base/button'
|
||||
import { SearchMenu } from '@/app/components/base/icons/src/vender/knowledge'
|
||||
import React from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
|
||||
type EmptySearchResultProps = {
|
||||
onResetKeywords: () => void
|
||||
}
|
||||
|
||||
const EmptySearchResult = ({
|
||||
onResetKeywords,
|
||||
}: EmptySearchResultProps & {
|
||||
className?: string
|
||||
}) => {
|
||||
const { t } = useTranslation()
|
||||
|
||||
return (
|
||||
<div className='flex size-full flex-col items-center justify-center gap-y-2 rounded-[10px] bg-background-section p-6'>
|
||||
<SearchMenu className='size-8 text-text-tertiary' />
|
||||
<div className='system-sm-regular text-text-secondary'>
|
||||
{t('datasetPipeline.onlineDrive.emptySearchResult')}
|
||||
</div>
|
||||
<Button
|
||||
variant='secondary-accent'
|
||||
size='small'
|
||||
onClick={onResetKeywords}
|
||||
className='px-1.5'
|
||||
>
|
||||
<span className='px-[3px]'>{t('datasetPipeline.onlineDrive.resetKeywords')}</span>
|
||||
</Button>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
export default React.memo(EmptySearchResult)
|
||||
@ -0,0 +1,42 @@
|
||||
import React from 'react'
|
||||
import { OnlineDriveFileType } from '@/models/pipeline'
|
||||
import { BucketsBlue, Folder } from '@/app/components/base/icons/src/public/knowledge/online-drive'
|
||||
import FileTypeIcon from '@/app/components/base/file-uploader/file-type-icon'
|
||||
import { getFileType } from './utils'
|
||||
import cn from '@/utils/classnames'
|
||||
|
||||
type FileIconProps = {
|
||||
type: OnlineDriveFileType
|
||||
fileName: string
|
||||
size?: 'sm' | 'md' | 'lg' | 'xl'
|
||||
className?: string
|
||||
}
|
||||
|
||||
const FileIcon = ({
|
||||
type,
|
||||
fileName,
|
||||
size = 'md',
|
||||
className,
|
||||
}: FileIconProps) => {
|
||||
if (type === OnlineDriveFileType.bucket) {
|
||||
return (
|
||||
<BucketsBlue className={cn('size-[18px]', className)} />
|
||||
)
|
||||
}
|
||||
|
||||
if (type === OnlineDriveFileType.folder) {
|
||||
return (
|
||||
<Folder className={cn('size-[18px]', className)} />
|
||||
)
|
||||
}
|
||||
|
||||
return (
|
||||
<FileTypeIcon
|
||||
size={size}
|
||||
type={getFileType(fileName)}
|
||||
className={cn('size-[18px]', className)}
|
||||
/>
|
||||
)
|
||||
}
|
||||
|
||||
export default React.memo(FileIcon)
|
||||
@ -0,0 +1,57 @@
|
||||
import type { OnlineDriveFile } from '@/models/pipeline'
|
||||
import Item from './item'
|
||||
import EmptyFolder from './empty-folder'
|
||||
import EmptySearchResult from './empty-search-result'
|
||||
|
||||
type FileListProps = {
|
||||
fileList: OnlineDriveFile[]
|
||||
selectedFileList: string[]
|
||||
keywords: string
|
||||
handleResetKeywords: () => void
|
||||
}
|
||||
|
||||
const List = ({
|
||||
fileList,
|
||||
selectedFileList,
|
||||
keywords,
|
||||
handleResetKeywords,
|
||||
}: FileListProps) => {
|
||||
const isEmptyFolder = fileList.length === 0 && keywords.length === 0
|
||||
const isSearchResultEmpty = fileList.length === 0 && keywords.length > 0
|
||||
|
||||
return (
|
||||
<div className='grow overflow-hidden p-1 pt-0'>
|
||||
{
|
||||
isEmptyFolder && (
|
||||
<EmptyFolder />
|
||||
)
|
||||
}
|
||||
{
|
||||
isSearchResultEmpty && (
|
||||
<EmptySearchResult onResetKeywords={handleResetKeywords} />
|
||||
)
|
||||
}
|
||||
{fileList.length > 0 && (
|
||||
<div className='flex h-full flex-col gap-y-px overflow-y-auto rounded-[10px] bg-background-section px-1 py-1.5'>
|
||||
{
|
||||
fileList.map((file) => {
|
||||
const isSelected = selectedFileList.includes(file.key)
|
||||
return (
|
||||
<Item
|
||||
key={file.key}
|
||||
file={file}
|
||||
isSelected={isSelected}
|
||||
onSelect={(file) => { console.log(file) }}
|
||||
onOpen={(file) => { console.log(file) }}
|
||||
disabled
|
||||
/>
|
||||
)
|
||||
})
|
||||
}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
export default List
|
||||
@ -0,0 +1,78 @@
|
||||
import Checkbox from '@/app/components/base/checkbox'
|
||||
import Radio from '@/app/components/base/radio/ui'
|
||||
import type { OnlineDriveFile } from '@/models/pipeline'
|
||||
import React, { useCallback } from 'react'
|
||||
import FileIcon from './file-icon'
|
||||
import { formatFileSize } from '@/utils/format'
|
||||
import Tooltip from '@/app/components/base/tooltip'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import cn from '@/utils/classnames'
|
||||
|
||||
type ItemProps = {
|
||||
file: OnlineDriveFile
|
||||
isSelected: boolean
|
||||
disabled?: boolean
|
||||
isMultipleChoice?: boolean
|
||||
onSelect: (file: OnlineDriveFile) => void
|
||||
onOpen: (file: OnlineDriveFile) => void
|
||||
}
|
||||
|
||||
const Item = ({
|
||||
file,
|
||||
isSelected,
|
||||
disabled = false,
|
||||
isMultipleChoice = true,
|
||||
onSelect,
|
||||
onOpen,
|
||||
}: ItemProps) => {
|
||||
const { t } = useTranslation()
|
||||
const isBucket = file.type === 'bucket'
|
||||
const isFolder = file.type === 'folder'
|
||||
|
||||
const Wrapper = disabled ? Tooltip : React.Fragment
|
||||
|
||||
const handleSelect = useCallback(() => {
|
||||
onSelect(file)
|
||||
}, [file, onSelect])
|
||||
|
||||
const handleClickItem = useCallback((e: React.MouseEvent<HTMLDivElement>) => {
|
||||
e.stopPropagation()
|
||||
if (disabled) return
|
||||
if (isBucket || isFolder)
|
||||
onOpen(file)
|
||||
onSelect(file)
|
||||
}, [disabled, file, isBucket, isFolder, onOpen, onSelect])
|
||||
|
||||
return (
|
||||
<div
|
||||
className='flex cursor-pointer items-center gap-2 rounded-md px-2 py-[3px] hover:bg-state-base-hover'
|
||||
onClick={handleClickItem}
|
||||
>
|
||||
{!isBucket && isMultipleChoice && (
|
||||
<Checkbox className='shrink-0' id={file.key} checked={isSelected} onCheck={handleSelect} />
|
||||
)}
|
||||
{!isBucket && !isMultipleChoice && (
|
||||
<Radio className='shrink-0' isChecked={isSelected} onCheck={handleSelect} />
|
||||
)}
|
||||
<Wrapper
|
||||
popupContent={t('datasetPipeline.onlineDrive.notSupportedFileType')}
|
||||
position='top-end'
|
||||
offset={{ mainAxis: 4, crossAxis: 104 }}
|
||||
>
|
||||
<div
|
||||
className={cn(
|
||||
'flex grow items-center gap-x-1 py-0.5',
|
||||
disabled && 'opacity-30',
|
||||
)}>
|
||||
<FileIcon type={file.type} fileName={file.key} className='shrink-0' />
|
||||
<span className='system-sm-medium grow text-text-secondary'>{file.key}</span>
|
||||
{!isFolder && file.size && (
|
||||
<span className='system-xs-regular shrink-0 text-text-tertiary'>{formatFileSize(file.size)}</span>
|
||||
)}
|
||||
</div>
|
||||
</Wrapper>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
export default React.memo(Item)
|
||||
@ -0,0 +1,51 @@
|
||||
import { FileAppearanceTypeEnum } from '@/app/components/base/file-uploader/types'
|
||||
import { FILE_EXTS } from '@/app/components/base/prompt-editor/constants'
|
||||
|
||||
export const getFileExtension = (fileName: string): string => {
|
||||
if (!fileName)
|
||||
return ''
|
||||
const parts = fileName.split('.')
|
||||
if (parts.length <= 1 || (parts[0] === '' && parts.length === 2))
|
||||
return ''
|
||||
|
||||
return parts[parts.length - 1].toLowerCase()
|
||||
}
|
||||
|
||||
export const getFileType = (fileName: string) => {
|
||||
const extension = getFileExtension(fileName)
|
||||
|
||||
if (extension === 'gif')
|
||||
return FileAppearanceTypeEnum.gif
|
||||
|
||||
if (FILE_EXTS.image.includes(extension.toUpperCase()))
|
||||
return FileAppearanceTypeEnum.image
|
||||
|
||||
if (FILE_EXTS.video.includes(extension.toUpperCase()))
|
||||
return FileAppearanceTypeEnum.video
|
||||
|
||||
if (FILE_EXTS.audio.includes(extension.toUpperCase()))
|
||||
return FileAppearanceTypeEnum.audio
|
||||
|
||||
if (extension === 'html' || extension === 'htm' || extension === 'xml' || extension === 'json')
|
||||
return FileAppearanceTypeEnum.code
|
||||
|
||||
if (extension === 'pdf')
|
||||
return FileAppearanceTypeEnum.pdf
|
||||
|
||||
if (extension === 'md' || extension === 'markdown' || extension === 'mdx')
|
||||
return FileAppearanceTypeEnum.markdown
|
||||
|
||||
if (extension === 'xlsx' || extension === 'xls' || extension === 'csv')
|
||||
return FileAppearanceTypeEnum.excel
|
||||
|
||||
if (extension === 'docx' || extension === 'doc')
|
||||
return FileAppearanceTypeEnum.word
|
||||
|
||||
if (extension === 'pptx' || extension === 'ppt')
|
||||
return FileAppearanceTypeEnum.ppt
|
||||
|
||||
if (FILE_EXTS.document.includes(extension.toUpperCase()))
|
||||
return FileAppearanceTypeEnum.document
|
||||
|
||||
return FileAppearanceTypeEnum.custom
|
||||
}
|
||||
@ -0,0 +1,48 @@
|
||||
import React from 'react'
|
||||
import Button from '@/app/components/base/button'
|
||||
import Divider from '@/app/components/base/divider'
|
||||
import { RiBookOpenLine, RiEqualizer2Line } from '@remixicon/react'
|
||||
|
||||
type HeaderProps = {
|
||||
onClickConfiguration?: () => void
|
||||
docTitle: string
|
||||
docLink: string
|
||||
}
|
||||
|
||||
const Header = ({
|
||||
onClickConfiguration,
|
||||
docTitle,
|
||||
docLink,
|
||||
}: HeaderProps) => {
|
||||
return (
|
||||
<div className='flex items-center gap-x-2'>
|
||||
<div className='flex shrink-0 grow items-center gap-x-1'>
|
||||
<div className='w-20 bg-black'>
|
||||
{/* placeholder */}
|
||||
</div>
|
||||
<Divider type='vertical' className='mx-1 h-3.5' />
|
||||
<Button
|
||||
variant='ghost'
|
||||
size='small'
|
||||
className='px-1'
|
||||
>
|
||||
<RiEqualizer2Line
|
||||
className='size-4'
|
||||
onClick={onClickConfiguration}
|
||||
/>
|
||||
</Button>
|
||||
</div>
|
||||
<a
|
||||
className='system-xs-medium flex items-center gap-x-1 overflow-hidden text-text-accent'
|
||||
href={docLink}
|
||||
target='_blank'
|
||||
rel='noopener noreferrer'
|
||||
>
|
||||
<RiBookOpenLine className='size-3.5 shrink-0' />
|
||||
<span className='grow truncate' title={docTitle}>{docTitle}</span>
|
||||
</a>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
export default React.memo(Header)
|
||||
@ -0,0 +1,61 @@
|
||||
import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types'
|
||||
import Header from './header'
|
||||
import { useCallback } from 'react'
|
||||
import FileList from './file-list'
|
||||
import type { OnlineDriveFile } from '@/models/pipeline'
|
||||
|
||||
type OnlineDriveProps = {
|
||||
nodeData: DataSourceNodeType
|
||||
prefix: string[]
|
||||
setPrefix: (prefix: string[]) => void
|
||||
keywords: string
|
||||
setKeywords: (keywords: string) => void
|
||||
startAfter: string
|
||||
setStartAfter: (startAfter: string) => void
|
||||
selectedFileList: string[]
|
||||
setSelectedFileList: (selectedFileList: string[]) => void
|
||||
fileList: OnlineDriveFile[]
|
||||
setFileList: (fileList: OnlineDriveFile[]) => void
|
||||
}
|
||||
|
||||
const OnlineDrive = ({
|
||||
nodeData,
|
||||
prefix,
|
||||
setPrefix,
|
||||
keywords,
|
||||
setKeywords,
|
||||
startAfter,
|
||||
setStartAfter,
|
||||
selectedFileList,
|
||||
setSelectedFileList,
|
||||
fileList,
|
||||
setFileList,
|
||||
}: OnlineDriveProps) => {
|
||||
const updateKeywords = useCallback((keywords: string) => {
|
||||
setKeywords(keywords)
|
||||
}, [setKeywords])
|
||||
|
||||
const resetPrefix = useCallback(() => {
|
||||
setKeywords('')
|
||||
}, [setKeywords])
|
||||
|
||||
return (
|
||||
<div className='flex flex-col gap-y-2'>
|
||||
<Header
|
||||
docTitle='Online Drive Docs'
|
||||
docLink='https://docs.dify.ai/'
|
||||
/>
|
||||
<FileList
|
||||
fileList={fileList}
|
||||
selectedFileList={selectedFileList}
|
||||
prefix={prefix}
|
||||
keywords={keywords}
|
||||
resetKeywords={resetPrefix}
|
||||
updateKeywords={updateKeywords}
|
||||
searchResultsLength={fileList.length}
|
||||
/>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
export default OnlineDrive
|
||||
@ -0,0 +1,39 @@
|
||||
'use client'
|
||||
import React from 'react'
|
||||
import cn from '@/utils/classnames'
|
||||
import Checkbox from '@/app/components/base/checkbox'
|
||||
import Tooltip from '@/app/components/base/tooltip'
|
||||
|
||||
type CheckboxWithLabelProps = {
|
||||
className?: string
|
||||
isChecked: boolean
|
||||
onChange: (isChecked: boolean) => void
|
||||
label: string
|
||||
labelClassName?: string
|
||||
tooltip?: string
|
||||
}
|
||||
|
||||
const CheckboxWithLabel = ({
|
||||
className = '',
|
||||
isChecked,
|
||||
onChange,
|
||||
label,
|
||||
labelClassName,
|
||||
tooltip,
|
||||
}: CheckboxWithLabelProps) => {
|
||||
return (
|
||||
<label className={cn('flex items-center space-x-2', className)}>
|
||||
<Checkbox checked={isChecked} onCheck={() => onChange(!isChecked)} />
|
||||
<div className={cn('system-sm-medium text-text-secondary', labelClassName)}>{label}</div>
|
||||
{tooltip && (
|
||||
<Tooltip
|
||||
popupContent={
|
||||
<div className='w-[200px]'>{tooltip}</div>
|
||||
}
|
||||
triggerClassName='ml-0.5 w-4 h-4'
|
||||
/>
|
||||
)}
|
||||
</label>
|
||||
)
|
||||
}
|
||||
export default React.memo(CheckboxWithLabel)
|
||||
@ -0,0 +1,80 @@
|
||||
'use client'
|
||||
import React, { useCallback } from 'react'
|
||||
import cn from '@/utils/classnames'
|
||||
import type { CrawlResultItem as CrawlResultItemType } from '@/models/datasets'
|
||||
import Checkbox from '@/app/components/base/checkbox'
|
||||
import Button from '@/app/components/base/button'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import Radio from '@/app/components/base/radio/ui'
|
||||
|
||||
type CrawledResultItemProps = {
|
||||
payload: CrawlResultItemType
|
||||
isChecked: boolean
|
||||
onCheckChange: (checked: boolean) => void
|
||||
isPreview: boolean
|
||||
showPreview: boolean
|
||||
onPreview: () => void
|
||||
isMultipleChoice?: boolean
|
||||
}
|
||||
|
||||
const CrawledResultItem = ({
|
||||
payload,
|
||||
isChecked,
|
||||
onCheckChange,
|
||||
isPreview,
|
||||
onPreview,
|
||||
showPreview,
|
||||
isMultipleChoice = true,
|
||||
}: CrawledResultItemProps) => {
|
||||
const { t } = useTranslation()
|
||||
|
||||
const handleCheckChange = useCallback(() => {
|
||||
onCheckChange(!isChecked)
|
||||
}, [isChecked, onCheckChange])
|
||||
|
||||
return (
|
||||
<div className={cn(
|
||||
'relative flex cursor-pointer gap-x-2 rounded-lg p-2',
|
||||
isPreview ? 'bg-state-base-active' : 'group hover:bg-state-base-hover',
|
||||
)}>
|
||||
{
|
||||
isMultipleChoice ? (
|
||||
<Checkbox
|
||||
className='shrink-0'
|
||||
checked={isChecked}
|
||||
onCheck={handleCheckChange}
|
||||
/>
|
||||
) : (
|
||||
<Radio
|
||||
isChecked={isChecked}
|
||||
onCheck={handleCheckChange}
|
||||
/>
|
||||
)
|
||||
}
|
||||
<div className='flex min-w-0 grow flex-col gap-y-0.5'>
|
||||
<div
|
||||
className='system-sm-medium truncate text-text-secondary'
|
||||
title={payload.title}
|
||||
>
|
||||
{payload.title}
|
||||
</div>
|
||||
<div
|
||||
className='system-xs-regular truncate text-text-tertiary'
|
||||
title={payload.source_url}
|
||||
>
|
||||
{payload.source_url}
|
||||
</div>
|
||||
</div>
|
||||
{showPreview && (
|
||||
<Button
|
||||
size='small'
|
||||
onClick={onPreview}
|
||||
className='system-xs-medium-uppercase right-2 top-2 hidden px-1.5 group-hover:absolute group-hover:block'
|
||||
>
|
||||
{t('datasetCreation.stepOne.website.preview')}
|
||||
</Button>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
export default React.memo(CrawledResultItem)
|
||||
@ -0,0 +1,93 @@
|
||||
'use client'
|
||||
import React, { useCallback } from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import cn from '@/utils/classnames'
|
||||
import type { CrawlResultItem } from '@/models/datasets'
|
||||
import CheckboxWithLabel from './checkbox-with-label'
|
||||
import CrawledResultItem from './crawled-result-item'
|
||||
|
||||
const I18N_PREFIX = 'datasetCreation.stepOne.website'
|
||||
|
||||
type CrawledResultProps = {
|
||||
className?: string
|
||||
previewIndex?: number
|
||||
list: CrawlResultItem[]
|
||||
checkedList: CrawlResultItem[]
|
||||
onSelectedChange: (selected: CrawlResultItem[]) => void
|
||||
onPreview?: (payload: CrawlResultItem, index: number) => void
|
||||
usedTime: number
|
||||
isMultipleChoice?: boolean
|
||||
}
|
||||
|
||||
const CrawledResult = ({
|
||||
className = '',
|
||||
previewIndex,
|
||||
list,
|
||||
checkedList,
|
||||
onSelectedChange,
|
||||
usedTime,
|
||||
onPreview,
|
||||
isMultipleChoice = true,
|
||||
}: CrawledResultProps) => {
|
||||
const { t } = useTranslation()
|
||||
|
||||
const isCheckAll = checkedList.length === list.length
|
||||
|
||||
const handleCheckedAll = useCallback(() => {
|
||||
if (!isCheckAll)
|
||||
onSelectedChange(list)
|
||||
|
||||
else
|
||||
onSelectedChange([])
|
||||
}, [isCheckAll, list, onSelectedChange])
|
||||
|
||||
const handleItemCheckChange = useCallback((item: CrawlResultItem) => {
|
||||
return (checked: boolean) => {
|
||||
if (checked)
|
||||
isMultipleChoice ? onSelectedChange([...checkedList, item]) : onSelectedChange([item])
|
||||
else
|
||||
onSelectedChange(checkedList.filter(checkedItem => checkedItem.source_url !== item.source_url))
|
||||
}
|
||||
}, [checkedList, onSelectedChange, isMultipleChoice])
|
||||
|
||||
const handlePreview = useCallback((index: number) => {
|
||||
if (!onPreview) return
|
||||
onPreview(list[index], index)
|
||||
}, [list, onPreview])
|
||||
|
||||
return (
|
||||
<div className={cn('flex flex-col gap-y-2', className)}>
|
||||
<div className='system-sm-medium pt-2 text-text-primary'>
|
||||
{t(`${I18N_PREFIX}.scrapTimeInfo`, {
|
||||
total: list.length,
|
||||
time: usedTime.toFixed(1),
|
||||
})}
|
||||
</div>
|
||||
<div className='overflow-hidden rounded-xl border border-components-panel-border bg-components-panel-bg'>
|
||||
{isMultipleChoice && (
|
||||
<div className='flex items-center px-4 py-2'>
|
||||
<CheckboxWithLabel
|
||||
isChecked={isCheckAll}
|
||||
onChange={handleCheckedAll} label={isCheckAll ? t(`${I18N_PREFIX}.resetAll`) : t(`${I18N_PREFIX}.selectAll`)}
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
<div className='flex flex-col gap-y-px border-t border-divider-subtle bg-background-default-subtle p-2'>
|
||||
{list.map((item, index) => (
|
||||
<CrawledResultItem
|
||||
key={item.source_url}
|
||||
payload={item}
|
||||
isChecked={checkedList.some(checkedItem => checkedItem.source_url === item.source_url)}
|
||||
onCheckChange={handleItemCheckChange(item)}
|
||||
isPreview={index === previewIndex}
|
||||
onPreview={handlePreview.bind(null, index)}
|
||||
showPreview={!!onPreview}
|
||||
isMultipleChoice={isMultipleChoice}
|
||||
/>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
export default React.memo(CrawledResult)
|
||||
@ -0,0 +1,89 @@
|
||||
'use client'
|
||||
import React from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import cn from '@/utils/classnames'
|
||||
|
||||
type CrawlingProps = {
|
||||
className?: string
|
||||
crawledNum: number
|
||||
totalNum: number
|
||||
}
|
||||
|
||||
type BlockProps = {
|
||||
className?: string
|
||||
}
|
||||
|
||||
type ItemProps = {
|
||||
firstLineWidth: string
|
||||
secondLineWidth: string
|
||||
}
|
||||
|
||||
const Block = React.memo(({
|
||||
className,
|
||||
}: BlockProps) => {
|
||||
return <div className={cn('bg-text-quaternary opacity-20', className)} />
|
||||
})
|
||||
|
||||
const Item = React.memo(({
|
||||
firstLineWidth,
|
||||
secondLineWidth,
|
||||
}: ItemProps) => {
|
||||
return (
|
||||
<div className='flex gap-x-2 px-2 py-[5px]'>
|
||||
<div className='py-0.5'>
|
||||
<Block className='size-4 rounded-[4px]' />
|
||||
</div>
|
||||
<div className='flex grow flex-col'>
|
||||
<div className='flex h-5 w-full items-center'>
|
||||
<Block className={cn('h-2.5 rounded-sm', firstLineWidth)} />
|
||||
</div>
|
||||
<div className='flex h-[18px] w-full items-center'>
|
||||
<Block className={cn('h-1.5 rounded-sm', secondLineWidth)} />
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
})
|
||||
|
||||
const Crawling = ({
|
||||
className = '',
|
||||
crawledNum,
|
||||
totalNum,
|
||||
}: CrawlingProps) => {
|
||||
const { t } = useTranslation()
|
||||
|
||||
const itemsConfig = [{
|
||||
firstLineWidth: 'w-[35%]',
|
||||
secondLineWidth: 'w-[50%]',
|
||||
}, {
|
||||
firstLineWidth: 'w-[40%]',
|
||||
secondLineWidth: 'w-[45%]',
|
||||
}, {
|
||||
firstLineWidth: 'w-[30%]',
|
||||
secondLineWidth: 'w-[36%]',
|
||||
}]
|
||||
|
||||
return (
|
||||
<div className={cn('mt-2 flex flex-col gap-y-2 pt-2', className)}>
|
||||
<div className='system-sm-medium text-text-primary'>
|
||||
{t('datasetCreation.stepOne.website.totalPageScraped')} {crawledNum}/{totalNum}
|
||||
</div>
|
||||
<div className='overflow-hidden rounded-xl border border-components-panel-border bg-components-panel-bg'>
|
||||
<div className='flex items-center gap-x-2 px-4 py-2'>
|
||||
<Block className='size-4 rounded-[4px]' />
|
||||
<Block className='h-2.5 w-14 rounded-sm' />
|
||||
</div>
|
||||
<div className='flex flex-col gap-px border-t border-divider-subtle bg-background-default-subtle p-2'>
|
||||
{itemsConfig.map((item, index) => (
|
||||
<Item
|
||||
key={index}
|
||||
firstLineWidth={item.firstLineWidth}
|
||||
secondLineWidth={item.secondLineWidth}
|
||||
/>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
export default React.memo(Crawling)
|
||||
@ -0,0 +1,34 @@
|
||||
import React from 'react'
|
||||
import cn from '@/utils/classnames'
|
||||
import { RiErrorWarningFill } from '@remixicon/react'
|
||||
|
||||
type ErrorMessageProps = {
|
||||
className?: string
|
||||
title: string
|
||||
errorMsg?: string
|
||||
}
|
||||
|
||||
const ErrorMessage = ({
|
||||
className,
|
||||
title,
|
||||
errorMsg,
|
||||
}: ErrorMessageProps) => {
|
||||
return (
|
||||
// eslint-disable-next-line tailwindcss/migration-from-tailwind-2
|
||||
<div className={cn(
|
||||
'flex gap-x-0.5 rounded-xl border-[0.5px] border-components-panel-border bg-opacity-40 bg-toast-error-bg p-2 shadow-xs shadow-shadow-shadow-3',
|
||||
className,
|
||||
)}>
|
||||
<div className='flex size-6 items-center justify-center'>
|
||||
<RiErrorWarningFill className='h-4 w-4 text-text-destructive' />
|
||||
</div>
|
||||
<div className='flex flex-col gap-y-0.5 py-1'>
|
||||
<div className='system-xs-medium text-text-primary'>{title}</div>
|
||||
{errorMsg && (
|
||||
<div className='system-xs-regular text-text-secondary'>{errorMsg}</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
export default React.memo(ErrorMessage)
|
||||
@ -0,0 +1,119 @@
|
||||
import Button from '@/app/components/base/button'
|
||||
import { useAppForm } from '@/app/components/base/form'
|
||||
import BaseField from '@/app/components/base/form/form-scenarios/base/field'
|
||||
import { ArrowDownRoundFill } from '@/app/components/base/icons/src/vender/solid/general'
|
||||
import cn from '@/utils/classnames'
|
||||
import { RiPlayLargeLine } from '@remixicon/react'
|
||||
import { useBoolean } from 'ahooks'
|
||||
import { useEffect, useMemo } from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import Toast from '@/app/components/base/toast'
|
||||
import type { RAGPipelineVariables } from '@/models/pipeline'
|
||||
import { useConfigurations, useInitialData } from '@/app/components/rag-pipeline/hooks/use-input-fields'
|
||||
import { generateZodSchema } from '@/app/components/base/form/form-scenarios/base/utils'
|
||||
|
||||
const I18N_PREFIX = 'datasetCreation.stepOne.website'
|
||||
|
||||
type OptionsProps = {
|
||||
variables: RAGPipelineVariables
|
||||
isRunning: boolean
|
||||
runDisabled?: boolean
|
||||
controlFoldOptions?: number
|
||||
onSubmit: (data: Record<string, any>) => void
|
||||
}
|
||||
|
||||
const Options = ({
|
||||
variables,
|
||||
isRunning,
|
||||
runDisabled,
|
||||
controlFoldOptions,
|
||||
onSubmit,
|
||||
}: OptionsProps) => {
|
||||
const { t } = useTranslation()
|
||||
const initialData = useInitialData(variables)
|
||||
const configurations = useConfigurations(variables)
|
||||
const schema = useMemo(() => {
|
||||
return generateZodSchema(configurations)
|
||||
}, [configurations])
|
||||
|
||||
const form = useAppForm({
|
||||
defaultValues: initialData,
|
||||
validators: {
|
||||
onSubmit: ({ value }) => {
|
||||
const result = schema.safeParse(value)
|
||||
if (!result.success) {
|
||||
const issues = result.error.issues
|
||||
const firstIssue = issues[0]
|
||||
const errorMessage = `"${firstIssue.path.join('.')}" ${firstIssue.message}`
|
||||
Toast.notify({
|
||||
type: 'error',
|
||||
message: errorMessage,
|
||||
})
|
||||
return errorMessage
|
||||
}
|
||||
return undefined
|
||||
},
|
||||
},
|
||||
onSubmit: ({ value }) => {
|
||||
onSubmit(value)
|
||||
},
|
||||
})
|
||||
|
||||
const [fold, {
|
||||
toggle: foldToggle,
|
||||
setTrue: foldHide,
|
||||
}] = useBoolean(false)
|
||||
|
||||
useEffect(() => {
|
||||
if (controlFoldOptions !== 0)
|
||||
foldHide()
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [controlFoldOptions])
|
||||
|
||||
return (
|
||||
<form
|
||||
className='w-full'
|
||||
onSubmit={(e) => {
|
||||
e.preventDefault()
|
||||
e.stopPropagation()
|
||||
form.handleSubmit()
|
||||
}}
|
||||
>
|
||||
<div className='flex items-center gap-x-1 px-4 py-2'>
|
||||
<div
|
||||
className='flex grow cursor-pointer select-none items-center gap-x-0.5'
|
||||
onClick={foldToggle}
|
||||
>
|
||||
<span className='system-sm-semibold-uppercase text-text-secondary'>
|
||||
{t(`${I18N_PREFIX}.options`)}
|
||||
</span>
|
||||
<ArrowDownRoundFill className={cn('h-4 w-4 shrink-0 text-text-tertiary', fold && '-rotate-90')} />
|
||||
</div>
|
||||
<Button
|
||||
variant='primary'
|
||||
onClick={form.handleSubmit}
|
||||
disabled={runDisabled || isRunning}
|
||||
loading={isRunning}
|
||||
className='shrink-0 gap-x-0.5'
|
||||
spinnerClassName='!ml-0'
|
||||
>
|
||||
<RiPlayLargeLine className='size-4' />
|
||||
<span className='px-0.5'>{!isRunning ? t(`${I18N_PREFIX}.run`) : t(`${I18N_PREFIX}.running`)}</span>
|
||||
</Button>
|
||||
</div>
|
||||
{!fold && (
|
||||
<div className='flex flex-col gap-3 border-t border-divider-subtle px-4 py-3'>
|
||||
{configurations.map((config, index) => {
|
||||
const FieldComponent = BaseField({
|
||||
initialData,
|
||||
config,
|
||||
})
|
||||
return <FieldComponent key={index} form={form} />
|
||||
})}
|
||||
</div>
|
||||
)}
|
||||
</form>
|
||||
)
|
||||
}
|
||||
|
||||
export default Options
|
||||
@ -0,0 +1,180 @@
|
||||
'use client'
|
||||
import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import type { CrawlResult, CrawlResultItem } from '@/models/datasets'
|
||||
import { CrawlStep } from '@/models/datasets'
|
||||
import Header from '@/app/components/datasets/create/website/base/header'
|
||||
import Options from './base/options'
|
||||
import Crawling from './base/crawling'
|
||||
import ErrorMessage from './base/error-message'
|
||||
import CrawledResult from './base/crawled-result'
|
||||
import {
|
||||
useDraftPipelinePreProcessingParams,
|
||||
usePublishedPipelinePreProcessingParams,
|
||||
} from '@/service/use-pipeline'
|
||||
import { useDatasetDetailContextWithSelector } from '@/context/dataset-detail'
|
||||
import { DatasourceType } from '@/models/pipeline'
|
||||
import { ssePost } from '@/service/base'
|
||||
import type {
|
||||
DataSourceNodeCompletedResponse,
|
||||
DataSourceNodeProcessingResponse,
|
||||
} from '@/types/pipeline'
|
||||
import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types'
|
||||
|
||||
const I18N_PREFIX = 'datasetCreation.stepOne.website'
|
||||
|
||||
export type WebsiteCrawlProps = {
|
||||
nodeId: string
|
||||
nodeData: DataSourceNodeType
|
||||
crawlResult: CrawlResult | undefined
|
||||
setCrawlResult: (payload: CrawlResult) => void
|
||||
step: CrawlStep
|
||||
setStep: (step: CrawlStep) => void
|
||||
checkedCrawlResult: CrawlResultItem[]
|
||||
onCheckedCrawlResultChange: (payload: CrawlResultItem[]) => void
|
||||
previewIndex?: number
|
||||
onPreview?: (payload: CrawlResultItem, index: number) => void
|
||||
isInPipeline?: boolean
|
||||
}
|
||||
|
||||
const WebsiteCrawl = ({
|
||||
nodeId,
|
||||
nodeData,
|
||||
crawlResult,
|
||||
setCrawlResult,
|
||||
step,
|
||||
setStep,
|
||||
checkedCrawlResult,
|
||||
onCheckedCrawlResultChange,
|
||||
previewIndex,
|
||||
onPreview,
|
||||
isInPipeline = false,
|
||||
}: WebsiteCrawlProps) => {
|
||||
const { t } = useTranslation()
|
||||
const [controlFoldOptions, setControlFoldOptions] = useState<number>(0)
|
||||
const [totalNum, setTotalNum] = useState(0)
|
||||
const [crawledNum, setCrawledNum] = useState(0)
|
||||
const pipelineId = useDatasetDetailContextWithSelector(s => s.dataset?.pipeline_id)
|
||||
|
||||
const usePreProcessingParams = useRef(!isInPipeline ? usePublishedPipelinePreProcessingParams : useDraftPipelinePreProcessingParams)
|
||||
const { data: paramsConfig, isFetching: isFetchingParams } = usePreProcessingParams.current({
|
||||
pipeline_id: pipelineId!,
|
||||
node_id: nodeId,
|
||||
}, !!pipelineId && !!nodeId)
|
||||
|
||||
useEffect(() => {
|
||||
if (step !== CrawlStep.init)
|
||||
setControlFoldOptions(Date.now())
|
||||
}, [step])
|
||||
|
||||
const isInit = step === CrawlStep.init
|
||||
const isCrawlFinished = step === CrawlStep.finished
|
||||
const isRunning = step === CrawlStep.running
|
||||
const [crawlErrorMessage, setCrawlErrorMessage] = useState('')
|
||||
const showError = isCrawlFinished && crawlErrorMessage
|
||||
|
||||
const datasourceNodeRunURL = !isInPipeline
|
||||
? `/rag/pipelines/${pipelineId}/workflows/published/datasource/nodes/${nodeId}/run`
|
||||
: `/rag/pipelines/${pipelineId}/workflows/draft/datasource/nodes/${nodeId}/run`
|
||||
|
||||
const handleRun = useCallback(async (value: Record<string, any>) => {
|
||||
setStep(CrawlStep.running)
|
||||
ssePost(
|
||||
datasourceNodeRunURL,
|
||||
{
|
||||
body: {
|
||||
inputs: value,
|
||||
datasource_type: DatasourceType.websiteCrawl,
|
||||
response_mode: 'streaming',
|
||||
},
|
||||
},
|
||||
{
|
||||
onDataSourceNodeProcessing: (data: DataSourceNodeProcessingResponse) => {
|
||||
setTotalNum(data.total ?? 0)
|
||||
setCrawledNum(data.completed ?? 0)
|
||||
},
|
||||
onDataSourceNodeCompleted: (data: DataSourceNodeCompletedResponse) => {
|
||||
const { data: crawlData, time_consuming } = data
|
||||
const crawlResultData = {
|
||||
data: crawlData.map((item: any) => {
|
||||
const { content, ...rest } = item
|
||||
return {
|
||||
markdown: content || '',
|
||||
...rest,
|
||||
} as CrawlResultItem
|
||||
}),
|
||||
time_consuming: time_consuming ?? 0,
|
||||
}
|
||||
setCrawlResult(crawlResultData)
|
||||
onCheckedCrawlResultChange(crawlData || []) // default select the crawl result
|
||||
setCrawlErrorMessage('')
|
||||
setStep(CrawlStep.finished)
|
||||
},
|
||||
onError: (message: string) => {
|
||||
setCrawlErrorMessage(message || t(`${I18N_PREFIX}.unknownError`))
|
||||
setStep(CrawlStep.finished)
|
||||
},
|
||||
},
|
||||
)
|
||||
}, [datasourceNodeRunURL, onCheckedCrawlResultChange, setCrawlResult, setStep, t])
|
||||
|
||||
const handleSubmit = useCallback((value: Record<string, any>) => {
|
||||
handleRun(value)
|
||||
}, [handleRun])
|
||||
|
||||
const headerInfo = useMemo(() => {
|
||||
return {
|
||||
title: nodeData.title,
|
||||
docTitle: 'How to use?',
|
||||
docLink: 'https://docs.dify.ai',
|
||||
}
|
||||
}, [nodeData])
|
||||
|
||||
return (
|
||||
<div className='flex flex-col'>
|
||||
<Header
|
||||
isInPipeline
|
||||
{...headerInfo}
|
||||
/>
|
||||
<div className='mt-2 rounded-xl border border-components-panel-border bg-background-default-subtle'>
|
||||
<Options
|
||||
variables={paramsConfig?.variables || []}
|
||||
isRunning={isRunning}
|
||||
runDisabled={isFetchingParams}
|
||||
controlFoldOptions={controlFoldOptions}
|
||||
onSubmit={handleSubmit}
|
||||
/>
|
||||
</div>
|
||||
{!isInit && (
|
||||
<div className='relative flex flex-col'>
|
||||
{isRunning && (
|
||||
<Crawling
|
||||
crawledNum={crawledNum}
|
||||
totalNum={totalNum}
|
||||
/>
|
||||
)}
|
||||
{showError && (
|
||||
<ErrorMessage
|
||||
className='mt-2'
|
||||
title={t(`${I18N_PREFIX}.exceptionErrorTitle`)}
|
||||
errorMsg={crawlErrorMessage}
|
||||
/>
|
||||
)}
|
||||
{isCrawlFinished && !showError && (
|
||||
<CrawledResult
|
||||
className='mt-2'
|
||||
list={crawlResult?.data || []}
|
||||
checkedList={checkedCrawlResult}
|
||||
onSelectedChange={onCheckedCrawlResultChange}
|
||||
usedTime={Number.parseFloat(crawlResult?.time_consuming as string) || 0}
|
||||
previewIndex={previewIndex}
|
||||
onPreview={onPreview}
|
||||
isMultipleChoice={!isInPipeline} // only support single choice in test run
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
export default React.memo(WebsiteCrawl)
|
||||
Reference in New Issue
Block a user