mirror of
https://github.com/langgenius/dify.git
synced 2026-05-02 08:28:03 +08:00
use base ui toast
This commit is contained in:
yyh
@ -1,10 +1,27 @@
|
||||
import type { MockedFunction } from 'vitest'
|
||||
import { fireEvent, render, screen, waitFor } from '@testing-library/react'
|
||||
import * as React from 'react'
|
||||
import { createEmptyDataset } from '@/service/datasets'
|
||||
import { useInvalidDatasetList } from '@/service/knowledge/use-dataset'
|
||||
import EmptyDatasetCreationModal from '../index'
|
||||
|
||||
const { mockNotify, mockToast } = vi.hoisted(() => {
|
||||
const mockNotify = vi.fn()
|
||||
const mockToast = Object.assign(mockNotify, {
|
||||
success: vi.fn((message, options) => mockNotify({ type: 'success', message, ...options })),
|
||||
error: vi.fn((message, options) => mockNotify({ type: 'error', message, ...options })),
|
||||
warning: vi.fn((message, options) => mockNotify({ type: 'warning', message, ...options })),
|
||||
info: vi.fn((message, options) => mockNotify({ type: 'info', message, ...options })),
|
||||
dismiss: vi.fn(),
|
||||
update: vi.fn(),
|
||||
promise: vi.fn(),
|
||||
})
|
||||
return { mockNotify, mockToast }
|
||||
})
|
||||
|
||||
vi.mock('@/app/components/base/ui/toast', () => ({
|
||||
toast: mockToast,
|
||||
}))
|
||||
|
||||
// Mock Next.js router
|
||||
const mockPush = vi.fn()
|
||||
vi.mock('@/next/navigation', () => ({
|
||||
@ -23,15 +40,6 @@ vi.mock('@/service/knowledge/use-dataset', () => ({
|
||||
useInvalidDatasetList: vi.fn(),
|
||||
}))
|
||||
|
||||
// Mock ToastContext - need to mock both createContext and useContext from use-context-selector
|
||||
const mockNotify = vi.fn()
|
||||
vi.mock('use-context-selector', () => ({
|
||||
createContext: vi.fn(() => ({
|
||||
Provider: ({ children }: { children: React.ReactNode }) => children,
|
||||
})),
|
||||
useContext: vi.fn(() => ({ notify: mockNotify })),
|
||||
}))
|
||||
|
||||
// Type cast mocked functions
|
||||
const mockCreateEmptyDataset = createEmptyDataset as MockedFunction<typeof createEmptyDataset>
|
||||
const mockInvalidDatasetList = vi.fn()
|
||||
|
||||
@ -2,14 +2,12 @@
|
||||
import * as React from 'react'
|
||||
import { useState } from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import { useContext } from 'use-context-selector'
|
||||
import { trackEvent } from '@/app/components/base/amplitude'
|
||||
import Button from '@/app/components/base/button'
|
||||
import Input from '@/app/components/base/input'
|
||||
import Modal from '@/app/components/base/modal'
|
||||
import { ToastContext } from '@/app/components/base/toast/context'
|
||||
import { toast } from '@/app/components/base/ui/toast'
|
||||
import { useRouter } from '@/next/navigation'
|
||||
|
||||
import { createEmptyDataset } from '@/service/datasets'
|
||||
import { useInvalidDatasetList } from '@/service/knowledge/use-dataset'
|
||||
import { cn } from '@/utils/classnames'
|
||||
@ -19,24 +17,18 @@ type IProps = {
|
||||
show: boolean
|
||||
onHide: () => void
|
||||
}
|
||||
|
||||
const EmptyDatasetCreationModal = ({
|
||||
show = false,
|
||||
onHide,
|
||||
}: IProps) => {
|
||||
const EmptyDatasetCreationModal = ({ show = false, onHide }: IProps) => {
|
||||
const [inputValue, setInputValue] = useState('')
|
||||
const { t } = useTranslation()
|
||||
const { notify } = useContext(ToastContext)
|
||||
const router = useRouter()
|
||||
const invalidDatasetList = useInvalidDatasetList()
|
||||
|
||||
const submit = async () => {
|
||||
if (!inputValue) {
|
||||
notify({ type: 'error', message: t('stepOne.modal.nameNotEmpty', { ns: 'datasetCreation' }) })
|
||||
toast.error(t('stepOne.modal.nameNotEmpty', { ns: 'datasetCreation' }))
|
||||
return
|
||||
}
|
||||
if (inputValue.length > 40) {
|
||||
notify({ type: 'error', message: t('stepOne.modal.nameLengthInvalid', { ns: 'datasetCreation' }) })
|
||||
toast.error(t('stepOne.modal.nameLengthInvalid', { ns: 'datasetCreation' }))
|
||||
return
|
||||
}
|
||||
try {
|
||||
@ -50,16 +42,11 @@ const EmptyDatasetCreationModal = ({
|
||||
router.push(`/datasets/${dataset.id}/documents`)
|
||||
}
|
||||
catch {
|
||||
notify({ type: 'error', message: t('stepOne.modal.failed', { ns: 'datasetCreation' }) })
|
||||
toast.error(t('stepOne.modal.failed', { ns: 'datasetCreation' }))
|
||||
}
|
||||
}
|
||||
|
||||
return (
|
||||
<Modal
|
||||
isShow={show}
|
||||
onClose={onHide}
|
||||
className={cn(s.modal, '!max-w-[520px]', 'px-8')}
|
||||
>
|
||||
<Modal isShow={show} onClose={onHide} className={cn(s.modal, '!max-w-[520px]', 'px-8')}>
|
||||
<div className={s.modalHeader}>
|
||||
<div className={s.title}>{t('stepOne.modal.title', { ns: 'datasetCreation' })}</div>
|
||||
<span className={s.close} onClick={onHide} />
|
||||
@ -76,5 +63,4 @@ const EmptyDatasetCreationModal = ({
|
||||
</Modal>
|
||||
)
|
||||
}
|
||||
|
||||
export default EmptyDatasetCreationModal
|
||||
|
||||
@ -2,7 +2,6 @@ import type { ReactNode } from 'react'
|
||||
import type { CustomFile, FileItem } from '@/models/datasets'
|
||||
import { act, render, renderHook, waitFor } from '@testing-library/react'
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import { ToastContext } from '@/app/components/base/toast/context'
|
||||
|
||||
import { PROGRESS_COMPLETE, PROGRESS_ERROR, PROGRESS_NOT_STARTED } from '../../constants'
|
||||
// Import after mocks
|
||||
@ -63,9 +62,9 @@ vi.mock('@/app/components/base/file-uploader/utils', () => ({
|
||||
|
||||
const createWrapper = () => {
|
||||
return ({ children }: { children: ReactNode }) => (
|
||||
<ToastContext.Provider value={{ notify: mockNotify, close: mockClose }}>
|
||||
<>
|
||||
{children}
|
||||
</ToastContext.Provider>
|
||||
</>
|
||||
)
|
||||
}
|
||||
|
||||
@ -385,9 +384,9 @@ describe('useFileUpload', () => {
|
||||
it('should set dragging true on dragenter', async () => {
|
||||
const { getByTestId } = await act(async () =>
|
||||
render(
|
||||
<ToastContext.Provider value={{ notify: mockNotify, close: mockClose }}>
|
||||
<>
|
||||
<TestDropzone options={defaultOptions} />
|
||||
</ToastContext.Provider>,
|
||||
</>,
|
||||
),
|
||||
)
|
||||
|
||||
@ -404,9 +403,9 @@ describe('useFileUpload', () => {
|
||||
it('should handle dragover event', async () => {
|
||||
const { getByTestId } = await act(async () =>
|
||||
render(
|
||||
<ToastContext.Provider value={{ notify: mockNotify, close: mockClose }}>
|
||||
<>
|
||||
<TestDropzone options={defaultOptions} />
|
||||
</ToastContext.Provider>,
|
||||
</>,
|
||||
),
|
||||
)
|
||||
|
||||
@ -423,9 +422,9 @@ describe('useFileUpload', () => {
|
||||
it('should set dragging false on dragleave from drag overlay', async () => {
|
||||
const { getByTestId, queryByTestId } = await act(async () =>
|
||||
render(
|
||||
<ToastContext.Provider value={{ notify: mockNotify, close: mockClose }}>
|
||||
<>
|
||||
<TestDropzone options={defaultOptions} />
|
||||
</ToastContext.Provider>,
|
||||
</>,
|
||||
),
|
||||
)
|
||||
|
||||
@ -454,9 +453,9 @@ describe('useFileUpload', () => {
|
||||
|
||||
const { getByTestId } = await act(async () =>
|
||||
render(
|
||||
<ToastContext.Provider value={{ notify: mockNotify, close: mockClose }}>
|
||||
<>
|
||||
<TestDropzone options={{ ...defaultOptions, prepareFileList }} />
|
||||
</ToastContext.Provider>,
|
||||
</>,
|
||||
),
|
||||
)
|
||||
|
||||
@ -486,9 +485,9 @@ describe('useFileUpload', () => {
|
||||
|
||||
const { getByTestId } = await act(async () =>
|
||||
render(
|
||||
<ToastContext.Provider value={{ notify: mockNotify, close: mockClose }}>
|
||||
<>
|
||||
<TestDropzone options={{ ...defaultOptions, prepareFileList }} />
|
||||
</ToastContext.Provider>,
|
||||
</>,
|
||||
),
|
||||
)
|
||||
|
||||
@ -509,9 +508,9 @@ describe('useFileUpload', () => {
|
||||
|
||||
const { getByTestId } = await act(async () =>
|
||||
render(
|
||||
<ToastContext.Provider value={{ notify: mockNotify, close: mockClose }}>
|
||||
<>
|
||||
<TestDropzone options={{ ...defaultOptions, supportBatchUpload: false, prepareFileList }} />
|
||||
</ToastContext.Provider>,
|
||||
</>,
|
||||
),
|
||||
)
|
||||
|
||||
@ -549,9 +548,9 @@ describe('useFileUpload', () => {
|
||||
|
||||
const { getByTestId } = await act(async () =>
|
||||
render(
|
||||
<ToastContext.Provider value={{ notify: mockNotify, close: mockClose }}>
|
||||
<>
|
||||
<TestDropzone options={{ ...defaultOptions, prepareFileList }} />
|
||||
</ToastContext.Provider>,
|
||||
</>,
|
||||
),
|
||||
)
|
||||
|
||||
@ -586,9 +585,9 @@ describe('useFileUpload', () => {
|
||||
|
||||
const { getByTestId } = await act(async () =>
|
||||
render(
|
||||
<ToastContext.Provider value={{ notify: mockNotify, close: mockClose }}>
|
||||
<>
|
||||
<TestDropzone options={{ ...defaultOptions, prepareFileList }} />
|
||||
</ToastContext.Provider>,
|
||||
</>,
|
||||
),
|
||||
)
|
||||
|
||||
@ -639,9 +638,9 @@ describe('useFileUpload', () => {
|
||||
|
||||
const { getByTestId } = await act(async () =>
|
||||
render(
|
||||
<ToastContext.Provider value={{ notify: mockNotify, close: mockClose }}>
|
||||
<>
|
||||
<TestDropzone options={{ ...defaultOptions, prepareFileList }} />
|
||||
</ToastContext.Provider>,
|
||||
</>,
|
||||
),
|
||||
)
|
||||
|
||||
@ -678,9 +677,9 @@ describe('useFileUpload', () => {
|
||||
|
||||
const { getByTestId } = await act(async () =>
|
||||
render(
|
||||
<ToastContext.Provider value={{ notify: mockNotify, close: mockClose }}>
|
||||
<>
|
||||
<TestDropzone options={{ ...defaultOptions, prepareFileList }} />
|
||||
</ToastContext.Provider>,
|
||||
</>,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@ -3,9 +3,8 @@ import type { RefObject } from 'react'
|
||||
import type { CustomFile as File, FileItem } from '@/models/datasets'
|
||||
import { useCallback, useEffect, useMemo, useRef, useState } from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import { useContext } from 'use-context-selector'
|
||||
import { getFileUploadErrorMessage } from '@/app/components/base/file-uploader/utils'
|
||||
import { ToastContext } from '@/app/components/base/toast/context'
|
||||
import { toast } from '@/app/components/base/ui/toast'
|
||||
import { IS_CE_EDITION } from '@/config'
|
||||
import { useLocale } from '@/context/i18n'
|
||||
import { LanguagesSupported } from '@/i18n-config/language'
|
||||
@ -70,7 +69,6 @@ export const useFileUpload = ({
|
||||
allowedExtensions,
|
||||
}: UseFileUploadOptions): UseFileUploadReturn => {
|
||||
const { t } = useTranslation()
|
||||
const { notify } = useContext(ToastContext)
|
||||
const locale = useLocale()
|
||||
|
||||
const [dragging, setDragging] = useState(false)
|
||||
@ -119,14 +117,14 @@ export const useFileUpload = ({
|
||||
const ext = `.${getFileExtension(file.name)}`
|
||||
const isValidType = acceptTypes.includes(ext.toLowerCase())
|
||||
if (!isValidType)
|
||||
notify({ type: 'error', message: t('stepOne.uploader.validation.typeError', { ns: 'datasetCreation' }) })
|
||||
toast.error(t('stepOne.uploader.validation.typeError', { ns: 'datasetCreation' }))
|
||||
|
||||
const isValidSize = size <= fileUploadConfig.file_size_limit * 1024 * 1024
|
||||
if (!isValidSize)
|
||||
notify({ type: 'error', message: t('stepOne.uploader.validation.size', { ns: 'datasetCreation', size: fileUploadConfig.file_size_limit }) })
|
||||
toast.error(t('stepOne.uploader.validation.size', { ns: 'datasetCreation', size: fileUploadConfig.file_size_limit }))
|
||||
|
||||
return isValidType && isValidSize
|
||||
}, [fileUploadConfig, notify, t, acceptTypes])
|
||||
}, [fileUploadConfig, t, acceptTypes])
|
||||
|
||||
const fileUpload = useCallback(async (fileItem: FileItem): Promise<FileItem> => {
|
||||
const formData = new FormData()
|
||||
@ -156,12 +154,12 @@ export const useFileUpload = ({
|
||||
})
|
||||
.catch((e) => {
|
||||
const errorMessage = getFileUploadErrorMessage(e, t('stepOne.uploader.failed', { ns: 'datasetCreation' }), t)
|
||||
notify({ type: 'error', message: errorMessage })
|
||||
toast.error(errorMessage)
|
||||
onFileUpdate(fileItem, PROGRESS_ERROR, fileListRef.current)
|
||||
return Promise.resolve({ ...fileItem })
|
||||
})
|
||||
.finally()
|
||||
}, [notify, onFileUpdate, t])
|
||||
}, [onFileUpdate, t])
|
||||
|
||||
const uploadBatchFiles = useCallback((bFiles: FileItem[]) => {
|
||||
bFiles.forEach(bf => (bf.progress = 0))
|
||||
@ -191,7 +189,7 @@ export const useFileUpload = ({
|
||||
return false
|
||||
|
||||
if (files.length + fileList.length > filesCountLimit && !IS_CE_EDITION) {
|
||||
notify({ type: 'error', message: t('stepOne.uploader.validation.filesNumber', { ns: 'datasetCreation', filesNumber: filesCountLimit }) })
|
||||
toast.error(t('stepOne.uploader.validation.filesNumber', { ns: 'datasetCreation', filesNumber: filesCountLimit }))
|
||||
return false
|
||||
}
|
||||
|
||||
@ -204,7 +202,7 @@ export const useFileUpload = ({
|
||||
prepareFileList(newFiles)
|
||||
fileListRef.current = newFiles
|
||||
uploadMultipleFiles(preparedFiles)
|
||||
}, [prepareFileList, uploadMultipleFiles, notify, t, fileList, fileUploadConfig])
|
||||
}, [prepareFileList, uploadMultipleFiles, t, fileList, fileUploadConfig])
|
||||
|
||||
const traverseFileEntry = useCallback(
|
||||
(entry: FileSystemEntry, prefix = ''): Promise<FileWithPath[]> => {
|
||||
|
||||
@ -14,7 +14,7 @@ const mocks = vi.hoisted(() => ({
|
||||
invalidDatasetList: vi.fn(),
|
||||
}))
|
||||
|
||||
vi.mock('@/app/components/base/toast', () => ({
|
||||
vi.mock('@/app/components/base/ui/toast', () => ({
|
||||
default: { notify: mocks.toastNotify },
|
||||
}))
|
||||
|
||||
|
||||
@ -1,26 +1,14 @@
|
||||
import type { DefaultModel, Model } from '@/app/components/header/account-setting/model-provider-page/declarations'
|
||||
import type { NotionPage } from '@/models/common'
|
||||
import type {
|
||||
ChunkingMode,
|
||||
CrawlOptions,
|
||||
CrawlResultItem,
|
||||
CreateDocumentReq,
|
||||
createDocumentResponse,
|
||||
CustomFile,
|
||||
FullDocumentDetail,
|
||||
ProcessRule,
|
||||
SummaryIndexSetting as SummaryIndexSettingType,
|
||||
} from '@/models/datasets'
|
||||
import type { ChunkingMode, CrawlOptions, CrawlResultItem, CreateDocumentReq, createDocumentResponse, CustomFile, FullDocumentDetail, ProcessRule, SummaryIndexSetting as SummaryIndexSettingType } from '@/models/datasets'
|
||||
import type { RetrievalConfig, RETRIEVE_METHOD } from '@/types/app'
|
||||
import { useCallback } from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import { trackEvent } from '@/app/components/base/amplitude'
|
||||
import Toast from '@/app/components/base/toast'
|
||||
import { toast } from '@/app/components/base/ui/toast'
|
||||
import { isReRankModelSelected } from '@/app/components/datasets/common/check-rerank-model'
|
||||
import { DataSourceProvider } from '@/models/common'
|
||||
import {
|
||||
DataSourceType,
|
||||
} from '@/models/datasets'
|
||||
import { DataSourceType } from '@/models/datasets'
|
||||
import { getNotionInfo, getWebsiteInfo, useCreateDocument, useCreateFirstDocument } from '@/service/knowledge/use-create-dataset'
|
||||
import { useInvalidDatasetList } from '@/service/knowledge/use-dataset'
|
||||
import { IndexingType } from './use-indexing-config'
|
||||
@ -46,7 +34,6 @@ export type UseDocumentCreationOptions = {
|
||||
onSave?: () => void
|
||||
mutateDatasetRes?: () => void
|
||||
}
|
||||
|
||||
export type ValidationParams = {
|
||||
segmentationType: string
|
||||
maxChunkLength: number
|
||||
@ -57,93 +44,42 @@ export type ValidationParams = {
|
||||
rerankModelList: Model[]
|
||||
retrievalConfig: RetrievalConfig
|
||||
}
|
||||
|
||||
export const useDocumentCreation = (options: UseDocumentCreationOptions) => {
|
||||
const { t } = useTranslation()
|
||||
const {
|
||||
datasetId,
|
||||
isSetting,
|
||||
documentDetail,
|
||||
dataSourceType,
|
||||
files,
|
||||
notionPages,
|
||||
notionCredentialId,
|
||||
websitePages,
|
||||
crawlOptions,
|
||||
websiteCrawlProvider = DataSourceProvider.jinaReader,
|
||||
websiteCrawlJobId = '',
|
||||
onStepChange,
|
||||
updateIndexingTypeCache,
|
||||
updateResultCache,
|
||||
updateRetrievalMethodCache,
|
||||
onSave,
|
||||
mutateDatasetRes,
|
||||
} = options
|
||||
|
||||
const { datasetId, isSetting, documentDetail, dataSourceType, files, notionPages, notionCredentialId, websitePages, crawlOptions, websiteCrawlProvider = DataSourceProvider.jinaReader, websiteCrawlJobId = '', onStepChange, updateIndexingTypeCache, updateResultCache, updateRetrievalMethodCache, onSave, mutateDatasetRes } = options
|
||||
const createFirstDocumentMutation = useCreateFirstDocument()
|
||||
const createDocumentMutation = useCreateDocument(datasetId!)
|
||||
const invalidDatasetList = useInvalidDatasetList()
|
||||
|
||||
const isCreating = createFirstDocumentMutation.isPending || createDocumentMutation.isPending
|
||||
|
||||
// Validate creation params
|
||||
const validateParams = useCallback((params: ValidationParams): boolean => {
|
||||
const {
|
||||
segmentationType,
|
||||
maxChunkLength,
|
||||
limitMaxChunkLength,
|
||||
overlap,
|
||||
indexType,
|
||||
embeddingModel,
|
||||
rerankModelList,
|
||||
retrievalConfig,
|
||||
} = params
|
||||
|
||||
const { segmentationType, maxChunkLength, limitMaxChunkLength, overlap, indexType, embeddingModel, rerankModelList, retrievalConfig } = params
|
||||
if (segmentationType === 'general' && overlap > maxChunkLength) {
|
||||
Toast.notify({ type: 'error', message: t('stepTwo.overlapCheck', { ns: 'datasetCreation' }) })
|
||||
toast.error(t('stepTwo.overlapCheck', { ns: 'datasetCreation' }))
|
||||
return false
|
||||
}
|
||||
|
||||
if (segmentationType === 'general' && maxChunkLength > limitMaxChunkLength) {
|
||||
Toast.notify({
|
||||
type: 'error',
|
||||
message: t('stepTwo.maxLengthCheck', { ns: 'datasetCreation', limit: limitMaxChunkLength }),
|
||||
})
|
||||
toast.error(t('stepTwo.maxLengthCheck', { ns: 'datasetCreation', limit: limitMaxChunkLength }))
|
||||
return false
|
||||
}
|
||||
|
||||
if (!isSetting) {
|
||||
if (indexType === IndexingType.QUALIFIED && (!embeddingModel.model || !embeddingModel.provider)) {
|
||||
Toast.notify({
|
||||
type: 'error',
|
||||
message: t('datasetConfig.embeddingModelRequired', { ns: 'appDebug' }),
|
||||
})
|
||||
toast.error(t('datasetConfig.embeddingModelRequired', { ns: 'appDebug' }))
|
||||
return false
|
||||
}
|
||||
|
||||
if (!isReRankModelSelected({
|
||||
rerankModelList,
|
||||
retrievalConfig,
|
||||
indexMethod: indexType,
|
||||
})) {
|
||||
Toast.notify({ type: 'error', message: t('datasetConfig.rerankModelRequired', { ns: 'appDebug' }) })
|
||||
toast.error(t('datasetConfig.rerankModelRequired', { ns: 'appDebug' }))
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
return true
|
||||
}, [t, isSetting])
|
||||
|
||||
// Build creation params
|
||||
const buildCreationParams = useCallback((
|
||||
currentDocForm: ChunkingMode,
|
||||
docLanguage: string,
|
||||
processRule: ProcessRule,
|
||||
retrievalConfig: RetrievalConfig,
|
||||
embeddingModel: DefaultModel,
|
||||
indexingTechnique: string,
|
||||
summaryIndexSetting?: SummaryIndexSettingType,
|
||||
): CreateDocumentReq | null => {
|
||||
const buildCreationParams = useCallback((currentDocForm: ChunkingMode, docLanguage: string, processRule: ProcessRule, retrievalConfig: RetrievalConfig, embeddingModel: DefaultModel, indexingTechnique: string, summaryIndexSetting?: SummaryIndexSettingType): CreateDocumentReq | null => {
|
||||
if (isSetting) {
|
||||
return {
|
||||
original_document_id: documentDetail?.id,
|
||||
@ -157,7 +93,6 @@ export const useDocumentCreation = (options: UseDocumentCreationOptions) => {
|
||||
indexing_technique: indexingTechnique,
|
||||
} as CreateDocumentReq
|
||||
}
|
||||
|
||||
const params: CreateDocumentReq = {
|
||||
data_source: {
|
||||
type: dataSourceType,
|
||||
@ -174,7 +109,6 @@ export const useDocumentCreation = (options: UseDocumentCreationOptions) => {
|
||||
embedding_model: embeddingModel.model,
|
||||
embedding_model_provider: embeddingModel.provider,
|
||||
} as CreateDocumentReq
|
||||
|
||||
// Add data source specific info
|
||||
if (dataSourceType === DataSourceType.FILE) {
|
||||
params.data_source!.info_list.file_info_list = {
|
||||
@ -183,7 +117,6 @@ export const useDocumentCreation = (options: UseDocumentCreationOptions) => {
|
||||
}
|
||||
if (dataSourceType === DataSourceType.NOTION)
|
||||
params.data_source!.info_list.notion_info_list = getNotionInfo(notionPages, notionCredentialId)
|
||||
|
||||
if (dataSourceType === DataSourceType.WEB) {
|
||||
params.data_source!.info_list.website_info_list = getWebsiteInfo({
|
||||
websiteCrawlProvider,
|
||||
@ -192,7 +125,6 @@ export const useDocumentCreation = (options: UseDocumentCreationOptions) => {
|
||||
crawlOptions,
|
||||
})
|
||||
}
|
||||
|
||||
return params
|
||||
}, [
|
||||
isSetting,
|
||||
@ -206,13 +138,8 @@ export const useDocumentCreation = (options: UseDocumentCreationOptions) => {
|
||||
websiteCrawlJobId,
|
||||
crawlOptions,
|
||||
])
|
||||
|
||||
// Execute creation
|
||||
const executeCreation = useCallback(async (
|
||||
params: CreateDocumentReq,
|
||||
indexType: IndexingType,
|
||||
retrievalConfig: RetrievalConfig,
|
||||
) => {
|
||||
const executeCreation = useCallback(async (params: CreateDocumentReq, indexType: IndexingType, retrievalConfig: RetrievalConfig) => {
|
||||
if (!datasetId) {
|
||||
await createFirstDocumentMutation.mutateAsync(params, {
|
||||
onSuccess(data) {
|
||||
@ -231,17 +158,13 @@ export const useDocumentCreation = (options: UseDocumentCreationOptions) => {
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
mutateDatasetRes?.()
|
||||
invalidDatasetList()
|
||||
|
||||
trackEvent('create_datasets', {
|
||||
data_source_type: dataSourceType,
|
||||
indexing_technique: indexType,
|
||||
})
|
||||
|
||||
onStepChange?.(+1)
|
||||
|
||||
if (isSetting)
|
||||
onSave?.()
|
||||
}, [
|
||||
@ -258,19 +181,14 @@ export const useDocumentCreation = (options: UseDocumentCreationOptions) => {
|
||||
isSetting,
|
||||
onSave,
|
||||
])
|
||||
|
||||
// Validate preview params
|
||||
const validatePreviewParams = useCallback((maxChunkLength: number): boolean => {
|
||||
if (maxChunkLength > MAXIMUM_CHUNK_TOKEN_LENGTH) {
|
||||
Toast.notify({
|
||||
type: 'error',
|
||||
message: t('stepTwo.maxLengthCheck', { ns: 'datasetCreation', limit: MAXIMUM_CHUNK_TOKEN_LENGTH }),
|
||||
})
|
||||
toast.error(t('stepTwo.maxLengthCheck', { ns: 'datasetCreation', limit: MAXIMUM_CHUNK_TOKEN_LENGTH }))
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}, [t])
|
||||
|
||||
return {
|
||||
isCreating,
|
||||
validateParams,
|
||||
@ -279,5 +197,4 @@ export const useDocumentCreation = (options: UseDocumentCreationOptions) => {
|
||||
validatePreviewParams,
|
||||
}
|
||||
}
|
||||
|
||||
export type DocumentCreation = ReturnType<typeof useDocumentCreation>
|
||||
|
||||
@ -1,11 +1,10 @@
|
||||
'use client'
|
||||
|
||||
import type { FC } from 'react'
|
||||
import type { StepTwoProps } from './types'
|
||||
import { useCallback, useEffect, useState } from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import Divider from '@/app/components/base/divider'
|
||||
import Toast from '@/app/components/base/toast'
|
||||
import { toast } from '@/app/components/base/ui/toast'
|
||||
import { useDatasetDetailContextWithSelector } from '@/context/dataset-detail'
|
||||
import { useLocale } from '@/context/i18n'
|
||||
import useBreakpoints, { MediaType } from '@/hooks/use-breakpoints'
|
||||
@ -18,34 +17,12 @@ import { GeneralChunkingOptions, IndexingModeSection, ParentChildOptions, Previe
|
||||
import { IndexingType, MAXIMUM_CHUNK_TOKEN_LENGTH, useDocumentCreation, useIndexingConfig, useIndexingEstimate, usePreviewState, useSegmentationState } from './hooks'
|
||||
|
||||
export { IndexingType }
|
||||
|
||||
const StepTwo: FC<StepTwoProps> = ({
|
||||
isSetting,
|
||||
documentDetail,
|
||||
isAPIKeySet,
|
||||
datasetId,
|
||||
indexingType: propsIndexingType,
|
||||
dataSourceType: inCreatePageDataSourceType,
|
||||
files,
|
||||
notionPages = [],
|
||||
notionCredentialId,
|
||||
websitePages = [],
|
||||
crawlOptions,
|
||||
websiteCrawlProvider = DataSourceProvider.jinaReader,
|
||||
websiteCrawlJobId = '',
|
||||
onStepChange,
|
||||
updateIndexingTypeCache,
|
||||
updateResultCache,
|
||||
onSave,
|
||||
onCancel,
|
||||
updateRetrievalMethodCache,
|
||||
}) => {
|
||||
const StepTwo: FC<StepTwoProps> = ({ isSetting, documentDetail, isAPIKeySet, datasetId, indexingType: propsIndexingType, dataSourceType: inCreatePageDataSourceType, files, notionPages = [], notionCredentialId, websitePages = [], crawlOptions, websiteCrawlProvider = DataSourceProvider.jinaReader, websiteCrawlJobId = '', onStepChange, updateIndexingTypeCache, updateResultCache, onSave, onCancel, updateRetrievalMethodCache }) => {
|
||||
const { t } = useTranslation()
|
||||
const locale = useLocale()
|
||||
const isMobile = useBreakpoints() === MediaType.mobile
|
||||
const currentDataset = useDatasetDetailContextWithSelector(s => s.dataset)
|
||||
const mutateDatasetRes = useDatasetDetailContextWithSelector(s => s.mutateDatasetRes)
|
||||
|
||||
// Computed flags
|
||||
const isInUpload = Boolean(currentDataset)
|
||||
const isUploadInEmptyDataset = isInUpload && !currentDataset?.doc_form
|
||||
@ -55,13 +32,11 @@ const StepTwo: FC<StepTwoProps> = ({
|
||||
const dataSourceType = isInCreatePage ? inCreatePageDataSourceType : (currentDataset?.data_source_type ?? inCreatePageDataSourceType)
|
||||
const hasSetIndexType = !!propsIndexingType
|
||||
const isModelAndRetrievalConfigDisabled = !!datasetId && !!currentDataset?.data_source_type
|
||||
|
||||
// Document form state
|
||||
const [docForm, setDocForm] = useState<ChunkingMode>((datasetId && documentDetail) ? documentDetail.doc_form as ChunkingMode : ChunkingMode.text)
|
||||
const [docLanguage, setDocLanguage] = useState<string>(() => (datasetId && documentDetail) ? documentDetail.doc_language : (locale !== LanguagesSupported[1] ? 'English' : 'Chinese Simplified'))
|
||||
const [isQAConfirmDialogOpen, setIsQAConfirmDialogOpen] = useState(false)
|
||||
const currentDocForm = currentDataset?.doc_form || docForm
|
||||
|
||||
// Custom hooks
|
||||
const segmentation = useSegmentationState({
|
||||
initialSegmentationType: currentDataset?.doc_form === ChunkingMode.parentChild ? ProcessMode.parentChild : ProcessMode.general,
|
||||
@ -111,7 +86,6 @@ const StepTwo: FC<StepTwoProps> = ({
|
||||
indexingTechnique: indexing.getIndexingTechnique() as IndexingType,
|
||||
processRule: segmentation.getProcessRule(currentDocForm),
|
||||
})
|
||||
|
||||
// Fetch default process rule
|
||||
const fetchDefaultProcessRuleMutation = useFetchDefaultProcessRule({
|
||||
onSuccess(data) {
|
||||
@ -123,7 +97,6 @@ const StepTwo: FC<StepTwoProps> = ({
|
||||
segmentation.setLimitMaxChunkLength(data.limits.indexing_max_segmentation_tokens_length)
|
||||
},
|
||||
})
|
||||
|
||||
// Event handlers
|
||||
const handleDocFormChange = useCallback((value: ChunkingMode) => {
|
||||
if (value === ChunkingMode.qa && indexing.indexType === IndexingType.ECONOMICAL) {
|
||||
@ -136,15 +109,13 @@ const StepTwo: FC<StepTwoProps> = ({
|
||||
segmentation.setSegmentationType(value === ChunkingMode.parentChild ? ProcessMode.parentChild : ProcessMode.general)
|
||||
estimateHook.reset()
|
||||
}, [indexing, segmentation, estimateHook])
|
||||
|
||||
const updatePreview = useCallback(() => {
|
||||
if (segmentation.segmentationType === ProcessMode.general && segmentation.maxChunkLength > MAXIMUM_CHUNK_TOKEN_LENGTH) {
|
||||
Toast.notify({ type: 'error', message: t('stepTwo.maxLengthCheck', { ns: 'datasetCreation', limit: MAXIMUM_CHUNK_TOKEN_LENGTH }) })
|
||||
toast.error(t('stepTwo.maxLengthCheck', { ns: 'datasetCreation', limit: MAXIMUM_CHUNK_TOKEN_LENGTH }))
|
||||
return
|
||||
}
|
||||
estimateHook.fetchEstimate()
|
||||
}, [segmentation, t, estimateHook])
|
||||
|
||||
const handleCreate = useCallback(async () => {
|
||||
const isValid = creation.validateParams({
|
||||
segmentationType: segmentation.segmentationType,
|
||||
@ -163,19 +134,19 @@ const StepTwo: FC<StepTwoProps> = ({
|
||||
return
|
||||
await creation.executeCreation(params, indexing.indexType, indexing.retrievalConfig)
|
||||
}, [creation, segmentation, indexing, currentDocForm, docLanguage])
|
||||
|
||||
const handlePickerChange = useCallback((selected: { id: string, name: string }) => {
|
||||
const handlePickerChange = useCallback((selected: {
|
||||
id: string
|
||||
name: string
|
||||
}) => {
|
||||
estimateHook.reset()
|
||||
preview.handlePreviewChange(selected)
|
||||
estimateHook.fetchEstimate()
|
||||
}, [estimateHook, preview])
|
||||
|
||||
const handleQAConfirm = useCallback(() => {
|
||||
setIsQAConfirmDialogOpen(false)
|
||||
indexing.setIndexType(IndexingType.QUALIFIED)
|
||||
setDocForm(ChunkingMode.qa)
|
||||
}, [indexing])
|
||||
|
||||
// Initialize rules
|
||||
useEffect(() => {
|
||||
if (!isSetting) {
|
||||
@ -187,83 +158,19 @@ const StepTwo: FC<StepTwoProps> = ({
|
||||
segmentation.applyConfigFromRules(rules, isHierarchical)
|
||||
segmentation.setSegmentationType(documentDetail.dataset_process_rule.mode)
|
||||
}
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [])
|
||||
|
||||
// Show options conditions
|
||||
const showGeneralOption = (isInUpload && [ChunkingMode.text, ChunkingMode.qa].includes(currentDataset!.doc_form)) || isUploadInEmptyDataset || isInInit
|
||||
const showParentChildOption = (isInUpload && currentDataset!.doc_form === ChunkingMode.parentChild) || isUploadInEmptyDataset || isInInit
|
||||
|
||||
return (
|
||||
<div className="flex h-full w-full">
|
||||
<div className={cn('relative h-full w-1/2 overflow-y-auto py-6', isMobile ? 'px-4' : 'px-12')}>
|
||||
<div className="mb-1 text-text-secondary system-md-semibold">{t('stepTwo.segmentation', { ns: 'datasetCreation' })}</div>
|
||||
{showGeneralOption && (
|
||||
<GeneralChunkingOptions
|
||||
segmentIdentifier={segmentation.segmentIdentifier}
|
||||
maxChunkLength={segmentation.maxChunkLength}
|
||||
overlap={segmentation.overlap}
|
||||
rules={segmentation.rules}
|
||||
currentDocForm={currentDocForm}
|
||||
docLanguage={docLanguage}
|
||||
isActive={[ChunkingMode.text, ChunkingMode.qa].includes(currentDocForm)}
|
||||
isInUpload={isInUpload}
|
||||
isNotUploadInEmptyDataset={isNotUploadInEmptyDataset}
|
||||
hasCurrentDatasetDocForm={!!currentDataset?.doc_form}
|
||||
onSegmentIdentifierChange={value => segmentation.setSegmentIdentifier(value, true)}
|
||||
onMaxChunkLengthChange={segmentation.setMaxChunkLength}
|
||||
onOverlapChange={segmentation.setOverlap}
|
||||
onRuleToggle={segmentation.toggleRule}
|
||||
onDocFormChange={handleDocFormChange}
|
||||
onDocLanguageChange={setDocLanguage}
|
||||
onPreview={updatePreview}
|
||||
onReset={segmentation.resetToDefaults}
|
||||
locale={locale}
|
||||
showSummaryIndexSetting={showSummaryIndexSetting}
|
||||
summaryIndexSetting={segmentation.summaryIndexSetting}
|
||||
onSummaryIndexSettingChange={segmentation.handleSummaryIndexSettingChange}
|
||||
/>
|
||||
)}
|
||||
{showParentChildOption && (
|
||||
<ParentChildOptions
|
||||
parentChildConfig={segmentation.parentChildConfig}
|
||||
rules={segmentation.rules}
|
||||
currentDocForm={currentDocForm}
|
||||
isActive={currentDocForm === ChunkingMode.parentChild}
|
||||
isInUpload={isInUpload}
|
||||
isNotUploadInEmptyDataset={isNotUploadInEmptyDataset}
|
||||
onDocFormChange={handleDocFormChange}
|
||||
onChunkForContextChange={segmentation.setChunkForContext}
|
||||
onParentDelimiterChange={v => segmentation.updateParentConfig('delimiter', v)}
|
||||
onParentMaxLengthChange={v => segmentation.updateParentConfig('maxLength', v)}
|
||||
onChildDelimiterChange={v => segmentation.updateChildConfig('delimiter', v)}
|
||||
onChildMaxLengthChange={v => segmentation.updateChildConfig('maxLength', v)}
|
||||
onRuleToggle={segmentation.toggleRule}
|
||||
onPreview={updatePreview}
|
||||
onReset={segmentation.resetToDefaults}
|
||||
showSummaryIndexSetting={showSummaryIndexSetting}
|
||||
summaryIndexSetting={segmentation.summaryIndexSetting}
|
||||
onSummaryIndexSettingChange={segmentation.handleSummaryIndexSettingChange}
|
||||
/>
|
||||
)}
|
||||
{showGeneralOption && (<GeneralChunkingOptions segmentIdentifier={segmentation.segmentIdentifier} maxChunkLength={segmentation.maxChunkLength} overlap={segmentation.overlap} rules={segmentation.rules} currentDocForm={currentDocForm} docLanguage={docLanguage} isActive={[ChunkingMode.text, ChunkingMode.qa].includes(currentDocForm)} isInUpload={isInUpload} isNotUploadInEmptyDataset={isNotUploadInEmptyDataset} hasCurrentDatasetDocForm={!!currentDataset?.doc_form} onSegmentIdentifierChange={value => segmentation.setSegmentIdentifier(value, true)} onMaxChunkLengthChange={segmentation.setMaxChunkLength} onOverlapChange={segmentation.setOverlap} onRuleToggle={segmentation.toggleRule} onDocFormChange={handleDocFormChange} onDocLanguageChange={setDocLanguage} onPreview={updatePreview} onReset={segmentation.resetToDefaults} locale={locale} showSummaryIndexSetting={showSummaryIndexSetting} summaryIndexSetting={segmentation.summaryIndexSetting} onSummaryIndexSettingChange={segmentation.handleSummaryIndexSettingChange} />)}
|
||||
{showParentChildOption && (<ParentChildOptions parentChildConfig={segmentation.parentChildConfig} rules={segmentation.rules} currentDocForm={currentDocForm} isActive={currentDocForm === ChunkingMode.parentChild} isInUpload={isInUpload} isNotUploadInEmptyDataset={isNotUploadInEmptyDataset} onDocFormChange={handleDocFormChange} onChunkForContextChange={segmentation.setChunkForContext} onParentDelimiterChange={v => segmentation.updateParentConfig('delimiter', v)} onParentMaxLengthChange={v => segmentation.updateParentConfig('maxLength', v)} onChildDelimiterChange={v => segmentation.updateChildConfig('delimiter', v)} onChildMaxLengthChange={v => segmentation.updateChildConfig('maxLength', v)} onRuleToggle={segmentation.toggleRule} onPreview={updatePreview} onReset={segmentation.resetToDefaults} showSummaryIndexSetting={showSummaryIndexSetting} summaryIndexSetting={segmentation.summaryIndexSetting} onSummaryIndexSettingChange={segmentation.handleSummaryIndexSettingChange} />)}
|
||||
<Divider className="my-5" />
|
||||
<IndexingModeSection
|
||||
indexType={indexing.indexType}
|
||||
hasSetIndexType={hasSetIndexType}
|
||||
docForm={docForm}
|
||||
embeddingModel={indexing.embeddingModel}
|
||||
embeddingModelList={indexing.embeddingModelList}
|
||||
retrievalConfig={indexing.retrievalConfig}
|
||||
showMultiModalTip={indexing.showMultiModalTip}
|
||||
isModelAndRetrievalConfigDisabled={isModelAndRetrievalConfigDisabled}
|
||||
datasetId={datasetId}
|
||||
isQAConfirmDialogOpen={isQAConfirmDialogOpen}
|
||||
onIndexTypeChange={indexing.setIndexType}
|
||||
onEmbeddingModelChange={indexing.setEmbeddingModel}
|
||||
onRetrievalConfigChange={indexing.setRetrievalConfig}
|
||||
onQAConfirmDialogClose={() => setIsQAConfirmDialogOpen(false)}
|
||||
onQAConfirmDialogConfirm={handleQAConfirm}
|
||||
/>
|
||||
<IndexingModeSection indexType={indexing.indexType} hasSetIndexType={hasSetIndexType} docForm={docForm} embeddingModel={indexing.embeddingModel} embeddingModelList={indexing.embeddingModelList} retrievalConfig={indexing.retrievalConfig} showMultiModalTip={indexing.showMultiModalTip} isModelAndRetrievalConfigDisabled={isModelAndRetrievalConfigDisabled} datasetId={datasetId} isQAConfirmDialogOpen={isQAConfirmDialogOpen} onIndexTypeChange={indexing.setIndexType} onEmbeddingModelChange={indexing.setEmbeddingModel} onRetrievalConfigChange={indexing.setRetrievalConfig} onQAConfirmDialogClose={() => setIsQAConfirmDialogOpen(false)} onQAConfirmDialogConfirm={handleQAConfirm} />
|
||||
<StepTwoFooter isSetting={isSetting} isCreating={creation.isCreating} onPrevious={() => onStepChange?.(-1)} onCreate={handleCreate} onCancel={onCancel} />
|
||||
</div>
|
||||
<PreviewPanel
|
||||
@ -273,7 +180,11 @@ const StepTwo: FC<StepTwoProps> = ({
|
||||
estimate={estimateHook.estimate}
|
||||
parentChildConfig={segmentation.parentChildConfig}
|
||||
isSetting={isSetting}
|
||||
pickerFiles={preview.getPreviewPickerItems() as Array<{ id: string, name: string, extension: string }>}
|
||||
pickerFiles={preview.getPreviewPickerItems() as Array<{
|
||||
id: string
|
||||
name: string
|
||||
extension: string
|
||||
}>}
|
||||
pickerValue={preview.getPreviewPickerValue()}
|
||||
isIdle={estimateHook.isIdle}
|
||||
isPending={estimateHook.isPending}
|
||||
@ -282,5 +193,4 @@ const StepTwo: FC<StepTwoProps> = ({
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
export default StepTwo
|
||||
|
||||
@ -4,7 +4,7 @@ import type { CrawlOptions, CrawlResultItem } from '@/models/datasets'
|
||||
import * as React from 'react'
|
||||
import { useCallback, useEffect, useRef, useState } from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import Toast from '@/app/components/base/toast'
|
||||
import { toast } from '@/app/components/base/ui/toast'
|
||||
import { ACCOUNT_SETTING_TAB } from '@/app/components/header/account-setting/constants'
|
||||
import { useModalContextSelector } from '@/context/modal-context'
|
||||
import { checkFirecrawlTaskStatus, createFirecrawlTask } from '@/service/datasets'
|
||||
@ -19,7 +19,6 @@ import Options from './options'
|
||||
|
||||
const ERROR_I18N_PREFIX = 'errorMsg'
|
||||
const I18N_PREFIX = 'stepOne.website'
|
||||
|
||||
type Props = {
|
||||
onPreview: (payload: CrawlResultItem) => void
|
||||
checkedCrawlResult: CrawlResultItem[]
|
||||
@ -28,20 +27,17 @@ type Props = {
|
||||
crawlOptions: CrawlOptions
|
||||
onCrawlOptionsChange: (payload: CrawlOptions) => void
|
||||
}
|
||||
|
||||
enum Step {
|
||||
init = 'init',
|
||||
running = 'running',
|
||||
finished = 'finished',
|
||||
}
|
||||
|
||||
type CrawlState = {
|
||||
current: number
|
||||
total: number
|
||||
data: CrawlResultItem[]
|
||||
time_consuming: number | string
|
||||
}
|
||||
|
||||
type CrawlFinishedResult = {
|
||||
isCancelled?: boolean
|
||||
isError: boolean
|
||||
@ -50,15 +46,7 @@ type CrawlFinishedResult = {
|
||||
data: CrawlResultItem[]
|
||||
}
|
||||
}
|
||||
|
||||
const FireCrawl: FC<Props> = ({
|
||||
onPreview,
|
||||
checkedCrawlResult,
|
||||
onCheckedCrawlResultChange,
|
||||
onJobIdChange,
|
||||
crawlOptions,
|
||||
onCrawlOptionsChange,
|
||||
}) => {
|
||||
const FireCrawl: FC<Props> = ({ onPreview, checkedCrawlResult, onCheckedCrawlResultChange, onJobIdChange, crawlOptions, onCrawlOptionsChange }) => {
|
||||
const { t } = useTranslation()
|
||||
const [step, setStep] = useState<Step>(Step.init)
|
||||
const [controlFoldOptions, setControlFoldOptions] = useState<number>(0)
|
||||
@ -78,7 +66,6 @@ const FireCrawl: FC<Props> = ({
|
||||
payload: ACCOUNT_SETTING_TAB.DATA_SOURCE,
|
||||
})
|
||||
}, [setShowAccountSettingModal])
|
||||
|
||||
const checkValid = useCallback((url: string) => {
|
||||
let errorMsg = ''
|
||||
if (!url) {
|
||||
@ -87,30 +74,25 @@ const FireCrawl: FC<Props> = ({
|
||||
field: 'url',
|
||||
})
|
||||
}
|
||||
|
||||
if (!errorMsg && !((url.startsWith('http://') || url.startsWith('https://'))))
|
||||
errorMsg = t(`${ERROR_I18N_PREFIX}.urlError`, { ns: 'common' })
|
||||
|
||||
if (!errorMsg && (crawlOptions.limit === null || crawlOptions.limit === undefined || crawlOptions.limit === '')) {
|
||||
errorMsg = t(`${ERROR_I18N_PREFIX}.fieldRequired`, {
|
||||
ns: 'common',
|
||||
field: t(`${I18N_PREFIX}.limit`, { ns: 'datasetCreation' }),
|
||||
})
|
||||
}
|
||||
|
||||
return {
|
||||
isValid: !errorMsg,
|
||||
errorMsg,
|
||||
}
|
||||
}, [crawlOptions, t])
|
||||
|
||||
const isInit = step === Step.init
|
||||
const isCrawlFinished = step === Step.finished
|
||||
const isRunning = step === Step.running
|
||||
const [crawlResult, setCrawlResult] = useState<CrawlState | undefined>(undefined)
|
||||
const [crawlErrorMessage, setCrawlErrorMessage] = useState('')
|
||||
const showError = isCrawlFinished && crawlErrorMessage
|
||||
|
||||
const waitForCrawlFinished = useCallback(async (jobId: string): Promise<CrawlFinishedResult> => {
|
||||
const cancelledResult: CrawlFinishedResult = {
|
||||
isCancelled: true,
|
||||
@ -119,7 +101,6 @@ const FireCrawl: FC<Props> = ({
|
||||
data: [],
|
||||
},
|
||||
}
|
||||
|
||||
try {
|
||||
const res = await checkFirecrawlTaskStatus(jobId) as any
|
||||
if (res.status === 'completed') {
|
||||
@ -171,14 +152,10 @@ const FireCrawl: FC<Props> = ({
|
||||
} satisfies CrawlFinishedResult
|
||||
}
|
||||
}, [crawlOptions.limit, onCheckedCrawlResultChange])
|
||||
|
||||
const handleRun = useCallback(async (url: string) => {
|
||||
const { isValid, errorMsg } = checkValid(url)
|
||||
if (!isValid) {
|
||||
Toast.notify({
|
||||
message: errorMsg!,
|
||||
type: 'error',
|
||||
})
|
||||
toast.error(errorMsg!)
|
||||
return
|
||||
}
|
||||
setStep(Step.running)
|
||||
@ -188,7 +165,6 @@ const FireCrawl: FC<Props> = ({
|
||||
}
|
||||
if (crawlOptions.max_depth === '')
|
||||
delete passToServerCrawlOptions.max_depth
|
||||
|
||||
const res = await createFirecrawlTask({
|
||||
url,
|
||||
options: passToServerCrawlOptions,
|
||||
@ -220,49 +196,22 @@ const FireCrawl: FC<Props> = ({
|
||||
setStep(Step.finished)
|
||||
}
|
||||
}, [checkValid, crawlOptions, onJobIdChange, t, waitForCrawlFinished, onCheckedCrawlResultChange])
|
||||
|
||||
return (
|
||||
<div>
|
||||
<Header
|
||||
onClickConfiguration={handleSetting}
|
||||
title={t(`${I18N_PREFIX}.firecrawlTitle`, { ns: 'datasetCreation' })}
|
||||
buttonText={t(`${I18N_PREFIX}.configureFirecrawl`, { ns: 'datasetCreation' })}
|
||||
docTitle={t(`${I18N_PREFIX}.firecrawlDoc`, { ns: 'datasetCreation' })}
|
||||
docLink="https://docs.firecrawl.dev/introduction"
|
||||
/>
|
||||
<Header onClickConfiguration={handleSetting} title={t(`${I18N_PREFIX}.firecrawlTitle`, { ns: 'datasetCreation' })} buttonText={t(`${I18N_PREFIX}.configureFirecrawl`, { ns: 'datasetCreation' })} docTitle={t(`${I18N_PREFIX}.firecrawlDoc`, { ns: 'datasetCreation' })} docLink="https://docs.firecrawl.dev/introduction" />
|
||||
<div className="mt-2 rounded-xl border border-components-panel-border bg-background-default-subtle p-4 pb-0">
|
||||
<UrlInput onRun={handleRun} isRunning={isRunning} />
|
||||
<OptionsWrap
|
||||
className="mt-4"
|
||||
controlFoldOptions={controlFoldOptions}
|
||||
>
|
||||
<OptionsWrap className="mt-4" controlFoldOptions={controlFoldOptions}>
|
||||
<Options className="mt-2" payload={crawlOptions} onChange={onCrawlOptionsChange} />
|
||||
</OptionsWrap>
|
||||
|
||||
{!isInit && (
|
||||
<div className="relative left-[-16px] mt-3 w-[calc(100%_+_32px)] rounded-b-xl">
|
||||
{isRunning
|
||||
&& (
|
||||
<Crawling
|
||||
className="mt-2"
|
||||
crawledNum={crawlResult?.current || 0}
|
||||
totalNum={crawlResult?.total || Number.parseFloat(crawlOptions.limit as string) || 0}
|
||||
/>
|
||||
)}
|
||||
{showError && (
|
||||
<ErrorMessage className="rounded-b-xl" title={t(`${I18N_PREFIX}.exceptionErrorTitle`, { ns: 'datasetCreation' })} errorMsg={crawlErrorMessage} />
|
||||
)}
|
||||
&& (<Crawling className="mt-2" crawledNum={crawlResult?.current || 0} totalNum={crawlResult?.total || Number.parseFloat(crawlOptions.limit as string) || 0} />)}
|
||||
{showError && (<ErrorMessage className="rounded-b-xl" title={t(`${I18N_PREFIX}.exceptionErrorTitle`, { ns: 'datasetCreation' })} errorMsg={crawlErrorMessage} />)}
|
||||
{isCrawlFinished && !showError
|
||||
&& (
|
||||
<CrawledResult
|
||||
className="mb-2"
|
||||
list={crawlResult?.data || []}
|
||||
checkedList={checkedCrawlResult}
|
||||
onSelectedChange={onCheckedCrawlResultChange}
|
||||
onPreview={onPreview}
|
||||
usedTime={Number.parseFloat(crawlResult?.time_consuming as string) || 0}
|
||||
/>
|
||||
)}
|
||||
&& (<CrawledResult className="mb-2" list={crawlResult?.data || []} checkedList={checkedCrawlResult} onSelectedChange={onCheckedCrawlResultChange} onPreview={onPreview} usedTime={Number.parseFloat(crawlResult?.time_consuming as string) || 0} />)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
@ -4,7 +4,7 @@ import type { CrawlOptions, CrawlResultItem } from '@/models/datasets'
|
||||
import * as React from 'react'
|
||||
import { useCallback, useEffect, useState } from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import Toast from '@/app/components/base/toast'
|
||||
import { toast } from '@/app/components/base/ui/toast'
|
||||
import { ACCOUNT_SETTING_TAB } from '@/app/components/header/account-setting/constants'
|
||||
import { useModalContext } from '@/context/modal-context'
|
||||
import { checkJinaReaderTaskStatus, createJinaReaderTask } from '@/service/datasets'
|
||||
@ -19,7 +19,6 @@ import Options from './options'
|
||||
|
||||
const ERROR_I18N_PREFIX = 'errorMsg'
|
||||
const I18N_PREFIX = 'stepOne.website'
|
||||
|
||||
type Props = {
|
||||
onPreview: (payload: CrawlResultItem) => void
|
||||
checkedCrawlResult: CrawlResultItem[]
|
||||
@ -28,21 +27,12 @@ type Props = {
|
||||
crawlOptions: CrawlOptions
|
||||
onCrawlOptionsChange: (payload: CrawlOptions) => void
|
||||
}
|
||||
|
||||
enum Step {
|
||||
init = 'init',
|
||||
running = 'running',
|
||||
finished = 'finished',
|
||||
}
|
||||
|
||||
const JinaReader: FC<Props> = ({
|
||||
onPreview,
|
||||
checkedCrawlResult,
|
||||
onCheckedCrawlResultChange,
|
||||
onJobIdChange,
|
||||
crawlOptions,
|
||||
onCrawlOptionsChange,
|
||||
}) => {
|
||||
const JinaReader: FC<Props> = ({ onPreview, checkedCrawlResult, onCheckedCrawlResultChange, onJobIdChange, crawlOptions, onCrawlOptionsChange }) => {
|
||||
const { t } = useTranslation()
|
||||
const [step, setStep] = useState<Step>(Step.init)
|
||||
const [controlFoldOptions, setControlFoldOptions] = useState<number>(0)
|
||||
@ -56,7 +46,6 @@ const JinaReader: FC<Props> = ({
|
||||
payload: ACCOUNT_SETTING_TAB.DATA_SOURCE,
|
||||
})
|
||||
}, [setShowAccountSettingModal])
|
||||
|
||||
const checkValid = useCallback((url: string) => {
|
||||
let errorMsg = ''
|
||||
if (!url) {
|
||||
@ -65,23 +54,19 @@ const JinaReader: FC<Props> = ({
|
||||
field: 'url',
|
||||
})
|
||||
}
|
||||
|
||||
if (!errorMsg && !((url.startsWith('http://') || url.startsWith('https://'))))
|
||||
errorMsg = t(`${ERROR_I18N_PREFIX}.urlError`, { ns: 'common' })
|
||||
|
||||
if (!errorMsg && (crawlOptions.limit === null || crawlOptions.limit === undefined || crawlOptions.limit === '')) {
|
||||
errorMsg = t(`${ERROR_I18N_PREFIX}.fieldRequired`, {
|
||||
ns: 'common',
|
||||
field: t(`${I18N_PREFIX}.limit`, { ns: 'datasetCreation' }),
|
||||
})
|
||||
}
|
||||
|
||||
return {
|
||||
isValid: !errorMsg,
|
||||
errorMsg,
|
||||
}
|
||||
}, [crawlOptions, t])
|
||||
|
||||
const isInit = step === Step.init
|
||||
const isCrawlFinished = step === Step.finished
|
||||
const isRunning = step === Step.running
|
||||
@ -93,7 +78,6 @@ const JinaReader: FC<Props> = ({
|
||||
} | undefined>(undefined)
|
||||
const [crawlErrorMessage, setCrawlErrorMessage] = useState('')
|
||||
const showError = isCrawlFinished && crawlErrorMessage
|
||||
|
||||
const waitForCrawlFinished = useCallback(async (jobId: string) => {
|
||||
try {
|
||||
const res = await checkJinaReaderTaskStatus(jobId) as any
|
||||
@ -135,14 +119,10 @@ const JinaReader: FC<Props> = ({
|
||||
}
|
||||
}
|
||||
}, [crawlOptions.limit, onCheckedCrawlResultChange])
|
||||
|
||||
const handleRun = useCallback(async (url: string) => {
|
||||
const { isValid, errorMsg } = checkValid(url)
|
||||
if (!isValid) {
|
||||
Toast.notify({
|
||||
message: errorMsg!,
|
||||
type: 'error',
|
||||
})
|
||||
toast.error(errorMsg!)
|
||||
return
|
||||
}
|
||||
setStep(Step.running)
|
||||
@ -152,7 +132,6 @@ const JinaReader: FC<Props> = ({
|
||||
url,
|
||||
options: crawlOptions,
|
||||
}) as any
|
||||
|
||||
if (res.data) {
|
||||
const { title, content, description, url } = res.data
|
||||
const data = {
|
||||
@ -192,49 +171,22 @@ const JinaReader: FC<Props> = ({
|
||||
setStep(Step.finished)
|
||||
}
|
||||
}, [checkValid, crawlOptions, onCheckedCrawlResultChange, onJobIdChange, t, waitForCrawlFinished])
|
||||
|
||||
return (
|
||||
<div>
|
||||
<Header
|
||||
onClickConfiguration={handleSetting}
|
||||
title={t(`${I18N_PREFIX}.jinaReaderTitle`, { ns: 'datasetCreation' })}
|
||||
buttonText={t(`${I18N_PREFIX}.configureJinaReader`, { ns: 'datasetCreation' })}
|
||||
docTitle={t(`${I18N_PREFIX}.jinaReaderDoc`, { ns: 'datasetCreation' })}
|
||||
docLink="https://jina.ai/reader"
|
||||
/>
|
||||
<Header onClickConfiguration={handleSetting} title={t(`${I18N_PREFIX}.jinaReaderTitle`, { ns: 'datasetCreation' })} buttonText={t(`${I18N_PREFIX}.configureJinaReader`, { ns: 'datasetCreation' })} docTitle={t(`${I18N_PREFIX}.jinaReaderDoc`, { ns: 'datasetCreation' })} docLink="https://jina.ai/reader" />
|
||||
<div className="mt-2 rounded-xl border border-components-panel-border bg-background-default-subtle p-4 pb-0">
|
||||
<UrlInput onRun={handleRun} isRunning={isRunning} />
|
||||
<OptionsWrap
|
||||
className="mt-4"
|
||||
controlFoldOptions={controlFoldOptions}
|
||||
>
|
||||
<OptionsWrap className="mt-4" controlFoldOptions={controlFoldOptions}>
|
||||
<Options className="mt-2" payload={crawlOptions} onChange={onCrawlOptionsChange} />
|
||||
</OptionsWrap>
|
||||
|
||||
{!isInit && (
|
||||
<div className="relative left-[-16px] mt-3 w-[calc(100%_+_32px)] rounded-b-xl">
|
||||
{isRunning
|
||||
&& (
|
||||
<Crawling
|
||||
className="mt-2"
|
||||
crawledNum={crawlResult?.current || 0}
|
||||
totalNum={crawlResult?.total || Number.parseFloat(crawlOptions.limit as string) || 0}
|
||||
/>
|
||||
)}
|
||||
{showError && (
|
||||
<ErrorMessage className="rounded-b-xl" title={t(`${I18N_PREFIX}.exceptionErrorTitle`, { ns: 'datasetCreation' })} errorMsg={crawlErrorMessage} />
|
||||
)}
|
||||
&& (<Crawling className="mt-2" crawledNum={crawlResult?.current || 0} totalNum={crawlResult?.total || Number.parseFloat(crawlOptions.limit as string) || 0} />)}
|
||||
{showError && (<ErrorMessage className="rounded-b-xl" title={t(`${I18N_PREFIX}.exceptionErrorTitle`, { ns: 'datasetCreation' })} errorMsg={crawlErrorMessage} />)}
|
||||
{isCrawlFinished && !showError
|
||||
&& (
|
||||
<CrawledResult
|
||||
className="mb-2"
|
||||
list={crawlResult?.data || []}
|
||||
checkedList={checkedCrawlResult}
|
||||
onSelectedChange={onCheckedCrawlResultChange}
|
||||
onPreview={onPreview}
|
||||
usedTime={Number.parseFloat(crawlResult?.time_consuming as string) || 0}
|
||||
/>
|
||||
)}
|
||||
&& (<CrawledResult className="mb-2" list={crawlResult?.data || []} checkedList={checkedCrawlResult} onSelectedChange={onCheckedCrawlResultChange} onPreview={onPreview} usedTime={Number.parseFloat(crawlResult?.time_consuming as string) || 0} />)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
@ -4,7 +4,7 @@ import type { CrawlOptions, CrawlResultItem } from '@/models/datasets'
|
||||
import * as React from 'react'
|
||||
import { useCallback, useEffect, useState } from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import Toast from '@/app/components/base/toast'
|
||||
import { toast } from '@/app/components/base/ui/toast'
|
||||
import { ACCOUNT_SETTING_TAB } from '@/app/components/header/account-setting/constants'
|
||||
import { useModalContext } from '@/context/modal-context'
|
||||
import { checkWatercrawlTaskStatus, createWatercrawlTask } from '@/service/datasets'
|
||||
@ -19,7 +19,6 @@ import Options from './options'
|
||||
|
||||
const ERROR_I18N_PREFIX = 'errorMsg'
|
||||
const I18N_PREFIX = 'stepOne.website'
|
||||
|
||||
type Props = {
|
||||
onPreview: (payload: CrawlResultItem) => void
|
||||
checkedCrawlResult: CrawlResultItem[]
|
||||
@ -28,21 +27,12 @@ type Props = {
|
||||
crawlOptions: CrawlOptions
|
||||
onCrawlOptionsChange: (payload: CrawlOptions) => void
|
||||
}
|
||||
|
||||
enum Step {
|
||||
init = 'init',
|
||||
running = 'running',
|
||||
finished = 'finished',
|
||||
}
|
||||
|
||||
const WaterCrawl: FC<Props> = ({
|
||||
onPreview,
|
||||
checkedCrawlResult,
|
||||
onCheckedCrawlResultChange,
|
||||
onJobIdChange,
|
||||
crawlOptions,
|
||||
onCrawlOptionsChange,
|
||||
}) => {
|
||||
const WaterCrawl: FC<Props> = ({ onPreview, checkedCrawlResult, onCheckedCrawlResultChange, onJobIdChange, crawlOptions, onCrawlOptionsChange }) => {
|
||||
const { t } = useTranslation()
|
||||
const [step, setStep] = useState<Step>(Step.init)
|
||||
const [controlFoldOptions, setControlFoldOptions] = useState<number>(0)
|
||||
@ -56,7 +46,6 @@ const WaterCrawl: FC<Props> = ({
|
||||
payload: ACCOUNT_SETTING_TAB.DATA_SOURCE,
|
||||
})
|
||||
}, [setShowAccountSettingModal])
|
||||
|
||||
const checkValid = useCallback((url: string) => {
|
||||
let errorMsg = ''
|
||||
if (!url) {
|
||||
@ -65,23 +54,19 @@ const WaterCrawl: FC<Props> = ({
|
||||
field: 'url',
|
||||
})
|
||||
}
|
||||
|
||||
if (!errorMsg && !((url.startsWith('http://') || url.startsWith('https://'))))
|
||||
errorMsg = t(`${ERROR_I18N_PREFIX}.urlError`, { ns: 'common' })
|
||||
|
||||
if (!errorMsg && (crawlOptions.limit === null || crawlOptions.limit === undefined || crawlOptions.limit === '')) {
|
||||
errorMsg = t(`${ERROR_I18N_PREFIX}.fieldRequired`, {
|
||||
ns: 'common',
|
||||
field: t(`${I18N_PREFIX}.limit`, { ns: 'datasetCreation' }),
|
||||
})
|
||||
}
|
||||
|
||||
return {
|
||||
isValid: !errorMsg,
|
||||
errorMsg,
|
||||
}
|
||||
}, [crawlOptions, t])
|
||||
|
||||
const isInit = step === Step.init
|
||||
const isCrawlFinished = step === Step.finished
|
||||
const isRunning = step === Step.running
|
||||
@ -93,7 +78,6 @@ const WaterCrawl: FC<Props> = ({
|
||||
} | undefined>(undefined)
|
||||
const [crawlErrorMessage, setCrawlErrorMessage] = useState('')
|
||||
const showError = isCrawlFinished && crawlErrorMessage
|
||||
|
||||
const waitForCrawlFinished = useCallback(async (jobId: string): Promise<any> => {
|
||||
try {
|
||||
const res = await checkWatercrawlTaskStatus(jobId) as any
|
||||
@ -127,20 +111,22 @@ const WaterCrawl: FC<Props> = ({
|
||||
}
|
||||
catch (error: unknown) {
|
||||
let errorMessage = ''
|
||||
|
||||
const maybeErrorWithJson = error as { json?: () => Promise<unknown>, message?: unknown } | null
|
||||
const maybeErrorWithJson = error as {
|
||||
json?: () => Promise<unknown>
|
||||
message?: unknown
|
||||
} | null
|
||||
if (maybeErrorWithJson?.json) {
|
||||
try {
|
||||
const errorBody = await maybeErrorWithJson.json() as { message?: unknown } | null
|
||||
const errorBody = await maybeErrorWithJson.json() as {
|
||||
message?: unknown
|
||||
} | null
|
||||
if (typeof errorBody?.message === 'string')
|
||||
errorMessage = errorBody.message
|
||||
}
|
||||
catch {}
|
||||
catch { }
|
||||
}
|
||||
|
||||
if (!errorMessage && typeof maybeErrorWithJson?.message === 'string')
|
||||
errorMessage = maybeErrorWithJson.message
|
||||
|
||||
return {
|
||||
isError: true,
|
||||
errorMessage,
|
||||
@ -150,14 +136,10 @@ const WaterCrawl: FC<Props> = ({
|
||||
}
|
||||
}
|
||||
}, [crawlOptions.limit, onCheckedCrawlResultChange])
|
||||
|
||||
const handleRun = useCallback(async (url: string) => {
|
||||
const { isValid, errorMsg } = checkValid(url)
|
||||
if (!isValid) {
|
||||
Toast.notify({
|
||||
message: errorMsg!,
|
||||
type: 'error',
|
||||
})
|
||||
toast.error(errorMsg!)
|
||||
return
|
||||
}
|
||||
setStep(Step.running)
|
||||
@ -167,7 +149,6 @@ const WaterCrawl: FC<Props> = ({
|
||||
}
|
||||
if (crawlOptions.max_depth === '')
|
||||
delete passToServerCrawlOptions.max_depth
|
||||
|
||||
const res = await createWatercrawlTask({
|
||||
url,
|
||||
options: passToServerCrawlOptions,
|
||||
@ -192,49 +173,22 @@ const WaterCrawl: FC<Props> = ({
|
||||
setStep(Step.finished)
|
||||
}
|
||||
}, [checkValid, crawlOptions, onCheckedCrawlResultChange, onJobIdChange, t, waitForCrawlFinished])
|
||||
|
||||
return (
|
||||
<div>
|
||||
<Header
|
||||
onClickConfiguration={handleSetting}
|
||||
title={t(`${I18N_PREFIX}.watercrawlTitle`, { ns: 'datasetCreation' })}
|
||||
buttonText={t(`${I18N_PREFIX}.configureWatercrawl`, { ns: 'datasetCreation' })}
|
||||
docTitle={t(`${I18N_PREFIX}.watercrawlDoc`, { ns: 'datasetCreation' })}
|
||||
docLink="https://docs.watercrawl.dev/"
|
||||
/>
|
||||
<Header onClickConfiguration={handleSetting} title={t(`${I18N_PREFIX}.watercrawlTitle`, { ns: 'datasetCreation' })} buttonText={t(`${I18N_PREFIX}.configureWatercrawl`, { ns: 'datasetCreation' })} docTitle={t(`${I18N_PREFIX}.watercrawlDoc`, { ns: 'datasetCreation' })} docLink="https://docs.watercrawl.dev/" />
|
||||
<div className="mt-2 rounded-xl border border-components-panel-border bg-background-default-subtle p-4 pb-0">
|
||||
<UrlInput onRun={handleRun} isRunning={isRunning} />
|
||||
<OptionsWrap
|
||||
className="mt-4"
|
||||
controlFoldOptions={controlFoldOptions}
|
||||
>
|
||||
<OptionsWrap className="mt-4" controlFoldOptions={controlFoldOptions}>
|
||||
<Options className="mt-2" payload={crawlOptions} onChange={onCrawlOptionsChange} />
|
||||
</OptionsWrap>
|
||||
|
||||
{!isInit && (
|
||||
<div className="relative left-[-16px] mt-3 w-[calc(100%_+_32px)] rounded-b-xl">
|
||||
{isRunning
|
||||
&& (
|
||||
<Crawling
|
||||
className="mt-2"
|
||||
crawledNum={crawlResult?.current || 0}
|
||||
totalNum={crawlResult?.total || Number.parseFloat(crawlOptions.limit as string) || 0}
|
||||
/>
|
||||
)}
|
||||
{showError && (
|
||||
<ErrorMessage className="rounded-b-xl" title={t(`${I18N_PREFIX}.exceptionErrorTitle`, { ns: 'datasetCreation' })} errorMsg={crawlErrorMessage} />
|
||||
)}
|
||||
&& (<Crawling className="mt-2" crawledNum={crawlResult?.current || 0} totalNum={crawlResult?.total || Number.parseFloat(crawlOptions.limit as string) || 0} />)}
|
||||
{showError && (<ErrorMessage className="rounded-b-xl" title={t(`${I18N_PREFIX}.exceptionErrorTitle`, { ns: 'datasetCreation' })} errorMsg={crawlErrorMessage} />)}
|
||||
{isCrawlFinished && !showError
|
||||
&& (
|
||||
<CrawledResult
|
||||
className="mb-2"
|
||||
list={crawlResult?.data || []}
|
||||
checkedList={checkedCrawlResult}
|
||||
onSelectedChange={onCheckedCrawlResultChange}
|
||||
onPreview={onPreview}
|
||||
usedTime={Number.parseFloat(crawlResult?.time_consuming as string) || 0}
|
||||
/>
|
||||
)}
|
||||
&& (<CrawledResult className="mb-2" list={crawlResult?.data || []} checkedList={checkedCrawlResult} onSelectedChange={onCheckedCrawlResultChange} onPreview={onPreview} usedTime={Number.parseFloat(crawlResult?.time_consuming as string) || 0} />)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
Reference in New Issue
Block a user