Merge main HEAD (segment 5) into sandboxed-agent-rebase

Resolve 83 conflicts: 10 backend, 62 frontend, 11 config/lock files.
Preserve sandbox/agent/collaboration features while adopting main's
UI refactorings (Dialog/AlertDialog/Popover), model provider updates,
and enterprise features.

Made-with: Cursor
This commit is contained in:
Novice
2026-03-23 14:20:06 +08:00
1671 changed files with 124822 additions and 22302 deletions

View File

@ -24,7 +24,7 @@ const IndexingTypeValues = {
}
// Mock next/link
vi.mock('next/link', () => {
vi.mock('@/next/link', () => {
return function MockLink({ children, href }: { children: React.ReactNode, href: string }) {
return <a href={href}>{children}</a>
}

View File

@ -16,18 +16,10 @@ import {
const mockPush = vi.fn()
const mockRouter = { push: mockPush }
vi.mock('next/navigation', () => ({
vi.mock('@/next/navigation', () => ({
useRouter: () => mockRouter,
}))
// Override global next/image auto-mock: test asserts on data-testid="next-image"
vi.mock('next/image', () => ({
default: ({ src, alt, className }: { src: string, alt: string, className?: string }) => (
// eslint-disable-next-line next/no-img-element
<img src={src} alt={alt} className={className} data-testid="next-image" />
),
}))
// Mock API service
const mockFetchIndexingStatusBatch = vi.fn()
vi.mock('@/service/datasets', () => ({
@ -979,9 +971,9 @@ describe('RuleDetail', () => {
})
it('should render correct icon for indexing type', () => {
render(<RuleDetail indexingType="high_quality" />)
const { container } = render(<RuleDetail indexingType="high_quality" />)
const images = screen.getAllByTestId('next-image')
const images = container.querySelectorAll('img')
expect(images.length).toBeGreaterThan(0)
})
})

View File

@ -6,8 +6,6 @@ import {
RiLoader2Fill,
RiTerminalBoxLine,
} from '@remixicon/react'
import Link from 'next/link'
import { useRouter } from 'next/navigation'
import { useMemo } from 'react'
import { useTranslation } from 'react-i18next'
import Button from '@/app/components/base/button'
@ -15,6 +13,8 @@ import Divider from '@/app/components/base/divider'
import { Plan } from '@/app/components/billing/type'
import { useProviderContext } from '@/context/provider-context'
import { useDatasetApiAccessUrl } from '@/hooks/use-api-access-url'
import Link from '@/next/link'
import { useRouter } from '@/next/navigation'
import { useProcessRule } from '@/service/knowledge/use-dataset'
import { useInvalidDocumentList } from '@/service/knowledge/use-document'
import IndexingProgressItem from './indexing-progress-item'

View File

@ -1,6 +1,5 @@
import type { FC } from 'react'
import type { ProcessRuleResponse } from '@/models/datasets'
import Image from 'next/image'
import { useCallback } from 'react'
import { useTranslation } from 'react-i18next'
import { FieldInfo } from '@/app/components/datasets/documents/detail/metadata'
@ -119,12 +118,12 @@ const RuleDetail: FC<RuleDetailProps> = ({ sourceData, indexingType, retrievalMe
<FieldInfo
label={t('stepTwo.indexMode', { ns: 'datasetCreation' })}
displayedValue={indexModeLabel}
valueIcon={<Image className="size-4" src={indexMethodIconSrc} alt="" />}
valueIcon={<img className="size-4" src={indexMethodIconSrc} alt="" />}
/>
<FieldInfo
label={t('form.retrievalSetting.title', { ns: 'datasetSettings' })}
displayedValue={retrievalLabel}
valueIcon={<Image className="size-4" src={retrievalIconSrc} alt="" />}
valueIcon={<img className="size-4" src={retrievalIconSrc} alt="" />}
/>
</div>
)

View File

@ -7,7 +7,7 @@ import EmptyDatasetCreationModal from '../index'
// Mock Next.js router
const mockPush = vi.fn()
vi.mock('next/navigation', () => ({
vi.mock('@/next/navigation', () => ({
useRouter: () => ({
push: mockPush,
}),

View File

@ -1,5 +1,4 @@
'use client'
import { useRouter } from 'next/navigation'
import * as React from 'react'
import { useState } from 'react'
import { useTranslation } from 'react-i18next'
@ -9,6 +8,7 @@ import Button from '@/app/components/base/button'
import Input from '@/app/components/base/input'
import Modal from '@/app/components/base/modal'
import { ToastContext } from '@/app/components/base/toast/context'
import { useRouter } from '@/next/navigation'
import { createEmptyDataset } from '@/service/datasets'
import { useInvalidDatasetList } from '@/service/knowledge/use-dataset'

View File

@ -58,7 +58,7 @@ vi.mock('@/app/components/datasets/common/document-file-icon', () => ({
}))
// Mock SimplePieChart
vi.mock('next/dynamic', () => ({
vi.mock('@/next/dynamic', () => ({
default: () => {
const Component = ({ percentage }: { percentage: number }) => (
<div data-testid="pie-chart">

View File

@ -17,7 +17,7 @@ vi.mock('@/types/app', () => ({
}))
// Mock SimplePieChart with dynamic import handling
vi.mock('next/dynamic', () => ({
vi.mock('@/next/dynamic', () => ({
default: () => {
const DynamicComponent = ({ percentage, stroke, fill }: { percentage: number, stroke: string, fill: string }) => (
<div data-testid="pie-chart" data-percentage={percentage} data-stroke={stroke} data-fill={fill}>

View File

@ -1,10 +1,10 @@
'use client'
import type { CustomFile as File, FileItem } from '@/models/datasets'
import { RiDeleteBinLine, RiErrorWarningFill } from '@remixicon/react'
import dynamic from 'next/dynamic'
import { useMemo } from 'react'
import DocumentFileIcon from '@/app/components/datasets/common/document-file-icon'
import useTheme from '@/hooks/use-theme'
import dynamic from '@/next/dynamic'
import { Theme } from '@/types/app'
import { formatFileSize, getFileExtension } from '@/utils/format'
import { PROGRESS_COMPLETE, PROGRESS_ERROR } from '../constants'

View File

@ -5,12 +5,12 @@ import Research from './assets/research-mod.svg'
import Selection from './assets/selection-mod.svg'
export const indexMethodIcon = {
high_quality: GoldIcon,
economical: Piggybank,
high_quality: GoldIcon.src,
economical: Piggybank.src,
}
export const retrievalIcon = {
vector: Selection,
fullText: Research,
hybrid: PatternRecognition,
vector: Selection.src,
fullText: Research.src,
hybrid: PatternRecognition.src,
}

View File

@ -6,7 +6,7 @@ import { ChunkingMode } from '@/models/datasets'
import { IndexingType } from '../../hooks'
import { IndexingModeSection } from '../indexing-mode-section'
vi.mock('next/link', () => ({
vi.mock('@/next/link', () => ({
default: ({ children, href, ...props }: { children?: React.ReactNode, href?: string, className?: string }) => <a href={href} {...props}>{children}</a>,
}))

View File

@ -1,4 +1,4 @@
import { render, screen } from '@testing-library/react'
import { fireEvent, render, screen } from '@testing-library/react'
import { beforeEach, describe, expect, it, vi } from 'vitest'
import { DelimiterInput, MaxLengthInput, OverlapInput } from '../inputs'
@ -47,19 +47,34 @@ describe('MaxLengthInput', () => {
it('should render number input', () => {
render(<MaxLengthInput onChange={vi.fn()} />)
const input = screen.getByRole('spinbutton')
const input = screen.getByRole('textbox')
expect(input).toBeInTheDocument()
})
it('should accept value prop', () => {
render(<MaxLengthInput value={500} onChange={vi.fn()} />)
expect(screen.getByDisplayValue('500')).toBeInTheDocument()
expect(screen.getByRole('textbox')).toHaveValue('500')
})
it('should have min of 1', () => {
render(<MaxLengthInput onChange={vi.fn()} />)
const input = screen.getByRole('spinbutton')
expect(input).toHaveAttribute('min', '1')
const input = screen.getByRole('textbox')
expect(input).toBeInTheDocument()
})
it('should reset to the minimum when users clear the value', () => {
const onChange = vi.fn()
render(<MaxLengthInput value={500} onChange={onChange} />)
fireEvent.change(screen.getByRole('textbox'), { target: { value: '' } })
expect(onChange).toHaveBeenCalledWith(1)
})
it('should clamp out-of-range text edits before updating state', () => {
const onChange = vi.fn()
render(<MaxLengthInput value={500} max={1000} onChange={onChange} />)
fireEvent.change(screen.getByRole('textbox'), { target: { value: '1200' } })
expect(onChange).toHaveBeenLastCalledWith(1000)
})
})
@ -75,18 +90,33 @@ describe('OverlapInput', () => {
it('should render number input', () => {
render(<OverlapInput onChange={vi.fn()} />)
const input = screen.getByRole('spinbutton')
const input = screen.getByRole('textbox')
expect(input).toBeInTheDocument()
})
it('should accept value prop', () => {
render(<OverlapInput value={50} onChange={vi.fn()} />)
expect(screen.getByDisplayValue('50')).toBeInTheDocument()
expect(screen.getByRole('textbox')).toHaveValue('50')
})
it('should have min of 1', () => {
render(<OverlapInput onChange={vi.fn()} />)
const input = screen.getByRole('spinbutton')
expect(input).toHaveAttribute('min', '1')
const input = screen.getByRole('textbox')
expect(input).toBeInTheDocument()
})
it('should reset to the minimum when users clear the value', () => {
const onChange = vi.fn()
render(<OverlapInput value={50} onChange={onChange} />)
fireEvent.change(screen.getByRole('textbox'), { target: { value: '' } })
expect(onChange).toHaveBeenCalledWith(1)
})
it('should clamp out-of-range text edits before updating state', () => {
const onChange = vi.fn()
render(<OverlapInput value={50} max={100} onChange={onChange} />)
fireEvent.change(screen.getByRole('textbox'), { target: { value: '150' } })
expect(onChange).toHaveBeenLastCalledWith(100)
})
})

View File

@ -2,13 +2,6 @@ import { fireEvent, render, screen } from '@testing-library/react'
import { beforeEach, describe, expect, it, vi } from 'vitest'
import { OptionCard, OptionCardHeader } from '../option-card'
// Override global next/image auto-mock: tests assert on rendered <img> elements
vi.mock('next/image', () => ({
default: ({ src, alt, ...props }: { src?: string, alt?: string, width?: number, height?: number }) => (
<img src={src} alt={alt} {...props} />
),
}))
describe('OptionCardHeader', () => {
const defaultProps = {
icon: <span data-testid="icon">icon</span>,

View File

@ -6,7 +6,6 @@ import {
RiAlertFill,
RiSearchEyeLine,
} from '@remixicon/react'
import Image from 'next/image'
import { useTranslation } from 'react-i18next'
import Button from '@/app/components/base/button'
import Checkbox from '@/app/components/base/checkbox'
@ -97,7 +96,7 @@ export const GeneralChunkingOptions: FC<GeneralChunkingOptionsProps> = ({
<OptionCard
className="mb-2 bg-background-section"
title={t('stepTwo.general', { ns: 'datasetCreation' })}
icon={<Image width={20} height={20} src={SettingCog} alt={t('stepTwo.general', { ns: 'datasetCreation' })} />}
icon={<img width={20} height={20} src={SettingCog.src} alt={t('stepTwo.general', { ns: 'datasetCreation' })} />}
activeHeaderClassName="bg-dataset-option-card-blue-gradient"
description={t('stepTwo.generalTip', { ns: 'datasetCreation' })}
isActive={isActive}

View File

@ -3,8 +3,6 @@
import type { FC } from 'react'
import type { DefaultModel, Model } from '@/app/components/header/account-setting/model-provider-page/declarations'
import type { RetrievalConfig } from '@/types/app'
import Image from 'next/image'
import Link from 'next/link'
import { useTranslation } from 'react-i18next'
import Badge from '@/app/components/base/badge'
import Button from '@/app/components/base/button'
@ -17,6 +15,7 @@ import RetrievalMethodConfig from '@/app/components/datasets/common/retrieval-me
import ModelSelector from '@/app/components/header/account-setting/model-provider-page/model-selector'
import { useDocLink } from '@/context/i18n'
import { ChunkingMode } from '@/models/datasets'
import Link from '@/next/link'
import { cn } from '@/utils/classnames'
import { indexMethodIcon } from '../../icons'
import { IndexingType } from '../hooks'
@ -98,7 +97,7 @@ export const IndexingModeSection: FC<IndexingModeSectionProps> = ({
</div>
)}
description={t('stepTwo.qualifiedTip', { ns: 'datasetCreation' })}
icon={<Image src={indexMethodIcon.high_quality} alt="" />}
icon={<img src={indexMethodIcon.high_quality} alt="" />}
isActive={!hasSetIndexType && indexType === IndexingType.QUALIFIED}
disabled={hasSetIndexType}
onSwitched={() => onIndexTypeChange(IndexingType.QUALIFIED)}
@ -143,7 +142,7 @@ export const IndexingModeSection: FC<IndexingModeSectionProps> = ({
className="h-full"
title={t('stepTwo.economical', { ns: 'datasetCreation' })}
description={t('stepTwo.economicalTip', { ns: 'datasetCreation' })}
icon={<Image src={indexMethodIcon.economical} alt="" />}
icon={<img src={indexMethodIcon.economical} alt="" />}
isActive={!hasSetIndexType && indexType === IndexingType.ECONOMICAL}
disabled={hasSetIndexType || docForm !== ChunkingMode.text}
onSwitched={() => onIndexTypeChange(IndexingType.ECONOMICAL)}

View File

@ -1,10 +1,18 @@
import type { FC, PropsWithChildren, ReactNode } from 'react'
import type { InputProps } from '@/app/components/base/input'
import type { InputNumberProps } from '@/app/components/base/input-number'
import type { NumberFieldInputProps, NumberFieldRootProps, NumberFieldSize } from '@/app/components/base/ui/number-field'
import { useTranslation } from 'react-i18next'
import Input from '@/app/components/base/input'
import { InputNumber } from '@/app/components/base/input-number'
import Tooltip from '@/app/components/base/tooltip'
import {
NumberField,
NumberFieldControls,
NumberFieldDecrement,
NumberFieldGroup,
NumberFieldIncrement,
NumberFieldInput,
NumberFieldUnit,
} from '@/app/components/base/ui/number-field'
import { env } from '@/env'
const TextLabel: FC<PropsWithChildren> = (props) => {
@ -46,7 +54,58 @@ export const DelimiterInput: FC<InputProps & { tooltip?: string }> = (props) =>
)
}
export const MaxLengthInput: FC<InputNumberProps> = (props) => {
type CompoundNumberInputProps = Omit<NumberFieldRootProps, 'children' | 'className' | 'onValueChange'> & Omit<NumberFieldInputProps, 'children' | 'size' | 'onChange'> & {
unit?: ReactNode
size?: NumberFieldSize
onChange: (value: number) => void
}
function CompoundNumberInput({
onChange,
unit,
size = 'large',
className,
...props
}: CompoundNumberInputProps) {
const { value, defaultValue, min, max, step, disabled, readOnly, required, id, name, onBlur, ...inputProps } = props
const emptyValue = defaultValue ?? min ?? 0
return (
<NumberField
value={value}
defaultValue={defaultValue}
min={min}
max={max}
step={step}
disabled={disabled}
readOnly={readOnly}
required={required}
id={id}
name={name}
onValueChange={value => onChange(value ?? emptyValue)}
>
<NumberFieldGroup size={size}>
<NumberFieldInput
{...inputProps}
size={size}
className={className}
onBlur={onBlur}
/>
{Boolean(unit) && (
<NumberFieldUnit size={size}>
{unit}
</NumberFieldUnit>
)}
<NumberFieldControls>
<NumberFieldIncrement size={size} />
<NumberFieldDecrement size={size} />
</NumberFieldControls>
</NumberFieldGroup>
</NumberField>
)
}
export const MaxLengthInput: FC<CompoundNumberInputProps> = (props) => {
const maxValue = env.NEXT_PUBLIC_INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH
const { t } = useTranslation()
@ -57,8 +116,7 @@ export const MaxLengthInput: FC<InputNumberProps> = (props) => {
</div>
)}
>
<InputNumber
type="number"
<CompoundNumberInput
size="large"
placeholder={`${maxValue}`}
max={maxValue}
@ -69,7 +127,7 @@ export const MaxLengthInput: FC<InputNumberProps> = (props) => {
)
}
export const OverlapInput: FC<InputNumberProps> = (props) => {
export const OverlapInput: FC<CompoundNumberInputProps> = (props) => {
const { t } = useTranslation()
return (
<FormField label={(
@ -85,8 +143,7 @@ export const OverlapInput: FC<InputNumberProps> = (props) => {
</div>
)}
>
<InputNumber
type="number"
<CompoundNumberInput
size="large"
placeholder={t('stepTwo.overlap', { ns: 'datasetCreation' }) || ''}
min={1}

View File

@ -1,5 +1,4 @@
import type { ComponentProps, FC, ReactNode } from 'react'
import Image from 'next/image'
import { cn } from '@/utils/classnames'
const TriangleArrow: FC<ComponentProps<'svg'>> = props => (
@ -23,7 +22,7 @@ export const OptionCardHeader: FC<OptionCardHeaderProps> = (props) => {
return (
<div className={cn('relative flex h-full overflow-hidden rounded-t-xl', isActive && activeClassName, !disabled && 'cursor-pointer')}>
<div className="relative flex size-14 items-center justify-center overflow-hidden">
{isActive && effectImg && <Image src={effectImg} className="absolute left-0 top-0 h-full w-full" alt="" width={56} height={56} />}
{isActive && effectImg && <img src={effectImg} className="absolute left-0 top-0 h-full w-full" alt="" width={56} height={56} />}
<div className="p-1">
<div className="flex size-8 justify-center rounded-lg border border-components-panel-border-subtle bg-background-default-dodge p-1.5 shadow-md">
{icon}

View File

@ -4,7 +4,6 @@ import type { FC } from 'react'
import type { ParentChildConfig } from '../hooks'
import type { ParentMode, PreProcessingRule, SummaryIndexSetting as SummaryIndexSettingType } from '@/models/datasets'
import { RiSearchEyeLine } from '@remixicon/react'
import Image from 'next/image'
import { useTranslation } from 'react-i18next'
import Button from '@/app/components/base/button'
import Checkbox from '@/app/components/base/checkbox'
@ -118,7 +117,7 @@ export const ParentChildOptions: FC<ParentChildOptionsProps> = ({
</div>
<RadioCard
className="mt-1"
icon={<Image src={Note} alt="" />}
icon={<img src={Note.src} alt="" />}
title={t('stepTwo.paragraph', { ns: 'datasetCreation' })}
description={t('stepTwo.paragraphTip', { ns: 'datasetCreation' })}
isChosen={parentChildConfig.chunkForContext === 'paragraph'}
@ -140,7 +139,7 @@ export const ParentChildOptions: FC<ParentChildOptionsProps> = ({
/>
<RadioCard
className="mt-2"
icon={<Image src={FileList} alt="" />}
icon={<img src={FileList.src} alt="" />}
title={t('stepTwo.fullDoc', { ns: 'datasetCreation' })}
description={t('stepTwo.fullDocTip', { ns: 'datasetCreation' })}
onChosen={() => onChunkForContextChange('full-doc')}

View File

@ -3,7 +3,7 @@ import { render, screen } from '@testing-library/react'
import { TopBar } from '../index'
// Mock next/link to capture href values
vi.mock('next/link', () => ({
vi.mock('@/next/link', () => ({
default: ({ children, href, replace, className }: { children: React.ReactNode, href: string, replace?: boolean, className?: string }) => (
<a href={href} data-replace={replace} className={className} data-testid="back-link">
{children}

View File

@ -1,9 +1,9 @@
import type { FC } from 'react'
import type { StepperProps } from '../stepper'
import { RiArrowLeftLine } from '@remixicon/react'
import Link from 'next/link'
import { useMemo } from 'react'
import { useTranslation } from 'react-i18next'
import Link from '@/next/link'
import { cn } from '@/utils/classnames'
import { Stepper } from '../stepper'

View File

@ -1,5 +1,5 @@
import type { CrawlOptions, CrawlResultItem } from '@/models/datasets'
import { fireEvent, render, screen, waitFor } from '@testing-library/react'
import { act, fireEvent, render, screen, waitFor } from '@testing-library/react'
import userEvent from '@testing-library/user-event'
import { beforeEach, describe, expect, it, vi } from 'vitest'
@ -55,6 +55,21 @@ const createMockCrawlResultItem = (overrides: Partial<CrawlResultItem> = {}): Cr
...overrides,
})
const createDeferred = <T,>() => {
let resolve!: (value: T | PromiseLike<T>) => void
let reject!: (reason?: unknown) => void
const promise = new Promise<T>((res, rej) => {
resolve = res
reject = rej
})
return {
promise,
resolve,
reject,
}
}
// FireCrawl Component Tests
describe('FireCrawl', () => {
@ -217,7 +232,7 @@ describe('FireCrawl', () => {
await user.click(runButton)
await waitFor(() => {
expect(mockCreateFirecrawlTask).toHaveBeenCalled()
expect(mockOnCheckedCrawlResultChange).toHaveBeenCalledWith([])
})
})
@ -241,7 +256,7 @@ describe('FireCrawl', () => {
await user.click(runButton)
await waitFor(() => {
expect(mockCreateFirecrawlTask).toHaveBeenCalled()
expect(mockOnCheckedCrawlResultChange).toHaveBeenCalledWith([])
})
})
})
@ -277,6 +292,10 @@ describe('FireCrawl', () => {
}),
})
})
await waitFor(() => {
expect(mockOnCheckedCrawlResultChange).toHaveBeenCalledWith([])
})
})
it('should call onJobIdChange with job_id from API response', async () => {
@ -301,6 +320,10 @@ describe('FireCrawl', () => {
await waitFor(() => {
expect(mockOnJobIdChange).toHaveBeenCalledWith('my-job-123')
})
await waitFor(() => {
expect(mockOnCheckedCrawlResultChange).toHaveBeenCalledWith([])
})
})
it('should remove empty max_depth from crawlOptions before sending to API', async () => {
@ -334,11 +357,23 @@ describe('FireCrawl', () => {
}),
})
})
await waitFor(() => {
expect(mockOnCheckedCrawlResultChange).toHaveBeenCalledWith([])
})
})
it('should show loading state while running', async () => {
const user = userEvent.setup()
mockCreateFirecrawlTask.mockImplementation(() => new Promise(() => {})) // Never resolves
const createTaskDeferred = createDeferred<{ job_id: string }>()
mockCreateFirecrawlTask.mockImplementation(() => createTaskDeferred.promise)
mockCheckFirecrawlTaskStatus.mockResolvedValueOnce({
status: 'completed',
data: [],
total: 0,
current: 0,
time_consuming: 1,
})
render(<FireCrawl {...defaultProps} />)
@ -352,6 +387,14 @@ describe('FireCrawl', () => {
await waitFor(() => {
expect(runButton).not.toHaveTextContent(/run/i)
})
await act(async () => {
createTaskDeferred.resolve({ job_id: 'test-job-id' })
})
await waitFor(() => {
expect(mockOnCheckedCrawlResultChange).toHaveBeenCalledWith([])
})
})
})
@ -656,7 +699,7 @@ describe('FireCrawl', () => {
await waitFor(() => {
// Total should be capped to limit (5)
expect(mockCheckFirecrawlTaskStatus).toHaveBeenCalled()
expect(mockOnCheckedCrawlResultChange).toHaveBeenCalledWith([])
})
})
})

View File

@ -2,7 +2,7 @@
import type { FC } from 'react'
import type { CrawlOptions, CrawlResultItem } from '@/models/datasets'
import * as React from 'react'
import { useCallback, useEffect, useState } from 'react'
import { useCallback, useEffect, useRef, useState } from 'react'
import { useTranslation } from 'react-i18next'
import Toast from '@/app/components/base/toast'
import { ACCOUNT_SETTING_TAB } from '@/app/components/header/account-setting/constants'
@ -35,6 +35,22 @@ enum Step {
finished = 'finished',
}
type CrawlState = {
current: number
total: number
data: CrawlResultItem[]
time_consuming: number | string
}
type CrawlFinishedResult = {
isCancelled?: boolean
isError: boolean
errorMessage?: string
data: Partial<CrawlState> & {
data: CrawlResultItem[]
}
}
const FireCrawl: FC<Props> = ({
onPreview,
checkedCrawlResult,
@ -46,10 +62,16 @@ const FireCrawl: FC<Props> = ({
const { t } = useTranslation()
const [step, setStep] = useState<Step>(Step.init)
const [controlFoldOptions, setControlFoldOptions] = useState<number>(0)
const isMountedRef = useRef(true)
useEffect(() => {
if (step !== Step.init)
setControlFoldOptions(Date.now())
}, [step])
useEffect(() => {
return () => {
isMountedRef.current = false
}
}, [])
const setShowAccountSettingModal = useModalContextSelector(s => s.setShowAccountSettingModal)
const handleSetting = useCallback(() => {
setShowAccountSettingModal({
@ -85,16 +107,19 @@ const FireCrawl: FC<Props> = ({
const isInit = step === Step.init
const isCrawlFinished = step === Step.finished
const isRunning = step === Step.running
const [crawlResult, setCrawlResult] = useState<{
current: number
total: number
data: CrawlResultItem[]
time_consuming: number | string
} | undefined>(undefined)
const [crawlResult, setCrawlResult] = useState<CrawlState | undefined>(undefined)
const [crawlErrorMessage, setCrawlErrorMessage] = useState('')
const showError = isCrawlFinished && crawlErrorMessage
const waitForCrawlFinished = useCallback(async (jobId: string) => {
const waitForCrawlFinished = useCallback(async (jobId: string): Promise<CrawlFinishedResult> => {
const cancelledResult: CrawlFinishedResult = {
isCancelled: true,
isError: false,
data: {
data: [],
},
}
try {
const res = await checkFirecrawlTaskStatus(jobId) as any
if (res.status === 'completed') {
@ -104,7 +129,7 @@ const FireCrawl: FC<Props> = ({
...res,
total: Math.min(res.total, Number.parseFloat(crawlOptions.limit as string)),
},
}
} satisfies CrawlFinishedResult
}
if (res.status === 'error' || !res.status) {
// can't get the error message from the firecrawl api
@ -114,12 +139,14 @@ const FireCrawl: FC<Props> = ({
data: {
data: [],
},
}
} satisfies CrawlFinishedResult
}
res.data = res.data.map((item: any) => ({
...item,
content: item.markdown,
}))
if (!isMountedRef.current)
return cancelledResult
// update the progress
setCrawlResult({
...res,
@ -127,17 +154,21 @@ const FireCrawl: FC<Props> = ({
})
onCheckedCrawlResultChange(res.data || []) // default select the crawl result
await sleep(2500)
if (!isMountedRef.current)
return cancelledResult
return await waitForCrawlFinished(jobId)
}
catch (e: any) {
const errorBody = await e.json()
if (!isMountedRef.current)
return cancelledResult
const errorBody = typeof e?.json === 'function' ? await e.json() : undefined
return {
isError: true,
errorMessage: errorBody.message,
errorMessage: errorBody?.message,
data: {
data: [],
},
}
} satisfies CrawlFinishedResult
}
}, [crawlOptions.limit, onCheckedCrawlResultChange])
@ -162,24 +193,31 @@ const FireCrawl: FC<Props> = ({
url,
options: passToServerCrawlOptions,
}) as any
if (!isMountedRef.current)
return
const jobId = res.job_id
onJobIdChange(jobId)
const { isError, data, errorMessage } = await waitForCrawlFinished(jobId)
const { isCancelled, isError, data, errorMessage } = await waitForCrawlFinished(jobId)
if (isCancelled || !isMountedRef.current)
return
if (isError) {
setCrawlErrorMessage(errorMessage || t(`${I18N_PREFIX}.unknownError`, { ns: 'datasetCreation' }))
}
else {
setCrawlResult(data)
setCrawlResult(data as CrawlState)
onCheckedCrawlResultChange(data.data || []) // default select the crawl result
setCrawlErrorMessage('')
}
}
catch (e) {
if (!isMountedRef.current)
return
setCrawlErrorMessage(t(`${I18N_PREFIX}.unknownError`, { ns: 'datasetCreation' })!)
console.log(e)
}
finally {
setStep(Step.finished)
if (isMountedRef.current)
setStep(Step.finished)
}
}, [checkValid, crawlOptions, onJobIdChange, t, waitForCrawlFinished, onCheckedCrawlResultChange])