mirror of
https://github.com/langgenius/dify.git
synced 2026-05-03 08:58:09 +08:00
Merge main HEAD (segment 5) into sandboxed-agent-rebase
Resolve 83 conflicts: 10 backend, 62 frontend, 11 config/lock files. Preserve sandbox/agent/collaboration features while adopting main's UI refactorings (Dialog/AlertDialog/Popover), model provider updates, and enterprise features. Made-with: Cursor
This commit is contained in:
@ -24,7 +24,7 @@ const IndexingTypeValues = {
|
||||
}
|
||||
|
||||
// Mock next/link
|
||||
vi.mock('next/link', () => {
|
||||
vi.mock('@/next/link', () => {
|
||||
return function MockLink({ children, href }: { children: React.ReactNode, href: string }) {
|
||||
return <a href={href}>{children}</a>
|
||||
}
|
||||
|
||||
@ -16,18 +16,10 @@ import {
|
||||
|
||||
const mockPush = vi.fn()
|
||||
const mockRouter = { push: mockPush }
|
||||
vi.mock('next/navigation', () => ({
|
||||
vi.mock('@/next/navigation', () => ({
|
||||
useRouter: () => mockRouter,
|
||||
}))
|
||||
|
||||
// Override global next/image auto-mock: test asserts on data-testid="next-image"
|
||||
vi.mock('next/image', () => ({
|
||||
default: ({ src, alt, className }: { src: string, alt: string, className?: string }) => (
|
||||
// eslint-disable-next-line next/no-img-element
|
||||
<img src={src} alt={alt} className={className} data-testid="next-image" />
|
||||
),
|
||||
}))
|
||||
|
||||
// Mock API service
|
||||
const mockFetchIndexingStatusBatch = vi.fn()
|
||||
vi.mock('@/service/datasets', () => ({
|
||||
@ -979,9 +971,9 @@ describe('RuleDetail', () => {
|
||||
})
|
||||
|
||||
it('should render correct icon for indexing type', () => {
|
||||
render(<RuleDetail indexingType="high_quality" />)
|
||||
const { container } = render(<RuleDetail indexingType="high_quality" />)
|
||||
|
||||
const images = screen.getAllByTestId('next-image')
|
||||
const images = container.querySelectorAll('img')
|
||||
expect(images.length).toBeGreaterThan(0)
|
||||
})
|
||||
})
|
||||
|
||||
@ -6,8 +6,6 @@ import {
|
||||
RiLoader2Fill,
|
||||
RiTerminalBoxLine,
|
||||
} from '@remixicon/react'
|
||||
import Link from 'next/link'
|
||||
import { useRouter } from 'next/navigation'
|
||||
import { useMemo } from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import Button from '@/app/components/base/button'
|
||||
@ -15,6 +13,8 @@ import Divider from '@/app/components/base/divider'
|
||||
import { Plan } from '@/app/components/billing/type'
|
||||
import { useProviderContext } from '@/context/provider-context'
|
||||
import { useDatasetApiAccessUrl } from '@/hooks/use-api-access-url'
|
||||
import Link from '@/next/link'
|
||||
import { useRouter } from '@/next/navigation'
|
||||
import { useProcessRule } from '@/service/knowledge/use-dataset'
|
||||
import { useInvalidDocumentList } from '@/service/knowledge/use-document'
|
||||
import IndexingProgressItem from './indexing-progress-item'
|
||||
|
||||
@ -1,6 +1,5 @@
|
||||
import type { FC } from 'react'
|
||||
import type { ProcessRuleResponse } from '@/models/datasets'
|
||||
import Image from 'next/image'
|
||||
import { useCallback } from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import { FieldInfo } from '@/app/components/datasets/documents/detail/metadata'
|
||||
@ -119,12 +118,12 @@ const RuleDetail: FC<RuleDetailProps> = ({ sourceData, indexingType, retrievalMe
|
||||
<FieldInfo
|
||||
label={t('stepTwo.indexMode', { ns: 'datasetCreation' })}
|
||||
displayedValue={indexModeLabel}
|
||||
valueIcon={<Image className="size-4" src={indexMethodIconSrc} alt="" />}
|
||||
valueIcon={<img className="size-4" src={indexMethodIconSrc} alt="" />}
|
||||
/>
|
||||
<FieldInfo
|
||||
label={t('form.retrievalSetting.title', { ns: 'datasetSettings' })}
|
||||
displayedValue={retrievalLabel}
|
||||
valueIcon={<Image className="size-4" src={retrievalIconSrc} alt="" />}
|
||||
valueIcon={<img className="size-4" src={retrievalIconSrc} alt="" />}
|
||||
/>
|
||||
</div>
|
||||
)
|
||||
|
||||
@ -7,7 +7,7 @@ import EmptyDatasetCreationModal from '../index'
|
||||
|
||||
// Mock Next.js router
|
||||
const mockPush = vi.fn()
|
||||
vi.mock('next/navigation', () => ({
|
||||
vi.mock('@/next/navigation', () => ({
|
||||
useRouter: () => ({
|
||||
push: mockPush,
|
||||
}),
|
||||
|
||||
@ -1,5 +1,4 @@
|
||||
'use client'
|
||||
import { useRouter } from 'next/navigation'
|
||||
import * as React from 'react'
|
||||
import { useState } from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
@ -9,6 +8,7 @@ import Button from '@/app/components/base/button'
|
||||
import Input from '@/app/components/base/input'
|
||||
import Modal from '@/app/components/base/modal'
|
||||
import { ToastContext } from '@/app/components/base/toast/context'
|
||||
import { useRouter } from '@/next/navigation'
|
||||
|
||||
import { createEmptyDataset } from '@/service/datasets'
|
||||
import { useInvalidDatasetList } from '@/service/knowledge/use-dataset'
|
||||
|
||||
@ -58,7 +58,7 @@ vi.mock('@/app/components/datasets/common/document-file-icon', () => ({
|
||||
}))
|
||||
|
||||
// Mock SimplePieChart
|
||||
vi.mock('next/dynamic', () => ({
|
||||
vi.mock('@/next/dynamic', () => ({
|
||||
default: () => {
|
||||
const Component = ({ percentage }: { percentage: number }) => (
|
||||
<div data-testid="pie-chart">
|
||||
|
||||
@ -17,7 +17,7 @@ vi.mock('@/types/app', () => ({
|
||||
}))
|
||||
|
||||
// Mock SimplePieChart with dynamic import handling
|
||||
vi.mock('next/dynamic', () => ({
|
||||
vi.mock('@/next/dynamic', () => ({
|
||||
default: () => {
|
||||
const DynamicComponent = ({ percentage, stroke, fill }: { percentage: number, stroke: string, fill: string }) => (
|
||||
<div data-testid="pie-chart" data-percentage={percentage} data-stroke={stroke} data-fill={fill}>
|
||||
|
||||
@ -1,10 +1,10 @@
|
||||
'use client'
|
||||
import type { CustomFile as File, FileItem } from '@/models/datasets'
|
||||
import { RiDeleteBinLine, RiErrorWarningFill } from '@remixicon/react'
|
||||
import dynamic from 'next/dynamic'
|
||||
import { useMemo } from 'react'
|
||||
import DocumentFileIcon from '@/app/components/datasets/common/document-file-icon'
|
||||
import useTheme from '@/hooks/use-theme'
|
||||
import dynamic from '@/next/dynamic'
|
||||
import { Theme } from '@/types/app'
|
||||
import { formatFileSize, getFileExtension } from '@/utils/format'
|
||||
import { PROGRESS_COMPLETE, PROGRESS_ERROR } from '../constants'
|
||||
|
||||
@ -5,12 +5,12 @@ import Research from './assets/research-mod.svg'
|
||||
import Selection from './assets/selection-mod.svg'
|
||||
|
||||
export const indexMethodIcon = {
|
||||
high_quality: GoldIcon,
|
||||
economical: Piggybank,
|
||||
high_quality: GoldIcon.src,
|
||||
economical: Piggybank.src,
|
||||
}
|
||||
|
||||
export const retrievalIcon = {
|
||||
vector: Selection,
|
||||
fullText: Research,
|
||||
hybrid: PatternRecognition,
|
||||
vector: Selection.src,
|
||||
fullText: Research.src,
|
||||
hybrid: PatternRecognition.src,
|
||||
}
|
||||
|
||||
@ -6,7 +6,7 @@ import { ChunkingMode } from '@/models/datasets'
|
||||
import { IndexingType } from '../../hooks'
|
||||
import { IndexingModeSection } from '../indexing-mode-section'
|
||||
|
||||
vi.mock('next/link', () => ({
|
||||
vi.mock('@/next/link', () => ({
|
||||
default: ({ children, href, ...props }: { children?: React.ReactNode, href?: string, className?: string }) => <a href={href} {...props}>{children}</a>,
|
||||
}))
|
||||
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
import { render, screen } from '@testing-library/react'
|
||||
import { fireEvent, render, screen } from '@testing-library/react'
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import { DelimiterInput, MaxLengthInput, OverlapInput } from '../inputs'
|
||||
|
||||
@ -47,19 +47,34 @@ describe('MaxLengthInput', () => {
|
||||
|
||||
it('should render number input', () => {
|
||||
render(<MaxLengthInput onChange={vi.fn()} />)
|
||||
const input = screen.getByRole('spinbutton')
|
||||
const input = screen.getByRole('textbox')
|
||||
expect(input).toBeInTheDocument()
|
||||
})
|
||||
|
||||
it('should accept value prop', () => {
|
||||
render(<MaxLengthInput value={500} onChange={vi.fn()} />)
|
||||
expect(screen.getByDisplayValue('500')).toBeInTheDocument()
|
||||
expect(screen.getByRole('textbox')).toHaveValue('500')
|
||||
})
|
||||
|
||||
it('should have min of 1', () => {
|
||||
render(<MaxLengthInput onChange={vi.fn()} />)
|
||||
const input = screen.getByRole('spinbutton')
|
||||
expect(input).toHaveAttribute('min', '1')
|
||||
const input = screen.getByRole('textbox')
|
||||
expect(input).toBeInTheDocument()
|
||||
})
|
||||
|
||||
it('should reset to the minimum when users clear the value', () => {
|
||||
const onChange = vi.fn()
|
||||
render(<MaxLengthInput value={500} onChange={onChange} />)
|
||||
fireEvent.change(screen.getByRole('textbox'), { target: { value: '' } })
|
||||
expect(onChange).toHaveBeenCalledWith(1)
|
||||
})
|
||||
|
||||
it('should clamp out-of-range text edits before updating state', () => {
|
||||
const onChange = vi.fn()
|
||||
render(<MaxLengthInput value={500} max={1000} onChange={onChange} />)
|
||||
|
||||
fireEvent.change(screen.getByRole('textbox'), { target: { value: '1200' } })
|
||||
expect(onChange).toHaveBeenLastCalledWith(1000)
|
||||
})
|
||||
})
|
||||
|
||||
@ -75,18 +90,33 @@ describe('OverlapInput', () => {
|
||||
|
||||
it('should render number input', () => {
|
||||
render(<OverlapInput onChange={vi.fn()} />)
|
||||
const input = screen.getByRole('spinbutton')
|
||||
const input = screen.getByRole('textbox')
|
||||
expect(input).toBeInTheDocument()
|
||||
})
|
||||
|
||||
it('should accept value prop', () => {
|
||||
render(<OverlapInput value={50} onChange={vi.fn()} />)
|
||||
expect(screen.getByDisplayValue('50')).toBeInTheDocument()
|
||||
expect(screen.getByRole('textbox')).toHaveValue('50')
|
||||
})
|
||||
|
||||
it('should have min of 1', () => {
|
||||
render(<OverlapInput onChange={vi.fn()} />)
|
||||
const input = screen.getByRole('spinbutton')
|
||||
expect(input).toHaveAttribute('min', '1')
|
||||
const input = screen.getByRole('textbox')
|
||||
expect(input).toBeInTheDocument()
|
||||
})
|
||||
|
||||
it('should reset to the minimum when users clear the value', () => {
|
||||
const onChange = vi.fn()
|
||||
render(<OverlapInput value={50} onChange={onChange} />)
|
||||
fireEvent.change(screen.getByRole('textbox'), { target: { value: '' } })
|
||||
expect(onChange).toHaveBeenCalledWith(1)
|
||||
})
|
||||
|
||||
it('should clamp out-of-range text edits before updating state', () => {
|
||||
const onChange = vi.fn()
|
||||
render(<OverlapInput value={50} max={100} onChange={onChange} />)
|
||||
|
||||
fireEvent.change(screen.getByRole('textbox'), { target: { value: '150' } })
|
||||
expect(onChange).toHaveBeenLastCalledWith(100)
|
||||
})
|
||||
})
|
||||
|
||||
@ -2,13 +2,6 @@ import { fireEvent, render, screen } from '@testing-library/react'
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import { OptionCard, OptionCardHeader } from '../option-card'
|
||||
|
||||
// Override global next/image auto-mock: tests assert on rendered <img> elements
|
||||
vi.mock('next/image', () => ({
|
||||
default: ({ src, alt, ...props }: { src?: string, alt?: string, width?: number, height?: number }) => (
|
||||
<img src={src} alt={alt} {...props} />
|
||||
),
|
||||
}))
|
||||
|
||||
describe('OptionCardHeader', () => {
|
||||
const defaultProps = {
|
||||
icon: <span data-testid="icon">icon</span>,
|
||||
|
||||
@ -6,7 +6,6 @@ import {
|
||||
RiAlertFill,
|
||||
RiSearchEyeLine,
|
||||
} from '@remixicon/react'
|
||||
import Image from 'next/image'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import Button from '@/app/components/base/button'
|
||||
import Checkbox from '@/app/components/base/checkbox'
|
||||
@ -97,7 +96,7 @@ export const GeneralChunkingOptions: FC<GeneralChunkingOptionsProps> = ({
|
||||
<OptionCard
|
||||
className="mb-2 bg-background-section"
|
||||
title={t('stepTwo.general', { ns: 'datasetCreation' })}
|
||||
icon={<Image width={20} height={20} src={SettingCog} alt={t('stepTwo.general', { ns: 'datasetCreation' })} />}
|
||||
icon={<img width={20} height={20} src={SettingCog.src} alt={t('stepTwo.general', { ns: 'datasetCreation' })} />}
|
||||
activeHeaderClassName="bg-dataset-option-card-blue-gradient"
|
||||
description={t('stepTwo.generalTip', { ns: 'datasetCreation' })}
|
||||
isActive={isActive}
|
||||
|
||||
@ -3,8 +3,6 @@
|
||||
import type { FC } from 'react'
|
||||
import type { DefaultModel, Model } from '@/app/components/header/account-setting/model-provider-page/declarations'
|
||||
import type { RetrievalConfig } from '@/types/app'
|
||||
import Image from 'next/image'
|
||||
import Link from 'next/link'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import Badge from '@/app/components/base/badge'
|
||||
import Button from '@/app/components/base/button'
|
||||
@ -17,6 +15,7 @@ import RetrievalMethodConfig from '@/app/components/datasets/common/retrieval-me
|
||||
import ModelSelector from '@/app/components/header/account-setting/model-provider-page/model-selector'
|
||||
import { useDocLink } from '@/context/i18n'
|
||||
import { ChunkingMode } from '@/models/datasets'
|
||||
import Link from '@/next/link'
|
||||
import { cn } from '@/utils/classnames'
|
||||
import { indexMethodIcon } from '../../icons'
|
||||
import { IndexingType } from '../hooks'
|
||||
@ -98,7 +97,7 @@ export const IndexingModeSection: FC<IndexingModeSectionProps> = ({
|
||||
</div>
|
||||
)}
|
||||
description={t('stepTwo.qualifiedTip', { ns: 'datasetCreation' })}
|
||||
icon={<Image src={indexMethodIcon.high_quality} alt="" />}
|
||||
icon={<img src={indexMethodIcon.high_quality} alt="" />}
|
||||
isActive={!hasSetIndexType && indexType === IndexingType.QUALIFIED}
|
||||
disabled={hasSetIndexType}
|
||||
onSwitched={() => onIndexTypeChange(IndexingType.QUALIFIED)}
|
||||
@ -143,7 +142,7 @@ export const IndexingModeSection: FC<IndexingModeSectionProps> = ({
|
||||
className="h-full"
|
||||
title={t('stepTwo.economical', { ns: 'datasetCreation' })}
|
||||
description={t('stepTwo.economicalTip', { ns: 'datasetCreation' })}
|
||||
icon={<Image src={indexMethodIcon.economical} alt="" />}
|
||||
icon={<img src={indexMethodIcon.economical} alt="" />}
|
||||
isActive={!hasSetIndexType && indexType === IndexingType.ECONOMICAL}
|
||||
disabled={hasSetIndexType || docForm !== ChunkingMode.text}
|
||||
onSwitched={() => onIndexTypeChange(IndexingType.ECONOMICAL)}
|
||||
|
||||
@ -1,10 +1,18 @@
|
||||
import type { FC, PropsWithChildren, ReactNode } from 'react'
|
||||
import type { InputProps } from '@/app/components/base/input'
|
||||
import type { InputNumberProps } from '@/app/components/base/input-number'
|
||||
import type { NumberFieldInputProps, NumberFieldRootProps, NumberFieldSize } from '@/app/components/base/ui/number-field'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import Input from '@/app/components/base/input'
|
||||
import { InputNumber } from '@/app/components/base/input-number'
|
||||
import Tooltip from '@/app/components/base/tooltip'
|
||||
import {
|
||||
NumberField,
|
||||
NumberFieldControls,
|
||||
NumberFieldDecrement,
|
||||
NumberFieldGroup,
|
||||
NumberFieldIncrement,
|
||||
NumberFieldInput,
|
||||
NumberFieldUnit,
|
||||
} from '@/app/components/base/ui/number-field'
|
||||
import { env } from '@/env'
|
||||
|
||||
const TextLabel: FC<PropsWithChildren> = (props) => {
|
||||
@ -46,7 +54,58 @@ export const DelimiterInput: FC<InputProps & { tooltip?: string }> = (props) =>
|
||||
)
|
||||
}
|
||||
|
||||
export const MaxLengthInput: FC<InputNumberProps> = (props) => {
|
||||
type CompoundNumberInputProps = Omit<NumberFieldRootProps, 'children' | 'className' | 'onValueChange'> & Omit<NumberFieldInputProps, 'children' | 'size' | 'onChange'> & {
|
||||
unit?: ReactNode
|
||||
size?: NumberFieldSize
|
||||
onChange: (value: number) => void
|
||||
}
|
||||
|
||||
function CompoundNumberInput({
|
||||
onChange,
|
||||
unit,
|
||||
size = 'large',
|
||||
className,
|
||||
...props
|
||||
}: CompoundNumberInputProps) {
|
||||
const { value, defaultValue, min, max, step, disabled, readOnly, required, id, name, onBlur, ...inputProps } = props
|
||||
const emptyValue = defaultValue ?? min ?? 0
|
||||
|
||||
return (
|
||||
<NumberField
|
||||
value={value}
|
||||
defaultValue={defaultValue}
|
||||
min={min}
|
||||
max={max}
|
||||
step={step}
|
||||
disabled={disabled}
|
||||
readOnly={readOnly}
|
||||
required={required}
|
||||
id={id}
|
||||
name={name}
|
||||
onValueChange={value => onChange(value ?? emptyValue)}
|
||||
>
|
||||
<NumberFieldGroup size={size}>
|
||||
<NumberFieldInput
|
||||
{...inputProps}
|
||||
size={size}
|
||||
className={className}
|
||||
onBlur={onBlur}
|
||||
/>
|
||||
{Boolean(unit) && (
|
||||
<NumberFieldUnit size={size}>
|
||||
{unit}
|
||||
</NumberFieldUnit>
|
||||
)}
|
||||
<NumberFieldControls>
|
||||
<NumberFieldIncrement size={size} />
|
||||
<NumberFieldDecrement size={size} />
|
||||
</NumberFieldControls>
|
||||
</NumberFieldGroup>
|
||||
</NumberField>
|
||||
)
|
||||
}
|
||||
|
||||
export const MaxLengthInput: FC<CompoundNumberInputProps> = (props) => {
|
||||
const maxValue = env.NEXT_PUBLIC_INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH
|
||||
|
||||
const { t } = useTranslation()
|
||||
@ -57,8 +116,7 @@ export const MaxLengthInput: FC<InputNumberProps> = (props) => {
|
||||
</div>
|
||||
)}
|
||||
>
|
||||
<InputNumber
|
||||
type="number"
|
||||
<CompoundNumberInput
|
||||
size="large"
|
||||
placeholder={`≤ ${maxValue}`}
|
||||
max={maxValue}
|
||||
@ -69,7 +127,7 @@ export const MaxLengthInput: FC<InputNumberProps> = (props) => {
|
||||
)
|
||||
}
|
||||
|
||||
export const OverlapInput: FC<InputNumberProps> = (props) => {
|
||||
export const OverlapInput: FC<CompoundNumberInputProps> = (props) => {
|
||||
const { t } = useTranslation()
|
||||
return (
|
||||
<FormField label={(
|
||||
@ -85,8 +143,7 @@ export const OverlapInput: FC<InputNumberProps> = (props) => {
|
||||
</div>
|
||||
)}
|
||||
>
|
||||
<InputNumber
|
||||
type="number"
|
||||
<CompoundNumberInput
|
||||
size="large"
|
||||
placeholder={t('stepTwo.overlap', { ns: 'datasetCreation' }) || ''}
|
||||
min={1}
|
||||
|
||||
@ -1,5 +1,4 @@
|
||||
import type { ComponentProps, FC, ReactNode } from 'react'
|
||||
import Image from 'next/image'
|
||||
import { cn } from '@/utils/classnames'
|
||||
|
||||
const TriangleArrow: FC<ComponentProps<'svg'>> = props => (
|
||||
@ -23,7 +22,7 @@ export const OptionCardHeader: FC<OptionCardHeaderProps> = (props) => {
|
||||
return (
|
||||
<div className={cn('relative flex h-full overflow-hidden rounded-t-xl', isActive && activeClassName, !disabled && 'cursor-pointer')}>
|
||||
<div className="relative flex size-14 items-center justify-center overflow-hidden">
|
||||
{isActive && effectImg && <Image src={effectImg} className="absolute left-0 top-0 h-full w-full" alt="" width={56} height={56} />}
|
||||
{isActive && effectImg && <img src={effectImg} className="absolute left-0 top-0 h-full w-full" alt="" width={56} height={56} />}
|
||||
<div className="p-1">
|
||||
<div className="flex size-8 justify-center rounded-lg border border-components-panel-border-subtle bg-background-default-dodge p-1.5 shadow-md">
|
||||
{icon}
|
||||
|
||||
@ -4,7 +4,6 @@ import type { FC } from 'react'
|
||||
import type { ParentChildConfig } from '../hooks'
|
||||
import type { ParentMode, PreProcessingRule, SummaryIndexSetting as SummaryIndexSettingType } from '@/models/datasets'
|
||||
import { RiSearchEyeLine } from '@remixicon/react'
|
||||
import Image from 'next/image'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import Button from '@/app/components/base/button'
|
||||
import Checkbox from '@/app/components/base/checkbox'
|
||||
@ -118,7 +117,7 @@ export const ParentChildOptions: FC<ParentChildOptionsProps> = ({
|
||||
</div>
|
||||
<RadioCard
|
||||
className="mt-1"
|
||||
icon={<Image src={Note} alt="" />}
|
||||
icon={<img src={Note.src} alt="" />}
|
||||
title={t('stepTwo.paragraph', { ns: 'datasetCreation' })}
|
||||
description={t('stepTwo.paragraphTip', { ns: 'datasetCreation' })}
|
||||
isChosen={parentChildConfig.chunkForContext === 'paragraph'}
|
||||
@ -140,7 +139,7 @@ export const ParentChildOptions: FC<ParentChildOptionsProps> = ({
|
||||
/>
|
||||
<RadioCard
|
||||
className="mt-2"
|
||||
icon={<Image src={FileList} alt="" />}
|
||||
icon={<img src={FileList.src} alt="" />}
|
||||
title={t('stepTwo.fullDoc', { ns: 'datasetCreation' })}
|
||||
description={t('stepTwo.fullDocTip', { ns: 'datasetCreation' })}
|
||||
onChosen={() => onChunkForContextChange('full-doc')}
|
||||
|
||||
@ -3,7 +3,7 @@ import { render, screen } from '@testing-library/react'
|
||||
import { TopBar } from '../index'
|
||||
|
||||
// Mock next/link to capture href values
|
||||
vi.mock('next/link', () => ({
|
||||
vi.mock('@/next/link', () => ({
|
||||
default: ({ children, href, replace, className }: { children: React.ReactNode, href: string, replace?: boolean, className?: string }) => (
|
||||
<a href={href} data-replace={replace} className={className} data-testid="back-link">
|
||||
{children}
|
||||
|
||||
@ -1,9 +1,9 @@
|
||||
import type { FC } from 'react'
|
||||
import type { StepperProps } from '../stepper'
|
||||
import { RiArrowLeftLine } from '@remixicon/react'
|
||||
import Link from 'next/link'
|
||||
import { useMemo } from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import Link from '@/next/link'
|
||||
import { cn } from '@/utils/classnames'
|
||||
import { Stepper } from '../stepper'
|
||||
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
import type { CrawlOptions, CrawlResultItem } from '@/models/datasets'
|
||||
import { fireEvent, render, screen, waitFor } from '@testing-library/react'
|
||||
import { act, fireEvent, render, screen, waitFor } from '@testing-library/react'
|
||||
import userEvent from '@testing-library/user-event'
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
|
||||
@ -55,6 +55,21 @@ const createMockCrawlResultItem = (overrides: Partial<CrawlResultItem> = {}): Cr
|
||||
...overrides,
|
||||
})
|
||||
|
||||
const createDeferred = <T,>() => {
|
||||
let resolve!: (value: T | PromiseLike<T>) => void
|
||||
let reject!: (reason?: unknown) => void
|
||||
const promise = new Promise<T>((res, rej) => {
|
||||
resolve = res
|
||||
reject = rej
|
||||
})
|
||||
|
||||
return {
|
||||
promise,
|
||||
resolve,
|
||||
reject,
|
||||
}
|
||||
}
|
||||
|
||||
// FireCrawl Component Tests
|
||||
|
||||
describe('FireCrawl', () => {
|
||||
@ -217,7 +232,7 @@ describe('FireCrawl', () => {
|
||||
await user.click(runButton)
|
||||
|
||||
await waitFor(() => {
|
||||
expect(mockCreateFirecrawlTask).toHaveBeenCalled()
|
||||
expect(mockOnCheckedCrawlResultChange).toHaveBeenCalledWith([])
|
||||
})
|
||||
})
|
||||
|
||||
@ -241,7 +256,7 @@ describe('FireCrawl', () => {
|
||||
await user.click(runButton)
|
||||
|
||||
await waitFor(() => {
|
||||
expect(mockCreateFirecrawlTask).toHaveBeenCalled()
|
||||
expect(mockOnCheckedCrawlResultChange).toHaveBeenCalledWith([])
|
||||
})
|
||||
})
|
||||
})
|
||||
@ -277,6 +292,10 @@ describe('FireCrawl', () => {
|
||||
}),
|
||||
})
|
||||
})
|
||||
|
||||
await waitFor(() => {
|
||||
expect(mockOnCheckedCrawlResultChange).toHaveBeenCalledWith([])
|
||||
})
|
||||
})
|
||||
|
||||
it('should call onJobIdChange with job_id from API response', async () => {
|
||||
@ -301,6 +320,10 @@ describe('FireCrawl', () => {
|
||||
await waitFor(() => {
|
||||
expect(mockOnJobIdChange).toHaveBeenCalledWith('my-job-123')
|
||||
})
|
||||
|
||||
await waitFor(() => {
|
||||
expect(mockOnCheckedCrawlResultChange).toHaveBeenCalledWith([])
|
||||
})
|
||||
})
|
||||
|
||||
it('should remove empty max_depth from crawlOptions before sending to API', async () => {
|
||||
@ -334,11 +357,23 @@ describe('FireCrawl', () => {
|
||||
}),
|
||||
})
|
||||
})
|
||||
|
||||
await waitFor(() => {
|
||||
expect(mockOnCheckedCrawlResultChange).toHaveBeenCalledWith([])
|
||||
})
|
||||
})
|
||||
|
||||
it('should show loading state while running', async () => {
|
||||
const user = userEvent.setup()
|
||||
mockCreateFirecrawlTask.mockImplementation(() => new Promise(() => {})) // Never resolves
|
||||
const createTaskDeferred = createDeferred<{ job_id: string }>()
|
||||
mockCreateFirecrawlTask.mockImplementation(() => createTaskDeferred.promise)
|
||||
mockCheckFirecrawlTaskStatus.mockResolvedValueOnce({
|
||||
status: 'completed',
|
||||
data: [],
|
||||
total: 0,
|
||||
current: 0,
|
||||
time_consuming: 1,
|
||||
})
|
||||
|
||||
render(<FireCrawl {...defaultProps} />)
|
||||
|
||||
@ -352,6 +387,14 @@ describe('FireCrawl', () => {
|
||||
await waitFor(() => {
|
||||
expect(runButton).not.toHaveTextContent(/run/i)
|
||||
})
|
||||
|
||||
await act(async () => {
|
||||
createTaskDeferred.resolve({ job_id: 'test-job-id' })
|
||||
})
|
||||
|
||||
await waitFor(() => {
|
||||
expect(mockOnCheckedCrawlResultChange).toHaveBeenCalledWith([])
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
@ -656,7 +699,7 @@ describe('FireCrawl', () => {
|
||||
|
||||
await waitFor(() => {
|
||||
// Total should be capped to limit (5)
|
||||
expect(mockCheckFirecrawlTaskStatus).toHaveBeenCalled()
|
||||
expect(mockOnCheckedCrawlResultChange).toHaveBeenCalledWith([])
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
@ -2,7 +2,7 @@
|
||||
import type { FC } from 'react'
|
||||
import type { CrawlOptions, CrawlResultItem } from '@/models/datasets'
|
||||
import * as React from 'react'
|
||||
import { useCallback, useEffect, useState } from 'react'
|
||||
import { useCallback, useEffect, useRef, useState } from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import Toast from '@/app/components/base/toast'
|
||||
import { ACCOUNT_SETTING_TAB } from '@/app/components/header/account-setting/constants'
|
||||
@ -35,6 +35,22 @@ enum Step {
|
||||
finished = 'finished',
|
||||
}
|
||||
|
||||
type CrawlState = {
|
||||
current: number
|
||||
total: number
|
||||
data: CrawlResultItem[]
|
||||
time_consuming: number | string
|
||||
}
|
||||
|
||||
type CrawlFinishedResult = {
|
||||
isCancelled?: boolean
|
||||
isError: boolean
|
||||
errorMessage?: string
|
||||
data: Partial<CrawlState> & {
|
||||
data: CrawlResultItem[]
|
||||
}
|
||||
}
|
||||
|
||||
const FireCrawl: FC<Props> = ({
|
||||
onPreview,
|
||||
checkedCrawlResult,
|
||||
@ -46,10 +62,16 @@ const FireCrawl: FC<Props> = ({
|
||||
const { t } = useTranslation()
|
||||
const [step, setStep] = useState<Step>(Step.init)
|
||||
const [controlFoldOptions, setControlFoldOptions] = useState<number>(0)
|
||||
const isMountedRef = useRef(true)
|
||||
useEffect(() => {
|
||||
if (step !== Step.init)
|
||||
setControlFoldOptions(Date.now())
|
||||
}, [step])
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
isMountedRef.current = false
|
||||
}
|
||||
}, [])
|
||||
const setShowAccountSettingModal = useModalContextSelector(s => s.setShowAccountSettingModal)
|
||||
const handleSetting = useCallback(() => {
|
||||
setShowAccountSettingModal({
|
||||
@ -85,16 +107,19 @@ const FireCrawl: FC<Props> = ({
|
||||
const isInit = step === Step.init
|
||||
const isCrawlFinished = step === Step.finished
|
||||
const isRunning = step === Step.running
|
||||
const [crawlResult, setCrawlResult] = useState<{
|
||||
current: number
|
||||
total: number
|
||||
data: CrawlResultItem[]
|
||||
time_consuming: number | string
|
||||
} | undefined>(undefined)
|
||||
const [crawlResult, setCrawlResult] = useState<CrawlState | undefined>(undefined)
|
||||
const [crawlErrorMessage, setCrawlErrorMessage] = useState('')
|
||||
const showError = isCrawlFinished && crawlErrorMessage
|
||||
|
||||
const waitForCrawlFinished = useCallback(async (jobId: string) => {
|
||||
const waitForCrawlFinished = useCallback(async (jobId: string): Promise<CrawlFinishedResult> => {
|
||||
const cancelledResult: CrawlFinishedResult = {
|
||||
isCancelled: true,
|
||||
isError: false,
|
||||
data: {
|
||||
data: [],
|
||||
},
|
||||
}
|
||||
|
||||
try {
|
||||
const res = await checkFirecrawlTaskStatus(jobId) as any
|
||||
if (res.status === 'completed') {
|
||||
@ -104,7 +129,7 @@ const FireCrawl: FC<Props> = ({
|
||||
...res,
|
||||
total: Math.min(res.total, Number.parseFloat(crawlOptions.limit as string)),
|
||||
},
|
||||
}
|
||||
} satisfies CrawlFinishedResult
|
||||
}
|
||||
if (res.status === 'error' || !res.status) {
|
||||
// can't get the error message from the firecrawl api
|
||||
@ -114,12 +139,14 @@ const FireCrawl: FC<Props> = ({
|
||||
data: {
|
||||
data: [],
|
||||
},
|
||||
}
|
||||
} satisfies CrawlFinishedResult
|
||||
}
|
||||
res.data = res.data.map((item: any) => ({
|
||||
...item,
|
||||
content: item.markdown,
|
||||
}))
|
||||
if (!isMountedRef.current)
|
||||
return cancelledResult
|
||||
// update the progress
|
||||
setCrawlResult({
|
||||
...res,
|
||||
@ -127,17 +154,21 @@ const FireCrawl: FC<Props> = ({
|
||||
})
|
||||
onCheckedCrawlResultChange(res.data || []) // default select the crawl result
|
||||
await sleep(2500)
|
||||
if (!isMountedRef.current)
|
||||
return cancelledResult
|
||||
return await waitForCrawlFinished(jobId)
|
||||
}
|
||||
catch (e: any) {
|
||||
const errorBody = await e.json()
|
||||
if (!isMountedRef.current)
|
||||
return cancelledResult
|
||||
const errorBody = typeof e?.json === 'function' ? await e.json() : undefined
|
||||
return {
|
||||
isError: true,
|
||||
errorMessage: errorBody.message,
|
||||
errorMessage: errorBody?.message,
|
||||
data: {
|
||||
data: [],
|
||||
},
|
||||
}
|
||||
} satisfies CrawlFinishedResult
|
||||
}
|
||||
}, [crawlOptions.limit, onCheckedCrawlResultChange])
|
||||
|
||||
@ -162,24 +193,31 @@ const FireCrawl: FC<Props> = ({
|
||||
url,
|
||||
options: passToServerCrawlOptions,
|
||||
}) as any
|
||||
if (!isMountedRef.current)
|
||||
return
|
||||
const jobId = res.job_id
|
||||
onJobIdChange(jobId)
|
||||
const { isError, data, errorMessage } = await waitForCrawlFinished(jobId)
|
||||
const { isCancelled, isError, data, errorMessage } = await waitForCrawlFinished(jobId)
|
||||
if (isCancelled || !isMountedRef.current)
|
||||
return
|
||||
if (isError) {
|
||||
setCrawlErrorMessage(errorMessage || t(`${I18N_PREFIX}.unknownError`, { ns: 'datasetCreation' }))
|
||||
}
|
||||
else {
|
||||
setCrawlResult(data)
|
||||
setCrawlResult(data as CrawlState)
|
||||
onCheckedCrawlResultChange(data.data || []) // default select the crawl result
|
||||
setCrawlErrorMessage('')
|
||||
}
|
||||
}
|
||||
catch (e) {
|
||||
if (!isMountedRef.current)
|
||||
return
|
||||
setCrawlErrorMessage(t(`${I18N_PREFIX}.unknownError`, { ns: 'datasetCreation' })!)
|
||||
console.log(e)
|
||||
}
|
||||
finally {
|
||||
setStep(Step.finished)
|
||||
if (isMountedRef.current)
|
||||
setStep(Step.finished)
|
||||
}
|
||||
}, [checkValid, crawlOptions, onJobIdChange, t, waitForCrawlFinished, onCheckedCrawlResultChange])
|
||||
|
||||
|
||||
Reference in New Issue
Block a user