merge main

This commit is contained in:
Joel
2024-10-28 10:51:02 +08:00
858 changed files with 16206 additions and 17932 deletions

View File

@ -12,7 +12,7 @@ import Button from '@/app/components/base/button'
import { ToastContext } from '@/app/components/base/toast'
import { createEmptyDataset } from '@/service/datasets'
type IProps = {
interface IProps {
show: boolean
onHide: () => void
}
@ -58,7 +58,7 @@ const EmptyDatasetCreationModal = ({
<div className={s.tip}>{t('datasetCreation.stepOne.modal.tip')}</div>
<div className={s.form}>
<div className={s.label}>{t('datasetCreation.stepOne.modal.input')}</div>
<Input className='!h-8' value={inputValue} placeholder={t('datasetCreation.stepOne.modal.placeholder') || ''} onChange={e => setInputValue(e.target.value)} />
<Input value={inputValue} placeholder={t('datasetCreation.stepOne.modal.placeholder') || ''} onChange={e => setInputValue(e.target.value)} />
</div>
<div className='flex flex-row-reverse'>
<Button className='w-24 ml-2' variant='primary' onClick={submit}>{t('datasetCreation.stepOne.modal.confirmButton')}</Button>

View File

@ -24,6 +24,7 @@ import {
fetchDefaultProcessRule,
} from '@/service/datasets'
import Button from '@/app/components/base/button'
import Input from '@/app/components/base/input'
import Loading from '@/app/components/base/loading'
import FloatRightContainer from '@/app/components/base/float-right-container'
import RetrievalMethodConfig from '@/app/components/datasets/common/retrieval-method-config'
@ -52,7 +53,7 @@ import { ModelTypeEnum } from '@/app/components/header/account-setting/model-pro
import { Globe01 } from '@/app/components/base/icons/src/vender/line/mapsAndTravel'
type ValueOf<T> = T[keyof T]
type StepTwoProps = {
interface StepTwoProps {
isSetting?: boolean
documentDetail?: FullDocumentDetail
isAPIKeySet: boolean
@ -202,7 +203,6 @@ const StepTwo = ({
}
const fetchFileIndexingEstimate = async (docForm = DocForm.TEXT, language?: string) => {
// eslint-disable-next-line ts/no-use-before-define
const res = await didFetchFileIndexingEstimate(getFileIndexingEstimateParams(docForm, language)!)
if (segmentationType === SegmentType.CUSTOM)
setCustomFileIndexingEstimate(res)
@ -211,6 +211,10 @@ const StepTwo = ({
}
const confirmChangeCustomConfig = () => {
if (segmentationType === SegmentType.CUSTOM && max > 4000) {
Toast.notify({ type: 'error', message: t('datasetCreation.stepTwo.maxLengthCheck') })
return
}
setCustomFileIndexingEstimate(null)
setShowPreview()
fetchFileIndexingEstimate()
@ -338,13 +342,17 @@ const StepTwo = ({
Toast.notify({ type: 'error', message: t('datasetCreation.stepTwo.overlapCheck') })
return
}
if (segmentationType === SegmentType.CUSTOM && max > 4000) {
Toast.notify({ type: 'error', message: t('datasetCreation.stepTwo.maxLengthCheck') })
return
}
if (isSetting) {
params = {
original_document_id: documentDetail?.id,
doc_form: docForm,
doc_language: docLanguage,
process_rule: getProcessRule(),
// eslint-disable-next-line ts/no-use-before-define
retrieval_model: retrievalConfig, // Readonly. If want to changed, just go to settings page.
embedding_model: embeddingModel.model, // Readonly
embedding_model_provider: embeddingModel.provider, // Readonly
@ -357,7 +365,7 @@ const StepTwo = ({
rerankDefaultModel,
isRerankDefaultModelValid: !!isRerankDefaultModelValid,
rerankModelList,
// eslint-disable-next-line ts/no-use-before-define
retrievalConfig,
indexMethod: indexMethod as string,
})
@ -367,7 +375,7 @@ const StepTwo = ({
}
const postRetrievalConfig = ensureRerankModelSelected({
rerankDefaultModel: rerankDefaultModel!,
// eslint-disable-next-line ts/no-use-before-define
retrievalConfig,
indexMethod: indexMethod as string,
})
@ -646,29 +654,26 @@ const StepTwo = ({
}
/>
</div>
<input
<Input
type="text"
className={s.input}
placeholder={t('datasetCreation.stepTwo.separatorPlaceholder') || ''}
value={segmentIdentifier}
onChange={e => doSetSegmentIdentifier(e.target.value)}
className='h-9'
placeholder={t('datasetCreation.stepTwo.separatorPlaceholder') || ''} value={segmentIdentifier}
onChange={e => setSegmentIdentifier(e.target.value)}
/>
</div>
</div>
<div className={s.formRow}>
<div className='w-full'>
<div className={s.label}>{t('datasetCreation.stepTwo.maxLength')}</div>
<div className='relative w-full'>
<input
type="number"
className={s.input}
placeholder={t('datasetCreation.stepTwo.maxLength') || ''}
value={max}
min={1}
onChange={e => setMax(parseInt(e.target.value.replace(/^0+/, ''), 10))}
/>
<div className='absolute top-2.5 right-2.5 text-text-tertiary system-sm-regular'>Tokens</div>
</div>
<Input
type="number"
className='h-9'
placeholder={t('datasetCreation.stepTwo.maxLength') || ''}
value={max}
max={4000}
min={1}
onChange={e => setMax(Number.parseInt(e.target.value.replace(/^0+/, ''), 10))}
/>
</div>
</div>
<div className={s.formRow}>
@ -683,17 +688,14 @@ const StepTwo = ({
}
/>
</div>
<div className='relative w-full'>
<input
type="number"
className={s.input}
placeholder={t('datasetCreation.stepTwo.overlap') || ''}
value={overlap}
min={1}
onChange={e => setOverlap(parseInt(e.target.value.replace(/^0+/, ''), 10))}
/>
<div className='absolute top-2.5 right-2.5 text-text-tertiary system-sm-regular'>Tokens</div>
</div>
<Input
type="number"
className='h-9'
placeholder={t('datasetCreation.stepTwo.overlap') || ''}
value={overlap}
min={1}
onChange={e => setOverlap(Number.parseInt(e.target.value.replace(/^0+/, ''), 10))}
/>
</div>
</div>
<div className={s.formRow}>

View File

@ -6,7 +6,7 @@ import cn from '@/utils/classnames'
import Popover from '@/app/components/base/popover'
import { languages } from '@/i18n/language'
export type ILanguageSelectProps = {
export interface ILanguageSelectProps {
currentLanguage: string
onSelect: (language: string) => void
disabled?: boolean
@ -24,7 +24,7 @@ const LanguageSelect: FC<ILanguageSelectProps> = ({
disabled={disabled}
htmlContent={
<div className='w-full py-1'>
{languages.filter(language => language.supported).map(({ prompt_name, name }) => (
{languages.filter(language => language.supported).map(({ prompt_name }) => (
<div
key={prompt_name}
className='py-2 px-3 mx-1 flex items-center gap-2 hover:bg-gray-100 rounded-lg cursor-pointer text-gray-700 text-sm'

View File

@ -0,0 +1,40 @@
'use client'
import type { FC } from 'react'
import React from 'react'
import cn from '@/utils/classnames'
import Checkbox from '@/app/components/base/checkbox'
import Tooltip from '@/app/components/base/tooltip'
interface Props {
className?: string
isChecked: boolean
onChange: (isChecked: boolean) => void
label: string
labelClassName?: string
tooltip?: string
}
const CheckboxWithLabel: FC<Props> = ({
className = '',
isChecked,
onChange,
label,
labelClassName,
tooltip,
}) => {
return (
<label className={cn(className, 'flex items-center h-7 space-x-2')}>
<Checkbox checked={isChecked} onCheck={() => onChange(!isChecked)} />
<div className={cn(labelClassName, 'text-sm font-normal text-gray-800')}>{label}</div>
{tooltip && (
<Tooltip
popupContent={
<div className='w-[200px]'>{tooltip}</div>
}
triggerClassName='ml-0.5 w-4 h-4'
/>
)}
</label>
)
}
export default React.memo(CheckboxWithLabel)

View File

@ -0,0 +1,30 @@
'use client'
import type { FC } from 'react'
import React from 'react'
import cn from '@/utils/classnames'
import { AlertTriangle } from '@/app/components/base/icons/src/vender/solid/alertsAndFeedback'
interface Props {
className?: string
title: string
errorMsg?: string
}
const ErrorMessage: FC<Props> = ({
className,
title,
errorMsg,
}) => {
return (
<div className={cn(className, 'py-2 px-4 border-t border-gray-200 bg-[#FFFAEB]')}>
<div className='flex items-center h-5'>
<AlertTriangle className='mr-2 w-4 h-4 text-[#F79009]' />
<div className='text-sm font-medium text-[#DC6803]'>{title}</div>
</div>
{errorMsg && (
<div className='mt-1 pl-6 leading-[18px] text-xs font-normal text-gray-700'>{errorMsg}</div>
)}
</div>
)
}
export default React.memo(ErrorMessage)

View File

@ -0,0 +1,54 @@
'use client'
import type { FC } from 'react'
import React from 'react'
import Input from './input'
import cn from '@/utils/classnames'
import Tooltip from '@/app/components/base/tooltip'
interface Props {
className?: string
label: string
labelClassName?: string
value: string | number
onChange: (value: string | number) => void
isRequired?: boolean
placeholder?: string
isNumber?: boolean
tooltip?: string
}
const Field: FC<Props> = ({
className,
label,
labelClassName,
value,
onChange,
isRequired = false,
placeholder = '',
isNumber = false,
tooltip,
}) => {
return (
<div className={cn(className)}>
<div className='flex py-[7px]'>
<div className={cn(labelClassName, 'flex items-center h-[18px] text-[13px] font-medium text-gray-900')}>{label} </div>
{isRequired && <span className='ml-0.5 text-xs font-semibold text-[#D92D20]'>*</span>}
{tooltip && (
<Tooltip
popupContent={
<div className='w-[200px]'>{tooltip}</div>
}
triggerClassName='ml-0.5 w-4 h-4'
/>
)}
</div>
<Input
value={value}
onChange={onChange}
placeholder={placeholder}
isNumber={isNumber}
/>
</div>
)
}
export default React.memo(Field)

View File

@ -0,0 +1,58 @@
'use client'
import type { FC } from 'react'
import React, { useCallback } from 'react'
interface Props {
value: string | number
onChange: (value: string | number) => void
placeholder?: string
isNumber?: boolean
}
const MIN_VALUE = 0
const Input: FC<Props> = ({
value,
onChange,
placeholder = '',
isNumber = false,
}) => {
const handleChange = useCallback((e: React.ChangeEvent<HTMLInputElement>) => {
const value = e.target.value
if (isNumber) {
let numberValue = Number.parseInt(value, 10) // integer only
if (isNaN(numberValue)) {
onChange('')
return
}
if (numberValue < MIN_VALUE)
numberValue = MIN_VALUE
onChange(numberValue)
return
}
onChange(value)
}, [isNumber, onChange])
const otherOption = (() => {
if (isNumber) {
return {
min: MIN_VALUE,
}
}
return {
}
})()
return (
<input
type={isNumber ? 'number' : 'text'}
{...otherOption}
value={value}
onChange={handleChange}
className='flex h-9 w-full py-1 px-2 rounded-lg text-xs leading-normal bg-gray-100 caret-primary-600 hover:bg-gray-100 focus:ring-1 focus:ring-inset focus:ring-gray-200 focus-visible:outline-none focus:bg-gray-50 placeholder:text-gray-400'
placeholder={placeholder}
/>
)
}
export default React.memo(Input)

View File

@ -0,0 +1,55 @@
'use client'
import { useBoolean } from 'ahooks'
import type { FC } from 'react'
import React, { useEffect } from 'react'
import { useTranslation } from 'react-i18next'
import cn from '@/utils/classnames'
import { Settings04 } from '@/app/components/base/icons/src/vender/line/general'
import { ChevronRight } from '@/app/components/base/icons/src/vender/line/arrows'
const I18N_PREFIX = 'datasetCreation.stepOne.website'
interface Props {
className?: string
children: React.ReactNode
controlFoldOptions?: number
}
const OptionsWrap: FC<Props> = ({
className = '',
children,
controlFoldOptions,
}) => {
const { t } = useTranslation()
const [fold, {
toggle: foldToggle,
setTrue: foldHide,
}] = useBoolean(false)
useEffect(() => {
if (controlFoldOptions)
foldHide()
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [controlFoldOptions])
return (
<div className={cn(className, !fold ? 'mb-0' : 'mb-3')}>
<div
className='flex justify-between items-center h-[26px] py-1 cursor-pointer select-none'
onClick={foldToggle}
>
<div className='flex items-center text-gray-700'>
<Settings04 className='mr-1 w-4 h-4' />
<div className='text-[13px] font-semibold text-gray-800 uppercase'>{t(`${I18N_PREFIX}.options`)}</div>
</div>
<ChevronRight className={cn(!fold && 'rotate-90', 'w-4 h-4 text-gray-500')} />
</div>
{!fold && (
<div className='mb-4'>
{children}
</div>
)}
</div>
)
}
export default React.memo(OptionsWrap)

View File

@ -0,0 +1,48 @@
'use client'
import type { FC } from 'react'
import React, { useCallback, useState } from 'react'
import { useTranslation } from 'react-i18next'
import Input from './input'
import Button from '@/app/components/base/button'
const I18N_PREFIX = 'datasetCreation.stepOne.website'
interface Props {
isRunning: boolean
onRun: (url: string) => void
}
const UrlInput: FC<Props> = ({
isRunning,
onRun,
}) => {
const { t } = useTranslation()
const [url, setUrl] = useState('')
const handleUrlChange = useCallback((url: string | number) => {
setUrl(url as string)
}, [])
const handleOnRun = useCallback(() => {
if (isRunning)
return
onRun(url)
}, [isRunning, onRun, url])
return (
<div className='flex items-center justify-between'>
<Input
value={url}
onChange={handleUrlChange}
placeholder='https://docs.dify.ai'
/>
<Button
variant='primary'
onClick={handleOnRun}
className='ml-2'
loading={isRunning}
>
{!isRunning ? t(`${I18N_PREFIX}.run`) : ''}
</Button>
</div>
)
}
export default React.memo(UrlInput)

View File

@ -0,0 +1,40 @@
'use client'
import type { FC } from 'react'
import React, { useCallback } from 'react'
import { useTranslation } from 'react-i18next'
import cn from '@/utils/classnames'
import type { CrawlResultItem as CrawlResultItemType } from '@/models/datasets'
import Checkbox from '@/app/components/base/checkbox'
interface Props {
payload: CrawlResultItemType
isChecked: boolean
isPreview: boolean
onCheckChange: (checked: boolean) => void
onPreview: () => void
}
const CrawledResultItem: FC<Props> = ({
isPreview,
payload,
isChecked,
onCheckChange,
onPreview,
}) => {
const { t } = useTranslation()
const handleCheckChange = useCallback(() => {
onCheckChange(!isChecked)
}, [isChecked, onCheckChange])
return (
<div className={cn(isPreview ? 'border-[#D1E0FF] bg-primary-50 shadow-xs' : 'group hover:bg-gray-100', 'rounded-md px-2 py-[5px] cursor-pointer border border-transparent')}>
<div className='flex items-center h-5'>
<Checkbox className='group-hover:border-2 group-hover:border-primary-600 mr-2 shrink-0' checked={isChecked} onCheck={handleCheckChange} />
<div className='grow w-0 truncate text-sm font-medium text-gray-700' title={payload.title}>{payload.title}</div>
<div onClick={onPreview} className='hidden group-hover:flex items-center h-6 px-2 text-xs rounded-md font-medium text-gray-500 uppercase hover:bg-gray-50'>{t('datasetCreation.stepOne.website.preview')}</div>
</div>
<div className='mt-0.5 truncate pl-6 leading-[18px] text-xs font-normal text-gray-500' title={payload.source_url}>{payload.source_url}</div>
</div>
)
}
export default React.memo(CrawledResultItem)

View File

@ -0,0 +1,87 @@
'use client'
import type { FC } from 'react'
import React, { useCallback } from 'react'
import { useTranslation } from 'react-i18next'
import CheckboxWithLabel from './base/checkbox-with-label'
import CrawledResultItem from './crawled-result-item'
import cn from '@/utils/classnames'
import type { CrawlResultItem } from '@/models/datasets'
const I18N_PREFIX = 'datasetCreation.stepOne.website'
interface Props {
className?: string
list: CrawlResultItem[]
checkedList: CrawlResultItem[]
onSelectedChange: (selected: CrawlResultItem[]) => void
onPreview: (payload: CrawlResultItem) => void
usedTime: number
}
const CrawledResult: FC<Props> = ({
className = '',
list,
checkedList,
onSelectedChange,
onPreview,
usedTime,
}) => {
const { t } = useTranslation()
const isCheckAll = checkedList.length === list.length
const handleCheckedAll = useCallback(() => {
if (!isCheckAll)
onSelectedChange(list)
else
onSelectedChange([])
}, [isCheckAll, list, onSelectedChange])
const handleItemCheckChange = useCallback((item: CrawlResultItem) => {
return (checked: boolean) => {
if (checked)
onSelectedChange([...checkedList, item])
else
onSelectedChange(checkedList.filter(checkedItem => checkedItem.source_url !== item.source_url))
}
}, [checkedList, onSelectedChange])
const [previewIndex, setPreviewIndex] = React.useState<number>(-1)
const handlePreview = useCallback((index: number) => {
return () => {
setPreviewIndex(index)
onPreview(list[index])
}
}, [list, onPreview])
return (
<div className={cn(className, 'border-t border-gray-200')}>
<div className='flex items-center justify-between h-[34px] px-4 bg-gray-50 shadow-xs border-b-[0.5px] border-black/8 text-xs font-normal text-gray-700'>
<CheckboxWithLabel
isChecked={isCheckAll}
onChange={handleCheckedAll} label={isCheckAll ? t(`${I18N_PREFIX}.resetAll`) : t(`${I18N_PREFIX}.selectAll`)}
labelClassName='!font-medium'
/>
<div>{t(`${I18N_PREFIX}.scrapTimeInfo`, {
total: list.length,
time: usedTime.toFixed(1),
})}</div>
</div>
<div className='p-2'>
{list.map((item, index) => (
<CrawledResultItem
key={item.source_url}
isPreview={index === previewIndex}
onPreview={handlePreview(index)}
payload={item}
isChecked={checkedList.some(checkedItem => checkedItem.source_url === item.source_url)}
onCheckChange={handleItemCheckChange(item)}
/>
))}
</div>
</div>
)
}
export default React.memo(CrawledResult)

View File

@ -0,0 +1,37 @@
'use client'
import type { FC } from 'react'
import React from 'react'
import { useTranslation } from 'react-i18next'
import cn from '@/utils/classnames'
import { RowStruct } from '@/app/components/base/icons/src/public/other'
interface Props {
className?: string
crawledNum: number
totalNum: number
}
const Crawling: FC<Props> = ({
className = '',
crawledNum,
totalNum,
}) => {
const { t } = useTranslation()
return (
<div className={cn(className, 'border-t border-gray-200')}>
<div className='flex items-center h-[34px] px-4 bg-gray-50 shadow-xs border-b-[0.5px] border-black/8 text-xs font-normal text-gray-700'>
{t('datasetCreation.stepOne.website.totalPageScraped')} {crawledNum}/{totalNum}
</div>
<div className='p-2'>
{['', '', '', ''].map((item, index) => (
<div className='py-[5px]' key={index}>
<RowStruct />
</div>
))}
</div>
</div>
)
}
export default React.memo(Crawling)

View File

@ -0,0 +1,24 @@
import type { CrawlResultItem } from '@/models/datasets'
const result: CrawlResultItem[] = [
{
title: 'Start the frontend Docker container separately',
markdown: 'Markdown 1',
description: 'Description 1',
source_url: 'https://example.com/1',
},
{
title: 'Advanced Tool Integration',
markdown: 'Markdown 2',
description: 'Description 2',
source_url: 'https://example.com/2',
},
{
title: 'Local Source Code Start | English | Dify',
markdown: 'Markdown 3',
description: 'Description 3',
source_url: 'https://example.com/3',
},
]
export default result