merge main

web/app/components/app-sidebar/basic.tsx

@@ -1,4 +1,5 @@
 import React from 'react'
+import { useTranslation } from 'react-i18next'
 import AppIcon from '../base/app-icon'
 import Tooltip from '@/app/components/base/tooltip'
 
@@ -6,6 +7,7 @@ export type IAppBasicProps = {
   iconType?: 'app' | 'api' | 'dataset' | 'webapp' | 'notion'
   icon?: string
   icon_background?: string | null
+  isExternal?: boolean
   name: string
   type: string | React.ReactNode
   hoverTip?: string
@@ -52,7 +54,9 @@ const ICON_MAP = {
   notion: <AppIcon innerIcon={NotionSvg} className='!border-[0.5px] !border-indigo-100 !bg-white' />,
 }
 
-export default function AppBasic({ icon, icon_background, name, type, hoverTip, textStyle, mode = 'expand', iconType = 'app' }: IAppBasicProps) {
+export default function AppBasic({ icon, icon_background, name, isExternal, type, hoverTip, textStyle, mode = 'expand', iconType = 'app' }: IAppBasicProps) {
+  const { t } = useTranslation()
+
   return (
     <div className="flex items-start p-1">
       {icon && icon_background && iconType === 'app' && (
@@ -83,6 +87,7 @@ export default function AppBasic({ icon, icon_background, name, type, hoverTip,
           }
         </div>
         <div className={`text-xs font-normal text-gray-500 group-hover:text-gray-700 break-all ${textStyle?.extra ?? ''}`}>{type}</div>
+        <div className='text-text-tertiary system-2xs-medium-uppercase'>{isExternal ? t('dataset.externalTag') : ''}</div>
       </div>}
     </div>
   )

web/app/components/app-sidebar/index.tsx

@@ -15,6 +15,7 @@ export type IAppDetailNavProps = {
   iconType?: 'app' | 'dataset' | 'notion'
   title: string
   desc: string
+  isExternal?: boolean
   icon: string
   icon_background: string
   navigation: Array<{
@@ -26,7 +27,7 @@ export type IAppDetailNavProps = {
   extraInfo?: (modeState: string) => React.ReactNode
 }
 
-const AppDetailNav = ({ title, desc, icon, icon_background, navigation, extraInfo, iconType = 'app' }: IAppDetailNavProps) => {
+const AppDetailNav = ({ title, desc, isExternal, icon, icon_background, navigation, extraInfo, iconType = 'app' }: IAppDetailNavProps) => {
   const { appSidebarExpand, setAppSiderbarExpand } = useAppStore(useShallow(state => ({
     appSidebarExpand: state.appSidebarExpand,
     setAppSiderbarExpand: state.setAppSiderbarExpand,
@@ -70,6 +71,7 @@ const AppDetailNav = ({ title, desc, icon, icon_background, navigation, extraInf
             icon_background={icon_background}
             name={title}
             type={desc}
+            isExternal={isExternal}
           />
         )}
       </div>

web/app/components/app/annotation/index.tsx

@@ -3,6 +3,7 @@ import type { FC } from 'react'
 import React, { useEffect, useState } from 'react'
 import { useTranslation } from 'react-i18next'
 import { Pagination } from 'react-headless-pagination'
+import { useDebounce } from 'ahooks'
 import { ArrowLeftIcon, ArrowRightIcon } from '@heroicons/react/24/outline'
 import Toast from '../../base/toast'
 import Filter from './filter'
@@ -67,10 +68,11 @@ const Annotation: FC<Props> = ({
 
   const [queryParams, setQueryParams] = useState<QueryParam>({})
   const [currPage, setCurrPage] = React.useState<number>(0)
+  const debouncedQueryParams = useDebounce(queryParams, { wait: 500 })
   const query = {
     page: currPage + 1,
     limit: APP_PAGE_LIMIT,
-    keyword: queryParams.keyword || '',
+    keyword: debouncedQueryParams.keyword || '',
   }
 
   const [controlUpdateList, setControlUpdateList] = useState(Date.now())
@@ -232,8 +234,8 @@ const Annotation: FC<Props> = ({
             middlePagesSiblingCount={1}
             setCurrentPage={setCurrPage}
             totalPages={Math.ceil(total / APP_PAGE_LIMIT)}
-            truncatableClassName="w-8 px-0.5 text-center"
-            truncatableText="..."
+            truncableClassName="w-8 px-0.5 text-center"
+            truncableText="..."
           >
             <Pagination.PrevButton
               disabled={currPage === 0}

web/app/components/app/configuration/dataset-config/card-item/item.tsx

@@ -5,6 +5,7 @@ import {
   RiDeleteBinLine,
   RiEditLine,
 } from '@remixicon/react'
+import { useTranslation } from 'react-i18next'
 import SettingsModal from '../settings-modal'
 import type { DataSet } from '@/models/datasets'
 import { DataSourceType } from '@/models/datasets'
@@ -33,6 +34,7 @@ const Item: FC<ItemProps> = ({
   const isMobile = media === MediaType.mobile
   const [showSettingsModal, setShowSettingsModal] = useState(false)
   const { formatIndexingTechniqueAndMethod } = useKnowledge()
+  const { t } = useTranslation()
 
   const handleSave = (newDataset: DataSet) => {
     onSave(newDataset)
@@ -65,9 +67,11 @@ const Item: FC<ItemProps> = ({
       <div className='grow'>
         <div className='flex items-center h-[18px]'>
           <div className='grow text-[13px] font-medium text-gray-800 truncate' title={config.name}>{config.name}</div>
-          <Badge
-            text={formatIndexingTechniqueAndMethod(config.indexing_technique, config.retrieval_model_dict?.search_method)}
-          />
+          {config.provider === 'external'
+            ? <Badge text={t('dataset.externalTag')}></Badge>
+            : <Badge
+              text={formatIndexingTechniqueAndMethod(config.indexing_technique, config.retrieval_model_dict?.search_method)}
+            />}
         </div>
       </div>
       <div className='hidden rounded-lg group-hover:flex items-center justify-end absolute right-0 top-0 bottom-0 pr-2 w-[124px] bg-gradient-to-r from-white/50 to-white to-50%'>

web/app/components/app/configuration/dataset-config/params-config/config-content.tsx

@@ -1,6 +1,6 @@
 'use client'
 
-import { memo, useEffect, useMemo } from 'react'
+import { memo, useCallback, useEffect, useMemo } from 'react'
 import type { FC } from 'react'
 import { useTranslation } from 'react-i18next'
 import WeightedScore from './weighted-score'
@@ -11,7 +11,7 @@ import type {
   DatasetConfigs,
 } from '@/models/debug'
 import ModelSelector from '@/app/components/header/account-setting/model-provider-page/model-selector'
-import { useModelListAndDefaultModelAndCurrentProviderAndModel } from '@/app/components/header/account-setting/model-provider-page/hooks'
+import { useCurrentProviderAndModel, useModelListAndDefaultModelAndCurrentProviderAndModel } from '@/app/components/header/account-setting/model-provider-page/hooks'
 import type { ModelConfig } from '@/app/components/workflow/types'
 import ModelParameterModal from '@/app/components/header/account-setting/model-provider-page/model-parameter-modal'
 import Tooltip from '@/app/components/base/tooltip'
@@ -23,6 +23,7 @@ import { RerankingModeEnum } from '@/models/datasets'
 import cn from '@/utils/classnames'
 import { useSelectedDatasetsMode } from '@/app/components/workflow/nodes/knowledge-retrieval/hooks'
 import Switch from '@/app/components/base/switch'
+import Toast from '@/app/components/base/toast'
 
 type Props = {
   datasetConfigs: DatasetConfigs
@@ -60,6 +61,24 @@ const ConfigContent: FC<Props> = ({
     modelList: rerankModelList,
     defaultModel: rerankDefaultModel,
   } = useModelListAndDefaultModelAndCurrentProviderAndModel(ModelTypeEnum.rerank)
+
+  const {
+    currentModel,
+  } = useCurrentProviderAndModel(
+    rerankModelList,
+    rerankDefaultModel
+      ? {
+        ...rerankDefaultModel,
+        provider: rerankDefaultModel.provider.provider,
+      }
+      : undefined,
+  )
+
+  const handleDisabledSwitchClick = useCallback(() => {
+    if (!currentModel)
+      Toast.notify({ type: 'error', message: t('workflow.errorMsg.rerankModelRequired') })
+  }, [currentModel, rerankDefaultModel, t])
+
   const rerankModel = (() => {
     if (datasetConfigs.reranking_model?.reranking_provider_name) {
       return {
@@ -174,6 +193,20 @@ const ConfigContent: FC<Props> = ({
               </div>
             )
           }
+          {
+            selectedDatasetsMode.mixtureInternalAndExternal && (
+              <div className='mt-4 system-xs-medium text-text-warning'>
+                {t('dataset.mixtureInternalAndExternalTip')}
+              </div>
+            )
+          }
+          {
+            selectedDatasetsMode.allExternal && (
+              <div className='mt-4 system-xs-medium text-text-warning'>
+                {t('dataset.allExternalTip')}
+              </div>
+            )
+          }
           {
             selectedDatasetsMode.mixtureHighQualityAndEconomic
             && (
@@ -217,27 +250,33 @@ const ConfigContent: FC<Props> = ({
                 <div className='flex items-center'>
                   {
                     selectedDatasetsMode.allEconomic && (
-                      <Switch
-                        size='md'
-                        defaultValue={showRerankModel}
-                        onChange={(v) => {
-                          onChange({
-                            ...datasetConfigs,
-                            reranking_enable: v,
-                          })
-                        }}
-                      />
+                      <div
+                        className='flex items-center'
+                        onClick={handleDisabledSwitchClick}
+                      >
+                        <Switch
+                          size='md'
+                          defaultValue={currentModel ? showRerankModel : false}
+                          disabled={!currentModel}
+                          onChange={(v) => {
+                            onChange({
+                              ...datasetConfigs,
+                              reranking_enable: v,
+                            })
+                          }}
+                        />
+                      </div>
                     )
                   }
-                  <div className='ml-2 leading-[32px] text-[13px] font-medium text-gray-900'>{t('common.modelProvider.rerankModel.key')}</div>
+                  <div className='leading-[32px] ml-1 text-text-secondary system-sm-semibold'>{t('common.modelProvider.rerankModel.key')}</div>
                   <Tooltip
                     popupContent={
                       <div className="w-[200px]">
                         {t('common.modelProvider.rerankModel.tip')}
                       </div>
                     }
-                    popupClassName='ml-0.5'
-                    triggerClassName='ml-0.5 w-3.5 h-3.5'
+                    popupClassName='ml-1'
+                    triggerClassName='ml-1 w-4 h-4'
                   />
                 </div>
                 <div>

web/app/components/app/configuration/dataset-config/params-config/index.tsx

@@ -39,13 +39,26 @@ const ParamsConfig = ({
   useEffect(() => {
     const {
       allEconomic,
+      allHighQuality,
+      allHighQualityFullTextSearch,
+      allHighQualityVectorSearch,
+      allExternal,
+      mixtureHighQualityAndEconomic,
+      inconsistentEmbeddingModel,
+      mixtureInternalAndExternal,
     } = getSelectedDatasetsMode(selectedDatasets)
     const { datasets, retrieval_model, score_threshold_enabled, ...restConfigs } = datasetConfigs
     let rerankEnable = restConfigs.reranking_enable
 
-    if (allEconomic && !restConfigs.reranking_model?.reranking_provider_name && rerankEnable === undefined)
+    if ((allEconomic && !restConfigs.reranking_model?.reranking_provider_name && rerankEnable === undefined) || allExternal)
       rerankEnable = false
 
+    if (allEconomic || allHighQuality || allHighQualityFullTextSearch || allHighQualityVectorSearch || (allExternal && selectedDatasets.length === 1))
+      setRerankSettingModalOpen(false)
+
+    if (mixtureHighQualityAndEconomic || inconsistentEmbeddingModel || mixtureInternalAndExternal || (allExternal && selectedDatasets.length > 1))
+      setRerankSettingModalOpen(true)
+
     setTempDataSetConfigs({
       ...getMultipleRetrievalConfig({
         top_k: restConfigs.top_k,

web/app/components/app/configuration/dataset-config/select-dataset/index.tsx

@@ -47,7 +47,7 @@ const SelectDataSet: FC<ISelectDataSetProps> = ({
         const { data, has_more } = await fetchDatasets({ url: '/datasets', params: { page } })
         setPage(getPage() + 1)
         setIsNoMore(!has_more)
-        const newList = [...(datasets || []), ...data.filter(item => item.indexing_technique)]
+        const newList = [...(datasets || []), ...data.filter(item => item.indexing_technique || item.provider === 'external')]
         setDataSets(newList)
         setLoaded(true)
         if (!selected.find(item => !item.name))
@@ -145,6 +145,11 @@ const SelectDataSet: FC<ISelectDataSetProps> = ({
                     />
                   )
                 }
+                {
+                  item.provider === 'external' && (
+                    <Badge text={t('dataset.externalTag')} />
+                  )
+                }
               </div>
             ))}
           </div>

web/app/components/app/configuration/dataset-config/settings-modal/index.tsx

@@ -5,8 +5,10 @@ import { useTranslation } from 'react-i18next'
 import { isEqual } from 'lodash-es'
 import { RiCloseLine } from '@remixicon/react'
 import { BookOpenIcon } from '@heroicons/react/24/outline'
+import { ApiConnectionMod } from '@/app/components/base/icons/src/vender/solid/development'
 import cn from '@/utils/classnames'
 import IndexMethodRadio from '@/app/components/datasets/settings/index-method-radio'
+import Divider from '@/app/components/base/divider'
 import Button from '@/app/components/base/button'
 import type { DataSet } from '@/models/datasets'
 import { useToastContext } from '@/app/components/base/toast'
@@ -14,6 +16,7 @@ import { updateDatasetSetting } from '@/service/datasets'
 import { useAppContext } from '@/context/app-context'
 import { useModalContext } from '@/context/modal-context'
 import type { RetrievalConfig } from '@/types/app'
+import RetrievalSettings from '@/app/components/datasets/external-knowledge-base/create/RetrievalSettings'
 import RetrievalMethodConfig from '@/app/components/datasets/common/retrieval-method-config'
 import EconomicalRetrievalMethodConfig from '@/app/components/datasets/common/economical-retrieval-method-config'
 import { ensureRerankModelSelected, isReRankModelSelected } from '@/app/components/datasets/common/check-rerank-model'
@@ -56,7 +59,10 @@ const SettingsModal: FC<SettingsModalProps> = ({
   const { t } = useTranslation()
   const { notify } = useToastContext()
   const ref = useRef(null)
-
+  const isExternal = currentDataset.provider === 'external'
+  const [topK, setTopK] = useState(currentDataset?.external_retrieval_model.top_k ?? 2)
+  const [scoreThreshold, setScoreThreshold] = useState(currentDataset?.external_retrieval_model.score_threshold ?? 0.5)
+  const [scoreThresholdEnabled, setScoreThresholdEnabled] = useState(currentDataset?.external_retrieval_model.score_threshold_enabled ?? false)
   const { setShowAccountSettingModal } = useModalContext()
   const [loading, setLoading] = useState(false)
   const { isCurrentWorkspaceDatasetOperator } = useAppContext()
@@ -73,6 +79,15 @@ const SettingsModal: FC<SettingsModalProps> = ({
   const [isHideChangedTip, setIsHideChangedTip] = useState(false)
   const isRetrievalChanged = !isEqual(retrievalConfig, localeCurrentDataset?.retrieval_model_dict) || indexMethod !== localeCurrentDataset?.indexing_technique
 
+  const handleSettingsChange = (data: { top_k?: number; score_threshold?: number; score_threshold_enabled?: boolean }) => {
+    if (data.top_k !== undefined)
+      setTopK(data.top_k)
+    if (data.score_threshold !== undefined)
+      setScoreThreshold(data.score_threshold)
+    if (data.score_threshold_enabled !== undefined)
+      setScoreThresholdEnabled(data.score_threshold_enabled)
+  }
+
   const handleSave = async () => {
     if (loading)
       return
@@ -113,6 +128,15 @@ const SettingsModal: FC<SettingsModalProps> = ({
           },
           embedding_model: localeCurrentDataset.embedding_model,
           embedding_model_provider: localeCurrentDataset.embedding_model_provider,
+          ...(isExternal && {
+            external_knowledge_id: currentDataset!.external_knowledge_info.external_knowledge_id,
+            external_knowledge_api_id: currentDataset!.external_knowledge_info.external_knowledge_api_id,
+            external_retrieval_model: {
+              top_k: topK,
+              score_threshold: scoreThreshold,
+              score_threshold_enabled: scoreThresholdEnabled,
+            },
+          }),
         },
       } as any
       if (permission === 'partial_members') {
@@ -178,7 +202,7 @@ const SettingsModal: FC<SettingsModalProps> = ({
       }}>
         <div className={cn(rowClass, 'items-center')}>
           <div className={labelClass}>
-            {t('datasetSettings.form.name')}
+            <div className='text-text-secondary system-sm-semibold'>{t('datasetSettings.form.name')}</div>
           </div>
           <input
             value={localeCurrentDataset.name}
@@ -189,7 +213,7 @@ const SettingsModal: FC<SettingsModalProps> = ({
         </div>
         <div className={cn(rowClass)}>
           <div className={labelClass}>
-            {t('datasetSettings.form.desc')}
+            <div className='text-text-secondary system-sm-semibold'>{t('datasetSettings.form.desc')}</div>
           </div>
           <div className='w-full'>
             <textarea
@@ -206,7 +230,7 @@ const SettingsModal: FC<SettingsModalProps> = ({
         </div>
         <div className={rowClass}>
           <div className={labelClass}>
-            <div>{t('datasetSettings.form.permissions')}</div>
+            <div className='text-text-secondary system-sm-semibold'>{t('datasetSettings.form.permissions')}</div>
           </div>
           <div className='w-full'>
             <PermissionSelector
@@ -219,24 +243,25 @@ const SettingsModal: FC<SettingsModalProps> = ({
             />
           </div>
         </div>
-        <div className="w-full h-0 border-b-[0.5px] border-b-gray-200 my-2"></div>
-        <div className={cn(rowClass)}>
-          <div className={labelClass}>
-            {t('datasetSettings.form.indexMethod')}
+        {currentDataset && currentDataset.indexing_technique && (
+          <div className={cn(rowClass)}>
+            <div className={labelClass}>
+              <div className='text-text-secondary system-sm-semibold'>{t('datasetSettings.form.indexMethod')}</div>
+            </div>
+            <div className='grow'>
+              <IndexMethodRadio
+                disable={!localeCurrentDataset?.embedding_available}
+                value={indexMethod}
+                onChange={v => setIndexMethod(v!)}
+                itemClassName='sm:!w-[280px]'
+              />
+            </div>
           </div>
-          <div className='grow'>
-            <IndexMethodRadio
-              disable={!localeCurrentDataset?.embedding_available}
-              value={indexMethod}
-              onChange={v => setIndexMethod(v!)}
-              itemClassName='sm:!w-[280px]'
-            />
-          </div>
-        </div>
+        )}
         {indexMethod === 'high_quality' && (
           <div className={cn(rowClass)}>
             <div className={labelClass}>
-              {t('datasetSettings.form.embeddingModel')}
+              <div className='text-text-secondary system-sm-semibold'>{t('datasetSettings.form.embeddingModel')}</div>
             </div>
             <div className='w-full'>
               <div className='w-full h-9 rounded-lg bg-gray-100 opacity-60'>
@@ -258,32 +283,75 @@ const SettingsModal: FC<SettingsModalProps> = ({
         )}
 
         {/* Retrieval Method Config */}
-        <div className={rowClass}>
-          <div className={cn(labelClass, 'w-auto min-w-[168px]')}>
-            <div>
-              <div>{t('datasetSettings.form.retrievalSetting.title')}</div>
-              <div className='leading-[18px] text-xs font-normal text-gray-500'>
-                <a target='_blank' rel='noopener noreferrer' href='https://docs.dify.ai/guides/knowledge-base/create-knowledge-and-upload-documents#id-4-retrieval-settings' className='text-[#155eef]'>{t('datasetSettings.form.retrievalSetting.learnMore')}</a>
-                {t('datasetSettings.form.retrievalSetting.description')}
+        {currentDataset?.provider === 'external'
+          ? <>
+            <div className={rowClass}><Divider/></div>
+            <div className={rowClass}>
+              <div className={labelClass}>
+                <div className='text-text-secondary system-sm-semibold'>{t('datasetSettings.form.retrievalSetting.title')}</div>
+              </div>
+              <RetrievalSettings
+                topK={topK}
+                scoreThreshold={scoreThreshold}
+                scoreThresholdEnabled={scoreThresholdEnabled}
+                onChange={handleSettingsChange}
+                isInRetrievalSetting={true}
+              />
+            </div>
+            <div className={rowClass}><Divider/></div>
+            <div className={rowClass}>
+              <div className={labelClass}>
+                <div className='text-text-secondary system-sm-semibold'>{t('datasetSettings.form.externalKnowledgeAPI')}</div>
+              </div>
+              <div className='w-full max-w-[480px]'>
+                <div className='flex h-full px-3 py-2 items-center gap-1 rounded-lg bg-components-input-bg-normal'>
+                  <ApiConnectionMod className='w-4 h-4 text-text-secondary' />
+                  <div className='overflow-hidden text-text-secondary text-ellipsis system-sm-medium'>
+                    {currentDataset?.external_knowledge_info.external_knowledge_api_name}
+                  </div>
+                  <div className='text-text-tertiary system-xs-regular'>·</div>
+                  <div className='text-text-tertiary system-xs-regular'>{currentDataset?.external_knowledge_info.external_knowledge_api_endpoint}</div>
+                </div>
               </div>
             </div>
-          </div>
-          <div>
-            {indexMethod === 'high_quality'
-              ? (
-                <RetrievalMethodConfig
-                  value={retrievalConfig}
-                  onChange={setRetrievalConfig}
-                />
-              )
-              : (
-                <EconomicalRetrievalMethodConfig
-                  value={retrievalConfig}
-                  onChange={setRetrievalConfig}
-                />
-              )}
-          </div>
-        </div>
+            <div className={rowClass}>
+              <div className={labelClass}>
+                <div className='text-text-secondary system-sm-semibold'>{t('datasetSettings.form.externalKnowledgeID')}</div>
+              </div>
+              <div className='w-full max-w-[480px]'>
+                <div className='flex h-full px-3 py-2 items-center gap-1 rounded-lg bg-components-input-bg-normal'>
+                  <div className='text-text-tertiary system-xs-regular'>{currentDataset?.external_knowledge_info.external_knowledge_id}</div>
+                </div>
+              </div>
+            </div>
+            <div className={rowClass}><Divider/></div>
+          </>
+          : <div className={rowClass}>
+            <div className={cn(labelClass, 'w-auto min-w-[168px]')}>
+              <div>
+                <div className='text-text-secondary system-sm-semibold'>{t('datasetSettings.form.retrievalSetting.title')}</div>
+                <div className='leading-[18px] text-xs font-normal text-gray-500'>
+                  <a target='_blank' rel='noopener noreferrer' href='https://docs.dify.ai/guides/knowledge-base/create-knowledge-and-upload-documents#id-4-retrieval-settings' className='text-[#155eef]'>{t('datasetSettings.form.retrievalSetting.learnMore')}</a>
+                  {t('datasetSettings.form.retrievalSetting.description')}
+                </div>
+              </div>
+            </div>
+            <div>
+              {indexMethod === 'high_quality'
+                ? (
+                  <RetrievalMethodConfig
+                    value={retrievalConfig}
+                    onChange={setRetrievalConfig}
+                  />
+                )
+                : (
+                  <EconomicalRetrievalMethodConfig
+                    value={retrievalConfig}
+                    onChange={setRetrievalConfig}
+                  />
+                )}
+            </div>
+          </div>}
       </div>
       {isRetrievalChanged && !isHideChangedTip && (
         <div className='absolute z-10 left-[30px] right-[30px] bottom-[76px] flex h-10 items-center px-3 rounded-lg border border-[#FEF0C7] bg-[#FFFAEB] shadow-lg justify-between'>

web/app/components/app/log/index.tsx

@@ -4,6 +4,7 @@ import React, { useState } from 'react'
 import useSWR from 'swr'
 import { usePathname } from 'next/navigation'
 import { Pagination } from 'react-headless-pagination'
+import { useDebounce } from 'ahooks'
 import { omit } from 'lodash-es'
 import dayjs from 'dayjs'
 import { ArrowLeftIcon, ArrowRightIcon } from '@heroicons/react/24/outline'
@@ -59,6 +60,7 @@ const Logs: FC<ILogsProps> = ({ appDetail }) => {
     sort_by: '-created_at',
   })
   const [currPage, setCurrPage] = React.useState<number>(0)
+  const debouncedQueryParams = useDebounce(queryParams, { wait: 500 })
 
   // Get the app type first
   const isChatMode = appDetail.mode !== 'completion'
@@ -66,14 +68,14 @@ const Logs: FC<ILogsProps> = ({ appDetail }) => {
   const query = {
     page: currPage + 1,
     limit: APP_PAGE_LIMIT,
-    ...(queryParams.period !== 'all'
+    ...(debouncedQueryParams.period !== 'all'
       ? {
-        start: dayjs().subtract(queryParams.period as number, 'day').startOf('day').format('YYYY-MM-DD HH:mm'),
+        start: dayjs().subtract(debouncedQueryParams.period as number, 'day').startOf('day').format('YYYY-MM-DD HH:mm'),
         end: dayjs().endOf('day').format('YYYY-MM-DD HH:mm'),
       }
       : {}),
-    ...(isChatMode ? { sort_by: queryParams.sort_by } : {}),
-    ...omit(queryParams, ['period']),
+    ...(isChatMode ? { sort_by: debouncedQueryParams.sort_by } : {}),
+    ...omit(debouncedQueryParams, ['period']),
   }
 
   const getWebAppType = (appType: AppMode) => {
@@ -119,8 +121,8 @@ const Logs: FC<ILogsProps> = ({ appDetail }) => {
             middlePagesSiblingCount={1}
             setCurrentPage={setCurrPage}
             totalPages={Math.ceil(total / APP_PAGE_LIMIT)}
-            truncatableClassName="w-8 px-0.5 text-center"
-            truncatableText="..."
+            truncableClassName="w-8 px-0.5 text-center"
+            truncableText="..."
           >
             <Pagination.PrevButton
               disabled={currPage === 0}

web/app/components/app/log/list.tsx

@@ -299,10 +299,14 @@ function DetailPanel<T extends ChatConversationFullDetailResponse | CompletionCo
     }))
   }, [items])
 
+  const fetchInitiated = useRef(false)
+
   useEffect(() => {
-    if (appDetail?.id && detail.id && appDetail?.mode !== 'completion')
+    if (appDetail?.id && detail.id && appDetail?.mode !== 'completion' && !fetchInitiated.current) {
+      fetchInitiated.current = true
       fetchData()
-  }, [appDetail?.id, detail.id, appDetail?.mode])
+    }
+  }, [appDetail?.id, detail.id, appDetail?.mode, fetchData])
 
   const isChatMode = appDetail?.mode !== 'completion'
   const isAdvanced = appDetail?.mode === 'advanced-chat'

web/app/components/app/overview/embedded/index.tsx

@@ -52,6 +52,10 @@ const OPTION_MAP = {
   #dify-chatbot-bubble-button {
     background-color: ${primaryColor} !important;
   }
+  #dify-chatbot-bubble-window {
+    width: 24rem !important;
+    height: 40rem !important;
+  }
 </style>`,
   },
   chromePlugin: {

web/app/components/app/workflow-log/index.tsx

@@ -4,6 +4,7 @@ import React, { useState } from 'react'
 import useSWR from 'swr'
 import { usePathname } from 'next/navigation'
 import { Pagination } from 'react-headless-pagination'
+import { useDebounce } from 'ahooks'
 import { ArrowLeftIcon, ArrowRightIcon } from '@heroicons/react/24/outline'
 import { Trans, useTranslation } from 'react-i18next'
 import Link from 'next/link'
@@ -51,12 +52,13 @@ const Logs: FC<ILogsProps> = ({ appDetail }) => {
   const { t } = useTranslation()
   const [queryParams, setQueryParams] = useState<QueryParam>({ status: 'all' })
   const [currPage, setCurrPage] = React.useState<number>(0)
+  const debouncedQueryParams = useDebounce(queryParams, { wait: 500 })
 
   const query = {
     page: currPage + 1,
     limit: APP_PAGE_LIMIT,
-    ...(queryParams.status !== 'all' ? { status: queryParams.status } : {}),
-    ...(queryParams.keyword ? { keyword: queryParams.keyword } : {}),
+    ...(debouncedQueryParams.status !== 'all' ? { status: debouncedQueryParams.status } : {}),
+    ...(debouncedQueryParams.keyword ? { keyword: debouncedQueryParams.keyword } : {}),
   }
 
   const getWebAppType = (appType: AppMode) => {
@@ -93,8 +95,8 @@ const Logs: FC<ILogsProps> = ({ appDetail }) => {
             middlePagesSiblingCount={1}
             setCurrentPage={setCurrPage}
             totalPages={Math.ceil(total / APP_PAGE_LIMIT)}
-            truncatableClassName="w-8 px-0.5 text-center"
-            truncatableText="..."
+            truncableClassName="w-8 px-0.5 text-center"
+            truncableText="..."
           >
             <Pagination.PrevButton
               disabled={currPage === 0}

web/app/components/base/chat/chat/citation/popup.tsx

@@ -100,7 +100,7 @@ const Popup: FC<PopupProps> = ({
                             />
                             <Tooltip
                               text={t('common.chat.citation.vectorHash')}
-                              data={source.index_node_hash.substring(0, 7)}
+                              data={source.index_node_hash?.substring(0, 7)}
                               icon={<BezierCurve03 className='mr-1 w-3 h-3' />}
                             />
                             {

web/app/components/base/chat/embedded-chatbot/theme/theme-context.ts

@@ -26,7 +26,7 @@ export class Theme {
     if (this.chatColorTheme !== null && this.chatColorTheme !== '') {
       this.primaryColor = this.chatColorTheme ?? '#1C64F2'
       this.backgroundHeaderColorStyle = `backgroundColor: ${this.primaryColor}`
-      this.backgroundButtonDefaultColorStyle = `backgroundColor: ${this.primaryColor}`
+      this.backgroundButtonDefaultColorStyle = `backgroundColor: ${this.primaryColor}; color: ${this.colorFontOnHeaderStyle};`
       this.roundedBackgroundColorStyle = `backgroundColor: ${hexToRGBA(this.primaryColor, 0.05)}`
       this.chatBubbleColorStyle = `backgroundColor: ${hexToRGBA(this.primaryColor, 0.15)}`
       this.chatBubbleColor = `${hexToRGBA(this.primaryColor, 0.15)}`

web/app/components/base/corner-label/index.tsx

@@ -0,0 +1,21 @@
+import { Corner } from '../icons/src/vender/solid/shapes'
+import cn from '@/utils/classnames'
+
+type CornerLabelProps = {
+  label: string
+  className?: string
+  labelClassName?: string
+}
+
+const CornerLabel: React.FC<CornerLabelProps> = ({ label, className, labelClassName }) => {
+  return (
+    <div className={cn('group/corner-label inline-flex items-start', className)}>
+      <Corner className='w-[13px] h-5 text-background-section group-hover/corner-label:text-background-section-burn' />
+      <div className={cn('flex py-1 pr-2 items-center gap-0.5 bg-background-section group-hover/corner-label:bg-background-section-burn', labelClassName)}>
+        <div className='text-text-tertiary system-2xs-medium-uppercase'>{label}</div>
+      </div>
+    </div>
+  )
+}
+
+export default CornerLabel

web/app/components/base/ga/index.tsx

@@ -1,6 +1,7 @@
 import type { FC } from 'react'
 import React from 'react'
 import Script from 'next/script'
+import { headers } from 'next/headers'
 import { IS_CE_EDITION } from '@/config'
 
 export enum GaType {
@@ -23,9 +24,16 @@ const GA: FC<IGAProps> = ({
   if (IS_CE_EDITION)
     return null
 
+  const nonce = process.env.NODE_ENV === 'production' ? headers().get('x-nonce') : ''
+
   return (
     <>
-      <Script strategy="beforeInteractive" async src={`https://www.googletagmanager.com/gtag/js?id=${gaIdMaps[gaType]}`}></Script>
+      <Script
+        strategy="beforeInteractive"
+        async
+        src={`https://www.googletagmanager.com/gtag/js?id=${gaIdMaps[gaType]}`}
+        nonce={nonce!}
+      ></Script>
       <Script
         id="ga-init"
         dangerouslySetInnerHTML={{
@@ -36,6 +44,7 @@ gtag('js', new Date());
 gtag('config', '${gaIdMaps[gaType]}');
           `,
         }}
+        nonce={nonce!}
       >
       </Script>
     </>

web/app/components/base/icons/assets/vender/solid/development/api-connection-mod.svg

@@ -0,0 +1,5 @@
+<svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg">
+<g id="Icon L">
+<path id="Vector" fill-rule="evenodd" clip-rule="evenodd" d="M7.99996 3.33333C5.42263 3.33333 3.33329 5.42267 3.33329 8C3.33329 10.5773 5.42263 12.6667 7.99996 12.6667C9.72643 12.6667 11.2348 11.7295 12.0427 10.3329C12.227 10.0141 12.6349 9.90523 12.9536 10.0896C13.2723 10.274 13.3812 10.6818 13.1968 11.0005C12.1604 12.7921 10.2216 14 7.99996 14C4.91159 14 2.36821 11.6666 2.03658 8.66667H1.33329C0.965103 8.66667 0.666626 8.36819 0.666626 8C0.666626 7.63181 0.965103 7.33333 1.33329 7.33333H2.03658C2.36821 4.33337 4.91159 2 7.99996 2C10.2216 2 12.1604 3.20785 13.1968 4.99952C13.3812 5.31823 13.2723 5.72605 12.9536 5.91041C12.6349 6.09477 12.227 5.98585 12.0427 5.66714C11.2348 4.27054 9.72643 3.33333 7.99996 3.33333ZM7.99996 6C6.89539 6 5.99996 6.89543 5.99996 8C5.99996 9.10455 6.89539 10 7.99996 10C9.1045 10 9.99996 9.10454 9.99996 8C9.99996 6.89543 9.10451 6 7.99996 6ZM4.66663 8C4.66663 6.15905 6.15901 4.66667 7.99996 4.66667C9.61257 4.66667 10.9578 5.81184 11.2666 7.33333H14.6666C15.0348 7.33333 15.3333 7.63181 15.3333 8C15.3333 8.36819 15.0348 8.66667 14.6666 8.66667H11.2666C10.9578 10.1881 9.61257 11.3333 7.99996 11.3333C6.159 11.3333 4.66663 9.84092 4.66663 8Z" fill="#354052"/>
+</g>
+</svg>

web/app/components/base/icons/assets/vender/solid/shapes/corner.svg

@@ -0,0 +1,3 @@
+<svg width="13" height="20" viewBox="0 0 13 20" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path id="Shape" d="M0 0H13V20C9.98017 20 7.26458 18.1615 6.14305 15.3576L0 0Z" fill="#F9FAFB"/>
+</svg>

web/app/components/base/icons/src/vender/solid/development/ApiConnectionMod.json

@@ -0,0 +1,38 @@
+{
+	"icon": {
+		"type": "element",
+		"isRootNode": true,
+		"name": "svg",
+		"attributes": {
+			"width": "16",
+			"height": "16",
+			"viewBox": "0 0 16 16",
+			"fill": "none",
+			"xmlns": "http://www.w3.org/2000/svg"
+		},
+		"children": [
+			{
+				"type": "element",
+				"name": "g",
+				"attributes": {
+					"id": "Icon L"
+				},
+				"children": [
+					{
+						"type": "element",
+						"name": "path",
+						"attributes": {
+							"id": "Vector",
+							"fill-rule": "evenodd",
+							"clip-rule": "evenodd",
+							"d": "M7.99996 3.33333C5.42263 3.33333 3.33329 5.42267 3.33329 8C3.33329 10.5773 5.42263 12.6667 7.99996 12.6667C9.72643 12.6667 11.2348 11.7295 12.0427 10.3329C12.227 10.0141 12.6349 9.90523 12.9536 10.0896C13.2723 10.274 13.3812 10.6818 13.1968 11.0005C12.1604 12.7921 10.2216 14 7.99996 14C4.91159 14 2.36821 11.6666 2.03658 8.66667H1.33329C0.965103 8.66667 0.666626 8.36819 0.666626 8C0.666626 7.63181 0.965103 7.33333 1.33329 7.33333H2.03658C2.36821 4.33337 4.91159 2 7.99996 2C10.2216 2 12.1604 3.20785 13.1968 4.99952C13.3812 5.31823 13.2723 5.72605 12.9536 5.91041C12.6349 6.09477 12.227 5.98585 12.0427 5.66714C11.2348 4.27054 9.72643 3.33333 7.99996 3.33333ZM7.99996 6C6.89539 6 5.99996 6.89543 5.99996 8C5.99996 9.10455 6.89539 10 7.99996 10C9.1045 10 9.99996 9.10454 9.99996 8C9.99996 6.89543 9.10451 6 7.99996 6ZM4.66663 8C4.66663 6.15905 6.15901 4.66667 7.99996 4.66667C9.61257 4.66667 10.9578 5.81184 11.2666 7.33333H14.6666C15.0348 7.33333 15.3333 7.63181 15.3333 8C15.3333 8.36819 15.0348 8.66667 14.6666 8.66667H11.2666C10.9578 10.1881 9.61257 11.3333 7.99996 11.3333C6.159 11.3333 4.66663 9.84092 4.66663 8Z",
+							"fill": "currentColor"
+						},
+						"children": []
+					}
+				]
+			}
+		]
+	},
+	"name": "ApiConnectionMod"
+}

web/app/components/base/icons/src/vender/solid/development/ApiConnectionMod.tsx

@@ -0,0 +1,16 @@
+// GENERATE BY script
+// DON NOT EDIT IT MANUALLY
+
+import * as React from 'react'
+import data from './ApiConnectionMod.json'
+import IconBase from '@/app/components/base/icons/IconBase'
+import type { IconBaseProps, IconData } from '@/app/components/base/icons/IconBase'
+
+const Icon = React.forwardRef<React.MutableRefObject<SVGElement>, Omit<IconBaseProps, 'data'>>((
+  props,
+  ref,
+) => <IconBase {...props} ref={ref} data={data as IconData} />)
+
+Icon.displayName = 'ApiConnectionMod'
+
+export default Icon

web/app/components/base/icons/src/vender/solid/development/index.ts

@@ -1,3 +1,4 @@
+export { default as ApiConnectionMod } from './ApiConnectionMod'
 export { default as ApiConnection } from './ApiConnection'
 export { default as BarChartSquare02 } from './BarChartSquare02'
 export { default as Container } from './Container'

web/app/components/base/icons/src/vender/solid/shapes/Corner.json

@@ -0,0 +1,27 @@
+{
+	"icon": {
+		"type": "element",
+		"isRootNode": true,
+		"name": "svg",
+		"attributes": {
+			"width": "13",
+			"height": "20",
+			"viewBox": "0 0 13 20",
+			"fill": "none",
+			"xmlns": "http://www.w3.org/2000/svg"
+		},
+		"children": [
+			{
+				"type": "element",
+				"name": "path",
+				"attributes": {
+					"id": "Shape",
+					"d": "M0 0H13V20C9.98017 20 7.26458 18.1615 6.14305 15.3576L0 0Z",
+					"fill": "currentColor"
+				},
+				"children": []
+			}
+		]
+	},
+	"name": "Corner"
+}

web/app/components/base/icons/src/vender/solid/shapes/Corner.tsx

@@ -0,0 +1,16 @@
+// GENERATE BY script
+// DON NOT EDIT IT MANUALLY
+
+import * as React from 'react'
+import data from './Corner.json'
+import IconBase from '@/app/components/base/icons/IconBase'
+import type { IconBaseProps, IconData } from '@/app/components/base/icons/IconBase'
+
+const Icon = React.forwardRef<React.MutableRefObject<SVGElement>, Omit<IconBaseProps, 'data'>>((
+  props,
+  ref,
+) => <IconBase {...props} ref={ref} data={data as IconData} />)
+
+Icon.displayName = 'Corner'
+
+export default Icon

web/app/components/base/icons/src/vender/solid/shapes/index.ts

@@ -1,2 +1,3 @@
+export { default as Corner } from './Corner'
 export { default as Star04 } from './Star04'
 export { default as Star06 } from './Star06'

web/app/components/base/markdown.tsx

@@ -245,7 +245,7 @@ export function Markdown(props: { content: string; className?: string }) {
   return (
     <div className={cn(props.className, 'markdown-body')}>
       <ReactMarkdown
-        remarkPlugins={[[RemarkGfm, RemarkMath, { singleDollarTextMath: false }], RemarkBreaks]}
+        remarkPlugins={[RemarkGfm, RemarkMath, RemarkBreaks]}
         rehypePlugins={[
           RehypeKatex,
           RehypeRaw as any,

web/app/components/base/modal/index.css

@@ -3,5 +3,5 @@
 }
 
 .modal-panel {
-  @apply w-full max-w-md transform rounded-2xl bg-white p-6 text-left align-middle shadow-xl transition-all;
+  @apply w-full max-w-[480px] transform rounded-2xl bg-white p-6 text-left align-middle shadow-xl transition-all;
 }

web/app/components/base/param-item/index.tsx

@@ -37,6 +37,7 @@ const ParamItem: FC<Props> = ({ className, id, name, noTooltip, tip, step = 0.1,
           <span className="mx-1 text-gray-900 text-[13px] leading-[18px] font-medium">{name}</span>
           {!noTooltip && (
             <Tooltip
+              triggerClassName='w-4 h-4 shrink-0'
               popupContent={<div className="w-[200px]">{tip}</div>}
             />
           )}

web/app/components/base/prompt-editor/plugins/workflow-variable-block/workflow-variable-block-replacement-block.tsx

@@ -12,7 +12,7 @@ import type { WorkflowVariableBlockType } from '../../types'
 import { CustomTextNode } from '../custom-text/node'
 import { $createWorkflowVariableBlockNode } from './node'
 import { WorkflowVariableBlockNode } from './index'
-import { VAR_REGEX as REGEX } from '@/config'
+import { VAR_REGEX as REGEX, resetReg } from '@/config'
 
 const WorkflowVariableBlockReplacementBlock = ({
   workflowNodesMap,
@@ -48,11 +48,12 @@ const WorkflowVariableBlockReplacementBlock = ({
   }, [])
 
   const transformListener = useCallback((textNode: any) => {
+    resetReg()
     return decoratorTransform(textNode, getMatch, createWorkflowVariableBlockNode)
   }, [createWorkflowVariableBlockNode, getMatch])
 
   useEffect(() => {
-    REGEX.lastIndex = 0
+    resetReg()
     return mergeRegister(
       editor.registerNodeTransform(CustomTextNode, transformListener),
     )

web/app/components/base/select/index.tsx

@@ -87,7 +87,7 @@ const Select: FC<ISelectProps> = ({
         <div className='group text-gray-800'>
           {allowSearch
             ? <Combobox.Input
-              className={`w-full rounded-lg border-0 ${bgClassName} py-1.5 pl-3 pr-10 shadow-sm sm:text-sm sm:leading-6 focus-visible:outline-none focus-visible:bg-gray-200 group-hover:bg-gray-200 cursor-not-allowed`}
+              className={`w-full rounded-lg border-0 ${bgClassName} py-1.5 pl-3 pr-10 shadow-sm sm:text-sm sm:leading-6 focus-visible:outline-none focus-visible:bg-gray-200 group-hover:bg-gray-200 ${disabled ? 'cursor-not-allowed' : 'cursor-pointer'}`}
               onChange={(event) => {
                 if (!disabled)
                   setQuery(event.target.value)

web/app/components/base/topbar/index.tsx

@@ -1,16 +0,0 @@
-'use client'
-
-import { AppProgressBar as ProgressBar } from 'next-nprogress-bar'
-
-const Topbar = () => {
-  return (
-    <>
-      <ProgressBar
-        height='2px'
-        color="#1C64F2FF"
-        options={{ showSpinner: false }}
-        shallowRouting />
-    </>)
-}
-
-export default Topbar

web/app/components/datasets/common/retrieval-param-config/index.tsx

@@ -1,6 +1,6 @@
 'use client'
 import type { FC } from 'react'
-import React from 'react'
+import React, { useCallback } from 'react'
 import { useTranslation } from 'react-i18next'
 
 import cn from '@/utils/classnames'
@@ -11,7 +11,7 @@ import Switch from '@/app/components/base/switch'
 import Tooltip from '@/app/components/base/tooltip'
 import type { RetrievalConfig } from '@/types/app'
 import ModelSelector from '@/app/components/header/account-setting/model-provider-page/model-selector'
-import { useModelListAndDefaultModel } from '@/app/components/header/account-setting/model-provider-page/hooks'
+import { useCurrentProviderAndModel, useModelListAndDefaultModel } from '@/app/components/header/account-setting/model-provider-page/hooks'
 import { ModelTypeEnum } from '@/app/components/header/account-setting/model-provider-page/declarations'
 import {
   DEFAULT_WEIGHTED_SCORE,
@@ -19,6 +19,7 @@ import {
   WeightedScoreEnum,
 } from '@/models/datasets'
 import WeightedScore from '@/app/components/app/configuration/dataset-config/params-config/weighted-score'
+import Toast from '@/app/components/base/toast'
 
 type Props = {
   type: RETRIEVE_METHOD
@@ -38,6 +39,24 @@ const RetrievalParamConfig: FC<Props> = ({
     defaultModel: rerankDefaultModel,
     modelList: rerankModelList,
   } = useModelListAndDefaultModel(ModelTypeEnum.rerank)
+
+  const {
+    currentModel,
+  } = useCurrentProviderAndModel(
+    rerankModelList,
+    rerankDefaultModel
+      ? {
+        ...rerankDefaultModel,
+        provider: rerankDefaultModel.provider.provider,
+      }
+      : undefined,
+  )
+
+  const handleDisabledSwitchClick = useCallback(() => {
+    if (!currentModel)
+      Toast.notify({ type: 'error', message: t('workflow.errorMsg.rerankModelRequired') })
+  }, [currentModel, rerankDefaultModel, t])
+
   const isHybridSearch = type === RETRIEVE_METHOD.hybrid
 
   const rerankModel = (() => {
@@ -99,16 +118,22 @@ const RetrievalParamConfig: FC<Props> = ({
         <div>
           <div className='flex h-8 items-center text-[13px] font-medium text-gray-900 space-x-2'>
             {canToggleRerankModalEnable && (
-              <Switch
-                size='md'
-                defaultValue={value.reranking_enable}
-                onChange={(v) => {
-                  onChange({
-                    ...value,
-                    reranking_enable: v,
-                  })
-                }}
-              />
+              <div
+                className='flex items-center'
+                onClick={handleDisabledSwitchClick}
+              >
+                <Switch
+                  size='md'
+                  defaultValue={currentModel ? value.reranking_enable : false}
+                  onChange={(v) => {
+                    onChange({
+                      ...value,
+                      reranking_enable: v,
+                    })
+                  }}
+                  disabled={!currentModel}
+                />
+              </div>
             )}
             <div className='flex items-center'>
               <span className='mr-0.5'>{t('common.modelProvider.rerankModel.key')}</span>

web/app/components/datasets/create/index.tsx

@@ -11,7 +11,7 @@ import { DataSourceType } from '@/models/datasets'
 import type { CrawlOptions, CrawlResultItem, DataSet, FileItem, createDocumentResponse } from '@/models/datasets'
 import { fetchDataSource } from '@/service/common'
 import { fetchDatasetDetail } from '@/service/datasets'
-import type { NotionPage } from '@/models/common'
+import { DataSourceProvider, type NotionPage } from '@/models/common'
 import { useModalContext } from '@/context/modal-context'
 import { useDefaultModel } from '@/app/components/header/account-setting/model-provider-page/hooks'
 
@@ -26,6 +26,7 @@ const DEFAULT_CRAWL_OPTIONS: CrawlOptions = {
   excludes: '',
   limit: 10,
   max_depth: '',
+  use_sitemap: true,
 }
 
 const DatasetUpdateForm = ({ datasetId }: DatasetUpdateFormProps) => {
@@ -51,7 +52,8 @@ const DatasetUpdateForm = ({ datasetId }: DatasetUpdateFormProps) => {
   const updateFileList = (preparedFiles: FileItem[]) => {
     setFiles(preparedFiles)
   }
-  const [fireCrawlJobId, setFireCrawlJobId] = useState('')
+  const [websiteCrawlProvider, setWebsiteCrawlProvider] = useState<DataSourceProvider>(DataSourceProvider.fireCrawl)
+  const [websiteCrawlJobId, setWebsiteCrawlJobId] = useState('')
 
   const updateFile = (fileItem: FileItem, progress: number, list: FileItem[]) => {
     const targetIndex = list.findIndex(file => file.fileID === fileItem.fileID)
@@ -137,7 +139,8 @@ const DatasetUpdateForm = ({ datasetId }: DatasetUpdateFormProps) => {
             onStepChange={nextStep}
             websitePages={websitePages}
             updateWebsitePages={setWebsitePages}
-            onFireCrawlJobIdChange={setFireCrawlJobId}
+            onWebsiteCrawlProviderChange={setWebsiteCrawlProvider}
+            onWebsiteCrawlJobIdChange={setWebsiteCrawlJobId}
             crawlOptions={crawlOptions}
             onCrawlOptionsChange={setCrawlOptions}
           />
@@ -151,7 +154,8 @@ const DatasetUpdateForm = ({ datasetId }: DatasetUpdateFormProps) => {
           files={fileList.map(file => file.file)}
           notionPages={notionPages}
           websitePages={websitePages}
-          fireCrawlJobId={fireCrawlJobId}
+          websiteCrawlProvider={websiteCrawlProvider}
+          websiteCrawlJobId={websiteCrawlJobId}
           onStepChange={changeStep}
           updateIndexingTypeCache={updateIndexingTypeCache}
           updateResultCache={updateResultCache}

web/app/components/datasets/create/step-one/index.tsx

@@ -10,7 +10,7 @@ import WebsitePreview from '../website/preview'
 import s from './index.module.css'
 import cn from '@/utils/classnames'
 import type { CrawlOptions, CrawlResultItem, FileItem } from '@/models/datasets'
-import type { NotionPage } from '@/models/common'
+import type { DataSourceProvider, NotionPage } from '@/models/common'
 import { DataSourceType } from '@/models/datasets'
 import Button from '@/app/components/base/button'
 import { NotionPageSelector } from '@/app/components/base/notion-page-selector'
@@ -33,7 +33,8 @@ type IStepOneProps = {
   changeType: (type: DataSourceType) => void
   websitePages?: CrawlResultItem[]
   updateWebsitePages: (value: CrawlResultItem[]) => void
-  onFireCrawlJobIdChange: (jobId: string) => void
+  onWebsiteCrawlProviderChange: (provider: DataSourceProvider) => void
+  onWebsiteCrawlJobIdChange: (jobId: string) => void
   crawlOptions: CrawlOptions
   onCrawlOptionsChange: (payload: CrawlOptions) => void
 }
@@ -69,7 +70,8 @@ const StepOne = ({
   updateNotionPages,
   websitePages = [],
   updateWebsitePages,
-  onFireCrawlJobIdChange,
+  onWebsiteCrawlProviderChange,
+  onWebsiteCrawlJobIdChange,
   crawlOptions,
   onCrawlOptionsChange,
 }: IStepOneProps) => {
@@ -229,7 +231,8 @@ const StepOne = ({
                   onPreview={setCurrentWebsite}
                   checkedCrawlResult={websitePages}
                   onCheckedCrawlResultChange={updateWebsitePages}
-                  onJobIdChange={onFireCrawlJobIdChange}
+                  onCrawlProviderChange={onWebsiteCrawlProviderChange}
+                  onJobIdChange={onWebsiteCrawlJobIdChange}
                   crawlOptions={crawlOptions}
                   onCrawlOptionsChange={onCrawlOptionsChange}
                 />

web/app/components/datasets/create/step-two/index.tsx

@@ -33,6 +33,7 @@ import { ensureRerankModelSelected, isReRankModelSelected } from '@/app/componen
 import Toast from '@/app/components/base/toast'
 import { formatNumber } from '@/utils/format'
 import type { NotionPage } from '@/models/common'
+import { DataSourceProvider } from '@/models/common'
 import { DataSourceType, DocForm } from '@/models/datasets'
 import NotionIcon from '@/app/components/base/notion-icon'
 import Switch from '@/app/components/base/switch'
@@ -63,7 +64,8 @@ type StepTwoProps = {
   notionPages?: NotionPage[]
   websitePages?: CrawlResultItem[]
   crawlOptions?: CrawlOptions
-  fireCrawlJobId?: string
+  websiteCrawlProvider?: DataSourceProvider
+  websiteCrawlJobId?: string
   onStepChange?: (delta: number) => void
   updateIndexingTypeCache?: (type: string) => void
   updateResultCache?: (res: createDocumentResponse) => void
@@ -94,7 +96,8 @@ const StepTwo = ({
   notionPages = [],
   websitePages = [],
   crawlOptions,
-  fireCrawlJobId = '',
+  websiteCrawlProvider = DataSourceProvider.fireCrawl,
+  websiteCrawlJobId = '',
   onStepChange,
   updateIndexingTypeCache,
   updateResultCache,
@@ -260,8 +263,8 @@ const StepTwo = ({
 
   const getWebsiteInfo = () => {
     return {
-      provider: 'firecrawl',
-      job_id: fireCrawlJobId,
+      provider: websiteCrawlProvider,
+      job_id: websiteCrawlJobId,
       urls: websitePages.map(page => page.source_url),
       only_main_content: crawlOptions?.only_main_content,
     }

web/app/components/datasets/create/website/base/checkbox-with-label.tsx

@@ -3,6 +3,7 @@ import type { FC } from 'react'
 import React from 'react'
 import cn from '@/utils/classnames'
 import Checkbox from '@/app/components/base/checkbox'
+import Tooltip from '@/app/components/base/tooltip'
 
 type Props = {
   className?: string
@@ -10,6 +11,7 @@ type Props = {
   onChange: (isChecked: boolean) => void
   label: string
   labelClassName?: string
+  tooltip?: string
 }
 
 const CheckboxWithLabel: FC<Props> = ({
@@ -18,11 +20,20 @@ const CheckboxWithLabel: FC<Props> = ({
   onChange,
   label,
   labelClassName,
+  tooltip,
 }) => {
   return (
     <label className={cn(className, 'flex items-center h-7 space-x-2')}>
       <Checkbox checked={isChecked} onCheck={() => onChange(!isChecked)} />
       <div className={cn(labelClassName, 'text-sm font-normal text-gray-800')}>{label}</div>
+      {tooltip && (
+        <Tooltip
+          popupContent={
+            <div className='w-[200px]'>{tooltip}</div>
+          }
+          triggerClassName='ml-0.5 w-4 h-4'
+        />
+      )}
     </label>
   )
 }

web/app/components/datasets/create/website/base/crawled-result.tsx

@@ -2,7 +2,7 @@
 import type { FC } from 'react'
 import React, { useCallback } from 'react'
 import { useTranslation } from 'react-i18next'
-import CheckboxWithLabel from './base/checkbox-with-label'
+import CheckboxWithLabel from './checkbox-with-label'
 import CrawledResultItem from './crawled-result-item'
 import cn from '@/utils/classnames'
 import type { CrawlResultItem } from '@/models/datasets'

web/app/components/datasets/create/website/firecrawl/index.tsx

@@ -2,13 +2,13 @@
 import type { FC } from 'react'
 import React, { useCallback, useEffect, useState } from 'react'
 import { useTranslation } from 'react-i18next'
+import UrlInput from '../base/url-input'
+import OptionsWrap from '../base/options-wrap'
+import CrawledResult from '../base/crawled-result'
+import Crawling from '../base/crawling'
+import ErrorMessage from '../base/error-message'
 import Header from './header'
-import UrlInput from './base/url-input'
-import OptionsWrap from './base/options-wrap'
 import Options from './options'
-import CrawledResult from './crawled-result'
-import Crawling from './crawling'
-import ErrorMessage from './base/error-message'
 import cn from '@/utils/classnames'
 import { useModalContext } from '@/context/modal-context'
 import type { CrawlOptions, CrawlResultItem } from '@/models/datasets'

web/app/components/datasets/create/website/firecrawl/options.tsx

@@ -2,8 +2,8 @@
 import type { FC } from 'react'
 import React, { useCallback } from 'react'
 import { useTranslation } from 'react-i18next'
-import CheckboxWithLabel from './base/checkbox-with-label'
-import Field from './base/field'
+import CheckboxWithLabel from '../base/checkbox-with-label'
+import Field from '../base/field'
 import cn from '@/utils/classnames'
 import type { CrawlOptions } from '@/models/datasets'
 

web/app/components/datasets/create/website/index.module.css

@@ -0,0 +1,6 @@
+.jinaLogo {
+  @apply w-4 h-4 bg-center bg-no-repeat inline-block;
+  background-color: #F5FAFF;
+  background-image: url(../assets/jina.png);
+  background-size: 16px;
+}

web/app/components/datasets/create/website/index.tsx

@@ -1,8 +1,12 @@
 'use client'
 import type { FC } from 'react'
 import React, { useCallback, useEffect, useState } from 'react'
+import { useTranslation } from 'react-i18next'
+import s from './index.module.css'
 import NoData from './no-data'
 import Firecrawl from './firecrawl'
+import JinaReader from './jina-reader'
+import cn from '@/utils/classnames'
 import { useModalContext } from '@/context/modal-context'
 import type { CrawlOptions, CrawlResultItem } from '@/models/datasets'
 import { fetchDataSources } from '@/service/datasets'
@@ -12,6 +16,7 @@ type Props = {
   onPreview: (payload: CrawlResultItem) => void
   checkedCrawlResult: CrawlResultItem[]
   onCheckedCrawlResultChange: (payload: CrawlResultItem[]) => void
+  onCrawlProviderChange: (provider: DataSourceProvider) => void
   onJobIdChange: (jobId: string) => void
   crawlOptions: CrawlOptions
   onCrawlOptionsChange: (payload: CrawlOptions) => void
@@ -21,17 +26,32 @@ const Website: FC<Props> = ({
   onPreview,
   checkedCrawlResult,
   onCheckedCrawlResultChange,
+  onCrawlProviderChange,
   onJobIdChange,
   crawlOptions,
   onCrawlOptionsChange,
 }) => {
+  const { t } = useTranslation()
   const { setShowAccountSettingModal } = useModalContext()
   const [isLoaded, setIsLoaded] = useState(false)
-  const [isSetFirecrawlApiKey, setIsSetFirecrawlApiKey] = useState(false)
+  const [selectedProvider, setSelectedProvider] = useState<DataSourceProvider>(DataSourceProvider.jinaReader)
+  const [sources, setSources] = useState<DataSourceItem[]>([])
+
+  useEffect(() => {
+    onCrawlProviderChange(selectedProvider)
+  }, [selectedProvider, onCrawlProviderChange])
+
   const checkSetApiKey = useCallback(async () => {
     const res = await fetchDataSources() as any
-    const isFirecrawlSet = res.sources.some((item: DataSourceItem) => item.provider === DataSourceProvider.fireCrawl)
-    setIsSetFirecrawlApiKey(isFirecrawlSet)
+    setSources(res.sources)
+
+    // If users have configured one of the providers, select it.
+    const availableProviders = res.sources.filter((item: DataSourceItem) =>
+      [DataSourceProvider.jinaReader, DataSourceProvider.fireCrawl].includes(item.provider),
+    )
+
+    if (availableProviders.length > 0)
+      setSelectedProvider(availableProviders[0].provider)
   }, [])
 
   useEffect(() => {
@@ -52,20 +72,66 @@ const Website: FC<Props> = ({
 
   return (
     <div>
-      {isSetFirecrawlApiKey
-        ? (
-          <Firecrawl
-            onPreview={onPreview}
-            checkedCrawlResult={checkedCrawlResult}
-            onCheckedCrawlResultChange={onCheckedCrawlResultChange}
-            onJobIdChange={onJobIdChange}
-            crawlOptions={crawlOptions}
-            onCrawlOptionsChange={onCrawlOptionsChange}
-          />
-        )
-        : (
-          <NoData onConfig={handleOnConfig} />
-        )}
+      <div className="mb-4">
+        <div className="font-medium text-gray-700 mb-2 h-6">
+          {t('datasetCreation.stepOne.website.chooseProvider')}
+        </div>
+        <div className="flex space-x-2">
+          <button
+            className={`px-4 py-2 text-sm font-medium rounded-md flex items-center justify-center ${
+              selectedProvider === DataSourceProvider.jinaReader
+                ? 'bg-primary-50 text-primary-600'
+                : 'bg-gray-100 text-gray-600 hover:bg-gray-200'
+            }`}
+            onClick={() => setSelectedProvider(DataSourceProvider.jinaReader)}
+          >
+            <span className={cn(s.jinaLogo, 'mr-2')} />
+            <span>Jina Reader</span>
+          </button>
+          <button
+            className={`px-4 py-2 text-sm font-medium rounded-md ${
+              selectedProvider === DataSourceProvider.fireCrawl
+                ? 'bg-primary-50 text-primary-600'
+                : 'bg-gray-100 text-gray-600 hover:bg-gray-200'
+            }`}
+            onClick={() => setSelectedProvider(DataSourceProvider.fireCrawl)}
+          >
+            🔥 Firecrawl
+          </button>
+        </div>
+      </div>
+
+      {
+        selectedProvider === DataSourceProvider.fireCrawl
+          ? sources.find(source => source.provider === DataSourceProvider.fireCrawl)
+            ? (
+              <Firecrawl
+                onPreview={onPreview}
+                checkedCrawlResult={checkedCrawlResult}
+                onCheckedCrawlResultChange={onCheckedCrawlResultChange}
+                onJobIdChange={onJobIdChange}
+                crawlOptions={crawlOptions}
+                onCrawlOptionsChange={onCrawlOptionsChange}
+              />
+            )
+            : (
+              <NoData onConfig={handleOnConfig} provider={selectedProvider} />
+            )
+          : sources.find(source => source.provider === DataSourceProvider.jinaReader)
+            ? (
+              <JinaReader
+                onPreview={onPreview}
+                checkedCrawlResult={checkedCrawlResult}
+                onCheckedCrawlResultChange={onCheckedCrawlResultChange}
+                onJobIdChange={onJobIdChange}
+                crawlOptions={crawlOptions}
+                onCrawlOptionsChange={onCrawlOptionsChange}
+              />
+            )
+            : (
+              <NoData onConfig={handleOnConfig} provider={selectedProvider} />
+            )
+      }
     </div>
   )
 }

web/app/components/datasets/create/website/jina-reader/header.tsx

@@ -0,0 +1,42 @@
+'use client'
+import type { FC } from 'react'
+import React from 'react'
+import { useTranslation } from 'react-i18next'
+import { Settings01 } from '@/app/components/base/icons/src/vender/line/general'
+import { BookOpen01 } from '@/app/components/base/icons/src/vender/line/education'
+
+const I18N_PREFIX = 'datasetCreation.stepOne.website'
+
+type Props = {
+  onSetting: () => void
+}
+
+const Header: FC<Props> = ({
+  onSetting,
+}) => {
+  const { t } = useTranslation()
+
+  return (
+    <div className='flex h-6 items-center justify-between'>
+      <div className='flex items-center'>
+        <div className='text-base font-medium text-gray-700'>{t(`${I18N_PREFIX}.jinaReaderTitle`)}</div>
+        <div className='ml-2 mr-1 w-px h-3.5 bg-gray-200'></div>
+        <div
+          className='p-1 rounded-md hover:bg-black/5 cursor-pointer'
+          onClick={onSetting}
+        >
+          <Settings01 className='w-3.5 h-3.5 text-gray-500' />
+        </div>
+      </div>
+      <a
+        href='https://jina.ai/reader'
+        target='_blank' rel='noopener noreferrer'
+        className='flex items-center text-xs text-primary-600'
+      >
+        <BookOpen01 className='mr-1 w-3.5 h-3.5 text-primary-600' />
+        {t(`${I18N_PREFIX}.jinaReaderDoc`)}
+      </a>
+    </div>
+  )
+}
+export default React.memo(Header)

web/app/components/datasets/create/website/jina-reader/index.tsx

@@ -0,0 +1,232 @@
+'use client'
+import type { FC } from 'react'
+import React, { useCallback, useEffect, useState } from 'react'
+import { useTranslation } from 'react-i18next'
+import UrlInput from '../base/url-input'
+import OptionsWrap from '../base/options-wrap'
+import CrawledResult from '../base/crawled-result'
+import Crawling from '../base/crawling'
+import ErrorMessage from '../base/error-message'
+import Header from './header'
+import Options from './options'
+import cn from '@/utils/classnames'
+import { useModalContext } from '@/context/modal-context'
+import Toast from '@/app/components/base/toast'
+import { checkJinaReaderTaskStatus, createJinaReaderTask } from '@/service/datasets'
+import { sleep } from '@/utils'
+import type { CrawlOptions, CrawlResultItem } from '@/models/datasets'
+
+const ERROR_I18N_PREFIX = 'common.errorMsg'
+const I18N_PREFIX = 'datasetCreation.stepOne.website'
+
+type Props = {
+  onPreview: (payload: CrawlResultItem) => void
+  checkedCrawlResult: CrawlResultItem[]
+  onCheckedCrawlResultChange: (payload: CrawlResultItem[]) => void
+  onJobIdChange: (jobId: string) => void
+  crawlOptions: CrawlOptions
+  onCrawlOptionsChange: (payload: CrawlOptions) => void
+}
+
+enum Step {
+  init = 'init',
+  running = 'running',
+  finished = 'finished',
+}
+
+const JinaReader: FC<Props> = ({
+  onPreview,
+  checkedCrawlResult,
+  onCheckedCrawlResultChange,
+  onJobIdChange,
+  crawlOptions,
+  onCrawlOptionsChange,
+}) => {
+  const { t } = useTranslation()
+  const [step, setStep] = useState<Step>(Step.init)
+  const [controlFoldOptions, setControlFoldOptions] = useState<number>(0)
+  useEffect(() => {
+    if (step !== Step.init)
+      setControlFoldOptions(Date.now())
+  }, [step])
+  const { setShowAccountSettingModal } = useModalContext()
+  const handleSetting = useCallback(() => {
+    setShowAccountSettingModal({
+      payload: 'data-source',
+    })
+  }, [setShowAccountSettingModal])
+
+  const checkValid = useCallback((url: string) => {
+    let errorMsg = ''
+    if (!url) {
+      errorMsg = t(`${ERROR_I18N_PREFIX}.fieldRequired`, {
+        field: 'url',
+      })
+    }
+
+    if (!errorMsg && !((url.startsWith('http://') || url.startsWith('https://'))))
+      errorMsg = t(`${ERROR_I18N_PREFIX}.urlError`)
+
+    if (!errorMsg && (crawlOptions.limit === null || crawlOptions.limit === undefined || crawlOptions.limit === '')) {
+      errorMsg = t(`${ERROR_I18N_PREFIX}.fieldRequired`, {
+        field: t(`${I18N_PREFIX}.limit`),
+      })
+    }
+
+    return {
+      isValid: !errorMsg,
+      errorMsg,
+    }
+  }, [crawlOptions, t])
+
+  const isInit = step === Step.init
+  const isCrawlFinished = step === Step.finished
+  const isRunning = step === Step.running
+  const [crawlResult, setCrawlResult] = useState<{
+    current: number
+    total: number
+    data: CrawlResultItem[]
+    time_consuming: number | string
+  } | undefined>(undefined)
+  const [crawlErrorMessage, setCrawlErrorMessage] = useState('')
+  const showError = isCrawlFinished && crawlErrorMessage
+
+  const waitForCrawlFinished = useCallback(async (jobId: string) => {
+    try {
+      const res = await checkJinaReaderTaskStatus(jobId) as any
+      console.log('res', res)
+      if (res.status === 'completed') {
+        return {
+          isError: false,
+          data: {
+            ...res,
+            total: Math.min(res.total, parseFloat(crawlOptions.limit as string)),
+          },
+        }
+      }
+      if (res.status === 'failed' || !res.status) {
+        return {
+          isError: true,
+          errorMessage: res.message,
+          data: {
+            data: [],
+          },
+        }
+      }
+      // update the progress
+      setCrawlResult({
+        ...res,
+        total: Math.min(res.total, parseFloat(crawlOptions.limit as string)),
+      })
+      onCheckedCrawlResultChange(res.data || []) // default select the crawl result
+      await sleep(2500)
+      return await waitForCrawlFinished(jobId)
+    }
+    catch (e: any) {
+      const errorBody = await e.json()
+      return {
+        isError: true,
+        errorMessage: errorBody.message,
+        data: {
+          data: [],
+        },
+      }
+    }
+  }, [crawlOptions.limit])
+
+  const handleRun = useCallback(async (url: string) => {
+    const { isValid, errorMsg } = checkValid(url)
+    if (!isValid) {
+      Toast.notify({
+        message: errorMsg!,
+        type: 'error',
+      })
+      return
+    }
+    setStep(Step.running)
+    try {
+      const startTime = Date.now()
+      const res = await createJinaReaderTask({
+        url,
+        options: crawlOptions,
+      }) as any
+
+      if (res.data) {
+        const data = {
+          current: 1,
+          total: 1,
+          data: [{
+            title: res.data.title,
+            markdown: res.data.content,
+            description: res.data.description,
+            source_url: res.data.url,
+          }],
+          time_consuming: (Date.now() - startTime) / 1000,
+        }
+        setCrawlResult(data)
+        onCheckedCrawlResultChange(data.data || [])
+        setCrawlErrorMessage('')
+      }
+      else if (res.job_id) {
+        const jobId = res.job_id
+        onJobIdChange(jobId)
+        const { isError, data, errorMessage } = await waitForCrawlFinished(jobId)
+        if (isError) {
+          setCrawlErrorMessage(errorMessage || t(`${I18N_PREFIX}.unknownError`))
+        }
+        else {
+          setCrawlResult(data)
+          onCheckedCrawlResultChange(data.data || []) // default select the crawl result
+          setCrawlErrorMessage('')
+        }
+      }
+    }
+    catch (e) {
+      setCrawlErrorMessage(t(`${I18N_PREFIX}.unknownError`)!)
+      console.log(e)
+    }
+    finally {
+      setStep(Step.finished)
+    }
+  }, [checkValid, crawlOptions, onJobIdChange, t, waitForCrawlFinished])
+
+  return (
+    <div>
+      <Header onSetting={handleSetting} />
+      <div className={cn('mt-2 p-4 pb-0 rounded-xl border border-gray-200')}>
+        <UrlInput onRun={handleRun} isRunning={isRunning} />
+        <OptionsWrap
+          className={cn('mt-4')}
+          controlFoldOptions={controlFoldOptions}
+        >
+          <Options className='mt-2' payload={crawlOptions} onChange={onCrawlOptionsChange} />
+        </OptionsWrap>
+
+        {!isInit && (
+          <div className='mt-3 relative left-[-16px] w-[calc(100%_+_32px)] rounded-b-xl'>
+            {isRunning
+              && <Crawling
+                className='mt-2'
+                crawledNum={crawlResult?.current || 0}
+                totalNum={crawlResult?.total || parseFloat(crawlOptions.limit as string) || 0}
+              />}
+            {showError && (
+              <ErrorMessage className='rounded-b-xl' title={t(`${I18N_PREFIX}.exceptionErrorTitle`)} errorMsg={crawlErrorMessage} />
+            )}
+            {isCrawlFinished && !showError
+              && <CrawledResult
+                className='mb-2'
+                list={crawlResult?.data || []}
+                checkedList={checkedCrawlResult}
+                onSelectedChange={onCheckedCrawlResultChange}
+                onPreview={onPreview}
+                usedTime={parseFloat(crawlResult?.time_consuming as string) || 0}
+              />
+            }
+          </div>
+        )}
+      </div>
+    </div>
+  )
+}
+export default React.memo(JinaReader)

web/app/components/datasets/create/website/jina-reader/options.tsx

@@ -0,0 +1,59 @@
+'use client'
+import type { FC } from 'react'
+import React, { useCallback } from 'react'
+import { useTranslation } from 'react-i18next'
+import CheckboxWithLabel from '../base/checkbox-with-label'
+import Field from '../base/field'
+import cn from '@/utils/classnames'
+import type { CrawlOptions } from '@/models/datasets'
+
+const I18N_PREFIX = 'datasetCreation.stepOne.website'
+
+type Props = {
+  className?: string
+  payload: CrawlOptions
+  onChange: (payload: CrawlOptions) => void
+}
+
+const Options: FC<Props> = ({
+  className = '',
+  payload,
+  onChange,
+}) => {
+  const { t } = useTranslation()
+
+  const handleChange = useCallback((key: keyof CrawlOptions) => {
+    return (value: any) => {
+      onChange({
+        ...payload,
+        [key]: value,
+      })
+    }
+  }, [payload, onChange])
+  return (
+    <div className={cn(className, ' space-y-2')}>
+      <CheckboxWithLabel
+        label={t(`${I18N_PREFIX}.crawlSubPage`)}
+        isChecked={payload.crawl_sub_pages}
+        onChange={handleChange('crawl_sub_pages')}
+      />
+      <CheckboxWithLabel
+        label={t(`${I18N_PREFIX}.useSitemap`)}
+        isChecked={payload.use_sitemap}
+        onChange={handleChange('use_sitemap')}
+        tooltip={t(`${I18N_PREFIX}.useSitemapTooltip`) as string}
+      />
+      <div className='flex justify-between space-x-4'>
+        <Field
+          className='grow shrink-0'
+          label={t(`${I18N_PREFIX}.limit`)}
+          value={payload.limit}
+          onChange={handleChange('limit')}
+          isNumber
+          isRequired
+        />
+      </div>
+    </div>
+  )
+}
+export default React.memo(Options)

web/app/components/datasets/create/website/no-data.tsx

@@ -2,35 +2,56 @@
 import type { FC } from 'react'
 import React from 'react'
 import { useTranslation } from 'react-i18next'
+import s from './index.module.css'
 import { Icon3Dots } from '@/app/components/base/icons/src/vender/line/others'
 import Button from '@/app/components/base/button'
+import { DataSourceProvider } from '@/models/common'
 
 const I18N_PREFIX = 'datasetCreation.stepOne.website'
 
 type Props = {
   onConfig: () => void
+  provider: DataSourceProvider
 }
 
 const NoData: FC<Props> = ({
   onConfig,
+  provider,
 }) => {
   const { t } = useTranslation()
 
+  const providerConfig = {
+    [DataSourceProvider.jinaReader]: {
+      emoji: <span className={s.jinaLogo} />,
+      title: t(`${I18N_PREFIX}.jinaReaderNotConfigured`),
+      description: t(`${I18N_PREFIX}.jinaReaderNotConfiguredDescription`),
+    },
+    [DataSourceProvider.fireCrawl]: {
+      emoji: '🔥',
+      title: t(`${I18N_PREFIX}.fireCrawlNotConfigured`),
+      description: t(`${I18N_PREFIX}.fireCrawlNotConfiguredDescription`),
+    },
+  }
+
+  const currentProvider = providerConfig[provider]
+
   return (
-    <div className='max-w-[640px] p-6 rounded-2xl bg-gray-50'>
-      <div className='flex w-11 h-11 items-center justify-center bg-gray-50 rounded-xl border-[0.5px] border-gray-100 shadow-lg'>
-        🔥
-      </div>
-      <div className='my-2'>
-        <span className='text-gray-700 font-semibold'>{t(`${I18N_PREFIX}.fireCrawlNotConfigured`)}<Icon3Dots className='inline relative -top-3 -left-1.5' /></span>
-        <div className='mt-1 pb-3 text-gray-500 text-[13px] font-normal'>
-          {t(`${I18N_PREFIX}.fireCrawlNotConfiguredDescription`)}
+    <>
+      <div className='max-w-[640px] p-6 rounded-2xl bg-gray-50 mt-4'>
+        <div className='flex w-11 h-11 items-center justify-center bg-gray-50 rounded-xl border-[0.5px] border-gray-100 shadow-lg'>
+          {currentProvider.emoji}
         </div>
+        <div className='my-2'>
+          <span className='text-gray-700 font-semibold'>{currentProvider.title}<Icon3Dots className='inline relative -top-3 -left-1.5' /></span>
+          <div className='mt-1 pb-3 text-gray-500 text-[13px] font-normal'>
+            {currentProvider.description}
+          </div>
+        </div>
+        <Button variant='primary' onClick={onConfig}>
+          {t(`${I18N_PREFIX}.configure`)}
+        </Button>
       </div>
-      <Button variant='primary' onClick={onConfig}>
-        {t(`${I18N_PREFIX}.configure`)}
-      </Button>
-    </div>
+    </>
   )
 }
 export default React.memo(NoData)

web/app/components/datasets/documents/detail/completed/SegmentCard.tsx

@@ -36,6 +36,12 @@ export type UsageScene = 'doc' | 'hitTesting'
 type ISegmentCardProps = {
   loading: boolean
   detail?: SegmentDetailModel & { document: { name: string } }
+  contentExternal?: string
+  refSource?: {
+    title: string
+    uri: string
+  }
+  isExternal?: boolean
   score?: number
   onClick?: () => void
   onChangeSwitch?: (segId: string, enabled: boolean) => Promise<void>
@@ -48,6 +54,9 @@ type ISegmentCardProps = {
 
 const SegmentCard: FC<ISegmentCardProps> = ({
   detail = {},
+  contentExternal,
+  isExternal,
+  refSource,
   score,
   onClick,
   onChangeSwitch,
@@ -88,6 +97,9 @@ const SegmentCard: FC<ISegmentCardProps> = ({
       )
     }
 
+    if (contentExternal)
+      return contentExternal
+
     return content
   }
 
@@ -199,16 +211,16 @@ const SegmentCard: FC<ISegmentCardProps> = ({
               </div>
               <div className={cn('w-full bg-gray-50 group-hover:bg-white')}>
                 <Divider />
-                <div className="relative flex items-center w-full">
+                <div className="relative flex items-center w-full pb-1">
                   <DocumentTitle
-                    name={detail?.document?.name || ''}
-                    extension={(detail?.document?.name || '').split('.').pop() || 'txt'}
+                    name={detail?.document?.name || refSource?.title || ''}
+                    extension={(detail?.document?.name || refSource?.title || '').split('.').pop() || 'txt'}
                     wrapperCls='w-full'
                     iconCls="!h-4 !w-4 !bg-contain"
                     textCls="text-xs text-gray-700 !font-normal overflow-hidden whitespace-nowrap text-ellipsis"
                   />
                   <div className={cn(s.chartLinkText, 'group-hover:inline-flex')}>
-                    {t('datasetHitTesting.viewChart')}
+                    {isExternal ? t('datasetHitTesting.viewDetail') : t('datasetHitTesting.viewChart')}
                     <ArrowUpRightIcon className="w-3 h-3 ml-1 stroke-current stroke-2" />
                   </div>
                 </div>

web/app/components/datasets/documents/list.tsx

@@ -122,6 +122,7 @@ export const OperationAction: FC<{
 }> = ({ embeddingAvailable, datasetId, detail, onUpdate, scene = 'list', className = '' }) => {
   const { id, enabled = false, archived = false, data_source_type } = detail || {}
   const [showModal, setShowModal] = useState(false)
+  const [deleting, setDeleting] = useState(false)
   const { notify } = useContext(ToastContext)
   const { t } = useTranslation()
   const router = useRouter()
@@ -153,6 +154,7 @@ export const OperationAction: FC<{
         break
       default:
         opApi = deleteDocument
+        setDeleting(true)
         break
     }
     const [e] = await asyncRunSafe<CommonResponse>(opApi({ datasetId, documentId: id }) as Promise<CommonResponse>)
@@ -160,6 +162,8 @@ export const OperationAction: FC<{
       notify({ type: 'success', message: t('common.actionMsg.modifiedSuccessfully') })
     else
       notify({ type: 'error', message: t('common.actionMsg.modifiedUnsuccessfully') })
+    if (operationName === 'delete')
+      setDeleting(false)
     onUpdate(operationName)
   }
 
@@ -295,6 +299,8 @@ export const OperationAction: FC<{
     {showModal
       && <Confirm
         isShow={showModal}
+        isLoading={deleting}
+        isDisabled={deleting}
         title={t('datasetDocuments.list.delete.title')}
         content={t('datasetDocuments.list.delete.content')}
         confirmText={t('common.operation.sure')}

web/app/components/datasets/external-api/declarations.ts

@@ -0,0 +1,16 @@
+export type CreateExternalAPIReq = {
+  name: string
+  settings: {
+    endpoint: string
+    api_key: string
+  }
+}
+
+export type FormSchema = {
+  variable: string
+  type: 'text' | 'secret'
+  label: {
+    [key: string]: string
+  }
+  required: boolean
+}

web/app/components/datasets/external-api/external-api-modal/Form.tsx

@@ -0,0 +1,90 @@
+import React, { useState } from 'react'
+import type { FC } from 'react'
+import { useTranslation } from 'react-i18next'
+import { RiBookOpenLine } from '@remixicon/react'
+import type { CreateExternalAPIReq, FormSchema } from '../declarations'
+import Input from '@/app/components/base/input'
+import cn from '@/utils/classnames'
+
+type FormProps = {
+  className?: string
+  itemClassName?: string
+  fieldLabelClassName?: string
+  value: CreateExternalAPIReq
+  onChange: (val: CreateExternalAPIReq) => void
+  formSchemas: FormSchema[]
+  inputClassName?: string
+}
+
+const Form: FC<FormProps> = React.memo(({
+  className,
+  itemClassName,
+  fieldLabelClassName,
+  value,
+  onChange,
+  formSchemas,
+  inputClassName,
+}) => {
+  const { t, i18n } = useTranslation()
+  const [changeKey, setChangeKey] = useState('')
+
+  const handleFormChange = (key: string, val: string) => {
+    setChangeKey(key)
+    if (key === 'name') {
+      onChange({ ...value, [key]: val })
+    }
+    else {
+      onChange({
+        ...value,
+        settings: {
+          ...value.settings,
+          [key]: val,
+        },
+      })
+    }
+  }
+
+  const renderField = (formSchema: FormSchema) => {
+    const { variable, type, label, required } = formSchema
+    const fieldValue = variable === 'name' ? value[variable] : (value.settings[variable as keyof typeof value.settings] || '')
+
+    return (
+      <div key={variable} className={cn(itemClassName, 'flex flex-col items-start gap-1 self-stretch')}>
+        <div className="flex justify-between items-center w-full">
+          <label className={cn(fieldLabelClassName, 'text-text-secondary system-sm-semibold')} htmlFor={variable}>
+            {label[i18n.language] || label.en_US}
+            {required && <span className='ml-1 text-red-500'>*</span>}
+          </label>
+          {variable === 'endpoint' && (
+            <a
+              href={'https://docs.dify.ai/guides/knowledge-base/external-knowledge-api-documentation' || '/'}
+              target='_blank'
+              rel='noopener noreferrer'
+              className='text-text-accent body-xs-regular flex items-center'
+            >
+              <RiBookOpenLine className='w-3 h-3 text-text-accent mr-1' />
+              {t('dataset.externalAPIPanelDocumentation')}
+            </a>
+          )}
+        </div>
+        <Input
+          type={type === 'secret' ? 'password' : 'text'}
+          id={variable}
+          name={variable}
+          value={fieldValue}
+          onChange={val => handleFormChange(variable, val.target.value)}
+          required={required}
+          className={cn(inputClassName)}
+        />
+      </div>
+    )
+  }
+
+  return (
+    <form className={cn('flex flex-col justify-center items-start gap-4 self-stretch', className)}>
+      {formSchemas.map(formSchema => renderField(formSchema))}
+    </form>
+  )
+})
+
+export default Form

web/app/components/datasets/external-api/external-api-modal/index.tsx

@@ -0,0 +1,218 @@
+import type { FC } from 'react'
+import {
+  memo,
+  useEffect,
+  useState,
+} from 'react'
+import { useTranslation } from 'react-i18next'
+import {
+  RiBook2Line,
+  RiCloseLine,
+  RiInformation2Line,
+  RiLock2Fill,
+} from '@remixicon/react'
+import type { CreateExternalAPIReq, FormSchema } from '../declarations'
+import Form from './Form'
+import ActionButton from '@/app/components/base/action-button'
+import Confirm from '@/app/components/base/confirm'
+import {
+  PortalToFollowElem,
+  PortalToFollowElemContent,
+} from '@/app/components/base/portal-to-follow-elem'
+import { createExternalAPI } from '@/service/datasets'
+import { useToastContext } from '@/app/components/base/toast'
+import Button from '@/app/components/base/button'
+import Tooltip from '@/app/components/base/tooltip'
+
+type AddExternalAPIModalProps = {
+  data?: CreateExternalAPIReq
+  onSave: (formValue: CreateExternalAPIReq) => void
+  onCancel: () => void
+  onEdit?: (formValue: CreateExternalAPIReq) => Promise<void>
+  datasetBindings?: { id: string; name: string }[]
+  isEditMode: boolean
+}
+
+const formSchemas: FormSchema[] = [
+  {
+    variable: 'name',
+    type: 'text',
+    label: {
+      en_US: 'Name',
+    },
+    required: true,
+  },
+  {
+    variable: 'endpoint',
+    type: 'text',
+    label: {
+      en_US: 'API Endpoint',
+    },
+    required: true,
+  },
+  {
+    variable: 'api_key',
+    type: 'secret',
+    label: {
+      en_US: 'API Key',
+    },
+    required: true,
+  },
+]
+
+const AddExternalAPIModal: FC<AddExternalAPIModalProps> = ({ data, onSave, onCancel, datasetBindings, isEditMode, onEdit }) => {
+  const { t } = useTranslation()
+  const { notify } = useToastContext()
+  const [loading, setLoading] = useState(false)
+  const [showConfirm, setShowConfirm] = useState(false)
+  const [formData, setFormData] = useState<CreateExternalAPIReq>({ name: '', settings: { endpoint: '', api_key: '' } })
+
+  useEffect(() => {
+    if (isEditMode && data)
+      setFormData(data)
+  }, [isEditMode, data])
+
+  const hasEmptyInputs = Object.values(formData).some(value =>
+    typeof value === 'string' ? value.trim() === '' : Object.values(value).some(v => v.trim() === ''),
+  )
+  const handleDataChange = (val: CreateExternalAPIReq) => {
+    setFormData(val)
+  }
+
+  const handleSave = async () => {
+    if (formData && formData.settings.api_key && formData.settings.api_key?.length < 5) {
+      notify({ type: 'error', message: t('common.apiBasedExtension.modal.apiKey.lengthError') })
+      setLoading(false)
+      return
+    }
+    try {
+      setLoading(true)
+      if (isEditMode && onEdit) {
+        await onEdit(
+          {
+            ...formData,
+            settings: { ...formData.settings, api_key: formData.settings.api_key ? '[__HIDDEN__]' : formData.settings.api_key },
+          },
+        )
+        notify({ type: 'success', message: 'External API updated successfully' })
+      }
+      else {
+        const res = await createExternalAPI({ body: formData })
+        if (res && res.id) {
+          notify({ type: 'success', message: 'External API saved successfully' })
+          onSave(res)
+        }
+      }
+      onCancel()
+    }
+    catch (error) {
+      console.error('Error saving/updating external API:', error)
+      notify({ type: 'error', message: 'Failed to save/update External API' })
+    }
+    finally {
+      setLoading(false)
+    }
+  }
+
+  return (
+    <PortalToFollowElem open>
+      <PortalToFollowElemContent className='w-full h-full z-[60]'>
+        <div className='fixed inset-0 flex items-center justify-center bg-black/[.25]'>
+          <div className='flex relative w-[480px] flex-col items-start bg-components-panel-bg rounded-2xl border-[0.5px] border-components-panel-border shadows-shadow-xl'>
+            <div className='flex flex-col pt-6 pl-6 pb-3 pr-14 items-start gap-2 self-stretch'>
+              <div className='self-stretch text-text-primary title-2xl-semi-bold flex-grow'>
+                {
+                  isEditMode ? t('dataset.editExternalAPIFormTitle') : t('dataset.createExternalAPI')
+                }
+              </div>
+              {isEditMode && (datasetBindings?.length ?? 0) > 0 && (
+                <div className='text-text-tertiary system-xs-regular flex items-center'>
+                  {t('dataset.editExternalAPIFormWarning.front')}
+                  <span className='text-text-accent cursor-pointer flex items-center'>
+                    &nbsp;{datasetBindings?.length} {t('dataset.editExternalAPIFormWarning.end')}&nbsp;
+                    <Tooltip
+                      popupClassName='flex items-center self-stretch w-[320px]'
+                      popupContent={
+                        <div className='p-1'>
+                          <div className='flex pt-1 pb-0.5 pl-2 pr-3 items-start self-stretch'>
+                            <div className='text-text-tertiary system-xs-medium-uppercase'>{`${datasetBindings?.length} ${t('dataset.editExternalAPITooltipTitle')}`}</div>
+                          </div>
+                          {datasetBindings?.map(binding => (
+                            <div key={binding.id} className='flex px-2 py-1 items-center gap-1 self-stretch'>
+                              <RiBook2Line className='w-4 h-4 text-text-secondary' />
+                              <div className='text-text-secondary system-sm-medium'>{binding.name}</div>
+                            </div>
+                          ))}
+                        </div>
+                      }
+                      asChild={false}
+                      position='bottom'
+                    >
+                      <RiInformation2Line className='w-3.5 h-3.5' />
+                    </Tooltip>
+                  </span>
+                </div>
+              )}
+            </div>
+            <ActionButton className='absolute top-5 right-5' onClick={onCancel}>
+              <RiCloseLine className='w-[18px] h-[18px] text-text-tertiary flex-shrink-0' />
+            </ActionButton>
+            <Form
+              value={formData}
+              onChange={handleDataChange}
+              formSchemas={formSchemas}
+              className='flex px-6 py-3 flex-col justify-center items-start gap-4 self-stretch'
+            />
+            <div className='flex p-6 pt-5 justify-end items-center gap-2 self-stretch'>
+              <Button type='button' variant='secondary' onClick={onCancel}>
+                {t('dataset.externalAPIForm.cancel')}
+              </Button>
+              <Button
+                type='submit'
+                variant='primary'
+                onClick={() => {
+                  if (isEditMode && (datasetBindings?.length ?? 0) > 0)
+                    setShowConfirm(true)
+                  else if (isEditMode && onEdit)
+                    onEdit(formData)
+
+                  else
+                    handleSave()
+                }}
+                disabled={hasEmptyInputs || loading}
+              >
+                {t('dataset.externalAPIForm.save')}
+              </Button>
+            </div>
+            <div className='flex px-2 py-3 justify-center items-center gap-1 self-stretch rounded-b-2xl
+              border-t-[0.5px] border-divider-subtle bg-background-soft text-text-tertiary system-xs-regular'
+            >
+              <RiLock2Fill className='w-3 h-3 text-text-quaternary' />
+              {t('dataset.externalAPIForm.encrypted.front')}
+              <a
+                className='text-text-accent'
+                target='_blank' rel='noopener noreferrer'
+                href='https://pycryptodome.readthedocs.io/en/latest/src/cipher/oaep.html'
+              >
+                PKCS1_OAEP
+              </a>
+              {t('dataset.externalAPIForm.encrypted.end')}
+            </div>
+          </div>
+          {showConfirm && (datasetBindings?.length ?? 0) > 0 && (
+            <Confirm
+              isShow={showConfirm}
+              type='warning'
+              title='Warning'
+              content={`${t('dataset.editExternalAPIConfirmWarningContent.front')} ${datasetBindings?.length} ${t('dataset.editExternalAPIConfirmWarningContent.end')}`}
+              onCancel={() => setShowConfirm(false)}
+              onConfirm={handleSave}
+            />
+          )}
+        </div>
+      </PortalToFollowElemContent>
+    </PortalToFollowElem>
+  )
+}
+
+export default memo(AddExternalAPIModal)

web/app/components/datasets/external-api/external-api-panel/index.tsx

@@ -0,0 +1,90 @@
+import React from 'react'
+import {
+  RiAddLine,
+  RiBookOpenLine,
+  RiCloseLine,
+} from '@remixicon/react'
+import { useTranslation } from 'react-i18next'
+import ExternalKnowledgeAPICard from '../external-knowledge-api-card'
+import cn from '@/utils/classnames'
+import { useExternalKnowledgeApi } from '@/context/external-knowledge-api-context'
+import ActionButton from '@/app/components/base/action-button'
+import Button from '@/app/components/base/button'
+import Loading from '@/app/components/base/loading'
+import { useModalContext } from '@/context/modal-context'
+
+type ExternalAPIPanelProps = {
+  onClose: () => void
+}
+
+const ExternalAPIPanel: React.FC<ExternalAPIPanelProps> = ({ onClose }) => {
+  const { t } = useTranslation()
+  const { setShowExternalKnowledgeAPIModal } = useModalContext()
+  const { externalKnowledgeApiList, mutateExternalKnowledgeApis, isLoading } = useExternalKnowledgeApi()
+
+  const handleOpenExternalAPIModal = () => {
+    setShowExternalKnowledgeAPIModal({
+      payload: { name: '', settings: { endpoint: '', api_key: '' } },
+      datasetBindings: [],
+      onSaveCallback: () => {
+        mutateExternalKnowledgeApis()
+      },
+      onCancelCallback: () => {
+        mutateExternalKnowledgeApis()
+      },
+      isEditMode: false,
+    })
+  }
+
+  return (
+    <div
+      tabIndex={-1}
+      className={cn('absolute top-14 right-0 bottom-2 flex z-10 outline-none')}
+    >
+      <div
+        className={cn(
+          'relative flex flex-col w-[420px] bg-components-panel-bg-alt rounded-l-2xl h-full border border-components-panel-border',
+        )}
+      >
+        <div className='flex items-start self-stretch p-4 pb-0'>
+          <div className='flex flex-col items-start gap-1 flex-grow'>
+            <div className='self-stretch text-text-primary system-xl-semibold'>{t('dataset.externalAPIPanelTitle')}</div>
+            <div className='self-stretch text-text-tertiary body-xs-regular'>{t('dataset.externalAPIPanelDescription')}</div>
+            <a className='flex justify-center items-center gap-1 self-stretch cursor-pointer' href='https://docs.dify.ai/guides/knowledge-base/external-knowledge-api-documentation' target='_blank'>
+              <RiBookOpenLine className='w-3 h-3 text-text-accent' />
+              <div className='flex-grow text-text-accent body-xs-regular'>{t('dataset.externalAPIPanelDocumentation')}</div>
+            </a>
+          </div>
+          <div className='flex items-center'>
+            <ActionButton onClick={() => onClose()}>
+              <RiCloseLine className='w-4 h-4 text-text-tertiary' />
+            </ActionButton>
+          </div>
+        </div>
+        <div className='flex px-4 py-3 flex-col justify-center items-start gap-2 self-stretch'>
+          <Button
+            variant={'primary'}
+            className='flex justify-center items-center px-3 py-2 gap-0.5'
+            onClick={handleOpenExternalAPIModal}
+          >
+            <RiAddLine className='w-4 h-4 text-components-button-primary-text' />
+            <div className='text-components-button-primary-text system-sm-medium'>{t('dataset.createExternalAPI')}</div>
+          </Button>
+        </div>
+        <div className='flex py-0 px-4 flex-col items-start gap-1 flex-grow self-stretch'>
+          {isLoading
+            ? (
+              <Loading />
+            )
+            : (
+              externalKnowledgeApiList.map(api => (
+                <ExternalKnowledgeAPICard key={api.id} api={api} />
+              ))
+            )}
+        </div>
+      </div>
+    </div>
+  )
+}
+
+export default ExternalAPIPanel

web/app/components/datasets/external-api/external-knowledge-api-card/index.tsx

@@ -0,0 +1,151 @@
+import React, { useState } from 'react'
+import { useTranslation } from 'react-i18next'
+import {
+  RiDeleteBinLine,
+  RiEditLine,
+} from '@remixicon/react'
+import type { CreateExternalAPIReq } from '../declarations'
+import type { ExternalAPIItem } from '@/models/datasets'
+import { checkUsageExternalAPI, deleteExternalAPI, fetchExternalAPI, updateExternalAPI } from '@/service/datasets'
+import { ApiConnectionMod } from '@/app/components/base/icons/src/vender/solid/development'
+import { useExternalKnowledgeApi } from '@/context/external-knowledge-api-context'
+import { useModalContext } from '@/context/modal-context'
+import ActionButton from '@/app/components/base/action-button'
+import Confirm from '@/app/components/base/confirm'
+
+type ExternalKnowledgeAPICardProps = {
+  api: ExternalAPIItem
+}
+
+const ExternalKnowledgeAPICard: React.FC<ExternalKnowledgeAPICardProps> = ({ api }) => {
+  const { setShowExternalKnowledgeAPIModal } = useModalContext()
+  const [showConfirm, setShowConfirm] = useState(false)
+  const [isHovered, setIsHovered] = useState(false)
+  const [usageCount, setUsageCount] = useState(0)
+  const { mutateExternalKnowledgeApis } = useExternalKnowledgeApi()
+
+  const { t } = useTranslation()
+
+  const handleEditClick = async () => {
+    try {
+      const response = await fetchExternalAPI({ apiTemplateId: api.id })
+      const formValue: CreateExternalAPIReq = {
+        name: response.name,
+        settings: {
+          endpoint: response.settings.endpoint,
+          api_key: response.settings.api_key,
+        },
+      }
+
+      setShowExternalKnowledgeAPIModal({
+        payload: formValue,
+        onSaveCallback: () => {
+          mutateExternalKnowledgeApis()
+        },
+        onCancelCallback: () => {
+          mutateExternalKnowledgeApis()
+        },
+        isEditMode: true,
+        datasetBindings: response.dataset_bindings,
+        onEditCallback: async (updatedData: CreateExternalAPIReq) => {
+          try {
+            await updateExternalAPI({
+              apiTemplateId: api.id,
+              body: {
+                ...response,
+                name: updatedData.name,
+                settings: {
+                  ...response.settings,
+                  endpoint: updatedData.settings.endpoint,
+                  api_key: updatedData.settings.api_key,
+                },
+              },
+            })
+            mutateExternalKnowledgeApis()
+          }
+          catch (error) {
+            console.error('Error updating external knowledge API:', error)
+          }
+        },
+      })
+    }
+    catch (error) {
+      console.error('Error fetching external knowledge API data:', error)
+    }
+  }
+
+  const handleDeleteClick = async () => {
+    try {
+      const usage = await checkUsageExternalAPI({ apiTemplateId: api.id })
+      if (usage.is_using)
+        setUsageCount(usage.count)
+
+      setShowConfirm(true)
+    }
+    catch (error) {
+      console.error('Error checking external API usage:', error)
+    }
+  }
+
+  const handleConfirmDelete = async () => {
+    try {
+      const response = await deleteExternalAPI({ apiTemplateId: api.id })
+      if (response && response.result === 'success') {
+        setShowConfirm(false)
+        mutateExternalKnowledgeApis()
+      }
+      else {
+        console.error('Failed to delete external API')
+      }
+    }
+    catch (error) {
+      console.error('Error deleting external knowledge API:', error)
+    }
+  }
+
+  return (
+    <>
+      <div className={`flex p-2 pl-3 items-start self-stretch rounded-lg border-[0.5px] 
+        border-components-panel-border-subtle bg-components-panel-on-panel-item-bg 
+        shadows-shadow-xs ${isHovered ? 'bg-state-destructive-hover border-state-destructive-border' : ''}`}
+      >
+        <div className='flex py-1 flex-col justify-center items-start gap-1.5 flex-grow'>
+          <div className='flex items-center gap-1 self-stretch text-text-secondary'>
+            <ApiConnectionMod className='w-4 h-4' />
+            <div className='system-sm-medium'>{api.name}</div>
+          </div>
+          <div className='self-stretch text-text-tertiary system-xs-regular'>{api.settings.endpoint}</div>
+        </div>
+        <div className='flex items-start gap-1'>
+          <ActionButton onClick={handleEditClick}>
+            <RiEditLine className='w-4 h-4 text-text-tertiary hover:text-text-secondary' />
+          </ActionButton>
+          <ActionButton
+            className='hover:bg-state-destructive-hover'
+            onClick={handleDeleteClick}
+            onMouseEnter={() => setIsHovered(true)}
+            onMouseLeave={() => setIsHovered(false)}
+          >
+            <RiDeleteBinLine className='w-4 h-4 text-text-tertiary hover:text-text-destructive' />
+          </ActionButton>
+        </div>
+      </div>
+      {showConfirm && (
+        <Confirm
+          isShow={showConfirm}
+          title={`${t('dataset.deleteExternalAPIConfirmWarningContent.title.front')} ${api.name}${t('dataset.deleteExternalAPIConfirmWarningContent.title.end')}`}
+          content={
+            usageCount > 0
+              ? `${t('dataset.deleteExternalAPIConfirmWarningContent.content.front')} ${usageCount} ${t('dataset.deleteExternalAPIConfirmWarningContent.content.end')}`
+              : t('dataset.deleteExternalAPIConfirmWarningContent.noConnectionContent')
+          }
+          type='warning'
+          onConfirm={handleConfirmDelete}
+          onCancel={() => setShowConfirm(false)}
+        />
+      )}
+    </>
+  )
+}
+
+export default ExternalKnowledgeAPICard

web/app/components/datasets/external-knowledge-base/connector/index.tsx

@@ -0,0 +1,36 @@
+'use client'
+
+import React, { useState } from 'react'
+import { useRouter } from 'next/navigation'
+import { useToastContext } from '@/app/components/base/toast'
+import ExternalKnowledgeBaseCreate from '@/app/components/datasets/external-knowledge-base/create'
+import type { CreateKnowledgeBaseReq } from '@/app/components/datasets/external-knowledge-base/create/declarations'
+import { createExternalKnowledgeBase } from '@/service/datasets'
+
+const ExternalKnowledgeBaseConnector = () => {
+  const { notify } = useToastContext()
+  const [loading, setLoading] = useState(false)
+  const router = useRouter()
+
+  const handleConnect = async (formValue: CreateKnowledgeBaseReq) => {
+    try {
+      setLoading(true)
+      const result = await createExternalKnowledgeBase({ body: formValue })
+      if (result && result.id) {
+        notify({ type: 'success', message: 'External Knowledge Base Connected Successfully' })
+        router.back()
+      }
+      else { throw new Error('Failed to create external knowledge base') }
+    }
+    catch (error) {
+      console.error('Error creating external knowledge base:', error)
+      notify({ type: 'error', message: 'Failed to connect External Knowledge Base' })
+    }
+    setLoading(false)
+  }
+  return (
+    <ExternalKnowledgeBaseCreate onConnect={handleConnect} loading={loading} />
+  )
+}
+
+export default ExternalKnowledgeBaseConnector

web/app/components/datasets/external-knowledge-base/create/ExternalApiSelect.tsx

@@ -0,0 +1,110 @@
+import React, { useEffect, useState } from 'react'
+import {
+  RiAddLine,
+  RiArrowDownSLine,
+} from '@remixicon/react'
+import { useTranslation } from 'react-i18next'
+import { useRouter } from 'next/navigation'
+import { ApiConnectionMod } from '@/app/components/base/icons/src/vender/solid/development'
+import { useModalContext } from '@/context/modal-context'
+import { useExternalKnowledgeApi } from '@/context/external-knowledge-api-context'
+
+type ApiItem = {
+  value: string
+  name: string
+  url: string
+}
+
+type ExternalApiSelectProps = {
+  items: ApiItem[]
+  value?: string
+  onSelect: (item: ApiItem) => void
+}
+
+const ExternalApiSelect: React.FC<ExternalApiSelectProps> = ({ items, value, onSelect }) => {
+  const { t } = useTranslation()
+  const [isOpen, setIsOpen] = useState(false)
+  const [selectedItem, setSelectedItem] = useState<ApiItem | null>(
+    items.find(item => item.value === value) || null,
+  )
+  const { setShowExternalKnowledgeAPIModal } = useModalContext()
+  const { mutateExternalKnowledgeApis } = useExternalKnowledgeApi()
+  const router = useRouter()
+
+  useEffect(() => {
+    const newSelectedItem = items.find(item => item.value === value) || null
+    setSelectedItem(newSelectedItem)
+  }, [value, items])
+
+  const handleAddNewAPI = () => {
+    setShowExternalKnowledgeAPIModal({
+      payload: { name: '', settings: { endpoint: '', api_key: '' } },
+      onSaveCallback: async () => {
+        mutateExternalKnowledgeApis()
+        router.refresh()
+      },
+      onCancelCallback: () => {
+        mutateExternalKnowledgeApis()
+      },
+      isEditMode: false,
+    })
+  }
+
+  const handleSelect = (item: ApiItem) => {
+    setSelectedItem(item)
+    onSelect(item)
+    setIsOpen(false)
+  }
+
+  return (
+    <div className="relative w-full">
+      <div
+        className={`flex items-center justify-between cursor-pointer px-2 py-1 gap-0.5 self-stretch rounded-lg 
+        bg-components-input-bg-normal hover:bg-state-base-hover-alt ${isOpen && 'bg-state-base-hover-alt'}`}
+        onClick={() => setIsOpen(!isOpen)}
+      >
+        {selectedItem
+          ? (
+            <div className="flex p-1 items-center gap-2 self-stretch rounded-lg">
+              <ApiConnectionMod className='text-text-secondary w-4 h-4' />
+              <div className='flex items-center flex-grow'>
+                <span className='text-components-input-text-filled text-ellipsis system-sm-regular overflow-hidden'>{selectedItem.name}</span>
+              </div>
+            </div>
+          )
+          : (
+            <span className='text-components-input-text-placeholder system-sm-regular'>{t('dataset.selectExternalKnowledgeAPI.placeholder')}</span>
+          )}
+        <RiArrowDownSLine className={`w-4 h-4 text-text-quaternary transition-transform ${isOpen ? 'text-text-secondary' : ''}`} />
+      </div>
+      {isOpen && (
+        <div className="absolute z-10 w-full mt-1 bg-components-panel-bg-blur border rounded-xl shadow-lg">
+          {items.map(item => (
+            <div
+              key={item.value}
+              className="flex p-1 items-center cursor-pointer"
+              onClick={() => handleSelect(item)}
+            >
+              <div className="flex p-2 items-center gap-2 self-stretch rounded-lg hover:bg-state-base-hover w-full">
+                <ApiConnectionMod className='text-text-secondary w-4 h-4' />
+                <span className='text-text-secondary text-ellipsis system-sm-medium overflow-hidden flex-grow'>{item.name}</span>
+                <span className='text-text-tertiary overflow-hidden text-right text-ellipsis system-xs-regular'>{item.url}</span>
+              </div>
+            </div>
+          ))}
+          <div className='flex p-1 flex-col items-start self-stretch'>
+            <div
+              className='flex p-2 items-center gap-2 self-stretch rounded-lg cursor-pointer hover:bg-state-base-hover'
+              onClick={handleAddNewAPI}
+            >
+              <RiAddLine className='text-text-secondary w-4 h-4' />
+              <span className='flex-grow overflow-hidden text-text-secondary text-ellipsis system-sm-medium'>{t('dataset.createNewExternalAPI')}</span>
+            </div>
+          </div>
+        </div>
+      )}
+    </div>
+  )
+}
+
+export default ExternalApiSelect

web/app/components/datasets/external-knowledge-base/create/ExternalApiSelection.tsx

@@ -0,0 +1,96 @@
+'use client'
+
+import React, { useEffect, useState } from 'react'
+import { useTranslation } from 'react-i18next'
+import { RiAddLine } from '@remixicon/react'
+import { useRouter } from 'next/navigation'
+import ExternalApiSelect from './ExternalApiSelect'
+import Input from '@/app/components/base/input'
+import Button from '@/app/components/base/button'
+import { useModalContext } from '@/context/modal-context'
+import { useExternalKnowledgeApi } from '@/context/external-knowledge-api-context'
+
+type ExternalApiSelectionProps = {
+  external_knowledge_api_id: string
+  external_knowledge_id: string
+  onChange: (data: { external_knowledge_api_id?: string; external_knowledge_id?: string }) => void
+}
+
+const ExternalApiSelection: React.FC<ExternalApiSelectionProps> = ({ external_knowledge_api_id, external_knowledge_id, onChange }) => {
+  const { t } = useTranslation()
+  const router = useRouter()
+  const { externalKnowledgeApiList } = useExternalKnowledgeApi()
+  const [selectedApiId, setSelectedApiId] = useState(external_knowledge_api_id)
+  const { setShowExternalKnowledgeAPIModal } = useModalContext()
+  const { mutateExternalKnowledgeApis } = useExternalKnowledgeApi()
+
+  const apiItems = externalKnowledgeApiList.map(api => ({
+    value: api.id,
+    name: api.name,
+    url: api.settings.endpoint,
+  }))
+
+  useEffect(() => {
+    if (apiItems.length > 0) {
+      const newSelectedId = external_knowledge_api_id || apiItems[0].value
+      setSelectedApiId(newSelectedId)
+      if (newSelectedId !== external_knowledge_api_id)
+        onChange({ external_knowledge_api_id: newSelectedId, external_knowledge_id })
+    }
+  }, [apiItems, external_knowledge_api_id, external_knowledge_id, onChange])
+
+  const handleAddNewAPI = () => {
+    setShowExternalKnowledgeAPIModal({
+      payload: { name: '', settings: { endpoint: '', api_key: '' } },
+      onSaveCallback: async () => {
+        mutateExternalKnowledgeApis()
+        router.refresh()
+      },
+      onCancelCallback: () => {
+        mutateExternalKnowledgeApis()
+      },
+      isEditMode: false,
+    })
+  }
+
+  useEffect(() => {
+    if (!external_knowledge_api_id && apiItems.length > 0)
+      onChange({ external_knowledge_api_id: apiItems[0].value, external_knowledge_id })
+  }, [])
+
+  return (
+    <form className='flex flex-col gap-4 self-stretch'>
+      <div className='flex flex-col gap-1 self-stretch'>
+        <div className='flex flex-col self-stretch'>
+          <label className='text-text-secondary system-sm-semibold'>{t('dataset.externalAPIPanelTitle')}</label>
+        </div>
+        {apiItems.length > 0
+          ? <ExternalApiSelect
+            items={apiItems}
+            value={selectedApiId}
+            onSelect={(e) => {
+              setSelectedApiId(e.value)
+              onChange({ external_knowledge_api_id: e.value, external_knowledge_id })
+            }}
+          />
+          : <Button variant={'tertiary'} onClick={handleAddNewAPI} className='justify-start gap-0.5'>
+            <RiAddLine className='w-4 h-4 text-text-tertiary' />
+            <span className='text-text-tertiary system-sm-regular'>{t('dataset.noExternalKnowledge')}</span>
+          </Button>
+        }
+      </div>
+      <div className='flex flex-col gap-1 self-stretch'>
+        <div className='flex flex-col self-stretch'>
+          <label className='text-text-secondary system-sm-semibold'>{t('dataset.externalKnowledgeId')}</label>
+        </div>
+        <Input
+          value={external_knowledge_id}
+          onChange={e => onChange({ external_knowledge_id: e.target.value, external_knowledge_api_id })}
+          placeholder={t('dataset.externalKnowledgeIdPlaceholder') ?? ''}
+        />
+      </div>
+    </form>
+  )
+}
+
+export default ExternalApiSelection

web/app/components/datasets/external-knowledge-base/create/InfoPanel.tsx

@@ -0,0 +1,33 @@
+import { RiBookOpenLine } from '@remixicon/react'
+import { useTranslation } from 'react-i18next'
+
+const InfoPanel = () => {
+  const { t } = useTranslation()
+
+  return (
+    <div className='flex w-[360px] pt-[108px] pb-2 pr-8 flex-col items-start'>
+      <div className='flex min-w-[240px] w-full p-6 flex-col items-start gap-3 self-stretch rounded-xl bg-background-section'>
+        <div className='flex p-1 w-10 h-10 justify-center items-center gap-2 flex-grow self-stretch rounded-lg border-[0.5px] border-components-card-border bg-components-card-bg'>
+          <RiBookOpenLine className='w-5 h-5 text-text-accent' />
+        </div>
+        <p className='flex flex-col items-start gap-2 self-stretch'>
+          <span className='self-stretch text-text-secondary system-xl-semibold'>
+            {t('dataset.connectDatasetIntro.title')}
+          </span>
+          <span className='text-text-tertiary system-sm-regular'>
+            {t('dataset.connectDatasetIntro.content.front')}
+            <a className='text-text-accent system-sm-regular ml-1' href='https://docs.dify.ai/guides/knowledge-base/external-knowledge-api-documentation' target='_blank' rel="noopener noreferrer">
+              {t('dataset.connectDatasetIntro.content.link')}
+            </a>
+            {t('dataset.connectDatasetIntro.content.end')}
+          </span>
+          <a className='self-stretch text-text-accent system-sm-regular' href='https://docs.dify.ai/guides/knowledge-base/connect-external-knowledge' target='_blank' rel="noopener noreferrer">
+            {t('dataset.connectDatasetIntro.learnMore')}
+          </a>
+        </p>
+      </div>
+    </div>
+  )
+}
+
+export default InfoPanel

web/app/components/datasets/external-knowledge-base/create/KnowledgeBaseInfo.tsx

@@ -0,0 +1,53 @@
+import React from 'react'
+import { useTranslation } from 'react-i18next'
+import Input from '@/app/components/base/input'
+
+type KnowledgeBaseInfoProps = {
+  name: string
+  description?: string
+  onChange: (data: { name?: string; description?: string }) => void
+}
+
+const KnowledgeBaseInfo: React.FC<KnowledgeBaseInfoProps> = ({ name, description, onChange }) => {
+  const { t } = useTranslation()
+
+  const handleNameChange = (e: React.ChangeEvent<HTMLInputElement>) => {
+    onChange({ name: e.target.value })
+  }
+
+  const handleDescriptionChange = (e: React.ChangeEvent<HTMLTextAreaElement>) => {
+    onChange({ description: e.target.value })
+  }
+
+  return (
+    <form className='flex flex-col gap-4 self-stretch'>
+      <div className='flex flex-col gap-4 self-stretch'>
+        <div className='flex flex-col gap-1 self-stretch'>
+          <div className='flex flex-col justify-center self-stretch'>
+            <label className='text-text-secondary system-sm-semibold'>{t('dataset.externalKnowledgeName')}</label>
+          </div>
+          <Input
+            value={name}
+            onChange={handleNameChange}
+            placeholder={t('dataset.externalKnowledgeNamePlaceholder') ?? ''}
+          />
+        </div>
+        <div className='flex flex-col gap-1 self-stretch'>
+          <div className='flex flex-col justify-center self-stretch'>
+            <label className='text-text-secondary system-sm-semibold'>{t('dataset.externalKnowledgeDescription')}</label>
+          </div>
+          <div className='flex flex-col gap-1 self-stretch'>
+            <textarea
+              value={description}
+              onChange={ e => handleDescriptionChange(e)}
+              placeholder={t('dataset.externalKnowledgeDescriptionPlaceholder') ?? ''}
+              className={`flex h-20 py-2 p-3 self-stretch items-start rounded-lg bg-components-input-bg-normal ${description ? 'text-components-input-text-filled' : 'text-components-input-text-placeholder'} system-sm-regular`}
+            />
+          </div>
+        </div>
+      </div>
+    </form>
+  )
+}
+
+export default KnowledgeBaseInfo

web/app/components/datasets/external-knowledge-base/create/RetrievalSettings.tsx

@@ -0,0 +1,67 @@
+import type { FC } from 'react'
+import React from 'react'
+import { useTranslation } from 'react-i18next'
+import TopKItem from '@/app/components/base/param-item/top-k-item'
+import ScoreThresholdItem from '@/app/components/base/param-item/score-threshold-item'
+import cn from '@/utils/classnames'
+
+type RetrievalSettingsProps = {
+  topK: number
+  scoreThreshold: number
+  scoreThresholdEnabled: boolean
+  isInHitTesting?: boolean
+  isInRetrievalSetting?: boolean
+  onChange: (data: { top_k?: number; score_threshold?: number; score_threshold_enabled?: boolean }) => void
+}
+
+const RetrievalSettings: FC<RetrievalSettingsProps> = ({
+  topK,
+  scoreThreshold,
+  scoreThresholdEnabled,
+  onChange,
+  isInHitTesting = false,
+  isInRetrievalSetting = false,
+}) => {
+  const { t } = useTranslation()
+
+  const handleScoreThresholdChange = (enabled: boolean) => {
+    onChange({ score_threshold_enabled: enabled })
+  }
+
+  return (
+    <div className={cn('flex flex-col gap-2 self-stretch', isInRetrievalSetting && 'w-full max-w-[480px]')}>
+      {!isInHitTesting && !isInRetrievalSetting && <div className='flex h-7 pt-1 flex-col gap-2 self-stretch'>
+        <label className='text-text-secondary system-sm-semibold'>{t('dataset.retrievalSettings')}</label>
+      </div>}
+      <div className={cn(
+        'flex gap-4 self-stretch',
+        {
+          'flex-col': isInHitTesting,
+          'flex-row': isInRetrievalSetting,
+          'flex-col sm:flex-row': !isInHitTesting && !isInRetrievalSetting,
+        },
+      )}>
+        <div className='flex flex-col gap-1 flex-grow'>
+          <TopKItem
+            className='grow'
+            value={topK}
+            onChange={(_key, v) => onChange({ top_k: v })}
+            enable={true}
+          />
+        </div>
+        <div className='flex flex-col gap-1 flex-grow'>
+          <ScoreThresholdItem
+            className='grow'
+            value={scoreThreshold}
+            onChange={(_key, v) => onChange({ score_threshold: v })}
+            enable={scoreThresholdEnabled}
+            hasSwitch={true}
+            onSwitchChange={(_key, v) => handleScoreThresholdChange(v)}
+          />
+        </div>
+      </div>
+    </div>
+  )
+}
+
+export default RetrievalSettings

web/app/components/datasets/external-knowledge-base/create/declarations.ts

@@ -0,0 +1,12 @@
+export type CreateKnowledgeBaseReq = {
+  name: string
+  description?: string
+  external_knowledge_api_id: string
+  provider: 'external'
+  external_knowledge_id: string
+  external_retrieval_model: {
+    top_k: number
+    score_threshold: number
+    score_threshold_enabled: boolean
+  }
+}

web/app/components/datasets/external-knowledge-base/create/index.tsx

@@ -0,0 +1,128 @@
+'use client'
+
+import { useCallback, useState } from 'react'
+import { useRouter } from 'next/navigation'
+import { RiArrowLeftLine, RiArrowRightLine } from '@remixicon/react'
+import { useTranslation } from 'react-i18next'
+import KnowledgeBaseInfo from './KnowledgeBaseInfo'
+import ExternalApiSelection from './ExternalApiSelection'
+import RetrievalSettings from './RetrievalSettings'
+import InfoPanel from './InfoPanel'
+import type { CreateKnowledgeBaseReq } from './declarations'
+import Divider from '@/app/components/base/divider'
+import Button from '@/app/components/base/button'
+
+type ExternalKnowledgeBaseCreateProps = {
+  onConnect: (formValue: CreateKnowledgeBaseReq) => void
+  loading: boolean
+}
+
+const ExternalKnowledgeBaseCreate: React.FC<ExternalKnowledgeBaseCreateProps> = ({ onConnect, loading }) => {
+  const { t } = useTranslation()
+  const router = useRouter()
+  const [formData, setFormData] = useState<CreateKnowledgeBaseReq>({
+    name: '',
+    description: '',
+    external_knowledge_api_id: '',
+    external_knowledge_id: '',
+    external_retrieval_model: {
+      top_k: 2,
+      score_threshold: 0.5,
+      score_threshold_enabled: false,
+    },
+    provider: 'external',
+
+  })
+
+  const navBackHandle = useCallback(() => {
+    router.replace('/datasets')
+  }, [router])
+
+  const handleFormChange = (newData: CreateKnowledgeBaseReq) => {
+    setFormData(newData)
+  }
+
+  const isFormValid = formData.name.trim() !== ''
+    && formData.external_knowledge_api_id !== ''
+    && formData.external_knowledge_id !== ''
+    && formData.external_retrieval_model.top_k !== undefined
+    && formData.external_retrieval_model.score_threshold !== undefined
+
+  return (
+    <div className='flex flex-col flex-grow self-stretch rounded-t-2xl border-t border-effects-highlight bg-components-panel-bg'>
+      <div className='flex justify-center flex-grow self-stretch'>
+        <div className='flex w-full max-w-[960px] px-14 py-0 flex-col items-center'>
+          <div className='flex w-full max-w-[640px] pt-6 pb-8 flex-col grow items-center gap-4'>
+            <div className='relative flex flex-col py-2 items-center gap-[2px] self-stretch'>
+              <div className='flex-grow text-text-primary system-xl-semibold self-stretch'>{t('dataset.connectDataset')}</div>
+              <p className='text-text-tertiary system-sm-regular'>
+                <span>{t('dataset.connectHelper.helper1')}</span>
+                <span className='text-text-secondary system-sm-medium'>{t('dataset.connectHelper.helper2')}</span>
+                <span>{t('dataset.connectHelper.helper3')}</span>
+                <a className='self-stretch text-text-accent system-sm-regular' href='https://docs.dify.ai/guides/knowledge-base/connect-external-knowledge' target='_blank' rel="noopener noreferrer">
+                  {t('dataset.connectHelper.helper4')}
+                </a>
+                <span>{t('dataset.connectHelper.helper5')} </span>
+              </p>
+              <Button
+                className='flex w-8 h-8 p-2 items-center justify-center absolute left-[-44px] top-1 rounded-full'
+                variant='tertiary'
+                onClick={navBackHandle}
+              >
+                <RiArrowLeftLine className='w-4 h-4 text-text-tertiary' />
+              </Button>
+            </div>
+            <KnowledgeBaseInfo
+              name={formData.name}
+              description={formData.description ?? ''}
+              onChange={data => handleFormChange({
+                ...formData,
+                ...data,
+              })}
+            />
+            <Divider />
+            <ExternalApiSelection
+              external_knowledge_api_id={formData.external_knowledge_api_id}
+              external_knowledge_id={formData.external_knowledge_id}
+              onChange={data => handleFormChange({
+                ...formData,
+                ...data,
+              })}
+            />
+            <RetrievalSettings
+              topK={formData.external_retrieval_model.top_k}
+              scoreThreshold={formData.external_retrieval_model.score_threshold}
+              scoreThresholdEnabled={formData.external_retrieval_model.score_threshold_enabled}
+              onChange={data => handleFormChange({
+                ...formData,
+                external_retrieval_model: {
+                  ...formData.external_retrieval_model,
+                  ...data,
+                },
+              })}
+            />
+            <div className='flex py-2 justify-end items-center gap-2 self-stretch'>
+              <Button variant='secondary' onClick={navBackHandle}>
+                <div className='text-components-button-secondary-text system-sm-medium'>{t('dataset.externalKnowledgeForm.cancel')}</div>
+              </Button>
+              <Button
+                variant='primary'
+                onClick={() => {
+                  onConnect(formData)
+                }}
+                disabled={!isFormValid}
+                loading={loading}
+              >
+                <div className='text-components-button-primary-text system-sm-medium'>{t('dataset.externalKnowledgeForm.connect')}</div>
+                <RiArrowRightLine className='w-4 h-4 text-components-button-primary-text' />
+              </Button>
+            </div>
+          </div>
+        </div>
+        <InfoPanel />
+      </div>
+    </div>
+  )
+}
+
+export default ExternalKnowledgeBaseCreate

web/app/components/datasets/hit-testing/hit-detail.tsx

@@ -26,12 +26,15 @@ const HitDetail: FC<IHitDetailProps> = ({ segInfo }) => {
       )
     }
 
-    return segInfo?.content
+    return <div className='mb-4 text-md text-gray-800 h-full'>{segInfo?.content}</div>
   }
 
   return (
-    <div className='overflow-x-auto'>
-      <div className="bg-gray-25 p-6">
+    segInfo?.id === 'external'
+      ? <div className='w-full overflow-x-auto px-2'>
+        <div className={s.segModalContent}>{renderContent()}</div>
+      </div>
+      : <div className='overflow-x-auto'>
         <div className="flex items-center">
           <SegmentIndexTag
             positionId={segInfo?.position || ''}
@@ -59,7 +62,6 @@ const HitDetail: FC<IHitDetailProps> = ({ segInfo }) => {
             })}
         </div>
       </div>
-    </div>
   )
 }
 

web/app/components/datasets/hit-testing/index.tsx

@@ -13,7 +13,7 @@ import s from './style.module.css'
 import HitDetail from './hit-detail'
 import ModifyRetrievalModal from './modify-retrieval-modal'
 import cn from '@/utils/classnames'
-import type { HitTestingResponse, HitTesting as HitTestingType } from '@/models/datasets'
+import type { ExternalKnowledgeBaseHitTestingResponse, ExternalKnowledgeBaseHitTesting as ExternalKnowledgeBaseHitTestingType, HitTestingResponse, HitTesting as HitTestingType } from '@/models/datasets'
 import Loading from '@/app/components/base/loading'
 import Modal from '@/app/components/base/modal'
 import Drawer from '@/app/components/base/drawer'
@@ -49,8 +49,10 @@ const HitTesting: FC<Props> = ({ datasetId }: Props) => {
   const isMobile = media === MediaType.mobile
 
   const [hitResult, setHitResult] = useState<HitTestingResponse | undefined>() // 初始化记录为空数组
+  const [externalHitResult, setExternalHitResult] = useState<ExternalKnowledgeBaseHitTestingResponse | undefined>()
   const [submitLoading, setSubmitLoading] = useState(false)
   const [currParagraph, setCurrParagraph] = useState<{ paraInfo?: HitTestingType; showModal: boolean }>({ showModal: false })
+  const [externalCurrParagraph, setExternalCurrParagraph] = useState<{ paraInfo?: ExternalKnowledgeBaseHitTestingType; showModal: boolean }>({ showModal: false })
   const [text, setText] = useState('')
 
   const [currPage, setCurrPage] = React.useState<number>(0)
@@ -66,12 +68,52 @@ const HitTesting: FC<Props> = ({ datasetId }: Props) => {
     setCurrParagraph({ paraInfo: detail, showModal: true })
   }
 
+  const onClickExternalCard = (detail: ExternalKnowledgeBaseHitTestingType) => {
+    setExternalCurrParagraph({ paraInfo: detail, showModal: true })
+  }
   const { dataset: currentDataset } = useContext(DatasetDetailContext)
+  const isExternal = currentDataset?.provider === 'external'
 
   const [retrievalConfig, setRetrievalConfig] = useState(currentDataset?.retrieval_model_dict as RetrievalConfig)
   const [isShowModifyRetrievalModal, setIsShowModifyRetrievalModal] = useState(false)
   const [isShowRightPanel, { setTrue: showRightPanel, setFalse: hideRightPanel, set: setShowRightPanel }] = useBoolean(!isMobile)
 
+  const renderHitResults = (results: any[], onClickCard: (record: any) => void) => (
+    <>
+      <div className='text-gray-600 font-semibold mb-4'>{t('datasetHitTesting.hit.title')}</div>
+      <div className='overflow-auto flex-1'>
+        <div className={s.cardWrapper}>
+          {results.map((record, idx) => (
+            <SegmentCard
+              key={idx}
+              loading={false}
+              refSource= {{
+                title: record.title,
+                uri: record.metadata ? record.metadata['x-amz-bedrock-kb-source-uri'] : '',
+              }}
+              isExternal={isExternal}
+              detail={record.segment}
+              contentExternal={record.content}
+              score={record.score}
+              scene='hitTesting'
+              className='h-[216px] mb-4'
+              onClick={() => onClickCard(record)}
+            />
+          ))}
+        </div>
+      </div>
+    </>
+  )
+
+  const renderEmptyState = () => (
+    <div className='h-full flex flex-col justify-center items-center'>
+      <div className={cn(docStyle.commonIcon, docStyle.targetIcon, '!bg-gray-200 !h-14 !w-14')} />
+      <div className='text-gray-300 text-[13px] mt-3'>
+        {t('datasetHitTesting.hit.emptyTip')}
+      </div>
+    </div>
+  )
+
   useEffect(() => {
     setShowRightPanel(!isMobile)
   }, [isMobile, setShowRightPanel])
@@ -86,12 +128,14 @@ const HitTesting: FC<Props> = ({ datasetId }: Props) => {
         <Textarea
           datasetId={datasetId}
           setHitResult={setHitResult}
+          setExternalHitResult={setExternalHitResult}
           onSubmit={showRightPanel}
           onUpdateList={recordsMutate}
           loading={submitLoading}
           setLoading={setSubmitLoading}
           setText={setText}
           text={text}
+          isExternal={isExternal}
           onClickRetrievalMethod={() => setIsShowModifyRetrievalModal(true)}
           retrievalConfig={retrievalConfig}
           isEconomy={currentDataset?.indexing_technique === 'economy'}
@@ -159,47 +203,42 @@ const HitTesting: FC<Props> = ({ datasetId }: Props) => {
                 className='h-[216px]'
               />
             </div>
-            : !hitResult?.records.length
-              ? (
-                <div className='h-full flex flex-col justify-center items-center'>
-                  <div className={cn(docStyle.commonIcon, docStyle.targetIcon, '!bg-gray-200 !h-14 !w-14')} />
-                  <div className='text-gray-300 text-[13px] mt-3'>
-                    {t('datasetHitTesting.hit.emptyTip')}
-                  </div>
-                </div>
-              )
-              : (
-                <>
-                  <div className='text-gray-600 font-semibold mb-4'>{t('datasetHitTesting.hit.title')}</div>
-                  <div className='overflow-auto flex-1'>
-                    <div className={s.cardWrapper}>
-                      {hitResult?.records.map((record, idx) => {
-                        return <SegmentCard
-                          key={idx}
-                          loading={false}
-                          detail={record.segment as any}
-                          score={record.score}
-                          scene='hitTesting'
-                          className='h-[216px] mb-4'
-                          onClick={() => onClickCard(record as any)}
-                        />
-                      })}
-                    </div>
-                  </div>
-                </>
-              )
+            : (
+              (() => {
+                if (!hitResult?.records.length && !externalHitResult?.records.length)
+                  return renderEmptyState()
+
+                if (hitResult?.records.length)
+                  return renderHitResults(hitResult.records, onClickCard)
+
+                return renderHitResults(externalHitResult?.records || [], onClickExternalCard)
+              })()
+            )
           }
         </div>
       </FloatRightContainer>
       <Modal
-        className='w-[520px] p-0'
+        className={isExternal ? 'py-10 px-8' : 'w-full'}
         closable
-        onClose={() => setCurrParagraph({ showModal: false })}
-        isShow={currParagraph.showModal}
+        onClose={() => {
+          setCurrParagraph({ showModal: false })
+          setExternalCurrParagraph({ showModal: false })
+        }}
+        isShow={currParagraph.showModal || externalCurrParagraph.showModal}
       >
-        {currParagraph.showModal && <HitDetail
-          segInfo={currParagraph.paraInfo?.segment}
-        />}
+        {currParagraph.showModal && (
+          <HitDetail
+            segInfo={currParagraph.paraInfo?.segment}
+          />
+        )}
+        {externalCurrParagraph.showModal && (
+          <HitDetail
+            segInfo={{
+              id: 'external',
+              content: externalCurrParagraph.paraInfo?.content,
+            }}
+          />
+        )}
       </Modal>
       <Drawer isOpen={isShowModifyRetrievalModal} onClose={() => setIsShowModifyRetrievalModal(false)} footer={null} mask={isMobile} panelClassname='mt-16 mx-2 sm:mr-2 mb-3 !p-0 !max-w-[640px] rounded-xl'>
         <ModifyRetrievalModal

web/app/components/datasets/hit-testing/modify-external-retrieval-modal.tsx

@@ -0,0 +1,71 @@
+import { useState } from 'react'
+import {
+  RiCloseLine,
+} from '@remixicon/react'
+import { useTranslation } from 'react-i18next'
+import RetrievalSettings from '../external-knowledge-base/create/RetrievalSettings'
+import Button from '@/app/components/base/button'
+import ActionButton from '@/app/components/base/action-button'
+
+type ModifyExternalRetrievalModalProps = {
+  onClose: () => void
+  onSave: (data: { top_k: number; score_threshold: number; score_threshold_enabled: boolean }) => void
+  initialTopK: number
+  initialScoreThreshold: number
+  initialScoreThresholdEnabled: boolean
+}
+
+const ModifyExternalRetrievalModal: React.FC<ModifyExternalRetrievalModalProps> = ({
+  onClose,
+  onSave,
+  initialTopK,
+  initialScoreThreshold,
+  initialScoreThresholdEnabled,
+}) => {
+  const { t } = useTranslation()
+  const [topK, setTopK] = useState(initialTopK)
+  const [scoreThreshold, setScoreThreshold] = useState(initialScoreThreshold)
+  const [scoreThresholdEnabled, setScoreThresholdEnabled] = useState(initialScoreThresholdEnabled)
+
+  const handleSettingsChange = (data: { top_k?: number; score_threshold?: number; score_threshold_enabled?: boolean }) => {
+    if (data.top_k !== undefined)
+      setTopK(data.top_k)
+    if (data.score_threshold !== undefined)
+      setScoreThreshold(data.score_threshold)
+    if (data.score_threshold_enabled !== undefined)
+      setScoreThresholdEnabled(data.score_threshold_enabled)
+  }
+
+  const handleSave = () => {
+    onSave({ top_k: topK, score_threshold: scoreThreshold, score_threshold_enabled: scoreThresholdEnabled })
+    onClose()
+  }
+
+  return (
+    <div className='absolute z-10 top-[36px] right-[14px] flex w-[320px] flex-col items-start rounded-2xl border-[0.5px]
+      border-components-panel-border bg-components-panel-bg shadows-shadow-2xl'
+    >
+      <div className='flex p-4 pb-2 items-center justify-between self-stretch'>
+        <div className='text-text-primary system-xl-semibold flex-grow'>{t('datasetHitTesting.settingTitle')}</div>
+        <ActionButton className='ml-auto' onClick={onClose}>
+          <RiCloseLine className='w-4 h-4 flex-shrink-0' />
+        </ActionButton>
+      </div>
+      <div className='flex p-4 pt-2 flex-col justify-center items-start gap-4 self-stretch'>
+        <RetrievalSettings
+          topK={topK}
+          scoreThreshold={scoreThreshold}
+          scoreThresholdEnabled={scoreThresholdEnabled}
+          onChange={handleSettingsChange}
+          isInHitTesting={true}
+        />
+      </div>
+      <div className='flex p-4 pt-2 justify-end items-end gap-1 w-full'>
+        <Button className='flex-shrink-0 min-w-[72px]' onClick={onClose}>{t('common.operation.cancel')}</Button>
+        <Button variant='primary' className='flex-shrink-0 min-w-[72px]' onClick={handleSave}>{t('common.operation.save')}</Button>
+      </div>
+    </div>
+  )
+}
+
+export default ModifyExternalRetrievalModal

web/app/components/datasets/hit-testing/textarea.tsx

@@ -1,12 +1,17 @@
+import React, { useState } from 'react'
 import { useTranslation } from 'react-i18next'
+import {
+  RiEqualizer2Line,
+} from '@remixicon/react'
 import Button from '../../base/button'
 import Tag from '../../base/tag'
 import { getIcon } from '../common/retrieval-method-info'
 import s from './style.module.css'
+import ModifyExternalRetrievalModal from './modify-external-retrieval-modal'
 import Tooltip from '@/app/components/base/tooltip'
 import cn from '@/utils/classnames'
-import type { HitTestingResponse } from '@/models/datasets'
-import { hitTesting } from '@/service/datasets'
+import type { ExternalKnowledgeBaseHitTestingResponse, HitTestingResponse } from '@/models/datasets'
+import { externalKnowledgeBaseHitTesting, hitTesting } from '@/service/datasets'
 import { asyncRunSafe } from '@/utils'
 import { RETRIEVE_METHOD, type RetrievalConfig } from '@/types/app'
 
@@ -14,10 +19,12 @@ type TextAreaWithButtonIProps = {
   datasetId: string
   onUpdateList: () => void
   setHitResult: (res: HitTestingResponse) => void
+  setExternalHitResult: (res: ExternalKnowledgeBaseHitTestingResponse) => void
   loading: boolean
   setLoading: (v: boolean) => void
   text: string
   setText: (v: string) => void
+  isExternal?: boolean
   onClickRetrievalMethod: () => void
   retrievalConfig: RetrievalConfig
   isEconomy: boolean
@@ -28,16 +35,29 @@ const TextAreaWithButton = ({
   datasetId,
   onUpdateList,
   setHitResult,
+  setExternalHitResult,
   setLoading,
   loading,
   text,
   setText,
+  isExternal = false,
   onClickRetrievalMethod,
   retrievalConfig,
   isEconomy,
   onSubmit: _onSubmit,
 }: TextAreaWithButtonIProps) => {
   const { t } = useTranslation()
+  const [isSettingsOpen, setIsSettingsOpen] = useState(false)
+  const [externalRetrievalSettings, setExternalRetrievalSettings] = useState({
+    top_k: 2,
+    score_threshold: 0.5,
+    score_threshold_enabled: false,
+  })
+
+  const handleSaveExternalRetrievalSettings = (data: { top_k: number; score_threshold: number; score_threshold_enabled: boolean }) => {
+    setExternalRetrievalSettings(data)
+    setIsSettingsOpen(false)
+  }
 
   function handleTextChange(event: any) {
     setText(event.target.value)
@@ -63,28 +83,70 @@ const TextAreaWithButton = ({
     _onSubmit && _onSubmit()
   }
 
+  const externalRetrievalTestingOnSubmit = async () => {
+    const [e, res] = await asyncRunSafe<ExternalKnowledgeBaseHitTestingResponse>(
+      externalKnowledgeBaseHitTesting({
+        datasetId,
+        query: text,
+        external_retrieval_model: {
+          top_k: externalRetrievalSettings.top_k,
+          score_threshold: externalRetrievalSettings.score_threshold,
+          score_threshold_enabled: externalRetrievalSettings.score_threshold_enabled,
+        },
+      }) as Promise<ExternalKnowledgeBaseHitTestingResponse>,
+    )
+    if (!e) {
+      setExternalHitResult(res)
+      onUpdateList?.()
+    }
+    setLoading(false)
+  }
+
   const retrievalMethod = isEconomy ? RETRIEVE_METHOD.invertedIndex : retrievalConfig.search_method
   const Icon = getIcon(retrievalMethod)
   return (
     <>
       <div className={s.wrapper}>
-        <div className='pt-2 rounded-tl-xl rounded-tr-xl bg-[#EEF4FF]'>
+        <div className='relative pt-2 rounded-tl-xl rounded-tr-xl bg-[#EEF4FF]'>
           <div className="px-4 pb-2 flex justify-between h-8 items-center">
             <span className="text-gray-800 font-semibold text-sm">
               {t('datasetHitTesting.input.title')}
             </span>
-            <Tooltip
-              popupContent={t('dataset.retrieval.changeRetrievalMethod')}
-            >
-              <div
-                onClick={onClickRetrievalMethod}
-                className='flex px-2 h-7 items-center space-x-1 bg-white hover:bg-[#ECE9FE] rounded-md shadow-sm cursor-pointer text-[#6927DA]'
+            {isExternal
+              ? <Button
+                variant='secondary'
+                size='small'
+                onClick={() => setIsSettingsOpen(!isSettingsOpen)}
               >
-                <Icon className='w-3.5 h-3.5'></Icon>
-                <div className='text-xs font-medium'>{t(`dataset.retrieval.${retrievalMethod}.title`)}</div>
-              </div>
-            </Tooltip>
+                <RiEqualizer2Line className='text-components-button-secondary-text w-3.5 h-3.5' />
+                <div className='flex px-[3px] justify-center items-center gap-1'>
+                  <span className='text-components-button-secondary-text system-xs-medium'>{t('datasetHitTesting.settingTitle')}</span>
+                </div>
+              </Button>
+              : <Tooltip
+                popupContent={t('dataset.retrieval.changeRetrievalMethod')}
+              >
+                <div
+                  onClick={onClickRetrievalMethod}
+                  className='flex px-2 h-7 items-center space-x-1 bg-white hover:bg-[#ECE9FE] rounded-md shadow-sm cursor-pointer text-[#6927DA]'
+                >
+                  <Icon className='w-3.5 h-3.5'></Icon>
+                  <div className='text-xs font-medium'>{t(`dataset.retrieval.${retrievalMethod}.title`)}</div>
+                </div>
+              </Tooltip>
+            }
           </div>
+          {
+            isSettingsOpen && (
+              <ModifyExternalRetrievalModal
+                onClose={() => setIsSettingsOpen(false)}
+                onSave={handleSaveExternalRetrievalSettings}
+                initialTopK={externalRetrievalSettings.top_k}
+                initialScoreThreshold={externalRetrievalSettings.score_threshold}
+                initialScoreThresholdEnabled={externalRetrievalSettings.score_threshold_enabled}
+              />
+            )
+          }
           <div className='h-2 rounded-tl-xl rounded-tr-xl bg-white'></div>
         </div>
         <div className='px-4 pb-11'>
@@ -122,7 +184,7 @@ const TextAreaWithButton = ({
 
             <div>
               <Button
-                onClick={onSubmit}
+                onClick={isExternal ? externalRetrievalTestingOnSubmit : onSubmit}
                 variant="primary"
                 loading={loading}
                 disabled={(!text?.length || text?.length > 200)}
@@ -132,7 +194,6 @@ const TextAreaWithButton = ({
             </div>
           </div>
         </div>
-
       </div>
     </>
   )

web/app/components/datasets/rename-modal/index.tsx

@@ -3,7 +3,6 @@
 import type { MouseEventHandler } from 'react'
 import { useState } from 'react'
 import { RiCloseLine } from '@remixicon/react'
-import { BookOpenIcon } from '@heroicons/react/24/outline'
 import { useContext } from 'use-context-selector'
 import { useTranslation } from 'react-i18next'
 import cn from '@/utils/classnames'
@@ -26,6 +25,8 @@ const RenameDatasetModal = ({ show, dataset, onSuccess, onClose }: RenameDataset
   const [loading, setLoading] = useState(false)
   const [name, setName] = useState<string>(dataset.name)
   const [description, setDescription] = useState<string>(dataset.description)
+  const [externalKnowledgeId, setExternalKnowledgeId] = useState<string>(dataset.external_knowledge_info.external_knowledge_id)
+  const [externalKnowledgeApiId, setExternalKnowledgeApiId] = useState<string>(dataset.external_knowledge_info.external_knowledge_api_id)
 
   const onConfirm: MouseEventHandler = async () => {
     if (!name.trim()) {
@@ -34,12 +35,17 @@ const RenameDatasetModal = ({ show, dataset, onSuccess, onClose }: RenameDataset
     }
     try {
       setLoading(true)
+      const body: Partial<DataSet> & { external_knowledge_id?: string; external_knowledge_api_id?: string } = {
+        name,
+        description,
+      }
+      if (externalKnowledgeId && externalKnowledgeApiId) {
+        body.external_knowledge_id = externalKnowledgeId
+        body.external_knowledge_api_id = externalKnowledgeApiId
+      }
       await updateDatasetSetting({
         datasetId: dataset.id,
-        body: {
-          name,
-          description,
-        },
+        body,
       })
       notify({ type: 'success', message: t('common.actionMsg.modifiedSuccessfully') })
       if (onSuccess)
@@ -87,10 +93,6 @@ const RenameDatasetModal = ({ show, dataset, onSuccess, onClose }: RenameDataset
               className='block px-3 py-2 w-full h-[88px] rounded-lg bg-gray-100 text-sm outline-none appearance-none resize-none'
               placeholder={t('datasetSettings.form.descPlaceholder') || ''}
             />
-            <a className='mt-2 flex items-center h-[18px] px-3 text-xs text-gray-500 hover:text-primary-600' href="https://docs.dify.ai/features/datasets#how-to-write-a-good-dataset-description" target='_blank' rel='noopener noreferrer'>
-              <BookOpenIcon className='w-3 h-[18px] mr-1' />
-              {t('datasetSettings.form.descWrite')}
-            </a>
           </div>
         </div>
       </div>

web/app/components/datasets/settings/form/index.tsx

@@ -2,17 +2,19 @@
 import { useState } from 'react'
 import { useMount } from 'ahooks'
 import { useContext } from 'use-context-selector'
-import { BookOpenIcon } from '@heroicons/react/24/outline'
 import { useTranslation } from 'react-i18next'
 import { useSWRConfig } from 'swr'
 import { unstable_serialize } from 'swr/infinite'
 import PermissionSelector from '../permission-selector'
 import IndexMethodRadio from '../index-method-radio'
+import RetrievalSettings from '../../external-knowledge-base/create/RetrievalSettings'
 import cn from '@/utils/classnames'
 import RetrievalMethodConfig from '@/app/components/datasets/common/retrieval-method-config'
 import EconomicalRetrievalMethodConfig from '@/app/components/datasets/common/economical-retrieval-method-config'
 import { ToastContext } from '@/app/components/base/toast'
 import Button from '@/app/components/base/button'
+import Divider from '@/app/components/base/divider'
+import { ApiConnectionMod } from '@/app/components/base/icons/src/vender/solid/development'
 import { updateDatasetSetting } from '@/service/datasets'
 import type { DataSetListResponse } from '@/models/datasets'
 import DatasetDetailContext from '@/context/dataset-detail'
@@ -55,6 +57,9 @@ const Form = () => {
   const [name, setName] = useState(currentDataset?.name ?? '')
   const [description, setDescription] = useState(currentDataset?.description ?? '')
   const [permission, setPermission] = useState(currentDataset?.permission)
+  const [topK, setTopK] = useState(currentDataset?.external_retrieval_model.top_k ?? 2)
+  const [scoreThreshold, setScoreThreshold] = useState(currentDataset?.external_retrieval_model.score_threshold ?? 0.5)
+  const [scoreThresholdEnabled, setScoreThresholdEnabled] = useState(currentDataset?.external_retrieval_model.score_threshold_enabled ?? false)
   const [selectedMemberIDs, setSelectedMemberIDs] = useState<string[]>(currentDataset?.partial_member_list || [])
   const [memberList, setMemberList] = useState<Member[]>([])
   const [indexMethod, setIndexMethod] = useState(currentDataset?.indexing_technique)
@@ -85,6 +90,15 @@ const Form = () => {
       setMemberList(accounts)
   }
 
+  const handleSettingsChange = (data: { top_k?: number; score_threshold?: number; score_threshold_enabled?: boolean }) => {
+    if (data.top_k !== undefined)
+      setTopK(data.top_k)
+    if (data.score_threshold !== undefined)
+      setScoreThreshold(data.score_threshold)
+    if (data.score_threshold_enabled !== undefined)
+      setScoreThresholdEnabled(data.score_threshold_enabled)
+  }
+
   useMount(() => {
     getMembers()
   })
@@ -132,6 +146,15 @@ const Form = () => {
           },
           embedding_model: embeddingModel.model,
           embedding_model_provider: embeddingModel.provider,
+          ...(currentDataset!.provider === 'external' && {
+            external_knowledge_id: currentDataset!.external_knowledge_info.external_knowledge_id,
+            external_knowledge_api_id: currentDataset!.external_knowledge_info.external_knowledge_api_id,
+            external_retrieval_model: {
+              top_k: topK,
+              score_threshold: scoreThreshold,
+              score_threshold_enabled: scoreThresholdEnabled,
+            },
+          }),
         },
       } as any
       if (permission === 'partial_members') {
@@ -161,7 +184,7 @@ const Form = () => {
     <div className='w-full sm:w-[800px] p-4 sm:px-16 sm:py-6'>
       <div className={rowClass}>
         <div className={labelClass}>
-          <div>{t('datasetSettings.form.name')}</div>
+          <div className='text-text-secondary system-sm-semibold'>{t('datasetSettings.form.name')}</div>
         </div>
         <div className='w-full max-w-[480px]'>
           <input
@@ -174,7 +197,7 @@ const Form = () => {
       </div>
       <div className={rowClass}>
         <div className={labelClass}>
-          <div>{t('datasetSettings.form.desc')}</div>
+          <div className='text-text-secondary system-sm-semibold'>{t('datasetSettings.form.desc')}</div>
         </div>
         <div className='w-full max-w-[480px]'>
           <textarea
@@ -184,15 +207,11 @@ const Form = () => {
             value={description}
             onChange={e => setDescription(e.target.value)}
           />
-          <a className='flex items-center h-[18px] px-3 text-xs text-gray-500' href="https://docs.dify.ai/features/datasets#how-to-write-a-good-dataset-description" target='_blank' rel='noopener noreferrer'>
-            <BookOpenIcon className='w-3 h-[18px] mr-1' />
-            {t('datasetSettings.form.descWrite')}
-          </a>
         </div>
       </div>
       <div className={rowClass}>
         <div className={labelClass}>
-          <div>{t('datasetSettings.form.permissions')}</div>
+          <div className='text-text-secondary system-sm-semibold'>{t('datasetSettings.form.permissions')}</div>
         </div>
         <div className='w-full sm:w-[480px]'>
           <PermissionSelector
@@ -210,7 +229,7 @@ const Form = () => {
           <div className='w-full h-0 border-b-[0.5px] border-b-gray-200 my-2' />
           <div className={rowClass}>
             <div className={labelClass}>
-              <div>{t('datasetSettings.form.indexMethod')}</div>
+              <div className='text-text-secondary system-sm-semibold'>{t('datasetSettings.form.indexMethod')}</div>
             </div>
             <div className='w-full sm:w-[480px]'>
               <IndexMethodRadio
@@ -225,7 +244,7 @@ const Form = () => {
       {indexMethod === 'high_quality' && (
         <div className={rowClass}>
           <div className={labelClass}>
-            <div>{t('datasetSettings.form.embeddingModel')}</div>
+            <div className='text-text-secondary system-sm-semibold'>{t('datasetSettings.form.embeddingModel')}</div>
           </div>
           <div className='w-[480px]'>
             <ModelSelector
@@ -240,38 +259,84 @@ const Form = () => {
         </div>
       )}
       {/* Retrieval Method Config */}
-      <div className={rowClass}>
-        <div className={labelClass}>
-          <div>
-            <div>{t('datasetSettings.form.retrievalSetting.title')}</div>
-            <div className='leading-[18px] text-xs font-normal text-gray-500'>
-              <a target='_blank' rel='noopener noreferrer' href='https://docs.dify.ai/guides/knowledge-base/create-knowledge-and-upload-documents#id-4-retrieval-settings' className='text-[#155eef]'>{t('datasetSettings.form.retrievalSetting.learnMore')}</a>
-              {t('datasetSettings.form.retrievalSetting.description')}
+      {currentDataset?.provider === 'external'
+        ? <>
+          <div className={rowClass}><Divider/></div>
+          <div className={rowClass}>
+            <div className={labelClass}>
+              <div className='text-text-secondary system-sm-semibold'>{t('datasetSettings.form.retrievalSetting.title')}</div>
+            </div>
+            <RetrievalSettings
+              topK={topK}
+              scoreThreshold={scoreThreshold}
+              scoreThresholdEnabled={scoreThresholdEnabled}
+              onChange={handleSettingsChange}
+              isInRetrievalSetting={true}
+            />
+          </div>
+          <div className={rowClass}><Divider/></div>
+          <div className={rowClass}>
+            <div className={labelClass}>
+              <div className='text-text-secondary system-sm-semibold'>{t('datasetSettings.form.externalKnowledgeAPI')}</div>
+            </div>
+            <div className='w-full max-w-[480px]'>
+              <div className='flex h-full px-3 py-2 items-center gap-1 rounded-lg bg-components-input-bg-normal'>
+                <ApiConnectionMod className='w-4 h-4 text-text-secondary' />
+                <div className='overflow-hidden text-text-secondary text-ellipsis system-sm-medium'>
+                  {currentDataset?.external_knowledge_info.external_knowledge_api_name}
+                </div>
+                <div className='text-text-tertiary system-xs-regular'>·</div>
+                <div className='text-text-tertiary system-xs-regular'>{currentDataset?.external_knowledge_info.external_knowledge_api_endpoint}</div>
+              </div>
             </div>
           </div>
+          <div className={rowClass}>
+            <div className={labelClass}>
+              <div className='text-text-secondary system-sm-semibold'>{t('datasetSettings.form.externalKnowledgeID')}</div>
+            </div>
+            <div className='w-full max-w-[480px]'>
+              <div className='flex h-full px-3 py-2 items-center gap-1 rounded-lg bg-components-input-bg-normal'>
+                <div className='text-text-tertiary system-xs-regular'>{currentDataset?.external_knowledge_info.external_knowledge_id}</div>
+              </div>
+            </div>
+          </div>
+          <div className={rowClass}><Divider/></div>
+        </>
+        : <div className={rowClass}>
+          <div className={labelClass}>
+            <div>
+              <div className='text-text-secondary system-sm-semibold'>{t('datasetSettings.form.retrievalSetting.title')}</div>
+              <div className='leading-[18px] text-xs font-normal text-gray-500'>
+                <a target='_blank' rel='noopener noreferrer' href='https://docs.dify.ai/guides/knowledge-base/create-knowledge-and-upload-documents#id-4-retrieval-settings' className='text-[#155eef]'>{t('datasetSettings.form.retrievalSetting.learnMore')}</a>
+                {t('datasetSettings.form.retrievalSetting.description')}
+              </div>
+            </div>
+          </div>
+          <div className='w-[480px]'>
+            {indexMethod === 'high_quality'
+              ? (
+                <RetrievalMethodConfig
+                  value={retrievalConfig}
+                  onChange={setRetrievalConfig}
+                />
+              )
+              : (
+                <EconomicalRetrievalMethodConfig
+                  value={retrievalConfig}
+                  onChange={setRetrievalConfig}
+                />
+              )}
+          </div>
         </div>
-        <div className='w-[480px]'>
-          {indexMethod === 'high_quality'
-            ? (
-              <RetrievalMethodConfig
-                value={retrievalConfig}
-                onChange={setRetrievalConfig}
-              />
-            )
-            : (
-              <EconomicalRetrievalMethodConfig
-                value={retrievalConfig}
-                onChange={setRetrievalConfig}
-              />
-            )}
-        </div>
-      </div>
+      }
       <div className={rowClass}>
         <div className={labelClass} />
         <div className='w-[480px]'>
           <Button
             className='min-w-24'
             variant='primary'
+            loading={loading}
+            disabled={loading}
             onClick={handleSave}
           >
             {t('datasetSettings.form.save')}

web/app/components/header/account-dropdown/index.tsx

@@ -102,7 +102,16 @@ export default function AppSelector({ isMobile }: IAppSelector) {
                   </Menu.Item>
                   <div className="px-1 py-1">
                     <Menu.Item>
-                      <div className={itemClassName} onClick={() => setShowAccountSettingModal({ payload: 'provider' })}>
+                      <Link
+                        className={classNames(itemClassName, 'group justify-between')}
+                        href='/account'
+                        target='_self' rel='noopener noreferrer'>
+                        <div>{t('common.account.account')}</div>
+                        <ArrowUpRight className='hidden w-[14px] h-[14px] text-gray-500 group-hover:flex' />
+                      </Link>
+                    </Menu.Item>
+                    <Menu.Item>
+                      <div className={itemClassName} onClick={() => setShowAccountSettingModal({ payload: 'members' })}>
                         <div>{t('common.userProfile.settings')}</div>
                       </div>
                     </Menu.Item>

web/app/components/header/account-setting/data-source-page/data-source-website/config-firecrawl-modal.tsx

@@ -9,7 +9,7 @@ import {
 import { Lock01 } from '@/app/components/base/icons/src/vender/solid/security'
 import Button from '@/app/components/base/button'
 import type { FirecrawlConfig } from '@/models/common'
-import Field from '@/app/components/datasets/create/website/firecrawl/base/field'
+import Field from '@/app/components/datasets/create/website/base/field'
 import Toast from '@/app/components/base/toast'
 import { createDataSourceApiKeyBinding } from '@/service/datasets'
 import { LinkExternal02 } from '@/app/components/base/icons/src/vender/line/general'

web/app/components/header/account-setting/data-source-page/data-source-website/config-jina-reader-modal.tsx

@@ -0,0 +1,140 @@
+'use client'
+import type { FC } from 'react'
+import React, { useCallback, useState } from 'react'
+import { useTranslation } from 'react-i18next'
+import {
+  PortalToFollowElem,
+  PortalToFollowElemContent,
+} from '@/app/components/base/portal-to-follow-elem'
+import { Lock01 } from '@/app/components/base/icons/src/vender/solid/security'
+import Button from '@/app/components/base/button'
+import { DataSourceProvider } from '@/models/common'
+import Field from '@/app/components/datasets/create/website/base/field'
+import Toast from '@/app/components/base/toast'
+import { createDataSourceApiKeyBinding } from '@/service/datasets'
+import { LinkExternal02 } from '@/app/components/base/icons/src/vender/line/general'
+type Props = {
+  onCancel: () => void
+  onSaved: () => void
+}
+
+const I18N_PREFIX = 'datasetCreation.jinaReader'
+
+const ConfigJinaReaderModal: FC<Props> = ({
+  onCancel,
+  onSaved,
+}) => {
+  const { t } = useTranslation()
+  const [isSaving, setIsSaving] = useState(false)
+  const [apiKey, setApiKey] = useState('')
+
+  const handleSave = useCallback(async () => {
+    if (isSaving)
+      return
+    let errorMsg = ''
+    if (!errorMsg) {
+      if (!apiKey) {
+        errorMsg = t('common.errorMsg.fieldRequired', {
+          field: 'API Key',
+        })
+      }
+    }
+
+    if (errorMsg) {
+      Toast.notify({
+        type: 'error',
+        message: errorMsg,
+      })
+      return
+    }
+    const postData = {
+      category: 'website',
+      provider: DataSourceProvider.jinaReader,
+      credentials: {
+        auth_type: 'bearer',
+        config: {
+          api_key: apiKey,
+        },
+      },
+    }
+    try {
+      setIsSaving(true)
+      await createDataSourceApiKeyBinding(postData)
+      Toast.notify({
+        type: 'success',
+        message: t('common.api.success'),
+      })
+    }
+    finally {
+      setIsSaving(false)
+    }
+
+    onSaved()
+  }, [apiKey, onSaved, t, isSaving])
+
+  return (
+    <PortalToFollowElem open>
+      <PortalToFollowElemContent className='w-full h-full z-[60]'>
+        <div className='fixed inset-0 flex items-center justify-center bg-black/[.25]'>
+          <div className='mx-2 w-[640px] max-h-[calc(100vh-120px)] bg-white shadow-xl rounded-2xl overflow-y-auto'>
+            <div className='px-8 pt-8'>
+              <div className='flex justify-between items-center mb-4'>
+                <div className='text-xl font-semibold text-gray-900'>{t(`${I18N_PREFIX}.configJinaReader`)}</div>
+              </div>
+
+              <div className='space-y-4'>
+                <Field
+                  label='API Key'
+                  labelClassName='!text-sm'
+                  isRequired
+                  value={apiKey}
+                  onChange={(value: string | number) => setApiKey(value as string)}
+                  placeholder={t(`${I18N_PREFIX}.apiKeyPlaceholder`)!}
+                />
+              </div>
+              <div className='my-8 flex justify-between items-center h-8'>
+                <a className='flex items-center space-x-1 leading-[18px] text-xs font-normal text-[#155EEF]' target='_blank' href='https://jina.ai/reader/'>
+                  <span>{t(`${I18N_PREFIX}.getApiKeyLinkText`)}</span>
+                  <LinkExternal02 className='w-3 h-3' />
+                </a>
+                <div className='flex'>
+                  <Button
+                    size='large'
+                    className='mr-2'
+                    onClick={onCancel}
+                  >
+                    {t('common.operation.cancel')}
+                  </Button>
+                  <Button
+                    variant='primary'
+                    size='large'
+                    onClick={handleSave}
+                    loading={isSaving}
+                  >
+                    {t('common.operation.save')}
+                  </Button>
+                </div>
+
+              </div>
+            </div>
+            <div className='border-t-[0.5px] border-t-black/5'>
+              <div className='flex justify-center items-center py-3 bg-gray-50 text-xs text-gray-500'>
+                <Lock01 className='mr-1 w-3 h-3 text-gray-500' />
+                {t('common.modelProvider.encrypted.front')}
+                <a
+                  className='text-primary-600 mx-1'
+                  target='_blank' rel='noopener noreferrer'
+                  href='https://pycryptodome.readthedocs.io/en/latest/src/cipher/oaep.html'
+                >
+                  PKCS1_OAEP
+                </a>
+                {t('common.modelProvider.encrypted.back')}
+              </div>
+            </div>
+          </div>
+        </div>
+      </PortalToFollowElemContent>
+    </PortalToFollowElem>
+  )
+}
+export default React.memo(ConfigJinaReaderModal)

web/app/components/header/account-setting/data-source-page/data-source-website/index.tsx

@@ -2,11 +2,12 @@
 import type { FC } from 'react'
 import React, { useCallback, useEffect, useState } from 'react'
 import { useTranslation } from 'react-i18next'
-import { useBoolean } from 'ahooks'
 import Panel from '../panel'
 import { DataSourceType } from '../panel/types'
 import ConfigFirecrawlModal from './config-firecrawl-modal'
+import ConfigJinaReaderModal from './config-jina-reader-modal'
 import cn from '@/utils/classnames'
+import s from '@/app/components/datasets/create/website/index.module.css'
 import { fetchDataSources, removeDataSourceApiKeyBinding } from '@/service/datasets'
 
 import type {
@@ -19,9 +20,11 @@ import {
 } from '@/models/common'
 import Toast from '@/app/components/base/toast'
 
-type Props = {}
+type Props = {
+  provider: DataSourceProvider
+}
 
-const DataSourceWebsite: FC<Props> = () => {
+const DataSourceWebsite: FC<Props> = ({ provider }) => {
   const { t } = useTranslation()
   const { isCurrentWorkspaceManager } = useAppContext()
   const [sources, setSources] = useState<DataSourceItem[]>([])
@@ -36,22 +39,26 @@ const DataSourceWebsite: FC<Props> = () => {
     // eslint-disable-next-line react-hooks/exhaustive-deps
   }, [])
 
-  const [isShowConfig, {
-    setTrue: showConfig,
-    setFalse: hideConfig,
-  }] = useBoolean(false)
+  const [configTarget, setConfigTarget] = useState<DataSourceProvider | null>(null)
+  const showConfig = useCallback((provider: DataSourceProvider) => {
+    setConfigTarget(provider)
+  }, [setConfigTarget])
+
+  const hideConfig = useCallback(() => {
+    setConfigTarget(null)
+  }, [setConfigTarget])
 
   const handleAdded = useCallback(() => {
     checkSetApiKey()
     hideConfig()
   }, [checkSetApiKey, hideConfig])
 
-  const getIdByProvider = (provider: string): string | undefined => {
+  const getIdByProvider = (provider: DataSourceProvider): string | undefined => {
     const source = sources.find(item => item.provider === provider)
     return source?.id
   }
 
-  const handleRemove = useCallback((provider: string) => {
+  const handleRemove = useCallback((provider: DataSourceProvider) => {
     return async () => {
       const dataSourceId = getIdByProvider(provider)
       if (dataSourceId) {
@@ -69,22 +76,34 @@ const DataSourceWebsite: FC<Props> = () => {
     <>
       <Panel
         type={DataSourceType.website}
-        isConfigured={sources.length > 0}
-        onConfigure={showConfig}
+        provider={provider}
+        isConfigured={sources.find(item => item.provider === provider) !== undefined}
+        onConfigure={() => showConfig(provider)}
         readOnly={!isCurrentWorkspaceManager}
-        configuredList={sources.map(item => ({
+        configuredList={sources.filter(item => item.provider === provider).map(item => ({
           id: item.id,
           logo: ({ className }: { className: string }) => (
-            <div className={cn(className, 'flex items-center justify-center w-5 h-5 bg-white border border-gray-100 text-xs font-medium text-gray-500 rounded ml-3')}>🔥</div>
+            item.provider === DataSourceProvider.fireCrawl
+              ? (
+                <div className={cn(className, 'flex items-center justify-center w-5 h-5 bg-white border border-gray-100 text-xs font-medium text-gray-500 rounded ml-3')}>🔥</div>
+              )
+              : (
+                <div className={cn(className, 'flex items-center justify-center w-5 h-5 bg-white border border-gray-100 text-xs font-medium text-gray-500 rounded ml-3')}>
+                  <span className={s.jinaLogo} />
+                </div>
+              )
           ),
-          name: 'Firecrawl',
+          name: item.provider === DataSourceProvider.fireCrawl ? 'Firecrawl' : 'Jina Reader',
           isActive: true,
         }))}
-        onRemove={handleRemove(DataSourceProvider.fireCrawl)}
+        onRemove={handleRemove(provider)}
       />
-      {isShowConfig && (
+      {configTarget === DataSourceProvider.fireCrawl && (
         <ConfigFirecrawlModal onSaved={handleAdded} onCancel={hideConfig} />
       )}
+      {configTarget === DataSourceProvider.jinaReader && (
+        <ConfigJinaReaderModal onSaved={handleAdded} onCancel={hideConfig} />
+      )}
     </>
 
   )

web/app/components/header/account-setting/data-source-page/index.tsx

@@ -3,6 +3,7 @@ import { useTranslation } from 'react-i18next'
 import DataSourceNotion from './data-source-notion'
 import DataSourceWebsite from './data-source-website'
 import { fetchDataSource } from '@/service/common'
+import { DataSourceProvider } from '@/models/common'
 
 export default function DataSourcePage() {
   const { t } = useTranslation()
@@ -13,7 +14,8 @@ export default function DataSourcePage() {
     <div className='mb-8'>
       <div className='mb-2 text-sm font-medium text-gray-900'>{t('common.dataSource.add')}</div>
       <DataSourceNotion workspaces={notionWorkspaces} />
-      <DataSourceWebsite />
+      <DataSourceWebsite provider={DataSourceProvider.jinaReader} />
+      <DataSourceWebsite provider={DataSourceProvider.fireCrawl} />
     </div>
   )
 }

web/app/components/header/account-setting/data-source-page/panel/index.tsx

@@ -8,10 +8,12 @@ import ConfigItem from './config-item'
 
 import s from './style.module.css'
 import { DataSourceType } from './types'
+import { DataSourceProvider } from '@/models/common'
 import cn from '@/utils/classnames'
 
 type Props = {
   type: DataSourceType
+  provider: DataSourceProvider
   isConfigured: boolean
   onConfigure: () => void
   readOnly: boolean
@@ -25,6 +27,7 @@ type Props = {
 
 const Panel: FC<Props> = ({
   type,
+  provider,
   isConfigured,
   onConfigure,
   readOnly,
@@ -46,7 +49,7 @@ const Panel: FC<Props> = ({
             <div className='text-sm font-medium text-gray-800'>{t(`common.dataSource.${type}.title`)}</div>
             {isWebsite && (
               <div className='ml-1 leading-[18px] px-1.5 rounded-md bg-white border border-gray-100 text-xs font-medium text-gray-700'>
-                <span className='text-gray-500'>{t('common.dataSource.website.with')}</span> 🔥 Firecrawl
+                <span className='text-gray-500'>{t('common.dataSource.website.with')}</span> { provider === DataSourceProvider.fireCrawl ? '🔥 Firecrawl' : 'Jina Reader'}
               </div>
             )}
           </div>

web/app/components/header/account-setting/index.tsx

@@ -2,10 +2,6 @@
 import { useTranslation } from 'react-i18next'
 import { useEffect, useRef, useState } from 'react'
 import {
-  RiAccountCircleFill,
-  RiAccountCircleLine,
-  RiApps2AddFill,
-  RiApps2AddLine,
   RiBrainFill,
   RiBrainLine,
   RiColorFilterFill,
@@ -20,9 +16,7 @@ import {
   RiPuzzle2Line,
   RiTranslate2,
 } from '@remixicon/react'
-import AccountPage from './account-page'
 import MembersPage from './members-page'
-import IntegrationsPage from './Integrations-page'
 import LanguagePage from './language-page'
 import ApiBasedExtensionPage from './api-based-extension-page'
 import DataSourcePage from './data-source-page'
@@ -55,7 +49,7 @@ type GroupItem = {
 
 export default function AccountSetting({
   onCancel,
-  activeTab = 'provider',
+  activeTab = 'members',
 }: IAccountSettingProps) {
   const [activeMenu, setActiveMenu] = useState(activeTab)
   const { t } = useTranslation()
@@ -120,18 +114,6 @@ export default function AccountSetting({
       key: 'account-group',
       name: t('common.settings.generalGroup'),
       items: [
-        {
-          key: 'account',
-          name: t('common.settings.account'),
-          icon: <RiAccountCircleLine className={iconClassName} />,
-          activeIcon: <RiAccountCircleFill className={iconClassName} />,
-        },
-        {
-          key: 'integrations',
-          name: t('common.settings.integrations'),
-          icon: <RiApps2AddLine className={iconClassName} />,
-          activeIcon: <RiApps2AddFill className={iconClassName} />,
-        },
         {
           key: 'language',
           name: t('common.settings.language'),
@@ -217,10 +199,8 @@ export default function AccountSetting({
             )}
           </div>
           <div className='px-4 sm:px-8 pt-2'>
-            {activeMenu === 'account' && <AccountPage />}
             {activeMenu === 'members' && <MembersPage />}
             {activeMenu === 'billing' && <BillingPage />}
-            {activeMenu === 'integrations' && <IntegrationsPage />}
             {activeMenu === 'language' && <LanguagePage />}
             {activeMenu === 'provider' && <ModelProviderPage />}
             {activeMenu === 'data-source' && <DataSourcePage />}

web/app/components/header/account-setting/model-provider-page/model-icon/index.tsx

@@ -26,7 +26,7 @@ const ModelIcon: FC<ModelIconProps> = ({
     return (
       <img
         alt='model-icon'
-        src={`${provider.icon_small[language] || provider.icon_small.en_US}?_token=${localStorage.getItem('console_token')}`}
+        src={`${provider.icon_small[language] || provider.icon_small.en_US}`}
         className={`w-4 h-4 ${className}`}
       />
     )

web/app/components/header/account-setting/model-provider-page/provider-icon/index.tsx

@@ -16,7 +16,7 @@ const ProviderIcon: FC<ProviderIconProps> = ({
     return (
       <img
         alt='provider-icon'
-        src={`${provider.icon_large[language] || provider.icon_large.en_US}?_token=${localStorage.getItem('console_token')}`}
+        src={`${provider.icon_large[language] || provider.icon_large.en_US}`}
         className={`w-auto h-6 ${className}`}
       />
     )

web/app/components/header/dataset-nav/index.tsx

@@ -51,7 +51,7 @@ const DatasetNav = () => {
       navs={datasetItems.map(dataset => ({
         id: dataset.id,
         name: dataset.name,
-        link: `/datasets/${dataset.id}/documents`,
+        link: dataset.provider === 'external' ? `/datasets/${dataset.id}/hitTesting` : `/datasets/${dataset.id}/documents`,
         icon: dataset.icon,
         icon_background: dataset.icon_background,
       })) as NavItem[]}

web/app/components/header/header-wrapper.tsx

@@ -11,7 +11,7 @@ const HeaderWrapper = ({
   children,
 }: HeaderWrapperProps) => {
   const pathname = usePathname()
-  const isBordered = ['/apps', '/datasets', '/datasets/create', '/tools'].includes(pathname)
+  const isBordered = ['/apps', '/datasets', '/datasets/create', '/tools', '/account'].includes(pathname)
 
   return (
     <div className={classNames(

web/app/components/swr-initor.tsx

@@ -4,6 +4,7 @@ import { SWRConfig } from 'swr'
 import { useEffect, useState } from 'react'
 import type { ReactNode } from 'react'
 import { useRouter, useSearchParams } from 'next/navigation'
+import useRefreshToken from '@/hooks/use-refresh-token'
 
 type SwrInitorProps = {
   children: ReactNode
@@ -13,18 +14,31 @@ const SwrInitor = ({
 }: SwrInitorProps) => {
   const router = useRouter()
   const searchParams = useSearchParams()
-  const consoleToken = searchParams.get('console_token')
+  const consoleToken = searchParams.get('access_token')
+  const refreshToken = searchParams.get('refresh_token')
   const consoleTokenFromLocalStorage = localStorage?.getItem('console_token')
+  const refreshTokenFromLocalStorage = localStorage?.getItem('refresh_token')
   const [init, setInit] = useState(false)
+  const { getNewAccessToken } = useRefreshToken()
 
   useEffect(() => {
-    if (!(consoleToken || consoleTokenFromLocalStorage))
+    if (!(consoleToken || refreshToken || consoleTokenFromLocalStorage || refreshTokenFromLocalStorage)) {
       router.replace('/signin')
-
-    if (consoleToken) {
-      localStorage?.setItem('console_token', consoleToken!)
-      router.replace('/apps', { forceOptimisticNavigation: false } as any)
+      return
     }
+    if (consoleTokenFromLocalStorage && refreshTokenFromLocalStorage)
+      getNewAccessToken(consoleTokenFromLocalStorage, refreshTokenFromLocalStorage)
+
+    if (consoleToken && refreshToken) {
+      localStorage.setItem('console_token', consoleToken)
+      localStorage.setItem('refresh_token', refreshToken)
+      getNewAccessToken(consoleToken, refreshToken).then(() => {
+        router.replace('/apps', { forceOptimisticNavigation: false } as any)
+      }).catch(() => {
+        router.replace('/signin')
+      })
+    }
+
     setInit(true)
   }, [])
 

web/app/components/workflow/hooks/use-workflow-start-run.tsx

@@ -1,17 +1,25 @@
 import { useCallback } from 'react'
 import { useStoreApi } from 'reactflow'
+import { useTranslation } from 'react-i18next'
 import { useWorkflowStore } from '../store'
 import {
   BlockEnum,
   WorkflowRunningStatus,
 } from '../types'
+import type { KnowledgeRetrievalNodeType } from '../nodes/knowledge-retrieval/types'
+import type { Node } from '../types'
+import { useWorkflow } from './use-workflow'
 import {
   useIsChatMode,
   useNodesSyncDraft,
   useWorkflowInteractions,
   useWorkflowRun,
 } from './index'
+import { ModelTypeEnum } from '@/app/components/header/account-setting/model-provider-page/declarations'
+import { useCurrentProviderAndModel, useModelListAndDefaultModelAndCurrentProviderAndModel } from '@/app/components/header/account-setting/model-provider-page/hooks'
 import { useFeaturesStore } from '@/app/components/base/features/hooks'
+import KnowledgeRetrievalDefault from '@/app/components/workflow/nodes/knowledge-retrieval/default'
+import Toast from '@/app/components/base/toast'
 
 export const useWorkflowStartRun = () => {
   const store = useStoreApi()
@@ -20,7 +28,26 @@ export const useWorkflowStartRun = () => {
   const isChatMode = useIsChatMode()
   const { handleCancelDebugAndPreviewPanel } = useWorkflowInteractions()
   const { handleRun } = useWorkflowRun()
+  const { isFromStartNode } = useWorkflow()
   const { doSyncWorkflowDraft } = useNodesSyncDraft()
+  const { checkValid: checkKnowledgeRetrievalValid } = KnowledgeRetrievalDefault
+  const { t } = useTranslation()
+  const {
+    modelList: rerankModelList,
+    defaultModel: rerankDefaultModel,
+  } = useModelListAndDefaultModelAndCurrentProviderAndModel(ModelTypeEnum.rerank)
+
+  const {
+    currentModel,
+  } = useCurrentProviderAndModel(
+    rerankModelList,
+    rerankDefaultModel
+      ? {
+        ...rerankDefaultModel,
+        provider: rerankDefaultModel.provider.provider,
+      }
+      : undefined,
+  )
 
   const handleWorkflowStartRunInWorkflow = useCallback(async () => {
     const {
@@ -33,6 +60,9 @@ export const useWorkflowStartRun = () => {
     const { getNodes } = store.getState()
     const nodes = getNodes()
     const startNode = nodes.find(node => node.data.type === BlockEnum.Start)
+    const knowledgeRetrievalNodes = nodes.filter((node: Node<KnowledgeRetrievalNodeType>) =>
+      node.data.type === BlockEnum.KnowledgeRetrieval,
+    )
     const startVariables = startNode?.data.variables || []
     const fileSettings = featuresStore!.getState().features.file
     const {
@@ -42,6 +72,31 @@ export const useWorkflowStartRun = () => {
       setShowEnvPanel,
     } = workflowStore.getState()
 
+    if (knowledgeRetrievalNodes.length > 0) {
+      for (const node of knowledgeRetrievalNodes) {
+        if (isFromStartNode(node.id)) {
+          const res = checkKnowledgeRetrievalValid(node.data, t)
+          if (!res.isValid || !currentModel || !rerankDefaultModel) {
+            const errorMessage = res.errorMessage
+            if (errorMessage) {
+              Toast.notify({
+                type: 'error',
+                message: errorMessage,
+              })
+              return false
+            }
+            else {
+              Toast.notify({
+                type: 'error',
+                message: t('appDebug.datasetConfig.rerankModelRequired'),
+              })
+              return false
+            }
+          }
+        }
+      }
+    }
+
     setShowEnvPanel(false)
 
     if (showDebugAndPreviewPanel) {

web/app/components/workflow/hooks/use-workflow.ts

@@ -235,6 +235,33 @@ export const useWorkflow = () => {
     return nodes.filter(node => node.parentId === nodeId)
   }, [store])
 
+  const isFromStartNode = useCallback((nodeId: string) => {
+    const { getNodes } = store.getState()
+    const nodes = getNodes()
+    const currentNode = nodes.find(node => node.id === nodeId)
+
+    if (!currentNode)
+      return false
+
+    if (currentNode.data.type === BlockEnum.Start)
+      return true
+
+    const checkPreviousNodes = (node: Node) => {
+      const previousNodes = getBeforeNodeById(node.id)
+
+      for (const prevNode of previousNodes) {
+        if (prevNode.data.type === BlockEnum.Start)
+          return true
+        if (checkPreviousNodes(prevNode))
+          return true
+      }
+
+      return false
+    }
+
+    return checkPreviousNodes(currentNode)
+  }, [store, getBeforeNodeById])
+
   const handleOutVarRenameChange = useCallback((nodeId: string, oldValeSelector: ValueSelector, newVarSelector: ValueSelector) => {
     const { getNodes, setNodes } = store.getState()
     const afterNodes = getAfterNodesInSameBranch(nodeId)
@@ -389,6 +416,7 @@ export const useWorkflow = () => {
     checkParallelLimit,
     checkNestedParallelLimit,
     isValidConnection,
+    isFromStartNode,
     formatTimeFromNow,
     getNode,
     getBeforeNodeById,

web/app/components/workflow/nodes/_base/components/variable/utils.ts

@@ -116,6 +116,19 @@ const formatItem = (
         variable: 'sys.files',
         type: VarType.arrayFile,
       })
+      res.vars.push({
+        variable: 'sys.app_id',
+        type: VarType.string,
+      })
+      res.vars.push({
+        variable: 'sys.workflow_id',
+        type: VarType.string,
+      })
+      res.vars.push({
+        variable: 'sys.workflow_run_id',
+        type: VarType.string,
+      })
+
       break
     }
 

web/app/components/workflow/nodes/iteration-start/index.tsx

@@ -9,7 +9,7 @@ const IterationStartNode = ({ id, data }: NodeProps) => {
   const { t } = useTranslation()
 
   return (
-    <div className='group flex nodrag items-center justify-center w-11 h-11 rounded-2xl border border-workflow-block-border bg-white'>
+    <div className='group flex nodrag items-center justify-center w-11 h-11 mt-1 rounded-2xl border border-workflow-block-border bg-white'>
       <Tooltip popupContent={t('workflow.blocks.iteration-start')} asChild={false}>
         <div className='flex items-center justify-center w-6 h-6 rounded-full border-[0.5px] border-components-panel-border-subtle bg-util-colors-blue-brand-blue-brand-500'>
           <RiHome5Fill className='w-3 h-3 text-text-primary-on-surface' />

web/app/components/workflow/nodes/knowledge-retrieval/components/dataset-item.tsx

@@ -1,13 +1,15 @@
 'use client'
 import type { FC } from 'react'
-import React, { useCallback } from 'react'
+import React, { useCallback, useState } from 'react'
 import { useBoolean } from 'ahooks'
 import {
   RiDeleteBinLine,
   RiEditLine,
 } from '@remixicon/react'
+import { useTranslation } from 'react-i18next'
 import type { DataSet } from '@/models/datasets'
 import { DataSourceType } from '@/models/datasets'
+import ActionButton, { ActionButtonState } from '@/app/components/base/action-button'
 import FileIcon from '@/app/components/base/file-icon'
 import { Folder } from '@/app/components/base/icons/src/vender/solid/files'
 import SettingsModal from '@/app/components/app/configuration/dataset-config/settings-modal'
@@ -30,8 +32,10 @@ const DatasetItem: FC<Props> = ({
   readonly,
 }) => {
   const media = useBreakpoints()
+  const { t } = useTranslation()
   const isMobile = media === MediaType.mobile
   const { formatIndexingTechniqueAndMethod } = useKnowledge()
+  const [isDeleteHovered, setIsDeleteHovered] = useState(false)
 
   const [isShowSettingsModal, {
     setTrue: showSettingsModal,
@@ -43,8 +47,18 @@ const DatasetItem: FC<Props> = ({
     hideSettingsModal()
   }, [hideSettingsModal, onChange])
 
+  const handleRemove = useCallback((e: React.MouseEvent) => {
+    e.stopPropagation()
+    onRemove()
+  }, [onRemove])
+
   return (
-    <div className='flex items-center h-10 justify-between rounded-xl px-2 bg-white border border-gray-200  cursor-pointer group/dataset-item'>
+    <div className={`flex items-center h-10 justify-between rounded-xl px-2 border-[0.5px] 
+      border-components-panel-border-subtle cursor-pointer group/dataset-item 
+      ${isDeleteHovered
+      ? 'bg-state-destructive-hover border-state-destructive-border'
+      : 'bg-components-panel-on-panel-item-bg hover:bg-components-panel-on-panel-item-bg-hover'
+    }`}>
       <div className='w-0 grow flex items-center space-x-1.5'>
         {
           payload.data_source_type === DataSourceType.NOTION
@@ -61,24 +75,36 @@ const DatasetItem: FC<Props> = ({
       </div>
       {!readonly && (
         <div className='hidden group-hover/dataset-item:flex shrink-0 ml-2  items-center space-x-1'>
-          <div
-            className='flex items-center justify-center w-6 h-6 hover:bg-black/5 rounded-md cursor-pointer'
-            onClick={showSettingsModal}
+          <ActionButton
+            onClick={(e) => {
+              e.stopPropagation()
+              showSettingsModal()
+            }}
           >
-            <RiEditLine className='w-4 h-4 text-gray-500' />
-          </div>
-          <div
-            className='flex items-center justify-center w-6 h-6 hover:bg-black/5 rounded-md cursor-pointer'
-            onClick={onRemove}
+            <RiEditLine className='w-4 h-4 flex-shrink-0 text-text-tertiary' />
+          </ActionButton>
+          <ActionButton
+            onClick={handleRemove}
+            state={ActionButtonState.Destructive}
+            onMouseEnter={() => setIsDeleteHovered(true)}
+            onMouseLeave={() => setIsDeleteHovered(false)}
           >
-            <RiDeleteBinLine className='w-4 h-4 text-gray-500' />
-          </div>
+            <RiDeleteBinLine className={`w-4 h-4 flex-shrink-0 ${isDeleteHovered ? 'text-text-destructive' : 'text-text-tertiary'}`} />
+          </ActionButton>
         </div>
       )}
-      <Badge
-        className='group-hover/dataset-item:hidden shrink-0'
-        text={formatIndexingTechniqueAndMethod(payload.indexing_technique, payload.retrieval_model_dict?.search_method)}
-      />
+      {
+        payload.indexing_technique && <Badge
+          className='group-hover/dataset-item:hidden shrink-0'
+          text={formatIndexingTechniqueAndMethod(payload.indexing_technique, payload.retrieval_model_dict?.search_method)}
+        />
+      }
+      {
+        payload.provider === 'external' && <Badge
+          className='group-hover/dataset-item:hidden shrink-0'
+          text={t('dataset.externalTag')}
+        />
+      }
 
       {isShowSettingsModal && (
         <Drawer isOpen={isShowSettingsModal} onClose={hideSettingsModal} footer={null} mask={isMobile} panelClassname='mt-16 mx-2 sm:mr-2 mb-3 !p-0 !max-w-[640px] rounded-xl'>

web/app/components/workflow/nodes/knowledge-retrieval/use-config.ts

@@ -136,6 +136,8 @@ const useConfig = (id: string, payload: KnowledgeRetrievalNodeType) => {
         top_k: multipleRetrievalConfig?.top_k || DATASET_DEFAULT.top_k,
         score_threshold: multipleRetrievalConfig?.score_threshold,
         reranking_model: multipleRetrievalConfig?.reranking_model,
+        reranking_mode: multipleRetrievalConfig?.reranking_mode,
+        weights: multipleRetrievalConfig?.weights,
       }
     })
     setInputs(newInput)
@@ -205,9 +207,11 @@ const useConfig = (id: string, payload: KnowledgeRetrievalNodeType) => {
 
   const handleOnDatasetsChange = useCallback((newDatasets: DataSet[]) => {
     const {
-      allEconomic,
       mixtureHighQualityAndEconomic,
+      mixtureInternalAndExternal,
       inconsistentEmbeddingModel,
+      allInternal,
+      allExternal,
     } = getSelectedDatasetsMode(newDatasets)
     const newInputs = produce(inputs, (draft) => {
       draft.dataset_ids = newDatasets.map(d => d.id)
@@ -220,7 +224,11 @@ const useConfig = (id: string, payload: KnowledgeRetrievalNodeType) => {
     setInputs(newInputs)
     setSelectedDatasets(newDatasets)
 
-    if (allEconomic || mixtureHighQualityAndEconomic || inconsistentEmbeddingModel)
+    if (
+      (allInternal && (mixtureHighQualityAndEconomic || inconsistentEmbeddingModel))
+      || mixtureInternalAndExternal
+      || (allExternal && newDatasets.length > 1)
+    )
       setRerankModelOpen(true)
   }, [inputs, setInputs, payload.retrieval_mode])
 

web/app/components/workflow/nodes/knowledge-retrieval/utils.ts

@@ -21,6 +21,9 @@ export const getSelectedDatasetsMode = (datasets: DataSet[]) => {
   let allHighQualityFullTextSearch = true
   let allEconomic = true
   let mixtureHighQualityAndEconomic = true
+  let allExternal = true
+  let allInternal = true
+  let mixtureInternalAndExternal = true
   let inconsistentEmbeddingModel = false
   if (!datasets.length) {
     allHighQuality = false
@@ -29,6 +32,9 @@ export const getSelectedDatasetsMode = (datasets: DataSet[]) => {
     allEconomic = false
     mixtureHighQualityAndEconomic = false
     inconsistentEmbeddingModel = false
+    allExternal = false
+    allInternal = false
+    mixtureInternalAndExternal = false
   }
   datasets.forEach((dataset) => {
     if (dataset.indexing_technique === 'economy') {
@@ -45,8 +51,21 @@ export const getSelectedDatasetsMode = (datasets: DataSet[]) => {
       if (dataset.retrieval_model_dict.search_method !== RETRIEVE_METHOD.fullText)
         allHighQualityFullTextSearch = false
     }
+    if (dataset.provider !== 'external') {
+      allExternal = false
+    }
+    else {
+      allInternal = false
+      allHighQuality = false
+      allHighQualityVectorSearch = false
+      allHighQualityFullTextSearch = false
+      mixtureHighQualityAndEconomic = false
+    }
   })
 
+  if (allExternal || allInternal)
+    mixtureInternalAndExternal = false
+
   if (allHighQuality || allEconomic)
     mixtureHighQualityAndEconomic = false
 
@@ -59,6 +78,9 @@ export const getSelectedDatasetsMode = (datasets: DataSet[]) => {
     allHighQualityFullTextSearch,
     allEconomic,
     mixtureHighQualityAndEconomic,
+    allInternal,
+    allExternal,
+    mixtureInternalAndExternal,
     inconsistentEmbeddingModel,
   } as SelectedDatasetsMode
 }
@@ -70,6 +92,9 @@ export const getMultipleRetrievalConfig = (multipleRetrievalConfig: MultipleRetr
     allHighQualityFullTextSearch,
     allEconomic,
     mixtureHighQualityAndEconomic,
+    allInternal,
+    allExternal,
+    mixtureInternalAndExternal,
     inconsistentEmbeddingModel,
   } = getSelectedDatasetsMode(selectedDatasets)
 
@@ -91,13 +116,13 @@ export const getMultipleRetrievalConfig = (multipleRetrievalConfig: MultipleRetr
     reranking_enable: allEconomic ? reranking_enable : true,
   }
 
-  if (allEconomic || mixtureHighQualityAndEconomic || inconsistentEmbeddingModel)
+  if (allEconomic || mixtureHighQualityAndEconomic || inconsistentEmbeddingModel || allExternal || mixtureInternalAndExternal)
     result.reranking_mode = RerankingModeEnum.RerankingModel
 
-  if (allHighQuality && !inconsistentEmbeddingModel && reranking_mode === undefined)
+  if (allHighQuality && !inconsistentEmbeddingModel && reranking_mode === undefined && allInternal)
     result.reranking_mode = RerankingModeEnum.WeightedScore
 
-  if (allHighQuality && !inconsistentEmbeddingModel && (reranking_mode === RerankingModeEnum.WeightedScore || reranking_mode === undefined) && !weights) {
+  if (allHighQuality && !inconsistentEmbeddingModel && (reranking_mode === RerankingModeEnum.WeightedScore || reranking_mode === undefined) && allInternal && !weights) {
     result.weights = {
       vector_setting: {
         vector_weight: allHighQualityVectorSearch

web/app/components/workflow/nodes/start/panel.tsx

@@ -121,6 +121,39 @@ const Panel: FC<NodePanelProps<StartNodeType>> = ({
                   </div>
                 }
               />
+              <VarItem
+                readonly
+                payload={{
+                  variable: 'sys.app_id',
+                } as any}
+                rightContent={
+                  <div className='text-xs font-normal text-gray-500'>
+                    String
+                  </div>
+                }
+              />
+              <VarItem
+                readonly
+                payload={{
+                  variable: 'sys.workflow_id',
+                } as any}
+                rightContent={
+                  <div className='text-xs font-normal text-gray-500'>
+                    String
+                  </div>
+                }
+              />
+              <VarItem
+                readonly
+                payload={{
+                  variable: 'sys.workflow_run_id',
+                } as any}
+                rightContent={
+                  <div className='text-xs font-normal text-gray-500'>
+                    String
+                  </div>
+                }
+              />
             </div>
 
           </>