Merge remote-tracking branch 'origin/main' into feat/support-agent-sandbox

This commit is contained in:
yyh
2026-02-05 15:33:01 +08:00
10 changed files with 323 additions and 80 deletions

View File

@ -39,7 +39,7 @@ jobs:
run: pnpm install --frozen-lockfile
- name: Run tests
run: pnpm test:coverage
run: pnpm test:ci
- name: Coverage Summary
if: always()

View File

@ -8,7 +8,6 @@ from sqlalchemy import delete, select
from core.db.session_factory import session_factory
from core.indexing_runner import DocumentIsPausedError, IndexingRunner
from core.rag.index_processor.index_processor_factory import IndexProcessorFactory
from extensions.ext_database import db
from libs.datetime_utils import naive_utc_now
from models.dataset import Dataset, Document, DocumentSegment
@ -27,7 +26,7 @@ def document_indexing_update_task(dataset_id: str, document_id: str):
logger.info(click.style(f"Start update document: {document_id}", fg="green"))
start_at = time.perf_counter()
with session_factory.create_session() as session:
with session_factory.create_session() as session, session.begin():
document = session.query(Document).where(Document.id == document_id, Document.dataset_id == dataset_id).first()
if not document:
@ -36,7 +35,6 @@ def document_indexing_update_task(dataset_id: str, document_id: str):
document.indexing_status = "parsing"
document.processing_started_at = naive_utc_now()
session.commit()
# delete all document segment and index
try:
@ -56,7 +54,7 @@ def document_indexing_update_task(dataset_id: str, document_id: str):
segment_ids = [segment.id for segment in segments]
segment_delete_stmt = delete(DocumentSegment).where(DocumentSegment.id.in_(segment_ids))
session.execute(segment_delete_stmt)
db.session.commit()
end_at = time.perf_counter()
logger.info(
click.style(

View File

@ -0,0 +1,182 @@
from unittest.mock import MagicMock, patch
import pytest
from faker import Faker
from models import Account, Tenant, TenantAccountJoin, TenantAccountRole
from models.dataset import Dataset, Document, DocumentSegment
from tasks.document_indexing_update_task import document_indexing_update_task
class TestDocumentIndexingUpdateTask:
@pytest.fixture
def mock_external_dependencies(self):
"""Patch external collaborators used by the update task.
- IndexProcessorFactory.init_index_processor().clean(...)
- IndexingRunner.run([...])
"""
with (
patch("tasks.document_indexing_update_task.IndexProcessorFactory") as mock_factory,
patch("tasks.document_indexing_update_task.IndexingRunner") as mock_runner,
):
processor_instance = MagicMock()
mock_factory.return_value.init_index_processor.return_value = processor_instance
runner_instance = MagicMock()
mock_runner.return_value = runner_instance
yield {
"factory": mock_factory,
"processor": processor_instance,
"runner": mock_runner,
"runner_instance": runner_instance,
}
def _create_dataset_document_with_segments(self, db_session_with_containers, *, segment_count: int = 2):
fake = Faker()
# Account and tenant
account = Account(
email=fake.email(),
name=fake.name(),
interface_language="en-US",
status="active",
)
db_session_with_containers.add(account)
db_session_with_containers.commit()
tenant = Tenant(name=fake.company(), status="normal")
db_session_with_containers.add(tenant)
db_session_with_containers.commit()
join = TenantAccountJoin(
tenant_id=tenant.id,
account_id=account.id,
role=TenantAccountRole.OWNER,
current=True,
)
db_session_with_containers.add(join)
db_session_with_containers.commit()
# Dataset and document
dataset = Dataset(
tenant_id=tenant.id,
name=fake.company(),
description=fake.text(max_nb_chars=64),
data_source_type="upload_file",
indexing_technique="high_quality",
created_by=account.id,
)
db_session_with_containers.add(dataset)
db_session_with_containers.commit()
document = Document(
tenant_id=tenant.id,
dataset_id=dataset.id,
position=0,
data_source_type="upload_file",
batch="test_batch",
name=fake.file_name(),
created_from="upload_file",
created_by=account.id,
indexing_status="waiting",
enabled=True,
doc_form="text_model",
)
db_session_with_containers.add(document)
db_session_with_containers.commit()
# Segments
node_ids = []
for i in range(segment_count):
node_id = f"node-{i + 1}"
seg = DocumentSegment(
tenant_id=tenant.id,
dataset_id=dataset.id,
document_id=document.id,
position=i,
content=fake.text(max_nb_chars=32),
answer=None,
word_count=10,
tokens=5,
index_node_id=node_id,
status="completed",
created_by=account.id,
)
db_session_with_containers.add(seg)
node_ids.append(node_id)
db_session_with_containers.commit()
# Refresh to ensure ORM state
db_session_with_containers.refresh(dataset)
db_session_with_containers.refresh(document)
return dataset, document, node_ids
def test_cleans_segments_and_reindexes(self, db_session_with_containers, mock_external_dependencies):
dataset, document, node_ids = self._create_dataset_document_with_segments(db_session_with_containers)
# Act
document_indexing_update_task(dataset.id, document.id)
# Ensure we see committed changes from another session
db_session_with_containers.expire_all()
# Assert document status updated before reindex
updated = db_session_with_containers.query(Document).where(Document.id == document.id).first()
assert updated.indexing_status == "parsing"
assert updated.processing_started_at is not None
# Segments should be deleted
remaining = (
db_session_with_containers.query(DocumentSegment).where(DocumentSegment.document_id == document.id).count()
)
assert remaining == 0
# Assert index processor clean was called with expected args
clean_call = mock_external_dependencies["processor"].clean.call_args
assert clean_call is not None
args, kwargs = clean_call
# args[0] is a Dataset instance (from another session) — validate by id
assert getattr(args[0], "id", None) == dataset.id
# args[1] should contain our node_ids
assert set(args[1]) == set(node_ids)
assert kwargs.get("with_keywords") is True
assert kwargs.get("delete_child_chunks") is True
# Assert indexing runner invoked with the updated document
run_call = mock_external_dependencies["runner_instance"].run.call_args
assert run_call is not None
run_docs = run_call[0][0]
assert len(run_docs) == 1
first = run_docs[0]
assert getattr(first, "id", None) == document.id
def test_clean_error_is_logged_and_indexing_continues(self, db_session_with_containers, mock_external_dependencies):
dataset, document, node_ids = self._create_dataset_document_with_segments(db_session_with_containers)
# Force clean to raise; task should continue to indexing
mock_external_dependencies["processor"].clean.side_effect = Exception("boom")
document_indexing_update_task(dataset.id, document.id)
# Ensure we see committed changes from another session
db_session_with_containers.expire_all()
# Indexing should still be triggered
mock_external_dependencies["runner_instance"].run.assert_called_once()
# Segments should remain (since clean failed before DB delete)
remaining = (
db_session_with_containers.query(DocumentSegment).where(DocumentSegment.document_id == document.id).count()
)
assert remaining > 0
def test_document_not_found_noop(self, db_session_with_containers, mock_external_dependencies):
fake = Faker()
# Act with non-existent document id
document_indexing_update_task(dataset_id=fake.uuid4(), document_id=fake.uuid4())
# Neither processor nor runner should be called
mock_external_dependencies["processor"].clean.assert_not_called()
mock_external_dependencies["runner_instance"].run.assert_not_called()

View File

@ -4,7 +4,7 @@ import type { FC } from 'react'
import { RiQuestionLine } from '@remixicon/react'
import { useBoolean } from 'ahooks'
import * as React from 'react'
import { useEffect, useRef, useState } from 'react'
import { useCallback, useEffect, useRef, useState } from 'react'
import { PortalToFollowElem, PortalToFollowElemContent, PortalToFollowElemTrigger } from '@/app/components/base/portal-to-follow-elem'
import { cn } from '@/utils/classnames'
import { tooltipManager } from './TooltipManager'
@ -61,6 +61,20 @@ const Tooltip: FC<TooltipProps> = ({
isHoverTriggerRef.current = isHoverTrigger
}, [isHoverTrigger])
const closeTimeoutRef = useRef<ReturnType<typeof setTimeout> | null>(null)
const clearCloseTimeout = useCallback(() => {
if (closeTimeoutRef.current) {
clearTimeout(closeTimeoutRef.current)
closeTimeoutRef.current = null
}
}, [])
useEffect(() => {
return () => {
clearCloseTimeout()
}
}, [clearCloseTimeout])
const close = () => setOpen(false)
const handleLeave = (isTrigger: boolean) => {
@ -71,7 +85,9 @@ const Tooltip: FC<TooltipProps> = ({
// give time to move to the popup
if (needsDelay) {
setTimeout(() => {
clearCloseTimeout()
closeTimeoutRef.current = setTimeout(() => {
closeTimeoutRef.current = null
if (!isHoverPopupRef.current && !isHoverTriggerRef.current) {
setOpen(false)
tooltipManager.clear(close)
@ -79,6 +95,7 @@ const Tooltip: FC<TooltipProps> = ({
}, 300)
}
else {
clearCloseTimeout()
setOpen(false)
tooltipManager.clear(close)
}
@ -95,6 +112,7 @@ const Tooltip: FC<TooltipProps> = ({
onClick={() => triggerMethod === 'click' && setOpen(v => !v)}
onMouseEnter={() => {
if (triggerMethod === 'hover') {
clearCloseTimeout()
setHoverTrigger()
tooltipManager.register(close)
setOpen(true)
@ -115,7 +133,12 @@ const Tooltip: FC<TooltipProps> = ({
!noDecoration && 'system-xs-regular relative max-w-[300px] break-words rounded-md bg-components-panel-bg px-3 py-2 text-left text-text-tertiary shadow-lg',
popupClassName,
)}
onMouseEnter={() => triggerMethod === 'hover' && setHoverPopup()}
onMouseEnter={() => {
if (triggerMethod === 'hover') {
clearCloseTimeout()
setHoverPopup()
}
}}
onMouseLeave={() => triggerMethod === 'hover' && handleLeave(false)}
>
{popupContent}

View File

@ -159,69 +159,74 @@ const Apps = ({
return (
<div className={cn(
'flex h-full flex-col border-l-[0.5px] border-divider-regular',
'flex h-full min-h-0 flex-col overflow-hidden border-l-[0.5px] border-divider-regular',
)}
>
{systemFeatures.enable_explore_banner && (
<div className="mt-4 px-12">
<Banner />
</div>
)}
<div className={cn(
'mt-6 flex items-center justify-between px-12',
)}
>
<div className="flex items-center">
<div className="system-xl-semibold grow truncate text-text-primary">{!hasFilterCondition ? t('apps.title', { ns: 'explore' }) : t('apps.resultNum', { num: searchFilteredList.length, ns: 'explore' })}</div>
{hasFilterCondition && (
<>
<div className="mx-3 h-4 w-px bg-divider-regular"></div>
<Button size="medium" onClick={handleResetFilter}>{t('apps.resetFilter', { ns: 'explore' })}</Button>
</>
)}
</div>
<Input
showLeftIcon
showClearIcon
wrapperClassName="w-[200px] self-start"
value={keywords}
onChange={e => handleKeywordsChange(e.target.value)}
onClear={() => handleKeywordsChange('')}
/>
</div>
<div className="flex flex-1 flex-col overflow-y-auto">
{systemFeatures.enable_explore_banner && (
<div className="mt-4 px-12">
<Banner />
</div>
)}
<div className="mt-2 px-12">
<Category
list={categories}
value={currCategory}
onChange={setCurrCategory}
allCategoriesEn={allCategoriesEn}
/>
</div>
<div className={cn(
'relative mt-4 flex flex-1 shrink-0 grow flex-col overflow-auto pb-6',
)}
>
<nav
className={cn(
s.appList,
'grid shrink-0 content-start gap-4 px-6 sm:px-12',
<div className="sticky top-0 z-10 bg-background-body">
<div className={cn(
'flex items-center justify-between px-12 pt-6',
)}
>
{searchFilteredList.map(app => (
<AppCard
key={app.app_id}
isExplore
app={app}
canCreate={hasEditPermission}
onCreate={() => {
setCurrApp(app)
setIsShowCreateModal(true)
}}
>
<div className="flex items-center">
<div className="system-xl-semibold grow truncate text-text-primary">{!hasFilterCondition ? t('apps.title', { ns: 'explore' }) : t('apps.resultNum', { num: searchFilteredList.length, ns: 'explore' })}</div>
{hasFilterCondition && (
<>
<div className="mx-3 h-4 w-px bg-divider-regular"></div>
<Button size="medium" onClick={handleResetFilter}>{t('apps.resetFilter', { ns: 'explore' })}</Button>
</>
)}
</div>
<Input
showLeftIcon
showClearIcon
wrapperClassName="w-[200px] self-start"
value={keywords}
onChange={e => handleKeywordsChange(e.target.value)}
onClear={() => handleKeywordsChange('')}
/>
))}
</nav>
</div>
<div className="px-12 pb-4 pt-2">
<Category
list={categories}
value={currCategory}
onChange={setCurrCategory}
allCategoriesEn={allCategoriesEn}
/>
</div>
</div>
<div className={cn(
'relative flex flex-1 shrink-0 grow flex-col pb-6',
)}
>
<nav
className={cn(
s.appList,
'grid shrink-0 content-start gap-4 px-6 sm:px-12',
)}
>
{searchFilteredList.map(app => (
<AppCard
key={app.app_id}
isExplore
app={app}
canCreate={hasEditPermission}
onCreate={() => {
setCurrApp(app)
setIsShowCreateModal(true)
}}
/>
))}
</nav>
</div>
</div>
{isShowCreateModal && (
<CreateAppModal

View File

@ -71,7 +71,7 @@ const Explore: FC<IExploreProps> = ({
}
>
<Sidebar controlUpdateInstalledApps={controlUpdateInstalledApps} />
<div className="w-0 grow">
<div className="h-full min-h-0 w-0 grow">
{children}
</div>
</ExploreContext.Provider>

View File

@ -599,20 +599,30 @@ describe('CommonCreateModal', () => {
},
})
mockUsePluginStore.mockReturnValue(detailWithCredentials)
const existingBuilder = createMockSubscriptionBuilder()
mockVerifyCredentials.mockImplementation((params, { onSuccess }) => {
onSuccess()
})
render(<CommonCreateModal {...defaultProps} />)
await waitFor(() => {
expect(mockCreateBuilder).toHaveBeenCalled()
})
render(<CommonCreateModal {...defaultProps} builder={existingBuilder} />)
fireEvent.click(screen.getByTestId('modal-confirm'))
await waitFor(() => {
expect(mockVerifyCredentials).toHaveBeenCalled()
expect(mockVerifyCredentials).toHaveBeenCalledWith(
expect.objectContaining({
provider: 'test-provider',
subscriptionBuilderId: existingBuilder.id,
}),
expect.objectContaining({
onSuccess: expect.any(Function),
onError: expect.any(Function),
}),
)
})
await waitFor(() => {
expect(screen.getByTestId('modal-confirm')).toHaveTextContent('pluginTrigger.modal.common.create')
})
})
@ -629,15 +639,12 @@ describe('CommonCreateModal', () => {
},
})
mockUsePluginStore.mockReturnValue(detailWithCredentials)
const existingBuilder = createMockSubscriptionBuilder()
mockVerifyCredentials.mockImplementation((params, { onError }) => {
onError(new Error('Verification failed'))
})
render(<CommonCreateModal {...defaultProps} />)
await waitFor(() => {
expect(mockCreateBuilder).toHaveBeenCalled()
})
render(<CommonCreateModal {...defaultProps} builder={existingBuilder} />)
fireEvent.click(screen.getByTestId('modal-confirm'))

View File

@ -4,6 +4,17 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import { DSLImportStatus } from '@/models/app'
import UpdateDSLModal from './update-dsl-modal'
class MockFileReader {
onload: ((this: FileReader, event: ProgressEvent<FileReader>) => void) | null = null
readAsText(_file: Blob) {
const event = { target: { result: 'test content' } } as unknown as ProgressEvent<FileReader>
this.onload?.call(this as unknown as FileReader, event)
}
}
vi.stubGlobal('FileReader', MockFileReader as unknown as typeof FileReader)
// Mock react-i18next
vi.mock('react-i18next', () => ({
useTranslation: () => ({

View File

@ -46,7 +46,8 @@
"uglify-embed": "node ./bin/uglify-embed",
"i18n:check": "tsx ./scripts/check-i18n.js",
"test": "vitest run",
"test:coverage": "vitest run --coverage --reporter=dot --silent=passed-only",
"test:coverage": "vitest run --coverage",
"test:ci": "vitest run --coverage --reporter vitest-tiny-reporter --silent=passed-only",
"test:watch": "vitest --watch",
"analyze-component": "node ./scripts/analyze-component.js",
"refactor-component": "node ./scripts/refactor-component.js",
@ -241,7 +242,8 @@
"vite": "7.3.1",
"vite-tsconfig-paths": "6.0.4",
"vitest": "4.0.17",
"vitest-canvas-mock": "1.1.3"
"vitest-canvas-mock": "1.1.3",
"vitest-tiny-reporter": "1.3.1"
},
"pnpm": {
"overrides": {

15
web/pnpm-lock.yaml generated
View File

@ -600,6 +600,9 @@ importers:
vitest-canvas-mock:
specifier: 1.1.3
version: 1.1.3(vitest@4.0.17)
vitest-tiny-reporter:
specifier: 1.3.1
version: 1.3.1(@vitest/runner@4.0.17)(vitest@4.0.17)
packages:
@ -7386,6 +7389,12 @@ packages:
peerDependencies:
vitest: ^3.0.0 || ^4.0.0
vitest-tiny-reporter@1.3.1:
resolution: {integrity: sha512-9WfLruQBbxm4EqMIS0jDZmQjvMgsWgHUso9mHQWgjA6hM3tEVhjdG8wYo7ePFh1XbwEFzEo3XUQqkGoKZ/Td2Q==}
peerDependencies:
'@vitest/runner': ^2.0.0 || ^3.0.2 || ^4.0.0
vitest: ^2.0.0 || ^3.0.2 || ^4.0.0
vitest@4.0.17:
resolution: {integrity: sha512-FQMeF0DJdWY0iOnbv466n/0BudNdKj1l5jYgl5JVTwjSsZSlqyXFt/9+1sEyhR6CLowbZpV7O1sCHrzBhucKKg==}
engines: {node: ^20.0.0 || ^22.0.0 || >=24.0.0}
@ -15493,6 +15502,12 @@ snapshots:
moo-color: 1.0.3
vitest: 4.0.17(@types/node@18.15.0)(@vitest/browser-playwright@4.0.17)(jiti@1.21.7)(jsdom@27.3.0(canvas@3.2.1))(sass@1.93.2)(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.2)
vitest-tiny-reporter@1.3.1(@vitest/runner@4.0.17)(vitest@4.0.17):
dependencies:
'@vitest/runner': 4.0.17
tinyrainbow: 3.0.3
vitest: 4.0.17(@types/node@18.15.0)(@vitest/browser-playwright@4.0.17)(jiti@1.21.7)(jsdom@27.3.0(canvas@3.2.1))(sass@1.93.2)(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.2)
vitest@4.0.17(@types/node@18.15.0)(@vitest/browser-playwright@4.0.17)(jiti@1.21.7)(jsdom@27.3.0(canvas@3.2.1))(sass@1.93.2)(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.2):
dependencies:
'@vitest/expect': 4.0.17