youngkingdom/dify
1
0
Fork 0
mirror of https://github.com/langgenius/dify.git synced 2026-05-05 18:08:07 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: chatflow support multimodal (#31293)

Browse Source 
Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>
This commit is contained in:
wangxiaolei
2026-01-27 00:24:48 +08:00
committed by GitHub
parent 5eaf0c733a
commit e48419937b
14 changed files with 1051 additions and 133 deletions
Download Patch File Download Diff File Expand all files Collapse all files

178
web/app/components/base/chat/chat/hooks.multimodal.spec.ts Normal file
View File

@ -0,0 +1,178 @@
/**
* Tests for multimodal image file handling in chat hooks.
* Tests the file object conversion logic without full hook integration.
*/
describe('Multimodal File Handling', () => {
describe('File type to MIME type mapping', () => {
it('should map image to image/png', () => {
const fileType: string = 'image'
const expectedMime = 'image/png'
const mimeType = fileType === 'image' ? 'image/png' : 'application/octet-stream'
expect(mimeType).toBe(expectedMime)
})
it('should map video to video/mp4', () => {
const fileType: string = 'video'
const expectedMime = 'video/mp4'
const mimeType = fileType === 'video' ? 'video/mp4' : 'application/octet-stream'
expect(mimeType).toBe(expectedMime)
})
it('should map audio to audio/mpeg', () => {
const fileType: string = 'audio'
const expectedMime = 'audio/mpeg'
const mimeType = fileType === 'audio' ? 'audio/mpeg' : 'application/octet-stream'
expect(mimeType).toBe(expectedMime)
})
it('should map unknown to application/octet-stream', () => {
const fileType: string = 'unknown'
const expectedMime = 'application/octet-stream'
const mimeType = ['image', 'video', 'audio'].includes(fileType) ? 'image/png' : 'application/octet-stream'
expect(mimeType).toBe(expectedMime)
})
})
describe('TransferMethod selection', () => {
it('should select remote_url for images', () => {
const fileType: string = 'image'
const transferMethod = fileType === 'image' ? 'remote_url' : 'local_file'
expect(transferMethod).toBe('remote_url')
})
it('should select local_file for non-images', () => {
const fileType: string = 'video'
const transferMethod = fileType === 'image' ? 'remote_url' : 'local_file'
expect(transferMethod).toBe('local_file')
})
})
describe('File extension mapping', () => {
it('should use .png extension for images', () => {
const fileType: string = 'image'
const expectedExtension = '.png'
const extension = fileType === 'image' ? 'png' : 'bin'
expect(extension).toBe(expectedExtension.replace('.', ''))
})
it('should use .mp4 extension for videos', () => {
const fileType: string = 'video'
const expectedExtension = '.mp4'
const extension = fileType === 'video' ? 'mp4' : 'bin'
expect(extension).toBe(expectedExtension.replace('.', ''))
})
it('should use .mp3 extension for audio', () => {
const fileType: string = 'audio'
const expectedExtension = '.mp3'
const extension = fileType === 'audio' ? 'mp3' : 'bin'
expect(extension).toBe(expectedExtension.replace('.', ''))
})
})
describe('File name generation', () => {
it('should generate correct file name for images', () => {
const fileType: string = 'image'
const expectedName = 'generated_image.png'
const fileName = `generated_${fileType}.${fileType === 'image' ? 'png' : 'bin'}`
expect(fileName).toBe(expectedName)
})
it('should generate correct file name for videos', () => {
const fileType: string = 'video'
const expectedName = 'generated_video.mp4'
const fileName = `generated_${fileType}.${fileType === 'video' ? 'mp4' : 'bin'}`
expect(fileName).toBe(expectedName)
})
it('should generate correct file name for audio', () => {
const fileType: string = 'audio'
const expectedName = 'generated_audio.mp3'
const fileName = `generated_${fileType}.${fileType === 'audio' ? 'mp3' : 'bin'}`
expect(fileName).toBe(expectedName)
})
})
describe('SupportFileType mapping', () => {
it('should map image type to image supportFileType', () => {
const fileType: string = 'image'
const supportFileType = fileType === 'image' ? 'image' : fileType === 'video' ? 'video' : fileType === 'audio' ? 'audio' : 'document'
expect(supportFileType).toBe('image')
})
it('should map video type to video supportFileType', () => {
const fileType: string = 'video'
const supportFileType = fileType === 'image' ? 'image' : fileType === 'video' ? 'video' : fileType === 'audio' ? 'audio' : 'document'
expect(supportFileType).toBe('video')
})
it('should map audio type to audio supportFileType', () => {
const fileType: string = 'audio'
const supportFileType = fileType === 'image' ? 'image' : fileType === 'video' ? 'video' : fileType === 'audio' ? 'audio' : 'document'
expect(supportFileType).toBe('audio')
})
it('should map unknown type to document supportFileType', () => {
const fileType: string = 'unknown'
const supportFileType = fileType === 'image' ? 'image' : fileType === 'video' ? 'video' : fileType === 'audio' ? 'audio' : 'document'
expect(supportFileType).toBe('document')
})
})
describe('File conversion logic', () => {
it('should detect existing transferMethod', () => {
const fileWithTransferMethod = {
id: 'file-123',
transferMethod: 'remote_url' as const,
type: 'image/png',
name: 'test.png',
size: 1024,
supportFileType: 'image',
progress: 100,
}
const hasTransferMethod = 'transferMethod' in fileWithTransferMethod
expect(hasTransferMethod).toBe(true)
})
it('should detect missing transferMethod', () => {
const fileWithoutTransferMethod = {
id: 'file-456',
type: 'image',
url: 'http://example.com/image.png',
belongs_to: 'assistant',
}
const hasTransferMethod = 'transferMethod' in fileWithoutTransferMethod
expect(hasTransferMethod).toBe(false)
})
it('should create file with size 0 for generated files', () => {
const expectedSize = 0
expect(expectedSize).toBe(0)
})
})
describe('Agent vs Non-Agent mode logic', () => {
it('should check for agent_thoughts to determine mode', () => {
const agentResponse: { agent_thoughts?: Array<Record<string, unknown>> } = {
agent_thoughts: [{}],
}
const isAgentMode = agentResponse.agent_thoughts && agentResponse.agent_thoughts.length > 0
expect(isAgentMode).toBe(true)
})
it('should detect non-agent mode when agent_thoughts is empty', () => {
const nonAgentResponse: { agent_thoughts?: Array<Record<string, unknown>> } = {
agent_thoughts: [],
}
const isAgentMode = nonAgentResponse.agent_thoughts && nonAgentResponse.agent_thoughts.length > 0
expect(isAgentMode).toBe(false)
})
it('should detect non-agent mode when agent_thoughts is undefined', () => {
const nonAgentResponse: { agent_thoughts?: Array<Record<string, unknown>> } = {}
const isAgentMode = nonAgentResponse.agent_thoughts && nonAgentResponse.agent_thoughts.length > 0
expect(isAgentMode).toBeFalsy()
})
})
})

35
web/app/components/base/chat/chat/hooks.ts
View File

@ -419,9 +419,40 @@ export const useChat = (
}
},
onFile(file) {
// Convert simple file type to MIME type for non-agent mode
// Backend sends: { id, type: "image", belongs_to, url }
// Frontend expects: { id, type: "image/png", transferMethod, url, uploadedId, supportFileType, name, size }
// Determine file type for MIME conversion
const fileType = (file as { type?: string }).type || 'image'
// If file already has transferMethod, use it as base and ensure all required fields exist
// Otherwise, create a new complete file object
const baseFile = ('transferMethod' in file) ? (file as Partial<FileEntity>) : null
const convertedFile: FileEntity = {
id: baseFile?.id || (file as { id: string }).id,
type: baseFile?.type || (fileType === 'image' ? 'image/png' : fileType === 'video' ? 'video/mp4' : fileType === 'audio' ? 'audio/mpeg' : 'application/octet-stream'),
transferMethod: (baseFile?.transferMethod as FileEntity['transferMethod']) || (fileType === 'image' ? 'remote_url' : 'local_file'),
uploadedId: baseFile?.uploadedId || (file as { id: string }).id,
supportFileType: baseFile?.supportFileType || (fileType === 'image' ? 'image' : fileType === 'video' ? 'video' : fileType === 'audio' ? 'audio' : 'document'),
progress: baseFile?.progress ?? 100,
name: baseFile?.name || `generated_${fileType}.${fileType === 'image' ? 'png' : fileType === 'video' ? 'mp4' : fileType === 'audio' ? 'mp3' : 'bin'}`,
url: baseFile?.url || (file as { url?: string }).url,
size: baseFile?.size ?? 0, // Generated files don't have a known size
}
// For agent mode, add files to the last thought
const lastThought = responseItem.agent_thoughts?.[responseItem.agent_thoughts?.length - 1]
if (lastThought)
responseItem.agent_thoughts![responseItem.agent_thoughts!.length - 1].message_files = [...(lastThought as any).message_files, file]
if (lastThought) {
const thought = lastThought as { message_files?: FileEntity[] }
responseItem.agent_thoughts![responseItem.agent_thoughts!.length - 1].message_files = [...(thought.message_files ?? []), convertedFile]
}
// For non-agent mode, add files directly to responseItem.message_files
else {
const currentFiles = (responseItem.message_files as FileEntity[] | undefined) ?? []
responseItem.message_files = [...currentFiles, convertedFile]
}
updateCurrentQAOnTree({
placeholderQuestionId,

93
web/app/components/datasets/hit-testing/index.spec.tsx
View File

@ -2039,8 +2039,13 @@ describe('Integration: Hit Testing Flow', () => {
renderWithProviders(<HitTestingPage datasetId="dataset-1" />)
// Wait for textbox with timeout for CI
const textarea = await waitFor(
() => screen.getByRole('textbox'),
{ timeout: 3000 },
)
// Type query
const textarea = screen.getByRole('textbox')
fireEvent.change(textarea, { target: { value: 'Test query' } })
// Find submit button by class
@ -2054,8 +2059,13 @@ describe('Integration: Hit Testing Flow', () => {
const { container } = renderWithProviders(<HitTestingPage datasetId="dataset-1" />)
// Wait for textbox with timeout for CI
const textarea = await waitFor(
() => screen.getByRole('textbox'),
{ timeout: 3000 },
)
// Type query
const textarea = screen.getByRole('textbox')
fireEvent.change(textarea, { target: { value: 'Test query' } })
// Component should still be functional - check for the main container
@ -2089,10 +2099,15 @@ describe('Integration: Hit Testing Flow', () => {
isLoading: false,
} as unknown as ReturnType<typeof useDatasetTestingRecords>)
const { container } = renderWithProviders(<HitTestingPage datasetId="dataset-1" />)
const { container: _container } = renderWithProviders(<HitTestingPage datasetId="dataset-1" />)
// Wait for textbox to be rendered with timeout for CI environment
const textarea = await waitFor(
() => screen.getByRole('textbox'),
{ timeout: 3000 },
)
// Type query
const textarea = screen.getByRole('textbox')
fireEvent.change(textarea, { target: { value: 'Test query' } })
// Submit
@ -2101,8 +2116,13 @@ describe('Integration: Hit Testing Flow', () => {
if (submitButton)
fireEvent.click(submitButton)
// Verify the component is still rendered after submission
expect(container.firstChild).toBeInTheDocument()
// Wait for the mutation to complete
await waitFor(
() => {
expect(mockHitTestingMutateAsync).toHaveBeenCalled()
},
{ timeout: 3000 },
)
})
it('should render ResultItem components for non-external results', async () => {
@ -2127,10 +2147,15 @@ describe('Integration: Hit Testing Flow', () => {
isLoading: false,
} as unknown as ReturnType<typeof useDatasetTestingRecords>)
const { container } = renderWithProviders(<HitTestingPage datasetId="dataset-1" />)
const { container: _container } = renderWithProviders(<HitTestingPage datasetId="dataset-1" />)
// Wait for component to be fully rendered with longer timeout
const textarea = await waitFor(
() => screen.getByRole('textbox'),
{ timeout: 3000 },
)
// Submit a query
const textarea = screen.getByRole('textbox')
fireEvent.change(textarea, { target: { value: 'Test query' } })
const buttons = screen.getAllByRole('button')
@ -2138,8 +2163,13 @@ describe('Integration: Hit Testing Flow', () => {
if (submitButton)
fireEvent.click(submitButton)
// Verify component is rendered after submission
expect(container.firstChild).toBeInTheDocument()
// Wait for mutation to complete with longer timeout
await waitFor(
() => {
expect(mockHitTestingMutateAsync).toHaveBeenCalled()
},
{ timeout: 3000 },
)
})
it('should render external results when dataset is external', async () => {
@ -2165,8 +2195,14 @@ describe('Integration: Hit Testing Flow', () => {
// Component should render
expect(container.firstChild).toBeInTheDocument()
// Wait for textbox with timeout for CI
const textarea = await waitFor(
() => screen.getByRole('textbox'),
{ timeout: 3000 },
)
// Type in textarea to verify component is functional
const textarea = screen.getByRole('textbox')
fireEvent.change(textarea, { target: { value: 'Test query' } })
const buttons = screen.getAllByRole('button')
@ -2174,9 +2210,13 @@ describe('Integration: Hit Testing Flow', () => {
if (submitButton)
fireEvent.click(submitButton)
await waitFor(() => {
expect(screen.getByRole('textbox')).toBeInTheDocument()
})
// Verify component is still functional after submission
await waitFor(
() => {
expect(screen.getByRole('textbox')).toBeInTheDocument()
},
{ timeout: 3000 },
)
})
})
@ -2260,8 +2300,13 @@ describe('renderHitResults Coverage', () => {
const { container } = renderWithProviders(<HitTestingPage datasetId="dataset-1" />)
// Wait for textbox with timeout for CI
const textarea = await waitFor(
() => screen.getByRole('textbox'),
{ timeout: 3000 },
)
// Enter query
const textarea = screen.getByRole('textbox')
fireEvent.change(textarea, { target: { value: 'test query' } })
// Submit
@ -2386,8 +2431,13 @@ describe('HitTestingPage Internal Functions Coverage', () => {
const { container } = renderWithProviders(<HitTestingPage datasetId="dataset-1" />)
// Wait for textbox with timeout for CI
const textarea = await waitFor(
() => screen.getByRole('textbox'),
{ timeout: 3000 },
)
// Enter query and submit
const textarea = screen.getByRole('textbox')
fireEvent.change(textarea, { target: { value: 'test query' } })
const buttons = screen.getAllByRole('button')
@ -2400,7 +2450,7 @@ describe('HitTestingPage Internal Functions Coverage', () => {
// Wait for state updates
await waitFor(() => {
expect(container.firstChild).toBeInTheDocument()
}, { timeout: 2000 })
}, { timeout: 3000 })
// Verify mutation was called
expect(mockHitTestingMutateAsync).toHaveBeenCalled()
@ -2445,8 +2495,13 @@ describe('HitTestingPage Internal Functions Coverage', () => {
const { container } = renderWithProviders(<HitTestingPage datasetId="dataset-1" />)
// Wait for textbox with timeout for CI
const textarea = await waitFor(
() => screen.getByRole('textbox'),
{ timeout: 3000 },
)
// Submit a query
const textarea = screen.getByRole('textbox')
fireEvent.change(textarea, { target: { value: 'test' } })
const buttons = screen.getAllByRole('button')
@ -2458,7 +2513,7 @@ describe('HitTestingPage Internal Functions Coverage', () => {
// Verify the component renders
await waitFor(() => {
expect(container.firstChild).toBeInTheDocument()
})
}, { timeout: 3000 })
})
})

136
web/app/components/plugins/marketplace/index.spec.tsx
View File

@ -162,6 +162,44 @@ vi.mock('@/utils/var', () => ({
getMarketplaceUrl: (path: string, _params?: Record<string, string | undefined>) => `https://marketplace.dify.ai${path}`,
}))
// Mock marketplace client used by marketplace utils
vi.mock('@/service/client', () => ({
marketplaceClient: {
collections: vi.fn(async (_args?: unknown, _opts?: { signal?: AbortSignal }) => ({
data: {
collections: [
{
name: 'collection-1',
label: { 'en-US': 'Collection 1' },
description: { 'en-US': 'Desc' },
rule: '',
created_at: '2024-01-01',
updated_at: '2024-01-01',
searchable: true,
search_params: { query: '', sort_by: 'install_count', sort_order: 'DESC' },
},
],
},
})),
collectionPlugins: vi.fn(async (_args?: unknown, _opts?: { signal?: AbortSignal }) => ({
data: {
plugins: [
{ type: 'plugin', org: 'test', name: 'plugin1', tags: [] },
],
},
})),
// Some utils paths may call searchAdvanced; provide a minimal stub
searchAdvanced: vi.fn(async (_args?: unknown, _opts?: { signal?: AbortSignal }) => ({
data: {
plugins: [
{ type: 'plugin', org: 'test', name: 'plugin1', tags: [] },
],
total: 1,
},
})),
},
}))
// Mock context/query-client
vi.mock('@/context/query-client', () => ({
TanstackQueryInitializer: ({ children }: { children: React.ReactNode }) => <div data-testid="query-initializer">{children}</div>,
@ -1474,7 +1512,24 @@ describe('flatMap Coverage', () => {
// ================================
// Async Utils Tests
// ================================
// Narrow mock surface and avoid any in tests
// Types are local to this spec to keep scope minimal
type FnMock = ReturnType<typeof vi.fn>
type MarketplaceClientMock = {
collectionPlugins: FnMock
collections: FnMock
}
describe('Async Utils', () => {
let marketplaceClientMock: MarketplaceClientMock
beforeAll(async () => {
const mod = await import('@/service/client')
marketplaceClientMock = mod.marketplaceClient as unknown as MarketplaceClientMock
})
beforeEach(() => {
vi.clearAllMocks()
})
@ -1490,12 +1545,10 @@ describe('Async Utils', () => {
{ type: 'plugin', org: 'test', name: 'plugin2' },
]
globalThis.fetch = vi.fn().mockResolvedValue(
new Response(JSON.stringify({ data: { plugins: mockPlugins } }), {
status: 200,
headers: { 'Content-Type': 'application/json' },
}),
)
// Adjusted to our mocked marketplaceClient instead of fetch
marketplaceClientMock.collectionPlugins.mockResolvedValueOnce({
data: { plugins: mockPlugins },
})
const { getMarketplacePluginsByCollectionId } = await import('./utils')
const result = await getMarketplacePluginsByCollectionId('test-collection', {
@ -1504,12 +1557,13 @@ describe('Async Utils', () => {
type: 'plugin',
})
expect(globalThis.fetch).toHaveBeenCalled()
expect(marketplaceClientMock.collectionPlugins).toHaveBeenCalled()
expect(result).toHaveLength(2)
})
it('should handle fetch error and return empty array', async () => {
globalThis.fetch = vi.fn().mockRejectedValue(new Error('Network error'))
// Simulate error from client
marketplaceClientMock.collectionPlugins.mockRejectedValueOnce(new Error('Network error'))
const { getMarketplacePluginsByCollectionId } = await import('./utils')
const result = await getMarketplacePluginsByCollectionId('test-collection')
@ -1519,25 +1573,18 @@ describe('Async Utils', () => {
it('should pass abort signal when provided', async () => {
const mockPlugins = [{ type: 'plugins', org: 'test', name: 'plugin1' }]
globalThis.fetch = vi.fn().mockResolvedValue(
new Response(JSON.stringify({ data: { plugins: mockPlugins } }), {
status: 200,
headers: { 'Content-Type': 'application/json' },
}),
)
// Our client mock receives the signal as second arg
marketplaceClientMock.collectionPlugins.mockResolvedValueOnce({
data: { plugins: mockPlugins },
})
const controller = new AbortController()
const { getMarketplacePluginsByCollectionId } = await import('./utils')
await getMarketplacePluginsByCollectionId('test-collection', {}, { signal: controller.signal })
// oRPC uses Request objects, so check that fetch was called with a Request containing the right URL
expect(globalThis.fetch).toHaveBeenCalledWith(
expect.any(Request),
expect.any(Object),
)
const call = vi.mocked(globalThis.fetch).mock.calls[0]
const request = call[0] as Request
expect(request.url).toContain('test-collection')
expect(marketplaceClientMock.collectionPlugins).toHaveBeenCalled()
const call = marketplaceClientMock.collectionPlugins.mock.calls[0]
expect(call[1]).toMatchObject({ signal: controller.signal })
})
})
@ -1548,23 +1595,17 @@ describe('Async Utils', () => {
]
const mockPlugins = [{ type: 'plugins', org: 'test', name: 'plugin1' }]
let callCount = 0
globalThis.fetch = vi.fn().mockImplementation(() => {
callCount++
if (callCount === 1) {
return Promise.resolve(
new Response(JSON.stringify({ data: { collections: mockCollections } }), {
status: 200,
headers: { 'Content-Type': 'application/json' },
}),
)
// Simulate two-step client calls: collections then collectionPlugins
let stage = 0
marketplaceClientMock.collections.mockImplementationOnce(async () => {
stage = 1
return { data: { collections: mockCollections } }
})
marketplaceClientMock.collectionPlugins.mockImplementation(async () => {
if (stage === 1) {
return { data: { plugins: mockPlugins } }
}
return Promise.resolve(
new Response(JSON.stringify({ data: { plugins: mockPlugins } }), {
status: 200,
headers: { 'Content-Type': 'application/json' },
}),
)
return { data: { plugins: [] } }
})
const { getMarketplaceCollectionsAndPlugins } = await import('./utils')
@ -1578,7 +1619,8 @@ describe('Async Utils', () => {
})
it('should handle fetch error and return empty data', async () => {
globalThis.fetch = vi.fn().mockRejectedValue(new Error('Network error'))
// Simulate client error
marketplaceClientMock.collections.mockRejectedValueOnce(new Error('Network error'))
const { getMarketplaceCollectionsAndPlugins } = await import('./utils')
const result = await getMarketplaceCollectionsAndPlugins()
@ -1588,24 +1630,16 @@ describe('Async Utils', () => {
})
it('should append condition and type to URL when provided', async () => {
globalThis.fetch = vi.fn().mockResolvedValue(
new Response(JSON.stringify({ data: { collections: [] } }), {
status: 200,
headers: { 'Content-Type': 'application/json' },
}),
)
// Assert that the client was called with query containing condition/type
const { getMarketplaceCollectionsAndPlugins } = await import('./utils')
await getMarketplaceCollectionsAndPlugins({
condition: 'category=tool',
type: 'bundle',
})
// oRPC uses Request objects, so check that fetch was called with a Request containing the right URL
expect(globalThis.fetch).toHaveBeenCalled()
const call = vi.mocked(globalThis.fetch).mock.calls[0]
const request = call[0] as Request
expect(request.url).toContain('condition=category%3Dtool')
expect(marketplaceClientMock.collections).toHaveBeenCalled()
const call = marketplaceClientMock.collections.mock.calls[0]
expect(call[0]).toMatchObject({ query: expect.objectContaining({ condition: 'category=tool', type: 'bundle' }) })
})
})
})