youngkingdom/dify
1
0
Fork 0
mirror of https://github.com/langgenius/dify.git synced 2026-07-14 17:07:03 +08:00
Code Issues Packages Projects Releases Wiki Activity

test(e2e): split agent builder glue

Browse Source
This commit is contained in:
yyh
2026-07-01 20:37:58 +08:00
parent f140a73a22
commit 2b8a80c4e2
17 changed files with 2344 additions and 2213 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
e2e/AGENTS.md
View File

@ -296,7 +296,9 @@ Use `support/naming.ts` for generated test resource names. New app, Agent, datas
Use `fixtures/test-materials/` for checked-in files that scenarios upload, preview, index, or retrieve. Keep these fixtures small and deterministic, and use `support/test-materials.ts` to resolve their absolute paths.
Use `support/preflight.ts` for scenarios that require optional external resources such as a stable model provider, plugin/tool credential, or knowledge base seed. Prefer an explicit `Given` step that returns a skipped result with a clear blocked-precondition reason over hidden setup in hooks.
Use `support/preflight.ts` for scenarios that require optional external resources such as a stable model provider, plugin/tool credential, or knowledge base seed. Prefer an explicit `Given` step that returns a skipped result with a clear blocked-precondition reason over hidden setup in hooks. Keep `support/preflight.ts` as the public barrel and put resource checks in the matching module under `support/preflight/`: `models.ts`, `agents.ts`, `datasets.ts`, `tools.ts`, or `access.ts`.
Keep Agent Builder step definitions grouped by user capability, not by DOM component or Cucumber keyword. `configure.steps.ts` owns common configure navigation, refresh, and draft assertions; `build-draft.steps.ts` owns Build mode checkout, apply, discard, and Build draft isolation; `files.steps.ts` owns Files upload and file-list assertions; `advanced-settings.steps.ts` owns Env and advanced settings behavior; `agent-edit.steps.ts` owns saved Agent detail display assertions; `publish.steps.ts` owns publish state; `access-point.steps.ts` owns Access Point behavior; `preflight.steps.ts` should remain the explicit `Given` entrypoint for preflight resources.
Use `the Agent Builder stable chat model is available` before scenarios that must run an Agent with a real model. `E2E_STABLE_MODEL_PROVIDER`, `E2E_STABLE_MODEL_NAME`, and optional `E2E_STABLE_MODEL_TYPE` define the single usable model for the E2E environment. The step defaults the type to `llm`, verifies the model is present and `active` through `/console/api/workspaces/current/models/model-types/{type}`, then stores it on `DifyWorld.agentBuilder.preflight.stableModel`. Any Agent Builder scenario that needs a usable model should explicitly apply this stored model during its own setup instead of hard-coding provider or model names in feature files or hooks.

383
e2e/features/step-definitions/agent-v2/advanced-settings.steps.ts Normal file
View File

@ -0,0 +1,383 @@
import type { DifyWorld } from '../../support/world'
import { Then, When } from '@cucumber/cucumber'
import { expect } from '@playwright/test'
import { getAgentComposerDraft } from '../../../support/agent'
import { agentBuilderFixedInputs } from '../../../support/agent-builder-resources'
import { skipBlockedPrecondition } from '../../../support/preflight'
import { getAgentBuilderTestMaterialPath } from '../../../support/test-materials'
import {
expectAgentEnvVariableHidden,
expectAgentEnvVariableVisible,
getAgentEnvVariables,
getAgentEnvVariableValue,
getCurrentAgentId,
getEnvVariableKey,
openAgentAdvancedSettings,
} from './configure-helpers'
When(
'I add the plain Agent v2 environment variable from Advanced Settings',
async function (this: DifyWorld) {
const page = this.getPage()
const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' })
await page.getByRole('button', { name: 'Advanced Settings' }).first().click()
await expect(advancedSettings.getByRole('heading', { name: 'Env Editor' })).toBeVisible()
await advancedSettings
.getByRole('textbox', { name: 'Key' })
.fill(agentBuilderFixedInputs.envPlainKey)
await advancedSettings
.getByRole('textbox', { name: 'Value' })
.fill(agentBuilderFixedInputs.envPlainValue)
await expect(advancedSettings.getByText('Plain', { exact: true })).toBeVisible()
},
)
When(
'I import the invalid Agent v2 environment file from Advanced Settings',
async function (this: DifyWorld) {
const page = this.getPage()
const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' })
const fileChooserPromise = page.waitForEvent('filechooser')
await advancedSettings.getByRole('button', { name: 'Import .env' }).click()
await (await fileChooserPromise).setFiles(getAgentBuilderTestMaterialPath('invalidEnv'))
},
)
When(
'I add the secondary plain Agent v2 environment variable from Advanced Settings',
async function (this: DifyWorld) {
const page = this.getPage()
const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' })
await advancedSettings.getByRole('button', { name: 'Add environment variable' }).click()
await advancedSettings
.getByRole('textbox', { name: 'Key' })
.last()
.fill(agentBuilderFixedInputs.envModeKey)
await advancedSettings
.getByRole('textbox', { name: 'Value' })
.last()
.fill(agentBuilderFixedInputs.envModeValue)
await expect(advancedSettings.getByText('Plain', { exact: true })).toHaveCount(2)
},
)
When(
'I delete the plain Agent v2 environment variable from Advanced Settings',
async function (this: DifyWorld) {
const page = this.getPage()
const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' })
await advancedSettings
.getByRole('button', { name: `Delete ${agentBuilderFixedInputs.envPlainKey}` })
.click()
},
)
When(
'I import the valid Agent v2 environment file from Advanced Settings',
async function (this: DifyWorld) {
const page = this.getPage()
const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' })
await page.getByRole('button', { name: 'Advanced Settings' }).first().click()
await expect(advancedSettings.getByRole('heading', { name: 'Env Editor' })).toBeVisible()
const fileChooserPromise = page.waitForEvent('filechooser')
await advancedSettings.getByRole('button', { name: 'Import .env' }).click()
await (await fileChooserPromise).setFiles(getAgentBuilderTestMaterialPath('validEnv'))
},
)
When('I expand Agent v2 Advanced Settings', async function (this: DifyWorld) {
const page = this.getPage()
const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' })
await page.getByRole('button', { name: 'Advanced Settings' }).first().click()
await expect(advancedSettings.getByRole('heading', { name: 'Env Editor' })).toBeVisible()
})
Then(
'Agent v2 Advanced Settings should describe supported entries while collapsed',
async function (this: DifyWorld) {
const page = this.getPage()
const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' })
await expect(advancedSettings).toBeVisible()
await expect(
advancedSettings.getByText('For power users. Env vars, sandbox & memory.'),
).toBeVisible()
await expect(advancedSettings.getByRole('heading', { name: 'Env Editor' }))
.not
.toBeVisible()
},
)
Then(
'the plain Agent v2 environment variable should be saved in the Agent v2 draft',
async function (this: DifyWorld) {
const agentId = getCurrentAgentId(this)
await expect
.poll(async () => {
const env = (await getAgentComposerDraft(agentId)).agent_soul?.env
const variable = env?.variables?.find(item =>
getEnvVariableKey(item) === agentBuilderFixedInputs.envPlainKey,
)
return {
secretCount: env?.secret_refs?.length ?? 0,
value: variable?.value,
}
}, {
timeout: 30_000,
})
.toEqual({
secretCount: 0,
value: agentBuilderFixedInputs.envPlainValue,
})
},
)
Then(
'the Agent v2 environment variables for deletion should be saved in the Agent v2 draft',
async function (this: DifyWorld) {
const agentId = getCurrentAgentId(this)
await expect
.poll(async () => {
const variables = await getAgentEnvVariables(agentId)
return {
modeValue: getAgentEnvVariableValue(variables, agentBuilderFixedInputs.envModeKey),
plainValue: getAgentEnvVariableValue(variables, agentBuilderFixedInputs.envPlainKey),
}
}, {
timeout: 30_000,
})
.toEqual({
modeValue: agentBuilderFixedInputs.envModeValue,
plainValue: agentBuilderFixedInputs.envPlainValue,
})
},
)
Then(
'the plain Agent v2 environment variable should be removed from the Agent v2 draft',
async function (this: DifyWorld) {
const agentId = getCurrentAgentId(this)
await expect
.poll(async () => {
const variables = await getAgentEnvVariables(agentId)
return {
modeValue: getAgentEnvVariableValue(variables, agentBuilderFixedInputs.envModeKey),
plainValue: getAgentEnvVariableValue(variables, agentBuilderFixedInputs.envPlainKey),
}
}, {
timeout: 30_000,
})
.toEqual({
modeValue: agentBuilderFixedInputs.envModeValue,
plainValue: undefined,
})
},
)
Then(
'the valid Agent v2 environment import should be saved in the Agent v2 draft',
async function (this: DifyWorld) {
const agentId = getCurrentAgentId(this)
await expect
.poll(async () => {
const variables = await getAgentEnvVariables(agentId)
return {
modeValue: getAgentEnvVariableValue(variables, agentBuilderFixedInputs.envModeKey),
plainValue: getAgentEnvVariableValue(variables, agentBuilderFixedInputs.envPlainKey),
}
}, {
timeout: 30_000,
})
.toEqual({
modeValue: agentBuilderFixedInputs.envModeValue,
plainValue: agentBuilderFixedInputs.envPlainValue,
})
},
)
Then(
'the invalid Agent v2 environment import should report skipped lines',
async function (this: DifyWorld) {
await expect(this.getPage().getByText('2 invalid .env lines were skipped.')).toBeVisible()
},
)
Then(
'the Agent v2 environment variables from the invalid import should be saved in the Agent v2 draft',
async function (this: DifyWorld) {
const agentId = getCurrentAgentId(this)
await expect
.poll(async () => {
const variables = await getAgentEnvVariables(agentId)
return {
importedValue: getAgentEnvVariableValue(
variables,
agentBuilderFixedInputs.envAfterInvalidImportKey,
),
plainValue: getAgentEnvVariableValue(variables, agentBuilderFixedInputs.envPlainKey),
}
}, {
timeout: 30_000,
})
.toEqual({
importedValue: agentBuilderFixedInputs.envAfterInvalidImportValue,
plainValue: agentBuilderFixedInputs.envPlainValue,
})
},
)
Then(
'I should see the supported Agent v2 Advanced Settings entries',
async function (this: DifyWorld) {
const page = this.getPage()
const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' })
const envEditor = advancedSettings.getByRole('region', { name: 'Env Editor' })
await expect(envEditor).toBeVisible()
await expect(envEditor.getByRole('button', { name: 'Import .env' })).toBeVisible()
await expect(envEditor.getByRole('button', { name: 'Add environment variable' }))
.toBeVisible()
await expect(envEditor.getByText('Key', { exact: true })).toBeVisible()
await expect(envEditor.getByText('Value', { exact: true })).toBeVisible()
await expect(envEditor.getByText('Scope', { exact: true })).toBeVisible()
},
)
Then('Agent v2 Content Moderation Settings should be available', async function (this: DifyWorld) {
const advancedSettings = this.getPage().getByRole('region', { name: 'Advanced Settings' })
const contentModeration = advancedSettings.getByRole('region', { name: 'Content moderation' })
try {
await expect(contentModeration).toBeVisible({ timeout: 3_000 })
}
catch {
return skipBlockedPrecondition(
this,
'Feature not enabled: Agent v2 Content Moderation Settings is not available in this build.',
)
}
})
Then(
'I should see the Agent v2 environment variables from the valid import in Advanced Settings',
async function (this: DifyWorld) {
const page = this.getPage()
const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' })
await page.getByRole('button', { name: 'Advanced Settings' }).first().click()
await expect(advancedSettings.getByRole('heading', { name: 'Env Editor' })).toBeVisible()
await expect.poll(
async () => advancedSettings.getByRole('textbox').evaluateAll(inputs =>
inputs.map(input => (input as HTMLInputElement).value),
),
{ timeout: 30_000 },
).toEqual(expect.arrayContaining([
agentBuilderFixedInputs.envPlainKey,
agentBuilderFixedInputs.envPlainValue,
agentBuilderFixedInputs.envModeKey,
agentBuilderFixedInputs.envModeValue,
]))
await expect(advancedSettings.getByText('Plain', { exact: true })).toHaveCount(2)
},
)
Then(
'I should not see the deleted Agent v2 environment variable in Advanced Settings',
async function (this: DifyWorld) {
const page = this.getPage()
const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' })
await page.getByRole('button', { name: 'Advanced Settings' }).first().click()
await expect(advancedSettings.getByRole('heading', { name: 'Env Editor' })).toBeVisible()
await expect.poll(
async () => advancedSettings.getByRole('textbox').evaluateAll(inputs =>
inputs.map(input => (input as HTMLInputElement).value),
),
{ timeout: 30_000 },
).toEqual(expect.arrayContaining([
agentBuilderFixedInputs.envModeKey,
agentBuilderFixedInputs.envModeValue,
]))
await expect.poll(
async () => advancedSettings.getByRole('textbox').evaluateAll(inputs =>
inputs.map(input => (input as HTMLInputElement).value),
),
{ timeout: 30_000 },
).not.toContain(agentBuilderFixedInputs.envPlainKey)
await expect(advancedSettings.getByText('Plain', { exact: true })).toHaveCount(1)
},
)
Then(
'I should see the plain Agent v2 environment variable in Advanced Settings',
async function (this: DifyWorld) {
const page = this.getPage()
const advancedSettings = await openAgentAdvancedSettings(page)
await expect(advancedSettings.getByRole('textbox', { name: 'Key' }))
.toHaveValue(agentBuilderFixedInputs.envPlainKey)
await expect(advancedSettings.getByRole('textbox', { name: 'Value' }))
.toHaveValue(agentBuilderFixedInputs.envPlainValue)
await expect(advancedSettings.getByText('Plain', { exact: true })).toBeVisible()
await expect(page.getByRole('button', { name: /^Build$/i })).toBeVisible()
},
)
Then(
'I should see the supported E2E environment variable in Advanced Settings',
async function (this: DifyWorld) {
await expectAgentEnvVariableVisible(
this,
agentBuilderFixedInputs.envPlainKey,
agentBuilderFixedInputs.envPlainValue,
)
},
)
Then(
'I should not see the supported E2E environment variable in Advanced Settings',
async function (this: DifyWorld) {
await expectAgentEnvVariableHidden(this, agentBuilderFixedInputs.envPlainKey)
},
)
Then(
'I should see the Agent v2 environment variables from the invalid import in Advanced Settings',
async function (this: DifyWorld) {
const page = this.getPage()
const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' })
await page.getByRole('button', { name: 'Advanced Settings' }).first().click()
await expect(advancedSettings.getByRole('heading', { name: 'Env Editor' })).toBeVisible()
await expect.poll(
async () => advancedSettings.getByRole('textbox').evaluateAll(inputs =>
inputs.map(input => (input as HTMLInputElement).value),
),
{ timeout: 30_000 },
).toEqual(expect.arrayContaining([
agentBuilderFixedInputs.envPlainKey,
agentBuilderFixedInputs.envPlainValue,
agentBuilderFixedInputs.envAfterInvalidImportKey,
agentBuilderFixedInputs.envAfterInvalidImportValue,
]))
await expect(page.getByRole('button', { name: /^Build$/i })).toBeVisible()
},
)

117
e2e/features/step-definitions/agent-v2/agent-edit.steps.ts Normal file
View File

@ -0,0 +1,117 @@
import type { DifyWorld } from '../../support/world'
import { Then } from '@cucumber/cucumber'
import { expect } from '@playwright/test'
import { agentBuilderExpectedTokens, agentBuilderFixedInputs, agentBuilderPreseededResources } from '../../../support/agent-builder-resources'
import { agentBuilderTestMaterials } from '../../../support/test-materials'
import { expectProviderToolActionVisible, openAgentKnowledgeRetrievalDialog } from './configure-helpers'
Then('I should see the Agent v2 full-config fixture sections', async function (this: DifyWorld) {
const page = this.getPage()
const stableModel = this.agentBuilder.preflight.stableModel
if (!stableModel)
throw new Error('Stable chat model preflight must run before asserting the full-config Agent.')
await expect(page.getByRole('heading', { name: 'Configure' })).toBeVisible({ timeout: 30_000 })
await expect(page.getByText(agentBuilderPreseededResources.fullConfigAgent, { exact: true }))
.toBeVisible()
await expect(page.getByText(stableModel.name, { exact: true })).toBeVisible()
const promptSection = page.getByRole('region', { name: 'Prompt' })
await expect(promptSection).toBeVisible()
await expect(promptSection).toContainText(agentBuilderExpectedTokens.agentReply)
const skillsSection = page.getByRole('region', { name: 'Skills' })
await expect(skillsSection).toBeVisible()
await expect(skillsSection.getByRole('button', {
exact: true,
name: agentBuilderPreseededResources.summarySkill,
})).toBeVisible()
const filesSection = page.getByRole('region', { name: 'Files' })
await expect(filesSection).toBeVisible()
await expect(filesSection.getByRole('button', {
exact: true,
name: agentBuilderTestMaterials.smallFile,
})).toBeVisible()
await expect(filesSection.getByRole('button', {
exact: true,
name: agentBuilderTestMaterials.specialFilename,
})).toBeVisible()
const toolsSection = page.getByRole('region', { name: 'Tools' })
await expect(toolsSection).toBeVisible()
await expectProviderToolActionVisible(
toolsSection,
agentBuilderPreseededResources.jsonReplaceTool,
)
const knowledgeSection = page.getByRole('region', { name: 'Knowledge Retrieval' })
await expect(knowledgeSection).toBeVisible()
await expect(knowledgeSection.getByText('Retrieval 1', { exact: true })).toBeVisible()
const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' })
await expect(advancedSettings).toBeVisible()
await expect(
advancedSettings.getByText('For power users. Env vars, sandbox & memory.'),
).toBeVisible()
})
Then('I should see the Agent v2 tool state fixture tools', async function (this: DifyWorld) {
const page = this.getPage()
const toolsSection = page.getByRole('region', { name: 'Tools' })
await expect(toolsSection).toBeVisible({ timeout: 30_000 })
await expect(toolsSection.getByRole('button', { exact: true, name: 'Not authorized' })).toBeVisible()
const { action: jsonReplaceAction, tool: jsonTool } = await expectProviderToolActionVisible(
toolsSection,
agentBuilderPreseededResources.jsonReplaceTool,
)
await jsonReplaceAction.hover()
await expect(toolsSection.getByRole('button', {
exact: true,
name: `Edit ${jsonTool.actionName}`,
})).toBeVisible()
await expect(toolsSection.getByRole('button', {
exact: true,
name: `Remove ${jsonTool.actionName}`,
})).toBeVisible()
await expectProviderToolActionVisible(
toolsSection,
agentBuilderPreseededResources.tavilySearchTool,
)
})
Then('I should see the Agent v2 dual retrieval fixture settings', async function (this: DifyWorld) {
const page = this.getPage()
const knowledgeSection = page.getByRole('region', { name: 'Knowledge Retrieval' })
await expect(knowledgeSection).toBeVisible({ timeout: 30_000 })
await expect(knowledgeSection.getByText('Retrieval 1', { exact: true })).toBeVisible()
await expect(knowledgeSection.getByText('Retrieval 2', { exact: true })).toBeVisible()
const agentDecideDialog = await openAgentKnowledgeRetrievalDialog(knowledgeSection, 'Retrieval 1')
await expect(agentDecideDialog.getByText(agentBuilderPreseededResources.agentKnowledgeBase, {
exact: true,
})).toBeVisible()
await expect(agentDecideDialog.getByRole('radio', {
exact: true,
name: 'Agent decide',
})).toBeChecked()
await agentDecideDialog.getByRole('button', { name: 'Close' }).click()
await expect(agentDecideDialog).not.toBeVisible()
const customQueryDialog = await openAgentKnowledgeRetrievalDialog(knowledgeSection, 'Retrieval 2')
await expect(customQueryDialog.getByText(agentBuilderPreseededResources.agentKnowledgeBase, {
exact: true,
})).toBeVisible()
await expect(customQueryDialog.getByRole('radio', {
exact: true,
name: 'Custom query',
})).toBeChecked()
await expect(customQueryDialog.getByRole('textbox', {
exact: true,
name: 'Custom query text',
})).toHaveValue(agentBuilderFixedInputs.customKnowledgeQuery)
})

291
e2e/features/step-definitions/agent-v2/build-draft.steps.ts Normal file
View File

@ -0,0 +1,291 @@
import type { DifyWorld } from '../../support/world'
import { readFile } from 'node:fs/promises'
import { Given, Then, When } from '@cucumber/cucumber'
import { expect } from '@playwright/test'
import {
createAgentSoulConfigWithModel,
getAgentComposerDraft,
normalAgentPrompt,
normalAgentSoulConfig,
saveAgentBuildDraft,
saveAgentComposerDraft,
updatedAgentPrompt,
updatedAgentSoulConfig,
uploadAgentConfigFileToDraft,
} from '../../../support/agent'
import { agentBuilderFixedInputs, agentBuilderPreseededResources } from '../../../support/agent-builder-resources'
import { skipBlockedPrecondition } from '../../../support/preflight'
import { agentBuilderTestMaterials, getAgentBuilderTestMaterialPath } from '../../../support/test-materials'
import {
getAgentEnvVariableValue,
getCurrentAgentId,
uploadSummaryConfigSkillForBuildDraft,
} from './configure-helpers'
Given(
'an Agent v2 Build draft adds the supported E2E files, skills, and env',
async function (this: DifyWorld) {
const agentId = getCurrentAgentId(this)
const configFile = await uploadAgentConfigFileToDraft({
agentId,
fileName: agentBuilderTestMaterials.smallFile,
filePath: getAgentBuilderTestMaterialPath('smallFile'),
})
const skill = await uploadSummaryConfigSkillForBuildDraft(this)
if (!configFile.file_id)
throw new Error('Agent v2 build draft config file fixture did not return a file_id.')
this.createdAgentConfigFiles.push({ agentId, name: configFile.name })
const normalConfig = this.agentBuilder.preflight.stableModel
? createAgentSoulConfigWithModel(normalAgentSoulConfig, this.agentBuilder.preflight.stableModel)
: normalAgentSoulConfig
const updatedConfig = this.agentBuilder.preflight.stableModel
? createAgentSoulConfigWithModel(updatedAgentSoulConfig, this.agentBuilder.preflight.stableModel)
: updatedAgentSoulConfig
await saveAgentComposerDraft(agentId, normalConfig)
await saveAgentBuildDraft(agentId, {
...updatedConfig,
config_files: [configFile],
config_skills: [{
...skill,
file_kind: skill.file_kind ?? 'tool_file',
}],
env: {
secret_refs: [],
variables: [{
id: agentBuilderFixedInputs.envPlainKey,
key: agentBuilderFixedInputs.envPlainKey,
name: agentBuilderFixedInputs.envPlainKey,
value: agentBuilderFixedInputs.envPlainValue,
variable: agentBuilderFixedInputs.envPlainKey,
}],
},
})
},
)
Given(
'an Agent v2 Build draft includes the existing e2e-summary-skill Skill',
async function (this: DifyWorld) {
const skill = await uploadSummaryConfigSkillForBuildDraft(this)
const normalConfig = this.agentBuilder.preflight.stableModel
? createAgentSoulConfigWithModel(normalAgentSoulConfig, this.agentBuilder.preflight.stableModel)
: normalAgentSoulConfig
const updatedConfig = this.agentBuilder.preflight.stableModel
? createAgentSoulConfigWithModel(updatedAgentSoulConfig, this.agentBuilder.preflight.stableModel)
: updatedAgentSoulConfig
const configSkills = [{
...skill,
file_kind: skill.file_kind ?? 'tool_file',
}]
await saveAgentComposerDraft(getCurrentAgentId(this), {
...normalConfig,
config_skills: configSkills,
})
await saveAgentBuildDraft(getCurrentAgentId(this), {
...updatedConfig,
config_skills: configSkills,
})
},
)
Given('an Agent v2 Build draft uses the updated E2E prompt', async function (this: DifyWorld) {
await saveAgentBuildDraft(getCurrentAgentId(this), updatedAgentSoulConfig)
})
Given(
'an Agent v2 Build draft uses the updated E2E prompt with the stable E2E model',
async function (this: DifyWorld) {
if (!this.agentBuilder.preflight.stableModel)
throw new Error('Create an Agent v2 Build draft with a stable model after stable model preflight.')
await saveAgentBuildDraft(
getCurrentAgentId(this),
createAgentSoulConfigWithModel(updatedAgentSoulConfig, this.agentBuilder.preflight.stableModel),
)
},
)
When('I generate an Agent v2 Build draft from the fixed instruction', async function (this: DifyWorld) {
const page = this.getPage()
const agentId = getCurrentAgentId(this)
const instruction = (await readFile(getAgentBuilderTestMaterialPath('buildInstruction'), 'utf8')).trim()
await page.getByRole('button', { name: 'Build' }).click()
await page.getByPlaceholder('Describe what your agent should do').fill(instruction)
const checkoutResponsePromise = page.waitForResponse(response => (
response.request().method() === 'POST'
&& new URL(response.url()).pathname.endsWith(`/console/api/agent/${agentId}/build-draft/checkout`)
))
const chatResponsePromise = page.waitForResponse(response => (
response.request().method() === 'POST'
&& new URL(response.url()).pathname.endsWith(`/console/api/agent/${agentId}/chat-messages`)
))
await page.getByRole('button', { name: 'Start build' }).click()
expect((await checkoutResponsePromise).ok()).toBe(true)
expect((await chatResponsePromise).ok()).toBe(true)
await expect(page.getByText('Build draft')).toBeVisible({ timeout: 120_000 })
await expect(page.getByRole('button', { name: 'Apply' })).toBeEnabled({ timeout: 120_000 })
await expect(page.getByRole('button', { name: 'Discard' })).toBeEnabled()
})
When('I discard the Agent v2 Build draft', async function (this: DifyWorld) {
await this.getPage().getByRole('button', { name: 'Discard' }).click()
})
When('I apply the Agent v2 Build draft', async function (this: DifyWorld) {
const page = this.getPage()
const agentId = getCurrentAgentId(this)
const applyButton = page.getByRole('button', { name: 'Apply' })
await expect(applyButton).toBeEnabled({ timeout: 30_000 })
const finalizeResponsePromise = page.waitForResponse(response => (
response.request().method() === 'POST'
&& new URL(response.url()).pathname.endsWith(`/console/api/agent/${agentId}/build-chat/finalize`)
), { timeout: 120_000 })
const applyResponsePromise = page.waitForResponse(response => (
response.request().method() === 'POST'
&& new URL(response.url()).pathname.endsWith(`/console/api/agent/${agentId}/build-draft/apply`)
), { timeout: 120_000 })
await applyButton.click()
expect((await finalizeResponsePromise).ok()).toBe(true)
expect((await applyResponsePromise).ok()).toBe(true)
await expect(page.getByText('Action succeeded')).toBeVisible()
})
Then('Agent v2 Build chat Dify Tool writeback should be available', async function (this: DifyWorld) {
const toolsSection = this.getPage().getByRole('region', { name: 'Tools' })
await expect(toolsSection).toBeVisible({ timeout: 30_000 })
return skipBlockedPrecondition(
this,
'Build draft Dify Tool writeback is not available: Build draft currently supports files, skills, and env only.',
)
})
Then('I should see the Agent v2 Build draft pending changes', async function (this: DifyWorld) {
const page = this.getPage()
await expect(page.getByText('Build draft')).toBeVisible({ timeout: 30_000 })
await expect(page.getByRole('button', { name: 'Apply' })).toBeEnabled()
await expect(page.getByRole('button', { name: 'Discard' })).toBeEnabled()
})
Then('I should see the Agent v2 Build mode confirmation state', async function (this: DifyWorld) {
const page = this.getPage()
await expect(page.getByText('Build mode', { exact: true })).toBeVisible()
await expect(
page.getByText('You\'re in build mode. Shape this setup through the chat on the right, then Apply.'),
).toBeVisible()
})
Then('I should see the e2e-summary-skill Skill in the Skills section', async function (this: DifyWorld) {
const skillsSection = this.getPage().getByRole('region', { name: 'Skills' })
await expect(skillsSection).toBeVisible({ timeout: 30_000 })
await expect(skillsSection.getByRole('button', {
exact: true,
name: agentBuilderPreseededResources.summarySkill,
})).toBeVisible()
})
Then('I should not see the e2e-summary-skill Skill in the Skills section', async function (this: DifyWorld) {
const skillsSection = this.getPage().getByRole('region', { name: 'Skills' })
await expect(skillsSection).toBeVisible({ timeout: 30_000 })
await expect(skillsSection.getByRole('button', {
exact: true,
name: agentBuilderPreseededResources.summarySkill,
})).toHaveCount(0)
})
Then('I should see one e2e-summary-skill Skill in the Skills section', async function (this: DifyWorld) {
const skillsSection = this.getPage().getByRole('region', { name: 'Skills' })
await expect(skillsSection).toBeVisible({ timeout: 30_000 })
await expect(skillsSection.getByRole('button', {
exact: true,
name: agentBuilderPreseededResources.summarySkill,
})).toHaveCount(1)
})
Then(
'the Agent v2 draft should include the supported Build draft config',
async function (this: DifyWorld) {
await expect.poll(
async () => {
const agentSoul = (await getAgentComposerDraft(getCurrentAgentId(this))).agent_soul
const variables = agentSoul?.env?.variables ?? []
return {
envValue: getAgentEnvVariableValue(variables, agentBuilderFixedInputs.envPlainKey),
fileNames: agentSoul?.config_files?.map(file => file.name) ?? [],
prompt: agentSoul?.prompt,
skillNames: agentSoul?.config_skills?.map(skill => skill.name) ?? [],
}
},
{ timeout: 30_000 },
).toEqual({
envValue: agentBuilderFixedInputs.envPlainValue,
fileNames: expect.arrayContaining([agentBuilderTestMaterials.smallFile]),
prompt: { system_prompt: updatedAgentPrompt },
skillNames: expect.arrayContaining([agentBuilderPreseededResources.summarySkill]),
})
},
)
Then(
'the Agent v2 draft should not include the supported Build draft config',
async function (this: DifyWorld) {
await expect.poll(
async () => {
const agentSoul = (await getAgentComposerDraft(getCurrentAgentId(this))).agent_soul
const variables = agentSoul?.env?.variables ?? []
return {
envValue: getAgentEnvVariableValue(variables, agentBuilderFixedInputs.envPlainKey),
fileNames: agentSoul?.config_files?.map(file => file.name) ?? [],
prompt: agentSoul?.prompt,
skillNames: agentSoul?.config_skills?.map(skill => skill.name) ?? [],
}
},
{ timeout: 30_000 },
).toEqual({
envValue: undefined,
fileNames: expect.not.arrayContaining([agentBuilderTestMaterials.smallFile]),
prompt: { system_prompt: normalAgentPrompt },
skillNames: expect.not.arrayContaining([agentBuilderPreseededResources.summarySkill]),
})
},
)
Then(
'the Agent v2 draft should include one e2e-summary-skill Skill',
async function (this: DifyWorld) {
await expect.poll(
async () => {
const agentSoul = (await getAgentComposerDraft(getCurrentAgentId(this))).agent_soul
return agentSoul?.config_skills?.filter(
skill => skill.name === agentBuilderPreseededResources.summarySkill,
).length ?? 0
},
{ timeout: 30_000 },
).toBe(1)
},
)
Then('the Agent v2 Build draft should no longer be active', async function (this: DifyWorld) {
const page = this.getPage()
await expect(page.getByText('Build draft')).not.toBeVisible()
await expect(page.getByRole('button', { name: 'Apply' })).not.toBeVisible()
await expect(page.getByRole('button', { name: 'Discard' })).not.toBeVisible()
})

241
e2e/features/step-definitions/agent-v2/configure-helpers.ts Normal file
View File

@ -0,0 +1,241 @@
import type { Locator } from '@playwright/test'
import type { AgentComposerEnvVariable } from '../../../support/agent'
import type { DifyWorld } from '../../support/world'
import { expect } from '@playwright/test'
import {
getAgentComposerDraft,
normalAgentPrompt,
uploadAgentConfigSkillToDraft,
} from '../../../support/agent'
import {
agentBuilderTestMaterials,
getAgentBuilderTestMaterialPath,
} from '../../../support/test-materials'
export const getCurrentAgentId = (world: DifyWorld) => {
const agentId = world.createdAgentIds.at(-1)
if (!agentId)
throw new Error('No Agent v2 ID found. Create an Agent v2 test agent first.')
return agentId
}
export const getPreseededAgent = (world: DifyWorld, name: string) => {
const resource = world.agentBuilder.preflight.preseededResources[name]
if (!resource || resource.kind !== 'agent') {
throw new Error(
`Preseeded Agent "${name}" is not available. Run the matching preflight step first.`,
)
}
return resource
}
const getPreseededToolDisplayParts = (displayName: string) => {
const [providerName, actionName] = displayName.split(' / ')
if (!providerName || !actionName)
throw new Error(`Preseeded tool display name must use "Provider / Action": ${displayName}`)
return { actionName, providerName }
}
export const getEnvVariableKey = (variable: AgentComposerEnvVariable) =>
variable.key ?? variable.name ?? variable.variable
export const getAgentEnvVariableValue = (
variables: AgentComposerEnvVariable[],
key: string,
) => variables.find(variable => getEnvVariableKey(variable) === key)?.value
export const getAgentEnvVariables = async (agentId: string) =>
(await getAgentComposerDraft(agentId)).agent_soul?.env?.variables ?? []
export const uploadAgentConfigFile = async (
world: DifyWorld,
material: keyof typeof agentBuilderTestMaterials,
) => {
const page = world.getPage()
const agentId = getCurrentAgentId(world)
const fileName = agentBuilderTestMaterials[material]
const filePath = getAgentBuilderTestMaterialPath(material)
await page.getByRole('button', { name: 'Add file' }).click()
const dialog = page.getByRole('dialog', { name: 'Upload file' })
await expect(dialog).toBeVisible()
const fileChooserPromise = page.waitForEvent('filechooser')
await dialog.getByRole('button', { name: 'browse' }).click()
await (await fileChooserPromise).setFiles(filePath)
await expect(dialog.getByText(fileName)).toBeVisible()
const commitResponsePromise = page.waitForResponse(response => (
response.request().method() === 'POST'
&& new URL(response.url()).pathname.endsWith(`/console/api/agent/${agentId}/config/files`)
))
await dialog.getByRole('button', { name: 'Upload' }).click()
const commitResponse = await commitResponsePromise
expect(commitResponse.status()).toBe(201)
const committed = await commitResponse.json() as { file?: { name?: string } }
await expect(dialog).not.toBeVisible({ timeout: 30_000 })
const committedName = committed.file?.name
if (!committedName)
throw new Error('Agent config file upload response did not include a file name.')
world.createdAgentConfigFiles.push({ agentId, name: committedName })
}
export const expectAgentConfigFileVisible = async (
world: DifyWorld,
material: keyof typeof agentBuilderTestMaterials,
) => {
await expect(
world.getPage().getByRole('button', {
exact: true,
name: agentBuilderTestMaterials[material],
}),
).toBeVisible({ timeout: 30_000 })
}
export const expectAgentConfigFileHidden = async (
world: DifyWorld,
material: keyof typeof agentBuilderTestMaterials,
) => {
await expect(
world.getPage().getByRole('button', {
exact: true,
name: agentBuilderTestMaterials[material],
}),
).not.toBeVisible()
}
export const expectAgentConfigFileSaved = async (
world: DifyWorld,
material: keyof typeof agentBuilderTestMaterials,
) => {
const agentId = getCurrentAgentId(world)
const fileName = agentBuilderTestMaterials[material]
await expect
.poll(async () => (
await getAgentComposerDraft(agentId)
).agent_soul?.config_files?.map(file => file.name) ?? [], {
timeout: 30_000,
})
.toContain(fileName)
}
export const uploadSummaryConfigSkillForBuildDraft = async (world: DifyWorld) => {
const agentId = getCurrentAgentId(world)
const skill = await uploadAgentConfigSkillToDraft({
agentId,
fileName: agentBuilderTestMaterials.summarySkill,
filePath: getAgentBuilderTestMaterialPath('summarySkill'),
})
if (!skill.file_id)
throw new Error('Agent v2 build draft Skill fixture did not return a file_id.')
world.createdAgentConfigSkills.push({ agentId, name: skill.name })
return skill
}
export const openAgentAdvancedSettings = async (page: ReturnType<DifyWorld['getPage']>) => {
const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' })
const envEditorHeading = advancedSettings.getByRole('heading', { name: 'Env Editor' })
if (!await envEditorHeading.isVisible().catch(() => false))
await page.getByRole('button', { name: 'Advanced Settings' }).first().click()
await expect(envEditorHeading).toBeVisible()
return advancedSettings
}
export const expectAgentEnvVariableVisible = async (
world: DifyWorld,
key: string,
value: string,
) => {
const advancedSettings = await openAgentAdvancedSettings(world.getPage())
await expect.poll(
async () => {
const text = await advancedSettings.textContent()
const inputValues = await advancedSettings.getByRole('textbox').evaluateAll(inputs =>
inputs.map(input => (input as HTMLInputElement).value),
)
return {
hasKey: inputValues.includes(key) || !!text?.includes(key),
hasValue: inputValues.includes(value) || !!text?.includes(value),
}
},
{ timeout: 30_000 },
).toEqual({
hasKey: true,
hasValue: true,
})
await expect(advancedSettings.getByText('Plain', { exact: true })).toBeVisible()
}
export const expectAgentEnvVariableHidden = async (
world: DifyWorld,
key: string,
) => {
const advancedSettings = await openAgentAdvancedSettings(world.getPage())
await expect.poll(
async () => {
const text = await advancedSettings.textContent()
const inputValues = await advancedSettings.getByRole('textbox').evaluateAll(inputs =>
inputs.map(input => (input as HTMLInputElement).value),
)
return inputValues.includes(key) || !!text?.includes(key)
},
{ timeout: 30_000 },
).toBe(false)
}
export const expectNormalAgentPromptDraft = async (world: DifyWorld) => {
await expect.poll(
async () => (await getAgentComposerDraft(getCurrentAgentId(world))).agent_soul?.prompt,
{ timeout: 30_000 },
).toEqual({ system_prompt: normalAgentPrompt })
}
export const expectProviderToolActionVisible = async (
toolsSection: Locator,
displayName: string,
) => {
const tool = getPreseededToolDisplayParts(displayName)
const provider = toolsSection.getByRole('button', {
exact: true,
name: tool.providerName,
})
await expect(provider).toBeVisible()
const action = toolsSection.getByText(tool.actionName, { exact: true })
if (!await action.isVisible())
await provider.click()
await expect(action).toBeVisible()
return { action, tool }
}
export const openAgentKnowledgeRetrievalDialog = async (knowledgeSection: Locator, name: string) => {
await knowledgeSection.getByText(name, { exact: true }).hover()
await knowledgeSection.getByRole('button', {
exact: true,
name: `Edit ${name}`,
}).click()
const dialog = knowledgeSection.page().getByRole('dialog', {
name: 'Knowledge Retrieval · Agent decide',
})
await expect(dialog).toBeVisible()
return dialog
}

1086
e2e/features/step-definitions/agent-v2/configure.steps.ts
View File

File diff suppressed because it is too large Load Diff

67
e2e/features/step-definitions/agent-v2/files.steps.ts Normal file
View File

@ -0,0 +1,67 @@
import type { DifyWorld } from '../../support/world'
import { Then, When } from '@cucumber/cucumber'
import { expect } from '@playwright/test'
import { agentBuilderFileTreeFixtureFileNames } from '../../../support/test-materials'
import {
expectAgentConfigFileHidden,
expectAgentConfigFileSaved,
expectAgentConfigFileVisible,
uploadAgentConfigFile,
} from './configure-helpers'
When('I upload the small Agent v2 file from the Files section', async function (this: DifyWorld) {
await uploadAgentConfigFile(this, 'smallFile')
})
When('I upload the special-name Agent v2 file from the Files section', async function (this: DifyWorld) {
await uploadAgentConfigFile(this, 'specialFilename')
})
Then(
'I should see the Agent v2 file fixture entries in the current flat Files list',
async function (this: DifyWorld) {
const page = this.getPage()
const filesSection = page.getByRole('region', { name: 'Files' })
const filesList = filesSection.getByLabel('Agent files')
await expect(filesSection).toBeVisible({ timeout: 30_000 })
await expect(filesList).toBeVisible()
for (const fileName of agentBuilderFileTreeFixtureFileNames) {
await expect(filesList.getByRole('button', {
exact: true,
name: fileName,
})).toBeVisible()
}
await expect(filesList.getByRole('button', { exact: true, name: 'assets' })).toHaveCount(0)
await expect(filesList.getByRole('button', { exact: true, name: 'docs' })).toHaveCount(0)
await expect(filesList.getByRole('button', { exact: true, name: 'public' })).toHaveCount(0)
await expect(filesList.getByRole('button', { exact: true, name: 'src' })).toHaveCount(0)
await expect(filesList.getByRole('button', { exact: true, name: 'web-game' })).toHaveCount(0)
},
)
Then('I should see the small Agent v2 file in the Files section', async function (this: DifyWorld) {
await expectAgentConfigFileVisible(this, 'smallFile')
})
Then('I should not see the small Agent v2 file in the Files section', async function (this: DifyWorld) {
await expectAgentConfigFileHidden(this, 'smallFile')
})
Then('I should see the special-name Agent v2 file in the Files section', async function (this: DifyWorld) {
await expectAgentConfigFileVisible(this, 'specialFilename')
})
Then(
'the small Agent v2 file should be saved in the Agent v2 draft',
async function (this: DifyWorld) {
await expectAgentConfigFileSaved(this, 'smallFile')
},
)
Then(
'the special-name Agent v2 file should be saved in the Agent v2 draft',
async function (this: DifyWorld) {
await expectAgentConfigFileSaved(this, 'specialFilename')
},
)

15
e2e/features/step-definitions/agent-v2/output-variables.steps.ts Normal file
View File

@ -0,0 +1,15 @@
import type { DifyWorld } from '../../support/world'
import { Then } from '@cucumber/cucumber'
import { expect } from '@playwright/test'
import { skipBlockedPrecondition } from '../../../support/preflight'
Then('Agent v2 standalone Output Variables should be available', async function (this: DifyWorld) {
const page = this.getPage()
await expect(page.getByRole('heading', { name: 'Configure' })).toBeVisible({ timeout: 30_000 })
return skipBlockedPrecondition(
this,
'Standalone Agent Output Variables are not available: output variables currently belong to Workflow Agent v2 nodes.',
)
})

27
e2e/features/step-definitions/agent-v2/publish.steps.ts Normal file
View File

@ -0,0 +1,27 @@
import type { DifyWorld } from '../../support/world'
import { Then, When } from '@cucumber/cucumber'
import { expect } from '@playwright/test'
import { getTestAgent } from '../../../support/agent'
import { waitForAgentConfigureAutosaved } from '../../../support/agent-configure'
import { getCurrentAgentId } from './configure-helpers'
When('I publish the Agent v2 draft', async function (this: DifyWorld) {
const page = this.getPage()
const publishButton = page.getByRole('button', { name: /^Publish(?: update)?$/ })
await expect(publishButton).toBeEnabled({ timeout: 30_000 })
await publishButton.click()
})
Then('the Agent v2 configuration should be saved automatically', async function (this: DifyWorld) {
await waitForAgentConfigureAutosaved(this.getPage())
})
Then('the Agent v2 draft should be published and up to date', async function (this: DifyWorld) {
const page = this.getPage()
const agentId = getCurrentAgentId(this)
await expect(page.getByRole('button', { name: 'Published' })).toBeVisible({ timeout: 30_000 })
await expect(page.getByText('Up to date')).toBeVisible()
await expect.poll(async () => (await getTestAgent(agentId)).active_config_is_published).toBe(true)
})

1132
e2e/support/preflight.ts
View File

File diff suppressed because it is too large Load Diff

166
e2e/support/preflight/access.ts Normal file
View File

@ -0,0 +1,166 @@
import type { DifyWorld } from '../../features/support/world'
import type { PreseededResource } from './common'
import { createApiContext, expectApiResponseOK } from '../api'
import { skipMissingPreseededAgent, skipMissingPreseededWorkflow } from './agents'
import { skipBlockedPrecondition } from './common'
type AgentApiAccessResponse = {
api_key_count: number
enabled: boolean
}
type AgentApiKeyListResponse = {
data: Array<{
id: string
}>
}
type AgentReferencingWorkflowsResponse = {
data: Array<{
app_id: string
app_name: string
node_ids?: string[]
}>
}
type PreseededAgentDetailResponse = {
active_config_is_published?: boolean
enable_site?: boolean
site?: {
access_token?: string | null
app_base_url?: string | null
code?: string | null
} | null
}
export async function skipMissingPreseededAgentBackendApiKey(
world: DifyWorld,
agentName: string,
): Promise<'skipped' | PreseededResource> {
const agent = await skipMissingPreseededAgent(world, agentName)
if (agent === 'skipped')
return agent
const ctx = await createApiContext()
try {
const accessResponse = await ctx.get(`/console/api/agent/${agent.id}/api-access`)
await expectApiResponseOK(accessResponse, `Check preseeded Agent API access ${agentName}`)
const access = (await accessResponse.json()) as AgentApiAccessResponse
if (!access.enabled || access.api_key_count < 1) {
return skipBlockedPrecondition(
world,
`Preseeded Agent "${agentName}" does not have Backend service API enabled with an API key.`,
)
}
const keyResponse = await ctx.get(`/console/api/agent/${agent.id}/api-keys`)
await expectApiResponseOK(keyResponse, `Check preseeded Agent API key ${agentName}`)
const keys = (await keyResponse.json()) as AgentApiKeyListResponse
const key = keys.data.at(0)
if (!key) {
return skipBlockedPrecondition(
world,
`Preseeded Agent "${agentName}" Backend service API key list is empty.`,
)
}
return {
id: key.id,
kind: 'api-key',
name: `${agentName} Backend service API key`,
}
}
finally {
await ctx.dispose()
}
}
export async function skipMissingPreseededAgentPublishedWebApp(
world: DifyWorld,
agentName: string,
): Promise<'skipped' | PreseededResource> {
const agent = await skipMissingPreseededAgent(world, agentName)
if (agent === 'skipped')
return agent
const ctx = await createApiContext()
try {
const response = await ctx.get(`/console/api/agent/${agent.id}`)
await expectApiResponseOK(response, `Check preseeded Agent published Web app ${agentName}`)
const detail = (await response.json()) as PreseededAgentDetailResponse
if (detail.active_config_is_published !== true) {
return skipBlockedPrecondition(world, `Preseeded Agent "${agentName}" is not published.`)
}
if (detail.enable_site !== true) {
return skipBlockedPrecondition(
world,
`Preseeded Agent "${agentName}" Web app is not enabled.`,
)
}
const siteToken = detail.site?.access_token ?? detail.site?.code
if (!siteToken || !detail.site?.app_base_url) {
return skipBlockedPrecondition(
world,
`Preseeded Agent "${agentName}" Web app URL is not available.`,
)
}
return {
id: agent.id,
kind: 'agent',
name: agent.name,
}
}
finally {
await ctx.dispose()
}
}
export async function skipMissingPreseededAgentWorkflowReference(
world: DifyWorld,
agentName: string,
workflowName: string,
): Promise<'skipped' | PreseededResource> {
const agent = await skipMissingPreseededAgent(world, agentName)
if (agent === 'skipped')
return agent
const workflow = await skipMissingPreseededWorkflow(world, workflowName)
if (workflow === 'skipped')
return workflow
const ctx = await createApiContext()
try {
const response = await ctx.get(`/console/api/agent/${agent.id}/referencing-workflows`)
await expectApiResponseOK(response, `Check preseeded Agent workflow reference ${agentName}`)
const references = (await response.json()) as AgentReferencingWorkflowsResponse
const reference = references.data.find(
item => item.app_id === workflow.id || item.app_name === workflow.name,
)
if (!reference) {
return skipBlockedPrecondition(
world,
`Preseeded Agent "${agentName}" is not referenced by workflow "${workflowName}".`,
)
}
if (!reference.node_ids || reference.node_ids.length < 1) {
return skipBlockedPrecondition(
world,
`Preseeded workflow "${workflowName}" does not expose Agent reference nodes for "${agentName}".`,
)
}
return {
id: workflow.id,
kind: 'workflow',
name: workflow.name,
}
}
finally {
await ctx.dispose()
}
}

472
e2e/support/preflight/agents.ts Normal file
View File

@ -0,0 +1,472 @@
import type { DifyWorld } from '../../features/support/world'
import type { AgentComposerResponse, PreseededResource } from './common'
import {
agentBuilderExpectedTokens,
agentBuilderFixedInputs,
agentBuilderPreseededResources,
} from '../agent-builder-resources'
import { createApiContext, expectApiResponseOK } from '../api'
import {
agentBuilderFileTreeFixtureFileNames,
agentBuilderFileTreeFixtureFiles,
agentBuilderTestMaterials,
} from '../test-materials'
import {
asArray,
asRecord,
asString,
buildQuery,
findConsoleResourceByName,
hasNamedOrKeyedEntry,
skipBlockedPrecondition,
} from './common'
import { skipMissingReadyPreseededDataset } from './datasets'
import { skipMissingAgentBuilderStableChatModel } from './models'
import {
findToolEntry,
hasToolEntry,
hasUnauthorizedToolCredentialState,
skipMissingPreseededTool,
splitToolDisplayName,
} from './tools'
type AgentDriveSkillListResponse = {
items: Array<{
name: string
path: string
}>
}
type AgentDriveFileListResponse = {
items?: Array<{
key: string
}>
}
const hasKnowledgeDataset = (
soul: Record<string, unknown>,
dataset: PreseededResource,
) => {
const knowledge = asRecord(soul.knowledge)
const sets = asArray(knowledge.sets)
return sets.some((set) => {
const datasets = asArray(asRecord(set).datasets)
return datasets.some((item) => {
const record = asRecord(item)
return record.id === dataset.id || record.name === dataset.name
})
})
}
const hasKnowledgeSet = (
soul: Record<string, unknown>,
dataset: PreseededResource,
{
queryMode,
queryValue,
}: {
queryMode: 'generated_query' | 'user_query'
queryValue?: string
},
) => {
const knowledge = asRecord(soul.knowledge)
const sets = asArray(knowledge.sets)
return sets.some((set) => {
const record = asRecord(set)
const query = asRecord(record.query)
const datasets = asArray(record.datasets)
const hasExpectedDataset = datasets.some((item) => {
const datasetRecord = asRecord(item)
return datasetRecord.id === dataset.id || datasetRecord.name === dataset.name
})
if (!hasExpectedDataset || query.mode !== queryMode)
return false
if (queryValue === undefined)
return true
return asString(query.value).trim() === queryValue
})
}
export async function skipMissingPreseededAgent(
world: DifyWorld,
resourceName: string,
): Promise<'skipped' | PreseededResource> {
const query = buildQuery({ limit: '20', name: resourceName, page: '1' })
const resource = await findConsoleResourceByName({
action: `Check preseeded Agent ${resourceName}`,
path: `/console/api/agent?${query}`,
resourceName,
})
if (!resource)
return skipBlockedPrecondition(world, `Preseeded Agent "${resourceName}" was not found.`)
return {
id: resource.id,
kind: 'agent',
name: resource.name,
}
}
export async function skipMissingPreseededWorkflow(
world: DifyWorld,
resourceName: string,
): Promise<'skipped' | PreseededResource> {
const query = buildQuery({ limit: '20', mode: 'workflow', name: resourceName, page: '1' })
const resource = await findConsoleResourceByName({
action: `Check preseeded workflow ${resourceName}`,
path: `/console/api/apps?${query}`,
resourceName,
})
if (!resource)
return skipBlockedPrecondition(world, `Preseeded workflow "${resourceName}" was not found.`)
return {
id: resource.id,
kind: 'workflow',
name: resource.name,
}
}
export async function skipMissingPreseededAgentDriveSkill(
world: DifyWorld,
agentName: string,
skillName: string,
): Promise<'skipped' | PreseededResource> {
const agent = await skipMissingPreseededAgent(world, agentName)
if (agent === 'skipped')
return agent
const ctx = await createApiContext()
try {
const response = await ctx.get(`/console/api/agent/${agent.id}/drive/skills`)
await expectApiResponseOK(response, `Check preseeded Agent skill ${skillName}`)
const body = (await response.json()) as AgentDriveSkillListResponse
const skill = body.items.find(item => item.name === skillName)
if (!skill) {
return skipBlockedPrecondition(
world,
`Preseeded Agent "${agentName}" does not include drive skill "${skillName}".`,
)
}
return {
id: skill.path,
kind: 'skill',
name: skill.name,
}
}
finally {
await ctx.dispose()
}
}
export async function skipMissingPreseededFullConfigAgentCoreConfiguration(
world: DifyWorld,
agentName: string,
): Promise<'skipped' | PreseededResource> {
const stableModel = await skipMissingAgentBuilderStableChatModel(world)
if (stableModel === 'skipped')
return stableModel
const agent = await skipMissingPreseededAgent(world, agentName)
if (agent === 'skipped')
return agent
const summarySkill = await skipMissingPreseededAgentDriveSkill(
world,
agentName,
agentBuilderPreseededResources.summarySkill,
)
if (summarySkill === 'skipped')
return summarySkill
const jsonTool = await skipMissingPreseededTool(
world,
agentBuilderPreseededResources.jsonReplaceTool,
)
if (jsonTool === 'skipped')
return jsonTool
const knowledgeBase = await skipMissingReadyPreseededDataset(
world,
agentBuilderPreseededResources.agentKnowledgeBase,
)
if (knowledgeBase === 'skipped')
return knowledgeBase
const ctx = await createApiContext()
try {
const response = await ctx.get(`/console/api/agent/${agent.id}/composer`)
await expectApiResponseOK(response, `Check preseeded Agent core configuration ${agentName}`)
const body = (await response.json()) as AgentComposerResponse
const soul = body.agent_soul ?? {}
const missing: string[] = []
const model = asRecord(soul.model)
if (model.model_provider !== stableModel.provider || model.model !== stableModel.name)
missing.push(`${agentBuilderPreseededResources.stableChatModel} model config`)
const prompt = asString(asRecord(soul.prompt).system_prompt)
if (!prompt.includes(agentBuilderExpectedTokens.agentReply))
missing.push(`Prompt token ${agentBuilderExpectedTokens.agentReply}`)
const files = asArray(asRecord(soul.files).files)
for (const fileName of [
agentBuilderTestMaterials.smallFile,
agentBuilderTestMaterials.specialFilename,
]) {
if (!hasNamedOrKeyedEntry(files, fileName))
missing.push(`file ${fileName}`)
}
const [providerName = '', toolName = ''] = jsonTool.id.split('/')
const parsedTool = splitToolDisplayName(agentBuilderPreseededResources.jsonReplaceTool)
if (
parsedTool.ok
&& !hasToolEntry(asArray(asRecord(soul.tools).dify_tools), {
providerDisplayName: parsedTool.providerName,
providerName,
toolDisplayName: parsedTool.toolName,
toolName,
})
) {
missing.push(agentBuilderPreseededResources.jsonReplaceTool)
}
if (!hasKnowledgeDataset(soul, knowledgeBase))
missing.push(agentBuilderPreseededResources.agentKnowledgeBase)
if (missing.length > 0) {
return skipBlockedPrecondition(
world,
`Preseeded Agent "${agentName}" is missing core fixture configuration: ${missing.join(', ')}.`,
)
}
return agent
}
finally {
await ctx.dispose()
}
}
export async function skipMissingPreseededToolStatesAgentConfiguration(
world: DifyWorld,
agentName: string,
): Promise<'skipped' | PreseededResource> {
const agent = await skipMissingPreseededAgent(world, agentName)
if (agent === 'skipped')
return agent
const summarySkill = await skipMissingPreseededAgentDriveSkill(
world,
agentName,
agentBuilderPreseededResources.summarySkill,
)
if (summarySkill === 'skipped')
return summarySkill
const jsonTool = await skipMissingPreseededTool(
world,
agentBuilderPreseededResources.jsonReplaceTool,
)
if (jsonTool === 'skipped')
return jsonTool
const tavilyTool = await skipMissingPreseededTool(
world,
agentBuilderPreseededResources.tavilySearchTool,
)
if (tavilyTool === 'skipped')
return tavilyTool
const ctx = await createApiContext()
try {
const response = await ctx.get(`/console/api/agent/${agent.id}/composer`)
await expectApiResponseOK(response, `Check preseeded Agent tool states ${agentName}`)
const body = (await response.json()) as AgentComposerResponse
const soul = body.agent_soul ?? {}
const toolItems = asArray(asRecord(soul.tools).dify_tools)
const missing: string[] = []
const [jsonProviderName = '', jsonToolName = ''] = jsonTool.id.split('/')
const parsedJsonTool = splitToolDisplayName(agentBuilderPreseededResources.jsonReplaceTool)
if (
parsedJsonTool.ok
&& !findToolEntry(toolItems, {
providerDisplayName: parsedJsonTool.providerName,
providerName: jsonProviderName,
toolDisplayName: parsedJsonTool.toolName,
toolName: jsonToolName,
})
) {
missing.push(agentBuilderPreseededResources.jsonReplaceTool)
}
const [tavilyProviderName = '', tavilyToolName = ''] = tavilyTool.id.split('/')
const parsedTavilyTool = splitToolDisplayName(agentBuilderPreseededResources.tavilySearchTool)
const tavilyEntry = parsedTavilyTool.ok
? findToolEntry(toolItems, {
providerDisplayName: parsedTavilyTool.providerName,
providerName: tavilyProviderName,
toolDisplayName: parsedTavilyTool.toolName,
toolName: tavilyToolName,
})
: undefined
if (!tavilyEntry) {
missing.push(agentBuilderPreseededResources.tavilySearchTool)
}
else if (!hasUnauthorizedToolCredentialState(tavilyEntry)) {
missing.push(`${agentBuilderPreseededResources.tavilySearchTool} unauthorized credential state`)
}
if (missing.length > 0) {
return skipBlockedPrecondition(
world,
`Preseeded Agent "${agentName}" is missing tool state fixture configuration: ${missing.join(', ')}.`,
)
}
return agent
}
finally {
await ctx.dispose()
}
}
export async function skipMissingPreseededDualRetrievalAgentConfiguration(
world: DifyWorld,
agentName: string,
): Promise<'skipped' | PreseededResource> {
const agent = await skipMissingPreseededAgent(world, agentName)
if (agent === 'skipped')
return agent
const knowledgeBase = await skipMissingReadyPreseededDataset(
world,
agentBuilderPreseededResources.agentKnowledgeBase,
)
if (knowledgeBase === 'skipped')
return knowledgeBase
const ctx = await createApiContext()
try {
const response = await ctx.get(`/console/api/agent/${agent.id}/composer`)
await expectApiResponseOK(response, `Check preseeded Agent dual retrieval ${agentName}`)
const body = (await response.json()) as AgentComposerResponse
const soul = body.agent_soul ?? {}
const missing: string[] = []
if (!hasKnowledgeSet(soul, knowledgeBase, { queryMode: 'generated_query' }))
missing.push('Agent decide Knowledge Retrieval')
if (
!hasKnowledgeSet(soul, knowledgeBase, {
queryMode: 'user_query',
queryValue: agentBuilderFixedInputs.customKnowledgeQuery,
})
) {
missing.push('Custom query Knowledge Retrieval')
}
if (missing.length > 0) {
return skipBlockedPrecondition(
world,
`Preseeded Agent "${agentName}" is missing dual retrieval fixture configuration: ${missing.join(', ')}.`,
)
}
return agent
}
finally {
await ctx.dispose()
}
}
export async function skipMissingPreseededAgentFileTreeFixture(
world: DifyWorld,
agentName: string,
): Promise<'skipped' | PreseededResource> {
const agent = await skipMissingPreseededAgent(world, agentName)
if (agent === 'skipped')
return agent
const ctx = await createApiContext()
try {
const query = buildQuery({ prefix: 'files/' })
const response = await ctx.get(`/console/api/agent/${agent.id}/drive/files?${query}`)
await expectApiResponseOK(response, `Check preseeded Agent file tree ${agentName}`)
const body = (await response.json()) as AgentDriveFileListResponse
const keys = (body.items ?? []).map(item => item.key)
const missingFiles = agentBuilderFileTreeFixtureFiles.filter(
filePath =>
!keys.some(key => key === `files/${filePath}` || key.endsWith(`/${filePath}`)),
)
if (missingFiles.length > 0) {
return skipBlockedPrecondition(
world,
`Preseeded Agent "${agentName}" is missing file tree fixture files: ${missingFiles.join(', ')}.`,
)
}
return {
id: agent.id,
kind: 'agent',
name: agent.name,
}
}
finally {
await ctx.dispose()
}
}
export async function skipMissingPreseededAgentFlatFileFixtureConfiguration(
world: DifyWorld,
agentName: string,
): Promise<'skipped' | PreseededResource> {
const agent = await skipMissingPreseededAgent(world, agentName)
if (agent === 'skipped')
return agent
const ctx = await createApiContext()
try {
const response = await ctx.get(`/console/api/agent/${agent.id}/composer`)
await expectApiResponseOK(response, `Check preseeded Agent flat file fixture ${agentName}`)
const body = (await response.json()) as AgentComposerResponse
const configFiles = Array.isArray(body.agent_soul?.config_files)
? body.agent_soul.config_files
: []
const fileNames = configFiles
.map(file => (typeof file === 'object' && file !== null && 'name' in file ? file.name : undefined))
.filter((name): name is string => typeof name === 'string')
const missingFiles = agentBuilderFileTreeFixtureFileNames.filter(fileName => !fileNames.includes(fileName))
if (missingFiles.length > 0) {
return skipBlockedPrecondition(
world,
`Preseeded Agent "${agentName}" is missing current flat Files fixture configuration: ${missingFiles.join(', ')}. Hierarchical Files display remains blocked until Agent config files support tree paths.`,
)
}
return {
id: agent.id,
kind: 'agent',
name: agent.name,
}
}
finally {
await ctx.dispose()
}
}

128
e2e/support/preflight/common.ts Normal file
View File

@ -0,0 +1,128 @@
import type { DifyWorld } from '../../features/support/world'
import { agentBuilderPreseededResources } from '../agent-builder-resources'
import { createApiContext, expectApiResponseOK } from '../api'
export type PreseededResource = NonNullable<
DifyWorld['agentBuilder']['preflight']['preseededResources'][string]
>
export type E2EResourcePrecondition
= | {
ok: true
value: string
}
| {
ok: false
reason: string
}
export type NamedResource = {
id: string
name: string
}
export type NamedResourceListResponse<T extends NamedResource = NamedResource> = {
data: T[]
}
export type LocalizedLabel = {
en_US?: string
zh_Hans?: string
}
export type AgentComposerResponse = {
agent_soul?: Record<string, unknown>
}
export const readRequiredEnvResource = (
envName: string,
description: string,
): E2EResourcePrecondition => {
const value = process.env[envName]?.trim()
if (value)
return { ok: true, value }
return {
ok: false,
reason: `${description} requires ${envName}.`,
}
}
export function skipBlockedPrecondition(world: DifyWorld, reason: string): 'skipped' {
const message = `Blocked precondition: ${reason}`
console.warn(`[e2e] ${message}`)
world.attach(message, 'text/plain')
return 'skipped'
}
export function skipMissingEnvResource(
world: DifyWorld,
envName: string,
description: string,
): 'skipped' | string {
const resource = readRequiredEnvResource(envName, description)
if (resource.ok)
return resource.value
return skipBlockedPrecondition(world, resource.reason)
}
export const requiredAgentBuilderPreseededResources = Object.values(agentBuilderPreseededResources)
export function skipMissingAgentBuilderPreseed(
world: DifyWorld,
resourceName: string,
envName: string,
): 'skipped' | string {
return skipMissingEnvResource(
world,
envName,
`Preseeded Agent Builder resource "${resourceName}"`,
)
}
export const findConsoleResourceByName = async <T extends NamedResource = NamedResource>({
action,
path,
resourceName,
}: {
action: string
path: string
resourceName: string
}) => {
const ctx = await createApiContext()
try {
const response = await ctx.get(path)
await expectApiResponseOK(response, action)
const body = (await response.json()) as NamedResourceListResponse<T>
return body.data.find(item => item.name === resourceName)
}
finally {
await ctx.dispose()
}
}
export const buildQuery = (params: Record<string, string>) => new URLSearchParams(params).toString()
export const matchesNameOrLabel = (value: string, name: string, label?: LocalizedLabel) =>
value === name || value === label?.en_US || value === label?.zh_Hans
export const isRecord = (value: unknown): value is Record<string, unknown> =>
typeof value === 'object' && value !== null && !Array.isArray(value)
export const asRecord = (value: unknown): Record<string, unknown> => (isRecord(value) ? value : {})
export const asArray = (value: unknown): unknown[] => (Array.isArray(value) ? value : [])
export const asString = (value: unknown) => (typeof value === 'string' ? value : '')
export const hasNamedOrKeyedEntry = (items: unknown[], expectedName: string) =>
items.some((item) => {
const record = asRecord(item)
const values = [record.name, record.drive_key, record.reference, record.file_id, record.id].map(
asString,
)
return values.some(value => value === expectedName || value.endsWith(`/${expectedName}`))
})

150
e2e/support/preflight/datasets.ts Normal file
View File

@ -0,0 +1,150 @@
import type { DifyWorld } from '../../features/support/world'
import type { NamedResource, PreseededResource } from './common'
import { createApiContext, expectApiResponseOK } from '../api'
import {
buildQuery,
findConsoleResourceByName,
skipBlockedPrecondition,
} from './common'
type DatasetResource = NamedResource & {
document_count: number
total_available_documents: number
}
type DocumentIndexingStatus
= | 'cleaning'
| 'completed'
| 'indexing'
| 'parsing'
| 'splitting'
| 'waiting'
type DatasetIndexingStatusResponse = {
data: Array<{
id: string
indexing_status?: string
}>
}
const completedDocumentIndexingStatus: DocumentIndexingStatus = 'completed'
const activeDocumentIndexingStatuses = new Set<string>([
'cleaning',
'indexing',
'parsing',
'splitting',
'waiting',
])
export const getPreseededDataset = async (resourceName: string) => {
const query = buildQuery({ keyword: resourceName, limit: '20', page: '1' })
return findConsoleResourceByName<DatasetResource>({
action: `Check preseeded dataset ${resourceName}`,
path: `/console/api/datasets?${query}`,
resourceName,
})
}
const getDatasetIndexingStatuses = async (datasetId: string, resourceName: string) => {
const ctx = await createApiContext()
try {
const response = await ctx.get(`/console/api/datasets/${datasetId}/indexing-status`)
await expectApiResponseOK(response, `Check preseeded dataset indexing status ${resourceName}`)
const body = (await response.json()) as DatasetIndexingStatusResponse
return body.data
}
finally {
await ctx.dispose()
}
}
export const toDatasetResource = (
resource: NamedResource,
): PreseededResource => ({
id: resource.id,
kind: 'dataset',
name: resource.name,
})
export async function skipMissingPreseededDataset(
world: DifyWorld,
resourceName: string,
): Promise<'skipped' | PreseededResource> {
const resource = await getPreseededDataset(resourceName)
if (!resource)
return skipBlockedPrecondition(world, `Preseeded dataset "${resourceName}" was not found.`)
return toDatasetResource(resource)
}
export async function skipMissingReadyPreseededDataset(
world: DifyWorld,
resourceName: string,
): Promise<'skipped' | PreseededResource> {
const resource = await getPreseededDataset(resourceName)
if (!resource)
return skipBlockedPrecondition(world, `Preseeded dataset "${resourceName}" was not found.`)
if (resource.document_count < 1) {
return skipBlockedPrecondition(world, `Preseeded dataset "${resourceName}" has no documents.`)
}
if (resource.total_available_documents !== resource.document_count) {
return skipBlockedPrecondition(
world,
`Preseeded dataset "${resourceName}" has ${resource.total_available_documents}/${resource.document_count} available documents.`,
)
}
const statuses = await getDatasetIndexingStatuses(resource.id, resourceName)
if (statuses.length < 1) {
return skipBlockedPrecondition(
world,
`Preseeded dataset "${resourceName}" has no document indexing status.`,
)
}
const incompleteStatus = statuses.find(
item => item.indexing_status !== completedDocumentIndexingStatus,
)
if (incompleteStatus) {
return skipBlockedPrecondition(
world,
`Preseeded dataset "${resourceName}" includes document ${incompleteStatus.id} with indexing status "${incompleteStatus.indexing_status ?? 'missing'}".`,
)
}
return toDatasetResource(resource)
}
export async function skipMissingIndexingPreseededDataset(
world: DifyWorld,
resourceName: string,
): Promise<'skipped' | PreseededResource> {
const resource = await getPreseededDataset(resourceName)
if (!resource)
return skipBlockedPrecondition(world, `Preseeded dataset "${resourceName}" was not found.`)
const statuses = await getDatasetIndexingStatuses(resource.id, resourceName)
const indexingStatus = statuses.find(item =>
activeDocumentIndexingStatuses.has(item.indexing_status ?? ''),
)
if (!indexingStatus) {
const actualStatuses
= statuses.map(item => item.indexing_status ?? 'missing').join(', ') || 'none'
return skipBlockedPrecondition(
world,
`Preseeded dataset "${resourceName}" is not indexing or queued; document statuses: ${actualStatuses}.`,
)
}
return toDatasetResource(resource)
}

6
e2e/support/preflight/index.ts Normal file
View File

@ -0,0 +1,6 @@
export * from './access'
export * from './agents'
export * from './common'
export * from './datasets'
export * from './models'
export * from './tools'

158
e2e/support/preflight/models.ts Normal file
View File

@ -0,0 +1,158 @@
import type { DifyWorld } from '../../features/support/world'
import { agentBuilderPreseededResources } from '../agent-builder-resources'
import { createApiContext, expectApiResponseOK } from '../api'
import { skipBlockedPrecondition } from './common'
const stableChatModelProviderEnv = 'E2E_STABLE_MODEL_PROVIDER'
const stableChatModelNameEnv = 'E2E_STABLE_MODEL_NAME'
const stableChatModelTypeEnv = 'E2E_STABLE_MODEL_TYPE'
const brokenChatModelProviderEnv = 'E2E_BROKEN_MODEL_PROVIDER'
const brokenChatModelNameEnv = 'E2E_BROKEN_MODEL_NAME'
const brokenChatModelTypeEnv = 'E2E_BROKEN_MODEL_TYPE'
const activeModelStatus = 'active'
const defaultStableChatModelType = 'llm'
const defaultBrokenChatModelName = agentBuilderPreseededResources.brokenModel
type ModelTypeListResponse = {
data: Array<{
provider: string
models: Array<{
label?: {
en_US?: string
zh_Hans?: string
}
model: string
status?: string
}>
status?: string
}>
}
type ModelPreflightConfig
= | {
ok: true
provider: string
resourceName: string
type: string
value: string
}
| {
ok: false
reason: string
}
export function readAgentBuilderStableChatModelConfig(): ModelPreflightConfig {
const provider = process.env[stableChatModelProviderEnv]?.trim()
const name = process.env[stableChatModelNameEnv]?.trim()
const type = process.env[stableChatModelTypeEnv]?.trim() || defaultStableChatModelType
const missing: string[] = []
if (!provider)
missing.push(stableChatModelProviderEnv)
if (!name)
missing.push(stableChatModelNameEnv)
if (!provider || !name) {
return {
ok: false,
reason: `${agentBuilderPreseededResources.stableChatModel} requires ${missing.join(', ')}.`,
}
}
return {
ok: true,
provider,
resourceName: agentBuilderPreseededResources.stableChatModel,
type,
value: name,
}
}
export function readAgentBuilderBrokenChatModelConfig(): ModelPreflightConfig {
const provider = process.env[brokenChatModelProviderEnv]?.trim()
const name = process.env[brokenChatModelNameEnv]?.trim() || defaultBrokenChatModelName
const type = process.env[brokenChatModelTypeEnv]?.trim() || defaultStableChatModelType
if (!provider) {
return {
ok: false,
reason: `${agentBuilderPreseededResources.brokenModelProvider} requires ${brokenChatModelProviderEnv}.`,
}
}
return {
ok: true,
provider,
resourceName: agentBuilderPreseededResources.brokenModelProvider,
type,
value: name,
}
}
async function skipMissingAgentBuilderModel(
world: DifyWorld,
config: ModelPreflightConfig,
{
requireActive,
}: {
requireActive: boolean
},
): Promise<'skipped' | NonNullable<DifyWorld['agentBuilder']['preflight']['stableModel']>> {
if (!config.ok)
return skipBlockedPrecondition(world, config.reason)
const ctx = await createApiContext()
try {
const response = await ctx.get(
`/console/api/workspaces/current/models/model-types/${config.type}`,
)
await expectApiResponseOK(response, `Check ${config.resourceName}`)
const body = (await response.json()) as ModelTypeListResponse
const provider = body.data.find(item => item.provider === config.provider)
const model = provider?.models.find(
item =>
item.model === config.value
|| item.label?.en_US === config.value
|| item.label?.zh_Hans === config.value,
)
if (!provider || !model) {
return skipBlockedPrecondition(
world,
`${config.resourceName} was not found as ${config.provider}/${config.value} (${config.type}).`,
)
}
if (requireActive && model.status !== activeModelStatus) {
return skipBlockedPrecondition(
world,
`${config.resourceName} is ${model.status ?? 'missing status'} instead of ${activeModelStatus}.`,
)
}
return {
name: model.model,
provider: provider.provider,
type: config.type,
}
}
finally {
await ctx.dispose()
}
}
export async function skipMissingAgentBuilderStableChatModel(
world: DifyWorld,
): Promise<'skipped' | NonNullable<DifyWorld['agentBuilder']['preflight']['stableModel']>> {
return skipMissingAgentBuilderModel(world, readAgentBuilderStableChatModelConfig(), {
requireActive: true,
})
}
export async function skipMissingAgentBuilderBrokenChatModel(
world: DifyWorld,
): Promise<'skipped' | NonNullable<DifyWorld['agentBuilder']['preflight']['stableModel']>> {
return skipMissingAgentBuilderModel(world, readAgentBuilderBrokenChatModelConfig(), {
requireActive: false,
})
}

114
e2e/support/preflight/tools.ts Normal file
View File

@ -0,0 +1,114 @@
import type { DifyWorld } from '../../features/support/world'
import type { LocalizedLabel, PreseededResource } from './common'
import { createApiContext, expectApiResponseOK } from '../api'
import {
asRecord,
asString,
matchesNameOrLabel,
skipBlockedPrecondition,
} from './common'
type BuiltinToolProvider = {
label?: LocalizedLabel
name: string
tools: Array<{
label?: LocalizedLabel
name: string
}>
}
export const splitToolDisplayName = (resourceName: string) => {
const [providerName, toolName] = resourceName.split('/').map(item => item.trim())
if (!providerName || !toolName) {
return {
ok: false as const,
reason: `Preseeded tool "${resourceName}" must use "Provider / Tool" format.`,
}
}
return {
ok: true as const,
providerName,
toolName,
}
}
export const findToolEntry = (
items: unknown[],
{
providerDisplayName,
providerName,
toolDisplayName,
toolName,
}: {
providerDisplayName: string
providerName: string
toolDisplayName: string
toolName: string
},
) =>
items.find((item) => {
const record = asRecord(item)
const providerValues = [record.provider_id, record.provider, record.plugin_id, record.name].map(
asString,
)
const toolValues = [record.tool_name, record.name].map(asString)
return (
providerValues.some(value => value === providerName || value === providerDisplayName)
&& toolValues.some(value => value === toolName || value === toolDisplayName)
)
})
export const hasToolEntry = (
items: unknown[],
tool: {
providerDisplayName: string
providerName: string
toolDisplayName: string
toolName: string
},
) => Boolean(findToolEntry(items, tool))
export const hasUnauthorizedToolCredentialState = (item: unknown) => {
const record = asRecord(item)
return asString(record.credential_type) === 'unauthorized'
}
export async function skipMissingPreseededTool(
world: DifyWorld,
resourceName: string,
): Promise<'skipped' | PreseededResource> {
const parsed = splitToolDisplayName(resourceName)
if (!parsed.ok)
return skipBlockedPrecondition(world, parsed.reason)
const ctx = await createApiContext()
try {
const response = await ctx.get('/console/api/workspaces/current/tools/builtin')
await expectApiResponseOK(response, `Check preseeded tool ${resourceName}`)
const providers = (await response.json()) as BuiltinToolProvider[]
const provider = providers.find(item =>
matchesNameOrLabel(parsed.providerName, item.name, item.label),
)
const tool = provider?.tools.find(item =>
matchesNameOrLabel(parsed.toolName, item.name, item.label),
)
if (!provider || !tool)
return skipBlockedPrecondition(world, `Preseeded tool "${resourceName}" was not found.`)
return {
id: `${provider.name}/${tool.name}`,
kind: 'tool',
name: resourceName,
}
}
finally {
await ctx.dispose()
}
}