diff --git a/e2e/AGENTS.md b/e2e/AGENTS.md index 90f741d4f34..1159f47abb9 100644 --- a/e2e/AGENTS.md +++ b/e2e/AGENTS.md @@ -296,7 +296,9 @@ Use `support/naming.ts` for generated test resource names. New app, Agent, datas Use `fixtures/test-materials/` for checked-in files that scenarios upload, preview, index, or retrieve. Keep these fixtures small and deterministic, and use `support/test-materials.ts` to resolve their absolute paths. -Use `support/preflight.ts` for scenarios that require optional external resources such as a stable model provider, plugin/tool credential, or knowledge base seed. Prefer an explicit `Given` step that returns a skipped result with a clear blocked-precondition reason over hidden setup in hooks. +Use `support/preflight.ts` for scenarios that require optional external resources such as a stable model provider, plugin/tool credential, or knowledge base seed. Prefer an explicit `Given` step that returns a skipped result with a clear blocked-precondition reason over hidden setup in hooks. Keep `support/preflight.ts` as the public barrel and put resource checks in the matching module under `support/preflight/`: `models.ts`, `agents.ts`, `datasets.ts`, `tools.ts`, or `access.ts`. + +Keep Agent Builder step definitions grouped by user capability, not by DOM component or Cucumber keyword. `configure.steps.ts` owns common configure navigation, refresh, and draft assertions; `build-draft.steps.ts` owns Build mode checkout, apply, discard, and Build draft isolation; `files.steps.ts` owns Files upload and file-list assertions; `advanced-settings.steps.ts` owns Env and advanced settings behavior; `agent-edit.steps.ts` owns saved Agent detail display assertions; `publish.steps.ts` owns publish state; `access-point.steps.ts` owns Access Point behavior; `preflight.steps.ts` should remain the explicit `Given` entrypoint for preflight resources. Use `the Agent Builder stable chat model is available` before scenarios that must run an Agent with a real model. `E2E_STABLE_MODEL_PROVIDER`, `E2E_STABLE_MODEL_NAME`, and optional `E2E_STABLE_MODEL_TYPE` define the single usable model for the E2E environment. The step defaults the type to `llm`, verifies the model is present and `active` through `/console/api/workspaces/current/models/model-types/{type}`, then stores it on `DifyWorld.agentBuilder.preflight.stableModel`. Any Agent Builder scenario that needs a usable model should explicitly apply this stored model during its own setup instead of hard-coding provider or model names in feature files or hooks. diff --git a/e2e/features/step-definitions/agent-v2/advanced-settings.steps.ts b/e2e/features/step-definitions/agent-v2/advanced-settings.steps.ts new file mode 100644 index 00000000000..80041582424 --- /dev/null +++ b/e2e/features/step-definitions/agent-v2/advanced-settings.steps.ts @@ -0,0 +1,383 @@ +import type { DifyWorld } from '../../support/world' +import { Then, When } from '@cucumber/cucumber' +import { expect } from '@playwright/test' +import { getAgentComposerDraft } from '../../../support/agent' +import { agentBuilderFixedInputs } from '../../../support/agent-builder-resources' +import { skipBlockedPrecondition } from '../../../support/preflight' +import { getAgentBuilderTestMaterialPath } from '../../../support/test-materials' +import { + expectAgentEnvVariableHidden, + expectAgentEnvVariableVisible, + getAgentEnvVariables, + getAgentEnvVariableValue, + getCurrentAgentId, + getEnvVariableKey, + openAgentAdvancedSettings, +} from './configure-helpers' + +When( + 'I add the plain Agent v2 environment variable from Advanced Settings', + async function (this: DifyWorld) { + const page = this.getPage() + const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' }) + + await page.getByRole('button', { name: 'Advanced Settings' }).first().click() + await expect(advancedSettings.getByRole('heading', { name: 'Env Editor' })).toBeVisible() + + await advancedSettings + .getByRole('textbox', { name: 'Key' }) + .fill(agentBuilderFixedInputs.envPlainKey) + await advancedSettings + .getByRole('textbox', { name: 'Value' }) + .fill(agentBuilderFixedInputs.envPlainValue) + await expect(advancedSettings.getByText('Plain', { exact: true })).toBeVisible() + }, +) + +When( + 'I import the invalid Agent v2 environment file from Advanced Settings', + async function (this: DifyWorld) { + const page = this.getPage() + const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' }) + + const fileChooserPromise = page.waitForEvent('filechooser') + await advancedSettings.getByRole('button', { name: 'Import .env' }).click() + await (await fileChooserPromise).setFiles(getAgentBuilderTestMaterialPath('invalidEnv')) + }, +) + +When( + 'I add the secondary plain Agent v2 environment variable from Advanced Settings', + async function (this: DifyWorld) { + const page = this.getPage() + const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' }) + + await advancedSettings.getByRole('button', { name: 'Add environment variable' }).click() + await advancedSettings + .getByRole('textbox', { name: 'Key' }) + .last() + .fill(agentBuilderFixedInputs.envModeKey) + await advancedSettings + .getByRole('textbox', { name: 'Value' }) + .last() + .fill(agentBuilderFixedInputs.envModeValue) + await expect(advancedSettings.getByText('Plain', { exact: true })).toHaveCount(2) + }, +) + +When( + 'I delete the plain Agent v2 environment variable from Advanced Settings', + async function (this: DifyWorld) { + const page = this.getPage() + const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' }) + + await advancedSettings + .getByRole('button', { name: `Delete ${agentBuilderFixedInputs.envPlainKey}` }) + .click() + }, +) + +When( + 'I import the valid Agent v2 environment file from Advanced Settings', + async function (this: DifyWorld) { + const page = this.getPage() + const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' }) + + await page.getByRole('button', { name: 'Advanced Settings' }).first().click() + await expect(advancedSettings.getByRole('heading', { name: 'Env Editor' })).toBeVisible() + + const fileChooserPromise = page.waitForEvent('filechooser') + await advancedSettings.getByRole('button', { name: 'Import .env' }).click() + await (await fileChooserPromise).setFiles(getAgentBuilderTestMaterialPath('validEnv')) + }, +) + +When('I expand Agent v2 Advanced Settings', async function (this: DifyWorld) { + const page = this.getPage() + const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' }) + + await page.getByRole('button', { name: 'Advanced Settings' }).first().click() + await expect(advancedSettings.getByRole('heading', { name: 'Env Editor' })).toBeVisible() +}) + +Then( + 'Agent v2 Advanced Settings should describe supported entries while collapsed', + async function (this: DifyWorld) { + const page = this.getPage() + const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' }) + + await expect(advancedSettings).toBeVisible() + await expect( + advancedSettings.getByText('For power users. Env vars, sandbox & memory.'), + ).toBeVisible() + await expect(advancedSettings.getByRole('heading', { name: 'Env Editor' })) + .not + .toBeVisible() + }, +) + +Then( + 'the plain Agent v2 environment variable should be saved in the Agent v2 draft', + async function (this: DifyWorld) { + const agentId = getCurrentAgentId(this) + + await expect + .poll(async () => { + const env = (await getAgentComposerDraft(agentId)).agent_soul?.env + const variable = env?.variables?.find(item => + getEnvVariableKey(item) === agentBuilderFixedInputs.envPlainKey, + ) + + return { + secretCount: env?.secret_refs?.length ?? 0, + value: variable?.value, + } + }, { + timeout: 30_000, + }) + .toEqual({ + secretCount: 0, + value: agentBuilderFixedInputs.envPlainValue, + }) + }, +) + +Then( + 'the Agent v2 environment variables for deletion should be saved in the Agent v2 draft', + async function (this: DifyWorld) { + const agentId = getCurrentAgentId(this) + + await expect + .poll(async () => { + const variables = await getAgentEnvVariables(agentId) + + return { + modeValue: getAgentEnvVariableValue(variables, agentBuilderFixedInputs.envModeKey), + plainValue: getAgentEnvVariableValue(variables, agentBuilderFixedInputs.envPlainKey), + } + }, { + timeout: 30_000, + }) + .toEqual({ + modeValue: agentBuilderFixedInputs.envModeValue, + plainValue: agentBuilderFixedInputs.envPlainValue, + }) + }, +) + +Then( + 'the plain Agent v2 environment variable should be removed from the Agent v2 draft', + async function (this: DifyWorld) { + const agentId = getCurrentAgentId(this) + + await expect + .poll(async () => { + const variables = await getAgentEnvVariables(agentId) + + return { + modeValue: getAgentEnvVariableValue(variables, agentBuilderFixedInputs.envModeKey), + plainValue: getAgentEnvVariableValue(variables, agentBuilderFixedInputs.envPlainKey), + } + }, { + timeout: 30_000, + }) + .toEqual({ + modeValue: agentBuilderFixedInputs.envModeValue, + plainValue: undefined, + }) + }, +) + +Then( + 'the valid Agent v2 environment import should be saved in the Agent v2 draft', + async function (this: DifyWorld) { + const agentId = getCurrentAgentId(this) + + await expect + .poll(async () => { + const variables = await getAgentEnvVariables(agentId) + + return { + modeValue: getAgentEnvVariableValue(variables, agentBuilderFixedInputs.envModeKey), + plainValue: getAgentEnvVariableValue(variables, agentBuilderFixedInputs.envPlainKey), + } + }, { + timeout: 30_000, + }) + .toEqual({ + modeValue: agentBuilderFixedInputs.envModeValue, + plainValue: agentBuilderFixedInputs.envPlainValue, + }) + }, +) + +Then( + 'the invalid Agent v2 environment import should report skipped lines', + async function (this: DifyWorld) { + await expect(this.getPage().getByText('2 invalid .env lines were skipped.')).toBeVisible() + }, +) + +Then( + 'the Agent v2 environment variables from the invalid import should be saved in the Agent v2 draft', + async function (this: DifyWorld) { + const agentId = getCurrentAgentId(this) + + await expect + .poll(async () => { + const variables = await getAgentEnvVariables(agentId) + + return { + importedValue: getAgentEnvVariableValue( + variables, + agentBuilderFixedInputs.envAfterInvalidImportKey, + ), + plainValue: getAgentEnvVariableValue(variables, agentBuilderFixedInputs.envPlainKey), + } + }, { + timeout: 30_000, + }) + .toEqual({ + importedValue: agentBuilderFixedInputs.envAfterInvalidImportValue, + plainValue: agentBuilderFixedInputs.envPlainValue, + }) + }, +) + +Then( + 'I should see the supported Agent v2 Advanced Settings entries', + async function (this: DifyWorld) { + const page = this.getPage() + const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' }) + const envEditor = advancedSettings.getByRole('region', { name: 'Env Editor' }) + + await expect(envEditor).toBeVisible() + await expect(envEditor.getByRole('button', { name: 'Import .env' })).toBeVisible() + await expect(envEditor.getByRole('button', { name: 'Add environment variable' })) + .toBeVisible() + await expect(envEditor.getByText('Key', { exact: true })).toBeVisible() + await expect(envEditor.getByText('Value', { exact: true })).toBeVisible() + await expect(envEditor.getByText('Scope', { exact: true })).toBeVisible() + }, +) + +Then('Agent v2 Content Moderation Settings should be available', async function (this: DifyWorld) { + const advancedSettings = this.getPage().getByRole('region', { name: 'Advanced Settings' }) + const contentModeration = advancedSettings.getByRole('region', { name: 'Content moderation' }) + + try { + await expect(contentModeration).toBeVisible({ timeout: 3_000 }) + } + catch { + return skipBlockedPrecondition( + this, + 'Feature not enabled: Agent v2 Content Moderation Settings is not available in this build.', + ) + } +}) + +Then( + 'I should see the Agent v2 environment variables from the valid import in Advanced Settings', + async function (this: DifyWorld) { + const page = this.getPage() + const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' }) + + await page.getByRole('button', { name: 'Advanced Settings' }).first().click() + await expect(advancedSettings.getByRole('heading', { name: 'Env Editor' })).toBeVisible() + await expect.poll( + async () => advancedSettings.getByRole('textbox').evaluateAll(inputs => + inputs.map(input => (input as HTMLInputElement).value), + ), + { timeout: 30_000 }, + ).toEqual(expect.arrayContaining([ + agentBuilderFixedInputs.envPlainKey, + agentBuilderFixedInputs.envPlainValue, + agentBuilderFixedInputs.envModeKey, + agentBuilderFixedInputs.envModeValue, + ])) + await expect(advancedSettings.getByText('Plain', { exact: true })).toHaveCount(2) + }, +) + +Then( + 'I should not see the deleted Agent v2 environment variable in Advanced Settings', + async function (this: DifyWorld) { + const page = this.getPage() + const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' }) + + await page.getByRole('button', { name: 'Advanced Settings' }).first().click() + await expect(advancedSettings.getByRole('heading', { name: 'Env Editor' })).toBeVisible() + await expect.poll( + async () => advancedSettings.getByRole('textbox').evaluateAll(inputs => + inputs.map(input => (input as HTMLInputElement).value), + ), + { timeout: 30_000 }, + ).toEqual(expect.arrayContaining([ + agentBuilderFixedInputs.envModeKey, + agentBuilderFixedInputs.envModeValue, + ])) + await expect.poll( + async () => advancedSettings.getByRole('textbox').evaluateAll(inputs => + inputs.map(input => (input as HTMLInputElement).value), + ), + { timeout: 30_000 }, + ).not.toContain(agentBuilderFixedInputs.envPlainKey) + await expect(advancedSettings.getByText('Plain', { exact: true })).toHaveCount(1) + }, +) + +Then( + 'I should see the plain Agent v2 environment variable in Advanced Settings', + async function (this: DifyWorld) { + const page = this.getPage() + const advancedSettings = await openAgentAdvancedSettings(page) + + await expect(advancedSettings.getByRole('textbox', { name: 'Key' })) + .toHaveValue(agentBuilderFixedInputs.envPlainKey) + await expect(advancedSettings.getByRole('textbox', { name: 'Value' })) + .toHaveValue(agentBuilderFixedInputs.envPlainValue) + await expect(advancedSettings.getByText('Plain', { exact: true })).toBeVisible() + await expect(page.getByRole('button', { name: /^Build$/i })).toBeVisible() + }, +) + +Then( + 'I should see the supported E2E environment variable in Advanced Settings', + async function (this: DifyWorld) { + await expectAgentEnvVariableVisible( + this, + agentBuilderFixedInputs.envPlainKey, + agentBuilderFixedInputs.envPlainValue, + ) + }, +) + +Then( + 'I should not see the supported E2E environment variable in Advanced Settings', + async function (this: DifyWorld) { + await expectAgentEnvVariableHidden(this, agentBuilderFixedInputs.envPlainKey) + }, +) + +Then( + 'I should see the Agent v2 environment variables from the invalid import in Advanced Settings', + async function (this: DifyWorld) { + const page = this.getPage() + const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' }) + + await page.getByRole('button', { name: 'Advanced Settings' }).first().click() + await expect(advancedSettings.getByRole('heading', { name: 'Env Editor' })).toBeVisible() + await expect.poll( + async () => advancedSettings.getByRole('textbox').evaluateAll(inputs => + inputs.map(input => (input as HTMLInputElement).value), + ), + { timeout: 30_000 }, + ).toEqual(expect.arrayContaining([ + agentBuilderFixedInputs.envPlainKey, + agentBuilderFixedInputs.envPlainValue, + agentBuilderFixedInputs.envAfterInvalidImportKey, + agentBuilderFixedInputs.envAfterInvalidImportValue, + ])) + await expect(page.getByRole('button', { name: /^Build$/i })).toBeVisible() + }, +) diff --git a/e2e/features/step-definitions/agent-v2/agent-edit.steps.ts b/e2e/features/step-definitions/agent-v2/agent-edit.steps.ts new file mode 100644 index 00000000000..ff65e1fb283 --- /dev/null +++ b/e2e/features/step-definitions/agent-v2/agent-edit.steps.ts @@ -0,0 +1,117 @@ +import type { DifyWorld } from '../../support/world' +import { Then } from '@cucumber/cucumber' +import { expect } from '@playwright/test' +import { agentBuilderExpectedTokens, agentBuilderFixedInputs, agentBuilderPreseededResources } from '../../../support/agent-builder-resources' +import { agentBuilderTestMaterials } from '../../../support/test-materials' +import { expectProviderToolActionVisible, openAgentKnowledgeRetrievalDialog } from './configure-helpers' + +Then('I should see the Agent v2 full-config fixture sections', async function (this: DifyWorld) { + const page = this.getPage() + const stableModel = this.agentBuilder.preflight.stableModel + if (!stableModel) + throw new Error('Stable chat model preflight must run before asserting the full-config Agent.') + + await expect(page.getByRole('heading', { name: 'Configure' })).toBeVisible({ timeout: 30_000 }) + await expect(page.getByText(agentBuilderPreseededResources.fullConfigAgent, { exact: true })) + .toBeVisible() + await expect(page.getByText(stableModel.name, { exact: true })).toBeVisible() + + const promptSection = page.getByRole('region', { name: 'Prompt' }) + await expect(promptSection).toBeVisible() + await expect(promptSection).toContainText(agentBuilderExpectedTokens.agentReply) + + const skillsSection = page.getByRole('region', { name: 'Skills' }) + await expect(skillsSection).toBeVisible() + await expect(skillsSection.getByRole('button', { + exact: true, + name: agentBuilderPreseededResources.summarySkill, + })).toBeVisible() + + const filesSection = page.getByRole('region', { name: 'Files' }) + await expect(filesSection).toBeVisible() + await expect(filesSection.getByRole('button', { + exact: true, + name: agentBuilderTestMaterials.smallFile, + })).toBeVisible() + await expect(filesSection.getByRole('button', { + exact: true, + name: agentBuilderTestMaterials.specialFilename, + })).toBeVisible() + + const toolsSection = page.getByRole('region', { name: 'Tools' }) + await expect(toolsSection).toBeVisible() + await expectProviderToolActionVisible( + toolsSection, + agentBuilderPreseededResources.jsonReplaceTool, + ) + + const knowledgeSection = page.getByRole('region', { name: 'Knowledge Retrieval' }) + await expect(knowledgeSection).toBeVisible() + await expect(knowledgeSection.getByText('Retrieval 1', { exact: true })).toBeVisible() + + const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' }) + await expect(advancedSettings).toBeVisible() + await expect( + advancedSettings.getByText('For power users. Env vars, sandbox & memory.'), + ).toBeVisible() +}) + +Then('I should see the Agent v2 tool state fixture tools', async function (this: DifyWorld) { + const page = this.getPage() + const toolsSection = page.getByRole('region', { name: 'Tools' }) + + await expect(toolsSection).toBeVisible({ timeout: 30_000 }) + await expect(toolsSection.getByRole('button', { exact: true, name: 'Not authorized' })).toBeVisible() + + const { action: jsonReplaceAction, tool: jsonTool } = await expectProviderToolActionVisible( + toolsSection, + agentBuilderPreseededResources.jsonReplaceTool, + ) + await jsonReplaceAction.hover() + await expect(toolsSection.getByRole('button', { + exact: true, + name: `Edit ${jsonTool.actionName}`, + })).toBeVisible() + await expect(toolsSection.getByRole('button', { + exact: true, + name: `Remove ${jsonTool.actionName}`, + })).toBeVisible() + + await expectProviderToolActionVisible( + toolsSection, + agentBuilderPreseededResources.tavilySearchTool, + ) +}) + +Then('I should see the Agent v2 dual retrieval fixture settings', async function (this: DifyWorld) { + const page = this.getPage() + const knowledgeSection = page.getByRole('region', { name: 'Knowledge Retrieval' }) + + await expect(knowledgeSection).toBeVisible({ timeout: 30_000 }) + await expect(knowledgeSection.getByText('Retrieval 1', { exact: true })).toBeVisible() + await expect(knowledgeSection.getByText('Retrieval 2', { exact: true })).toBeVisible() + + const agentDecideDialog = await openAgentKnowledgeRetrievalDialog(knowledgeSection, 'Retrieval 1') + await expect(agentDecideDialog.getByText(agentBuilderPreseededResources.agentKnowledgeBase, { + exact: true, + })).toBeVisible() + await expect(agentDecideDialog.getByRole('radio', { + exact: true, + name: 'Agent decide', + })).toBeChecked() + await agentDecideDialog.getByRole('button', { name: 'Close' }).click() + await expect(agentDecideDialog).not.toBeVisible() + + const customQueryDialog = await openAgentKnowledgeRetrievalDialog(knowledgeSection, 'Retrieval 2') + await expect(customQueryDialog.getByText(agentBuilderPreseededResources.agentKnowledgeBase, { + exact: true, + })).toBeVisible() + await expect(customQueryDialog.getByRole('radio', { + exact: true, + name: 'Custom query', + })).toBeChecked() + await expect(customQueryDialog.getByRole('textbox', { + exact: true, + name: 'Custom query text', + })).toHaveValue(agentBuilderFixedInputs.customKnowledgeQuery) +}) diff --git a/e2e/features/step-definitions/agent-v2/build-draft.steps.ts b/e2e/features/step-definitions/agent-v2/build-draft.steps.ts new file mode 100644 index 00000000000..728f9916b20 --- /dev/null +++ b/e2e/features/step-definitions/agent-v2/build-draft.steps.ts @@ -0,0 +1,291 @@ +import type { DifyWorld } from '../../support/world' +import { readFile } from 'node:fs/promises' +import { Given, Then, When } from '@cucumber/cucumber' +import { expect } from '@playwright/test' +import { + createAgentSoulConfigWithModel, + getAgentComposerDraft, + normalAgentPrompt, + normalAgentSoulConfig, + saveAgentBuildDraft, + saveAgentComposerDraft, + updatedAgentPrompt, + updatedAgentSoulConfig, + uploadAgentConfigFileToDraft, +} from '../../../support/agent' +import { agentBuilderFixedInputs, agentBuilderPreseededResources } from '../../../support/agent-builder-resources' +import { skipBlockedPrecondition } from '../../../support/preflight' +import { agentBuilderTestMaterials, getAgentBuilderTestMaterialPath } from '../../../support/test-materials' +import { + getAgentEnvVariableValue, + getCurrentAgentId, + uploadSummaryConfigSkillForBuildDraft, +} from './configure-helpers' + +Given( + 'an Agent v2 Build draft adds the supported E2E files, skills, and env', + async function (this: DifyWorld) { + const agentId = getCurrentAgentId(this) + const configFile = await uploadAgentConfigFileToDraft({ + agentId, + fileName: agentBuilderTestMaterials.smallFile, + filePath: getAgentBuilderTestMaterialPath('smallFile'), + }) + const skill = await uploadSummaryConfigSkillForBuildDraft(this) + + if (!configFile.file_id) + throw new Error('Agent v2 build draft config file fixture did not return a file_id.') + this.createdAgentConfigFiles.push({ agentId, name: configFile.name }) + + const normalConfig = this.agentBuilder.preflight.stableModel + ? createAgentSoulConfigWithModel(normalAgentSoulConfig, this.agentBuilder.preflight.stableModel) + : normalAgentSoulConfig + const updatedConfig = this.agentBuilder.preflight.stableModel + ? createAgentSoulConfigWithModel(updatedAgentSoulConfig, this.agentBuilder.preflight.stableModel) + : updatedAgentSoulConfig + + await saveAgentComposerDraft(agentId, normalConfig) + await saveAgentBuildDraft(agentId, { + ...updatedConfig, + config_files: [configFile], + config_skills: [{ + ...skill, + file_kind: skill.file_kind ?? 'tool_file', + }], + env: { + secret_refs: [], + variables: [{ + id: agentBuilderFixedInputs.envPlainKey, + key: agentBuilderFixedInputs.envPlainKey, + name: agentBuilderFixedInputs.envPlainKey, + value: agentBuilderFixedInputs.envPlainValue, + variable: agentBuilderFixedInputs.envPlainKey, + }], + }, + }) + }, +) + +Given( + 'an Agent v2 Build draft includes the existing e2e-summary-skill Skill', + async function (this: DifyWorld) { + const skill = await uploadSummaryConfigSkillForBuildDraft(this) + const normalConfig = this.agentBuilder.preflight.stableModel + ? createAgentSoulConfigWithModel(normalAgentSoulConfig, this.agentBuilder.preflight.stableModel) + : normalAgentSoulConfig + const updatedConfig = this.agentBuilder.preflight.stableModel + ? createAgentSoulConfigWithModel(updatedAgentSoulConfig, this.agentBuilder.preflight.stableModel) + : updatedAgentSoulConfig + const configSkills = [{ + ...skill, + file_kind: skill.file_kind ?? 'tool_file', + }] + + await saveAgentComposerDraft(getCurrentAgentId(this), { + ...normalConfig, + config_skills: configSkills, + }) + await saveAgentBuildDraft(getCurrentAgentId(this), { + ...updatedConfig, + config_skills: configSkills, + }) + }, +) + +Given('an Agent v2 Build draft uses the updated E2E prompt', async function (this: DifyWorld) { + await saveAgentBuildDraft(getCurrentAgentId(this), updatedAgentSoulConfig) +}) + +Given( + 'an Agent v2 Build draft uses the updated E2E prompt with the stable E2E model', + async function (this: DifyWorld) { + if (!this.agentBuilder.preflight.stableModel) + throw new Error('Create an Agent v2 Build draft with a stable model after stable model preflight.') + + await saveAgentBuildDraft( + getCurrentAgentId(this), + createAgentSoulConfigWithModel(updatedAgentSoulConfig, this.agentBuilder.preflight.stableModel), + ) + }, +) + +When('I generate an Agent v2 Build draft from the fixed instruction', async function (this: DifyWorld) { + const page = this.getPage() + const agentId = getCurrentAgentId(this) + const instruction = (await readFile(getAgentBuilderTestMaterialPath('buildInstruction'), 'utf8')).trim() + + await page.getByRole('button', { name: 'Build' }).click() + await page.getByPlaceholder('Describe what your agent should do').fill(instruction) + + const checkoutResponsePromise = page.waitForResponse(response => ( + response.request().method() === 'POST' + && new URL(response.url()).pathname.endsWith(`/console/api/agent/${agentId}/build-draft/checkout`) + )) + const chatResponsePromise = page.waitForResponse(response => ( + response.request().method() === 'POST' + && new URL(response.url()).pathname.endsWith(`/console/api/agent/${agentId}/chat-messages`) + )) + + await page.getByRole('button', { name: 'Start build' }).click() + expect((await checkoutResponsePromise).ok()).toBe(true) + expect((await chatResponsePromise).ok()).toBe(true) + await expect(page.getByText('Build draft')).toBeVisible({ timeout: 120_000 }) + await expect(page.getByRole('button', { name: 'Apply' })).toBeEnabled({ timeout: 120_000 }) + await expect(page.getByRole('button', { name: 'Discard' })).toBeEnabled() +}) + +When('I discard the Agent v2 Build draft', async function (this: DifyWorld) { + await this.getPage().getByRole('button', { name: 'Discard' }).click() +}) + +When('I apply the Agent v2 Build draft', async function (this: DifyWorld) { + const page = this.getPage() + const agentId = getCurrentAgentId(this) + const applyButton = page.getByRole('button', { name: 'Apply' }) + + await expect(applyButton).toBeEnabled({ timeout: 30_000 }) + const finalizeResponsePromise = page.waitForResponse(response => ( + response.request().method() === 'POST' + && new URL(response.url()).pathname.endsWith(`/console/api/agent/${agentId}/build-chat/finalize`) + ), { timeout: 120_000 }) + const applyResponsePromise = page.waitForResponse(response => ( + response.request().method() === 'POST' + && new URL(response.url()).pathname.endsWith(`/console/api/agent/${agentId}/build-draft/apply`) + ), { timeout: 120_000 }) + + await applyButton.click() + expect((await finalizeResponsePromise).ok()).toBe(true) + expect((await applyResponsePromise).ok()).toBe(true) + await expect(page.getByText('Action succeeded')).toBeVisible() +}) + +Then('Agent v2 Build chat Dify Tool writeback should be available', async function (this: DifyWorld) { + const toolsSection = this.getPage().getByRole('region', { name: 'Tools' }) + + await expect(toolsSection).toBeVisible({ timeout: 30_000 }) + + return skipBlockedPrecondition( + this, + 'Build draft Dify Tool writeback is not available: Build draft currently supports files, skills, and env only.', + ) +}) + +Then('I should see the Agent v2 Build draft pending changes', async function (this: DifyWorld) { + const page = this.getPage() + + await expect(page.getByText('Build draft')).toBeVisible({ timeout: 30_000 }) + await expect(page.getByRole('button', { name: 'Apply' })).toBeEnabled() + await expect(page.getByRole('button', { name: 'Discard' })).toBeEnabled() +}) + +Then('I should see the Agent v2 Build mode confirmation state', async function (this: DifyWorld) { + const page = this.getPage() + + await expect(page.getByText('Build mode', { exact: true })).toBeVisible() + await expect( + page.getByText('You\'re in build mode. Shape this setup through the chat on the right, then Apply.'), + ).toBeVisible() +}) + +Then('I should see the e2e-summary-skill Skill in the Skills section', async function (this: DifyWorld) { + const skillsSection = this.getPage().getByRole('region', { name: 'Skills' }) + + await expect(skillsSection).toBeVisible({ timeout: 30_000 }) + await expect(skillsSection.getByRole('button', { + exact: true, + name: agentBuilderPreseededResources.summarySkill, + })).toBeVisible() +}) + +Then('I should not see the e2e-summary-skill Skill in the Skills section', async function (this: DifyWorld) { + const skillsSection = this.getPage().getByRole('region', { name: 'Skills' }) + + await expect(skillsSection).toBeVisible({ timeout: 30_000 }) + await expect(skillsSection.getByRole('button', { + exact: true, + name: agentBuilderPreseededResources.summarySkill, + })).toHaveCount(0) +}) + +Then('I should see one e2e-summary-skill Skill in the Skills section', async function (this: DifyWorld) { + const skillsSection = this.getPage().getByRole('region', { name: 'Skills' }) + + await expect(skillsSection).toBeVisible({ timeout: 30_000 }) + await expect(skillsSection.getByRole('button', { + exact: true, + name: agentBuilderPreseededResources.summarySkill, + })).toHaveCount(1) +}) + +Then( + 'the Agent v2 draft should include the supported Build draft config', + async function (this: DifyWorld) { + await expect.poll( + async () => { + const agentSoul = (await getAgentComposerDraft(getCurrentAgentId(this))).agent_soul + const variables = agentSoul?.env?.variables ?? [] + + return { + envValue: getAgentEnvVariableValue(variables, agentBuilderFixedInputs.envPlainKey), + fileNames: agentSoul?.config_files?.map(file => file.name) ?? [], + prompt: agentSoul?.prompt, + skillNames: agentSoul?.config_skills?.map(skill => skill.name) ?? [], + } + }, + { timeout: 30_000 }, + ).toEqual({ + envValue: agentBuilderFixedInputs.envPlainValue, + fileNames: expect.arrayContaining([agentBuilderTestMaterials.smallFile]), + prompt: { system_prompt: updatedAgentPrompt }, + skillNames: expect.arrayContaining([agentBuilderPreseededResources.summarySkill]), + }) + }, +) + +Then( + 'the Agent v2 draft should not include the supported Build draft config', + async function (this: DifyWorld) { + await expect.poll( + async () => { + const agentSoul = (await getAgentComposerDraft(getCurrentAgentId(this))).agent_soul + const variables = agentSoul?.env?.variables ?? [] + + return { + envValue: getAgentEnvVariableValue(variables, agentBuilderFixedInputs.envPlainKey), + fileNames: agentSoul?.config_files?.map(file => file.name) ?? [], + prompt: agentSoul?.prompt, + skillNames: agentSoul?.config_skills?.map(skill => skill.name) ?? [], + } + }, + { timeout: 30_000 }, + ).toEqual({ + envValue: undefined, + fileNames: expect.not.arrayContaining([agentBuilderTestMaterials.smallFile]), + prompt: { system_prompt: normalAgentPrompt }, + skillNames: expect.not.arrayContaining([agentBuilderPreseededResources.summarySkill]), + }) + }, +) + +Then( + 'the Agent v2 draft should include one e2e-summary-skill Skill', + async function (this: DifyWorld) { + await expect.poll( + async () => { + const agentSoul = (await getAgentComposerDraft(getCurrentAgentId(this))).agent_soul + return agentSoul?.config_skills?.filter( + skill => skill.name === agentBuilderPreseededResources.summarySkill, + ).length ?? 0 + }, + { timeout: 30_000 }, + ).toBe(1) + }, +) + +Then('the Agent v2 Build draft should no longer be active', async function (this: DifyWorld) { + const page = this.getPage() + + await expect(page.getByText('Build draft')).not.toBeVisible() + await expect(page.getByRole('button', { name: 'Apply' })).not.toBeVisible() + await expect(page.getByRole('button', { name: 'Discard' })).not.toBeVisible() +}) diff --git a/e2e/features/step-definitions/agent-v2/configure-helpers.ts b/e2e/features/step-definitions/agent-v2/configure-helpers.ts new file mode 100644 index 00000000000..19d5fcbc478 --- /dev/null +++ b/e2e/features/step-definitions/agent-v2/configure-helpers.ts @@ -0,0 +1,241 @@ +import type { Locator } from '@playwright/test' +import type { AgentComposerEnvVariable } from '../../../support/agent' +import type { DifyWorld } from '../../support/world' +import { expect } from '@playwright/test' +import { + getAgentComposerDraft, + normalAgentPrompt, + uploadAgentConfigSkillToDraft, +} from '../../../support/agent' +import { + agentBuilderTestMaterials, + getAgentBuilderTestMaterialPath, +} from '../../../support/test-materials' + +export const getCurrentAgentId = (world: DifyWorld) => { + const agentId = world.createdAgentIds.at(-1) + if (!agentId) + throw new Error('No Agent v2 ID found. Create an Agent v2 test agent first.') + + return agentId +} + +export const getPreseededAgent = (world: DifyWorld, name: string) => { + const resource = world.agentBuilder.preflight.preseededResources[name] + if (!resource || resource.kind !== 'agent') { + throw new Error( + `Preseeded Agent "${name}" is not available. Run the matching preflight step first.`, + ) + } + + return resource +} + +const getPreseededToolDisplayParts = (displayName: string) => { + const [providerName, actionName] = displayName.split(' / ') + if (!providerName || !actionName) + throw new Error(`Preseeded tool display name must use "Provider / Action": ${displayName}`) + + return { actionName, providerName } +} + +export const getEnvVariableKey = (variable: AgentComposerEnvVariable) => + variable.key ?? variable.name ?? variable.variable + +export const getAgentEnvVariableValue = ( + variables: AgentComposerEnvVariable[], + key: string, +) => variables.find(variable => getEnvVariableKey(variable) === key)?.value + +export const getAgentEnvVariables = async (agentId: string) => + (await getAgentComposerDraft(agentId)).agent_soul?.env?.variables ?? [] + +export const uploadAgentConfigFile = async ( + world: DifyWorld, + material: keyof typeof agentBuilderTestMaterials, +) => { + const page = world.getPage() + const agentId = getCurrentAgentId(world) + const fileName = agentBuilderTestMaterials[material] + const filePath = getAgentBuilderTestMaterialPath(material) + + await page.getByRole('button', { name: 'Add file' }).click() + const dialog = page.getByRole('dialog', { name: 'Upload file' }) + await expect(dialog).toBeVisible() + + const fileChooserPromise = page.waitForEvent('filechooser') + await dialog.getByRole('button', { name: 'browse' }).click() + await (await fileChooserPromise).setFiles(filePath) + await expect(dialog.getByText(fileName)).toBeVisible() + + const commitResponsePromise = page.waitForResponse(response => ( + response.request().method() === 'POST' + && new URL(response.url()).pathname.endsWith(`/console/api/agent/${agentId}/config/files`) + )) + await dialog.getByRole('button', { name: 'Upload' }).click() + const commitResponse = await commitResponsePromise + expect(commitResponse.status()).toBe(201) + const committed = await commitResponse.json() as { file?: { name?: string } } + await expect(dialog).not.toBeVisible({ timeout: 30_000 }) + + const committedName = committed.file?.name + if (!committedName) + throw new Error('Agent config file upload response did not include a file name.') + + world.createdAgentConfigFiles.push({ agentId, name: committedName }) +} + +export const expectAgentConfigFileVisible = async ( + world: DifyWorld, + material: keyof typeof agentBuilderTestMaterials, +) => { + await expect( + world.getPage().getByRole('button', { + exact: true, + name: agentBuilderTestMaterials[material], + }), + ).toBeVisible({ timeout: 30_000 }) +} + +export const expectAgentConfigFileHidden = async ( + world: DifyWorld, + material: keyof typeof agentBuilderTestMaterials, +) => { + await expect( + world.getPage().getByRole('button', { + exact: true, + name: agentBuilderTestMaterials[material], + }), + ).not.toBeVisible() +} + +export const expectAgentConfigFileSaved = async ( + world: DifyWorld, + material: keyof typeof agentBuilderTestMaterials, +) => { + const agentId = getCurrentAgentId(world) + const fileName = agentBuilderTestMaterials[material] + + await expect + .poll(async () => ( + await getAgentComposerDraft(agentId) + ).agent_soul?.config_files?.map(file => file.name) ?? [], { + timeout: 30_000, + }) + .toContain(fileName) +} + +export const uploadSummaryConfigSkillForBuildDraft = async (world: DifyWorld) => { + const agentId = getCurrentAgentId(world) + const skill = await uploadAgentConfigSkillToDraft({ + agentId, + fileName: agentBuilderTestMaterials.summarySkill, + filePath: getAgentBuilderTestMaterialPath('summarySkill'), + }) + + if (!skill.file_id) + throw new Error('Agent v2 build draft Skill fixture did not return a file_id.') + + world.createdAgentConfigSkills.push({ agentId, name: skill.name }) + + return skill +} + +export const openAgentAdvancedSettings = async (page: ReturnType) => { + const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' }) + const envEditorHeading = advancedSettings.getByRole('heading', { name: 'Env Editor' }) + + if (!await envEditorHeading.isVisible().catch(() => false)) + await page.getByRole('button', { name: 'Advanced Settings' }).first().click() + + await expect(envEditorHeading).toBeVisible() + + return advancedSettings +} + +export const expectAgentEnvVariableVisible = async ( + world: DifyWorld, + key: string, + value: string, +) => { + const advancedSettings = await openAgentAdvancedSettings(world.getPage()) + + await expect.poll( + async () => { + const text = await advancedSettings.textContent() + const inputValues = await advancedSettings.getByRole('textbox').evaluateAll(inputs => + inputs.map(input => (input as HTMLInputElement).value), + ) + + return { + hasKey: inputValues.includes(key) || !!text?.includes(key), + hasValue: inputValues.includes(value) || !!text?.includes(value), + } + }, + { timeout: 30_000 }, + ).toEqual({ + hasKey: true, + hasValue: true, + }) + await expect(advancedSettings.getByText('Plain', { exact: true })).toBeVisible() +} + +export const expectAgentEnvVariableHidden = async ( + world: DifyWorld, + key: string, +) => { + const advancedSettings = await openAgentAdvancedSettings(world.getPage()) + + await expect.poll( + async () => { + const text = await advancedSettings.textContent() + const inputValues = await advancedSettings.getByRole('textbox').evaluateAll(inputs => + inputs.map(input => (input as HTMLInputElement).value), + ) + + return inputValues.includes(key) || !!text?.includes(key) + }, + { timeout: 30_000 }, + ).toBe(false) +} + +export const expectNormalAgentPromptDraft = async (world: DifyWorld) => { + await expect.poll( + async () => (await getAgentComposerDraft(getCurrentAgentId(world))).agent_soul?.prompt, + { timeout: 30_000 }, + ).toEqual({ system_prompt: normalAgentPrompt }) +} + +export const expectProviderToolActionVisible = async ( + toolsSection: Locator, + displayName: string, +) => { + const tool = getPreseededToolDisplayParts(displayName) + const provider = toolsSection.getByRole('button', { + exact: true, + name: tool.providerName, + }) + await expect(provider).toBeVisible() + + const action = toolsSection.getByText(tool.actionName, { exact: true }) + if (!await action.isVisible()) + await provider.click() + await expect(action).toBeVisible() + + return { action, tool } +} + +export const openAgentKnowledgeRetrievalDialog = async (knowledgeSection: Locator, name: string) => { + await knowledgeSection.getByText(name, { exact: true }).hover() + await knowledgeSection.getByRole('button', { + exact: true, + name: `Edit ${name}`, + }).click() + + const dialog = knowledgeSection.page().getByRole('dialog', { + name: 'Knowledge Retrieval · Agent decide', + }) + await expect(dialog).toBeVisible() + + return dialog +} diff --git a/e2e/features/step-definitions/agent-v2/configure.steps.ts b/e2e/features/step-definitions/agent-v2/configure.steps.ts index d3f766f6472..b89e7461879 100644 --- a/e2e/features/step-definitions/agent-v2/configure.steps.ts +++ b/e2e/features/step-definitions/agent-v2/configure.steps.ts @@ -1,7 +1,4 @@ -import type { Locator } from '@playwright/test' -import type { AgentComposerEnvVariable } from '../../../support/agent' import type { DifyWorld } from '../../support/world' -import { readFile } from 'node:fs/promises' import { Given, Then, When } from '@cucumber/cucumber' import { expect } from '@playwright/test' import { @@ -11,257 +8,18 @@ import { getAgentComposerDraft, getAgentConfigurePath, getAgentDriveSkills, - getTestAgent, normalAgentPrompt, normalAgentSoulConfig, - saveAgentBuildDraft, saveAgentComposerDraft, updatedAgentPrompt, - updatedAgentSoulConfig, - uploadAgentConfigFileToDraft, - uploadAgentConfigSkillToDraft, uploadAgentDriveSkill, } from '../../../support/agent' +import { agentBuilderTestMaterials, getAgentBuilderTestMaterialPath } from '../../../support/test-materials' import { - agentBuilderExpectedTokens, - agentBuilderFixedInputs, - agentBuilderPreseededResources, -} from '../../../support/agent-builder-resources' -import { waitForAgentConfigureAutosaved } from '../../../support/agent-configure' -import { skipBlockedPrecondition } from '../../../support/preflight' -import { - agentBuilderFileTreeFixtureFileNames, - agentBuilderTestMaterials, - getAgentBuilderTestMaterialPath, -} from '../../../support/test-materials' - -const getCurrentAgentId = (world: DifyWorld) => { - const agentId = world.createdAgentIds.at(-1) - if (!agentId) - throw new Error('No Agent v2 ID found. Create an Agent v2 test agent first.') - - return agentId -} - -const getPreseededAgent = (world: DifyWorld, name: string) => { - const resource = world.agentBuilder.preflight.preseededResources[name] - if (!resource || resource.kind !== 'agent') { - throw new Error( - `Preseeded Agent "${name}" is not available. Run the matching preflight step first.`, - ) - } - - return resource -} - -const getPreseededToolDisplayParts = (displayName: string) => { - const [providerName, actionName] = displayName.split(' / ') - if (!providerName || !actionName) - throw new Error(`Preseeded tool display name must use "Provider / Action": ${displayName}`) - - return { actionName, providerName } -} - -const getEnvVariableKey = (variable: AgentComposerEnvVariable) => - variable.key ?? variable.name ?? variable.variable - -const getAgentEnvVariableValue = ( - variables: AgentComposerEnvVariable[], - key: string, -) => variables.find(variable => getEnvVariableKey(variable) === key)?.value - -const getAgentEnvVariables = async (agentId: string) => - (await getAgentComposerDraft(agentId)).agent_soul?.env?.variables ?? [] - -const uploadAgentConfigFile = async ( - world: DifyWorld, - material: keyof typeof agentBuilderTestMaterials, -) => { - const page = world.getPage() - const agentId = getCurrentAgentId(world) - const fileName = agentBuilderTestMaterials[material] - const filePath = getAgentBuilderTestMaterialPath(material) - - await page.getByRole('button', { name: 'Add file' }).click() - const dialog = page.getByRole('dialog', { name: 'Upload file' }) - await expect(dialog).toBeVisible() - - const fileChooserPromise = page.waitForEvent('filechooser') - await dialog.getByRole('button', { name: 'browse' }).click() - await (await fileChooserPromise).setFiles(filePath) - await expect(dialog.getByText(fileName)).toBeVisible() - - const commitResponsePromise = page.waitForResponse(response => ( - response.request().method() === 'POST' - && new URL(response.url()).pathname.endsWith(`/console/api/agent/${agentId}/config/files`) - )) - await dialog.getByRole('button', { name: 'Upload' }).click() - const commitResponse = await commitResponsePromise - expect(commitResponse.status()).toBe(201) - const committed = await commitResponse.json() as { file?: { name?: string } } - await expect(dialog).not.toBeVisible({ timeout: 30_000 }) - - const committedName = committed.file?.name - if (!committedName) - throw new Error('Agent config file upload response did not include a file name.') - - world.createdAgentConfigFiles.push({ agentId, name: committedName }) -} - -const expectAgentConfigFileVisible = async ( - world: DifyWorld, - material: keyof typeof agentBuilderTestMaterials, -) => { - await expect( - world.getPage().getByRole('button', { - exact: true, - name: agentBuilderTestMaterials[material], - }), - ).toBeVisible({ timeout: 30_000 }) -} - -const expectAgentConfigFileHidden = async ( - world: DifyWorld, - material: keyof typeof agentBuilderTestMaterials, -) => { - await expect( - world.getPage().getByRole('button', { - exact: true, - name: agentBuilderTestMaterials[material], - }), - ).not.toBeVisible() -} - -const expectAgentConfigFileSaved = async ( - world: DifyWorld, - material: keyof typeof agentBuilderTestMaterials, -) => { - const agentId = getCurrentAgentId(world) - const fileName = agentBuilderTestMaterials[material] - - await expect - .poll(async () => ( - await getAgentComposerDraft(agentId) - ).agent_soul?.config_files?.map(file => file.name) ?? [], { - timeout: 30_000, - }) - .toContain(fileName) -} - -const uploadSummaryConfigSkillForBuildDraft = async (world: DifyWorld) => { - const agentId = getCurrentAgentId(world) - const skill = await uploadAgentConfigSkillToDraft({ - agentId, - fileName: agentBuilderTestMaterials.summarySkill, - filePath: getAgentBuilderTestMaterialPath('summarySkill'), - }) - - if (!skill.file_id) - throw new Error('Agent v2 build draft Skill fixture did not return a file_id.') - - world.createdAgentConfigSkills.push({ agentId, name: skill.name }) - - return skill -} - -const openAgentAdvancedSettings = async (page: ReturnType) => { - const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' }) - const envEditorHeading = advancedSettings.getByRole('heading', { name: 'Env Editor' }) - - if (!await envEditorHeading.isVisible().catch(() => false)) - await page.getByRole('button', { name: 'Advanced Settings' }).first().click() - - await expect(envEditorHeading).toBeVisible() - - return advancedSettings -} - -const expectAgentEnvVariableVisible = async ( - world: DifyWorld, - key: string, - value: string, -) => { - const advancedSettings = await openAgentAdvancedSettings(world.getPage()) - - await expect.poll( - async () => { - const text = await advancedSettings.textContent() - const inputValues = await advancedSettings.getByRole('textbox').evaluateAll(inputs => - inputs.map(input => (input as HTMLInputElement).value), - ) - - return { - hasKey: inputValues.includes(key) || !!text?.includes(key), - hasValue: inputValues.includes(value) || !!text?.includes(value), - } - }, - { timeout: 30_000 }, - ).toEqual({ - hasKey: true, - hasValue: true, - }) - await expect(advancedSettings.getByText('Plain', { exact: true })).toBeVisible() -} - -const expectAgentEnvVariableHidden = async ( - world: DifyWorld, - key: string, -) => { - const advancedSettings = await openAgentAdvancedSettings(world.getPage()) - - await expect.poll( - async () => { - const text = await advancedSettings.textContent() - const inputValues = await advancedSettings.getByRole('textbox').evaluateAll(inputs => - inputs.map(input => (input as HTMLInputElement).value), - ) - - return inputValues.includes(key) || !!text?.includes(key) - }, - { timeout: 30_000 }, - ).toBe(false) -} - -const expectNormalAgentPromptDraft = async (world: DifyWorld) => { - await expect.poll( - async () => (await getAgentComposerDraft(getCurrentAgentId(world))).agent_soul?.prompt, - { timeout: 30_000 }, - ).toEqual({ system_prompt: normalAgentPrompt }) -} - -const expectProviderToolActionVisible = async ( - toolsSection: Locator, - displayName: string, -) => { - const tool = getPreseededToolDisplayParts(displayName) - const provider = toolsSection.getByRole('button', { - exact: true, - name: tool.providerName, - }) - await expect(provider).toBeVisible() - - const action = toolsSection.getByText(tool.actionName, { exact: true }) - if (!await action.isVisible()) - await provider.click() - await expect(action).toBeVisible() - - return { action, tool } -} - -const openAgentKnowledgeRetrievalDialog = async (knowledgeSection: Locator, name: string) => { - await knowledgeSection.getByText(name, { exact: true }).hover() - await knowledgeSection.getByRole('button', { - exact: true, - name: `Edit ${name}`, - }).click() - - const dialog = knowledgeSection.page().getByRole('dialog', { - name: 'Knowledge Retrieval · Agent decide', - }) - await expect(dialog).toBeVisible() - - return dialog -} + expectNormalAgentPromptDraft, + getCurrentAgentId, + getPreseededAgent, +} from './configure-helpers' Given('an Agent v2 test agent has been created via API', async function (this: DifyWorld) { const agent = await createTestAgent() @@ -319,93 +77,6 @@ Then('the Agent v2 test agent should include drive skill {string}', async functi expect(skills.map(skill => skill.name)).toContain(skillName) }) -Given( - 'an Agent v2 Build draft adds the supported E2E files, skills, and env', - async function (this: DifyWorld) { - const agentId = getCurrentAgentId(this) - const configFile = await uploadAgentConfigFileToDraft({ - agentId, - fileName: agentBuilderTestMaterials.smallFile, - filePath: getAgentBuilderTestMaterialPath('smallFile'), - }) - const skill = await uploadSummaryConfigSkillForBuildDraft(this) - - if (!configFile.file_id) - throw new Error('Agent v2 build draft config file fixture did not return a file_id.') - this.createdAgentConfigFiles.push({ agentId, name: configFile.name }) - - const normalConfig = this.agentBuilder.preflight.stableModel - ? createAgentSoulConfigWithModel(normalAgentSoulConfig, this.agentBuilder.preflight.stableModel) - : normalAgentSoulConfig - const updatedConfig = this.agentBuilder.preflight.stableModel - ? createAgentSoulConfigWithModel(updatedAgentSoulConfig, this.agentBuilder.preflight.stableModel) - : updatedAgentSoulConfig - - await saveAgentComposerDraft(agentId, normalConfig) - await saveAgentBuildDraft(agentId, { - ...updatedConfig, - config_files: [configFile], - config_skills: [{ - ...skill, - file_kind: skill.file_kind ?? 'tool_file', - }], - env: { - secret_refs: [], - variables: [{ - id: agentBuilderFixedInputs.envPlainKey, - key: agentBuilderFixedInputs.envPlainKey, - name: agentBuilderFixedInputs.envPlainKey, - value: agentBuilderFixedInputs.envPlainValue, - variable: agentBuilderFixedInputs.envPlainKey, - }], - }, - }) - }, -) - -Given( - 'an Agent v2 Build draft includes the existing e2e-summary-skill Skill', - async function (this: DifyWorld) { - const skill = await uploadSummaryConfigSkillForBuildDraft(this) - const normalConfig = this.agentBuilder.preflight.stableModel - ? createAgentSoulConfigWithModel(normalAgentSoulConfig, this.agentBuilder.preflight.stableModel) - : normalAgentSoulConfig - const updatedConfig = this.agentBuilder.preflight.stableModel - ? createAgentSoulConfigWithModel(updatedAgentSoulConfig, this.agentBuilder.preflight.stableModel) - : updatedAgentSoulConfig - const configSkills = [{ - ...skill, - file_kind: skill.file_kind ?? 'tool_file', - }] - - await saveAgentComposerDraft(getCurrentAgentId(this), { - ...normalConfig, - config_skills: configSkills, - }) - await saveAgentBuildDraft(getCurrentAgentId(this), { - ...updatedConfig, - config_skills: configSkills, - }) - }, -) - -Given('an Agent v2 Build draft uses the updated E2E prompt', async function (this: DifyWorld) { - await saveAgentBuildDraft(getCurrentAgentId(this), updatedAgentSoulConfig) -}) - -Given( - 'an Agent v2 Build draft uses the updated E2E prompt with the stable E2E model', - async function (this: DifyWorld) { - if (!this.agentBuilder.preflight.stableModel) - throw new Error('Create an Agent v2 Build draft with a stable model after stable model preflight.') - - await saveAgentBuildDraft( - getCurrentAgentId(this), - createAgentSoulConfigWithModel(updatedAgentSoulConfig, this.agentBuilder.preflight.stableModel), - ) - }, -) - When('I open the Agent v2 configure page', async function (this: DifyWorld) { await this.getPage().goto(getAgentConfigurePath(getCurrentAgentId(this))) }) @@ -419,31 +90,6 @@ When('I switch to the Agent v2 Configure section', async function (this: DifyWor await expect(page.getByRole('heading', { name: 'Configure' })).toBeVisible({ timeout: 30_000 }) }) -When('I generate an Agent v2 Build draft from the fixed instruction', async function (this: DifyWorld) { - const page = this.getPage() - const agentId = getCurrentAgentId(this) - const instruction = (await readFile(getAgentBuilderTestMaterialPath('buildInstruction'), 'utf8')).trim() - - await page.getByRole('button', { name: 'Build' }).click() - await page.getByPlaceholder('Describe what your agent should do').fill(instruction) - - const checkoutResponsePromise = page.waitForResponse(response => ( - response.request().method() === 'POST' - && new URL(response.url()).pathname.endsWith(`/console/api/agent/${agentId}/build-draft/checkout`) - )) - const chatResponsePromise = page.waitForResponse(response => ( - response.request().method() === 'POST' - && new URL(response.url()).pathname.endsWith(`/console/api/agent/${agentId}/chat-messages`) - )) - - await page.getByRole('button', { name: 'Start build' }).click() - expect((await checkoutResponsePromise).ok()).toBe(true) - expect((await chatResponsePromise).ok()).toBe(true) - await expect(page.getByText('Build draft')).toBeVisible({ timeout: 120_000 }) - await expect(page.getByRole('button', { name: 'Apply' })).toBeEnabled({ timeout: 120_000 }) - await expect(page.getByRole('button', { name: 'Discard' })).toBeEnabled() -}) - When('I open the Agent v2 configure page from the Agent Roster', async function (this: DifyWorld) { const page = this.getPage() const agentId = getCurrentAgentId(this) @@ -478,132 +124,6 @@ When('I fill the Agent v2 prompt editor with the normal E2E prompt', async funct await promptSection.getByRole('textbox', { name: 'Prompt' }).fill(normalAgentPrompt) }) -When('I discard the Agent v2 Build draft', async function (this: DifyWorld) { - await this.getPage().getByRole('button', { name: 'Discard' }).click() -}) - -When('I apply the Agent v2 Build draft', async function (this: DifyWorld) { - const page = this.getPage() - const agentId = getCurrentAgentId(this) - const applyButton = page.getByRole('button', { name: 'Apply' }) - - await expect(applyButton).toBeEnabled({ timeout: 30_000 }) - const finalizeResponsePromise = page.waitForResponse(response => ( - response.request().method() === 'POST' - && new URL(response.url()).pathname.endsWith(`/console/api/agent/${agentId}/build-chat/finalize`) - ), { timeout: 120_000 }) - const applyResponsePromise = page.waitForResponse(response => ( - response.request().method() === 'POST' - && new URL(response.url()).pathname.endsWith(`/console/api/agent/${agentId}/build-draft/apply`) - ), { timeout: 120_000 }) - - await applyButton.click() - expect((await finalizeResponsePromise).ok()).toBe(true) - expect((await applyResponsePromise).ok()).toBe(true) - await expect(page.getByText('Action succeeded')).toBeVisible() -}) - -When('I publish the Agent v2 draft', async function (this: DifyWorld) { - const page = this.getPage() - const publishButton = page.getByRole('button', { name: /^Publish(?: update)?$/ }) - - await expect(publishButton).toBeEnabled({ timeout: 30_000 }) - await publishButton.click() -}) - -When('I upload the small Agent v2 file from the Files section', async function (this: DifyWorld) { - await uploadAgentConfigFile(this, 'smallFile') -}) - -When('I upload the special-name Agent v2 file from the Files section', async function (this: DifyWorld) { - await uploadAgentConfigFile(this, 'specialFilename') -}) - -When( - 'I add the plain Agent v2 environment variable from Advanced Settings', - async function (this: DifyWorld) { - const page = this.getPage() - const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' }) - - await page.getByRole('button', { name: 'Advanced Settings' }).first().click() - await expect(advancedSettings.getByRole('heading', { name: 'Env Editor' })).toBeVisible() - - await advancedSettings - .getByRole('textbox', { name: 'Key' }) - .fill(agentBuilderFixedInputs.envPlainKey) - await advancedSettings - .getByRole('textbox', { name: 'Value' }) - .fill(agentBuilderFixedInputs.envPlainValue) - await expect(advancedSettings.getByText('Plain', { exact: true })).toBeVisible() - }, -) - -When( - 'I import the invalid Agent v2 environment file from Advanced Settings', - async function (this: DifyWorld) { - const page = this.getPage() - const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' }) - - const fileChooserPromise = page.waitForEvent('filechooser') - await advancedSettings.getByRole('button', { name: 'Import .env' }).click() - await (await fileChooserPromise).setFiles(getAgentBuilderTestMaterialPath('invalidEnv')) - }, -) - -When( - 'I add the secondary plain Agent v2 environment variable from Advanced Settings', - async function (this: DifyWorld) { - const page = this.getPage() - const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' }) - - await advancedSettings.getByRole('button', { name: 'Add environment variable' }).click() - await advancedSettings - .getByRole('textbox', { name: 'Key' }) - .last() - .fill(agentBuilderFixedInputs.envModeKey) - await advancedSettings - .getByRole('textbox', { name: 'Value' }) - .last() - .fill(agentBuilderFixedInputs.envModeValue) - await expect(advancedSettings.getByText('Plain', { exact: true })).toHaveCount(2) - }, -) - -When( - 'I delete the plain Agent v2 environment variable from Advanced Settings', - async function (this: DifyWorld) { - const page = this.getPage() - const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' }) - - await advancedSettings - .getByRole('button', { name: `Delete ${agentBuilderFixedInputs.envPlainKey}` }) - .click() - }, -) - -When( - 'I import the valid Agent v2 environment file from Advanced Settings', - async function (this: DifyWorld) { - const page = this.getPage() - const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' }) - - await page.getByRole('button', { name: 'Advanced Settings' }).first().click() - await expect(advancedSettings.getByRole('heading', { name: 'Env Editor' })).toBeVisible() - - const fileChooserPromise = page.waitForEvent('filechooser') - await advancedSettings.getByRole('button', { name: 'Import .env' }).click() - await (await fileChooserPromise).setFiles(getAgentBuilderTestMaterialPath('validEnv')) - }, -) - -When('I expand Agent v2 Advanced Settings', async function (this: DifyWorld) { - const page = this.getPage() - const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' }) - - await page.getByRole('button', { name: 'Advanced Settings' }).first().click() - await expect(advancedSettings.getByRole('heading', { name: 'Env Editor' })).toBeVisible() -}) - Then('I should be on the Agent v2 configure page', async function (this: DifyWorld) { const agentId = getCurrentAgentId(this) @@ -620,142 +140,6 @@ Then('I should see the Agent v2 configure workspace', async function (this: Dify await expect(page.getByText(this.lastCreatedAgentName!)).toBeVisible() }) -Then('I should see the Agent v2 full-config fixture sections', async function (this: DifyWorld) { - const page = this.getPage() - const stableModel = this.agentBuilder.preflight.stableModel - if (!stableModel) - throw new Error('Stable chat model preflight must run before asserting the full-config Agent.') - - await expect(page.getByRole('heading', { name: 'Configure' })).toBeVisible({ timeout: 30_000 }) - await expect(page.getByText(agentBuilderPreseededResources.fullConfigAgent, { exact: true })) - .toBeVisible() - await expect(page.getByText(stableModel.name, { exact: true })).toBeVisible() - - const promptSection = page.getByRole('region', { name: 'Prompt' }) - await expect(promptSection).toBeVisible() - await expect(promptSection).toContainText(agentBuilderExpectedTokens.agentReply) - - const skillsSection = page.getByRole('region', { name: 'Skills' }) - await expect(skillsSection).toBeVisible() - await expect(skillsSection.getByRole('button', { - exact: true, - name: agentBuilderPreseededResources.summarySkill, - })).toBeVisible() - - const filesSection = page.getByRole('region', { name: 'Files' }) - await expect(filesSection).toBeVisible() - await expect(filesSection.getByRole('button', { - exact: true, - name: agentBuilderTestMaterials.smallFile, - })).toBeVisible() - await expect(filesSection.getByRole('button', { - exact: true, - name: agentBuilderTestMaterials.specialFilename, - })).toBeVisible() - - const toolsSection = page.getByRole('region', { name: 'Tools' }) - await expect(toolsSection).toBeVisible() - await expectProviderToolActionVisible( - toolsSection, - agentBuilderPreseededResources.jsonReplaceTool, - ) - - const knowledgeSection = page.getByRole('region', { name: 'Knowledge Retrieval' }) - await expect(knowledgeSection).toBeVisible() - await expect(knowledgeSection.getByText('Retrieval 1', { exact: true })).toBeVisible() - - const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' }) - await expect(advancedSettings).toBeVisible() - await expect( - advancedSettings.getByText('For power users. Env vars, sandbox & memory.'), - ).toBeVisible() -}) - -Then('I should see the Agent v2 tool state fixture tools', async function (this: DifyWorld) { - const page = this.getPage() - const toolsSection = page.getByRole('region', { name: 'Tools' }) - - await expect(toolsSection).toBeVisible({ timeout: 30_000 }) - await expect(toolsSection.getByRole('button', { exact: true, name: 'Not authorized' })).toBeVisible() - - const { action: jsonReplaceAction, tool: jsonTool } = await expectProviderToolActionVisible( - toolsSection, - agentBuilderPreseededResources.jsonReplaceTool, - ) - await jsonReplaceAction.hover() - await expect(toolsSection.getByRole('button', { - exact: true, - name: `Edit ${jsonTool.actionName}`, - })).toBeVisible() - await expect(toolsSection.getByRole('button', { - exact: true, - name: `Remove ${jsonTool.actionName}`, - })).toBeVisible() - - await expectProviderToolActionVisible( - toolsSection, - agentBuilderPreseededResources.tavilySearchTool, - ) -}) - -Then( - 'I should see the Agent v2 file fixture entries in the current flat Files list', - async function (this: DifyWorld) { - const page = this.getPage() - const filesSection = page.getByRole('region', { name: 'Files' }) - const filesList = filesSection.getByLabel('Agent files') - - await expect(filesSection).toBeVisible({ timeout: 30_000 }) - await expect(filesList).toBeVisible() - - for (const fileName of agentBuilderFileTreeFixtureFileNames) { - await expect(filesList.getByRole('button', { - exact: true, - name: fileName, - })).toBeVisible() - } - - await expect(filesList.getByRole('button', { exact: true, name: 'assets' })).toHaveCount(0) - await expect(filesList.getByRole('button', { exact: true, name: 'docs' })).toHaveCount(0) - await expect(filesList.getByRole('button', { exact: true, name: 'public' })).toHaveCount(0) - await expect(filesList.getByRole('button', { exact: true, name: 'src' })).toHaveCount(0) - await expect(filesList.getByRole('button', { exact: true, name: 'web-game' })).toHaveCount(0) - }, -) - -Then('I should see the Agent v2 dual retrieval fixture settings', async function (this: DifyWorld) { - const page = this.getPage() - const knowledgeSection = page.getByRole('region', { name: 'Knowledge Retrieval' }) - - await expect(knowledgeSection).toBeVisible({ timeout: 30_000 }) - await expect(knowledgeSection.getByText('Retrieval 1', { exact: true })).toBeVisible() - await expect(knowledgeSection.getByText('Retrieval 2', { exact: true })).toBeVisible() - - const agentDecideDialog = await openAgentKnowledgeRetrievalDialog(knowledgeSection, 'Retrieval 1') - await expect(agentDecideDialog.getByText(agentBuilderPreseededResources.agentKnowledgeBase, { - exact: true, - })).toBeVisible() - await expect(agentDecideDialog.getByRole('radio', { - exact: true, - name: 'Agent decide', - })).toBeChecked() - await agentDecideDialog.getByRole('button', { name: 'Close' }).click() - await expect(agentDecideDialog).not.toBeVisible() - - const customQueryDialog = await openAgentKnowledgeRetrievalDialog(knowledgeSection, 'Retrieval 2') - await expect(customQueryDialog.getByText(agentBuilderPreseededResources.agentKnowledgeBase, { - exact: true, - })).toBeVisible() - await expect(customQueryDialog.getByRole('radio', { - exact: true, - name: 'Custom query', - })).toBeChecked() - await expect(customQueryDialog.getByRole('textbox', { - exact: true, - name: 'Custom query text', - })).toHaveValue(agentBuilderFixedInputs.customKnowledgeQuery) -}) - Then( 'I should see the normal E2E prompt in the Agent v2 prompt editor', async function (this: DifyWorld) { @@ -798,380 +182,6 @@ Then( }, ) -Then('I should see the small Agent v2 file in the Files section', async function (this: DifyWorld) { - await expectAgentConfigFileVisible(this, 'smallFile') -}) - -Then('I should not see the small Agent v2 file in the Files section', async function (this: DifyWorld) { - await expectAgentConfigFileHidden(this, 'smallFile') -}) - -Then('I should see the special-name Agent v2 file in the Files section', async function (this: DifyWorld) { - await expectAgentConfigFileVisible(this, 'specialFilename') -}) - -Then('I should see the e2e-summary-skill Skill in the Skills section', async function (this: DifyWorld) { - const skillsSection = this.getPage().getByRole('region', { name: 'Skills' }) - - await expect(skillsSection.getByRole('button', { - exact: true, - name: agentBuilderPreseededResources.summarySkill, - })).toBeVisible({ timeout: 30_000 }) -}) - -Then('I should not see the e2e-summary-skill Skill in the Skills section', async function (this: DifyWorld) { - const skillsSection = this.getPage().getByRole('region', { name: 'Skills' }) - - await expect(skillsSection.getByRole('button', { - exact: true, - name: agentBuilderPreseededResources.summarySkill, - })).not.toBeVisible() -}) - -Then('I should see one e2e-summary-skill Skill in the Skills section', async function (this: DifyWorld) { - const skillsSection = this.getPage().getByRole('region', { name: 'Skills' }) - - await expect(skillsSection.getByRole('button', { - exact: true, - name: agentBuilderPreseededResources.summarySkill, - })).toHaveCount(1) -}) - -Then( - 'the small Agent v2 file should be saved in the Agent v2 draft', - async function (this: DifyWorld) { - await expectAgentConfigFileSaved(this, 'smallFile') - }, -) - -Then( - 'the special-name Agent v2 file should be saved in the Agent v2 draft', - async function (this: DifyWorld) { - await expectAgentConfigFileSaved(this, 'specialFilename') - }, -) - -Then( - 'Agent v2 Advanced Settings should describe supported entries while collapsed', - async function (this: DifyWorld) { - const page = this.getPage() - const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' }) - - await expect(advancedSettings).toBeVisible() - await expect( - advancedSettings.getByText('For power users. Env vars, sandbox & memory.'), - ).toBeVisible() - await expect(advancedSettings.getByRole('heading', { name: 'Env Editor' })) - .not - .toBeVisible() - }, -) - -Then( - 'the plain Agent v2 environment variable should be saved in the Agent v2 draft', - async function (this: DifyWorld) { - const agentId = getCurrentAgentId(this) - - await expect - .poll(async () => { - const env = (await getAgentComposerDraft(agentId)).agent_soul?.env - const variable = env?.variables?.find(item => - getEnvVariableKey(item) === agentBuilderFixedInputs.envPlainKey, - ) - - return { - secretCount: env?.secret_refs?.length ?? 0, - value: variable?.value, - } - }, { - timeout: 30_000, - }) - .toEqual({ - secretCount: 0, - value: agentBuilderFixedInputs.envPlainValue, - }) - }, -) - -Then( - 'the Agent v2 environment variables for deletion should be saved in the Agent v2 draft', - async function (this: DifyWorld) { - const agentId = getCurrentAgentId(this) - - await expect - .poll(async () => { - const variables = await getAgentEnvVariables(agentId) - - return { - modeValue: getAgentEnvVariableValue(variables, agentBuilderFixedInputs.envModeKey), - plainValue: getAgentEnvVariableValue(variables, agentBuilderFixedInputs.envPlainKey), - } - }, { - timeout: 30_000, - }) - .toEqual({ - modeValue: agentBuilderFixedInputs.envModeValue, - plainValue: agentBuilderFixedInputs.envPlainValue, - }) - }, -) - -Then( - 'the plain Agent v2 environment variable should be removed from the Agent v2 draft', - async function (this: DifyWorld) { - const agentId = getCurrentAgentId(this) - - await expect - .poll(async () => { - const variables = await getAgentEnvVariables(agentId) - - return { - modeValue: getAgentEnvVariableValue(variables, agentBuilderFixedInputs.envModeKey), - plainValue: getAgentEnvVariableValue(variables, agentBuilderFixedInputs.envPlainKey), - } - }, { - timeout: 30_000, - }) - .toEqual({ - modeValue: agentBuilderFixedInputs.envModeValue, - plainValue: undefined, - }) - }, -) - -Then( - 'the valid Agent v2 environment import should be saved in the Agent v2 draft', - async function (this: DifyWorld) { - const agentId = getCurrentAgentId(this) - - await expect - .poll(async () => { - const variables = await getAgentEnvVariables(agentId) - - return { - modeValue: getAgentEnvVariableValue(variables, agentBuilderFixedInputs.envModeKey), - plainValue: getAgentEnvVariableValue(variables, agentBuilderFixedInputs.envPlainKey), - } - }, { - timeout: 30_000, - }) - .toEqual({ - modeValue: agentBuilderFixedInputs.envModeValue, - plainValue: agentBuilderFixedInputs.envPlainValue, - }) - }, -) - -Then( - 'the invalid Agent v2 environment import should report skipped lines', - async function (this: DifyWorld) { - await expect(this.getPage().getByText('2 invalid .env lines were skipped.')).toBeVisible() - }, -) - -Then( - 'the Agent v2 environment variables from the invalid import should be saved in the Agent v2 draft', - async function (this: DifyWorld) { - const agentId = getCurrentAgentId(this) - - await expect - .poll(async () => { - const variables = await getAgentEnvVariables(agentId) - - return { - importedValue: getAgentEnvVariableValue( - variables, - agentBuilderFixedInputs.envAfterInvalidImportKey, - ), - plainValue: getAgentEnvVariableValue(variables, agentBuilderFixedInputs.envPlainKey), - } - }, { - timeout: 30_000, - }) - .toEqual({ - importedValue: agentBuilderFixedInputs.envAfterInvalidImportValue, - plainValue: agentBuilderFixedInputs.envPlainValue, - }) - }, -) - -Then( - 'I should see the supported Agent v2 Advanced Settings entries', - async function (this: DifyWorld) { - const page = this.getPage() - const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' }) - const envEditor = advancedSettings.getByRole('region', { name: 'Env Editor' }) - - await expect(envEditor).toBeVisible() - await expect(envEditor.getByRole('button', { name: 'Import .env' })).toBeVisible() - await expect(envEditor.getByRole('button', { name: 'Add environment variable' })) - .toBeVisible() - await expect(envEditor.getByText('Key', { exact: true })).toBeVisible() - await expect(envEditor.getByText('Value', { exact: true })).toBeVisible() - await expect(envEditor.getByText('Scope', { exact: true })).toBeVisible() - }, -) - -Then('Agent v2 Content Moderation Settings should be available', async function (this: DifyWorld) { - const advancedSettings = this.getPage().getByRole('region', { name: 'Advanced Settings' }) - const contentModeration = advancedSettings.getByRole('region', { name: 'Content moderation' }) - - try { - await expect(contentModeration).toBeVisible({ timeout: 3_000 }) - } - catch { - return skipBlockedPrecondition( - this, - 'Feature not enabled: Agent v2 Content Moderation Settings is not available in this build.', - ) - } -}) - -Then('Agent v2 Build chat Dify Tool writeback should be available', async function (this: DifyWorld) { - const toolsSection = this.getPage().getByRole('region', { name: 'Tools' }) - - await expect(toolsSection).toBeVisible({ timeout: 30_000 }) - - return skipBlockedPrecondition( - this, - 'Build draft Dify Tool writeback is not available: Build draft currently supports files, skills, and env only.', - ) -}) - -Then('Agent v2 standalone Output Variables should be available', async function (this: DifyWorld) { - const page = this.getPage() - - await expect(page.getByRole('heading', { name: 'Configure' })).toBeVisible({ timeout: 30_000 }) - - return skipBlockedPrecondition( - this, - 'Standalone Agent Output Variables are not available: output variables currently belong to Workflow Agent v2 nodes.', - ) -}) - -Then( - 'I should see the Agent v2 environment variables from the valid import in Advanced Settings', - async function (this: DifyWorld) { - const page = this.getPage() - const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' }) - - await page.getByRole('button', { name: 'Advanced Settings' }).first().click() - await expect(advancedSettings.getByRole('heading', { name: 'Env Editor' })).toBeVisible() - await expect.poll( - async () => advancedSettings.getByRole('textbox').evaluateAll(inputs => - inputs.map(input => (input as HTMLInputElement).value), - ), - { timeout: 30_000 }, - ).toEqual(expect.arrayContaining([ - agentBuilderFixedInputs.envPlainKey, - agentBuilderFixedInputs.envPlainValue, - agentBuilderFixedInputs.envModeKey, - agentBuilderFixedInputs.envModeValue, - ])) - await expect(advancedSettings.getByText('Plain', { exact: true })).toHaveCount(2) - }, -) - -Then( - 'I should not see the deleted Agent v2 environment variable in Advanced Settings', - async function (this: DifyWorld) { - const page = this.getPage() - const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' }) - - await page.getByRole('button', { name: 'Advanced Settings' }).first().click() - await expect(advancedSettings.getByRole('heading', { name: 'Env Editor' })).toBeVisible() - await expect.poll( - async () => advancedSettings.getByRole('textbox').evaluateAll(inputs => - inputs.map(input => (input as HTMLInputElement).value), - ), - { timeout: 30_000 }, - ).toEqual(expect.arrayContaining([ - agentBuilderFixedInputs.envModeKey, - agentBuilderFixedInputs.envModeValue, - ])) - await expect.poll( - async () => advancedSettings.getByRole('textbox').evaluateAll(inputs => - inputs.map(input => (input as HTMLInputElement).value), - ), - { timeout: 30_000 }, - ).not.toContain(agentBuilderFixedInputs.envPlainKey) - await expect(advancedSettings.getByText('Plain', { exact: true })).toHaveCount(1) - }, -) - -Then( - 'I should see the plain Agent v2 environment variable in Advanced Settings', - async function (this: DifyWorld) { - const page = this.getPage() - const advancedSettings = await openAgentAdvancedSettings(page) - - await expect(advancedSettings.getByRole('textbox', { name: 'Key' })) - .toHaveValue(agentBuilderFixedInputs.envPlainKey) - await expect(advancedSettings.getByRole('textbox', { name: 'Value' })) - .toHaveValue(agentBuilderFixedInputs.envPlainValue) - await expect(advancedSettings.getByText('Plain', { exact: true })).toBeVisible() - await expect(page.getByRole('button', { name: /^Build$/i })).toBeVisible() - }, -) - -Then( - 'I should see the supported E2E environment variable in Advanced Settings', - async function (this: DifyWorld) { - await expectAgentEnvVariableVisible( - this, - agentBuilderFixedInputs.envPlainKey, - agentBuilderFixedInputs.envPlainValue, - ) - }, -) - -Then( - 'I should not see the supported E2E environment variable in Advanced Settings', - async function (this: DifyWorld) { - await expectAgentEnvVariableHidden(this, agentBuilderFixedInputs.envPlainKey) - }, -) - -Then( - 'I should see the Agent v2 environment variables from the invalid import in Advanced Settings', - async function (this: DifyWorld) { - const page = this.getPage() - const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' }) - - await page.getByRole('button', { name: 'Advanced Settings' }).first().click() - await expect(advancedSettings.getByRole('heading', { name: 'Env Editor' })).toBeVisible() - await expect.poll( - async () => advancedSettings.getByRole('textbox').evaluateAll(inputs => - inputs.map(input => (input as HTMLInputElement).value), - ), - { timeout: 30_000 }, - ).toEqual(expect.arrayContaining([ - agentBuilderFixedInputs.envPlainKey, - agentBuilderFixedInputs.envPlainValue, - agentBuilderFixedInputs.envAfterInvalidImportKey, - agentBuilderFixedInputs.envAfterInvalidImportValue, - ])) - await expect(page.getByRole('button', { name: /^Build$/i })).toBeVisible() - }, -) - -Then('I should see the Agent v2 Build draft pending changes', async function (this: DifyWorld) { - const page = this.getPage() - - await expect(page.getByText('Build draft')).toBeVisible({ timeout: 30_000 }) - await expect(page.getByRole('button', { name: 'Apply' })).toBeEnabled() - await expect(page.getByRole('button', { name: 'Discard' })).toBeEnabled() -}) - -Then('I should see the Agent v2 Build mode confirmation state', async function (this: DifyWorld) { - const page = this.getPage() - - await expect(page.getByText('Build mode', { exact: true })).toBeVisible() - await expect( - page.getByText('You\'re in build mode. Shape this setup through the chat on the right, then Apply.'), - ).toBeVisible() -}) - Then( 'the normal Agent v2 draft should still use the normal E2E prompt', async function (this: DifyWorld) { @@ -1222,89 +232,3 @@ Then( }) }, ) - -Then( - 'the Agent v2 draft should include the supported Build draft config', - async function (this: DifyWorld) { - await expect.poll( - async () => { - const agentSoul = (await getAgentComposerDraft(getCurrentAgentId(this))).agent_soul - const variables = agentSoul?.env?.variables ?? [] - - return { - envValue: getAgentEnvVariableValue(variables, agentBuilderFixedInputs.envPlainKey), - fileNames: agentSoul?.config_files?.map(file => file.name) ?? [], - prompt: agentSoul?.prompt, - skillNames: agentSoul?.config_skills?.map(skill => skill.name) ?? [], - } - }, - { timeout: 30_000 }, - ).toEqual({ - envValue: agentBuilderFixedInputs.envPlainValue, - fileNames: expect.arrayContaining([agentBuilderTestMaterials.smallFile]), - prompt: { system_prompt: updatedAgentPrompt }, - skillNames: expect.arrayContaining([agentBuilderPreseededResources.summarySkill]), - }) - }, -) - -Then( - 'the Agent v2 draft should not include the supported Build draft config', - async function (this: DifyWorld) { - await expect.poll( - async () => { - const agentSoul = (await getAgentComposerDraft(getCurrentAgentId(this))).agent_soul - const variables = agentSoul?.env?.variables ?? [] - - return { - envValue: getAgentEnvVariableValue(variables, agentBuilderFixedInputs.envPlainKey), - fileNames: agentSoul?.config_files?.map(file => file.name) ?? [], - prompt: agentSoul?.prompt, - skillNames: agentSoul?.config_skills?.map(skill => skill.name) ?? [], - } - }, - { timeout: 30_000 }, - ).toEqual({ - envValue: undefined, - fileNames: expect.not.arrayContaining([agentBuilderTestMaterials.smallFile]), - prompt: { system_prompt: normalAgentPrompt }, - skillNames: expect.not.arrayContaining([agentBuilderPreseededResources.summarySkill]), - }) - }, -) - -Then( - 'the Agent v2 draft should include one e2e-summary-skill Skill', - async function (this: DifyWorld) { - await expect.poll( - async () => { - const agentSoul = (await getAgentComposerDraft(getCurrentAgentId(this))).agent_soul - return agentSoul?.config_skills?.filter( - skill => skill.name === agentBuilderPreseededResources.summarySkill, - ).length ?? 0 - }, - { timeout: 30_000 }, - ).toBe(1) - }, -) - -Then('the Agent v2 Build draft should no longer be active', async function (this: DifyWorld) { - const page = this.getPage() - - await expect(page.getByText('Build draft')).not.toBeVisible() - await expect(page.getByRole('button', { name: 'Apply' })).not.toBeVisible() - await expect(page.getByRole('button', { name: 'Discard' })).not.toBeVisible() -}) - -Then('the Agent v2 configuration should be saved automatically', async function (this: DifyWorld) { - await waitForAgentConfigureAutosaved(this.getPage()) -}) - -Then('the Agent v2 draft should be published and up to date', async function (this: DifyWorld) { - const page = this.getPage() - const agentId = getCurrentAgentId(this) - - await expect(page.getByRole('button', { name: 'Published' })).toBeVisible({ timeout: 30_000 }) - await expect(page.getByText('Up to date')).toBeVisible() - await expect.poll(async () => (await getTestAgent(agentId)).active_config_is_published).toBe(true) -}) diff --git a/e2e/features/step-definitions/agent-v2/files.steps.ts b/e2e/features/step-definitions/agent-v2/files.steps.ts new file mode 100644 index 00000000000..8c77b59d4f3 --- /dev/null +++ b/e2e/features/step-definitions/agent-v2/files.steps.ts @@ -0,0 +1,67 @@ +import type { DifyWorld } from '../../support/world' +import { Then, When } from '@cucumber/cucumber' +import { expect } from '@playwright/test' +import { agentBuilderFileTreeFixtureFileNames } from '../../../support/test-materials' +import { + expectAgentConfigFileHidden, + expectAgentConfigFileSaved, + expectAgentConfigFileVisible, + uploadAgentConfigFile, +} from './configure-helpers' + +When('I upload the small Agent v2 file from the Files section', async function (this: DifyWorld) { + await uploadAgentConfigFile(this, 'smallFile') +}) + +When('I upload the special-name Agent v2 file from the Files section', async function (this: DifyWorld) { + await uploadAgentConfigFile(this, 'specialFilename') +}) + +Then( + 'I should see the Agent v2 file fixture entries in the current flat Files list', + async function (this: DifyWorld) { + const page = this.getPage() + const filesSection = page.getByRole('region', { name: 'Files' }) + const filesList = filesSection.getByLabel('Agent files') + + await expect(filesSection).toBeVisible({ timeout: 30_000 }) + await expect(filesList).toBeVisible() + + for (const fileName of agentBuilderFileTreeFixtureFileNames) { + await expect(filesList.getByRole('button', { + exact: true, + name: fileName, + })).toBeVisible() + } + + await expect(filesList.getByRole('button', { exact: true, name: 'assets' })).toHaveCount(0) + await expect(filesList.getByRole('button', { exact: true, name: 'docs' })).toHaveCount(0) + await expect(filesList.getByRole('button', { exact: true, name: 'public' })).toHaveCount(0) + await expect(filesList.getByRole('button', { exact: true, name: 'src' })).toHaveCount(0) + await expect(filesList.getByRole('button', { exact: true, name: 'web-game' })).toHaveCount(0) + }, +) +Then('I should see the small Agent v2 file in the Files section', async function (this: DifyWorld) { + await expectAgentConfigFileVisible(this, 'smallFile') +}) + +Then('I should not see the small Agent v2 file in the Files section', async function (this: DifyWorld) { + await expectAgentConfigFileHidden(this, 'smallFile') +}) + +Then('I should see the special-name Agent v2 file in the Files section', async function (this: DifyWorld) { + await expectAgentConfigFileVisible(this, 'specialFilename') +}) +Then( + 'the small Agent v2 file should be saved in the Agent v2 draft', + async function (this: DifyWorld) { + await expectAgentConfigFileSaved(this, 'smallFile') + }, +) + +Then( + 'the special-name Agent v2 file should be saved in the Agent v2 draft', + async function (this: DifyWorld) { + await expectAgentConfigFileSaved(this, 'specialFilename') + }, +) diff --git a/e2e/features/step-definitions/agent-v2/output-variables.steps.ts b/e2e/features/step-definitions/agent-v2/output-variables.steps.ts new file mode 100644 index 00000000000..b091ff46edf --- /dev/null +++ b/e2e/features/step-definitions/agent-v2/output-variables.steps.ts @@ -0,0 +1,15 @@ +import type { DifyWorld } from '../../support/world' +import { Then } from '@cucumber/cucumber' +import { expect } from '@playwright/test' +import { skipBlockedPrecondition } from '../../../support/preflight' + +Then('Agent v2 standalone Output Variables should be available', async function (this: DifyWorld) { + const page = this.getPage() + + await expect(page.getByRole('heading', { name: 'Configure' })).toBeVisible({ timeout: 30_000 }) + + return skipBlockedPrecondition( + this, + 'Standalone Agent Output Variables are not available: output variables currently belong to Workflow Agent v2 nodes.', + ) +}) diff --git a/e2e/features/step-definitions/agent-v2/publish.steps.ts b/e2e/features/step-definitions/agent-v2/publish.steps.ts new file mode 100644 index 00000000000..13699f6578a --- /dev/null +++ b/e2e/features/step-definitions/agent-v2/publish.steps.ts @@ -0,0 +1,27 @@ +import type { DifyWorld } from '../../support/world' +import { Then, When } from '@cucumber/cucumber' +import { expect } from '@playwright/test' +import { getTestAgent } from '../../../support/agent' +import { waitForAgentConfigureAutosaved } from '../../../support/agent-configure' +import { getCurrentAgentId } from './configure-helpers' + +When('I publish the Agent v2 draft', async function (this: DifyWorld) { + const page = this.getPage() + const publishButton = page.getByRole('button', { name: /^Publish(?: update)?$/ }) + + await expect(publishButton).toBeEnabled({ timeout: 30_000 }) + await publishButton.click() +}) + +Then('the Agent v2 configuration should be saved automatically', async function (this: DifyWorld) { + await waitForAgentConfigureAutosaved(this.getPage()) +}) + +Then('the Agent v2 draft should be published and up to date', async function (this: DifyWorld) { + const page = this.getPage() + const agentId = getCurrentAgentId(this) + + await expect(page.getByRole('button', { name: 'Published' })).toBeVisible({ timeout: 30_000 }) + await expect(page.getByText('Up to date')).toBeVisible() + await expect.poll(async () => (await getTestAgent(agentId)).active_config_is_published).toBe(true) +}) diff --git a/e2e/support/preflight.ts b/e2e/support/preflight.ts index 4fa6f6fa860..fa1640a966b 100644 --- a/e2e/support/preflight.ts +++ b/e2e/support/preflight.ts @@ -1,1131 +1 @@ -import type { DifyWorld } from '../features/support/world' -import { - agentBuilderExpectedTokens, - agentBuilderFixedInputs, - agentBuilderPreseededResources, -} from './agent-builder-resources' -import { createApiContext, expectApiResponseOK } from './api' -import { - agentBuilderFileTreeFixtureFileNames, - agentBuilderFileTreeFixtureFiles, - agentBuilderTestMaterials, -} from './test-materials' - -const stableChatModelProviderEnv = 'E2E_STABLE_MODEL_PROVIDER' -const stableChatModelNameEnv = 'E2E_STABLE_MODEL_NAME' -const stableChatModelTypeEnv = 'E2E_STABLE_MODEL_TYPE' -const brokenChatModelProviderEnv = 'E2E_BROKEN_MODEL_PROVIDER' -const brokenChatModelNameEnv = 'E2E_BROKEN_MODEL_NAME' -const brokenChatModelTypeEnv = 'E2E_BROKEN_MODEL_TYPE' -const activeModelStatus = 'active' -const defaultStableChatModelType = 'llm' -const defaultBrokenChatModelName = agentBuilderPreseededResources.brokenModel - -export type E2EResourcePrecondition - = | { - ok: true - value: string - } - | { - ok: false - reason: string - } - -export const readRequiredEnvResource = ( - envName: string, - description: string, -): E2EResourcePrecondition => { - const value = process.env[envName]?.trim() - if (value) - return { ok: true, value } - - return { - ok: false, - reason: `${description} requires ${envName}.`, - } -} - -export function skipBlockedPrecondition(world: DifyWorld, reason: string): 'skipped' { - const message = `Blocked precondition: ${reason}` - console.warn(`[e2e] ${message}`) - world.attach(message, 'text/plain') - return 'skipped' -} - -export function skipMissingEnvResource( - world: DifyWorld, - envName: string, - description: string, -): 'skipped' | string { - const resource = readRequiredEnvResource(envName, description) - if (resource.ok) - return resource.value - - return skipBlockedPrecondition(world, resource.reason) -} - -export const requiredAgentBuilderPreseededResources = Object.values(agentBuilderPreseededResources) - -export function skipMissingAgentBuilderPreseed( - world: DifyWorld, - resourceName: string, - envName: string, -): 'skipped' | string { - return skipMissingEnvResource( - world, - envName, - `Preseeded Agent Builder resource "${resourceName}"`, - ) -} - -type ModelTypeListResponse = { - data: Array<{ - provider: string - models: Array<{ - label?: { - en_US?: string - zh_Hans?: string - } - model: string - status?: string - }> - status?: string - }> -} - -type NamedResource = { - id: string - name: string -} - -type DatasetResource = NamedResource & { - document_count: number - total_available_documents: number -} - -type NamedResourceListResponse = { - data: T[] -} - -type DocumentIndexingStatus - = | 'cleaning' - | 'completed' - | 'indexing' - | 'parsing' - | 'splitting' - | 'waiting' - -type DatasetIndexingStatusResponse = { - data: Array<{ - id: string - indexing_status?: string - }> -} - -const completedDocumentIndexingStatus: DocumentIndexingStatus = 'completed' -const activeDocumentIndexingStatuses = new Set([ - 'cleaning', - 'indexing', - 'parsing', - 'splitting', - 'waiting', -]) - -type LocalizedLabel = { - en_US?: string - zh_Hans?: string -} - -type BuiltinToolProvider = { - label?: LocalizedLabel - name: string - tools: Array<{ - label?: LocalizedLabel - name: string - }> -} - -type AgentDriveSkillListResponse = { - items: Array<{ - name: string - path: string - }> -} - -type AgentDriveFileListResponse = { - items?: Array<{ - key: string - }> -} - -type AgentComposerResponse = { - agent_soul?: Record -} - -type AgentApiAccessResponse = { - api_key_count: number - enabled: boolean -} - -type AgentApiKeyListResponse = { - data: Array<{ - id: string - }> -} - -type AgentReferencingWorkflowsResponse = { - data: Array<{ - app_id: string - app_name: string - node_ids?: string[] - }> -} - -type PreseededAgentDetailResponse = { - active_config_is_published?: boolean - enable_site?: boolean - site?: { - access_token?: string | null - app_base_url?: string | null - code?: string | null - } | null -} - -const findConsoleResourceByName = async ({ - action, - path, - resourceName, -}: { - action: string - path: string - resourceName: string -}) => { - const ctx = await createApiContext() - try { - const response = await ctx.get(path) - await expectApiResponseOK(response, action) - const body = (await response.json()) as NamedResourceListResponse - - return body.data.find(item => item.name === resourceName) - } - finally { - await ctx.dispose() - } -} - -const buildQuery = (params: Record) => new URLSearchParams(params).toString() - -const matchesNameOrLabel = (value: string, name: string, label?: LocalizedLabel) => - value === name || value === label?.en_US || value === label?.zh_Hans - -const isRecord = (value: unknown): value is Record => - typeof value === 'object' && value !== null && !Array.isArray(value) - -const asRecord = (value: unknown): Record => (isRecord(value) ? value : {}) - -const asArray = (value: unknown): unknown[] => (Array.isArray(value) ? value : []) - -const asString = (value: unknown) => (typeof value === 'string' ? value : '') - -const hasNamedOrKeyedEntry = (items: unknown[], expectedName: string) => - items.some((item) => { - const record = asRecord(item) - const values = [record.name, record.drive_key, record.reference, record.file_id, record.id].map( - asString, - ) - - return values.some(value => value === expectedName || value.endsWith(`/${expectedName}`)) - }) - -const findToolEntry = ( - items: unknown[], - { - providerDisplayName, - providerName, - toolDisplayName, - toolName, - }: { - providerDisplayName: string - providerName: string - toolDisplayName: string - toolName: string - }, -) => - items.find((item) => { - const record = asRecord(item) - const providerValues = [record.provider_id, record.provider, record.plugin_id, record.name].map( - asString, - ) - const toolValues = [record.tool_name, record.name].map(asString) - - return ( - providerValues.some(value => value === providerName || value === providerDisplayName) - && toolValues.some(value => value === toolName || value === toolDisplayName) - ) - }) - -const hasToolEntry = ( - items: unknown[], - tool: { - providerDisplayName: string - providerName: string - toolDisplayName: string - toolName: string - }, -) => Boolean(findToolEntry(items, tool)) - -const hasUnauthorizedToolCredentialState = (item: unknown) => { - const record = asRecord(item) - - return asString(record.credential_type) === 'unauthorized' -} - -const hasKnowledgeDataset = ( - soul: Record, - dataset: NonNullable, -) => { - const knowledge = asRecord(soul.knowledge) - const sets = asArray(knowledge.sets) - - return sets.some((set) => { - const datasets = asArray(asRecord(set).datasets) - - return datasets.some((item) => { - const record = asRecord(item) - return record.id === dataset.id || record.name === dataset.name - }) - }) -} - -const hasKnowledgeSet = ( - soul: Record, - dataset: NonNullable, - { - queryMode, - queryValue, - }: { - queryMode: 'generated_query' | 'user_query' - queryValue?: string - }, -) => { - const knowledge = asRecord(soul.knowledge) - const sets = asArray(knowledge.sets) - - return sets.some((set) => { - const record = asRecord(set) - const query = asRecord(record.query) - const datasets = asArray(record.datasets) - const hasExpectedDataset = datasets.some((item) => { - const datasetRecord = asRecord(item) - return datasetRecord.id === dataset.id || datasetRecord.name === dataset.name - }) - - if (!hasExpectedDataset || query.mode !== queryMode) - return false - if (queryValue === undefined) - return true - - return asString(query.value).trim() === queryValue - }) -} - -const getPreseededDataset = async (resourceName: string) => { - const query = buildQuery({ keyword: resourceName, limit: '20', page: '1' }) - - return findConsoleResourceByName({ - action: `Check preseeded dataset ${resourceName}`, - path: `/console/api/datasets?${query}`, - resourceName, - }) -} - -const getDatasetIndexingStatuses = async (datasetId: string, resourceName: string) => { - const ctx = await createApiContext() - try { - const response = await ctx.get(`/console/api/datasets/${datasetId}/indexing-status`) - await expectApiResponseOK(response, `Check preseeded dataset indexing status ${resourceName}`) - const body = (await response.json()) as DatasetIndexingStatusResponse - - return body.data - } - finally { - await ctx.dispose() - } -} - -const toDatasetResource = ( - resource: NamedResource, -): NonNullable => ({ - id: resource.id, - kind: 'dataset', - name: resource.name, -}) - -const splitToolDisplayName = (resourceName: string) => { - const [providerName, toolName] = resourceName.split('/').map(item => item.trim()) - - if (!providerName || !toolName) { - return { - ok: false as const, - reason: `Preseeded tool "${resourceName}" must use "Provider / Tool" format.`, - } - } - - return { - ok: true as const, - providerName, - toolName, - } -} - -export async function skipMissingPreseededAgent( - world: DifyWorld, - resourceName: string, -): Promise<'skipped' | NonNullable> { - const query = buildQuery({ limit: '20', name: resourceName, page: '1' }) - const resource = await findConsoleResourceByName({ - action: `Check preseeded Agent ${resourceName}`, - path: `/console/api/agent?${query}`, - resourceName, - }) - - if (!resource) - return skipBlockedPrecondition(world, `Preseeded Agent "${resourceName}" was not found.`) - - return { - id: resource.id, - kind: 'agent', - name: resource.name, - } -} - -export async function skipMissingPreseededWorkflow( - world: DifyWorld, - resourceName: string, -): Promise<'skipped' | NonNullable> { - const query = buildQuery({ limit: '20', mode: 'workflow', name: resourceName, page: '1' }) - const resource = await findConsoleResourceByName({ - action: `Check preseeded workflow ${resourceName}`, - path: `/console/api/apps?${query}`, - resourceName, - }) - - if (!resource) - return skipBlockedPrecondition(world, `Preseeded workflow "${resourceName}" was not found.`) - - return { - id: resource.id, - kind: 'workflow', - name: resource.name, - } -} - -export async function skipMissingPreseededDataset( - world: DifyWorld, - resourceName: string, -): Promise<'skipped' | NonNullable> { - const resource = await getPreseededDataset(resourceName) - - if (!resource) - return skipBlockedPrecondition(world, `Preseeded dataset "${resourceName}" was not found.`) - - return toDatasetResource(resource) -} - -export async function skipMissingReadyPreseededDataset( - world: DifyWorld, - resourceName: string, -): Promise<'skipped' | NonNullable> { - const resource = await getPreseededDataset(resourceName) - - if (!resource) - return skipBlockedPrecondition(world, `Preseeded dataset "${resourceName}" was not found.`) - - if (resource.document_count < 1) { - return skipBlockedPrecondition(world, `Preseeded dataset "${resourceName}" has no documents.`) - } - - if (resource.total_available_documents !== resource.document_count) { - return skipBlockedPrecondition( - world, - `Preseeded dataset "${resourceName}" has ${resource.total_available_documents}/${resource.document_count} available documents.`, - ) - } - - const statuses = await getDatasetIndexingStatuses(resource.id, resourceName) - if (statuses.length < 1) { - return skipBlockedPrecondition( - world, - `Preseeded dataset "${resourceName}" has no document indexing status.`, - ) - } - - const incompleteStatus = statuses.find( - item => item.indexing_status !== completedDocumentIndexingStatus, - ) - if (incompleteStatus) { - return skipBlockedPrecondition( - world, - `Preseeded dataset "${resourceName}" includes document ${incompleteStatus.id} with indexing status "${incompleteStatus.indexing_status ?? 'missing'}".`, - ) - } - - return toDatasetResource(resource) -} - -export async function skipMissingIndexingPreseededDataset( - world: DifyWorld, - resourceName: string, -): Promise<'skipped' | NonNullable> { - const resource = await getPreseededDataset(resourceName) - - if (!resource) - return skipBlockedPrecondition(world, `Preseeded dataset "${resourceName}" was not found.`) - - const statuses = await getDatasetIndexingStatuses(resource.id, resourceName) - const indexingStatus = statuses.find(item => - activeDocumentIndexingStatuses.has(item.indexing_status ?? ''), - ) - - if (!indexingStatus) { - const actualStatuses - = statuses.map(item => item.indexing_status ?? 'missing').join(', ') || 'none' - - return skipBlockedPrecondition( - world, - `Preseeded dataset "${resourceName}" is not indexing or queued; document statuses: ${actualStatuses}.`, - ) - } - - return toDatasetResource(resource) -} - -export async function skipMissingPreseededTool( - world: DifyWorld, - resourceName: string, -): Promise<'skipped' | NonNullable> { - const parsed = splitToolDisplayName(resourceName) - if (!parsed.ok) - return skipBlockedPrecondition(world, parsed.reason) - - const ctx = await createApiContext() - try { - const response = await ctx.get('/console/api/workspaces/current/tools/builtin') - await expectApiResponseOK(response, `Check preseeded tool ${resourceName}`) - const providers = (await response.json()) as BuiltinToolProvider[] - const provider = providers.find(item => - matchesNameOrLabel(parsed.providerName, item.name, item.label), - ) - const tool = provider?.tools.find(item => - matchesNameOrLabel(parsed.toolName, item.name, item.label), - ) - - if (!provider || !tool) - return skipBlockedPrecondition(world, `Preseeded tool "${resourceName}" was not found.`) - - return { - id: `${provider.name}/${tool.name}`, - kind: 'tool', - name: resourceName, - } - } - finally { - await ctx.dispose() - } -} - -export async function skipMissingPreseededAgentDriveSkill( - world: DifyWorld, - agentName: string, - skillName: string, -): Promise<'skipped' | NonNullable> { - const agent = await skipMissingPreseededAgent(world, agentName) - if (agent === 'skipped') - return agent - - const ctx = await createApiContext() - try { - const response = await ctx.get(`/console/api/agent/${agent.id}/drive/skills`) - await expectApiResponseOK(response, `Check preseeded Agent skill ${skillName}`) - const body = (await response.json()) as AgentDriveSkillListResponse - const skill = body.items.find(item => item.name === skillName) - - if (!skill) { - return skipBlockedPrecondition( - world, - `Preseeded Agent "${agentName}" does not include drive skill "${skillName}".`, - ) - } - - return { - id: skill.path, - kind: 'skill', - name: skill.name, - } - } - finally { - await ctx.dispose() - } -} - -export async function skipMissingPreseededFullConfigAgentCoreConfiguration( - world: DifyWorld, - agentName: string, -): Promise<'skipped' | NonNullable> { - const stableModel = await skipMissingAgentBuilderStableChatModel(world) - if (stableModel === 'skipped') - return stableModel - - const agent = await skipMissingPreseededAgent(world, agentName) - if (agent === 'skipped') - return agent - - const summarySkill = await skipMissingPreseededAgentDriveSkill( - world, - agentName, - agentBuilderPreseededResources.summarySkill, - ) - if (summarySkill === 'skipped') - return summarySkill - - const jsonTool = await skipMissingPreseededTool( - world, - agentBuilderPreseededResources.jsonReplaceTool, - ) - if (jsonTool === 'skipped') - return jsonTool - - const knowledgeBase = await skipMissingReadyPreseededDataset( - world, - agentBuilderPreseededResources.agentKnowledgeBase, - ) - if (knowledgeBase === 'skipped') - return knowledgeBase - - const ctx = await createApiContext() - try { - const response = await ctx.get(`/console/api/agent/${agent.id}/composer`) - await expectApiResponseOK(response, `Check preseeded Agent core configuration ${agentName}`) - const body = (await response.json()) as AgentComposerResponse - const soul = body.agent_soul ?? {} - const missing: string[] = [] - - const model = asRecord(soul.model) - if (model.model_provider !== stableModel.provider || model.model !== stableModel.name) - missing.push(`${agentBuilderPreseededResources.stableChatModel} model config`) - - const prompt = asString(asRecord(soul.prompt).system_prompt) - if (!prompt.includes(agentBuilderExpectedTokens.agentReply)) - missing.push(`Prompt token ${agentBuilderExpectedTokens.agentReply}`) - - const files = asArray(asRecord(soul.files).files) - for (const fileName of [ - agentBuilderTestMaterials.smallFile, - agentBuilderTestMaterials.specialFilename, - ]) { - if (!hasNamedOrKeyedEntry(files, fileName)) - missing.push(`file ${fileName}`) - } - - const [providerName = '', toolName = ''] = jsonTool.id.split('/') - const parsedTool = splitToolDisplayName(agentBuilderPreseededResources.jsonReplaceTool) - if ( - parsedTool.ok - && !hasToolEntry(asArray(asRecord(soul.tools).dify_tools), { - providerDisplayName: parsedTool.providerName, - providerName, - toolDisplayName: parsedTool.toolName, - toolName, - }) - ) { - missing.push(agentBuilderPreseededResources.jsonReplaceTool) - } - - if (!hasKnowledgeDataset(soul, knowledgeBase)) - missing.push(agentBuilderPreseededResources.agentKnowledgeBase) - - if (missing.length > 0) { - return skipBlockedPrecondition( - world, - `Preseeded Agent "${agentName}" is missing core fixture configuration: ${missing.join(', ')}.`, - ) - } - - return agent - } - finally { - await ctx.dispose() - } -} - -export async function skipMissingPreseededToolStatesAgentConfiguration( - world: DifyWorld, - agentName: string, -): Promise<'skipped' | NonNullable> { - const agent = await skipMissingPreseededAgent(world, agentName) - if (agent === 'skipped') - return agent - - const summarySkill = await skipMissingPreseededAgentDriveSkill( - world, - agentName, - agentBuilderPreseededResources.summarySkill, - ) - if (summarySkill === 'skipped') - return summarySkill - - const jsonTool = await skipMissingPreseededTool( - world, - agentBuilderPreseededResources.jsonReplaceTool, - ) - if (jsonTool === 'skipped') - return jsonTool - - const tavilyTool = await skipMissingPreseededTool( - world, - agentBuilderPreseededResources.tavilySearchTool, - ) - if (tavilyTool === 'skipped') - return tavilyTool - - const ctx = await createApiContext() - try { - const response = await ctx.get(`/console/api/agent/${agent.id}/composer`) - await expectApiResponseOK(response, `Check preseeded Agent tool states ${agentName}`) - const body = (await response.json()) as AgentComposerResponse - const soul = body.agent_soul ?? {} - const toolItems = asArray(asRecord(soul.tools).dify_tools) - const missing: string[] = [] - - const [jsonProviderName = '', jsonToolName = ''] = jsonTool.id.split('/') - const parsedJsonTool = splitToolDisplayName(agentBuilderPreseededResources.jsonReplaceTool) - if ( - parsedJsonTool.ok - && !findToolEntry(toolItems, { - providerDisplayName: parsedJsonTool.providerName, - providerName: jsonProviderName, - toolDisplayName: parsedJsonTool.toolName, - toolName: jsonToolName, - }) - ) { - missing.push(agentBuilderPreseededResources.jsonReplaceTool) - } - - const [tavilyProviderName = '', tavilyToolName = ''] = tavilyTool.id.split('/') - const parsedTavilyTool = splitToolDisplayName(agentBuilderPreseededResources.tavilySearchTool) - const tavilyEntry = parsedTavilyTool.ok - ? findToolEntry(toolItems, { - providerDisplayName: parsedTavilyTool.providerName, - providerName: tavilyProviderName, - toolDisplayName: parsedTavilyTool.toolName, - toolName: tavilyToolName, - }) - : undefined - - if (!tavilyEntry) { - missing.push(agentBuilderPreseededResources.tavilySearchTool) - } - else if (!hasUnauthorizedToolCredentialState(tavilyEntry)) { - missing.push(`${agentBuilderPreseededResources.tavilySearchTool} unauthorized credential state`) - } - - if (missing.length > 0) { - return skipBlockedPrecondition( - world, - `Preseeded Agent "${agentName}" is missing tool state fixture configuration: ${missing.join(', ')}.`, - ) - } - - return agent - } - finally { - await ctx.dispose() - } -} - -export async function skipMissingPreseededDualRetrievalAgentConfiguration( - world: DifyWorld, - agentName: string, -): Promise<'skipped' | NonNullable> { - const agent = await skipMissingPreseededAgent(world, agentName) - if (agent === 'skipped') - return agent - - const knowledgeBase = await skipMissingReadyPreseededDataset( - world, - agentBuilderPreseededResources.agentKnowledgeBase, - ) - if (knowledgeBase === 'skipped') - return knowledgeBase - - const ctx = await createApiContext() - try { - const response = await ctx.get(`/console/api/agent/${agent.id}/composer`) - await expectApiResponseOK(response, `Check preseeded Agent dual retrieval ${agentName}`) - const body = (await response.json()) as AgentComposerResponse - const soul = body.agent_soul ?? {} - const missing: string[] = [] - - if (!hasKnowledgeSet(soul, knowledgeBase, { queryMode: 'generated_query' })) - missing.push('Agent decide Knowledge Retrieval') - - if ( - !hasKnowledgeSet(soul, knowledgeBase, { - queryMode: 'user_query', - queryValue: agentBuilderFixedInputs.customKnowledgeQuery, - }) - ) { - missing.push('Custom query Knowledge Retrieval') - } - - if (missing.length > 0) { - return skipBlockedPrecondition( - world, - `Preseeded Agent "${agentName}" is missing dual retrieval fixture configuration: ${missing.join(', ')}.`, - ) - } - - return agent - } - finally { - await ctx.dispose() - } -} - -export async function skipMissingPreseededAgentFileTreeFixture( - world: DifyWorld, - agentName: string, -): Promise<'skipped' | NonNullable> { - const agent = await skipMissingPreseededAgent(world, agentName) - if (agent === 'skipped') - return agent - - const ctx = await createApiContext() - try { - const query = buildQuery({ prefix: 'files/' }) - const response = await ctx.get(`/console/api/agent/${agent.id}/drive/files?${query}`) - await expectApiResponseOK(response, `Check preseeded Agent file tree ${agentName}`) - const body = (await response.json()) as AgentDriveFileListResponse - const keys = (body.items ?? []).map(item => item.key) - const missingFiles = agentBuilderFileTreeFixtureFiles.filter( - filePath => - !keys.some(key => key === `files/${filePath}` || key.endsWith(`/${filePath}`)), - ) - - if (missingFiles.length > 0) { - return skipBlockedPrecondition( - world, - `Preseeded Agent "${agentName}" is missing file tree fixture files: ${missingFiles.join(', ')}.`, - ) - } - - return { - id: agent.id, - kind: 'agent', - name: agent.name, - } - } - finally { - await ctx.dispose() - } -} - -export async function skipMissingPreseededAgentFlatFileFixtureConfiguration( - world: DifyWorld, - agentName: string, -): Promise<'skipped' | NonNullable> { - const agent = await skipMissingPreseededAgent(world, agentName) - if (agent === 'skipped') - return agent - - const ctx = await createApiContext() - try { - const response = await ctx.get(`/console/api/agent/${agent.id}/composer`) - await expectApiResponseOK(response, `Check preseeded Agent flat file fixture ${agentName}`) - const body = (await response.json()) as AgentComposerResponse - const configFiles = Array.isArray(body.agent_soul?.config_files) - ? body.agent_soul.config_files - : [] - const fileNames = configFiles - .map(file => (typeof file === 'object' && file !== null && 'name' in file ? file.name : undefined)) - .filter((name): name is string => typeof name === 'string') - const missingFiles = agentBuilderFileTreeFixtureFileNames.filter(fileName => !fileNames.includes(fileName)) - - if (missingFiles.length > 0) { - return skipBlockedPrecondition( - world, - `Preseeded Agent "${agentName}" is missing current flat Files fixture configuration: ${missingFiles.join(', ')}. Hierarchical Files display remains blocked until Agent config files support tree paths.`, - ) - } - - return { - id: agent.id, - kind: 'agent', - name: agent.name, - } - } - finally { - await ctx.dispose() - } -} - -export async function skipMissingPreseededAgentBackendApiKey( - world: DifyWorld, - agentName: string, -): Promise<'skipped' | NonNullable> { - const agent = await skipMissingPreseededAgent(world, agentName) - if (agent === 'skipped') - return agent - - const ctx = await createApiContext() - try { - const accessResponse = await ctx.get(`/console/api/agent/${agent.id}/api-access`) - await expectApiResponseOK(accessResponse, `Check preseeded Agent API access ${agentName}`) - const access = (await accessResponse.json()) as AgentApiAccessResponse - if (!access.enabled || access.api_key_count < 1) { - return skipBlockedPrecondition( - world, - `Preseeded Agent "${agentName}" does not have Backend service API enabled with an API key.`, - ) - } - - const keyResponse = await ctx.get(`/console/api/agent/${agent.id}/api-keys`) - await expectApiResponseOK(keyResponse, `Check preseeded Agent API key ${agentName}`) - const keys = (await keyResponse.json()) as AgentApiKeyListResponse - const key = keys.data.at(0) - if (!key) { - return skipBlockedPrecondition( - world, - `Preseeded Agent "${agentName}" Backend service API key list is empty.`, - ) - } - - return { - id: key.id, - kind: 'api-key', - name: `${agentName} Backend service API key`, - } - } - finally { - await ctx.dispose() - } -} - -export async function skipMissingPreseededAgentPublishedWebApp( - world: DifyWorld, - agentName: string, -): Promise<'skipped' | NonNullable> { - const agent = await skipMissingPreseededAgent(world, agentName) - if (agent === 'skipped') - return agent - - const ctx = await createApiContext() - try { - const response = await ctx.get(`/console/api/agent/${agent.id}`) - await expectApiResponseOK(response, `Check preseeded Agent published Web app ${agentName}`) - const detail = (await response.json()) as PreseededAgentDetailResponse - if (detail.active_config_is_published !== true) { - return skipBlockedPrecondition(world, `Preseeded Agent "${agentName}" is not published.`) - } - - if (detail.enable_site !== true) { - return skipBlockedPrecondition( - world, - `Preseeded Agent "${agentName}" Web app is not enabled.`, - ) - } - - const siteToken = detail.site?.access_token ?? detail.site?.code - if (!siteToken || !detail.site?.app_base_url) { - return skipBlockedPrecondition( - world, - `Preseeded Agent "${agentName}" Web app URL is not available.`, - ) - } - - return { - id: agent.id, - kind: 'agent', - name: agent.name, - } - } - finally { - await ctx.dispose() - } -} - -export async function skipMissingPreseededAgentWorkflowReference( - world: DifyWorld, - agentName: string, - workflowName: string, -): Promise<'skipped' | NonNullable> { - const agent = await skipMissingPreseededAgent(world, agentName) - if (agent === 'skipped') - return agent - - const workflow = await skipMissingPreseededWorkflow(world, workflowName) - if (workflow === 'skipped') - return workflow - - const ctx = await createApiContext() - try { - const response = await ctx.get(`/console/api/agent/${agent.id}/referencing-workflows`) - await expectApiResponseOK(response, `Check preseeded Agent workflow reference ${agentName}`) - const references = (await response.json()) as AgentReferencingWorkflowsResponse - const reference = references.data.find( - item => item.app_id === workflow.id || item.app_name === workflow.name, - ) - - if (!reference) { - return skipBlockedPrecondition( - world, - `Preseeded Agent "${agentName}" is not referenced by workflow "${workflowName}".`, - ) - } - - if (!reference.node_ids || reference.node_ids.length < 1) { - return skipBlockedPrecondition( - world, - `Preseeded workflow "${workflowName}" does not expose Agent reference nodes for "${agentName}".`, - ) - } - - return { - id: workflow.id, - kind: 'workflow', - name: workflow.name, - } - } - finally { - await ctx.dispose() - } -} - -type ModelPreflightConfig - = | { - ok: true - provider: string - resourceName: string - type: string - value: string - } - | { - ok: false - reason: string - } - -export function readAgentBuilderStableChatModelConfig(): ModelPreflightConfig { - const provider = process.env[stableChatModelProviderEnv]?.trim() - const name = process.env[stableChatModelNameEnv]?.trim() - const type = process.env[stableChatModelTypeEnv]?.trim() || defaultStableChatModelType - - const missing: string[] = [] - if (!provider) - missing.push(stableChatModelProviderEnv) - if (!name) - missing.push(stableChatModelNameEnv) - - if (!provider || !name) { - return { - ok: false, - reason: `${agentBuilderPreseededResources.stableChatModel} requires ${missing.join(', ')}.`, - } - } - - return { - ok: true, - provider, - resourceName: agentBuilderPreseededResources.stableChatModel, - type, - value: name, - } -} - -export function readAgentBuilderBrokenChatModelConfig(): ModelPreflightConfig { - const provider = process.env[brokenChatModelProviderEnv]?.trim() - const name = process.env[brokenChatModelNameEnv]?.trim() || defaultBrokenChatModelName - const type = process.env[brokenChatModelTypeEnv]?.trim() || defaultStableChatModelType - - if (!provider) { - return { - ok: false, - reason: `${agentBuilderPreseededResources.brokenModelProvider} requires ${brokenChatModelProviderEnv}.`, - } - } - - return { - ok: true, - provider, - resourceName: agentBuilderPreseededResources.brokenModelProvider, - type, - value: name, - } -} - -async function skipMissingAgentBuilderModel( - world: DifyWorld, - config: ModelPreflightConfig, - { - requireActive, - }: { - requireActive: boolean - }, -): Promise<'skipped' | NonNullable> { - if (!config.ok) - return skipBlockedPrecondition(world, config.reason) - - const ctx = await createApiContext() - try { - const response = await ctx.get( - `/console/api/workspaces/current/models/model-types/${config.type}`, - ) - await expectApiResponseOK(response, `Check ${config.resourceName}`) - const body = (await response.json()) as ModelTypeListResponse - const provider = body.data.find(item => item.provider === config.provider) - const model = provider?.models.find( - item => - item.model === config.value - || item.label?.en_US === config.value - || item.label?.zh_Hans === config.value, - ) - - if (!provider || !model) { - return skipBlockedPrecondition( - world, - `${config.resourceName} was not found as ${config.provider}/${config.value} (${config.type}).`, - ) - } - - if (requireActive && model.status !== activeModelStatus) { - return skipBlockedPrecondition( - world, - `${config.resourceName} is ${model.status ?? 'missing status'} instead of ${activeModelStatus}.`, - ) - } - - return { - name: model.model, - provider: provider.provider, - type: config.type, - } - } - finally { - await ctx.dispose() - } -} - -export async function skipMissingAgentBuilderStableChatModel( - world: DifyWorld, -): Promise<'skipped' | NonNullable> { - return skipMissingAgentBuilderModel(world, readAgentBuilderStableChatModelConfig(), { - requireActive: true, - }) -} - -export async function skipMissingAgentBuilderBrokenChatModel( - world: DifyWorld, -): Promise<'skipped' | NonNullable> { - return skipMissingAgentBuilderModel(world, readAgentBuilderBrokenChatModelConfig(), { - requireActive: false, - }) -} +export * from './preflight/index' diff --git a/e2e/support/preflight/access.ts b/e2e/support/preflight/access.ts new file mode 100644 index 00000000000..e04be75ec79 --- /dev/null +++ b/e2e/support/preflight/access.ts @@ -0,0 +1,166 @@ +import type { DifyWorld } from '../../features/support/world' +import type { PreseededResource } from './common' +import { createApiContext, expectApiResponseOK } from '../api' +import { skipMissingPreseededAgent, skipMissingPreseededWorkflow } from './agents' +import { skipBlockedPrecondition } from './common' + +type AgentApiAccessResponse = { + api_key_count: number + enabled: boolean +} + +type AgentApiKeyListResponse = { + data: Array<{ + id: string + }> +} + +type AgentReferencingWorkflowsResponse = { + data: Array<{ + app_id: string + app_name: string + node_ids?: string[] + }> +} + +type PreseededAgentDetailResponse = { + active_config_is_published?: boolean + enable_site?: boolean + site?: { + access_token?: string | null + app_base_url?: string | null + code?: string | null + } | null +} + +export async function skipMissingPreseededAgentBackendApiKey( + world: DifyWorld, + agentName: string, +): Promise<'skipped' | PreseededResource> { + const agent = await skipMissingPreseededAgent(world, agentName) + if (agent === 'skipped') + return agent + + const ctx = await createApiContext() + try { + const accessResponse = await ctx.get(`/console/api/agent/${agent.id}/api-access`) + await expectApiResponseOK(accessResponse, `Check preseeded Agent API access ${agentName}`) + const access = (await accessResponse.json()) as AgentApiAccessResponse + if (!access.enabled || access.api_key_count < 1) { + return skipBlockedPrecondition( + world, + `Preseeded Agent "${agentName}" does not have Backend service API enabled with an API key.`, + ) + } + + const keyResponse = await ctx.get(`/console/api/agent/${agent.id}/api-keys`) + await expectApiResponseOK(keyResponse, `Check preseeded Agent API key ${agentName}`) + const keys = (await keyResponse.json()) as AgentApiKeyListResponse + const key = keys.data.at(0) + if (!key) { + return skipBlockedPrecondition( + world, + `Preseeded Agent "${agentName}" Backend service API key list is empty.`, + ) + } + + return { + id: key.id, + kind: 'api-key', + name: `${agentName} Backend service API key`, + } + } + finally { + await ctx.dispose() + } +} + +export async function skipMissingPreseededAgentPublishedWebApp( + world: DifyWorld, + agentName: string, +): Promise<'skipped' | PreseededResource> { + const agent = await skipMissingPreseededAgent(world, agentName) + if (agent === 'skipped') + return agent + + const ctx = await createApiContext() + try { + const response = await ctx.get(`/console/api/agent/${agent.id}`) + await expectApiResponseOK(response, `Check preseeded Agent published Web app ${agentName}`) + const detail = (await response.json()) as PreseededAgentDetailResponse + if (detail.active_config_is_published !== true) { + return skipBlockedPrecondition(world, `Preseeded Agent "${agentName}" is not published.`) + } + + if (detail.enable_site !== true) { + return skipBlockedPrecondition( + world, + `Preseeded Agent "${agentName}" Web app is not enabled.`, + ) + } + + const siteToken = detail.site?.access_token ?? detail.site?.code + if (!siteToken || !detail.site?.app_base_url) { + return skipBlockedPrecondition( + world, + `Preseeded Agent "${agentName}" Web app URL is not available.`, + ) + } + + return { + id: agent.id, + kind: 'agent', + name: agent.name, + } + } + finally { + await ctx.dispose() + } +} + +export async function skipMissingPreseededAgentWorkflowReference( + world: DifyWorld, + agentName: string, + workflowName: string, +): Promise<'skipped' | PreseededResource> { + const agent = await skipMissingPreseededAgent(world, agentName) + if (agent === 'skipped') + return agent + + const workflow = await skipMissingPreseededWorkflow(world, workflowName) + if (workflow === 'skipped') + return workflow + + const ctx = await createApiContext() + try { + const response = await ctx.get(`/console/api/agent/${agent.id}/referencing-workflows`) + await expectApiResponseOK(response, `Check preseeded Agent workflow reference ${agentName}`) + const references = (await response.json()) as AgentReferencingWorkflowsResponse + const reference = references.data.find( + item => item.app_id === workflow.id || item.app_name === workflow.name, + ) + + if (!reference) { + return skipBlockedPrecondition( + world, + `Preseeded Agent "${agentName}" is not referenced by workflow "${workflowName}".`, + ) + } + + if (!reference.node_ids || reference.node_ids.length < 1) { + return skipBlockedPrecondition( + world, + `Preseeded workflow "${workflowName}" does not expose Agent reference nodes for "${agentName}".`, + ) + } + + return { + id: workflow.id, + kind: 'workflow', + name: workflow.name, + } + } + finally { + await ctx.dispose() + } +} diff --git a/e2e/support/preflight/agents.ts b/e2e/support/preflight/agents.ts new file mode 100644 index 00000000000..90bdde7a12e --- /dev/null +++ b/e2e/support/preflight/agents.ts @@ -0,0 +1,472 @@ +import type { DifyWorld } from '../../features/support/world' +import type { AgentComposerResponse, PreseededResource } from './common' +import { + agentBuilderExpectedTokens, + agentBuilderFixedInputs, + agentBuilderPreseededResources, +} from '../agent-builder-resources' +import { createApiContext, expectApiResponseOK } from '../api' +import { + agentBuilderFileTreeFixtureFileNames, + agentBuilderFileTreeFixtureFiles, + agentBuilderTestMaterials, +} from '../test-materials' +import { + + asArray, + asRecord, + asString, + buildQuery, + findConsoleResourceByName, + hasNamedOrKeyedEntry, + + skipBlockedPrecondition, +} from './common' +import { skipMissingReadyPreseededDataset } from './datasets' +import { skipMissingAgentBuilderStableChatModel } from './models' +import { + findToolEntry, + hasToolEntry, + hasUnauthorizedToolCredentialState, + skipMissingPreseededTool, + splitToolDisplayName, +} from './tools' + +type AgentDriveSkillListResponse = { + items: Array<{ + name: string + path: string + }> +} + +type AgentDriveFileListResponse = { + items?: Array<{ + key: string + }> +} + +const hasKnowledgeDataset = ( + soul: Record, + dataset: PreseededResource, +) => { + const knowledge = asRecord(soul.knowledge) + const sets = asArray(knowledge.sets) + + return sets.some((set) => { + const datasets = asArray(asRecord(set).datasets) + + return datasets.some((item) => { + const record = asRecord(item) + return record.id === dataset.id || record.name === dataset.name + }) + }) +} + +const hasKnowledgeSet = ( + soul: Record, + dataset: PreseededResource, + { + queryMode, + queryValue, + }: { + queryMode: 'generated_query' | 'user_query' + queryValue?: string + }, +) => { + const knowledge = asRecord(soul.knowledge) + const sets = asArray(knowledge.sets) + + return sets.some((set) => { + const record = asRecord(set) + const query = asRecord(record.query) + const datasets = asArray(record.datasets) + const hasExpectedDataset = datasets.some((item) => { + const datasetRecord = asRecord(item) + return datasetRecord.id === dataset.id || datasetRecord.name === dataset.name + }) + + if (!hasExpectedDataset || query.mode !== queryMode) + return false + if (queryValue === undefined) + return true + + return asString(query.value).trim() === queryValue + }) +} + +export async function skipMissingPreseededAgent( + world: DifyWorld, + resourceName: string, +): Promise<'skipped' | PreseededResource> { + const query = buildQuery({ limit: '20', name: resourceName, page: '1' }) + const resource = await findConsoleResourceByName({ + action: `Check preseeded Agent ${resourceName}`, + path: `/console/api/agent?${query}`, + resourceName, + }) + + if (!resource) + return skipBlockedPrecondition(world, `Preseeded Agent "${resourceName}" was not found.`) + + return { + id: resource.id, + kind: 'agent', + name: resource.name, + } +} + +export async function skipMissingPreseededWorkflow( + world: DifyWorld, + resourceName: string, +): Promise<'skipped' | PreseededResource> { + const query = buildQuery({ limit: '20', mode: 'workflow', name: resourceName, page: '1' }) + const resource = await findConsoleResourceByName({ + action: `Check preseeded workflow ${resourceName}`, + path: `/console/api/apps?${query}`, + resourceName, + }) + + if (!resource) + return skipBlockedPrecondition(world, `Preseeded workflow "${resourceName}" was not found.`) + + return { + id: resource.id, + kind: 'workflow', + name: resource.name, + } +} + +export async function skipMissingPreseededAgentDriveSkill( + world: DifyWorld, + agentName: string, + skillName: string, +): Promise<'skipped' | PreseededResource> { + const agent = await skipMissingPreseededAgent(world, agentName) + if (agent === 'skipped') + return agent + + const ctx = await createApiContext() + try { + const response = await ctx.get(`/console/api/agent/${agent.id}/drive/skills`) + await expectApiResponseOK(response, `Check preseeded Agent skill ${skillName}`) + const body = (await response.json()) as AgentDriveSkillListResponse + const skill = body.items.find(item => item.name === skillName) + + if (!skill) { + return skipBlockedPrecondition( + world, + `Preseeded Agent "${agentName}" does not include drive skill "${skillName}".`, + ) + } + + return { + id: skill.path, + kind: 'skill', + name: skill.name, + } + } + finally { + await ctx.dispose() + } +} + +export async function skipMissingPreseededFullConfigAgentCoreConfiguration( + world: DifyWorld, + agentName: string, +): Promise<'skipped' | PreseededResource> { + const stableModel = await skipMissingAgentBuilderStableChatModel(world) + if (stableModel === 'skipped') + return stableModel + + const agent = await skipMissingPreseededAgent(world, agentName) + if (agent === 'skipped') + return agent + + const summarySkill = await skipMissingPreseededAgentDriveSkill( + world, + agentName, + agentBuilderPreseededResources.summarySkill, + ) + if (summarySkill === 'skipped') + return summarySkill + + const jsonTool = await skipMissingPreseededTool( + world, + agentBuilderPreseededResources.jsonReplaceTool, + ) + if (jsonTool === 'skipped') + return jsonTool + + const knowledgeBase = await skipMissingReadyPreseededDataset( + world, + agentBuilderPreseededResources.agentKnowledgeBase, + ) + if (knowledgeBase === 'skipped') + return knowledgeBase + + const ctx = await createApiContext() + try { + const response = await ctx.get(`/console/api/agent/${agent.id}/composer`) + await expectApiResponseOK(response, `Check preseeded Agent core configuration ${agentName}`) + const body = (await response.json()) as AgentComposerResponse + const soul = body.agent_soul ?? {} + const missing: string[] = [] + + const model = asRecord(soul.model) + if (model.model_provider !== stableModel.provider || model.model !== stableModel.name) + missing.push(`${agentBuilderPreseededResources.stableChatModel} model config`) + + const prompt = asString(asRecord(soul.prompt).system_prompt) + if (!prompt.includes(agentBuilderExpectedTokens.agentReply)) + missing.push(`Prompt token ${agentBuilderExpectedTokens.agentReply}`) + + const files = asArray(asRecord(soul.files).files) + for (const fileName of [ + agentBuilderTestMaterials.smallFile, + agentBuilderTestMaterials.specialFilename, + ]) { + if (!hasNamedOrKeyedEntry(files, fileName)) + missing.push(`file ${fileName}`) + } + + const [providerName = '', toolName = ''] = jsonTool.id.split('/') + const parsedTool = splitToolDisplayName(agentBuilderPreseededResources.jsonReplaceTool) + if ( + parsedTool.ok + && !hasToolEntry(asArray(asRecord(soul.tools).dify_tools), { + providerDisplayName: parsedTool.providerName, + providerName, + toolDisplayName: parsedTool.toolName, + toolName, + }) + ) { + missing.push(agentBuilderPreseededResources.jsonReplaceTool) + } + + if (!hasKnowledgeDataset(soul, knowledgeBase)) + missing.push(agentBuilderPreseededResources.agentKnowledgeBase) + + if (missing.length > 0) { + return skipBlockedPrecondition( + world, + `Preseeded Agent "${agentName}" is missing core fixture configuration: ${missing.join(', ')}.`, + ) + } + + return agent + } + finally { + await ctx.dispose() + } +} + +export async function skipMissingPreseededToolStatesAgentConfiguration( + world: DifyWorld, + agentName: string, +): Promise<'skipped' | PreseededResource> { + const agent = await skipMissingPreseededAgent(world, agentName) + if (agent === 'skipped') + return agent + + const summarySkill = await skipMissingPreseededAgentDriveSkill( + world, + agentName, + agentBuilderPreseededResources.summarySkill, + ) + if (summarySkill === 'skipped') + return summarySkill + + const jsonTool = await skipMissingPreseededTool( + world, + agentBuilderPreseededResources.jsonReplaceTool, + ) + if (jsonTool === 'skipped') + return jsonTool + + const tavilyTool = await skipMissingPreseededTool( + world, + agentBuilderPreseededResources.tavilySearchTool, + ) + if (tavilyTool === 'skipped') + return tavilyTool + + const ctx = await createApiContext() + try { + const response = await ctx.get(`/console/api/agent/${agent.id}/composer`) + await expectApiResponseOK(response, `Check preseeded Agent tool states ${agentName}`) + const body = (await response.json()) as AgentComposerResponse + const soul = body.agent_soul ?? {} + const toolItems = asArray(asRecord(soul.tools).dify_tools) + const missing: string[] = [] + + const [jsonProviderName = '', jsonToolName = ''] = jsonTool.id.split('/') + const parsedJsonTool = splitToolDisplayName(agentBuilderPreseededResources.jsonReplaceTool) + if ( + parsedJsonTool.ok + && !findToolEntry(toolItems, { + providerDisplayName: parsedJsonTool.providerName, + providerName: jsonProviderName, + toolDisplayName: parsedJsonTool.toolName, + toolName: jsonToolName, + }) + ) { + missing.push(agentBuilderPreseededResources.jsonReplaceTool) + } + + const [tavilyProviderName = '', tavilyToolName = ''] = tavilyTool.id.split('/') + const parsedTavilyTool = splitToolDisplayName(agentBuilderPreseededResources.tavilySearchTool) + const tavilyEntry = parsedTavilyTool.ok + ? findToolEntry(toolItems, { + providerDisplayName: parsedTavilyTool.providerName, + providerName: tavilyProviderName, + toolDisplayName: parsedTavilyTool.toolName, + toolName: tavilyToolName, + }) + : undefined + + if (!tavilyEntry) { + missing.push(agentBuilderPreseededResources.tavilySearchTool) + } + else if (!hasUnauthorizedToolCredentialState(tavilyEntry)) { + missing.push(`${agentBuilderPreseededResources.tavilySearchTool} unauthorized credential state`) + } + + if (missing.length > 0) { + return skipBlockedPrecondition( + world, + `Preseeded Agent "${agentName}" is missing tool state fixture configuration: ${missing.join(', ')}.`, + ) + } + + return agent + } + finally { + await ctx.dispose() + } +} + +export async function skipMissingPreseededDualRetrievalAgentConfiguration( + world: DifyWorld, + agentName: string, +): Promise<'skipped' | PreseededResource> { + const agent = await skipMissingPreseededAgent(world, agentName) + if (agent === 'skipped') + return agent + + const knowledgeBase = await skipMissingReadyPreseededDataset( + world, + agentBuilderPreseededResources.agentKnowledgeBase, + ) + if (knowledgeBase === 'skipped') + return knowledgeBase + + const ctx = await createApiContext() + try { + const response = await ctx.get(`/console/api/agent/${agent.id}/composer`) + await expectApiResponseOK(response, `Check preseeded Agent dual retrieval ${agentName}`) + const body = (await response.json()) as AgentComposerResponse + const soul = body.agent_soul ?? {} + const missing: string[] = [] + + if (!hasKnowledgeSet(soul, knowledgeBase, { queryMode: 'generated_query' })) + missing.push('Agent decide Knowledge Retrieval') + + if ( + !hasKnowledgeSet(soul, knowledgeBase, { + queryMode: 'user_query', + queryValue: agentBuilderFixedInputs.customKnowledgeQuery, + }) + ) { + missing.push('Custom query Knowledge Retrieval') + } + + if (missing.length > 0) { + return skipBlockedPrecondition( + world, + `Preseeded Agent "${agentName}" is missing dual retrieval fixture configuration: ${missing.join(', ')}.`, + ) + } + + return agent + } + finally { + await ctx.dispose() + } +} + +export async function skipMissingPreseededAgentFileTreeFixture( + world: DifyWorld, + agentName: string, +): Promise<'skipped' | PreseededResource> { + const agent = await skipMissingPreseededAgent(world, agentName) + if (agent === 'skipped') + return agent + + const ctx = await createApiContext() + try { + const query = buildQuery({ prefix: 'files/' }) + const response = await ctx.get(`/console/api/agent/${agent.id}/drive/files?${query}`) + await expectApiResponseOK(response, `Check preseeded Agent file tree ${agentName}`) + const body = (await response.json()) as AgentDriveFileListResponse + const keys = (body.items ?? []).map(item => item.key) + const missingFiles = agentBuilderFileTreeFixtureFiles.filter( + filePath => + !keys.some(key => key === `files/${filePath}` || key.endsWith(`/${filePath}`)), + ) + + if (missingFiles.length > 0) { + return skipBlockedPrecondition( + world, + `Preseeded Agent "${agentName}" is missing file tree fixture files: ${missingFiles.join(', ')}.`, + ) + } + + return { + id: agent.id, + kind: 'agent', + name: agent.name, + } + } + finally { + await ctx.dispose() + } +} + +export async function skipMissingPreseededAgentFlatFileFixtureConfiguration( + world: DifyWorld, + agentName: string, +): Promise<'skipped' | PreseededResource> { + const agent = await skipMissingPreseededAgent(world, agentName) + if (agent === 'skipped') + return agent + + const ctx = await createApiContext() + try { + const response = await ctx.get(`/console/api/agent/${agent.id}/composer`) + await expectApiResponseOK(response, `Check preseeded Agent flat file fixture ${agentName}`) + const body = (await response.json()) as AgentComposerResponse + const configFiles = Array.isArray(body.agent_soul?.config_files) + ? body.agent_soul.config_files + : [] + const fileNames = configFiles + .map(file => (typeof file === 'object' && file !== null && 'name' in file ? file.name : undefined)) + .filter((name): name is string => typeof name === 'string') + const missingFiles = agentBuilderFileTreeFixtureFileNames.filter(fileName => !fileNames.includes(fileName)) + + if (missingFiles.length > 0) { + return skipBlockedPrecondition( + world, + `Preseeded Agent "${agentName}" is missing current flat Files fixture configuration: ${missingFiles.join(', ')}. Hierarchical Files display remains blocked until Agent config files support tree paths.`, + ) + } + + return { + id: agent.id, + kind: 'agent', + name: agent.name, + } + } + finally { + await ctx.dispose() + } +} diff --git a/e2e/support/preflight/common.ts b/e2e/support/preflight/common.ts new file mode 100644 index 00000000000..90b491338b7 --- /dev/null +++ b/e2e/support/preflight/common.ts @@ -0,0 +1,128 @@ +import type { DifyWorld } from '../../features/support/world' +import { agentBuilderPreseededResources } from '../agent-builder-resources' +import { createApiContext, expectApiResponseOK } from '../api' + +export type PreseededResource = NonNullable< + DifyWorld['agentBuilder']['preflight']['preseededResources'][string] +> + +export type E2EResourcePrecondition + = | { + ok: true + value: string + } + | { + ok: false + reason: string + } + +export type NamedResource = { + id: string + name: string +} + +export type NamedResourceListResponse = { + data: T[] +} + +export type LocalizedLabel = { + en_US?: string + zh_Hans?: string +} + +export type AgentComposerResponse = { + agent_soul?: Record +} + +export const readRequiredEnvResource = ( + envName: string, + description: string, +): E2EResourcePrecondition => { + const value = process.env[envName]?.trim() + if (value) + return { ok: true, value } + + return { + ok: false, + reason: `${description} requires ${envName}.`, + } +} + +export function skipBlockedPrecondition(world: DifyWorld, reason: string): 'skipped' { + const message = `Blocked precondition: ${reason}` + console.warn(`[e2e] ${message}`) + world.attach(message, 'text/plain') + return 'skipped' +} + +export function skipMissingEnvResource( + world: DifyWorld, + envName: string, + description: string, +): 'skipped' | string { + const resource = readRequiredEnvResource(envName, description) + if (resource.ok) + return resource.value + + return skipBlockedPrecondition(world, resource.reason) +} + +export const requiredAgentBuilderPreseededResources = Object.values(agentBuilderPreseededResources) + +export function skipMissingAgentBuilderPreseed( + world: DifyWorld, + resourceName: string, + envName: string, +): 'skipped' | string { + return skipMissingEnvResource( + world, + envName, + `Preseeded Agent Builder resource "${resourceName}"`, + ) +} + +export const findConsoleResourceByName = async ({ + action, + path, + resourceName, +}: { + action: string + path: string + resourceName: string +}) => { + const ctx = await createApiContext() + try { + const response = await ctx.get(path) + await expectApiResponseOK(response, action) + const body = (await response.json()) as NamedResourceListResponse + + return body.data.find(item => item.name === resourceName) + } + finally { + await ctx.dispose() + } +} + +export const buildQuery = (params: Record) => new URLSearchParams(params).toString() + +export const matchesNameOrLabel = (value: string, name: string, label?: LocalizedLabel) => + value === name || value === label?.en_US || value === label?.zh_Hans + +export const isRecord = (value: unknown): value is Record => + typeof value === 'object' && value !== null && !Array.isArray(value) + +export const asRecord = (value: unknown): Record => (isRecord(value) ? value : {}) + +export const asArray = (value: unknown): unknown[] => (Array.isArray(value) ? value : []) + +export const asString = (value: unknown) => (typeof value === 'string' ? value : '') + +export const hasNamedOrKeyedEntry = (items: unknown[], expectedName: string) => + items.some((item) => { + const record = asRecord(item) + const values = [record.name, record.drive_key, record.reference, record.file_id, record.id].map( + asString, + ) + + return values.some(value => value === expectedName || value.endsWith(`/${expectedName}`)) + }) diff --git a/e2e/support/preflight/datasets.ts b/e2e/support/preflight/datasets.ts new file mode 100644 index 00000000000..81c2bf5208e --- /dev/null +++ b/e2e/support/preflight/datasets.ts @@ -0,0 +1,150 @@ +import type { DifyWorld } from '../../features/support/world' +import type { NamedResource, PreseededResource } from './common' +import { createApiContext, expectApiResponseOK } from '../api' +import { + buildQuery, + findConsoleResourceByName, + + skipBlockedPrecondition, +} from './common' + +type DatasetResource = NamedResource & { + document_count: number + total_available_documents: number +} + +type DocumentIndexingStatus + = | 'cleaning' + | 'completed' + | 'indexing' + | 'parsing' + | 'splitting' + | 'waiting' + +type DatasetIndexingStatusResponse = { + data: Array<{ + id: string + indexing_status?: string + }> +} + +const completedDocumentIndexingStatus: DocumentIndexingStatus = 'completed' +const activeDocumentIndexingStatuses = new Set([ + 'cleaning', + 'indexing', + 'parsing', + 'splitting', + 'waiting', +]) + +export const getPreseededDataset = async (resourceName: string) => { + const query = buildQuery({ keyword: resourceName, limit: '20', page: '1' }) + + return findConsoleResourceByName({ + action: `Check preseeded dataset ${resourceName}`, + path: `/console/api/datasets?${query}`, + resourceName, + }) +} + +const getDatasetIndexingStatuses = async (datasetId: string, resourceName: string) => { + const ctx = await createApiContext() + try { + const response = await ctx.get(`/console/api/datasets/${datasetId}/indexing-status`) + await expectApiResponseOK(response, `Check preseeded dataset indexing status ${resourceName}`) + const body = (await response.json()) as DatasetIndexingStatusResponse + + return body.data + } + finally { + await ctx.dispose() + } +} + +export const toDatasetResource = ( + resource: NamedResource, +): PreseededResource => ({ + id: resource.id, + kind: 'dataset', + name: resource.name, +}) + +export async function skipMissingPreseededDataset( + world: DifyWorld, + resourceName: string, +): Promise<'skipped' | PreseededResource> { + const resource = await getPreseededDataset(resourceName) + + if (!resource) + return skipBlockedPrecondition(world, `Preseeded dataset "${resourceName}" was not found.`) + + return toDatasetResource(resource) +} + +export async function skipMissingReadyPreseededDataset( + world: DifyWorld, + resourceName: string, +): Promise<'skipped' | PreseededResource> { + const resource = await getPreseededDataset(resourceName) + + if (!resource) + return skipBlockedPrecondition(world, `Preseeded dataset "${resourceName}" was not found.`) + + if (resource.document_count < 1) { + return skipBlockedPrecondition(world, `Preseeded dataset "${resourceName}" has no documents.`) + } + + if (resource.total_available_documents !== resource.document_count) { + return skipBlockedPrecondition( + world, + `Preseeded dataset "${resourceName}" has ${resource.total_available_documents}/${resource.document_count} available documents.`, + ) + } + + const statuses = await getDatasetIndexingStatuses(resource.id, resourceName) + if (statuses.length < 1) { + return skipBlockedPrecondition( + world, + `Preseeded dataset "${resourceName}" has no document indexing status.`, + ) + } + + const incompleteStatus = statuses.find( + item => item.indexing_status !== completedDocumentIndexingStatus, + ) + if (incompleteStatus) { + return skipBlockedPrecondition( + world, + `Preseeded dataset "${resourceName}" includes document ${incompleteStatus.id} with indexing status "${incompleteStatus.indexing_status ?? 'missing'}".`, + ) + } + + return toDatasetResource(resource) +} + +export async function skipMissingIndexingPreseededDataset( + world: DifyWorld, + resourceName: string, +): Promise<'skipped' | PreseededResource> { + const resource = await getPreseededDataset(resourceName) + + if (!resource) + return skipBlockedPrecondition(world, `Preseeded dataset "${resourceName}" was not found.`) + + const statuses = await getDatasetIndexingStatuses(resource.id, resourceName) + const indexingStatus = statuses.find(item => + activeDocumentIndexingStatuses.has(item.indexing_status ?? ''), + ) + + if (!indexingStatus) { + const actualStatuses + = statuses.map(item => item.indexing_status ?? 'missing').join(', ') || 'none' + + return skipBlockedPrecondition( + world, + `Preseeded dataset "${resourceName}" is not indexing or queued; document statuses: ${actualStatuses}.`, + ) + } + + return toDatasetResource(resource) +} diff --git a/e2e/support/preflight/index.ts b/e2e/support/preflight/index.ts new file mode 100644 index 00000000000..4c3ada4720a --- /dev/null +++ b/e2e/support/preflight/index.ts @@ -0,0 +1,6 @@ +export * from './access' +export * from './agents' +export * from './common' +export * from './datasets' +export * from './models' +export * from './tools' diff --git a/e2e/support/preflight/models.ts b/e2e/support/preflight/models.ts new file mode 100644 index 00000000000..05e544c3ab2 --- /dev/null +++ b/e2e/support/preflight/models.ts @@ -0,0 +1,158 @@ +import type { DifyWorld } from '../../features/support/world' +import { agentBuilderPreseededResources } from '../agent-builder-resources' +import { createApiContext, expectApiResponseOK } from '../api' +import { skipBlockedPrecondition } from './common' + +const stableChatModelProviderEnv = 'E2E_STABLE_MODEL_PROVIDER' +const stableChatModelNameEnv = 'E2E_STABLE_MODEL_NAME' +const stableChatModelTypeEnv = 'E2E_STABLE_MODEL_TYPE' +const brokenChatModelProviderEnv = 'E2E_BROKEN_MODEL_PROVIDER' +const brokenChatModelNameEnv = 'E2E_BROKEN_MODEL_NAME' +const brokenChatModelTypeEnv = 'E2E_BROKEN_MODEL_TYPE' +const activeModelStatus = 'active' +const defaultStableChatModelType = 'llm' +const defaultBrokenChatModelName = agentBuilderPreseededResources.brokenModel + +type ModelTypeListResponse = { + data: Array<{ + provider: string + models: Array<{ + label?: { + en_US?: string + zh_Hans?: string + } + model: string + status?: string + }> + status?: string + }> +} + +type ModelPreflightConfig + = | { + ok: true + provider: string + resourceName: string + type: string + value: string + } + | { + ok: false + reason: string + } + +export function readAgentBuilderStableChatModelConfig(): ModelPreflightConfig { + const provider = process.env[stableChatModelProviderEnv]?.trim() + const name = process.env[stableChatModelNameEnv]?.trim() + const type = process.env[stableChatModelTypeEnv]?.trim() || defaultStableChatModelType + + const missing: string[] = [] + if (!provider) + missing.push(stableChatModelProviderEnv) + if (!name) + missing.push(stableChatModelNameEnv) + + if (!provider || !name) { + return { + ok: false, + reason: `${agentBuilderPreseededResources.stableChatModel} requires ${missing.join(', ')}.`, + } + } + + return { + ok: true, + provider, + resourceName: agentBuilderPreseededResources.stableChatModel, + type, + value: name, + } +} + +export function readAgentBuilderBrokenChatModelConfig(): ModelPreflightConfig { + const provider = process.env[brokenChatModelProviderEnv]?.trim() + const name = process.env[brokenChatModelNameEnv]?.trim() || defaultBrokenChatModelName + const type = process.env[brokenChatModelTypeEnv]?.trim() || defaultStableChatModelType + + if (!provider) { + return { + ok: false, + reason: `${agentBuilderPreseededResources.brokenModelProvider} requires ${brokenChatModelProviderEnv}.`, + } + } + + return { + ok: true, + provider, + resourceName: agentBuilderPreseededResources.brokenModelProvider, + type, + value: name, + } +} + +async function skipMissingAgentBuilderModel( + world: DifyWorld, + config: ModelPreflightConfig, + { + requireActive, + }: { + requireActive: boolean + }, +): Promise<'skipped' | NonNullable> { + if (!config.ok) + return skipBlockedPrecondition(world, config.reason) + + const ctx = await createApiContext() + try { + const response = await ctx.get( + `/console/api/workspaces/current/models/model-types/${config.type}`, + ) + await expectApiResponseOK(response, `Check ${config.resourceName}`) + const body = (await response.json()) as ModelTypeListResponse + const provider = body.data.find(item => item.provider === config.provider) + const model = provider?.models.find( + item => + item.model === config.value + || item.label?.en_US === config.value + || item.label?.zh_Hans === config.value, + ) + + if (!provider || !model) { + return skipBlockedPrecondition( + world, + `${config.resourceName} was not found as ${config.provider}/${config.value} (${config.type}).`, + ) + } + + if (requireActive && model.status !== activeModelStatus) { + return skipBlockedPrecondition( + world, + `${config.resourceName} is ${model.status ?? 'missing status'} instead of ${activeModelStatus}.`, + ) + } + + return { + name: model.model, + provider: provider.provider, + type: config.type, + } + } + finally { + await ctx.dispose() + } +} + +export async function skipMissingAgentBuilderStableChatModel( + world: DifyWorld, +): Promise<'skipped' | NonNullable> { + return skipMissingAgentBuilderModel(world, readAgentBuilderStableChatModelConfig(), { + requireActive: true, + }) +} + +export async function skipMissingAgentBuilderBrokenChatModel( + world: DifyWorld, +): Promise<'skipped' | NonNullable> { + return skipMissingAgentBuilderModel(world, readAgentBuilderBrokenChatModelConfig(), { + requireActive: false, + }) +} diff --git a/e2e/support/preflight/tools.ts b/e2e/support/preflight/tools.ts new file mode 100644 index 00000000000..b188c9fe24e --- /dev/null +++ b/e2e/support/preflight/tools.ts @@ -0,0 +1,114 @@ +import type { DifyWorld } from '../../features/support/world' +import type { LocalizedLabel, PreseededResource } from './common' +import { createApiContext, expectApiResponseOK } from '../api' +import { + asRecord, + asString, + + matchesNameOrLabel, + + skipBlockedPrecondition, +} from './common' + +type BuiltinToolProvider = { + label?: LocalizedLabel + name: string + tools: Array<{ + label?: LocalizedLabel + name: string + }> +} + +export const splitToolDisplayName = (resourceName: string) => { + const [providerName, toolName] = resourceName.split('/').map(item => item.trim()) + + if (!providerName || !toolName) { + return { + ok: false as const, + reason: `Preseeded tool "${resourceName}" must use "Provider / Tool" format.`, + } + } + + return { + ok: true as const, + providerName, + toolName, + } +} + +export const findToolEntry = ( + items: unknown[], + { + providerDisplayName, + providerName, + toolDisplayName, + toolName, + }: { + providerDisplayName: string + providerName: string + toolDisplayName: string + toolName: string + }, +) => + items.find((item) => { + const record = asRecord(item) + const providerValues = [record.provider_id, record.provider, record.plugin_id, record.name].map( + asString, + ) + const toolValues = [record.tool_name, record.name].map(asString) + + return ( + providerValues.some(value => value === providerName || value === providerDisplayName) + && toolValues.some(value => value === toolName || value === toolDisplayName) + ) + }) + +export const hasToolEntry = ( + items: unknown[], + tool: { + providerDisplayName: string + providerName: string + toolDisplayName: string + toolName: string + }, +) => Boolean(findToolEntry(items, tool)) + +export const hasUnauthorizedToolCredentialState = (item: unknown) => { + const record = asRecord(item) + + return asString(record.credential_type) === 'unauthorized' +} + +export async function skipMissingPreseededTool( + world: DifyWorld, + resourceName: string, +): Promise<'skipped' | PreseededResource> { + const parsed = splitToolDisplayName(resourceName) + if (!parsed.ok) + return skipBlockedPrecondition(world, parsed.reason) + + const ctx = await createApiContext() + try { + const response = await ctx.get('/console/api/workspaces/current/tools/builtin') + await expectApiResponseOK(response, `Check preseeded tool ${resourceName}`) + const providers = (await response.json()) as BuiltinToolProvider[] + const provider = providers.find(item => + matchesNameOrLabel(parsed.providerName, item.name, item.label), + ) + const tool = provider?.tools.find(item => + matchesNameOrLabel(parsed.toolName, item.name, item.label), + ) + + if (!provider || !tool) + return skipBlockedPrecondition(world, `Preseeded tool "${resourceName}" was not found.`) + + return { + id: `${provider.name}/${tool.name}`, + kind: 'tool', + name: resourceName, + } + } + finally { + await ctx.dispose() + } +}