mirror of
https://github.com/langgenius/dify.git
synced 2026-07-15 09:27:36 +08:00
Merge remote-tracking branch 'origin/main' into feat/agent-v2
This commit is contained in:
yyh
@ -5,7 +5,7 @@ description: Write, update, or review Dify end-to-end tests under `e2e/` that us
|
||||
|
||||
# Dify E2E Cucumber + Playwright
|
||||
|
||||
Use this skill for Dify's repository-level E2E suite in `e2e/`. Use [`e2e/AGENTS.md`](../../../e2e/AGENTS.md) as the canonical guide for local architecture and conventions, then apply Playwright/Cucumber best practices only where they fit the current suite.
|
||||
Use this skill for Dify's repository-level E2E suite in `e2e/`. Use [`e2e/AGENTS.md`](../../../e2e/AGENTS.md) as the canonical package guide for local architecture and conventions, then read any feature-scoped `AGENTS.md` that owns the target area. Apply Playwright/Cucumber best practices only where they fit the current suite.
|
||||
|
||||
## Scope
|
||||
|
||||
@ -25,6 +25,8 @@ Use this skill for Dify's repository-level E2E suite in `e2e/`. Use [`e2e/AGENTS
|
||||
4. Read [`references/cucumber-best-practices.md`](references/cucumber-best-practices.md) only when scenario wording, step granularity, tags, or expression design are involved.
|
||||
5. Re-check official Playwright or Cucumber docs with the available documentation tools before introducing a new framework pattern.
|
||||
|
||||
Keep this skill focused on Cucumber, Playwright, and package-level E2E guidance. Put feature-specific conventions in the owning feature's `AGENTS.md` instead of adding them here.
|
||||
|
||||
## Local Rules
|
||||
|
||||
- `e2e/` uses Cucumber for scenarios and Playwright as the browser layer.
|
||||
@ -59,7 +61,7 @@ Use this skill for Dify's repository-level E2E suite in `e2e/`. Use [`e2e/AGENTS
|
||||
- Do not use `waitForTimeout`, manual polling, or raw visibility checks when a locator action or retrying assertion already expresses the behavior.
|
||||
5. Validate narrowly.
|
||||
- Run the narrowest tagged scenario or flow that exercises the change.
|
||||
- Run `pnpm -C e2e check`.
|
||||
- Run `vpr lint --fix --quiet` from the repository root and `pnpm -C e2e type-check`.
|
||||
- Broaden verification only when the change affects hooks, tags, setup, or shared step semantics.
|
||||
|
||||
## Review Checklist
|
||||
|
||||
@ -39,9 +39,11 @@ from controllers.console.wraps import (
|
||||
)
|
||||
from extensions.ext_database import db
|
||||
from fields.agent_fields import (
|
||||
AgentConfigDraftSummaryResponse,
|
||||
AgentConfigSnapshotDetailResponse,
|
||||
AgentConfigSnapshotListResponse,
|
||||
AgentConfigSnapshotRestoreResponse,
|
||||
AgentConfigSnapshotSummaryResponse,
|
||||
AgentInviteOptionsResponse,
|
||||
AgentLogListResponse,
|
||||
AgentLogMessageListResponse,
|
||||
@ -50,11 +52,13 @@ from fields.agent_fields import (
|
||||
AgentRosterListResponse,
|
||||
AgentStatisticSummaryEnvelopeResponse,
|
||||
)
|
||||
from fields.base import ResponseModel
|
||||
from libs.datetime_utils import parse_time_range
|
||||
from libs.helper import dump_response
|
||||
from libs.login import login_required
|
||||
from models import Account
|
||||
from models.agent import Agent, AgentStatus
|
||||
from models.agent_config_entities import AgentSoulConfig
|
||||
from models.enums import ApiTokenType
|
||||
from models.model import ApiToken, App, IconType
|
||||
from services.agent.composer_service import AgentComposerService
|
||||
@ -264,21 +268,21 @@ class AgentPublishPayload(BaseModel):
|
||||
version_note: str | None = Field(default=None, description="Optional note for this published Agent version")
|
||||
|
||||
|
||||
class AgentPublishResponse(BaseModel):
|
||||
class AgentPublishResponse(ResponseModel):
|
||||
result: str
|
||||
active_config_snapshot_id: str
|
||||
active_config_snapshot: dict[str, object] | None = None
|
||||
draft: dict[str, object] | None = None
|
||||
active_config_snapshot: AgentConfigSnapshotSummaryResponse | None = None
|
||||
draft: AgentConfigDraftSummaryResponse | None = None
|
||||
|
||||
|
||||
class AgentBuildDraftCheckoutPayload(BaseModel):
|
||||
force: bool = Field(default=False, description="Overwrite the existing current-user build draft")
|
||||
|
||||
|
||||
class AgentBuildDraftResponse(BaseModel):
|
||||
class AgentBuildDraftResponse(ResponseModel):
|
||||
variant: str
|
||||
draft: dict[str, object]
|
||||
agent_soul: dict[str, object]
|
||||
draft: AgentConfigDraftSummaryResponse
|
||||
agent_soul: AgentSoulConfig
|
||||
|
||||
|
||||
class AgentBuildDraftApplyResponse(BaseModel):
|
||||
|
||||
@ -13080,8 +13080,8 @@ default (the config form sends the full desired feature state on save).
|
||||
|
||||
| Name | Type | Description | Required |
|
||||
| ---- | ---- | ----------- | -------- |
|
||||
| agent_soul | object | | Yes |
|
||||
| draft | object | | Yes |
|
||||
| agent_soul | [AgentSoulConfig](#agentsoulconfig) | | Yes |
|
||||
| draft | [AgentConfigDraftSummaryResponse](#agentconfigdraftsummaryresponse) | | Yes |
|
||||
| variant | string | | Yes |
|
||||
|
||||
#### AgentCliToolAuthorizationStatus
|
||||
@ -14156,9 +14156,9 @@ section may be empty, which is how callers express "no knowledge layer".
|
||||
|
||||
| Name | Type | Description | Required |
|
||||
| ---- | ---- | ----------- | -------- |
|
||||
| active_config_snapshot | object | | No |
|
||||
| active_config_snapshot | [AgentConfigSnapshotSummaryResponse](#agentconfigsnapshotsummaryresponse) | | No |
|
||||
| active_config_snapshot_id | string | | Yes |
|
||||
| draft | object | | No |
|
||||
| draft | [AgentConfigDraftSummaryResponse](#agentconfigdraftsummaryresponse) | | No |
|
||||
| result | string | | Yes |
|
||||
|
||||
#### AgentPublishedReferenceResponse
|
||||
|
||||
1
e2e/.gitignore
vendored
1
e2e/.gitignore
vendored
@ -4,3 +4,4 @@ playwright-report/
|
||||
test-results/
|
||||
cucumber-report/
|
||||
.logs/
|
||||
.generated-test-materials/
|
||||
|
||||
@ -26,12 +26,18 @@ Install Playwright browsers once:
|
||||
```bash
|
||||
pnpm install
|
||||
pnpm -C e2e e2e:install
|
||||
pnpm -C e2e check
|
||||
```
|
||||
|
||||
`pnpm install` is resolved through the repository workspace and uses the shared root lockfile plus `pnpm-workspace.yaml`.
|
||||
|
||||
Use `pnpm -C e2e check` as the default local verification step after editing E2E TypeScript, Cucumber support code, or feature glue. It runs formatting, linting, and type checks for this package.
|
||||
Run only one `pnpm -C e2e e2e*` process against a local workspace at a time. Separate runner processes share the frontend port, backend port, auth bootstrap state, and log paths; running them in parallel can create startup or authorization failures that are not scenario failures.
|
||||
|
||||
Use root lint plus the package type check as the default local verification step after editing E2E TypeScript, Cucumber support code, or feature glue:
|
||||
|
||||
```bash
|
||||
vpr lint --fix --quiet
|
||||
pnpm -C e2e type-check
|
||||
```
|
||||
|
||||
Common commands:
|
||||
|
||||
@ -68,8 +74,8 @@ flowchart TD
|
||||
C --> D["Cucumber loads config, steps, and support modules"]
|
||||
D --> E["BeforeAll bootstraps shared auth state via /install"]
|
||||
E --> F{"Which command is running?"}
|
||||
F -->|`pnpm -C e2e e2e`| G["Run config default tags: not @fresh and not @skip"]
|
||||
F -->|`pnpm -C e2e e2e:full*`| H["Override tags to not @skip"]
|
||||
F -->|`pnpm -C e2e e2e`| G["Run config default tags: not @fresh and not @skip and not @preview"]
|
||||
F -->|`pnpm -C e2e e2e:full*`| H["Override tags to not @skip and not @preview"]
|
||||
G --> I["Per-scenario BrowserContext from shared browser"]
|
||||
H --> I
|
||||
I --> J["Failure artifacts written to cucumber-report/artifacts"]
|
||||
@ -99,7 +105,7 @@ Behavior depends on instance state:
|
||||
- uninitialized instance: completes install and stores authenticated state
|
||||
- initialized instance: signs in and reuses authenticated state
|
||||
|
||||
Because of that, the `@fresh` install scenario only runs in the `pnpm -C e2e e2e:full*` flows. The default `pnpm -C e2e e2e*` flows exclude `@fresh` via Cucumber config tags so they can be re-run against an already initialized instance.
|
||||
Because of that, the `@fresh` install scenario only runs in the `pnpm -C e2e e2e:full*` flows. The default `pnpm -C e2e e2e*` flows exclude `@fresh` and `@preview` via Cucumber config tags so they can be re-run against an already initialized instance while keeping Builder Preview scenarios opt-in.
|
||||
|
||||
Reset all persisted E2E state:
|
||||
|
||||
@ -174,7 +180,7 @@ open cucumber-report/report.html
|
||||
1. Add step definitions under `features/step-definitions/<capability>/`
|
||||
1. Reuse existing steps from `common/` and other definition files before writing new ones
|
||||
1. Run with `pnpm -C e2e e2e -- --tags @your-tag` to verify
|
||||
1. Run `pnpm -C e2e check` before committing
|
||||
1. Run `vpr lint --fix --quiet` from the repository root and `pnpm -C e2e type-check` before committing
|
||||
|
||||
### Feature file conventions
|
||||
|
||||
@ -278,7 +284,7 @@ When('I fill in the app name in the dialog', async function (this: DifyWorld) {
|
||||
|
||||
### Failure diagnostics
|
||||
|
||||
The `After` hook automatically captures on failure:
|
||||
The `After` hook automatically captures diagnostics for failed, ambiguous, pending, undefined, or unknown scenarios:
|
||||
|
||||
- Full-page screenshot (PNG)
|
||||
- Page HTML dump
|
||||
@ -286,6 +292,28 @@ The `After` hook automatically captures on failure:
|
||||
|
||||
Artifacts are saved to `cucumber-report/artifacts/` and attached to the HTML report. No extra code needed in step definitions.
|
||||
|
||||
Skipped preflight scenarios should attach the blocked-precondition reason to the skipped step and should not create screenshot or HTML artifacts.
|
||||
|
||||
### Seed resources and preflight checks
|
||||
|
||||
Use `support/naming.ts` for generated test resource names. New app, Agent, dataset, file, or credential seeds should start with `E2E` so local and shared environments can identify disposable resources.
|
||||
|
||||
Use `fixtures/test-materials/` for checked-in files that scenarios upload, preview, index, or retrieve. Keep these fixtures small and deterministic, and use `support/test-materials.ts` to resolve their absolute paths.
|
||||
|
||||
Use scoped feature support for scenarios that require optional external resources such as a model provider, plugin/tool credential, knowledge base seed, or fixed app. Prefer an explicit `Given` step that returns a skipped result with a clear blocked-precondition reason over hidden setup in hooks.
|
||||
|
||||
Treat preflight checks as read-only readiness checks. A preflight step may query the environment, record typed state on `DifyWorld`, attach a blocked-precondition reason, or return `skipped`; it must not create, repair, publish, reconfigure, or mutate shared seed resources. Treat the preflight suite as a readiness and drift report, not as a seed manager. Long-lived resources belong to the environment seed/setup process and need an explicit owner outside individual scenarios.
|
||||
|
||||
Keep package-level support limited to broadly reusable primitives such as API clients, naming, fixture path resolution, and cleanup helpers. Feature-specific seed contracts and preflight checks belong under the owning feature's support folder.
|
||||
|
||||
Use generated API contracts for Console/Web/Service API request, response, and payload shapes. Import the concrete type directly from `@dify/contracts/.../types.gen` when it exists, and do not hand-write duplicate response shapes or wrap generated types in local aliases just to preserve an older helper name. Keep local E2E types only for scenario state, fixture registries, helper input options, preflight resource state, and intentionally narrowed test view models that are not complete API responses.
|
||||
|
||||
Use typed cleanup fields on `DifyWorld` for resource types created by scenarios, and use `DifyWorld.registerCleanup(...)` when a scenario creates any resource type that is not covered by typed cleanup fields. Typed cleanup should remove child or referencing resources before their owners, such as Agent files before Agents and workflow apps before Agents they reference. Cleanup failures should be attached to the report instead of being swallowed silently. Cleanup callbacks run after typed cleanup queues, even when the scenario fails.
|
||||
|
||||
Scenario-owned setup may create disposable apps, Agents, files, credentials, drafts, or access toggles when the scenario owns their lifecycle and cleanup. Do not use scenario setup to silently fix or complete a shared preseeded resource; if a fixed resource is missing or drifted, report it as blocked and route it to the seed owner.
|
||||
|
||||
Feature-specific seed contracts, resource readiness rules, tags, and scenario ownership can be documented in one scoped `AGENTS.md` at the feature root when a module becomes large enough to need it. Do not add deeper `AGENTS.md` files unless the nested module becomes independently owned.
|
||||
|
||||
## Reusing existing steps
|
||||
|
||||
Before writing a new step definition, inspect the existing step definition files first. Reuse a matching step when the wording and behavior already fit, and only add a new step when the scenario needs a genuinely new user action or assertion. Steps in `common/` are designed for broad reuse across all features.
|
||||
@ -294,13 +322,3 @@ Or browse the step definition files directly:
|
||||
|
||||
- `features/step-definitions/common/` — auth guards and navigation assertions shared by all features
|
||||
- `features/step-definitions/<capability>/` — domain-specific steps scoped to a single feature area
|
||||
|
||||
## Agent v2 scenarios
|
||||
|
||||
Agent v2 scenarios live under `features/agent-v2/` and use the `@agent-v2` capability tag.
|
||||
|
||||
The E2E web environment enables Agent v2 through `NEXT_PUBLIC_ENABLE_AGENT_V2=true` in `scripts/common.ts`, because `/roster` routes are guarded by that feature flag.
|
||||
|
||||
Use `support/agent.ts` for Agent v2 API fixtures. It owns roster-shaped Agent IDs, configure/access route helpers, composer draft sync, build-draft helpers, publish, API access toggles, and Agent cleanup. Store created roster Agent IDs in `DifyWorld.createdAgentIds`; the shared `After` hook deletes them after each scenario.
|
||||
|
||||
Keep Agent v2 step definitions under `features/step-definitions/agent-v2/`. Prefer API setup for prerequisite state, then use Playwright only for user-observable navigation, editing, and assertions.
|
||||
|
||||
@ -1,5 +1,9 @@
|
||||
import type { IConfiguration } from '@cucumber/cucumber'
|
||||
|
||||
const hasCliTags = process.argv.some(arg => arg === '--tags' || arg.startsWith('--tags='))
|
||||
const defaultTags = process.env.E2E_CUCUMBER_TAGS
|
||||
|| (hasCliTags ? undefined : 'not @fresh and not @skip and not @preview')
|
||||
|
||||
const config = {
|
||||
format: [
|
||||
'progress-bar',
|
||||
@ -10,7 +14,7 @@ const config = {
|
||||
import: ['./tsx-register.js', 'features/**/*.ts'],
|
||||
parallel: 1,
|
||||
paths: ['features/**/*.feature'],
|
||||
tags: process.env.E2E_CUCUMBER_TAGS || 'not @fresh and not @skip',
|
||||
...(defaultTags ? { tags: defaultTags } : {}),
|
||||
timeout: 60_000,
|
||||
} satisfies Partial<IConfiguration> & {
|
||||
timeout: number
|
||||
|
||||
218
e2e/features/agent-v2/AGENTS.md
Normal file
218
e2e/features/agent-v2/AGENTS.md
Normal file
@ -0,0 +1,218 @@
|
||||
# Agent V2 E2E
|
||||
|
||||
This file scopes Agent v2 E2E conventions for `e2e/features/agent-v2/` and its step definitions under `e2e/features/step-definitions/agent-v2/`. Keep package-wide runner, lifecycle, hook, fixture, locator, assertion, and cleanup rules in `e2e/AGENTS.md`.
|
||||
|
||||
Do not add deeper `AGENTS.md` files unless an Agent v2 submodule becomes independently owned.
|
||||
|
||||
## Scope
|
||||
|
||||
Agent v2 scenarios live under `features/agent-v2/` and use the `@agent-v2` capability tag.
|
||||
|
||||
The E2E web environment enables Agent v2 through `NEXT_PUBLIC_ENABLE_AGENT_V2=true` in `scripts/common.ts`, because `/roster` routes are guarded by that feature flag.
|
||||
|
||||
Preview/Test Run scenarios are not part of the current build-mode slice unless explicitly requested. Current Agent v2 coverage should prioritize Configure, Build draft, saved configuration display, publish state, Access Point, preflight, files, advanced settings, and other build-mode behavior. Published Web app runtime is not Builder Preview; keep it as a separate `@web-app-runtime` slice because it exercises the public app surface and real model-backed responses after publish.
|
||||
|
||||
Use API setup for prerequisite state, then use Playwright only for user-observable navigation, editing, and assertions. Do not make assertions pass by mirroring the current implementation blindly; if a failure exposes a product ambiguity, resource gap, or test-quality problem, identify the owner before changing the test.
|
||||
|
||||
## Tags
|
||||
|
||||
Use tags in three layers:
|
||||
|
||||
- Capability tags describe the product area: `@build`, `@files`, `@advanced-settings`, `@agent-edit`, `@publish`, `@access-point`, `@output-variables`, and similar tags.
|
||||
- Execution-scope tags describe how the scenario should be selected: `@core`, `@infra`, `@web-app-runtime`, `@service-api-runtime`, `@preview`, and `@feature-gated`.
|
||||
- Narrow fixture or sub-surface tags describe a specific dependency or slice: `@stable-model`, `@skill-fixture`, `@web-app-access`, `@workflow-reference`, `@files-limits`, and similar tags.
|
||||
|
||||
- `@agent-v2` — required capability tag for all Agent v2 scenarios.
|
||||
- `@core` — stable non-runtime scenario expected to run in the regular Agent v2 suite when its explicit preconditions are met. Do not apply `@core` to Preview/Test Run, Web app chat runtime, or Backend service API chat runtime scenarios.
|
||||
- `@infra` — infrastructure or readiness checks.
|
||||
- `@build` — Build mode and Build draft behavior.
|
||||
- `@build-unavailable-resources` — feature-gated Build chat recovery when the user requests unavailable Skills or Tools.
|
||||
- `@files` — Files section upload, display, and fixture behavior.
|
||||
- `@files-limits` — feature-gated file format, size, count, and in-progress upload limit behavior.
|
||||
- `@knowledge` — Knowledge Retrieval configuration display, persistence, and reference cleanup.
|
||||
- `@advanced-settings` — Env Editor, Content Moderation, and related Advanced Settings behavior.
|
||||
- `@agent-create` — Agent Roster creation and initial Configure navigation.
|
||||
- `@agent-edit` — saved Agent detail/configuration display surfaces.
|
||||
- `@publish` — publish and publish-bar state.
|
||||
- `@access-point` — Web app, Backend service API, and Workflow access surfaces.
|
||||
- `@stable-model` — active model fixture dependency. Apply this to every scenario that includes `the Agent Builder stable chat model is available` or otherwise requires an active model configured in the workspace.
|
||||
- `@tool-fixture` — preseeded Tool dependency such as `JSON Process / JSON Replace` or `Tavily / Tavily Search`.
|
||||
- `@skill-fixture` — checked-in or preseeded Skill dependency such as `e2e-summary-skill`.
|
||||
- `@knowledge-fixture` — preseeded dataset dependency such as `E2E Agent Knowledge Base`.
|
||||
- `@full-config-agent` — fixed `E2E New Agent Builder Full Config` Agent dependency.
|
||||
- `@tool-states-agent` — fixed `E2E New Agent Builder Tool States` Agent dependency.
|
||||
- `@file-tree-fixture` — fixed file-tree Agent drive/config-files dependency.
|
||||
- `@dual-retrieval-fixture` — fixed dual Knowledge Retrieval Agent dependency.
|
||||
- `@backend-api-access` — fixed or scenario-owned Backend service API access dependency.
|
||||
- `@published-web-app` — fixed or scenario-owned published Web app access dependency.
|
||||
- `@web-app-runtime` — published public Web app runtime behavior. Use it for scenarios that open the public Web app and assert real chat responses. Access Point URL, launch, customization, and settings surfaces remain `@access-point` behavior unless they send messages through the public Web app.
|
||||
- `@service-api-runtime` — Backend service API runtime behavior. Use it for scenarios that call the published service API and assert real chat responses. Endpoint display, copy, API key, and API reference surfaces remain `@access-point` behavior.
|
||||
- `@feature-gated` — product capability is optional. This tag alone does not skip execution; the scenario must include an explicit step that returns `skipped` with a blocked-precondition reason when the feature is unavailable.
|
||||
|
||||
Use feature-level `@core` only when every scenario in the file is stable, non-runtime, and not feature-gated. If a feature file mixes stable scenarios with runtime, Preview, or feature-gated scenarios, put `@core` only on the stable scenarios. Keep runtime tags scenario-level so the regular core suite cannot inherit them accidentally.
|
||||
|
||||
## Step Organization
|
||||
|
||||
Keep Agent v2 step definitions grouped by user capability, not by DOM component or Cucumber keyword:
|
||||
|
||||
- `configure.steps.ts` — common configure navigation, refresh, autosave, and normal draft assertions.
|
||||
- `build-draft.steps.ts` — Build mode checkout, apply, discard, supported writeback, and Build draft isolation.
|
||||
- `files.steps.ts` — Files upload, display, and fixture-list assertions.
|
||||
- `knowledge.steps.ts` — Knowledge Retrieval configuration persistence and reference cleanup.
|
||||
- `tools.steps.ts` — Tools selector, search, and configuration-boundary behavior.
|
||||
- `advanced-settings.steps.ts` — common Advanced Settings shell and supported-entry assertions.
|
||||
- `env-editor.steps.ts` — Env Editor add, import, delete, persistence, and restored-display behavior.
|
||||
- `content-moderation.steps.ts` — Content Moderation availability, keyword settings, and feature-gated assertions.
|
||||
- `agent-roster.steps.ts` — Agent Roster creation and Roster-level user actions.
|
||||
- `agent-edit.steps.ts` — saved Agent detail display assertions.
|
||||
- `publish.steps.ts` — publish and publish-bar assertions.
|
||||
- `access-point.steps.ts` — common Access Point navigation and overview.
|
||||
- `access-point-web-app.steps.ts` — Web app access entrypoints and public Web app assertions.
|
||||
- `access-point-service-api.steps.ts` — Backend service API entrypoints, keys, API reference, and service requests.
|
||||
- `access-point-workflow.steps.ts` — Workflow access references.
|
||||
- `preflight.steps.ts` — explicit `Given` entrypoints for Agent Builder preflight resources.
|
||||
|
||||
Cucumber step definitions are globally registered. Do not duplicate the same step text across files, even if one is written as `Given` and another as `Then`.
|
||||
|
||||
## World State
|
||||
|
||||
`DifyWorld` owns generic scenario state such as `page`, `context`, errors, downloads, cleanup queues, and created resource IDs.
|
||||
|
||||
Agent v2 business state belongs under `world.agentBuilder`; do not keep adding Agent v2-specific fields to the top level of `DifyWorld`.
|
||||
|
||||
Use the existing namespace shape:
|
||||
|
||||
- `world.agentBuilder.preflight.stableModel`
|
||||
- `world.agentBuilder.preflight.brokenModel`
|
||||
- `world.agentBuilder.preflight.preseededResources`
|
||||
- `world.agentBuilder.accessPoint.serviceApiBaseURL`
|
||||
- `world.agentBuilder.accessPoint.generatedApiKey`
|
||||
- `world.agentBuilder.accessPoint.serviceApiResponse`
|
||||
- `world.agentBuilder.accessPoint.apiReferencePage`
|
||||
- `world.agentBuilder.accessPoint.webAppPage`
|
||||
- `world.agentBuilder.accessPoint.webAppURL`
|
||||
- `world.agentBuilder.accessPoint.workflowReferencePage`
|
||||
- `world.agentBuilder.accessPoint.composerDraftSnapshot`
|
||||
- `world.agentBuilder.configure.concurrentPage`
|
||||
- `world.agentBuilder.workflow.agentConsolePage`
|
||||
- `world.agentBuilder.workflow.outputVariables`
|
||||
|
||||
Use `features/agent-v2/support/agent.ts` for Agent v2 core API fixtures. It owns roster-shaped Agent IDs, configure/access route helpers, composer draft sync, version details, workflow references, publish, and Agent cleanup. Use `features/agent-v2/support/agent-soul.ts` for reusable Agent Soul fixture configuration, prompts, and model/dataset config builders. Use `features/agent-v2/support/agent-build-draft.ts` for Build draft checkout/save/discard API helpers. Use `features/agent-v2/support/agent-drive.ts` for Agent drive/config file and Skill upload plus cleanup helpers. Use `features/agent-v2/support/access-point.ts` for Web app access, Backend service API access, API keys, and service API request helpers. Store created roster Agent IDs in `DifyWorld.createdAgentIds`; the shared `After` hook deletes them after each scenario.
|
||||
|
||||
Use `DifyWorld.createdAgentDriveFiles` for Agent drive files committed during a scenario and `DifyWorld.createdBuiltinToolCredentials` for built-in tool credentials created during a scenario. The shared `After` hook deletes Agent drive files first so cleanup also works for scenarios that upload into a preseeded Agent.
|
||||
|
||||
## Setup Boundary
|
||||
|
||||
Use `a basic configured Agent v2 test agent has been created via API` when a scenario only needs a created Agent with a composer draft. Do not use that basic shell for runtime, model, tool, skill, knowledge, environment variable, moderation, or output-variable coverage until those resources have explicit seed helpers and readiness checks.
|
||||
|
||||
Use `a runnable Agent v2 test agent has been created via API` after `the Agent Builder stable chat model is available` when a scenario needs a real model-backed Agent. The step writes the preflight model into the Agent Soul model config through `features/agent-v2/support/agent-soul.ts` with deterministic E2E model settings; do not duplicate provider/model payload construction in individual steps.
|
||||
|
||||
Use `the Agent v2 configuration should be saved automatically` after UI edits that rely on Configure autosave. It waits for the user-visible publish bar saved state; do not replace it with network-idle waits or internal store checks.
|
||||
|
||||
API setup is acceptable for creating scenario-owned Agents, enabling Backend service API, writing composer drafts, seeding Build drafts, and preparing fixed state. The scenario must still assert user-visible behavior or a real persisted product contract through the public Console API. Do not assert only that a setup API call succeeded.
|
||||
|
||||
Do not use scenario API setup to repair an environment-owned Agent Builder seed. If a scenario depends on a fixed Agent, dataset, workflow, Skill, Tool, credential, published Web app, or active model, use the matching preflight step to verify it and block when it is missing or drifted. Create or mutate resources only when they are scenario-owned and registered for cleanup.
|
||||
|
||||
## API Contract Types
|
||||
|
||||
Agent v2 support helpers consume Console API contracts from `@dify/contracts/api/console/.../types.gen`. When a generated request, response, or payload type exists, import and use that exact type name at the helper boundary. Do not keep an old local response type name as an alias for the generated type.
|
||||
|
||||
Keep local types for Agent v2 E2E-owned state only, such as `DifyWorld.agentBuilder` state, scenario preflight resource records, fixture registry entries, helper input options, and deliberately narrowed test view models. If an endpoint response needs a field that the generated contract does not expose yet, fix the backend schema and regenerate contracts before broadening E2E types.
|
||||
|
||||
## Build Mode And Preview
|
||||
|
||||
Build mode scope means:
|
||||
|
||||
- Configure page behavior.
|
||||
- Build draft checkout, pending state, apply, discard, and route isolation.
|
||||
- Supported Build draft writeback for files, skills, and env.
|
||||
- Saved configuration display after apply/discard/refresh.
|
||||
|
||||
Preview/Test Run scope means:
|
||||
|
||||
- Model-backed runtime responses.
|
||||
- Duplicate run prevention.
|
||||
- Runtime failure recovery.
|
||||
- Tool/knowledge hit behavior proven through Agent replies.
|
||||
|
||||
Keep Preview/Test Run scenarios out of the current build-mode slice unless the task explicitly reintroduces them.
|
||||
|
||||
## Preflight Resources
|
||||
|
||||
Agent Builder resource checks live under `features/agent-v2/support/preflight/`. Import from the specific module that owns the resource contract; do not add a preflight barrel file:
|
||||
|
||||
- `models.ts`
|
||||
- `agents.ts`
|
||||
- `datasets.ts`
|
||||
- `tools.ts`
|
||||
- `access.ts`
|
||||
|
||||
`preflight.steps.ts` should remain the explicit `Given` entrypoint. Do not move preflight into hidden hooks.
|
||||
|
||||
Agent Builder preflight is read-only. It checks long-lived seed resources and records their IDs or normalized metadata for later steps, but it must not create missing resources, toggle fixed access settings, upload missing Skills/files, publish fixed Agents, or patch model/provider credentials. Seed creation and repair belong to the environment setup process, not to Cucumber scenarios.
|
||||
|
||||
Treat preseeded Agent Builder resources as environment contracts. Preflight can report that a stable model, dataset, Skill, Tool credential, fixed Agent, published Web app, or workflow reference is missing, inactive, not indexed, or drifted, but it must not repair that drift during a scenario. Seed scripts, CI bootstrap, or the documented environment maintenance flow own creating and keeping those resources valid.
|
||||
|
||||
Use `the Agent Builder stable chat model is available` before scenarios that need a real Agent Soul model configuration. This includes true runtime scenarios, model-backed build-mode assertions, and Workflow Agent v2 node setup because the backend rejects Agent nodes without model config. Do not add the model preflight to pure navigation or identity checks unless the setup API itself requires model config. `E2E_STABLE_MODEL_PROVIDER`, `E2E_STABLE_MODEL_NAME`, and optional `E2E_STABLE_MODEL_TYPE` are selectors for a model already configured in the workspace; they are not provider credentials. The step defaults to `openai` / `gpt-5.4-mini` / `llm`, verifies the selected model is present and `active` through `/console/api/workspaces/current/models/model-types/{type}`, then stores it on `DifyWorld.agentBuilder.preflight.stableModel`.
|
||||
|
||||
Keep `@stable-model` on Build draft apply scenarios that click `Apply`. The current product path calls `/build-chat/finalize` before applying the draft, and the backend returns `model is required` when the Agent Soul has no model config. Discard-only and pending-draft isolation scenarios can stay model-free when they do not finalize the Build draft.
|
||||
|
||||
Do not pass model provider API keys through Cucumber or Playwright env vars. Provider credentials belong to the Dify environment seed/admin setup. If the selected provider/model is missing or inactive, the scenario must be blocked by preflight instead of trying to create or patch provider credentials during the test.
|
||||
|
||||
Override the default selector only when a scenario or environment explicitly needs a different stable model:
|
||||
|
||||
```bash
|
||||
E2E_STABLE_MODEL_PROVIDER=openai
|
||||
E2E_STABLE_MODEL_NAME=gpt-5.4-mini
|
||||
E2E_STABLE_MODEL_TYPE=llm
|
||||
```
|
||||
|
||||
Dify may expose OpenAI as either `openai` or a plugin provider ID such as `langgenius/openai/openai`. The preflight accepts both forms for selection and stores the actual Console API provider ID for Agent Soul setup.
|
||||
|
||||
Use `the Agent Builder broken chat model is available` before model-recovery scenarios that intentionally start from an invalid model. The step requires `E2E_BROKEN_MODEL_PROVIDER`, defaults `E2E_BROKEN_MODEL_NAME` to `e2e-broken-model`, defaults `E2E_BROKEN_MODEL_TYPE` to `llm`, and only verifies that the model entry exists. The scenario must still assert the user-visible failure and recovery behavior.
|
||||
|
||||
Use `the Agent Builder preseeded Agent "{name}" is available`, `the Agent Builder preseeded workflow "{name}" is available`, `the Agent Builder preseeded dataset "{name}" is available`, and `the Agent Builder preseeded tool "{provider} / {tool}" is available` when a scenario depends on a fixed environment resource. These steps verify the resource through Console APIs, store the result in `DifyWorld.agentBuilder.preflight.preseededResources`, and return `skipped` when the resource is missing.
|
||||
|
||||
Use `the Agent Builder preseeded dataset "{name}" is indexed and ready` for knowledge retrieval scenarios that require a completed knowledge base. It verifies that the dataset exists, has documents, all listed documents are available, and every document indexing status is `completed`. For `E2E Agent Knowledge Base`, it also verifies through the Console segment list API that at least one enabled segment contains `AGENT_KNOWLEDGE_PASS`.
|
||||
|
||||
Use `the Agent Builder preseeded dataset "{name}" is indexing` for failure-recovery scenarios that require an indexing or queued knowledge base. It verifies at least one document is in `waiting`, `parsing`, `cleaning`, `splitting`, or `indexing`.
|
||||
|
||||
`e2e-summary-skill` has two separate E2E contracts. The checked-in package under `e2e/fixtures/test-materials/e2e-summary-skill/SKILL.md` is used by scenario-owned Agents to verify that the Skill package can be uploaded to an Agent drive. Fixed display/configuration scenarios use `e2e-summary-skill` as a preseeded resource: the environment-owned `E2E New Agent Builder Full Config` Agent must already include that drive-backed Skill before the scenario starts. Do not mutate fixed preseeded Agents during a scenario to add missing Skills.
|
||||
|
||||
Use `the Agent Builder preseeded Agent "{agent}" includes drive skill "{skill}"` to verify that a fixed Agent has a drive-backed Skill attached. If it is missing, return a blocked precondition owned by seed/product instead of uploading the Skill into the fixed Agent.
|
||||
|
||||
Use `the Agent Builder preseeded Agent "{agent}" has Backend service API access with an API key` to verify that a fixed Agent has Backend service API enabled and at least one key. The API key step does not validate a human-readable key name because the Console API key response does not expose one.
|
||||
|
||||
Use `the Agent Builder preseeded Agent "{agent}" includes the core fixture configuration` for the fixed Full Config Agent prerequisite. It composes the stable model, Summary Skill, JSON Replace tool, and indexed knowledge-base preflights, then reads `/console/api/agent/{agent_id}/composer` to verify the Agent Soul contains the selected model, prompt success token, required file fixtures, JSON Replace tool entry, and knowledge dataset reference. Do not use this step for Agent node output variables; those live in workflow node-job `declared_outputs`, not the roster Agent App composer response.
|
||||
|
||||
Use `the Agent Builder preseeded Agent "{agent}" includes the tool state fixture configuration` for the fixed Tool States Agent prerequisite. It composes the Summary Skill, JSON Replace tool, and Tavily Search tool preflights, then reads `/console/api/agent/{agent_id}/composer` to verify the Agent Soul includes JSON Replace, Tavily Search, and a Tavily credential reference. This proves the seed is configured to exercise tool status UI; keep actual invalid-credential errors in dependent user-visible configuration or runtime scenarios.
|
||||
|
||||
Use `the Agent Builder preseeded Agent "{agent}" includes the dual retrieval fixture configuration` for the fixed Dual Retrieval Agent prerequisite. It composes the indexed knowledge-base preflight, then reads `/console/api/agent/{agent_id}/composer` to verify `agent_soul.knowledge.sets` includes both an Agent-decide generated query set and a custom user-query set using the fixed custom query.
|
||||
|
||||
Use `the Agent Builder preseeded Agent "{agent}" includes the file tree fixture files` for file-tree display prerequisites. It verifies the Agent drive contains every file from `agentBuilderFileTreeFixtureFiles` through `/console/api/agent/{agent_id}/drive/files?prefix=files/`.
|
||||
|
||||
Use `the Agent Builder preseeded Agent "{agent}" includes the current flat file fixture configuration` for the current Agent Edit Files section. Agent config files are still a flat `config_files` list and reject path separators, so this preflight verifies the fixture file basenames are present in the Agent Soul. Treat this as partial coverage for tree-display requirements until the product supports hierarchical config files in the visible Files section.
|
||||
|
||||
Use `the Agent Builder preseeded Agent "{agent}" has published Web app access` to verify that a fixed Agent is published, Web app access is enabled, and the Agent detail response includes the site token and base URL needed to open the Web app.
|
||||
|
||||
Use `the Agent Builder preseeded Agent "{agent}" is referenced by workflow "{workflow}"` to verify Workflow access prerequisites. It checks both fixed resources exist, then uses `/console/api/agent/{agent_id}/referencing-workflows`, the same Console API used by the Access Point Workflow references table, to verify the workflow references the Agent through at least one published Agent node.
|
||||
|
||||
Run `pnpm -C e2e e2e -- --tags @agent-v2-preflight` against a seeded environment to verify Agent Builder preseeded resource readiness before running dependent scenarios. Keep each resource as a separate preflight scenario so a missing resource marks only its dependent precondition as blocked instead of hiding the rest of the readiness report.
|
||||
|
||||
## Blocked And Partial Policy
|
||||
|
||||
Use explicit skipped steps for missing resources, disabled feature flags, and product capabilities that are not currently implemented. `@feature-gated` is only a label; it is not execution semantics.
|
||||
|
||||
Blocked messages should be specific enough to route ownership:
|
||||
|
||||
```text
|
||||
Blocked precondition: <resource> missing <capability>. Owner: seed/product. Remediation: <what to seed or decide>.
|
||||
```
|
||||
|
||||
Order blocked steps by the real owner of the first unresolved condition. If a scenario is not automatable yet because the product behavior or test fixture contract is undefined, put that explicit availability step before model/tool/dataset preflights so the report is not masked by an unrelated missing seed. If the scenario is otherwise automatable and only depends on an external seed resource, run the matching resource preflight before creating scenario-owned state. When an availability check must inspect a real Configure UI surface, create only scenario-owned disposable state first and rely on the shared cleanup path after the skipped result.
|
||||
|
||||
Use partial coverage only when current product behavior is intentionally narrower than the written requirement and the test still asserts a real user-visible behavior. Example: Files are currently flat in Agent config files, so the flat Files list can be asserted while tree display remains blocked until product support exists.
|
||||
|
||||
File format, size, count, and in-progress upload limit cases are feature-gated until the product exposes stable Agent config file restrictions and user-visible recovery/error states. Do not convert `@files-limits` scenarios to passing tests by relying on default environment behavior; first align the product contract or seed configuration.
|
||||
|
||||
Do not mark a scenario as complete if it only proves setup state and does not assert the user-visible behavior or persisted product contract required by the case.
|
||||
193
e2e/features/agent-v2/access-point.feature
Normal file
193
e2e/features/agent-v2/access-point.feature
Normal file
@ -0,0 +1,193 @@
|
||||
@agent-v2 @authenticated @access-point
|
||||
Feature: Agent v2 Access Point
|
||||
@core
|
||||
Scenario: Access Point shows the available Agent v2 access surfaces
|
||||
Given I am signed in as the default E2E admin
|
||||
And an Agent v2 test agent has been created via API
|
||||
When I open the Agent v2 configure page from the Agent Roster
|
||||
And I switch to the Agent v2 Access Point section
|
||||
Then I should see the Agent v2 Access Point overview
|
||||
|
||||
@core @web-app-access
|
||||
Scenario: Web app access URL can be copied without changing orchestration
|
||||
Given I am signed in as the default E2E admin
|
||||
And a basic configured Agent v2 test agent has been created via API
|
||||
And Agent v2 Web app access has been enabled via API
|
||||
When I open the Agent v2 configure page from the Agent Roster
|
||||
And I switch to the Agent v2 Access Point section
|
||||
Then I should see the Agent v2 Web app access URL
|
||||
And I record the current Agent v2 orchestration draft
|
||||
When I copy the Agent v2 Web app access URL
|
||||
Then the Agent v2 Web app access URL should show it was copied
|
||||
And the current Agent v2 orchestration draft should be unchanged
|
||||
|
||||
@core @web-app-access @published-web-app
|
||||
Scenario: Published Web app can be launched from Access Point
|
||||
Given I am signed in as the default E2E admin
|
||||
And a basic configured Agent v2 test agent has been created via API
|
||||
And the Agent v2 draft has been published via API
|
||||
And Agent v2 Web app access has been enabled via API
|
||||
When I open the Agent v2 configure page from the Agent Roster
|
||||
And I switch to the Agent v2 Access Point section
|
||||
Then I should see the Agent v2 Web app access URL
|
||||
And I record the current Agent v2 orchestration draft
|
||||
When I launch the Agent v2 Web app
|
||||
Then the Agent v2 Web app should open in a new tab
|
||||
And the current Agent v2 orchestration draft should be unchanged
|
||||
|
||||
@core @web-app-access
|
||||
Scenario: Web app Embedded configuration opens from Access Point
|
||||
Given I am signed in as the default E2E admin
|
||||
And a basic configured Agent v2 test agent has been created via API
|
||||
And Agent v2 Web app access has been enabled via API
|
||||
When I open the Agent v2 configure page from the Agent Roster
|
||||
And I switch to the Agent v2 Access Point section
|
||||
And I record the current Agent v2 orchestration draft
|
||||
And I open Agent v2 Embedded configuration
|
||||
Then I should see the Agent v2 Embedded configuration dialog
|
||||
And the current Agent v2 orchestration draft should be unchanged
|
||||
|
||||
@core @web-app-access
|
||||
Scenario: Web app customization opens from Access Point
|
||||
Given I am signed in as the default E2E admin
|
||||
And a basic configured Agent v2 test agent has been created via API
|
||||
And Agent v2 Web app access has been enabled via API
|
||||
When I open the Agent v2 configure page from the Agent Roster
|
||||
And I switch to the Agent v2 Access Point section
|
||||
And I record the current Agent v2 orchestration draft
|
||||
And I open Agent v2 Web app customization
|
||||
Then I should see the Agent v2 Web app customization dialog
|
||||
And the current Agent v2 orchestration draft should be unchanged
|
||||
|
||||
@core @web-app-access
|
||||
Scenario: Web app settings open from Access Point without changing orchestration
|
||||
Given I am signed in as the default E2E admin
|
||||
And a basic configured Agent v2 test agent has been created via API
|
||||
And Agent v2 Web app access has been enabled via API
|
||||
When I open the Agent v2 configure page from the Agent Roster
|
||||
And I switch to the Agent v2 Access Point section
|
||||
And I record the current Agent v2 orchestration draft
|
||||
And I open Agent v2 Web app settings
|
||||
Then I should see the Agent v2 Web app settings dialog
|
||||
And the current Agent v2 orchestration draft should be unchanged
|
||||
|
||||
@core @web-app-access @published-web-app
|
||||
Scenario: Web app access can be disabled and restored from Access Point
|
||||
Given I am signed in as the default E2E admin
|
||||
And a basic configured Agent v2 test agent has been created via API
|
||||
And the Agent v2 draft has been published via API
|
||||
And Agent v2 Web app access has been enabled via API
|
||||
When I open the Agent v2 configure page from the Agent Roster
|
||||
And I switch to the Agent v2 Access Point section
|
||||
And I disable Agent v2 Web app access
|
||||
Then Agent v2 Web app access should be out of service
|
||||
When I enable Agent v2 Web app access
|
||||
Then Agent v2 Web app access should be in service
|
||||
When I refresh the current page
|
||||
Then Agent v2 Web app access should be in service
|
||||
|
||||
@web-app-access @published-web-app @feature-gated
|
||||
Scenario: Disabled Web app public URL shows an unavailable state
|
||||
Given I am signed in as the default E2E admin
|
||||
And Agent v2 disabled Web app public unavailable state is available
|
||||
And a basic configured Agent v2 test agent has been created via API
|
||||
And the Agent v2 draft has been published via API
|
||||
And Agent v2 Web app access has been enabled via API
|
||||
When I open the Agent v2 configure page from the Agent Roster
|
||||
And I switch to the Agent v2 Access Point section
|
||||
And I disable Agent v2 Web app access
|
||||
Then Agent v2 Web app access should be out of service
|
||||
When I open the disabled Agent v2 Web app URL
|
||||
Then the disabled Agent v2 Web app should show an unavailable state
|
||||
|
||||
@core @workflow-reference
|
||||
Scenario: Workflow access shows the referencing workflow
|
||||
Given I am signed in as the default E2E admin
|
||||
And the Agent Builder preseeded Agent "E2E Agent With Workflow Reference" is available
|
||||
And the Agent Builder preseeded workflow "E2E Agent Reference Workflow" is available
|
||||
And the Agent Builder preseeded Agent "E2E Agent With Workflow Reference" is referenced by workflow "E2E Agent Reference Workflow"
|
||||
When I open the preseeded Agent v2 Access Point page for "E2E Agent With Workflow Reference" from the Agent Roster
|
||||
Then I should see the Agent v2 Workflow access reference for "E2E Agent Reference Workflow"
|
||||
When I open the Agent v2 Workflow access reference for "E2E Agent Reference Workflow"
|
||||
Then the Agent v2 Workflow access reference for "E2E Agent Reference Workflow" should open in Studio
|
||||
|
||||
@core @backend-api-access
|
||||
Scenario: Backend service API endpoint can be copied
|
||||
Given I am signed in as the default E2E admin
|
||||
And an Agent v2 test agent has been created via API
|
||||
And Agent v2 Backend service API access has been enabled via API
|
||||
When I open the Agent v2 configure page from the Agent Roster
|
||||
And I switch to the Agent v2 Access Point section
|
||||
Then I should see the Agent v2 Backend service API endpoint
|
||||
When I copy the Agent v2 Backend service API endpoint
|
||||
Then the Agent v2 Backend service API endpoint should show it was copied
|
||||
|
||||
@core @backend-api-access
|
||||
Scenario: Backend service API keys are managed without exposing existing secrets
|
||||
Given I am signed in as the default E2E admin
|
||||
And an Agent v2 test agent has been created via API
|
||||
And Agent v2 Backend service API access has been enabled with a key via API
|
||||
When I open the Agent v2 configure page from the Agent Roster
|
||||
And I switch to the Agent v2 Access Point section
|
||||
And I open Agent v2 API key management
|
||||
Then Agent v2 API keys should not expose a secret by default
|
||||
When I create a new Agent v2 API key
|
||||
Then I should see the newly generated Agent v2 API key once
|
||||
When I copy the newly generated Agent v2 API key
|
||||
Then the newly generated Agent v2 API key should show it was copied
|
||||
When I close the newly generated Agent v2 API key
|
||||
Then the Agent v2 API key list should not expose the full generated secret
|
||||
|
||||
@core @backend-api-access
|
||||
Scenario: Backend service API Reference opens from Access Point
|
||||
Given I am signed in as the default E2E admin
|
||||
And an Agent v2 test agent has been created via API
|
||||
And Agent v2 Backend service API access has been enabled via API
|
||||
When I open the Agent v2 configure page from the Agent Roster
|
||||
And I switch to the Agent v2 Access Point section
|
||||
And I open the Agent v2 API Reference
|
||||
Then the Agent v2 API Reference should open in a new tab
|
||||
|
||||
@core @backend-api-access
|
||||
Scenario: Backend service API access can be disabled and restored from Access Point
|
||||
Given I am signed in as the default E2E admin
|
||||
And an Agent v2 test agent has been created via API
|
||||
And Agent v2 Backend service API access has been enabled via API
|
||||
When I open the Agent v2 configure page from the Agent Roster
|
||||
And I switch to the Agent v2 Access Point section
|
||||
And I disable Agent v2 Backend service API access
|
||||
Then Agent v2 Backend service API access should be out of service
|
||||
When I enable Agent v2 Backend service API access
|
||||
Then Agent v2 Backend service API access should be in service
|
||||
When I refresh the current page
|
||||
Then Agent v2 Backend service API access should be in service
|
||||
|
||||
@service-api-runtime @backend-api-access @stable-model
|
||||
Scenario: Published Agent v2 answers through Backend service API
|
||||
Given I am signed in as the default E2E admin
|
||||
And the Agent Builder stable chat model is available
|
||||
And a runnable Agent v2 test agent has been created via API
|
||||
And Agent v2 Backend service API access has been enabled with a key via API
|
||||
When I open the Agent v2 configure page
|
||||
And I publish the Agent v2 draft
|
||||
Then the Agent v2 draft should be published and up to date
|
||||
When I send the Agent v2 Backend service API minimal request
|
||||
Then the Agent v2 Backend service API request should succeed with the normal E2E marker
|
||||
|
||||
@service-api-runtime @backend-api-access @stable-model
|
||||
Scenario: Disabled Backend service API rejects requests and restored access succeeds
|
||||
Given I am signed in as the default E2E admin
|
||||
And the Agent Builder stable chat model is available
|
||||
And a runnable Agent v2 test agent has been created via API
|
||||
And the Agent v2 draft has been published via API
|
||||
And Agent v2 Backend service API access has been enabled with a key via API
|
||||
When I open the Agent v2 configure page from the Agent Roster
|
||||
And I switch to the Agent v2 Access Point section
|
||||
And I disable Agent v2 Backend service API access
|
||||
Then Agent v2 Backend service API access should be out of service
|
||||
When I send the Agent v2 Backend service API minimal request
|
||||
Then the Agent v2 Backend service API request should be rejected while disabled
|
||||
When I enable Agent v2 Backend service API access
|
||||
Then Agent v2 Backend service API access should be in service
|
||||
When I send the Agent v2 Backend service API minimal request
|
||||
Then the Agent v2 Backend service API request should succeed with the normal E2E marker
|
||||
76
e2e/features/agent-v2/advanced-settings.feature
Normal file
76
e2e/features/agent-v2/advanced-settings.feature
Normal file
@ -0,0 +1,76 @@
|
||||
@agent-v2 @authenticated @advanced-settings
|
||||
Feature: Agent v2 advanced settings
|
||||
@core
|
||||
Scenario: Advanced Settings exposes supported configuration entries
|
||||
Given I am signed in as the default E2E admin
|
||||
And a basic configured Agent v2 test agent has been created via API
|
||||
When I open the Agent v2 configure page
|
||||
Then Agent v2 Advanced Settings should describe supported entries while collapsed
|
||||
When I expand Agent v2 Advanced Settings
|
||||
Then I should see the supported Agent v2 Advanced Settings entries
|
||||
When I collapse Agent v2 Advanced Settings
|
||||
Then Agent v2 Advanced Settings should describe supported entries while collapsed
|
||||
|
||||
@core
|
||||
Scenario: Plain environment variables are saved and restored
|
||||
Given I am signed in as the default E2E admin
|
||||
And a basic configured Agent v2 test agent has been created via API
|
||||
When I open the Agent v2 configure page
|
||||
And I add the plain Agent v2 environment variable from Advanced Settings
|
||||
Then the plain Agent v2 environment variable should be saved in the Agent v2 draft
|
||||
And the Agent v2 configuration should be saved automatically
|
||||
When I refresh the current page
|
||||
Then I should see the plain Agent v2 environment variable in Advanced Settings
|
||||
|
||||
@core
|
||||
Scenario: Valid environment imports are saved and restored
|
||||
Given I am signed in as the default E2E admin
|
||||
And a basic configured Agent v2 test agent has been created via API
|
||||
When I open the Agent v2 configure page
|
||||
And I import the valid Agent v2 environment file from Advanced Settings
|
||||
Then the valid Agent v2 environment import should be saved in the Agent v2 draft
|
||||
And the Agent v2 configuration should be saved automatically
|
||||
When I refresh the current page
|
||||
Then I should see the Agent v2 environment variables from the valid import in Advanced Settings
|
||||
|
||||
@core
|
||||
Scenario: Deleted environment variables are removed after refresh
|
||||
Given I am signed in as the default E2E admin
|
||||
And a basic configured Agent v2 test agent has been created via API
|
||||
When I open the Agent v2 configure page
|
||||
And I add the plain Agent v2 environment variable from Advanced Settings
|
||||
And I add the secondary plain Agent v2 environment variable from Advanced Settings
|
||||
Then the Agent v2 environment variables for deletion should be saved in the Agent v2 draft
|
||||
When I delete the plain Agent v2 environment variable from Advanced Settings
|
||||
Then the plain Agent v2 environment variable should be removed from the Agent v2 draft
|
||||
And the Agent v2 configuration should be saved automatically
|
||||
When I refresh the current page
|
||||
Then I should not see the deleted Agent v2 environment variable in Advanced Settings
|
||||
|
||||
@core
|
||||
Scenario: Invalid environment imports report skipped lines and keep existing variables
|
||||
Given I am signed in as the default E2E admin
|
||||
And a basic configured Agent v2 test agent has been created via API
|
||||
When I open the Agent v2 configure page
|
||||
And I add the plain Agent v2 environment variable from Advanced Settings
|
||||
Then the plain Agent v2 environment variable should be saved in the Agent v2 draft
|
||||
When I import the invalid Agent v2 environment file from Advanced Settings
|
||||
Then the invalid Agent v2 environment import should report skipped lines
|
||||
And the Agent v2 environment variables from the invalid import should be saved in the Agent v2 draft
|
||||
And the Agent v2 configuration should be saved automatically
|
||||
When I refresh the current page
|
||||
Then I should see the Agent v2 environment variables from the invalid import in Advanced Settings
|
||||
|
||||
@content-moderation @feature-gated
|
||||
Scenario: Content Moderation keyword preset replies are saved and restored
|
||||
Given I am signed in as the default E2E admin
|
||||
And a basic configured Agent v2 test agent has been created via API
|
||||
When I open the Agent v2 configure page
|
||||
And I expand Agent v2 Advanced Settings
|
||||
Then Agent v2 Content Moderation Settings should be available
|
||||
When I configure Agent v2 Content Moderation keyword preset replies
|
||||
Then Agent v2 Content Moderation keyword preset replies should be saved in the Agent v2 draft
|
||||
And the Agent v2 configuration should be saved automatically
|
||||
When I refresh the current page
|
||||
And I expand Agent v2 Advanced Settings
|
||||
Then I should see the Agent v2 Content Moderation keyword preset replies in Advanced Settings
|
||||
70
e2e/features/agent-v2/agent-edit.feature
Normal file
70
e2e/features/agent-v2/agent-edit.feature
Normal file
@ -0,0 +1,70 @@
|
||||
@agent-v2 @authenticated @agent-edit
|
||||
Feature: Agent v2 Agent Edit page
|
||||
@core @stable-model @full-config-agent
|
||||
Scenario: Saved orchestration sections are visible on the Agent Edit page
|
||||
Given I am signed in as the default E2E admin
|
||||
And the Agent Builder stable chat model is available
|
||||
And the Agent Builder preseeded Agent "E2E New Agent Builder Full Config" is available
|
||||
And the Agent Builder preseeded Agent "E2E New Agent Builder Full Config" includes the core fixture configuration
|
||||
When I open the preseeded Agent v2 configure page for "E2E New Agent Builder Full Config" from the Agent Roster
|
||||
Then I should see the Agent v2 full-config fixture sections
|
||||
|
||||
@core @stable-model @full-config-agent
|
||||
Scenario: Duplicated Agent inherits configuration without changing the original Agent
|
||||
Given I am signed in as the default E2E admin
|
||||
And the Agent Builder stable chat model is available
|
||||
And the Agent Builder preseeded Agent "E2E New Agent Builder Full Config" is available
|
||||
And the Agent Builder preseeded Agent "E2E New Agent Builder Full Config" includes the core fixture configuration
|
||||
When I duplicate the preseeded Agent v2 "E2E New Agent Builder Full Config" from the Agent Roster
|
||||
Then the duplicated Agent v2 should inherit the full-config fixture from "E2E New Agent Builder Full Config"
|
||||
When I open the Agent v2 configure page
|
||||
And I fill the Agent v2 prompt editor with the updated E2E prompt
|
||||
Then the Agent v2 configuration should be saved automatically
|
||||
And the normal Agent v2 draft should use the updated E2E prompt
|
||||
And the preseeded Agent v2 "E2E New Agent Builder Full Config" should still use the normal E2E prompt
|
||||
|
||||
@core @tool-states-agent
|
||||
Scenario: Tool states are visible on the Agent Edit page
|
||||
Given I am signed in as the default E2E admin
|
||||
And the Agent Builder preseeded Agent "E2E New Agent Builder Tool States" is available
|
||||
And the Agent Builder preseeded Agent "E2E New Agent Builder Tool States" includes the tool state fixture configuration
|
||||
When I open the preseeded Agent v2 configure page for "E2E New Agent Builder Tool States" from the Agent Roster
|
||||
Then I should see the Agent v2 tool state fixture tools
|
||||
|
||||
@tool-error-state @tool-states-agent @feature-gated
|
||||
Scenario: Tool credential error states are visible on the Agent Edit page
|
||||
Given I am signed in as the default E2E admin
|
||||
And Agent v2 Tool credential error state is available
|
||||
And the Agent Builder preseeded Agent "E2E New Agent Builder Tool States" is available
|
||||
And the Agent Builder preseeded Agent "E2E New Agent Builder Tool States" includes the tool state fixture configuration
|
||||
When I open the preseeded Agent v2 configure page for "E2E New Agent Builder Tool States" from the Agent Roster
|
||||
Then Agent v2 Tool credential error state should be available
|
||||
|
||||
@core @file-tree-fixture
|
||||
Scenario: File fixture entries are visible in the current flat Files list
|
||||
Given I am signed in as the default E2E admin
|
||||
And the Agent Builder preseeded Agent "E2E Agent With File Tree" is available
|
||||
And the Agent Builder preseeded Agent "E2E Agent With File Tree" includes the file tree fixture files
|
||||
And the Agent Builder preseeded Agent "E2E Agent With File Tree" includes the current flat file fixture configuration
|
||||
When I open the preseeded Agent v2 configure page for "E2E Agent With File Tree" from the Agent Roster
|
||||
Then I should see the Agent v2 file fixture entries in the current flat Files list
|
||||
|
||||
@core @dual-retrieval-fixture
|
||||
Scenario: Dual Knowledge Retrieval settings are visible on the Agent Edit page
|
||||
Given I am signed in as the default E2E admin
|
||||
And the Agent Builder preseeded Agent "E2E Agent With Dual Retrieval" is available
|
||||
And the Agent Builder preseeded Agent "E2E Agent With Dual Retrieval" includes the dual retrieval fixture configuration
|
||||
When I open the preseeded Agent v2 configure page for "E2E Agent With Dual Retrieval" from the Agent Roster
|
||||
Then I should see the Agent v2 dual retrieval fixture settings
|
||||
|
||||
@core @stable-model
|
||||
Scenario: Agent Edit opens the same Agent in Agent Console
|
||||
Given I am signed in as the default E2E admin
|
||||
And the Agent Builder stable chat model is available
|
||||
And a workflow app with an Agent v2 node has been created via API
|
||||
When I open the app from the app list
|
||||
And I open the Agent v2 workflow node panel
|
||||
And I open the Agent v2 workflow Agent details
|
||||
Then I should see the Agent v2 workflow Agent details for the created Agent
|
||||
When I open the Agent v2 workflow Agent in Agent Console
|
||||
Then the Agent v2 Agent Console should open for the same workflow Agent
|
||||
154
e2e/features/agent-v2/build-draft.feature
Normal file
154
e2e/features/agent-v2/build-draft.feature
Normal file
@ -0,0 +1,154 @@
|
||||
@agent-v2 @authenticated @build
|
||||
Feature: Agent v2 build draft
|
||||
@core @stable-model @tool-fixture @skill-fixture
|
||||
Scenario: Generating a Build draft leaves the normal Agent configuration unchanged
|
||||
Given I am signed in as the default E2E admin
|
||||
And the Agent Builder stable chat model is available
|
||||
And the Agent Builder preseeded tool "JSON Process / JSON Replace" is available
|
||||
And a runnable Agent v2 test agent has been created via API
|
||||
And the e2e-summary-skill Skill is available to the Agent v2 test agent
|
||||
When I open the Agent v2 configure page
|
||||
And I generate an Agent v2 Build draft from the fixed instruction
|
||||
Then I should see the Agent v2 Build draft pending changes
|
||||
And I should see the Agent v2 Build mode confirmation state
|
||||
And the normal Agent v2 draft should still use the normal E2E prompt
|
||||
And the normal Agent v2 draft should not include the e2e-summary-skill Skill
|
||||
And the normal Agent v2 draft should not include the Agent Builder JSON Replace tool
|
||||
|
||||
@core
|
||||
Scenario: Discarding a Build draft keeps the original Agent configuration
|
||||
Given I am signed in as the default E2E admin
|
||||
And an Agent v2 test agent has been created via API
|
||||
And the Agent v2 composer draft uses the normal E2E prompt
|
||||
And an Agent v2 Build draft uses the updated E2E prompt
|
||||
When I open the Agent v2 configure page
|
||||
Then I should see the Agent v2 Build draft pending changes
|
||||
And I should see the updated E2E prompt in the Agent v2 prompt editor
|
||||
And the normal Agent v2 draft should still use the normal E2E prompt
|
||||
When I discard the Agent v2 Build draft
|
||||
Then I should see the normal E2E prompt in the Agent v2 prompt editor
|
||||
And the Agent v2 Build draft should no longer be active
|
||||
When I refresh the current page
|
||||
Then I should see the normal E2E prompt in the Agent v2 prompt editor
|
||||
And the Agent v2 Build draft should no longer be active
|
||||
|
||||
@core @skill-fixture
|
||||
Scenario: Discarding a Build draft does not apply supported configuration changes
|
||||
Given I am signed in as the default E2E admin
|
||||
And a basic configured Agent v2 test agent has been created via API
|
||||
And an Agent v2 Build draft adds the supported E2E files, skills, and env
|
||||
When I open the Agent v2 configure page
|
||||
Then I should see the Agent v2 Build draft pending changes
|
||||
And I should see the small Agent v2 file in the Files section
|
||||
And I should see the e2e-summary-skill Skill in the Skills section
|
||||
And I should see the supported E2E environment variable in Advanced Settings
|
||||
And the normal Agent v2 draft should still use the normal E2E prompt
|
||||
When I discard the Agent v2 Build draft
|
||||
Then I should see the normal E2E prompt in the Agent v2 prompt editor
|
||||
And I should not see the small Agent v2 file in the Files section
|
||||
And I should not see the e2e-summary-skill Skill in the Skills section
|
||||
And I should not see the supported E2E environment variable in Advanced Settings
|
||||
And the Agent v2 draft should not include the supported Build draft config
|
||||
And the Agent v2 Build draft should no longer be active
|
||||
When I refresh the current page
|
||||
Then I should see the normal E2E prompt in the Agent v2 prompt editor
|
||||
And I should not see the small Agent v2 file in the Files section
|
||||
And I should not see the e2e-summary-skill Skill in the Skills section
|
||||
And I should not see the supported E2E environment variable in Advanced Settings
|
||||
And the Agent v2 draft should not include the supported Build draft config
|
||||
And the Agent v2 Build draft should no longer be active
|
||||
|
||||
@core @stable-model
|
||||
Scenario: Applying a pending Build draft updates the normal Agent configuration
|
||||
Given I am signed in as the default E2E admin
|
||||
And the Agent Builder stable chat model is available
|
||||
And a runnable Agent v2 test agent has been created via API
|
||||
And an Agent v2 Build draft uses the updated E2E prompt with the stable E2E model
|
||||
When I open the Agent v2 configure page
|
||||
Then I should see the Agent v2 Build draft pending changes
|
||||
And I should see the updated E2E prompt in the Agent v2 prompt editor
|
||||
And the normal Agent v2 draft should still use the normal E2E prompt
|
||||
When I apply the Agent v2 Build draft
|
||||
Then I should see the updated E2E prompt in the Agent v2 prompt editor
|
||||
And the normal Agent v2 draft should use the updated E2E prompt
|
||||
And the Agent v2 Build draft should no longer be active
|
||||
When I refresh the current page
|
||||
Then I should see the updated E2E prompt in the Agent v2 prompt editor
|
||||
And the Agent v2 Build draft should no longer be active
|
||||
|
||||
@core @stable-model @skill-fixture
|
||||
Scenario: Applying a Build draft updates supported configuration sections
|
||||
Given I am signed in as the default E2E admin
|
||||
And the Agent Builder stable chat model is available
|
||||
And a runnable Agent v2 test agent has been created via API
|
||||
And an Agent v2 Build draft adds the supported E2E files, skills, and env
|
||||
When I open the Agent v2 configure page
|
||||
Then I should see the Agent v2 Build draft pending changes
|
||||
And I should see the updated E2E prompt in the Agent v2 prompt editor
|
||||
And I should see the small Agent v2 file in the Files section
|
||||
And I should see the e2e-summary-skill Skill in the Skills section
|
||||
And I should see the supported E2E environment variable in Advanced Settings
|
||||
And the normal Agent v2 draft should still use the normal E2E prompt
|
||||
When I apply the Agent v2 Build draft
|
||||
Then I should see the updated E2E prompt in the Agent v2 prompt editor
|
||||
And I should see the small Agent v2 file in the Files section
|
||||
And I should see the e2e-summary-skill Skill in the Skills section
|
||||
And I should see the supported E2E environment variable in Advanced Settings
|
||||
And the Agent v2 draft should include the supported Build draft config
|
||||
And the Agent v2 Build draft should no longer be active
|
||||
When I refresh the current page
|
||||
Then I should see the updated E2E prompt in the Agent v2 prompt editor
|
||||
And I should see the small Agent v2 file in the Files section
|
||||
And I should see the e2e-summary-skill Skill in the Skills section
|
||||
And I should see the supported E2E environment variable in Advanced Settings
|
||||
And the Agent v2 Build draft should no longer be active
|
||||
|
||||
@core @stable-model @skill-fixture
|
||||
Scenario: Applying a Build draft with an existing Skill keeps a single Skill entry
|
||||
Given I am signed in as the default E2E admin
|
||||
And the Agent Builder stable chat model is available
|
||||
And a runnable Agent v2 test agent has been created via API
|
||||
And an Agent v2 Build draft includes the existing e2e-summary-skill Skill
|
||||
When I open the Agent v2 configure page
|
||||
Then I should see the Agent v2 Build draft pending changes
|
||||
And I should see one e2e-summary-skill Skill in the Skills section
|
||||
And the Agent v2 draft should include one e2e-summary-skill Skill
|
||||
When I apply the Agent v2 Build draft
|
||||
Then I should see one e2e-summary-skill Skill in the Skills section
|
||||
And the Agent v2 draft should include one e2e-summary-skill Skill
|
||||
And the Agent v2 Build draft should no longer be active
|
||||
When I refresh the current page
|
||||
Then I should see one e2e-summary-skill Skill in the Skills section
|
||||
And the Agent v2 draft should include one e2e-summary-skill Skill
|
||||
|
||||
@core
|
||||
Scenario: Pending Build draft remains protected after leaving Configure
|
||||
Given I am signed in as the default E2E admin
|
||||
And a basic configured Agent v2 test agent has been created via API
|
||||
And an Agent v2 Build draft uses the updated E2E prompt
|
||||
When I open the Agent v2 configure page
|
||||
Then I should see the Agent v2 Build draft pending changes
|
||||
And I should see the updated E2E prompt in the Agent v2 prompt editor
|
||||
And the normal Agent v2 draft should still use the normal E2E prompt
|
||||
When I switch to the Agent v2 Access Point section
|
||||
And I switch to the Agent v2 Configure section
|
||||
Then I should see the Agent v2 Build draft pending changes
|
||||
And I should see the updated E2E prompt in the Agent v2 prompt editor
|
||||
And the normal Agent v2 draft should still use the normal E2E prompt
|
||||
|
||||
@build-tool-writeback @feature-gated
|
||||
Scenario: Applying a Build draft can add Dify Tools to the Agent configuration
|
||||
Given I am signed in as the default E2E admin
|
||||
And Agent v2 Build chat Dify Tool writeback is available
|
||||
And a basic configured Agent v2 test agent has been created via API
|
||||
When I open the Agent v2 configure page
|
||||
Then Agent v2 Build chat Dify Tool writeback should be available
|
||||
|
||||
@build-unavailable-resources @feature-gated @stable-model
|
||||
Scenario: Build chat reports unavailable Skill or Tool requests clearly
|
||||
Given I am signed in as the default E2E admin
|
||||
And Agent v2 Build chat unavailable Skill and Tool recovery is available
|
||||
And the Agent Builder stable chat model is available
|
||||
And a runnable Agent v2 test agent has been created via API
|
||||
When I open the Agent v2 configure page
|
||||
Then Agent v2 Build chat unavailable Skill and Tool recovery should be available
|
||||
54
e2e/features/agent-v2/configure-persistence.feature
Normal file
54
e2e/features/agent-v2/configure-persistence.feature
Normal file
@ -0,0 +1,54 @@
|
||||
@agent-v2 @authenticated @core
|
||||
Feature: Agent v2 configure persistence
|
||||
@configure-persistence @stable-model
|
||||
Scenario: Selecting a stable model in Configure persists after refresh
|
||||
Given I am signed in as the default E2E admin
|
||||
And the Agent Builder stable chat model is available
|
||||
And an Agent v2 test agent has been created via API
|
||||
When I open the Agent v2 configure page
|
||||
And I select the stable E2E model in the Agent v2 model selector
|
||||
And I fill the Agent v2 prompt editor with the normal E2E prompt
|
||||
Then the Agent v2 configuration should be saved automatically
|
||||
And the Agent v2 draft should use the stable E2E model
|
||||
And the normal Agent v2 draft should use the normal E2E prompt
|
||||
When I refresh the current page
|
||||
Then I should see the stable E2E model in the Agent v2 model selector
|
||||
And I should see the normal E2E prompt in the Agent v2 prompt editor
|
||||
And the Agent v2 draft should use the stable E2E model
|
||||
|
||||
@configure-persistence @stable-model
|
||||
Scenario: Persisted Agent v2 instructions remain visible after refresh
|
||||
Given I am signed in as the default E2E admin
|
||||
And the Agent Builder stable chat model is available
|
||||
And a runnable Agent v2 test agent has been created via API
|
||||
When I open the Agent v2 configure page
|
||||
And I fill the Agent v2 prompt editor with the normal E2E prompt
|
||||
Then the normal Agent v2 draft should use the normal E2E prompt
|
||||
And the Agent v2 draft should use the stable E2E model
|
||||
And the Agent v2 configuration should be saved automatically
|
||||
When I refresh the current page
|
||||
Then I should see the normal E2E prompt in the Agent v2 prompt editor
|
||||
And I should see the stable E2E model in the Agent v2 model selector
|
||||
And the Agent v2 draft should use the stable E2E model
|
||||
|
||||
@configure-persistence
|
||||
Scenario: Leaving Configure immediately after editing preserves prompt changes
|
||||
Given I am signed in as the default E2E admin
|
||||
And a basic configured Agent v2 test agent has been created via API
|
||||
When I open the Agent v2 configure page
|
||||
And I fill the Agent v2 prompt editor with the updated E2E prompt
|
||||
And I leave the Agent v2 configure page immediately after editing
|
||||
When I open the Agent v2 configure page from the Agent Roster
|
||||
Then I should see the updated E2E prompt in the Agent v2 prompt editor
|
||||
And the normal Agent v2 draft should use the updated E2E prompt
|
||||
|
||||
@configure-persistence
|
||||
Scenario: Concurrent Agent v2 edits converge to one clear saved draft after refresh
|
||||
Given I am signed in as the default E2E admin
|
||||
And a basic configured Agent v2 test agent has been created via API
|
||||
When I open the Agent v2 configure page
|
||||
And I open the same Agent v2 configure page in another tab
|
||||
And I save the Agent v2 prompt from the first configure tab
|
||||
And I save the Agent v2 prompt from the second configure tab
|
||||
When I refresh both Agent v2 configure tabs
|
||||
Then both Agent v2 configure tabs and the Agent v2 draft should show one saved concurrent prompt
|
||||
9
e2e/features/agent-v2/configure-validation.feature
Normal file
9
e2e/features/agent-v2/configure-validation.feature
Normal file
@ -0,0 +1,9 @@
|
||||
@agent-v2 @authenticated @preview
|
||||
Feature: Agent v2 configure validation
|
||||
Scenario: Preview is unavailable until a required model is configured
|
||||
Given I am signed in as the default E2E admin
|
||||
And an Agent v2 test agent has been created via API
|
||||
And the Agent v2 composer draft uses the normal E2E prompt
|
||||
When I open the Agent v2 configure page
|
||||
Then Agent v2 Preview should be unavailable until a model is configured
|
||||
And I should see the normal E2E prompt in the Agent v2 prompt editor
|
||||
77
e2e/features/agent-v2/files.feature
Normal file
77
e2e/features/agent-v2/files.feature
Normal file
@ -0,0 +1,77 @@
|
||||
@agent-v2 @authenticated @files
|
||||
Feature: Agent v2 files
|
||||
@core
|
||||
Scenario: Uploading a small file keeps it in the Agent configuration
|
||||
Given I am signed in as the default E2E admin
|
||||
And a basic configured Agent v2 test agent has been created via API
|
||||
When I open the Agent v2 configure page
|
||||
And I upload the small Agent v2 file from the Files section
|
||||
Then I should see the small Agent v2 file in the Files section
|
||||
And the small Agent v2 file should be saved in the Agent v2 draft
|
||||
And the Agent v2 configuration should be saved automatically
|
||||
When I refresh the current page
|
||||
Then I should see the small Agent v2 file in the Files section
|
||||
|
||||
@core
|
||||
Scenario: Uploading an empty file keeps a zero-byte file in the Agent configuration
|
||||
Given I am signed in as the default E2E admin
|
||||
And a basic configured Agent v2 test agent has been created via API
|
||||
When I open the Agent v2 configure page
|
||||
And I upload the empty Agent v2 file from the Files section
|
||||
Then I should see the empty Agent v2 file in the Files section
|
||||
And the empty Agent v2 file should be saved as a zero-byte file in the Agent v2 draft
|
||||
And the Agent v2 configuration should be saved automatically
|
||||
When I refresh the current page
|
||||
Then I should see the empty Agent v2 file in the Files section
|
||||
|
||||
@core
|
||||
Scenario: Uploading a special-name file keeps the filename readable
|
||||
Given I am signed in as the default E2E admin
|
||||
And a basic configured Agent v2 test agent has been created via API
|
||||
When I open the Agent v2 configure page
|
||||
And I upload the special-name Agent v2 file from the Files section
|
||||
Then I should see the special-name Agent v2 file in the Files section
|
||||
And the special-name Agent v2 file should be saved in the Agent v2 draft
|
||||
And the Agent v2 configuration should be saved automatically
|
||||
When I refresh the current page
|
||||
Then I should see the special-name Agent v2 file in the Files section
|
||||
|
||||
@files-limits @feature-gated
|
||||
Scenario: Unsupported Agent v2 file formats show a clear rejection reason
|
||||
Given I am signed in as the default E2E admin
|
||||
And Agent v2 unsupported file format rejection is available
|
||||
And a basic configured Agent v2 test agent has been created via API
|
||||
When I open the Agent v2 configure page
|
||||
Then Agent v2 unsupported file format rejection should be available
|
||||
|
||||
@files-limits @feature-gated
|
||||
Scenario: Oversized Agent v2 files show a clear rejection reason
|
||||
Given I am signed in as the default E2E admin
|
||||
And Agent v2 oversized file rejection is available
|
||||
And a basic configured Agent v2 test agent has been created via API
|
||||
When I open the Agent v2 configure page
|
||||
Then Agent v2 oversized file rejection should be available
|
||||
|
||||
@files-limits @feature-gated
|
||||
Scenario: Agent v2 single-batch file count limits are enforced
|
||||
Given I am signed in as the default E2E admin
|
||||
And Agent v2 single-batch file count limits are available
|
||||
And a basic configured Agent v2 test agent has been created via API
|
||||
When I open the Agent v2 configure page
|
||||
Then Agent v2 single-batch file count limits should be available
|
||||
|
||||
@files-limits @feature-gated
|
||||
Scenario: Agent v2 total file count limits are enforced
|
||||
Given I am signed in as the default E2E admin
|
||||
And Agent v2 total file count limits are available
|
||||
And a basic configured Agent v2 test agent has been created via API
|
||||
When I open the Agent v2 configure page
|
||||
Then Agent v2 total file count limits should be available
|
||||
|
||||
@files-limits @feature-gated
|
||||
Scenario: Leaving during Agent v2 file upload keeps a recoverable state
|
||||
Given I am signed in as the default E2E admin
|
||||
And Agent v2 in-progress file upload recovery is available
|
||||
And a basic configured Agent v2 test agent has been created via API
|
||||
When I open the Agent v2 configure page
|
||||
Then Agent v2 in-progress file upload recovery should be available
|
||||
69
e2e/features/agent-v2/knowledge.feature
Normal file
69
e2e/features/agent-v2/knowledge.feature
Normal file
@ -0,0 +1,69 @@
|
||||
@agent-v2 @authenticated @knowledge @knowledge-fixture
|
||||
Feature: Agent v2 Knowledge Retrieval
|
||||
@core
|
||||
Scenario: Agent decide Knowledge Retrieval settings are saved and restored
|
||||
Given I am signed in as the default E2E admin
|
||||
And the Agent Builder preseeded dataset "E2E Agent Knowledge Base" is indexed and ready
|
||||
And a basic configured Agent v2 test agent has been created via API
|
||||
When I open the Agent v2 configure page
|
||||
And I add the Agent Builder knowledge base as an Agent decide Knowledge Retrieval
|
||||
Then the Agent v2 Agent decide Knowledge Retrieval should be saved in the Agent v2 draft
|
||||
And the Agent v2 configuration should be saved automatically
|
||||
When I refresh the current page
|
||||
Then I should see the Agent v2 Agent decide Knowledge Retrieval settings
|
||||
|
||||
@core
|
||||
Scenario: Custom query Knowledge Retrieval settings are saved and restored
|
||||
Given I am signed in as the default E2E admin
|
||||
And the Agent Builder preseeded dataset "E2E Agent Knowledge Base" is indexed and ready
|
||||
And a basic configured Agent v2 test agent has been created via API
|
||||
When I open the Agent v2 configure page
|
||||
And I add the Agent Builder knowledge base as a Custom query Knowledge Retrieval
|
||||
Then the Agent v2 Custom query Knowledge Retrieval should be saved in the Agent v2 draft
|
||||
And the Agent v2 configuration should be saved automatically
|
||||
When I refresh the current page
|
||||
Then I should see the Agent v2 Custom query Knowledge Retrieval settings
|
||||
|
||||
@service-api-runtime @stable-model @backend-api-access
|
||||
Scenario: Agent decide Knowledge Retrieval answers through Backend service API
|
||||
Given I am signed in as the default E2E admin
|
||||
And the Agent Builder stable chat model is available
|
||||
And the Agent Builder preseeded dataset "E2E Agent Knowledge Base" is indexed and ready
|
||||
And a runnable Agent v2 test agent has been created via API
|
||||
And Agent v2 Backend service API access has been enabled with a key via API
|
||||
When I open the Agent v2 configure page
|
||||
And I add the Agent Builder knowledge base as an Agent decide Knowledge Retrieval
|
||||
Then the Agent v2 Agent decide Knowledge Retrieval should be saved in the Agent v2 draft
|
||||
And the Agent v2 configuration should be saved automatically
|
||||
When I publish the Agent v2 draft
|
||||
Then the Agent v2 draft should be published and up to date
|
||||
When I send the Agent v2 Backend service API knowledge request
|
||||
Then the Agent v2 Backend service API response should include the knowledge E2E marker
|
||||
|
||||
@service-api-runtime @stable-model @backend-api-access
|
||||
Scenario: Custom query Knowledge Retrieval answers through Backend service API
|
||||
Given I am signed in as the default E2E admin
|
||||
And the Agent Builder stable chat model is available
|
||||
And the Agent Builder preseeded dataset "E2E Agent Knowledge Base" is indexed and ready
|
||||
And a runnable Agent v2 test agent has been created via API
|
||||
And Agent v2 Backend service API access has been enabled with a key via API
|
||||
When I open the Agent v2 configure page
|
||||
And I add the Agent Builder knowledge base as a Custom query Knowledge Retrieval
|
||||
Then the Agent v2 Custom query Knowledge Retrieval should be saved in the Agent v2 draft
|
||||
And the Agent v2 configuration should be saved automatically
|
||||
When I publish the Agent v2 draft
|
||||
Then the Agent v2 draft should be published and up to date
|
||||
When I send the Agent v2 Backend service API knowledge request
|
||||
Then the Agent v2 Backend service API response should include the knowledge E2E marker
|
||||
|
||||
@core
|
||||
Scenario: Removing Knowledge Retrieval clears the saved dataset reference
|
||||
Given I am signed in as the default E2E admin
|
||||
And the Agent Builder preseeded dataset "E2E Agent Knowledge Base" is indexed and ready
|
||||
And a knowledge-backed Agent v2 test agent has been created via API
|
||||
When I open the Agent v2 configure page
|
||||
Then I should see the Agent v2 Knowledge Retrieval "Retrieval 1"
|
||||
When I remove the Agent v2 Knowledge Retrieval "Retrieval 1"
|
||||
Then the Agent v2 configuration should be saved automatically
|
||||
And the Agent v2 draft should no longer reference the Agent Builder knowledge base
|
||||
And I should not see the Agent v2 Knowledge Retrieval "Retrieval 1"
|
||||
85
e2e/features/agent-v2/output-variables.feature
Normal file
85
e2e/features/agent-v2/output-variables.feature
Normal file
@ -0,0 +1,85 @@
|
||||
@agent-v2 @authenticated @output-variables
|
||||
Feature: Agent v2 output variables
|
||||
@standalone-output-variables @feature-gated
|
||||
Scenario: Standalone Agent configure exposes Output Variables
|
||||
Given I am signed in as the default E2E admin
|
||||
And Agent v2 standalone Output Variables are available
|
||||
And a basic configured Agent v2 test agent has been created via API
|
||||
When I open the Agent v2 configure page
|
||||
Then Agent v2 standalone Output Variables should be available
|
||||
|
||||
@core @stable-model
|
||||
Scenario: Workflow Agent v2 output variables persist after refresh
|
||||
Given I am signed in as the default E2E admin
|
||||
And the Agent Builder stable chat model is available
|
||||
And a workflow app with an Agent v2 node has been created via API
|
||||
When I open the app from the app list
|
||||
And I open the Agent v2 workflow node panel
|
||||
And I add these Agent v2 workflow node output variables
|
||||
| name | type |
|
||||
| e2e_summary | string |
|
||||
| e2e_report_pdf | file |
|
||||
| e2e_topics | array[string] |
|
||||
Then the Agent v2 workflow node output variables should be saved in the workflow draft
|
||||
When I refresh the current page
|
||||
And I open the Agent v2 workflow node panel
|
||||
Then I should see the Agent v2 workflow node output variables
|
||||
|
||||
@core @stable-model
|
||||
Scenario: Workflow Agent v2 nested object output variables persist after refresh
|
||||
Given I am signed in as the default E2E admin
|
||||
And the Agent Builder stable chat model is available
|
||||
And a workflow app with an Agent v2 node has been created via API
|
||||
When I open the app from the app list
|
||||
And I open the Agent v2 workflow node panel
|
||||
And I add a required Agent v2 workflow node object output variable with text and analysis fields
|
||||
Then the Agent v2 workflow node nested object output variable should be saved in the workflow draft
|
||||
When I refresh the current page
|
||||
And I open the Agent v2 workflow node panel
|
||||
Then I should see the Agent v2 workflow node nested object output variable
|
||||
|
||||
@core @stable-model
|
||||
Scenario: Workflow Agent v2 prompt output reference stays synced when renamed
|
||||
Given I am signed in as the default E2E admin
|
||||
And the Agent Builder stable chat model is available
|
||||
And a workflow app with an Agent v2 node has been created via API
|
||||
When I open the app from the app list
|
||||
And I open the Agent v2 workflow node panel
|
||||
And I insert a file output reference from the Agent v2 workflow node task editor
|
||||
Then the Agent v2 workflow node task should reference the file output
|
||||
When I rename the Agent v2 workflow node task output reference
|
||||
Then the Agent v2 workflow node task should reference the renamed file output
|
||||
When I refresh the current page
|
||||
And I open the Agent v2 workflow node panel
|
||||
Then the Agent v2 workflow node task should reference the renamed file output
|
||||
|
||||
@output-reference-delete @feature-gated @stable-model
|
||||
Scenario: Workflow Agent v2 prompt output reference deletion remains explicit
|
||||
Given I am signed in as the default E2E admin
|
||||
And Agent v2 workflow task output reference deletion consistency is available
|
||||
And the Agent Builder stable chat model is available
|
||||
And a workflow app with an Agent v2 node has been created via API
|
||||
When I open the app from the app list
|
||||
And I open the Agent v2 workflow node panel
|
||||
And I insert a file output reference from the Agent v2 workflow node task editor
|
||||
Then Agent v2 workflow task output reference deletion consistency should be available
|
||||
|
||||
@output-retry-strategy @feature-gated @stable-model
|
||||
Scenario: Workflow Agent v2 output retry strategy can be saved after refresh
|
||||
Given I am signed in as the default E2E admin
|
||||
And Agent v2 workflow output retry strategy is available
|
||||
And the Agent Builder stable chat model is available
|
||||
And a workflow app with an Agent v2 node has been created via API
|
||||
When I open the app from the app list
|
||||
And I open the Agent v2 workflow node panel
|
||||
Then Agent v2 workflow output retry strategy should be available
|
||||
|
||||
@output-retry-validation @feature-gated @stable-model
|
||||
Scenario: Workflow Agent v2 output retry count validation is enforced
|
||||
Given I am signed in as the default E2E admin
|
||||
And Agent v2 workflow output retry count validation is available
|
||||
And the Agent Builder stable chat model is available
|
||||
And a workflow app with an Agent v2 node has been created via API
|
||||
When I open the app from the app list
|
||||
And I open the Agent v2 workflow node panel
|
||||
Then Agent v2 workflow output retry count validation should be available
|
||||
125
e2e/features/agent-v2/preflight.feature
Normal file
125
e2e/features/agent-v2/preflight.feature
Normal file
@ -0,0 +1,125 @@
|
||||
@agent-v2 @authenticated @infra @agent-v2-preflight
|
||||
Feature: Agent Builder preseeded environment
|
||||
@agent-lifecycle
|
||||
Scenario: Agent lifecycle permissions are available
|
||||
Given I am signed in as the default E2E admin
|
||||
And an Agent v2 test agent has been created via API
|
||||
And the Agent v2 composer draft uses the normal E2E prompt
|
||||
When I open the Agent v2 configure page
|
||||
And I publish the Agent v2 draft
|
||||
Then the Agent v2 draft should be published and up to date
|
||||
|
||||
@stable-model
|
||||
Scenario: Stable chat model is available
|
||||
Given I am signed in as the default E2E admin
|
||||
And the Agent Builder stable chat model is available
|
||||
|
||||
@broken-model
|
||||
Scenario: Broken chat model is available for recovery scenarios
|
||||
Given I am signed in as the default E2E admin
|
||||
And the Agent Builder broken chat model is available
|
||||
|
||||
@tool-fixture
|
||||
Scenario: JSON Replace tool is available
|
||||
Given I am signed in as the default E2E admin
|
||||
And the Agent Builder preseeded tool "JSON Process / JSON Replace" is available
|
||||
|
||||
@tool-fixture
|
||||
Scenario: Tavily Search tool is available
|
||||
Given I am signed in as the default E2E admin
|
||||
And the Agent Builder preseeded tool "Tavily / Tavily Search" is available
|
||||
|
||||
@skill-fixture
|
||||
Scenario: Summary Skill package fixture uploads to Agent drive
|
||||
Given I am signed in as the default E2E admin
|
||||
And an Agent v2 test agent has been created via API
|
||||
And the e2e-summary-skill Skill is available to the Agent v2 test agent
|
||||
Then the Agent v2 test agent should include drive skill "e2e-summary-skill"
|
||||
|
||||
@knowledge-fixture
|
||||
Scenario: Agent knowledge base is available
|
||||
Given I am signed in as the default E2E admin
|
||||
And the Agent Builder preseeded dataset "E2E Agent Knowledge Base" is indexed and ready
|
||||
|
||||
@knowledge-fixture
|
||||
Scenario: Indexing knowledge base is available
|
||||
Given I am signed in as the default E2E admin
|
||||
And the Agent Builder preseeded dataset "E2E Agent Knowledge Base Indexing" is indexing
|
||||
|
||||
@full-config-agent
|
||||
Scenario: Full config Agent is available
|
||||
Given I am signed in as the default E2E admin
|
||||
And the Agent Builder preseeded Agent "E2E New Agent Builder Full Config" is available
|
||||
|
||||
@full-config-agent @skill-fixture
|
||||
Scenario: Full config Agent includes the summary Skill
|
||||
Given I am signed in as the default E2E admin
|
||||
And the Agent Builder preseeded Agent "E2E New Agent Builder Full Config" includes drive skill "e2e-summary-skill"
|
||||
|
||||
@full-config-agent @stable-model @tool-fixture @skill-fixture @knowledge-fixture
|
||||
Scenario: Full config Agent includes core fixture configuration
|
||||
Given I am signed in as the default E2E admin
|
||||
And the Agent Builder preseeded Agent "E2E New Agent Builder Full Config" includes the core fixture configuration
|
||||
|
||||
@content-moderation @feature-gated
|
||||
Scenario: Content Moderation Settings is enabled
|
||||
Given I am signed in as the default E2E admin
|
||||
And a basic configured Agent v2 test agent has been created via API
|
||||
When I open the Agent v2 configure page
|
||||
And I expand Agent v2 Advanced Settings
|
||||
Then Agent v2 Content Moderation Settings should be available
|
||||
|
||||
@tool-states-agent
|
||||
Scenario: Tool states Agent is available
|
||||
Given I am signed in as the default E2E admin
|
||||
And the Agent Builder preseeded Agent "E2E New Agent Builder Tool States" is available
|
||||
|
||||
@tool-states-agent @tool-fixture @skill-fixture
|
||||
Scenario: Tool states Agent includes tool state fixture configuration
|
||||
Given I am signed in as the default E2E admin
|
||||
And the Agent Builder preseeded Agent "E2E New Agent Builder Tool States" includes the tool state fixture configuration
|
||||
|
||||
@file-tree-fixture
|
||||
Scenario: File tree Agent includes fixture files
|
||||
Given I am signed in as the default E2E admin
|
||||
And the Agent Builder preseeded Agent "E2E Agent With File Tree" includes the file tree fixture files
|
||||
|
||||
@dual-retrieval-fixture
|
||||
Scenario: Dual retrieval Agent is available
|
||||
Given I am signed in as the default E2E admin
|
||||
And the Agent Builder preseeded Agent "E2E Agent With Dual Retrieval" is available
|
||||
|
||||
@dual-retrieval-fixture @knowledge-fixture
|
||||
Scenario: Dual retrieval Agent includes dual retrieval fixture configuration
|
||||
Given I am signed in as the default E2E admin
|
||||
And the Agent Builder preseeded Agent "E2E Agent With Dual Retrieval" includes the dual retrieval fixture configuration
|
||||
|
||||
@published-web-app
|
||||
Scenario: Published Web app Agent exposes Web app access
|
||||
Given I am signed in as the default E2E admin
|
||||
And the Agent Builder preseeded Agent "E2E Agent Published Web App" has published Web app access
|
||||
|
||||
@backend-api-access
|
||||
Scenario: Backend API-enabled Agent is available
|
||||
Given I am signed in as the default E2E admin
|
||||
And the Agent Builder preseeded Agent "E2E Agent Backend API Enabled" is available
|
||||
|
||||
@backend-api-access
|
||||
Scenario: Backend API-enabled Agent exposes API access with a key
|
||||
Given I am signed in as the default E2E admin
|
||||
And the Agent Builder preseeded Agent "E2E Agent Backend API Enabled" has Backend service API access with an API key
|
||||
|
||||
@workflow-reference
|
||||
Scenario: Workflow reference Agent is available
|
||||
Given I am signed in as the default E2E admin
|
||||
And the Agent Builder preseeded Agent "E2E Agent With Workflow Reference" is available
|
||||
|
||||
@workflow-reference
|
||||
Scenario: Reference workflow is available
|
||||
Given I am signed in as the default E2E admin
|
||||
And the Agent Builder preseeded workflow "E2E Agent Reference Workflow" is available
|
||||
|
||||
@workflow-reference
|
||||
Scenario: Workflow reference Agent is used by the reference workflow
|
||||
Given I am signed in as the default E2E admin
|
||||
And the Agent Builder preseeded Agent "E2E Agent With Workflow Reference" is referenced by workflow "E2E Agent Reference Workflow"
|
||||
110
e2e/features/agent-v2/publish.feature
Normal file
110
e2e/features/agent-v2/publish.feature
Normal file
@ -0,0 +1,110 @@
|
||||
@agent-v2 @authenticated @publish
|
||||
Feature: Agent v2 publish
|
||||
@core @stable-model
|
||||
Scenario: Publish a configured Agent v2 draft
|
||||
Given I am signed in as the default E2E admin
|
||||
And the Agent Builder stable chat model is available
|
||||
And a runnable Agent v2 test agent has been created via API
|
||||
When I open the Agent v2 configure page
|
||||
And I publish the Agent v2 draft
|
||||
Then the Agent v2 draft should be published and up to date
|
||||
|
||||
@core @stable-model
|
||||
Scenario: Publish action follows unpublished changes
|
||||
Given I am signed in as the default E2E admin
|
||||
And the Agent Builder stable chat model is available
|
||||
And a runnable Agent v2 test agent has been created via API
|
||||
When I open the Agent v2 configure page
|
||||
Then the Agent v2 publish action should be available for unpublished changes
|
||||
When I publish the Agent v2 draft
|
||||
Then the Agent v2 draft should be published and up to date
|
||||
And the Agent v2 publish action should be unavailable while up to date
|
||||
When I fill the Agent v2 prompt editor with the updated E2E prompt
|
||||
Then the Agent v2 configuration should be saved automatically
|
||||
And the Agent v2 publish action should be available for unpublished changes
|
||||
|
||||
@core @stable-model
|
||||
Scenario: Published Agent v2 version remains isolated from draft edits
|
||||
Given I am signed in as the default E2E admin
|
||||
And the Agent Builder stable chat model is available
|
||||
And a runnable Agent v2 test agent has been created via API
|
||||
When I open the Agent v2 configure page
|
||||
And I publish the Agent v2 draft
|
||||
Then the Agent v2 draft should be published and up to date
|
||||
When I fill the Agent v2 prompt editor with the updated E2E prompt
|
||||
Then the Agent v2 configuration should be saved automatically
|
||||
And the normal Agent v2 draft should use the updated E2E prompt
|
||||
And the active published Agent v2 version should still use the normal E2E prompt
|
||||
|
||||
@core @stable-model
|
||||
Scenario: Restoring a published Agent v2 version shows the restored configuration in Builder
|
||||
Given I am signed in as the default E2E admin
|
||||
And the Agent Builder stable chat model is available
|
||||
And a runnable Agent v2 test agent has been created via API
|
||||
When I open the Agent v2 configure page
|
||||
And I publish the Agent v2 draft
|
||||
Then the Agent v2 draft should be published and up to date
|
||||
When I fill the Agent v2 prompt editor with the updated E2E prompt
|
||||
Then the Agent v2 configuration should be saved automatically
|
||||
And the normal Agent v2 draft should use the updated E2E prompt
|
||||
When I publish the Agent v2 draft
|
||||
Then the Agent v2 draft should be published and up to date
|
||||
When I open the Agent v2 version history
|
||||
And I select Agent v2 published version 1
|
||||
Then the selected Agent v2 version should be displayed in view-only mode
|
||||
And I should see the normal E2E prompt in the Agent v2 prompt editor
|
||||
When I restore the selected Agent v2 version
|
||||
Then I should see the normal E2E prompt in the Agent v2 prompt editor
|
||||
And the normal Agent v2 draft should use the normal E2E prompt
|
||||
And the Agent v2 publish action should be available for unpublished changes
|
||||
|
||||
@web-app-runtime @published-web-app @stable-model
|
||||
Scenario: Published Agent v2 answers through Web app
|
||||
Given I am signed in as the default E2E admin
|
||||
And the Agent Builder stable chat model is available
|
||||
And a runnable Agent v2 test agent has been created via API
|
||||
And Agent v2 Web app access has been enabled via API
|
||||
When I open the Agent v2 configure page
|
||||
And I publish the Agent v2 draft
|
||||
Then the Agent v2 draft should be published and up to date
|
||||
When I open the Agent v2 Web app URL
|
||||
And I send an E2E message in the Agent v2 Web app
|
||||
Then the Agent v2 Web app response should include the normal E2E marker
|
||||
When I close the Agent v2 Web app
|
||||
|
||||
@web-app-runtime @published-web-app @stable-model
|
||||
Scenario: Published Web app remains isolated from unpublished Agent v2 draft edits
|
||||
Given I am signed in as the default E2E admin
|
||||
And the Agent Builder stable chat model is available
|
||||
And a runnable Agent v2 test agent has been created via API
|
||||
And Agent v2 Web app access has been enabled via API
|
||||
When I open the Agent v2 configure page
|
||||
And I publish the Agent v2 draft
|
||||
Then the Agent v2 draft should be published and up to date
|
||||
When I fill the Agent v2 prompt editor with the updated E2E prompt
|
||||
Then the Agent v2 configuration should be saved automatically
|
||||
And the normal Agent v2 draft should use the updated E2E prompt
|
||||
When I open the Agent v2 Web app URL
|
||||
And I send an E2E message in the Agent v2 Web app
|
||||
Then the Agent v2 Web app response should include the normal E2E marker
|
||||
And the Agent v2 Web app response should not include the updated E2E marker
|
||||
When I close the Agent v2 Web app
|
||||
|
||||
@web-app-runtime @published-web-app @stable-model
|
||||
Scenario: Published Web app uses the latest Agent v2 published configuration
|
||||
Given I am signed in as the default E2E admin
|
||||
And the Agent Builder stable chat model is available
|
||||
And a runnable Agent v2 test agent has been created via API
|
||||
And Agent v2 Web app access has been enabled via API
|
||||
When I open the Agent v2 configure page
|
||||
And I publish the Agent v2 draft
|
||||
Then the Agent v2 draft should be published and up to date
|
||||
When I fill the Agent v2 prompt editor with the updated E2E prompt
|
||||
Then the Agent v2 configuration should be saved automatically
|
||||
And the normal Agent v2 draft should use the updated E2E prompt
|
||||
When I publish the Agent v2 draft
|
||||
Then the Agent v2 draft should be published and up to date
|
||||
When I open the Agent v2 Web app URL
|
||||
And I send an E2E message in the Agent v2 Web app
|
||||
Then the Agent v2 Web app response should include the updated E2E marker
|
||||
When I close the Agent v2 Web app
|
||||
7
e2e/features/agent-v2/roster-create.feature
Normal file
7
e2e/features/agent-v2/roster-create.feature
Normal file
@ -0,0 +1,7 @@
|
||||
@agent-v2 @authenticated @agent-create @core
|
||||
Feature: Agent v2 Roster creation
|
||||
Scenario: Create an Agent from the Agent Roster
|
||||
Given I am signed in as the default E2E admin
|
||||
When I create an Agent v2 test agent from the Agent Roster
|
||||
Then the created Agent v2 should open in Configure
|
||||
And I should see the Agent v2 configure workspace
|
||||
104
e2e/features/agent-v2/support/access-point.ts
Normal file
104
e2e/features/agent-v2/support/access-point.ts
Normal file
@ -0,0 +1,104 @@
|
||||
import type {
|
||||
AgentApiAccessResponse,
|
||||
ApiKeyItem,
|
||||
} from '@dify/contracts/api/console/agent/types.gen'
|
||||
import type {
|
||||
ChatRequestPayloadWithUser,
|
||||
PostChatMessagesResponse,
|
||||
} from '@dify/contracts/api/service/types.gen'
|
||||
import { request } from '@playwright/test'
|
||||
import { createApiContext, expectApiResponseOK, setAppSiteEnabled } from '../../../support/api'
|
||||
import { getTestAgent } from './agent'
|
||||
|
||||
export type AgentServiceApiChatResult = {
|
||||
body: PostChatMessagesResponse | unknown
|
||||
ok: boolean
|
||||
status: number
|
||||
}
|
||||
|
||||
export async function setAgentSiteAccessAndGetURL(
|
||||
agentId: string,
|
||||
enabled: boolean,
|
||||
): Promise<string> {
|
||||
const agent = await getTestAgent(agentId)
|
||||
const appId = agent.app_id ?? agent.backing_app_id
|
||||
if (!appId)
|
||||
throw new Error(`Agent v2 ${agentId} does not expose a backing app ID.`)
|
||||
|
||||
const appDetail = await setAppSiteEnabled(appId, enabled)
|
||||
const token = agent.site?.access_token ?? agent.site?.code ?? appDetail.site.access_token
|
||||
const baseURL = agent.site?.app_base_url ?? appDetail.site.app_base_url
|
||||
|
||||
return `${baseURL.replace(/\/$/, '')}/agent/${token}`
|
||||
}
|
||||
|
||||
export async function setAgentApiAccess(
|
||||
agentId: string,
|
||||
enabled: boolean,
|
||||
): Promise<AgentApiAccessResponse> {
|
||||
const ctx = await createApiContext()
|
||||
try {
|
||||
const response = await ctx.post(`/console/api/agent/${agentId}/api-enable`, {
|
||||
data: { enable_api: enabled },
|
||||
})
|
||||
await expectApiResponseOK(
|
||||
response,
|
||||
`${enabled ? 'Enable' : 'Disable'} Agent v2 API access for ${agentId}`,
|
||||
)
|
||||
return (await response.json()) as AgentApiAccessResponse
|
||||
}
|
||||
finally {
|
||||
await ctx.dispose()
|
||||
}
|
||||
}
|
||||
|
||||
export async function createAgentApiKey(agentId: string): Promise<ApiKeyItem> {
|
||||
const ctx = await createApiContext()
|
||||
try {
|
||||
const response = await ctx.post(`/console/api/agent/${agentId}/api-keys`)
|
||||
await expectApiResponseOK(response, `Create Agent v2 API key for ${agentId}`)
|
||||
return (await response.json()) as ApiKeyItem
|
||||
}
|
||||
finally {
|
||||
await ctx.dispose()
|
||||
}
|
||||
}
|
||||
|
||||
export async function sendAgentServiceApiChatMessage({
|
||||
apiKey,
|
||||
query = 'Please reply with the test success marker.',
|
||||
serviceApiBaseURL,
|
||||
}: {
|
||||
apiKey: string
|
||||
query?: string
|
||||
serviceApiBaseURL: string
|
||||
}): Promise<AgentServiceApiChatResult> {
|
||||
const ctx = await request.newContext()
|
||||
const body = {
|
||||
inputs: {},
|
||||
query,
|
||||
response_mode: 'blocking',
|
||||
user: 'e2e-agent-access-point',
|
||||
} satisfies ChatRequestPayloadWithUser
|
||||
|
||||
try {
|
||||
const response = await ctx.post(`${serviceApiBaseURL.replace(/\/$/, '')}/chat-messages`, {
|
||||
data: body,
|
||||
headers: {
|
||||
Authorization: `Bearer ${apiKey}`,
|
||||
},
|
||||
})
|
||||
const responseBody = await response.json().catch(async () => ({
|
||||
message: await response.text().catch(() => ''),
|
||||
}))
|
||||
|
||||
return {
|
||||
body: responseBody as PostChatMessagesResponse | unknown,
|
||||
ok: response.ok(),
|
||||
status: response.status(),
|
||||
}
|
||||
}
|
||||
finally {
|
||||
await ctx.dispose()
|
||||
}
|
||||
}
|
||||
51
e2e/features/agent-v2/support/agent-build-draft.ts
Normal file
51
e2e/features/agent-v2/support/agent-build-draft.ts
Normal file
@ -0,0 +1,51 @@
|
||||
import type {
|
||||
AgentBuildDraftResponse,
|
||||
AgentSoulConfig,
|
||||
} from '@dify/contracts/api/console/agent/types.gen'
|
||||
import { createApiContext, expectApiResponseOK } from '../../../support/api'
|
||||
|
||||
export async function checkoutAgentBuildDraft(agentId: string): Promise<AgentBuildDraftResponse> {
|
||||
const ctx = await createApiContext()
|
||||
try {
|
||||
const response = await ctx.post(`/console/api/agent/${agentId}/build-draft/checkout`, {
|
||||
data: { force: true },
|
||||
})
|
||||
await expectApiResponseOK(response, `Checkout Agent v2 build draft for ${agentId}`)
|
||||
return (await response.json()) as AgentBuildDraftResponse
|
||||
}
|
||||
finally {
|
||||
await ctx.dispose()
|
||||
}
|
||||
}
|
||||
|
||||
export async function saveAgentBuildDraft(
|
||||
agentId: string,
|
||||
agentSoul: AgentSoulConfig,
|
||||
): Promise<AgentBuildDraftResponse> {
|
||||
const ctx = await createApiContext()
|
||||
try {
|
||||
const response = await ctx.put(`/console/api/agent/${agentId}/build-draft`, {
|
||||
data: {
|
||||
agent_soul: agentSoul,
|
||||
save_strategy: 'save_to_current_version',
|
||||
variant: 'agent_app',
|
||||
},
|
||||
})
|
||||
await expectApiResponseOK(response, `Save Agent v2 build draft for ${agentId}`)
|
||||
return (await response.json()) as AgentBuildDraftResponse
|
||||
}
|
||||
finally {
|
||||
await ctx.dispose()
|
||||
}
|
||||
}
|
||||
|
||||
export async function discardAgentBuildDraft(agentId: string): Promise<void> {
|
||||
const ctx = await createApiContext()
|
||||
try {
|
||||
const response = await ctx.delete(`/console/api/agent/${agentId}/build-draft`)
|
||||
await expectApiResponseOK(response, `Discard Agent v2 build draft for ${agentId}`)
|
||||
}
|
||||
finally {
|
||||
await ctx.dispose()
|
||||
}
|
||||
}
|
||||
45
e2e/features/agent-v2/support/agent-builder-resources.ts
Normal file
45
e2e/features/agent-v2/support/agent-builder-resources.ts
Normal file
@ -0,0 +1,45 @@
|
||||
export const agentBuilderPreseededResources = {
|
||||
stableChatModel: 'E2E Stable Chat Model',
|
||||
summarySkill: 'e2e-summary-skill',
|
||||
jsonReplaceTool: 'JSON Process / JSON Replace',
|
||||
tavilySearchTool: 'Tavily / Tavily Search',
|
||||
agentKnowledgeBase: 'E2E Agent Knowledge Base',
|
||||
indexingKnowledgeBase: 'E2E Agent Knowledge Base Indexing',
|
||||
brokenModelProvider: 'E2E Broken Model Provider',
|
||||
brokenModel: 'e2e-broken-model',
|
||||
fullConfigAgent: 'E2E New Agent Builder Full Config',
|
||||
toolStatesAgent: 'E2E New Agent Builder Tool States',
|
||||
fileTreeAgent: 'E2E Agent With File Tree',
|
||||
dualRetrievalAgent: 'E2E Agent With Dual Retrieval',
|
||||
publishedWebAppAgent: 'E2E Agent Published Web App',
|
||||
backendApiEnabledAgent: 'E2E Agent Backend API Enabled',
|
||||
workflowReferenceAgent: 'E2E Agent With Workflow Reference',
|
||||
referenceWorkflow: 'E2E Agent Reference Workflow',
|
||||
backendApiKey: 'E2E Backend API Key',
|
||||
} as const
|
||||
|
||||
export const agentBuilderFixedInputs = {
|
||||
tavilyInvalidApiKey: 'E2E_INVALID_TAVILY_API_KEY_DO_NOT_USE',
|
||||
missingSkillSearch: 'E2E_NOT_EXIST_SKILL',
|
||||
missingToolSearch: 'E2E_NOT_EXIST_TOOL',
|
||||
missingToolSearchWithSuffix: 'E2E_NOT_EXIST_TOOL_12345',
|
||||
customKnowledgeQuery: 'Dify Agent E2E 测试暗号',
|
||||
envPlainKey: 'E2E_AGENT_FLAG',
|
||||
envPlainValue: 'enabled',
|
||||
envModeKey: 'E2E_AGENT_MODE',
|
||||
envModeValue: 'plain',
|
||||
envAfterInvalidImportKey: 'E2E_AGENT_AFTER_INVALID',
|
||||
envAfterInvalidImportValue: 'still-valid',
|
||||
moderationKeyword: 'E2E_BLOCKED_KEYWORD',
|
||||
inputModerationReply: 'E2E_INPUT_BLOCKED_REPLY',
|
||||
outputModerationReply: 'E2E_OUTPUT_BLOCKED_REPLY',
|
||||
backendApiUser: 'e2e-agent-access-point',
|
||||
} as const
|
||||
|
||||
export const agentBuilderExpectedTokens = {
|
||||
agentReply: 'AGENT_E2E_PASS',
|
||||
updatedAgentReply: 'E2E_AGENT_UPDATED',
|
||||
knowledgeReply: 'AGENT_KNOWLEDGE_PASS',
|
||||
jsonToolBefore: 'JSON_TOOL_E2E',
|
||||
jsonToolAfter: 'E2E_AFTER',
|
||||
} as const
|
||||
291
e2e/features/agent-v2/support/agent-drive.ts
Normal file
291
e2e/features/agent-v2/support/agent-drive.ts
Normal file
@ -0,0 +1,291 @@
|
||||
import type {
|
||||
AgentConfigFileRefConfig,
|
||||
AgentConfigFileUploadResponse,
|
||||
AgentConfigSkillRefConfig,
|
||||
AgentConfigSkillUploadResponse,
|
||||
AgentDriveSkillItemResponse,
|
||||
AgentDriveSkillListResponse,
|
||||
AgentSkillUploadResponse,
|
||||
} from '@dify/contracts/api/console/agent/types.gen'
|
||||
import { Buffer } from 'node:buffer'
|
||||
import { readFile } from 'node:fs/promises'
|
||||
import path from 'node:path'
|
||||
import { createApiContext, expectApiResponseOK } from '../../../support/api'
|
||||
|
||||
export type UploadedConsoleFile = {
|
||||
id: string
|
||||
mime_type?: string | null
|
||||
name: string
|
||||
size?: number | null
|
||||
}
|
||||
|
||||
const crc32Table = new Uint32Array(256)
|
||||
for (let i = 0; i < crc32Table.length; i++) {
|
||||
let c = i
|
||||
for (let k = 0; k < 8; k++)
|
||||
c = c & 1 ? 0xEDB88320 ^ (c >>> 1) : c >>> 1
|
||||
crc32Table[i] = c >>> 0
|
||||
}
|
||||
|
||||
const crc32 = (buffer: Buffer) => {
|
||||
let crc = 0xFFFFFFFF
|
||||
for (const byte of buffer)
|
||||
crc = crc32Table[(crc ^ byte) & 0xFF]! ^ (crc >>> 8)
|
||||
return (crc ^ 0xFFFFFFFF) >>> 0
|
||||
}
|
||||
|
||||
const createSingleFileZip = ({
|
||||
content,
|
||||
entryName,
|
||||
}: {
|
||||
content: Buffer
|
||||
entryName: string
|
||||
}) => {
|
||||
const entryNameBuffer = Buffer.from(entryName)
|
||||
const checksum = crc32(content)
|
||||
const localHeader = Buffer.alloc(30)
|
||||
localHeader.writeUInt32LE(0x04034B50, 0)
|
||||
localHeader.writeUInt16LE(20, 4)
|
||||
localHeader.writeUInt16LE(0, 6)
|
||||
localHeader.writeUInt16LE(0, 8)
|
||||
localHeader.writeUInt16LE(0, 10)
|
||||
localHeader.writeUInt16LE(0, 12)
|
||||
localHeader.writeUInt32LE(checksum, 14)
|
||||
localHeader.writeUInt32LE(content.length, 18)
|
||||
localHeader.writeUInt32LE(content.length, 22)
|
||||
localHeader.writeUInt16LE(entryNameBuffer.length, 26)
|
||||
localHeader.writeUInt16LE(0, 28)
|
||||
|
||||
const centralDirectoryOffset = localHeader.length + entryNameBuffer.length + content.length
|
||||
const centralDirectoryHeader = Buffer.alloc(46)
|
||||
centralDirectoryHeader.writeUInt32LE(0x02014B50, 0)
|
||||
centralDirectoryHeader.writeUInt16LE(20, 4)
|
||||
centralDirectoryHeader.writeUInt16LE(20, 6)
|
||||
centralDirectoryHeader.writeUInt16LE(0, 8)
|
||||
centralDirectoryHeader.writeUInt16LE(0, 10)
|
||||
centralDirectoryHeader.writeUInt16LE(0, 12)
|
||||
centralDirectoryHeader.writeUInt16LE(0, 14)
|
||||
centralDirectoryHeader.writeUInt32LE(checksum, 16)
|
||||
centralDirectoryHeader.writeUInt32LE(content.length, 20)
|
||||
centralDirectoryHeader.writeUInt32LE(content.length, 24)
|
||||
centralDirectoryHeader.writeUInt16LE(entryNameBuffer.length, 28)
|
||||
centralDirectoryHeader.writeUInt16LE(0, 30)
|
||||
centralDirectoryHeader.writeUInt16LE(0, 32)
|
||||
centralDirectoryHeader.writeUInt16LE(0, 34)
|
||||
centralDirectoryHeader.writeUInt16LE(0, 36)
|
||||
centralDirectoryHeader.writeUInt32LE(0, 38)
|
||||
centralDirectoryHeader.writeUInt32LE(0, 42)
|
||||
|
||||
const centralDirectorySize = centralDirectoryHeader.length + entryNameBuffer.length
|
||||
const endOfCentralDirectory = Buffer.alloc(22)
|
||||
endOfCentralDirectory.writeUInt32LE(0x06054B50, 0)
|
||||
endOfCentralDirectory.writeUInt16LE(0, 4)
|
||||
endOfCentralDirectory.writeUInt16LE(0, 6)
|
||||
endOfCentralDirectory.writeUInt16LE(1, 8)
|
||||
endOfCentralDirectory.writeUInt16LE(1, 10)
|
||||
endOfCentralDirectory.writeUInt32LE(centralDirectorySize, 12)
|
||||
endOfCentralDirectory.writeUInt32LE(centralDirectoryOffset, 16)
|
||||
endOfCentralDirectory.writeUInt16LE(0, 20)
|
||||
|
||||
return Buffer.concat([
|
||||
localHeader,
|
||||
entryNameBuffer,
|
||||
content,
|
||||
centralDirectoryHeader,
|
||||
entryNameBuffer,
|
||||
endOfCentralDirectory,
|
||||
])
|
||||
}
|
||||
|
||||
const toSkillArchiveUpload = async ({
|
||||
fileName,
|
||||
filePath,
|
||||
}: {
|
||||
fileName: string
|
||||
filePath: string
|
||||
}) => {
|
||||
if (fileName.endsWith('.zip') || fileName.endsWith('.skill')) {
|
||||
return {
|
||||
buffer: await readFile(filePath),
|
||||
name: path.basename(fileName),
|
||||
}
|
||||
}
|
||||
const sourceDirName = path.basename(path.dirname(fileName))
|
||||
const archiveBaseName = sourceDirName && sourceDirName !== '.'
|
||||
? sourceDirName
|
||||
: path.basename(fileName, path.extname(fileName))
|
||||
|
||||
return {
|
||||
buffer: createSingleFileZip({
|
||||
content: await readFile(filePath),
|
||||
entryName: 'SKILL.md',
|
||||
}),
|
||||
name: `${archiveBaseName}.skill`,
|
||||
}
|
||||
}
|
||||
|
||||
export async function uploadAgentDriveSkill({
|
||||
agentId,
|
||||
fileName,
|
||||
filePath,
|
||||
}: {
|
||||
agentId: string
|
||||
fileName: string
|
||||
filePath: string
|
||||
}): Promise<AgentSkillUploadResponse> {
|
||||
const ctx = await createApiContext()
|
||||
try {
|
||||
const upload = await toSkillArchiveUpload({ fileName, filePath })
|
||||
const response = await ctx.post(`/console/api/agent/${agentId}/skills/upload`, {
|
||||
multipart: {
|
||||
file: {
|
||||
buffer: upload.buffer,
|
||||
mimeType: 'application/zip',
|
||||
name: upload.name,
|
||||
},
|
||||
},
|
||||
})
|
||||
await expectApiResponseOK(response, `Upload Agent v2 drive skill ${fileName} for ${agentId}`)
|
||||
return (await response.json()) as AgentSkillUploadResponse
|
||||
}
|
||||
finally {
|
||||
await ctx.dispose()
|
||||
}
|
||||
}
|
||||
|
||||
export async function uploadAgentConfigFileToDraft({
|
||||
agentId,
|
||||
fileName,
|
||||
filePath,
|
||||
}: {
|
||||
agentId: string
|
||||
fileName: string
|
||||
filePath: string
|
||||
}): Promise<AgentConfigFileRefConfig> {
|
||||
const ctx = await createApiContext()
|
||||
try {
|
||||
const uploadResponse = await ctx.post('/console/api/files/upload', {
|
||||
multipart: {
|
||||
file: {
|
||||
buffer: await readFile(filePath),
|
||||
mimeType: 'text/plain',
|
||||
name: fileName,
|
||||
},
|
||||
},
|
||||
})
|
||||
await expectApiResponseOK(uploadResponse, `Upload Agent v2 config source file ${fileName}`)
|
||||
const uploadedFile = (await uploadResponse.json()) as UploadedConsoleFile
|
||||
|
||||
const commitResponse = await ctx.post(`/console/api/agent/${agentId}/config/files`, {
|
||||
data: {
|
||||
upload_file_id: uploadedFile.id,
|
||||
},
|
||||
})
|
||||
await expectApiResponseOK(commitResponse, `Commit Agent v2 config file ${fileName} for ${agentId}`)
|
||||
const body = (await commitResponse.json()) as AgentConfigFileUploadResponse
|
||||
const file = body.file
|
||||
if (!file.file_id)
|
||||
throw new Error(`Agent v2 config file ${fileName} did not return a file_id.`)
|
||||
|
||||
return {
|
||||
file_id: file.file_id,
|
||||
file_kind: 'upload_file',
|
||||
hash: file.hash,
|
||||
mime_type: file.mime_type,
|
||||
name: file.name,
|
||||
size: file.size,
|
||||
}
|
||||
}
|
||||
finally {
|
||||
await ctx.dispose()
|
||||
}
|
||||
}
|
||||
|
||||
export async function uploadAgentConfigSkillToDraft({
|
||||
agentId,
|
||||
fileName,
|
||||
filePath,
|
||||
}: {
|
||||
agentId: string
|
||||
fileName: string
|
||||
filePath: string
|
||||
}): Promise<AgentConfigSkillRefConfig> {
|
||||
const ctx = await createApiContext()
|
||||
try {
|
||||
const upload = await toSkillArchiveUpload({ fileName, filePath })
|
||||
const response = await ctx.post(`/console/api/agent/${agentId}/config/skills/upload`, {
|
||||
multipart: {
|
||||
file: {
|
||||
buffer: upload.buffer,
|
||||
mimeType: 'application/zip',
|
||||
name: upload.name,
|
||||
},
|
||||
},
|
||||
})
|
||||
await expectApiResponseOK(response, `Upload Agent v2 config skill ${fileName} for ${agentId}`)
|
||||
const body = (await response.json()) as AgentConfigSkillUploadResponse
|
||||
const skill = body.skill
|
||||
if (!skill.file_id)
|
||||
throw new Error(`Agent v2 config skill ${fileName} did not return a file_id.`)
|
||||
|
||||
return {
|
||||
description: skill.description,
|
||||
file_id: skill.file_id,
|
||||
file_kind: 'tool_file',
|
||||
hash: skill.hash,
|
||||
mime_type: skill.mime_type,
|
||||
name: skill.name,
|
||||
size: skill.size,
|
||||
}
|
||||
}
|
||||
finally {
|
||||
await ctx.dispose()
|
||||
}
|
||||
}
|
||||
|
||||
export async function getAgentDriveSkills(agentId: string): Promise<AgentDriveSkillItemResponse[]> {
|
||||
const ctx = await createApiContext()
|
||||
try {
|
||||
const response = await ctx.get(`/console/api/agent/${agentId}/drive/skills`)
|
||||
await expectApiResponseOK(response, `Get Agent v2 drive skills for ${agentId}`)
|
||||
const body = (await response.json()) as AgentDriveSkillListResponse
|
||||
return body.items ?? []
|
||||
}
|
||||
finally {
|
||||
await ctx.dispose()
|
||||
}
|
||||
}
|
||||
|
||||
export async function deleteAgentConfigFile(agentId: string, name: string): Promise<void> {
|
||||
const ctx = await createApiContext()
|
||||
try {
|
||||
const response = await ctx.delete(`/console/api/agent/${agentId}/config/files/${encodeURIComponent(name)}`)
|
||||
await expectApiResponseOK(response, `Delete Agent v2 config file ${name} for ${agentId}`)
|
||||
}
|
||||
finally {
|
||||
await ctx.dispose()
|
||||
}
|
||||
}
|
||||
|
||||
export async function deleteAgentConfigSkill(agentId: string, name: string): Promise<void> {
|
||||
const ctx = await createApiContext()
|
||||
try {
|
||||
const response = await ctx.delete(`/console/api/agent/${agentId}/config/skills/${encodeURIComponent(name)}`)
|
||||
await expectApiResponseOK(response, `Delete Agent v2 config skill ${name} for ${agentId}`)
|
||||
}
|
||||
finally {
|
||||
await ctx.dispose()
|
||||
}
|
||||
}
|
||||
|
||||
export async function deleteAgentDriveFile(agentId: string, key: string): Promise<void> {
|
||||
const ctx = await createApiContext()
|
||||
try {
|
||||
const searchParams = new URLSearchParams({ key })
|
||||
const response = await ctx.delete(`/console/api/agent/${agentId}/files?${searchParams}`)
|
||||
await expectApiResponseOK(response, `Delete Agent v2 drive file ${key} for ${agentId}`)
|
||||
}
|
||||
finally {
|
||||
await ctx.dispose()
|
||||
}
|
||||
}
|
||||
105
e2e/features/agent-v2/support/agent-soul.ts
Normal file
105
e2e/features/agent-v2/support/agent-soul.ts
Normal file
@ -0,0 +1,105 @@
|
||||
import type {
|
||||
AgentKnowledgeDatasetConfig,
|
||||
AgentSoulConfig,
|
||||
} from '@dify/contracts/api/console/agent/types.gen'
|
||||
|
||||
export type AgentComposerEnvVariable = NonNullable<
|
||||
NonNullable<AgentSoulConfig['env']>['variables']
|
||||
>[number]
|
||||
|
||||
export type AgentModelSelection = {
|
||||
name: string
|
||||
provider: string
|
||||
}
|
||||
|
||||
export const defaultAgentSoulConfig: AgentSoulConfig = {
|
||||
prompt: {
|
||||
system_prompt: 'You are a Dify Agent E2E test assistant.',
|
||||
},
|
||||
}
|
||||
|
||||
export const normalAgentPrompt
|
||||
= 'You are a Dify Agent E2E test assistant. Reply briefly to every user message, and always include AGENT_E2E_PASS in your response.'
|
||||
|
||||
export const updatedAgentPrompt
|
||||
= 'You are a Dify Agent E2E test assistant. Every response must start with E2E_AGENT_UPDATED.'
|
||||
|
||||
export const concurrentFirstAgentPrompt
|
||||
= 'You are a Dify Agent E2E concurrent edit assistant. Always include E2E_CONCURRENT_FIRST in saved instructions.'
|
||||
|
||||
export const concurrentSecondAgentPrompt
|
||||
= 'You are a Dify Agent E2E concurrent edit assistant. Always include E2E_CONCURRENT_SECOND in saved instructions.'
|
||||
|
||||
export const normalAgentSoulConfig: AgentSoulConfig = {
|
||||
prompt: {
|
||||
system_prompt: normalAgentPrompt,
|
||||
},
|
||||
}
|
||||
|
||||
export const updatedAgentSoulConfig: AgentSoulConfig = {
|
||||
prompt: {
|
||||
system_prompt: updatedAgentPrompt,
|
||||
},
|
||||
}
|
||||
|
||||
const getAgentModelPluginId = (provider: string) => {
|
||||
const [organization, pluginName] = provider.split('/').filter(Boolean)
|
||||
|
||||
if (organization && pluginName)
|
||||
return `${organization}/${pluginName}`
|
||||
|
||||
return provider ? `langgenius/${provider}` : ''
|
||||
}
|
||||
|
||||
const getExistingModelConfig = (agentSoul: AgentSoulConfig) => {
|
||||
const model = agentSoul.model
|
||||
|
||||
if (model && typeof model === 'object' && !Array.isArray(model))
|
||||
return model as Record<string, unknown>
|
||||
|
||||
return {}
|
||||
}
|
||||
|
||||
export function createAgentSoulConfigWithModel(
|
||||
agentSoul: AgentSoulConfig,
|
||||
model: AgentModelSelection,
|
||||
): AgentSoulConfig {
|
||||
return {
|
||||
...agentSoul,
|
||||
model: {
|
||||
...getExistingModelConfig(agentSoul),
|
||||
plugin_id: getAgentModelPluginId(model.provider),
|
||||
model_provider: model.provider,
|
||||
model: model.name,
|
||||
model_settings: {
|
||||
temperature: 0,
|
||||
max_tokens: 512,
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
export function createAgentSoulConfigWithKnowledgeDataset(
|
||||
agentSoul: AgentSoulConfig,
|
||||
dataset: AgentKnowledgeDatasetConfig,
|
||||
): AgentSoulConfig {
|
||||
return {
|
||||
...agentSoul,
|
||||
knowledge: {
|
||||
sets: [
|
||||
{
|
||||
datasets: [dataset],
|
||||
id: 'e2e-knowledge-retrieval',
|
||||
name: 'Retrieval 1',
|
||||
query: {
|
||||
mode: 'generated_query',
|
||||
},
|
||||
retrieval: {
|
||||
mode: 'multiple',
|
||||
top_k: 4,
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
}
|
||||
}
|
||||
170
e2e/features/agent-v2/support/agent.ts
Normal file
170
e2e/features/agent-v2/support/agent.ts
Normal file
@ -0,0 +1,170 @@
|
||||
import type {
|
||||
AgentAppComposerResponse,
|
||||
AgentAppDetailWithSite,
|
||||
AgentConfigSnapshotDetailResponse,
|
||||
AgentReferencingWorkflowResponse,
|
||||
AgentReferencingWorkflowsResponse,
|
||||
AgentSoulConfig,
|
||||
} from '@dify/contracts/api/console/agent/types.gen'
|
||||
import { createApiContext, expectApiResponseOK } from '../../../support/api'
|
||||
import { assertE2EResourceName, createE2EResourceName } from '../../../support/naming'
|
||||
import { defaultAgentSoulConfig, normalAgentSoulConfig } from './agent-soul'
|
||||
|
||||
export type AgentSeed = Pick<
|
||||
AgentAppDetailWithSite,
|
||||
| 'active_config_is_published'
|
||||
| 'app_id'
|
||||
| 'backing_app_id'
|
||||
| 'description'
|
||||
| 'enable_site'
|
||||
| 'id'
|
||||
| 'name'
|
||||
| 'role'
|
||||
| 'site'
|
||||
> & {
|
||||
active_config_snapshot_id?: string | null
|
||||
}
|
||||
|
||||
export type CreateTestAgentOptions = {
|
||||
description?: string
|
||||
name?: string
|
||||
role?: string
|
||||
}
|
||||
|
||||
export const getAgentConfigurePath = (agentId: string) => `/roster/agent/${agentId}/configure`
|
||||
export const getAgentAccessPath = (agentId: string) => `/roster/agent/${agentId}/access`
|
||||
|
||||
export async function createTestAgent({
|
||||
description = 'Created by Dify E2E.',
|
||||
name = createE2EResourceName('Agent'),
|
||||
role = 'E2E test assistant',
|
||||
}: CreateTestAgentOptions = {}): Promise<AgentSeed> {
|
||||
assertE2EResourceName(name, 'Agent')
|
||||
const ctx = await createApiContext()
|
||||
try {
|
||||
const response = await ctx.post('/console/api/agent', {
|
||||
data: {
|
||||
description,
|
||||
icon: '🤖',
|
||||
icon_background: '#FFEAD5',
|
||||
icon_type: 'emoji',
|
||||
name,
|
||||
role,
|
||||
},
|
||||
})
|
||||
await expectApiResponseOK(response, 'Create Agent v2 test agent')
|
||||
return (await response.json()) as AgentSeed
|
||||
}
|
||||
finally {
|
||||
await ctx.dispose()
|
||||
}
|
||||
}
|
||||
|
||||
export async function createConfiguredTestAgent({
|
||||
agentSoul = normalAgentSoulConfig,
|
||||
seed,
|
||||
}: {
|
||||
agentSoul?: AgentSoulConfig
|
||||
seed?: CreateTestAgentOptions
|
||||
} = {}): Promise<AgentSeed> {
|
||||
const agent = await createTestAgent(seed)
|
||||
await saveAgentComposerDraft(agent.id, agentSoul)
|
||||
return agent
|
||||
}
|
||||
|
||||
export async function getTestAgent(agentId: string): Promise<AgentSeed> {
|
||||
const ctx = await createApiContext()
|
||||
try {
|
||||
const response = await ctx.get(`/console/api/agent/${agentId}`)
|
||||
await expectApiResponseOK(response, `Get Agent v2 test agent ${agentId}`)
|
||||
return (await response.json()) as AgentSeed
|
||||
}
|
||||
finally {
|
||||
await ctx.dispose()
|
||||
}
|
||||
}
|
||||
|
||||
export async function getAgentVersionDetail(
|
||||
agentId: string,
|
||||
versionId: string,
|
||||
): Promise<AgentConfigSnapshotDetailResponse> {
|
||||
const ctx = await createApiContext()
|
||||
try {
|
||||
const response = await ctx.get(`/console/api/agent/${agentId}/versions/${versionId}`)
|
||||
await expectApiResponseOK(response, `Get Agent v2 version ${versionId} for ${agentId}`)
|
||||
return (await response.json()) as AgentConfigSnapshotDetailResponse
|
||||
}
|
||||
finally {
|
||||
await ctx.dispose()
|
||||
}
|
||||
}
|
||||
|
||||
export async function deleteTestAgent(agentId: string): Promise<void> {
|
||||
const ctx = await createApiContext()
|
||||
try {
|
||||
const response = await ctx.delete(`/console/api/agent/${agentId}`)
|
||||
await expectApiResponseOK(response, `Delete Agent v2 test agent ${agentId}`)
|
||||
}
|
||||
finally {
|
||||
await ctx.dispose()
|
||||
}
|
||||
}
|
||||
|
||||
export async function saveAgentComposerDraft(
|
||||
agentId: string,
|
||||
agentSoul: AgentSoulConfig = defaultAgentSoulConfig,
|
||||
): Promise<AgentAppComposerResponse> {
|
||||
const ctx = await createApiContext()
|
||||
try {
|
||||
const response = await ctx.put(`/console/api/agent/${agentId}/composer`, {
|
||||
data: {
|
||||
agent_soul: agentSoul,
|
||||
save_strategy: 'save_to_current_version',
|
||||
variant: 'agent_app',
|
||||
},
|
||||
})
|
||||
await expectApiResponseOK(response, `Save Agent v2 composer draft for ${agentId}`)
|
||||
return (await response.json()) as AgentAppComposerResponse
|
||||
}
|
||||
finally {
|
||||
await ctx.dispose()
|
||||
}
|
||||
}
|
||||
|
||||
export async function getAgentReferencingWorkflows(agentId: string): Promise<AgentReferencingWorkflowResponse[]> {
|
||||
const ctx = await createApiContext()
|
||||
try {
|
||||
const response = await ctx.get(`/console/api/agent/${agentId}/referencing-workflows`)
|
||||
await expectApiResponseOK(response, `Get Agent v2 referencing workflows for ${agentId}`)
|
||||
const body = (await response.json()) as AgentReferencingWorkflowsResponse
|
||||
return body.data ?? []
|
||||
}
|
||||
finally {
|
||||
await ctx.dispose()
|
||||
}
|
||||
}
|
||||
|
||||
export async function getAgentComposerDraft(agentId: string): Promise<AgentAppComposerResponse> {
|
||||
const ctx = await createApiContext()
|
||||
try {
|
||||
const response = await ctx.get(`/console/api/agent/${agentId}/composer`)
|
||||
await expectApiResponseOK(response, `Get Agent v2 composer draft for ${agentId}`)
|
||||
return (await response.json()) as AgentAppComposerResponse
|
||||
}
|
||||
finally {
|
||||
await ctx.dispose()
|
||||
}
|
||||
}
|
||||
|
||||
export async function publishAgent(agentId: string, versionNote = 'E2E publish'): Promise<void> {
|
||||
const ctx = await createApiContext()
|
||||
try {
|
||||
const response = await ctx.post(`/console/api/agent/${agentId}/publish`, {
|
||||
data: { version_note: versionNote },
|
||||
})
|
||||
await expectApiResponseOK(response, `Publish Agent v2 test agent ${agentId}`)
|
||||
}
|
||||
finally {
|
||||
await ctx.dispose()
|
||||
}
|
||||
}
|
||||
143
e2e/features/agent-v2/support/preflight/access.ts
Normal file
143
e2e/features/agent-v2/support/preflight/access.ts
Normal file
@ -0,0 +1,143 @@
|
||||
import type {
|
||||
AgentApiAccessResponse,
|
||||
AgentAppDetailWithSite,
|
||||
AgentReferencingWorkflowsResponse,
|
||||
ApiKeyList,
|
||||
} from '@dify/contracts/api/console/agent/types.gen'
|
||||
import type { DifyWorld } from '../../../support/world'
|
||||
import type { PreseededResource } from './common'
|
||||
import { createApiContext, expectApiResponseOK } from '../../../../support/api'
|
||||
import { skipMissingPreseededAgent, skipMissingPreseededWorkflow } from './agents'
|
||||
import { skipBlockedPrecondition } from './common'
|
||||
|
||||
export async function skipMissingPreseededAgentBackendApiKey(
|
||||
world: DifyWorld,
|
||||
agentName: string,
|
||||
): Promise<'skipped' | PreseededResource> {
|
||||
const agent = await skipMissingPreseededAgent(world, agentName)
|
||||
if (agent === 'skipped')
|
||||
return agent
|
||||
|
||||
const ctx = await createApiContext()
|
||||
try {
|
||||
const accessResponse = await ctx.get(`/console/api/agent/${agent.id}/api-access`)
|
||||
await expectApiResponseOK(accessResponse, `Check preseeded Agent API access ${agentName}`)
|
||||
const access = (await accessResponse.json()) as AgentApiAccessResponse
|
||||
if (!access.enabled || access.api_key_count < 1) {
|
||||
return skipBlockedPrecondition(
|
||||
world,
|
||||
`Preseeded Agent "${agentName}" does not have Backend service API enabled with an API key.`,
|
||||
)
|
||||
}
|
||||
|
||||
const keyResponse = await ctx.get(`/console/api/agent/${agent.id}/api-keys`)
|
||||
await expectApiResponseOK(keyResponse, `Check preseeded Agent API key ${agentName}`)
|
||||
const keys = (await keyResponse.json()) as ApiKeyList
|
||||
const key = keys.data.at(0)
|
||||
if (!key) {
|
||||
return skipBlockedPrecondition(
|
||||
world,
|
||||
`Preseeded Agent "${agentName}" Backend service API key list is empty.`,
|
||||
)
|
||||
}
|
||||
|
||||
return {
|
||||
id: key.id,
|
||||
kind: 'api-key',
|
||||
name: `${agentName} Backend service API key`,
|
||||
}
|
||||
}
|
||||
finally {
|
||||
await ctx.dispose()
|
||||
}
|
||||
}
|
||||
|
||||
export async function skipMissingPreseededAgentPublishedWebApp(
|
||||
world: DifyWorld,
|
||||
agentName: string,
|
||||
): Promise<'skipped' | PreseededResource> {
|
||||
const agent = await skipMissingPreseededAgent(world, agentName)
|
||||
if (agent === 'skipped')
|
||||
return agent
|
||||
|
||||
const ctx = await createApiContext()
|
||||
try {
|
||||
const response = await ctx.get(`/console/api/agent/${agent.id}`)
|
||||
await expectApiResponseOK(response, `Check preseeded Agent published Web app ${agentName}`)
|
||||
const detail = (await response.json()) as AgentAppDetailWithSite
|
||||
if (detail.active_config_is_published !== true) {
|
||||
return skipBlockedPrecondition(world, `Preseeded Agent "${agentName}" is not published.`)
|
||||
}
|
||||
|
||||
if (detail.enable_site !== true) {
|
||||
return skipBlockedPrecondition(
|
||||
world,
|
||||
`Preseeded Agent "${agentName}" Web app is not enabled.`,
|
||||
)
|
||||
}
|
||||
|
||||
const siteToken = detail.site?.access_token ?? detail.site?.code
|
||||
if (!siteToken || !detail.site?.app_base_url) {
|
||||
return skipBlockedPrecondition(
|
||||
world,
|
||||
`Preseeded Agent "${agentName}" Web app URL is not available.`,
|
||||
)
|
||||
}
|
||||
|
||||
return {
|
||||
id: agent.id,
|
||||
kind: 'agent',
|
||||
name: agent.name,
|
||||
}
|
||||
}
|
||||
finally {
|
||||
await ctx.dispose()
|
||||
}
|
||||
}
|
||||
|
||||
export async function skipMissingPreseededAgentWorkflowReference(
|
||||
world: DifyWorld,
|
||||
agentName: string,
|
||||
workflowName: string,
|
||||
): Promise<'skipped' | PreseededResource> {
|
||||
const agent = await skipMissingPreseededAgent(world, agentName)
|
||||
if (agent === 'skipped')
|
||||
return agent
|
||||
|
||||
const workflow = await skipMissingPreseededWorkflow(world, workflowName)
|
||||
if (workflow === 'skipped')
|
||||
return workflow
|
||||
|
||||
const ctx = await createApiContext()
|
||||
try {
|
||||
const response = await ctx.get(`/console/api/agent/${agent.id}/referencing-workflows`)
|
||||
await expectApiResponseOK(response, `Check preseeded Agent workflow reference ${agentName}`)
|
||||
const references = (await response.json()) as AgentReferencingWorkflowsResponse
|
||||
const reference = references.data?.find(
|
||||
item => item.app_id === workflow.id || item.app_name === workflow.name,
|
||||
)
|
||||
|
||||
if (!reference) {
|
||||
return skipBlockedPrecondition(
|
||||
world,
|
||||
`Preseeded Agent "${agentName}" is not referenced by workflow "${workflowName}".`,
|
||||
)
|
||||
}
|
||||
|
||||
if (!reference.node_ids || reference.node_ids.length < 1) {
|
||||
return skipBlockedPrecondition(
|
||||
world,
|
||||
`Preseeded workflow "${workflowName}" does not expose Agent reference nodes for "${agentName}".`,
|
||||
)
|
||||
}
|
||||
|
||||
return {
|
||||
id: workflow.id,
|
||||
kind: 'workflow',
|
||||
name: workflow.name,
|
||||
}
|
||||
}
|
||||
finally {
|
||||
await ctx.dispose()
|
||||
}
|
||||
}
|
||||
470
e2e/features/agent-v2/support/preflight/agents.ts
Normal file
470
e2e/features/agent-v2/support/preflight/agents.ts
Normal file
@ -0,0 +1,470 @@
|
||||
import type {
|
||||
AgentAppComposerResponse,
|
||||
AgentDriveListResponse,
|
||||
AgentDriveSkillListResponse,
|
||||
} from '@dify/contracts/api/console/agent/types.gen'
|
||||
import type { DifyWorld } from '../../../support/world'
|
||||
import type { PreseededResource } from './common'
|
||||
import { createApiContext, expectApiResponseOK } from '../../../../support/api'
|
||||
import {
|
||||
agentBuilderExpectedTokens,
|
||||
agentBuilderFixedInputs,
|
||||
agentBuilderPreseededResources,
|
||||
} from '../agent-builder-resources'
|
||||
import {
|
||||
agentBuilderFileTreeFixtureFileNames,
|
||||
agentBuilderFileTreeFixtureFiles,
|
||||
agentBuilderTestMaterials,
|
||||
} from '../test-materials'
|
||||
import {
|
||||
asArray,
|
||||
asRecord,
|
||||
asString,
|
||||
buildQuery,
|
||||
findConsoleResourceByName,
|
||||
hasNamedOrKeyedEntry,
|
||||
skipBlockedPrecondition,
|
||||
} from './common'
|
||||
import { skipMissingReadyPreseededDataset } from './datasets'
|
||||
import { skipMissingAgentBuilderStableChatModel } from './models'
|
||||
import {
|
||||
findToolEntry,
|
||||
hasToolEntry,
|
||||
hasUnauthorizedToolCredentialState,
|
||||
skipMissingPreseededTool,
|
||||
splitToolDisplayName,
|
||||
} from './tools'
|
||||
|
||||
const hasKnowledgeDataset = (
|
||||
soul: Record<string, unknown>,
|
||||
dataset: PreseededResource,
|
||||
) => {
|
||||
const knowledge = asRecord(soul.knowledge)
|
||||
const sets = asArray(knowledge.sets)
|
||||
|
||||
return sets.some((set) => {
|
||||
const datasets = asArray(asRecord(set).datasets)
|
||||
|
||||
return datasets.some((item) => {
|
||||
const record = asRecord(item)
|
||||
return record.id === dataset.id || record.name === dataset.name
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
const hasKnowledgeSet = (
|
||||
soul: Record<string, unknown>,
|
||||
dataset: PreseededResource,
|
||||
{
|
||||
queryMode,
|
||||
queryValue,
|
||||
}: {
|
||||
queryMode: 'generated_query' | 'user_query'
|
||||
queryValue?: string
|
||||
},
|
||||
) => {
|
||||
const knowledge = asRecord(soul.knowledge)
|
||||
const sets = asArray(knowledge.sets)
|
||||
|
||||
return sets.some((set) => {
|
||||
const record = asRecord(set)
|
||||
const query = asRecord(record.query)
|
||||
const datasets = asArray(record.datasets)
|
||||
const hasExpectedDataset = datasets.some((item) => {
|
||||
const datasetRecord = asRecord(item)
|
||||
return datasetRecord.id === dataset.id || datasetRecord.name === dataset.name
|
||||
})
|
||||
|
||||
if (!hasExpectedDataset || query.mode !== queryMode)
|
||||
return false
|
||||
if (queryValue === undefined)
|
||||
return true
|
||||
|
||||
return asString(query.value).trim() === queryValue
|
||||
})
|
||||
}
|
||||
|
||||
export async function skipMissingPreseededAgent(
|
||||
world: DifyWorld,
|
||||
resourceName: string,
|
||||
): Promise<'skipped' | PreseededResource> {
|
||||
const query = buildQuery({ limit: '20', name: resourceName, page: '1' })
|
||||
const resource = await findConsoleResourceByName({
|
||||
action: `Check preseeded Agent ${resourceName}`,
|
||||
path: `/console/api/agent?${query}`,
|
||||
resourceName,
|
||||
})
|
||||
|
||||
if (!resource)
|
||||
return skipBlockedPrecondition(world, `Preseeded Agent "${resourceName}" was not found.`)
|
||||
|
||||
return {
|
||||
id: resource.id,
|
||||
kind: 'agent',
|
||||
name: resource.name,
|
||||
}
|
||||
}
|
||||
|
||||
export async function skipMissingPreseededWorkflow(
|
||||
world: DifyWorld,
|
||||
resourceName: string,
|
||||
): Promise<'skipped' | PreseededResource> {
|
||||
const query = buildQuery({ limit: '20', mode: 'workflow', name: resourceName, page: '1' })
|
||||
const resource = await findConsoleResourceByName({
|
||||
action: `Check preseeded workflow ${resourceName}`,
|
||||
path: `/console/api/apps?${query}`,
|
||||
resourceName,
|
||||
})
|
||||
|
||||
if (!resource)
|
||||
return skipBlockedPrecondition(world, `Preseeded workflow "${resourceName}" was not found.`)
|
||||
|
||||
return {
|
||||
id: resource.id,
|
||||
kind: 'workflow',
|
||||
name: resource.name,
|
||||
}
|
||||
}
|
||||
|
||||
export async function skipMissingPreseededAgentDriveSkill(
|
||||
world: DifyWorld,
|
||||
agentName: string,
|
||||
skillName: string,
|
||||
): Promise<'skipped' | PreseededResource> {
|
||||
const agent = await skipMissingPreseededAgent(world, agentName)
|
||||
if (agent === 'skipped')
|
||||
return agent
|
||||
|
||||
const ctx = await createApiContext()
|
||||
try {
|
||||
const response = await ctx.get(`/console/api/agent/${agent.id}/drive/skills`)
|
||||
await expectApiResponseOK(response, `Check preseeded Agent skill ${skillName}`)
|
||||
const body = (await response.json()) as AgentDriveSkillListResponse
|
||||
const skill = body.items?.find(item => item.name === skillName)
|
||||
|
||||
if (!skill) {
|
||||
return skipBlockedPrecondition(
|
||||
world,
|
||||
`Preseeded Agent "${agentName}" does not include drive skill "${skillName}".`,
|
||||
)
|
||||
}
|
||||
|
||||
return {
|
||||
id: skill.path,
|
||||
kind: 'skill',
|
||||
name: skill.name,
|
||||
}
|
||||
}
|
||||
finally {
|
||||
await ctx.dispose()
|
||||
}
|
||||
}
|
||||
|
||||
export async function skipMissingPreseededFullConfigAgentCoreConfiguration(
|
||||
world: DifyWorld,
|
||||
agentName: string,
|
||||
): Promise<'skipped' | PreseededResource> {
|
||||
const stableModel = await skipMissingAgentBuilderStableChatModel(world)
|
||||
if (stableModel === 'skipped')
|
||||
return stableModel
|
||||
|
||||
const agent = await skipMissingPreseededAgent(world, agentName)
|
||||
if (agent === 'skipped')
|
||||
return agent
|
||||
|
||||
const summarySkill = await skipMissingPreseededAgentDriveSkill(
|
||||
world,
|
||||
agentName,
|
||||
agentBuilderPreseededResources.summarySkill,
|
||||
)
|
||||
if (summarySkill === 'skipped')
|
||||
return summarySkill
|
||||
|
||||
const jsonTool = await skipMissingPreseededTool(
|
||||
world,
|
||||
agentBuilderPreseededResources.jsonReplaceTool,
|
||||
)
|
||||
if (jsonTool === 'skipped')
|
||||
return jsonTool
|
||||
|
||||
const knowledgeBase = await skipMissingReadyPreseededDataset(
|
||||
world,
|
||||
agentBuilderPreseededResources.agentKnowledgeBase,
|
||||
)
|
||||
if (knowledgeBase === 'skipped')
|
||||
return knowledgeBase
|
||||
|
||||
const ctx = await createApiContext()
|
||||
try {
|
||||
const response = await ctx.get(`/console/api/agent/${agent.id}/composer`)
|
||||
await expectApiResponseOK(response, `Check preseeded Agent core configuration ${agentName}`)
|
||||
const body = (await response.json()) as AgentAppComposerResponse
|
||||
const soul = body.agent_soul ?? {}
|
||||
const missing: string[] = []
|
||||
|
||||
const model = asRecord(soul.model)
|
||||
if (model.model_provider !== stableModel.provider || model.model !== stableModel.name)
|
||||
missing.push(`${agentBuilderPreseededResources.stableChatModel} model config`)
|
||||
|
||||
const prompt = asString(asRecord(soul.prompt).system_prompt)
|
||||
if (!prompt.includes(agentBuilderExpectedTokens.agentReply))
|
||||
missing.push(`Prompt token ${agentBuilderExpectedTokens.agentReply}`)
|
||||
|
||||
const files = asArray(soul.config_files)
|
||||
for (const fileName of [
|
||||
agentBuilderTestMaterials.smallFile,
|
||||
agentBuilderTestMaterials.specialFilename,
|
||||
]) {
|
||||
if (!hasNamedOrKeyedEntry(files, fileName))
|
||||
missing.push(`file ${fileName}`)
|
||||
}
|
||||
|
||||
const skills = asArray(soul.config_skills)
|
||||
if (!hasNamedOrKeyedEntry(skills, agentBuilderPreseededResources.summarySkill))
|
||||
missing.push(agentBuilderPreseededResources.summarySkill)
|
||||
|
||||
const [providerName = '', toolName = ''] = jsonTool.id.split('/')
|
||||
const parsedTool = splitToolDisplayName(agentBuilderPreseededResources.jsonReplaceTool)
|
||||
if (
|
||||
parsedTool.ok
|
||||
&& !hasToolEntry(asArray(asRecord(soul.tools).dify_tools), {
|
||||
providerDisplayName: parsedTool.providerName,
|
||||
providerName,
|
||||
toolDisplayName: parsedTool.toolName,
|
||||
toolName,
|
||||
})
|
||||
) {
|
||||
missing.push(agentBuilderPreseededResources.jsonReplaceTool)
|
||||
}
|
||||
|
||||
if (!hasKnowledgeDataset(soul, knowledgeBase))
|
||||
missing.push(agentBuilderPreseededResources.agentKnowledgeBase)
|
||||
|
||||
if (missing.length > 0) {
|
||||
return skipBlockedPrecondition(
|
||||
world,
|
||||
`Preseeded Agent "${agentName}" is missing core fixture configuration: ${missing.join(', ')}.`,
|
||||
)
|
||||
}
|
||||
|
||||
return agent
|
||||
}
|
||||
finally {
|
||||
await ctx.dispose()
|
||||
}
|
||||
}
|
||||
|
||||
export async function skipMissingPreseededToolStatesAgentConfiguration(
|
||||
world: DifyWorld,
|
||||
agentName: string,
|
||||
): Promise<'skipped' | PreseededResource> {
|
||||
const agent = await skipMissingPreseededAgent(world, agentName)
|
||||
if (agent === 'skipped')
|
||||
return agent
|
||||
|
||||
const summarySkill = await skipMissingPreseededAgentDriveSkill(
|
||||
world,
|
||||
agentName,
|
||||
agentBuilderPreseededResources.summarySkill,
|
||||
)
|
||||
if (summarySkill === 'skipped')
|
||||
return summarySkill
|
||||
|
||||
const jsonTool = await skipMissingPreseededTool(
|
||||
world,
|
||||
agentBuilderPreseededResources.jsonReplaceTool,
|
||||
)
|
||||
if (jsonTool === 'skipped')
|
||||
return jsonTool
|
||||
|
||||
const tavilyTool = await skipMissingPreseededTool(
|
||||
world,
|
||||
agentBuilderPreseededResources.tavilySearchTool,
|
||||
)
|
||||
if (tavilyTool === 'skipped')
|
||||
return tavilyTool
|
||||
|
||||
const ctx = await createApiContext()
|
||||
try {
|
||||
const response = await ctx.get(`/console/api/agent/${agent.id}/composer`)
|
||||
await expectApiResponseOK(response, `Check preseeded Agent tool states ${agentName}`)
|
||||
const body = (await response.json()) as AgentAppComposerResponse
|
||||
const soul = body.agent_soul ?? {}
|
||||
const toolItems = asArray(asRecord(soul.tools).dify_tools)
|
||||
const missing: string[] = []
|
||||
|
||||
const skills = asArray(soul.config_skills)
|
||||
if (!hasNamedOrKeyedEntry(skills, agentBuilderPreseededResources.summarySkill))
|
||||
missing.push(agentBuilderPreseededResources.summarySkill)
|
||||
|
||||
const [jsonProviderName = '', jsonToolName = ''] = jsonTool.id.split('/')
|
||||
const parsedJsonTool = splitToolDisplayName(agentBuilderPreseededResources.jsonReplaceTool)
|
||||
if (
|
||||
parsedJsonTool.ok
|
||||
&& !findToolEntry(toolItems, {
|
||||
providerDisplayName: parsedJsonTool.providerName,
|
||||
providerName: jsonProviderName,
|
||||
toolDisplayName: parsedJsonTool.toolName,
|
||||
toolName: jsonToolName,
|
||||
})
|
||||
) {
|
||||
missing.push(agentBuilderPreseededResources.jsonReplaceTool)
|
||||
}
|
||||
|
||||
const [tavilyProviderName = '', tavilyToolName = ''] = tavilyTool.id.split('/')
|
||||
const parsedTavilyTool = splitToolDisplayName(agentBuilderPreseededResources.tavilySearchTool)
|
||||
const tavilyEntry = parsedTavilyTool.ok
|
||||
? findToolEntry(toolItems, {
|
||||
providerDisplayName: parsedTavilyTool.providerName,
|
||||
providerName: tavilyProviderName,
|
||||
toolDisplayName: parsedTavilyTool.toolName,
|
||||
toolName: tavilyToolName,
|
||||
})
|
||||
: undefined
|
||||
|
||||
if (!tavilyEntry) {
|
||||
missing.push(agentBuilderPreseededResources.tavilySearchTool)
|
||||
}
|
||||
else if (!hasUnauthorizedToolCredentialState(tavilyEntry)) {
|
||||
missing.push(`${agentBuilderPreseededResources.tavilySearchTool} unauthorized credential state`)
|
||||
}
|
||||
|
||||
if (missing.length > 0) {
|
||||
return skipBlockedPrecondition(
|
||||
world,
|
||||
`Preseeded Agent "${agentName}" is missing tool state fixture configuration: ${missing.join(', ')}.`,
|
||||
)
|
||||
}
|
||||
|
||||
return agent
|
||||
}
|
||||
finally {
|
||||
await ctx.dispose()
|
||||
}
|
||||
}
|
||||
|
||||
export async function skipMissingPreseededDualRetrievalAgentConfiguration(
|
||||
world: DifyWorld,
|
||||
agentName: string,
|
||||
): Promise<'skipped' | PreseededResource> {
|
||||
const agent = await skipMissingPreseededAgent(world, agentName)
|
||||
if (agent === 'skipped')
|
||||
return agent
|
||||
|
||||
const knowledgeBase = await skipMissingReadyPreseededDataset(
|
||||
world,
|
||||
agentBuilderPreseededResources.agentKnowledgeBase,
|
||||
)
|
||||
if (knowledgeBase === 'skipped')
|
||||
return knowledgeBase
|
||||
|
||||
const ctx = await createApiContext()
|
||||
try {
|
||||
const response = await ctx.get(`/console/api/agent/${agent.id}/composer`)
|
||||
await expectApiResponseOK(response, `Check preseeded Agent dual retrieval ${agentName}`)
|
||||
const body = (await response.json()) as AgentAppComposerResponse
|
||||
const soul = body.agent_soul ?? {}
|
||||
const missing: string[] = []
|
||||
|
||||
if (!hasKnowledgeSet(soul, knowledgeBase, { queryMode: 'generated_query' }))
|
||||
missing.push('Agent decide Knowledge Retrieval')
|
||||
|
||||
if (
|
||||
!hasKnowledgeSet(soul, knowledgeBase, {
|
||||
queryMode: 'user_query',
|
||||
queryValue: agentBuilderFixedInputs.customKnowledgeQuery,
|
||||
})
|
||||
) {
|
||||
missing.push('Custom query Knowledge Retrieval')
|
||||
}
|
||||
|
||||
if (missing.length > 0) {
|
||||
return skipBlockedPrecondition(
|
||||
world,
|
||||
`Preseeded Agent "${agentName}" is missing dual retrieval fixture configuration: ${missing.join(', ')}.`,
|
||||
)
|
||||
}
|
||||
|
||||
return agent
|
||||
}
|
||||
finally {
|
||||
await ctx.dispose()
|
||||
}
|
||||
}
|
||||
|
||||
export async function skipMissingPreseededAgentFileTreeFixture(
|
||||
world: DifyWorld,
|
||||
agentName: string,
|
||||
): Promise<'skipped' | PreseededResource> {
|
||||
const agent = await skipMissingPreseededAgent(world, agentName)
|
||||
if (agent === 'skipped')
|
||||
return agent
|
||||
|
||||
const ctx = await createApiContext()
|
||||
try {
|
||||
const query = buildQuery({ prefix: 'files/' })
|
||||
const response = await ctx.get(`/console/api/agent/${agent.id}/drive/files?${query}`)
|
||||
await expectApiResponseOK(response, `Check preseeded Agent file tree ${agentName}`)
|
||||
const body = (await response.json()) as AgentDriveListResponse
|
||||
const keys = (body.items ?? []).map(item => item.key)
|
||||
const missingFiles = agentBuilderFileTreeFixtureFiles.filter(
|
||||
filePath =>
|
||||
!keys.some(key => key === `files/${filePath}` || key.endsWith(`/${filePath}`)),
|
||||
)
|
||||
|
||||
if (missingFiles.length > 0) {
|
||||
return skipBlockedPrecondition(
|
||||
world,
|
||||
`Preseeded Agent "${agentName}" is missing file tree fixture files: ${missingFiles.join(', ')}.`,
|
||||
)
|
||||
}
|
||||
|
||||
return {
|
||||
id: agent.id,
|
||||
kind: 'agent',
|
||||
name: agent.name,
|
||||
}
|
||||
}
|
||||
finally {
|
||||
await ctx.dispose()
|
||||
}
|
||||
}
|
||||
|
||||
export async function skipMissingPreseededAgentFlatFileFixtureConfiguration(
|
||||
world: DifyWorld,
|
||||
agentName: string,
|
||||
): Promise<'skipped' | PreseededResource> {
|
||||
const agent = await skipMissingPreseededAgent(world, agentName)
|
||||
if (agent === 'skipped')
|
||||
return agent
|
||||
|
||||
const ctx = await createApiContext()
|
||||
try {
|
||||
const response = await ctx.get(`/console/api/agent/${agent.id}/composer`)
|
||||
await expectApiResponseOK(response, `Check preseeded Agent flat file fixture ${agentName}`)
|
||||
const body = (await response.json()) as AgentAppComposerResponse
|
||||
const configFiles = Array.isArray(body.agent_soul?.config_files)
|
||||
? body.agent_soul.config_files
|
||||
: []
|
||||
const fileNames = configFiles
|
||||
.map(file => (typeof file === 'object' && file !== null && 'name' in file ? file.name : undefined))
|
||||
.filter((name): name is string => typeof name === 'string')
|
||||
const missingFiles = agentBuilderFileTreeFixtureFileNames.filter(fileName => !fileNames.includes(fileName))
|
||||
|
||||
if (missingFiles.length > 0) {
|
||||
return skipBlockedPrecondition(
|
||||
world,
|
||||
`Preseeded Agent "${agentName}" is missing current flat Files fixture configuration: ${missingFiles.join(', ')}. Hierarchical Files display remains blocked until Agent config files support tree paths.`,
|
||||
)
|
||||
}
|
||||
|
||||
return {
|
||||
id: agent.id,
|
||||
kind: 'agent',
|
||||
name: agent.name,
|
||||
}
|
||||
}
|
||||
finally {
|
||||
await ctx.dispose()
|
||||
}
|
||||
}
|
||||
133
e2e/features/agent-v2/support/preflight/common.ts
Normal file
133
e2e/features/agent-v2/support/preflight/common.ts
Normal file
@ -0,0 +1,133 @@
|
||||
import type { DifyWorld } from '../../../support/world'
|
||||
import { createApiContext, expectApiResponseOK } from '../../../../support/api'
|
||||
import { agentBuilderPreseededResources } from '../agent-builder-resources'
|
||||
|
||||
export type PreseededResource = NonNullable<
|
||||
DifyWorld['agentBuilder']['preflight']['preseededResources'][string]
|
||||
>
|
||||
|
||||
export type E2EResourcePrecondition
|
||||
= | {
|
||||
ok: true
|
||||
value: string
|
||||
}
|
||||
| {
|
||||
ok: false
|
||||
reason: string
|
||||
}
|
||||
|
||||
export type NamedResource = {
|
||||
id: string
|
||||
name: string
|
||||
}
|
||||
|
||||
export type NamedResourceCollection<T extends NamedResource = NamedResource> = {
|
||||
data: T[]
|
||||
}
|
||||
|
||||
export type LocalizedLabel = {
|
||||
en_US?: string
|
||||
zh_Hans?: string
|
||||
}
|
||||
|
||||
export const readRequiredEnvResource = (
|
||||
envName: string,
|
||||
description: string,
|
||||
): E2EResourcePrecondition => {
|
||||
const value = process.env[envName]?.trim()
|
||||
if (value)
|
||||
return { ok: true, value }
|
||||
|
||||
return {
|
||||
ok: false,
|
||||
reason: `${description} requires ${envName}.`,
|
||||
}
|
||||
}
|
||||
|
||||
export function skipBlockedPrecondition(
|
||||
world: DifyWorld,
|
||||
reason: string,
|
||||
options: {
|
||||
owner?: string
|
||||
remediation?: string
|
||||
} = {},
|
||||
): 'skipped' {
|
||||
const owner = options.owner ?? 'seed/product'
|
||||
const remediation = options.remediation ?? 'Seed the required resource or align the product capability before running this scenario.'
|
||||
const message = `Blocked precondition: ${reason} Owner: ${owner}. Remediation: ${remediation}`
|
||||
console.warn(`[e2e] ${message}`)
|
||||
world.attach(message, 'text/plain')
|
||||
return 'skipped'
|
||||
}
|
||||
|
||||
export function skipMissingEnvResource(
|
||||
world: DifyWorld,
|
||||
envName: string,
|
||||
description: string,
|
||||
): 'skipped' | string {
|
||||
const resource = readRequiredEnvResource(envName, description)
|
||||
if (resource.ok)
|
||||
return resource.value
|
||||
|
||||
return skipBlockedPrecondition(world, resource.reason)
|
||||
}
|
||||
|
||||
export const requiredAgentBuilderPreseededResources = Object.values(agentBuilderPreseededResources)
|
||||
|
||||
export function skipMissingAgentBuilderPreseed(
|
||||
world: DifyWorld,
|
||||
resourceName: string,
|
||||
envName: string,
|
||||
): 'skipped' | string {
|
||||
return skipMissingEnvResource(
|
||||
world,
|
||||
envName,
|
||||
`Preseeded Agent Builder resource "${resourceName}"`,
|
||||
)
|
||||
}
|
||||
|
||||
export const findConsoleResourceByName = async <T extends NamedResource = NamedResource>({
|
||||
action,
|
||||
path,
|
||||
resourceName,
|
||||
}: {
|
||||
action: string
|
||||
path: string
|
||||
resourceName: string
|
||||
}) => {
|
||||
const ctx = await createApiContext()
|
||||
try {
|
||||
const response = await ctx.get(path)
|
||||
await expectApiResponseOK(response, action)
|
||||
const body = (await response.json()) as NamedResourceCollection<T>
|
||||
|
||||
return body.data.find(item => item.name === resourceName)
|
||||
}
|
||||
finally {
|
||||
await ctx.dispose()
|
||||
}
|
||||
}
|
||||
|
||||
export const buildQuery = (params: Record<string, string>) => new URLSearchParams(params).toString()
|
||||
|
||||
export const matchesNameOrLabel = (value: string, name: string, label?: LocalizedLabel) =>
|
||||
value === name || value === label?.en_US || value === label?.zh_Hans
|
||||
|
||||
export const isRecord = (value: unknown): value is Record<string, unknown> =>
|
||||
typeof value === 'object' && value !== null && !Array.isArray(value)
|
||||
|
||||
export const asRecord = (value: unknown): Record<string, unknown> => (isRecord(value) ? value : {})
|
||||
|
||||
export const asArray = (value: unknown): unknown[] => (Array.isArray(value) ? value : [])
|
||||
|
||||
export const asString = (value: unknown) => (typeof value === 'string' ? value : '')
|
||||
|
||||
export const hasNamedOrKeyedEntry = (items: unknown[], expectedName: string) =>
|
||||
items.some((item) => {
|
||||
const record = asRecord(item)
|
||||
const values = [record.name, record.drive_key, record.reference, record.file_id, record.id].map(
|
||||
asString,
|
||||
)
|
||||
|
||||
return values.some(value => value === expectedName || value.endsWith(`/${expectedName}`))
|
||||
})
|
||||
232
e2e/features/agent-v2/support/preflight/datasets.ts
Normal file
232
e2e/features/agent-v2/support/preflight/datasets.ts
Normal file
@ -0,0 +1,232 @@
|
||||
import type {
|
||||
ConsoleSegmentListResponse,
|
||||
DatasetListItemResponse,
|
||||
DocumentStatusListResponse,
|
||||
DocumentWithSegmentsListResponse,
|
||||
} from '@dify/contracts/api/console/datasets/types.gen'
|
||||
import type { DifyWorld } from '../../../support/world'
|
||||
import type { PreseededResource } from './common'
|
||||
import { createApiContext, expectApiResponseOK } from '../../../../support/api'
|
||||
import {
|
||||
agentBuilderExpectedTokens,
|
||||
agentBuilderPreseededResources,
|
||||
} from '../agent-builder-resources'
|
||||
import {
|
||||
buildQuery,
|
||||
findConsoleResourceByName,
|
||||
|
||||
skipBlockedPrecondition,
|
||||
} from './common'
|
||||
|
||||
type DocumentIndexingStatus
|
||||
= | 'cleaning'
|
||||
| 'completed'
|
||||
| 'indexing'
|
||||
| 'parsing'
|
||||
| 'splitting'
|
||||
| 'waiting'
|
||||
|
||||
const completedDocumentIndexingStatus: DocumentIndexingStatus = 'completed'
|
||||
const activeDocumentIndexingStatuses = new Set<string>([
|
||||
'cleaning',
|
||||
'indexing',
|
||||
'parsing',
|
||||
'splitting',
|
||||
'waiting',
|
||||
])
|
||||
|
||||
export const getPreseededDataset = async (resourceName: string) => {
|
||||
const query = buildQuery({ keyword: resourceName, limit: '20', page: '1' })
|
||||
|
||||
return findConsoleResourceByName<DatasetListItemResponse>({
|
||||
action: `Check preseeded dataset ${resourceName}`,
|
||||
path: `/console/api/datasets?${query}`,
|
||||
resourceName,
|
||||
})
|
||||
}
|
||||
|
||||
const getDatasetIndexingStatuses = async (datasetId: string, resourceName: string) => {
|
||||
const ctx = await createApiContext()
|
||||
try {
|
||||
const response = await ctx.get(`/console/api/datasets/${datasetId}/indexing-status`)
|
||||
await expectApiResponseOK(response, `Check preseeded dataset indexing status ${resourceName}`)
|
||||
const body = (await response.json()) as DocumentStatusListResponse
|
||||
|
||||
return body.data
|
||||
}
|
||||
finally {
|
||||
await ctx.dispose()
|
||||
}
|
||||
}
|
||||
|
||||
const getDatasetDocuments = async (datasetId: string, resourceName: string) => {
|
||||
const documents: DocumentWithSegmentsListResponse['data'] = []
|
||||
const ctx = await createApiContext()
|
||||
try {
|
||||
let page = 1
|
||||
let hasMore = true
|
||||
|
||||
while (hasMore) {
|
||||
const query = buildQuery({ limit: '100', page: String(page) })
|
||||
const response = await ctx.get(`/console/api/datasets/${datasetId}/documents?${query}`)
|
||||
await expectApiResponseOK(response, `List preseeded dataset documents ${resourceName}`)
|
||||
const body = (await response.json()) as DocumentWithSegmentsListResponse
|
||||
|
||||
documents.push(...body.data)
|
||||
hasMore = body.has_more
|
||||
page += 1
|
||||
}
|
||||
|
||||
return documents
|
||||
}
|
||||
finally {
|
||||
await ctx.dispose()
|
||||
}
|
||||
}
|
||||
|
||||
const datasetHasEnabledSegmentContainingToken = async (
|
||||
datasetId: string,
|
||||
resourceName: string,
|
||||
expectedToken: string,
|
||||
) => {
|
||||
const documents = await getDatasetDocuments(datasetId, resourceName)
|
||||
const ctx = await createApiContext()
|
||||
try {
|
||||
for (const document of documents) {
|
||||
const query = buildQuery({
|
||||
enabled: 'true',
|
||||
keyword: expectedToken,
|
||||
limit: '20',
|
||||
page: '1',
|
||||
})
|
||||
const response = await ctx.get(
|
||||
`/console/api/datasets/${datasetId}/documents/${document.id}/segments?${query}`,
|
||||
)
|
||||
await expectApiResponseOK(
|
||||
response,
|
||||
`Check preseeded dataset segment content ${resourceName}`,
|
||||
)
|
||||
const body = (await response.json()) as ConsoleSegmentListResponse
|
||||
const matchingSegment = body.data.find(
|
||||
segment =>
|
||||
segment.enabled
|
||||
&& (
|
||||
segment.content.includes(expectedToken)
|
||||
|| segment.keywords?.some(keyword => keyword.includes(expectedToken))
|
||||
),
|
||||
)
|
||||
|
||||
if (matchingSegment)
|
||||
return true
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
finally {
|
||||
await ctx.dispose()
|
||||
}
|
||||
}
|
||||
|
||||
export const toDatasetResource = (resource: DatasetListItemResponse): PreseededResource => ({
|
||||
id: resource.id,
|
||||
kind: 'dataset',
|
||||
name: resource.name,
|
||||
})
|
||||
|
||||
export async function skipMissingPreseededDataset(
|
||||
world: DifyWorld,
|
||||
resourceName: string,
|
||||
): Promise<'skipped' | PreseededResource> {
|
||||
const resource = await getPreseededDataset(resourceName)
|
||||
|
||||
if (!resource)
|
||||
return skipBlockedPrecondition(world, `Preseeded dataset "${resourceName}" was not found.`)
|
||||
|
||||
return toDatasetResource(resource)
|
||||
}
|
||||
|
||||
export async function skipMissingReadyPreseededDataset(
|
||||
world: DifyWorld,
|
||||
resourceName: string,
|
||||
): Promise<'skipped' | PreseededResource> {
|
||||
const resource = await getPreseededDataset(resourceName)
|
||||
|
||||
if (!resource)
|
||||
return skipBlockedPrecondition(world, `Preseeded dataset "${resourceName}" was not found.`)
|
||||
|
||||
if (resource.document_count < 1) {
|
||||
return skipBlockedPrecondition(world, `Preseeded dataset "${resourceName}" has no documents.`)
|
||||
}
|
||||
|
||||
if (resource.total_available_documents !== resource.document_count) {
|
||||
return skipBlockedPrecondition(
|
||||
world,
|
||||
`Preseeded dataset "${resourceName}" has ${resource.total_available_documents}/${resource.document_count} available documents.`,
|
||||
)
|
||||
}
|
||||
|
||||
const statuses = await getDatasetIndexingStatuses(resource.id, resourceName)
|
||||
if (statuses.length < 1) {
|
||||
return skipBlockedPrecondition(
|
||||
world,
|
||||
`Preseeded dataset "${resourceName}" has no document indexing status.`,
|
||||
)
|
||||
}
|
||||
|
||||
const incompleteStatus = statuses.find(
|
||||
item => item.indexing_status !== completedDocumentIndexingStatus,
|
||||
)
|
||||
if (incompleteStatus) {
|
||||
return skipBlockedPrecondition(
|
||||
world,
|
||||
`Preseeded dataset "${resourceName}" includes document ${incompleteStatus.id} with indexing status "${incompleteStatus.indexing_status ?? 'missing'}".`,
|
||||
)
|
||||
}
|
||||
|
||||
if (resourceName === agentBuilderPreseededResources.agentKnowledgeBase) {
|
||||
const hasExpectedToken = await datasetHasEnabledSegmentContainingToken(
|
||||
resource.id,
|
||||
resourceName,
|
||||
agentBuilderExpectedTokens.knowledgeReply,
|
||||
)
|
||||
|
||||
if (!hasExpectedToken) {
|
||||
return skipBlockedPrecondition(
|
||||
world,
|
||||
`Preseeded dataset "${resourceName}" has no enabled segment containing "${agentBuilderExpectedTokens.knowledgeReply}".`,
|
||||
{
|
||||
remediation: `Seed the dataset from the Agent Builder knowledge fixture and wait until an enabled segment contains "${agentBuilderExpectedTokens.knowledgeReply}".`,
|
||||
},
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
return toDatasetResource(resource)
|
||||
}
|
||||
|
||||
export async function skipMissingIndexingPreseededDataset(
|
||||
world: DifyWorld,
|
||||
resourceName: string,
|
||||
): Promise<'skipped' | PreseededResource> {
|
||||
const resource = await getPreseededDataset(resourceName)
|
||||
|
||||
if (!resource)
|
||||
return skipBlockedPrecondition(world, `Preseeded dataset "${resourceName}" was not found.`)
|
||||
|
||||
const statuses = await getDatasetIndexingStatuses(resource.id, resourceName)
|
||||
const indexingStatus = statuses.find(item =>
|
||||
activeDocumentIndexingStatuses.has(item.indexing_status ?? ''),
|
||||
)
|
||||
|
||||
if (!indexingStatus) {
|
||||
const actualStatuses
|
||||
= statuses.map(item => item.indexing_status ?? 'missing').join(', ') || 'none'
|
||||
|
||||
return skipBlockedPrecondition(
|
||||
world,
|
||||
`Preseeded dataset "${resourceName}" is not indexing or queued; document statuses: ${actualStatuses}.`,
|
||||
)
|
||||
}
|
||||
|
||||
return toDatasetResource(resource)
|
||||
}
|
||||
138
e2e/features/agent-v2/support/preflight/models.ts
Normal file
138
e2e/features/agent-v2/support/preflight/models.ts
Normal file
@ -0,0 +1,138 @@
|
||||
import type { ProviderWithModelsDataResponse } from '@dify/contracts/api/console/workspaces/types.gen'
|
||||
import type { DifyWorld } from '../../../support/world'
|
||||
import { createApiContext, expectApiResponseOK } from '../../../../support/api'
|
||||
import { agentBuilderPreseededResources } from '../agent-builder-resources'
|
||||
import { skipBlockedPrecondition } from './common'
|
||||
|
||||
const stableChatModelProviderEnv = 'E2E_STABLE_MODEL_PROVIDER'
|
||||
const stableChatModelNameEnv = 'E2E_STABLE_MODEL_NAME'
|
||||
const stableChatModelTypeEnv = 'E2E_STABLE_MODEL_TYPE'
|
||||
const brokenChatModelProviderEnv = 'E2E_BROKEN_MODEL_PROVIDER'
|
||||
const brokenChatModelNameEnv = 'E2E_BROKEN_MODEL_NAME'
|
||||
const brokenChatModelTypeEnv = 'E2E_BROKEN_MODEL_TYPE'
|
||||
const activeModelStatus = 'active'
|
||||
const defaultStableChatModelProvider = 'openai'
|
||||
const defaultStableChatModelName = 'gpt-5.4-mini'
|
||||
const defaultStableChatModelType = 'llm'
|
||||
const defaultBrokenChatModelName = agentBuilderPreseededResources.brokenModel
|
||||
|
||||
const getProviderAlias = (provider: string) => provider.split('/').filter(Boolean).at(-1) ?? provider
|
||||
|
||||
const matchesProvider = (actual: string, expected: string) =>
|
||||
actual === expected || getProviderAlias(actual) === getProviderAlias(expected)
|
||||
|
||||
type ModelPreflightConfig
|
||||
= | {
|
||||
ok: true
|
||||
provider: string
|
||||
resourceName: string
|
||||
type: string
|
||||
value: string
|
||||
}
|
||||
| {
|
||||
ok: false
|
||||
reason: string
|
||||
}
|
||||
|
||||
export function readAgentBuilderStableChatModelConfig(): ModelPreflightConfig {
|
||||
const provider = process.env[stableChatModelProviderEnv]?.trim() || defaultStableChatModelProvider
|
||||
const name = process.env[stableChatModelNameEnv]?.trim() || defaultStableChatModelName
|
||||
const type = process.env[stableChatModelTypeEnv]?.trim() || defaultStableChatModelType
|
||||
|
||||
return {
|
||||
ok: true,
|
||||
provider,
|
||||
resourceName: agentBuilderPreseededResources.stableChatModel,
|
||||
type,
|
||||
value: name,
|
||||
}
|
||||
}
|
||||
|
||||
export function readAgentBuilderBrokenChatModelConfig(): ModelPreflightConfig {
|
||||
const provider = process.env[brokenChatModelProviderEnv]?.trim()
|
||||
const name = process.env[brokenChatModelNameEnv]?.trim() || defaultBrokenChatModelName
|
||||
const type = process.env[brokenChatModelTypeEnv]?.trim() || defaultStableChatModelType
|
||||
|
||||
if (!provider) {
|
||||
return {
|
||||
ok: false,
|
||||
reason: `${agentBuilderPreseededResources.brokenModelProvider} requires ${brokenChatModelProviderEnv}.`,
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
ok: true,
|
||||
provider,
|
||||
resourceName: agentBuilderPreseededResources.brokenModelProvider,
|
||||
type,
|
||||
value: name,
|
||||
}
|
||||
}
|
||||
|
||||
async function skipMissingAgentBuilderModel(
|
||||
world: DifyWorld,
|
||||
config: ModelPreflightConfig,
|
||||
{
|
||||
requireActive,
|
||||
}: {
|
||||
requireActive: boolean
|
||||
},
|
||||
): Promise<'skipped' | NonNullable<DifyWorld['agentBuilder']['preflight']['stableModel']>> {
|
||||
if (!config.ok)
|
||||
return skipBlockedPrecondition(world, config.reason)
|
||||
|
||||
const ctx = await createApiContext()
|
||||
try {
|
||||
const response = await ctx.get(
|
||||
`/console/api/workspaces/current/models/model-types/${config.type}`,
|
||||
)
|
||||
await expectApiResponseOK(response, `Check ${config.resourceName}`)
|
||||
const body = (await response.json()) as ProviderWithModelsDataResponse
|
||||
const provider = body.data.find(item => matchesProvider(item.provider, config.provider))
|
||||
const model = provider?.models.find(
|
||||
item =>
|
||||
item.model === config.value
|
||||
|| item.label?.en_US === config.value
|
||||
|| item.label?.zh_Hans === config.value,
|
||||
)
|
||||
|
||||
if (!provider || !model) {
|
||||
return skipBlockedPrecondition(
|
||||
world,
|
||||
`${config.resourceName} was not found as ${config.provider}/${config.value} (${config.type}).`,
|
||||
)
|
||||
}
|
||||
|
||||
if (requireActive && model.status !== activeModelStatus) {
|
||||
return skipBlockedPrecondition(
|
||||
world,
|
||||
`${config.resourceName} is ${model.status ?? 'missing status'} instead of ${activeModelStatus}.`,
|
||||
)
|
||||
}
|
||||
|
||||
return {
|
||||
name: model.model,
|
||||
provider: provider.provider,
|
||||
type: config.type,
|
||||
}
|
||||
}
|
||||
finally {
|
||||
await ctx.dispose()
|
||||
}
|
||||
}
|
||||
|
||||
export async function skipMissingAgentBuilderStableChatModel(
|
||||
world: DifyWorld,
|
||||
): Promise<'skipped' | NonNullable<DifyWorld['agentBuilder']['preflight']['stableModel']>> {
|
||||
return skipMissingAgentBuilderModel(world, readAgentBuilderStableChatModelConfig(), {
|
||||
requireActive: true,
|
||||
})
|
||||
}
|
||||
|
||||
export async function skipMissingAgentBuilderBrokenChatModel(
|
||||
world: DifyWorld,
|
||||
): Promise<'skipped' | NonNullable<DifyWorld['agentBuilder']['preflight']['stableModel']>> {
|
||||
return skipMissingAgentBuilderModel(world, readAgentBuilderBrokenChatModelConfig(), {
|
||||
requireActive: false,
|
||||
})
|
||||
}
|
||||
114
e2e/features/agent-v2/support/preflight/tools.ts
Normal file
114
e2e/features/agent-v2/support/preflight/tools.ts
Normal file
@ -0,0 +1,114 @@
|
||||
import type { DifyWorld } from '../../../support/world'
|
||||
import type { LocalizedLabel, PreseededResource } from './common'
|
||||
import { createApiContext, expectApiResponseOK } from '../../../../support/api'
|
||||
import {
|
||||
asRecord,
|
||||
asString,
|
||||
|
||||
matchesNameOrLabel,
|
||||
|
||||
skipBlockedPrecondition,
|
||||
} from './common'
|
||||
|
||||
type BuiltinToolProvider = {
|
||||
label?: LocalizedLabel
|
||||
name: string
|
||||
tools: Array<{
|
||||
label?: LocalizedLabel
|
||||
name: string
|
||||
}>
|
||||
}
|
||||
|
||||
export const splitToolDisplayName = (resourceName: string) => {
|
||||
const [providerName, toolName] = resourceName.split('/').map(item => item.trim())
|
||||
|
||||
if (!providerName || !toolName) {
|
||||
return {
|
||||
ok: false as const,
|
||||
reason: `Preseeded tool "${resourceName}" must use "Provider / Tool" format.`,
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
ok: true as const,
|
||||
providerName,
|
||||
toolName,
|
||||
}
|
||||
}
|
||||
|
||||
export const findToolEntry = (
|
||||
items: unknown[],
|
||||
{
|
||||
providerDisplayName,
|
||||
providerName,
|
||||
toolDisplayName,
|
||||
toolName,
|
||||
}: {
|
||||
providerDisplayName: string
|
||||
providerName: string
|
||||
toolDisplayName: string
|
||||
toolName: string
|
||||
},
|
||||
) =>
|
||||
items.find((item) => {
|
||||
const record = asRecord(item)
|
||||
const providerValues = [record.provider_id, record.provider, record.plugin_id, record.name].map(
|
||||
asString,
|
||||
)
|
||||
const toolValues = [record.tool_name, record.name].map(asString)
|
||||
|
||||
return (
|
||||
providerValues.some(value => value === providerName || value === providerDisplayName)
|
||||
&& toolValues.some(value => value === toolName || value === toolDisplayName)
|
||||
)
|
||||
})
|
||||
|
||||
export const hasToolEntry = (
|
||||
items: unknown[],
|
||||
tool: {
|
||||
providerDisplayName: string
|
||||
providerName: string
|
||||
toolDisplayName: string
|
||||
toolName: string
|
||||
},
|
||||
) => Boolean(findToolEntry(items, tool))
|
||||
|
||||
export const hasUnauthorizedToolCredentialState = (item: unknown) => {
|
||||
const record = asRecord(item)
|
||||
|
||||
return asString(record.credential_type) === 'unauthorized'
|
||||
}
|
||||
|
||||
export async function skipMissingPreseededTool(
|
||||
world: DifyWorld,
|
||||
resourceName: string,
|
||||
): Promise<'skipped' | PreseededResource> {
|
||||
const parsed = splitToolDisplayName(resourceName)
|
||||
if (!parsed.ok)
|
||||
return skipBlockedPrecondition(world, parsed.reason)
|
||||
|
||||
const ctx = await createApiContext()
|
||||
try {
|
||||
const response = await ctx.get('/console/api/workspaces/current/tools/builtin')
|
||||
await expectApiResponseOK(response, `Check preseeded tool ${resourceName}`)
|
||||
const providers = (await response.json()) as BuiltinToolProvider[]
|
||||
const provider = providers.find(item =>
|
||||
matchesNameOrLabel(parsed.providerName, item.name, item.label),
|
||||
)
|
||||
const tool = provider?.tools.find(item =>
|
||||
matchesNameOrLabel(parsed.toolName, item.name, item.label),
|
||||
)
|
||||
|
||||
if (!provider || !tool)
|
||||
return skipBlockedPrecondition(world, `Preseeded tool "${resourceName}" was not found.`)
|
||||
|
||||
return {
|
||||
id: `${provider.name}/${tool.name}`,
|
||||
kind: 'tool',
|
||||
name: resourceName,
|
||||
}
|
||||
}
|
||||
finally {
|
||||
await ctx.dispose()
|
||||
}
|
||||
}
|
||||
52
e2e/features/agent-v2/support/test-materials.ts
Normal file
52
e2e/features/agent-v2/support/test-materials.ts
Normal file
@ -0,0 +1,52 @@
|
||||
import path from 'node:path'
|
||||
import { getGeneratedTextMaterialPath, getTestMaterialPath } from '../../../support/test-materials'
|
||||
|
||||
export const agentBuilderTestMaterials = {
|
||||
smallFile: 'agent-small-file.txt',
|
||||
knowledgeSource: 'agent-knowledge-source.txt',
|
||||
emptyFile: 'agent-empty-file.txt',
|
||||
unsupportedFile: 'agent-unsupported-file.exe',
|
||||
specialFilename: 'agent-special-filename-中文 @#$%.txt',
|
||||
validEnv: 'agent-valid.env',
|
||||
invalidEnv: 'agent-invalid.env',
|
||||
buildInstruction: 'agent-build-instruction.txt',
|
||||
summarySkill: 'e2e-summary-skill/SKILL.md',
|
||||
fileTreeFixture: 'file_tree_fixture',
|
||||
countBatch5: 'count_batch_5_valid_files',
|
||||
countBatch6: 'count_batch_6_valid_files',
|
||||
countTotal50: 'count_total_50_valid_files',
|
||||
countTotalExtra1: 'count_total_extra_1_valid_file',
|
||||
} as const
|
||||
|
||||
export const agentBuilderGeneratedTestMaterials = {
|
||||
slowUploadFile: 'agent-slow-upload-file.txt',
|
||||
tooLargeFile: 'agent-too-large-file.txt',
|
||||
} as const
|
||||
|
||||
export const agentBuilderFileTreeFixtureFiles = [
|
||||
'assets/sample.csv',
|
||||
'docs/中文说明.md',
|
||||
'public/index.html',
|
||||
'src/main.txt',
|
||||
'web-game/README.md',
|
||||
] as const
|
||||
|
||||
export const agentBuilderFileTreeFixtureFileNames = agentBuilderFileTreeFixtureFiles
|
||||
.map(filePath => path.basename(filePath))
|
||||
|
||||
export const getAgentBuilderTestMaterialPath = (material: keyof typeof agentBuilderTestMaterials) =>
|
||||
getTestMaterialPath(agentBuilderTestMaterials[material])
|
||||
|
||||
export const getTooLargeAgentFilePath = () =>
|
||||
getGeneratedTextMaterialPath({
|
||||
fileName: 'agent-too-large-file.txt',
|
||||
sizeBytes: 16 * 1024 * 1024,
|
||||
seedText: 'E2E_TOO_LARGE_FILE_FIXTURE',
|
||||
})
|
||||
|
||||
export const getSlowUploadAgentFilePath = () =>
|
||||
getGeneratedTextMaterialPath({
|
||||
fileName: 'agent-slow-upload-file.txt',
|
||||
sizeBytes: 2 * 1024 * 1024,
|
||||
seedText: 'E2E_SLOW_UPLOAD_FILE_FIXTURE',
|
||||
})
|
||||
25
e2e/features/agent-v2/support/tools.ts
Normal file
25
e2e/features/agent-v2/support/tools.ts
Normal file
@ -0,0 +1,25 @@
|
||||
import type { DifyWorld } from '../../support/world'
|
||||
import { splitToolDisplayName } from './preflight/tools'
|
||||
|
||||
export const getPreseededToolContract = (world: DifyWorld, resourceName: string) => {
|
||||
const resource = world.agentBuilder.preflight.preseededResources[resourceName]
|
||||
if (!resource || resource.kind !== 'tool') {
|
||||
throw new Error(
|
||||
`Preseeded tool "${resourceName}" is not available. Run the matching preflight step first.`,
|
||||
)
|
||||
}
|
||||
|
||||
const parsedDisplayName = splitToolDisplayName(resource.name)
|
||||
const parsedToolId = splitToolDisplayName(resource.id)
|
||||
if (!parsedDisplayName.ok)
|
||||
throw new Error(parsedDisplayName.reason)
|
||||
if (!parsedToolId.ok)
|
||||
throw new Error(parsedToolId.reason)
|
||||
|
||||
return {
|
||||
providerDisplayName: parsedDisplayName.providerName,
|
||||
providerName: parsedToolId.providerName,
|
||||
toolDisplayName: parsedDisplayName.toolName,
|
||||
toolName: parsedToolId.toolName,
|
||||
}
|
||||
}
|
||||
33
e2e/features/agent-v2/tools.feature
Normal file
33
e2e/features/agent-v2/tools.feature
Normal file
@ -0,0 +1,33 @@
|
||||
@agent-v2 @authenticated @tools
|
||||
Feature: Agent v2 tools
|
||||
@core @tool-fixture
|
||||
Scenario: JSON Replace tool is saved after adding it from the Tools selector
|
||||
Given I am signed in as the default E2E admin
|
||||
And the Agent Builder preseeded tool "JSON Process / JSON Replace" is available
|
||||
And a basic configured Agent v2 test agent has been created via API
|
||||
When I open the Agent v2 configure page
|
||||
And I add the Agent Builder JSON Replace tool from the Tools selector
|
||||
Then the Agent v2 JSON Replace tool should be saved in the Agent v2 draft
|
||||
And the Agent v2 configuration should be saved automatically
|
||||
When I refresh the current page
|
||||
Then I should see the Agent v2 JSON Replace tool in the Tools section
|
||||
|
||||
@service-api-runtime @stable-model @tool-fixture
|
||||
Scenario: JSON Replace tool runtime returns the replacement marker
|
||||
Given I am signed in as the default E2E admin
|
||||
And Agent v2 JSON Replace runtime verification is available
|
||||
And the Agent Builder stable chat model is available
|
||||
And the Agent Builder preseeded tool "JSON Process / JSON Replace" is available
|
||||
And a runnable Agent v2 test agent has been created via API
|
||||
When I open the Agent v2 configure page
|
||||
Then Agent v2 JSON Replace runtime verification should be available
|
||||
|
||||
@core
|
||||
Scenario: Tool selector shows an empty state for a missing tool search
|
||||
Given I am signed in as the default E2E admin
|
||||
And a basic configured Agent v2 test agent has been created via API
|
||||
When I open the Agent v2 configure page
|
||||
And I search for the missing Agent v2 tool from the Tools selector
|
||||
Then I should see the Agent v2 tool selector empty state
|
||||
When I clear the Agent v2 tool selector search
|
||||
Then I should see the Agent v2 tool selector ready for another search
|
||||
@ -0,0 +1,36 @@
|
||||
import type { DifyWorld } from '../../support/world'
|
||||
|
||||
export const getCurrentAgentId = (world: DifyWorld) => {
|
||||
const agentId = world.createdAgentIds.at(-1)
|
||||
if (!agentId)
|
||||
throw new Error('No Agent v2 ID found. Create an Agent v2 test agent first.')
|
||||
|
||||
return agentId
|
||||
}
|
||||
|
||||
export const getPreseededResource = (
|
||||
world: DifyWorld,
|
||||
name: string,
|
||||
kind: 'agent' | 'workflow',
|
||||
) => {
|
||||
const resource = world.agentBuilder.preflight.preseededResources[name]
|
||||
if (!resource || resource.kind !== kind) {
|
||||
throw new Error(
|
||||
`Preseeded ${kind} "${name}" is not available. Run the matching preflight step first.`,
|
||||
)
|
||||
}
|
||||
|
||||
return resource
|
||||
}
|
||||
|
||||
export const getAccessRegion = (world: DifyWorld) =>
|
||||
world.getPage().getByRole('region', { name: 'Access Point' })
|
||||
|
||||
export const getWebAppCard = (world: DifyWorld) =>
|
||||
getAccessRegion(world).locator('article').filter({ hasText: 'Web app' }).first()
|
||||
|
||||
export const getServiceApiCard = (world: DifyWorld) =>
|
||||
getAccessRegion(world).locator('article').filter({ hasText: 'Backend service API' }).first()
|
||||
|
||||
export const getDialog = (world: DifyWorld, name: string | RegExp) =>
|
||||
world.getPage().getByRole('dialog', { name })
|
||||
@ -0,0 +1,268 @@
|
||||
import type { DifyWorld } from '../../support/world'
|
||||
import { Given, Then, When } from '@cucumber/cucumber'
|
||||
import { expect } from '@playwright/test'
|
||||
import {
|
||||
createAgentApiKey,
|
||||
sendAgentServiceApiChatMessage,
|
||||
setAgentApiAccess,
|
||||
} from '../../agent-v2/support/access-point'
|
||||
import { agentBuilderExpectedTokens, agentBuilderFixedInputs } from '../../agent-v2/support/agent-builder-resources'
|
||||
import { getCurrentAgentId, getServiceApiCard } from './access-point-helpers'
|
||||
|
||||
async function enableAgentApiAccessWithKey(world: DifyWorld) {
|
||||
const agentId = getCurrentAgentId(world)
|
||||
const apiAccess = await setAgentApiAccess(agentId, true)
|
||||
const apiKey = await createAgentApiKey(agentId)
|
||||
|
||||
world.agentBuilder.accessPoint.serviceApiBaseURL = apiAccess.service_api_base_url
|
||||
world.agentBuilder.accessPoint.generatedApiKey = apiKey.token
|
||||
}
|
||||
|
||||
Given(
|
||||
'Agent v2 Backend service API access has been enabled via API',
|
||||
async function (this: DifyWorld) {
|
||||
const apiAccess = await setAgentApiAccess(getCurrentAgentId(this), true)
|
||||
|
||||
this.agentBuilder.accessPoint.serviceApiBaseURL = apiAccess.service_api_base_url
|
||||
},
|
||||
)
|
||||
|
||||
Given(
|
||||
'Agent v2 Backend service API access has been enabled with a key via API',
|
||||
async function (this: DifyWorld) {
|
||||
await enableAgentApiAccessWithKey(this)
|
||||
},
|
||||
)
|
||||
|
||||
Then('I should see the Agent v2 Backend service API endpoint', async function (this: DifyWorld) {
|
||||
const serviceApiCard = getServiceApiCard(this)
|
||||
|
||||
if (!this.agentBuilder.accessPoint.serviceApiBaseURL)
|
||||
throw new Error('No Agent v2 service API endpoint found. Enable Backend service API first.')
|
||||
|
||||
await expect(serviceApiCard.getByRole('heading', { name: 'Backend service API' })).toBeVisible({
|
||||
timeout: 30_000,
|
||||
})
|
||||
await expect(serviceApiCard.getByText('Service API Endpoint')).toBeVisible()
|
||||
await expect(serviceApiCard.getByText(this.agentBuilder.accessPoint.serviceApiBaseURL)).toBeVisible()
|
||||
await expect(serviceApiCard.getByLabel('Copy service API endpoint')).toBeEnabled()
|
||||
})
|
||||
|
||||
When('I copy the Agent v2 Backend service API endpoint', async function (this: DifyWorld) {
|
||||
await getServiceApiCard(this).getByLabel('Copy service API endpoint').click()
|
||||
})
|
||||
|
||||
Then(
|
||||
'the Agent v2 Backend service API endpoint should show it was copied',
|
||||
async function (this: DifyWorld) {
|
||||
await expect(this.getPage().getByLabel('Copied')).toBeVisible()
|
||||
},
|
||||
)
|
||||
|
||||
When('I open Agent v2 API key management', async function (this: DifyWorld) {
|
||||
await getServiceApiCard(this)
|
||||
.getByRole('button', { name: /^API Key\b/ })
|
||||
.click()
|
||||
})
|
||||
|
||||
Then('Agent v2 API keys should not expose a secret by default', async function (this: DifyWorld) {
|
||||
const page = this.getPage()
|
||||
const dialog = page.getByRole('dialog', { name: /API Secret key/i })
|
||||
const existingSecret = this.agentBuilder.accessPoint.generatedApiKey
|
||||
|
||||
await expect(dialog).toBeVisible()
|
||||
await expect(dialog.getByText('Secret Key', { exact: true })).toBeVisible()
|
||||
await expect(dialog.getByText('CREATED', { exact: true })).toBeVisible()
|
||||
await expect(dialog.getByText('LAST USED', { exact: true })).toBeVisible()
|
||||
await expect(dialog.getByRole('button', { name: 'Create new Secret key' })).toBeVisible()
|
||||
if (existingSecret)
|
||||
await expect(dialog.getByText(existingSecret, { exact: true })).not.toBeVisible()
|
||||
await expect(dialog.getByText(/^app-/)).not.toBeVisible()
|
||||
await expect(page.getByRole('dialog', { name: 'Internal Server Error' })).not.toBeVisible()
|
||||
})
|
||||
|
||||
When('I create a new Agent v2 API key', async function (this: DifyWorld) {
|
||||
const dialog = this.getPage().getByRole('dialog', { name: /API Secret key/i })
|
||||
|
||||
await dialog.getByRole('button', { name: 'Create new Secret key' }).click()
|
||||
})
|
||||
|
||||
Then('I should see the newly generated Agent v2 API key once', async function (this: DifyWorld) {
|
||||
const generatedKeyDialog = this.getPage()
|
||||
.getByRole('dialog', { name: /API Secret key/i })
|
||||
.last()
|
||||
const generatedKey = generatedKeyDialog.getByText(/^app-/)
|
||||
|
||||
await expect(generatedKeyDialog).toBeVisible()
|
||||
await expect(
|
||||
generatedKeyDialog.getByText('Keep this key in a secure and accessible place.'),
|
||||
).toBeVisible()
|
||||
await expect(generatedKey).toBeVisible()
|
||||
await expect(generatedKeyDialog.getByLabel('Copy')).toBeVisible()
|
||||
|
||||
this.agentBuilder.accessPoint.generatedApiKey = (await generatedKey.textContent())?.trim()
|
||||
if (!this.agentBuilder.accessPoint.generatedApiKey)
|
||||
throw new Error('Generated Agent v2 API key was empty.')
|
||||
})
|
||||
|
||||
When('I copy the newly generated Agent v2 API key', async function (this: DifyWorld) {
|
||||
const generatedKeyDialog = this.getPage()
|
||||
.getByRole('dialog', { name: /API Secret key/i })
|
||||
.last()
|
||||
|
||||
await generatedKeyDialog.getByLabel('Copy').first().click()
|
||||
})
|
||||
|
||||
Then(
|
||||
'the newly generated Agent v2 API key should show it was copied',
|
||||
async function (this: DifyWorld) {
|
||||
const generatedKeyDialog = this.getPage()
|
||||
.getByRole('dialog', { name: /API Secret key/i })
|
||||
.last()
|
||||
|
||||
await expect(generatedKeyDialog.getByLabel('Copied')).toBeVisible()
|
||||
},
|
||||
)
|
||||
|
||||
When('I close the newly generated Agent v2 API key', async function (this: DifyWorld) {
|
||||
const page = this.getPage()
|
||||
const generatedKeyDialog = page.getByRole('dialog', { name: /API Secret key/i }).last()
|
||||
|
||||
await generatedKeyDialog.getByRole('button', { name: 'OK' }).click()
|
||||
await expect(page.getByText('Keep this key in a secure and accessible place.')).not.toBeVisible()
|
||||
})
|
||||
|
||||
Then(
|
||||
'the Agent v2 API key list should not expose the full generated secret',
|
||||
async function (this: DifyWorld) {
|
||||
const fullSecret = this.agentBuilder.accessPoint.generatedApiKey
|
||||
if (!fullSecret)
|
||||
throw new Error('No generated Agent v2 API key found.')
|
||||
|
||||
const apiKeyDialog = this.getPage().getByRole('dialog', { name: /API Secret key/i })
|
||||
|
||||
await expect(apiKeyDialog).toBeVisible()
|
||||
await expect(apiKeyDialog.getByText(fullSecret, { exact: true })).not.toBeVisible()
|
||||
await expect(apiKeyDialog.getByText(/^app-/)).not.toBeVisible()
|
||||
await expect(apiKeyDialog.getByLabel('Copy').first()).toBeVisible()
|
||||
},
|
||||
)
|
||||
|
||||
When('I close Agent v2 API key management', async function (this: DifyWorld) {
|
||||
const apiKeyDialog = this.getPage().getByRole('dialog', { name: /API Secret key/i })
|
||||
|
||||
await apiKeyDialog.getByLabel('Close').click()
|
||||
await expect(apiKeyDialog).not.toBeVisible()
|
||||
})
|
||||
|
||||
When('I open the Agent v2 API Reference', async function (this: DifyWorld) {
|
||||
const page = this.getPage()
|
||||
const apiReferenceLink = page.getByRole('link', { name: 'API Reference' })
|
||||
|
||||
await expect(apiReferenceLink).toBeVisible()
|
||||
await expect(apiReferenceLink).toHaveAttribute('href', /\/use-dify\/publish\/developing-with-apis/)
|
||||
await expect(apiReferenceLink).toHaveAttribute('target', '_blank')
|
||||
|
||||
const [apiReferencePage] = await Promise.all([
|
||||
page.waitForEvent('popup'),
|
||||
apiReferenceLink.click(),
|
||||
])
|
||||
|
||||
this.agentBuilder.accessPoint.apiReferencePage = apiReferencePage
|
||||
})
|
||||
|
||||
Then('the Agent v2 API Reference should open in a new tab', async function (this: DifyWorld) {
|
||||
const apiReferencePage = this.agentBuilder.accessPoint.apiReferencePage
|
||||
if (!apiReferencePage)
|
||||
throw new Error('No Agent v2 API Reference page was opened.')
|
||||
|
||||
await expect(apiReferencePage).toHaveURL(/developing-with-apis/)
|
||||
await apiReferencePage.close()
|
||||
this.agentBuilder.accessPoint.apiReferencePage = undefined
|
||||
})
|
||||
|
||||
When('I disable Agent v2 Backend service API access', async function (this: DifyWorld) {
|
||||
await getServiceApiCard(this).getByLabel('Toggle Backend service API access').click()
|
||||
})
|
||||
|
||||
Then('Agent v2 Backend service API access should be out of service', async function (this: DifyWorld) {
|
||||
const serviceApiCard = getServiceApiCard(this)
|
||||
|
||||
await expect(serviceApiCard.getByText('Out of service')).toBeVisible({ timeout: 30_000 })
|
||||
})
|
||||
|
||||
When('I enable Agent v2 Backend service API access', async function (this: DifyWorld) {
|
||||
await getServiceApiCard(this).getByLabel('Toggle Backend service API access').click()
|
||||
})
|
||||
|
||||
Then('Agent v2 Backend service API access should be in service', async function (this: DifyWorld) {
|
||||
const serviceApiCard = getServiceApiCard(this)
|
||||
|
||||
await expect(serviceApiCard.getByText('In service')).toBeVisible({ timeout: 30_000 })
|
||||
})
|
||||
|
||||
When('I send the Agent v2 Backend service API minimal request', async function (this: DifyWorld) {
|
||||
const serviceApiBaseURL = this.agentBuilder.accessPoint.serviceApiBaseURL
|
||||
const apiKey = this.agentBuilder.accessPoint.generatedApiKey
|
||||
if (!serviceApiBaseURL)
|
||||
throw new Error('No Agent v2 service API endpoint found. Enable Backend service API first.')
|
||||
if (!apiKey)
|
||||
throw new Error('No Agent v2 API key found. Create a Backend service API key first.')
|
||||
|
||||
this.agentBuilder.accessPoint.serviceApiResponse = await sendAgentServiceApiChatMessage({
|
||||
apiKey,
|
||||
serviceApiBaseURL,
|
||||
})
|
||||
})
|
||||
|
||||
When('I send the Agent v2 Backend service API knowledge request', async function (this: DifyWorld) {
|
||||
const serviceApiBaseURL = this.agentBuilder.accessPoint.serviceApiBaseURL
|
||||
const apiKey = this.agentBuilder.accessPoint.generatedApiKey
|
||||
if (!serviceApiBaseURL)
|
||||
throw new Error('No Agent v2 service API endpoint found. Enable Backend service API first.')
|
||||
if (!apiKey)
|
||||
throw new Error('No Agent v2 API key found. Create a Backend service API key first.')
|
||||
|
||||
this.agentBuilder.accessPoint.serviceApiResponse = await sendAgentServiceApiChatMessage({
|
||||
apiKey,
|
||||
query: agentBuilderFixedInputs.customKnowledgeQuery,
|
||||
serviceApiBaseURL,
|
||||
})
|
||||
})
|
||||
|
||||
Then(
|
||||
'the Agent v2 Backend service API request should be rejected while disabled',
|
||||
async function (this: DifyWorld) {
|
||||
const response = this.agentBuilder.accessPoint.serviceApiResponse
|
||||
if (!response)
|
||||
throw new Error('No Agent v2 Backend service API response was recorded.')
|
||||
|
||||
expect(response.ok).toBe(false)
|
||||
expect(response.status).toBe(403)
|
||||
expect(JSON.stringify(response.body).toLowerCase()).toContain('disabled')
|
||||
},
|
||||
)
|
||||
|
||||
Then(
|
||||
'the Agent v2 Backend service API response should include the knowledge E2E marker',
|
||||
async function (this: DifyWorld) {
|
||||
const response = this.agentBuilder.accessPoint.serviceApiResponse
|
||||
if (!response)
|
||||
throw new Error('No Agent v2 Backend service API response was recorded.')
|
||||
|
||||
expect(response.ok).toBe(true)
|
||||
expect(JSON.stringify(response.body)).toContain(agentBuilderExpectedTokens.knowledgeReply)
|
||||
},
|
||||
)
|
||||
|
||||
Then(
|
||||
'the Agent v2 Backend service API request should succeed with the normal E2E marker',
|
||||
async function (this: DifyWorld) {
|
||||
const response = this.agentBuilder.accessPoint.serviceApiResponse
|
||||
if (!response)
|
||||
throw new Error('No Agent v2 Backend service API response was recorded.')
|
||||
|
||||
expect(response.ok).toBe(true)
|
||||
expect(JSON.stringify(response.body)).toContain(agentBuilderExpectedTokens.agentReply)
|
||||
},
|
||||
)
|
||||
@ -0,0 +1,288 @@
|
||||
import type { DifyWorld } from '../../support/world'
|
||||
import { Given, Then, When } from '@cucumber/cucumber'
|
||||
import { expect } from '@playwright/test'
|
||||
import { setAgentSiteAccessAndGetURL } from '../../agent-v2/support/access-point'
|
||||
import { getAgentComposerDraft } from '../../agent-v2/support/agent'
|
||||
import { agentBuilderExpectedTokens } from '../../agent-v2/support/agent-builder-resources'
|
||||
import { skipBlockedPrecondition } from '../../agent-v2/support/preflight/common'
|
||||
import {
|
||||
getCurrentAgentId,
|
||||
getDialog,
|
||||
getWebAppCard,
|
||||
} from './access-point-helpers'
|
||||
|
||||
Given(
|
||||
'Agent v2 Web app access has been enabled via API',
|
||||
async function (this: DifyWorld) {
|
||||
this.agentBuilder.accessPoint.webAppURL = await setAgentSiteAccessAndGetURL(
|
||||
getCurrentAgentId(this),
|
||||
true,
|
||||
)
|
||||
},
|
||||
)
|
||||
|
||||
Then('I should see the Agent v2 Web app access URL', async function (this: DifyWorld) {
|
||||
const webAppCard = getWebAppCard(this)
|
||||
|
||||
await expect(webAppCard.getByRole('heading', { name: 'Web app' })).toBeVisible()
|
||||
await expect(webAppCard.getByText('Access URL')).toBeVisible()
|
||||
await expect(webAppCard.getByLabel('Copy access URL')).toBeEnabled()
|
||||
await expect(webAppCard.getByRole('link', { name: 'Launch' })).toBeVisible()
|
||||
})
|
||||
|
||||
Then(
|
||||
'I record the current Agent v2 orchestration draft',
|
||||
async function (this: DifyWorld) {
|
||||
const draft = await getAgentComposerDraft(getCurrentAgentId(this))
|
||||
|
||||
this.agentBuilder.accessPoint.composerDraftSnapshot = JSON.stringify(draft.agent_soul ?? {})
|
||||
},
|
||||
)
|
||||
|
||||
When('I copy the Agent v2 Web app access URL', async function (this: DifyWorld) {
|
||||
await getWebAppCard(this).getByLabel('Copy access URL').click()
|
||||
})
|
||||
|
||||
Then('the Agent v2 Web app access URL should show it was copied', async function (this: DifyWorld) {
|
||||
await expect(this.getPage().getByLabel('Copied')).toBeVisible()
|
||||
})
|
||||
|
||||
When('I launch the Agent v2 Web app', async function (this: DifyWorld) {
|
||||
const launchLink = getWebAppCard(this).getByRole('link', { name: 'Launch' })
|
||||
const href = await launchLink.getAttribute('href')
|
||||
if (!href)
|
||||
throw new Error('Agent v2 Web app Launch link does not expose an href.')
|
||||
|
||||
const [webAppPage] = await Promise.all([
|
||||
this.getPage().waitForEvent('popup'),
|
||||
launchLink.click(),
|
||||
])
|
||||
|
||||
this.agentBuilder.accessPoint.webAppURL = href
|
||||
this.agentBuilder.accessPoint.webAppPage = webAppPage
|
||||
})
|
||||
|
||||
When('I open the Agent v2 Web app URL', async function (this: DifyWorld) {
|
||||
const webAppURL = this.agentBuilder.accessPoint.webAppURL
|
||||
if (!webAppURL)
|
||||
throw new Error('No Agent v2 Web app URL was recorded.')
|
||||
if (!this.context)
|
||||
throw new Error('Playwright browser context has not been initialized.')
|
||||
|
||||
const webAppPage = await this.context.newPage()
|
||||
await webAppPage.goto(webAppURL)
|
||||
|
||||
this.agentBuilder.accessPoint.webAppPage = webAppPage
|
||||
})
|
||||
|
||||
When('I send an E2E message in the Agent v2 Web app', async function (this: DifyWorld) {
|
||||
const webAppPage = this.agentBuilder.accessPoint.webAppPage
|
||||
if (!webAppPage)
|
||||
throw new Error('No Agent v2 Web app page was opened.')
|
||||
|
||||
const messageInput = webAppPage.getByRole('textbox').last()
|
||||
await expect(messageInput).toBeEditable({ timeout: 30_000 })
|
||||
await messageInput.fill('Please reply with the test success marker.')
|
||||
await messageInput.press('Enter')
|
||||
})
|
||||
|
||||
Then('the Agent v2 Web app should open in a new tab', async function (this: DifyWorld) {
|
||||
const webAppPage = this.agentBuilder.accessPoint.webAppPage
|
||||
const webAppURL = this.agentBuilder.accessPoint.webAppURL
|
||||
if (!webAppPage || !webAppURL)
|
||||
throw new Error('No Agent v2 Web app page was opened.')
|
||||
|
||||
await expect(webAppPage).toHaveURL(webAppURL)
|
||||
await expect(webAppPage.getByRole('textbox').last()).toBeEditable({ timeout: 30_000 })
|
||||
await webAppPage.close()
|
||||
this.agentBuilder.accessPoint.webAppPage = undefined
|
||||
this.agentBuilder.accessPoint.webAppURL = undefined
|
||||
})
|
||||
|
||||
Then(
|
||||
'the Agent v2 Web app response should include the updated E2E marker',
|
||||
async function (this: DifyWorld) {
|
||||
const webAppPage = this.agentBuilder.accessPoint.webAppPage
|
||||
if (!webAppPage)
|
||||
throw new Error('No Agent v2 Web app page was opened.')
|
||||
|
||||
await expect(webAppPage.getByText(agentBuilderExpectedTokens.updatedAgentReply))
|
||||
.toBeVisible({ timeout: 120_000 })
|
||||
},
|
||||
)
|
||||
|
||||
Then(
|
||||
'the Agent v2 Web app response should include the normal E2E marker',
|
||||
async function (this: DifyWorld) {
|
||||
const webAppPage = this.agentBuilder.accessPoint.webAppPage
|
||||
if (!webAppPage)
|
||||
throw new Error('No Agent v2 Web app page was opened.')
|
||||
|
||||
await expect(webAppPage.getByText(agentBuilderExpectedTokens.agentReply))
|
||||
.toBeVisible({ timeout: 120_000 })
|
||||
},
|
||||
)
|
||||
|
||||
Then(
|
||||
'the Agent v2 Web app response should not include the updated E2E marker',
|
||||
async function (this: DifyWorld) {
|
||||
const webAppPage = this.agentBuilder.accessPoint.webAppPage
|
||||
if (!webAppPage)
|
||||
throw new Error('No Agent v2 Web app page was opened.')
|
||||
|
||||
await expect(webAppPage.getByText(agentBuilderExpectedTokens.updatedAgentReply))
|
||||
.not
|
||||
.toBeVisible()
|
||||
},
|
||||
)
|
||||
|
||||
When('I close the Agent v2 Web app', async function (this: DifyWorld) {
|
||||
const webAppPage = this.agentBuilder.accessPoint.webAppPage
|
||||
if (!webAppPage)
|
||||
throw new Error('No Agent v2 Web app page was opened.')
|
||||
|
||||
await webAppPage.close()
|
||||
this.agentBuilder.accessPoint.webAppPage = undefined
|
||||
})
|
||||
|
||||
When('I open Agent v2 Embedded configuration', async function (this: DifyWorld) {
|
||||
await getWebAppCard(this).getByRole('button', { name: 'Embedded' }).click()
|
||||
})
|
||||
|
||||
Then('I should see the Agent v2 Embedded configuration dialog', async function (this: DifyWorld) {
|
||||
const dialog = getDialog(this, 'Embed on website')
|
||||
|
||||
await expect(dialog).toBeVisible()
|
||||
await expect(dialog.getByText('Embed on website')).toBeVisible()
|
||||
await expect(dialog.getByText(/iframe|script/i)).toBeVisible()
|
||||
})
|
||||
|
||||
When('I open Agent v2 Web app customization', async function (this: DifyWorld) {
|
||||
await getWebAppCard(this).getByRole('button', { name: 'Customize' }).click()
|
||||
})
|
||||
|
||||
Then('I should see the Agent v2 Web app customization dialog', async function (this: DifyWorld) {
|
||||
const dialog = getDialog(this, 'Customize AI web app')
|
||||
|
||||
await expect(dialog).toBeVisible()
|
||||
await expect(dialog.getByText('Customize AI web app')).toBeVisible()
|
||||
await expect(dialog.getByText(/NEXT_PUBLIC_APP_ID|NEXT_PUBLIC_API_URL/)).toBeVisible()
|
||||
})
|
||||
|
||||
When('I open Agent v2 Web app settings', async function (this: DifyWorld) {
|
||||
await getWebAppCard(this).getByRole('button', { name: 'Settings' }).click()
|
||||
})
|
||||
|
||||
Then('I should see the Agent v2 Web app settings dialog', async function (this: DifyWorld) {
|
||||
const dialog = getDialog(this, 'Web App Settings')
|
||||
|
||||
await expect(dialog).toBeVisible()
|
||||
await expect(dialog.getByRole('heading', { name: 'Web App Settings' })).toBeVisible()
|
||||
await expect(dialog.getByText('web app Name')).toBeVisible()
|
||||
await expect(dialog.getByText('web app Description')).toBeVisible()
|
||||
})
|
||||
|
||||
Then(
|
||||
'the current Agent v2 orchestration draft should be unchanged',
|
||||
async function (this: DifyWorld) {
|
||||
const snapshot = this.agentBuilder.accessPoint.composerDraftSnapshot
|
||||
if (!snapshot)
|
||||
throw new Error('No Agent v2 orchestration draft snapshot was recorded.')
|
||||
|
||||
const draft = await getAgentComposerDraft(getCurrentAgentId(this))
|
||||
|
||||
expect(JSON.stringify(draft.agent_soul ?? {})).toBe(snapshot)
|
||||
},
|
||||
)
|
||||
|
||||
When('I disable Agent v2 Web app access', async function (this: DifyWorld) {
|
||||
const webAppCard = getWebAppCard(this)
|
||||
const launchLink = webAppCard.getByRole('link', { name: 'Launch' })
|
||||
const href = await launchLink.getAttribute('href')
|
||||
if (!href)
|
||||
throw new Error('Agent v2 Web app Launch link does not expose an href.')
|
||||
|
||||
this.agentBuilder.accessPoint.webAppURL = href
|
||||
|
||||
await webAppCard.getByLabel('Toggle Web app access').click()
|
||||
})
|
||||
|
||||
Then('Agent v2 Web app access should be out of service', async function (this: DifyWorld) {
|
||||
const webAppCard = getWebAppCard(this)
|
||||
|
||||
await expect(webAppCard.getByText('Out of service')).toBeVisible()
|
||||
await expect(webAppCard.getByRole('button', { name: 'Launch' })).toBeDisabled()
|
||||
})
|
||||
|
||||
Given(
|
||||
'Agent v2 disabled Web app public unavailable state is available',
|
||||
async function (this: DifyWorld) {
|
||||
return skipBlockedPrecondition(
|
||||
this,
|
||||
'Disabled Agent v2 Web app public URL does not expose a stable user-visible unavailable state; the current route redirects to Web app sign-in.',
|
||||
{
|
||||
owner: 'product',
|
||||
remediation: 'Define and implement the disabled public Web app UX before enabling this scenario.',
|
||||
},
|
||||
)
|
||||
},
|
||||
)
|
||||
|
||||
When('I open the disabled Agent v2 Web app URL', async function (this: DifyWorld) {
|
||||
const webAppURL = this.agentBuilder.accessPoint.webAppURL
|
||||
if (!webAppURL)
|
||||
throw new Error('No Agent v2 Web app URL was recorded.')
|
||||
if (!this.context)
|
||||
throw new Error('Playwright browser context has not been initialized.')
|
||||
|
||||
const webAppPage = await this.context.newPage()
|
||||
await webAppPage.goto(webAppURL)
|
||||
|
||||
this.agentBuilder.accessPoint.webAppPage = webAppPage
|
||||
})
|
||||
|
||||
Then('the disabled Agent v2 Web app should show an unavailable state', async function (this: DifyWorld) {
|
||||
const webAppPage = this.agentBuilder.accessPoint.webAppPage
|
||||
if (!webAppPage)
|
||||
throw new Error('No Agent v2 Web app page was opened.')
|
||||
|
||||
await expect(webAppPage.getByText(/app is unavailable|site is disabled/i)).toBeVisible({
|
||||
timeout: 30_000,
|
||||
})
|
||||
await webAppPage.close()
|
||||
this.agentBuilder.accessPoint.webAppPage = undefined
|
||||
})
|
||||
|
||||
When('I enable Agent v2 Web app access', async function (this: DifyWorld) {
|
||||
await getWebAppCard(this).getByLabel('Toggle Web app access').click()
|
||||
})
|
||||
|
||||
Then('Agent v2 Web app access should be in service', async function (this: DifyWorld) {
|
||||
const webAppCard = getWebAppCard(this)
|
||||
|
||||
await expect(webAppCard.getByText('In service')).toBeVisible()
|
||||
await expect(webAppCard.getByRole('link', { name: 'Launch' })).toBeVisible()
|
||||
})
|
||||
|
||||
When('I open the restored Agent v2 Web app URL', async function (this: DifyWorld) {
|
||||
const webAppURL = this.agentBuilder.accessPoint.webAppURL
|
||||
if (!webAppURL)
|
||||
throw new Error('No Agent v2 Web app URL was recorded.')
|
||||
if (!this.context)
|
||||
throw new Error('Playwright browser context has not been initialized.')
|
||||
|
||||
const webAppPage = await this.context.newPage()
|
||||
await webAppPage.goto(webAppURL)
|
||||
|
||||
this.agentBuilder.accessPoint.webAppPage = webAppPage
|
||||
})
|
||||
|
||||
Then('the restored Agent v2 Web app should not show an unavailable state', async function (this: DifyWorld) {
|
||||
const webAppPage = this.agentBuilder.accessPoint.webAppPage
|
||||
if (!webAppPage)
|
||||
throw new Error('No Agent v2 Web app page was opened.')
|
||||
|
||||
await expect(webAppPage.getByText(/app is unavailable|site is disabled/i)).not.toBeVisible()
|
||||
await webAppPage.close()
|
||||
this.agentBuilder.accessPoint.webAppPage = undefined
|
||||
})
|
||||
@ -0,0 +1,70 @@
|
||||
import type { DifyWorld } from '../../support/world'
|
||||
import { Then, When } from '@cucumber/cucumber'
|
||||
import { expect } from '@playwright/test'
|
||||
import { getAgentReferencingWorkflows } from '../../agent-v2/support/agent'
|
||||
import { agentBuilderPreseededResources } from '../../agent-v2/support/agent-builder-resources'
|
||||
import { getAccessRegion, getPreseededResource } from './access-point-helpers'
|
||||
|
||||
Then(
|
||||
'I should see the Agent v2 Workflow access reference for {string}',
|
||||
async function (this: DifyWorld, workflowName: string) {
|
||||
const workflow = getPreseededResource(this, workflowName, 'workflow')
|
||||
const agent = getPreseededResource(
|
||||
this,
|
||||
agentBuilderPreseededResources.workflowReferenceAgent,
|
||||
'agent',
|
||||
)
|
||||
const references = await getAgentReferencingWorkflows(agent.id)
|
||||
const reference = references.find(item => item.app_id === workflow.id || item.app_name === workflow.name)
|
||||
if (!reference)
|
||||
throw new Error(`Agent "${agent.name}" does not reference workflow "${workflow.name}".`)
|
||||
|
||||
const accessRegion = getAccessRegion(this)
|
||||
const workflowSection = accessRegion.getByRole('region', { name: 'Workflow access' })
|
||||
const row = workflowSection.getByRole('row').filter({ hasText: workflowName })
|
||||
const nodeCount = reference.node_ids?.length ?? 0
|
||||
|
||||
await expect(accessRegion.getByRole('columnheader', { name: 'Name' })).toBeVisible()
|
||||
await expect(accessRegion.getByRole('columnheader', { name: 'Version' })).toBeVisible()
|
||||
await expect(accessRegion.getByRole('columnheader', { name: 'Nodes' })).toBeVisible()
|
||||
await expect(accessRegion.getByRole('columnheader', { name: 'Last updated' })).toBeVisible()
|
||||
await expect(accessRegion.getByRole('columnheader', { name: 'Actions' })).toBeVisible()
|
||||
await expect(row).toBeVisible({ timeout: 30_000 })
|
||||
await expect(row.getByText(reference.workflow_version, { exact: true })).toBeVisible()
|
||||
await expect(row.getByText(new RegExp(`^${nodeCount} nodes?$`))).toBeVisible()
|
||||
if (reference.app_updated_at == null)
|
||||
await expect(row.getByText('N/A', { exact: true })).toBeVisible()
|
||||
else
|
||||
await expect(row.getByText('N/A', { exact: true })).not.toBeVisible()
|
||||
await expect(row.getByRole('link', { name: `Open ${workflowName} in Studio` })).toBeVisible()
|
||||
},
|
||||
)
|
||||
|
||||
When(
|
||||
'I open the Agent v2 Workflow access reference for {string}',
|
||||
async function (this: DifyWorld, workflowName: string) {
|
||||
const workflowLink = this.getPage().getByRole('link', { name: `Open ${workflowName} in Studio` })
|
||||
|
||||
const [workflowPage] = await Promise.all([
|
||||
this.getPage().waitForEvent('popup'),
|
||||
workflowLink.click(),
|
||||
])
|
||||
|
||||
this.agentBuilder.accessPoint.workflowReferencePage = workflowPage
|
||||
},
|
||||
)
|
||||
|
||||
Then(
|
||||
'the Agent v2 Workflow access reference for {string} should open in Studio',
|
||||
async function (this: DifyWorld, workflowName: string) {
|
||||
const workflowPage = this.agentBuilder.accessPoint.workflowReferencePage
|
||||
if (!workflowPage)
|
||||
throw new Error('No Agent v2 Workflow access reference page was opened.')
|
||||
|
||||
const workflow = getPreseededResource(this, workflowName, 'workflow')
|
||||
|
||||
await expect(workflowPage).toHaveURL(new RegExp(`/app/${workflow.id}/workflow(?:\\?.*)?$`))
|
||||
await workflowPage.close()
|
||||
this.agentBuilder.accessPoint.workflowReferencePage = undefined
|
||||
},
|
||||
)
|
||||
72
e2e/features/step-definitions/agent-v2/access-point.steps.ts
Normal file
72
e2e/features/step-definitions/agent-v2/access-point.steps.ts
Normal file
@ -0,0 +1,72 @@
|
||||
import type { DifyWorld } from '../../support/world'
|
||||
import { Given, Then, When } from '@cucumber/cucumber'
|
||||
import { expect } from '@playwright/test'
|
||||
import { getAgentAccessPath, publishAgent } from '../../agent-v2/support/agent'
|
||||
import {
|
||||
getAccessRegion,
|
||||
getCurrentAgentId,
|
||||
getPreseededResource,
|
||||
} from './access-point-helpers'
|
||||
|
||||
Given('the Agent v2 draft has been published via API', async function (this: DifyWorld) {
|
||||
await publishAgent(getCurrentAgentId(this))
|
||||
})
|
||||
|
||||
When('I open the Agent v2 Access Point page', async function (this: DifyWorld) {
|
||||
await this.getPage().goto(getAgentAccessPath(getCurrentAgentId(this)))
|
||||
})
|
||||
|
||||
When(
|
||||
'I open the preseeded Agent v2 Access Point page for {string} from the Agent Roster',
|
||||
async function (this: DifyWorld, agentName: string) {
|
||||
const page = this.getPage()
|
||||
const agent = getPreseededResource(this, agentName, 'agent')
|
||||
|
||||
await page.goto('/roster')
|
||||
await page.getByRole('link', { name: agentName }).click()
|
||||
await expect(page).toHaveURL(new RegExp(`/roster/agent/${agent.id}/configure(?:\\?.*)?$`))
|
||||
await page.getByRole('link', { name: 'Access Point' }).click()
|
||||
await expect(page).toHaveURL(new RegExp(`/roster/agent/${agent.id}/access(?:\\?.*)?$`))
|
||||
await expect(page.getByRole('region', { name: 'Access Point' })).toBeVisible({
|
||||
timeout: 30_000,
|
||||
})
|
||||
},
|
||||
)
|
||||
|
||||
When('I switch to the Agent v2 Access Point section', async function (this: DifyWorld) {
|
||||
const page = this.getPage()
|
||||
const agentId = getCurrentAgentId(this)
|
||||
|
||||
await page.getByRole('link', { name: 'Access Point' }).click()
|
||||
await expect(page).toHaveURL(new RegExp(`/roster/agent/${agentId}/access(?:\\?.*)?$`))
|
||||
await expect(page.getByRole('region', { name: 'Access Point' })).toBeVisible()
|
||||
})
|
||||
|
||||
Then('I should see the Agent v2 Access Point overview', async function (this: DifyWorld) {
|
||||
const accessRegion = getAccessRegion(this)
|
||||
|
||||
await expect(accessRegion).toBeVisible({ timeout: 30_000 })
|
||||
await expect(accessRegion.getByRole('heading', { name: 'Access Point' })).toBeVisible()
|
||||
await expect(accessRegion.getByRole('heading', { name: 'Web app' })).toBeVisible()
|
||||
await expect(accessRegion.getByText('Access URL')).toBeVisible()
|
||||
await expect(accessRegion.getByLabel('Copy access URL')).toBeVisible()
|
||||
await expect(accessRegion.getByLabel('Toggle Web app access')).toBeVisible()
|
||||
await expect(accessRegion.getByRole('link', { name: 'Launch' })).toBeVisible()
|
||||
await expect(accessRegion.getByRole('button', { name: 'Embedded' })).toBeVisible()
|
||||
await expect(accessRegion.getByRole('button', { name: 'Customize' })).toBeVisible()
|
||||
await expect(accessRegion.getByRole('button', { name: 'Settings' })).toBeVisible()
|
||||
await expect(accessRegion.getByRole('heading', { name: 'Backend service API' })).toBeVisible()
|
||||
await expect(accessRegion.getByText('Service API Endpoint')).toBeVisible()
|
||||
await expect(accessRegion.getByLabel('Copy service API endpoint')).toBeVisible()
|
||||
await expect(accessRegion.getByLabel('Toggle Backend service API access')).toBeVisible()
|
||||
await expect(accessRegion.getByRole('button', { name: /^API Key\b/ })).toBeVisible()
|
||||
await expect(accessRegion.getByRole('link', { name: 'API Reference' })).toBeVisible()
|
||||
await expect(accessRegion.getByText(/^(?:In|Out of) service$/i)).toHaveCount(2)
|
||||
await expect(accessRegion.getByRole('heading', { name: 'Workflow access' })).toBeVisible()
|
||||
await expect(accessRegion.getByRole('columnheader', { name: 'Name' })).toBeVisible()
|
||||
await expect(accessRegion.getByRole('columnheader', { name: 'Version' })).toBeVisible()
|
||||
await expect(accessRegion.getByRole('columnheader', { name: 'Nodes' })).toBeVisible()
|
||||
await expect(accessRegion.getByRole('columnheader', { name: 'Last updated' })).toBeVisible()
|
||||
await expect(accessRegion.getByRole('columnheader', { name: 'Actions' })).toBeVisible()
|
||||
await expect(accessRegion.getByText('No workflow references yet.')).toBeVisible()
|
||||
})
|
||||
@ -0,0 +1,54 @@
|
||||
import type { DifyWorld } from '../../support/world'
|
||||
import { Then, When } from '@cucumber/cucumber'
|
||||
import { expect } from '@playwright/test'
|
||||
import { openAgentAdvancedSettings } from './configure-helpers'
|
||||
|
||||
When('I expand Agent v2 Advanced Settings', async function (this: DifyWorld) {
|
||||
const page = this.getPage()
|
||||
const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' })
|
||||
|
||||
await page.getByRole('button', { name: 'Advanced Settings' }).first().click()
|
||||
await expect(advancedSettings.getByRole('heading', { name: 'Env Editor' })).toBeVisible()
|
||||
})
|
||||
|
||||
When('I collapse Agent v2 Advanced Settings', async function (this: DifyWorld) {
|
||||
const page = this.getPage()
|
||||
const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' })
|
||||
|
||||
await page.getByRole('button', { name: 'Advanced Settings' }).first().click()
|
||||
await expect(advancedSettings.getByRole('heading', { name: 'Env Editor' }))
|
||||
.not
|
||||
.toBeVisible()
|
||||
})
|
||||
|
||||
Then(
|
||||
'Agent v2 Advanced Settings should describe supported entries while collapsed',
|
||||
async function (this: DifyWorld) {
|
||||
const page = this.getPage()
|
||||
const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' })
|
||||
|
||||
await expect(advancedSettings).toBeVisible()
|
||||
await expect(
|
||||
advancedSettings.getByText('For power users. Env vars, sandbox & memory.'),
|
||||
).toBeVisible()
|
||||
await expect(advancedSettings.getByRole('heading', { name: 'Env Editor' }))
|
||||
.not
|
||||
.toBeVisible()
|
||||
},
|
||||
)
|
||||
|
||||
Then(
|
||||
'I should see the supported Agent v2 Advanced Settings entries',
|
||||
async function (this: DifyWorld) {
|
||||
const advancedSettings = await openAgentAdvancedSettings(this.getPage())
|
||||
const envEditor = advancedSettings.getByRole('region', { name: 'Env Editor' })
|
||||
|
||||
await expect(envEditor).toBeVisible()
|
||||
await expect(envEditor.getByRole('button', { name: 'Import .env' })).toBeVisible()
|
||||
await expect(envEditor.getByRole('button', { name: 'Add environment variable' }))
|
||||
.toBeVisible()
|
||||
await expect(envEditor.getByText('Key', { exact: true })).toBeVisible()
|
||||
await expect(envEditor.getByText('Value', { exact: true })).toBeVisible()
|
||||
await expect(envEditor.getByText('Scope', { exact: true })).toBeVisible()
|
||||
},
|
||||
)
|
||||
290
e2e/features/step-definitions/agent-v2/agent-edit.steps.ts
Normal file
290
e2e/features/step-definitions/agent-v2/agent-edit.steps.ts
Normal file
@ -0,0 +1,290 @@
|
||||
import type { PostAgentByAgentIdCopyResponse } from '@dify/contracts/api/console/agent/types.gen'
|
||||
import type { DifyWorld } from '../../support/world'
|
||||
import { Given, Then, When } from '@cucumber/cucumber'
|
||||
import { expect } from '@playwright/test'
|
||||
import { createE2EResourceName } from '../../../support/naming'
|
||||
import {
|
||||
getAgentComposerDraft,
|
||||
getTestAgent,
|
||||
} from '../../agent-v2/support/agent'
|
||||
import { agentBuilderExpectedTokens, agentBuilderFixedInputs, agentBuilderPreseededResources } from '../../agent-v2/support/agent-builder-resources'
|
||||
import { normalAgentPrompt } from '../../agent-v2/support/agent-soul'
|
||||
import {
|
||||
asArray,
|
||||
asRecord,
|
||||
asString,
|
||||
skipBlockedPrecondition,
|
||||
} from '../../agent-v2/support/preflight/common'
|
||||
import { agentBuilderTestMaterials } from '../../agent-v2/support/test-materials'
|
||||
import {
|
||||
expectProviderToolActionVisible,
|
||||
getCurrentAgentId,
|
||||
getPreseededAgent,
|
||||
openAgentKnowledgeRetrievalDialog,
|
||||
} from './configure-helpers'
|
||||
|
||||
const getComposerInheritanceSnapshot = async (agentId: string) => {
|
||||
const draft = await getAgentComposerDraft(agentId)
|
||||
const soul = draft.agent_soul ?? {}
|
||||
const model = asRecord(soul.model)
|
||||
const prompt = asRecord(soul.prompt)
|
||||
const files = asArray(soul.config_files)
|
||||
const skills = asArray(soul.config_skills)
|
||||
const tools = asArray(asRecord(soul.tools).dify_tools)
|
||||
const knowledgeSets = asArray(asRecord(soul.knowledge).sets)
|
||||
|
||||
return {
|
||||
fileNames: files.map(file => asString(asRecord(file).name)).filter(Boolean).sort(),
|
||||
knowledgeDatasetNames: knowledgeSets
|
||||
.flatMap(set => asArray(asRecord(set).datasets))
|
||||
.map(dataset => asString(asRecord(dataset).name))
|
||||
.filter(Boolean)
|
||||
.sort(),
|
||||
model: {
|
||||
name: asString(model.model),
|
||||
provider: asString(model.model_provider),
|
||||
},
|
||||
prompt: asString(prompt.system_prompt),
|
||||
skillNames: skills.map(skill => asString(asRecord(skill).name)).filter(Boolean).sort(),
|
||||
toolSignatures: tools
|
||||
.map((tool) => {
|
||||
const record = asRecord(tool)
|
||||
const provider = asString(record.provider_id)
|
||||
|| asString(record.provider)
|
||||
|| asString(record.plugin_id)
|
||||
|| asString(record.name)
|
||||
const toolName = asString(record.tool_name) || asString(record.name)
|
||||
|
||||
return `${provider}/${toolName}`
|
||||
})
|
||||
.filter(signature => signature !== '/')
|
||||
.sort(),
|
||||
}
|
||||
}
|
||||
|
||||
When(
|
||||
'I duplicate the preseeded Agent v2 {string} from the Agent Roster',
|
||||
async function (this: DifyWorld, agentName: string) {
|
||||
const page = this.getPage()
|
||||
const agent = getPreseededAgent(this, agentName)
|
||||
const copyName = createE2EResourceName('Agent', 'copy')
|
||||
|
||||
await page.goto('/roster')
|
||||
const card = page.locator('article').filter({
|
||||
has: page.getByRole('link', { name: agentName }),
|
||||
}).first()
|
||||
|
||||
await expect(card).toBeVisible({ timeout: 30_000 })
|
||||
await card.hover()
|
||||
await card.getByLabel(`More actions for ${agentName}`).click()
|
||||
await page.getByRole('menuitem', { name: 'Duplicate' }).click()
|
||||
|
||||
const dialog = page.getByRole('dialog', { name: 'Duplicate agent' })
|
||||
await expect(dialog).toBeVisible()
|
||||
await dialog.getByRole('textbox', { name: /Name/ }).fill(copyName)
|
||||
|
||||
const copyResponsePromise = page.waitForResponse(response => (
|
||||
response.request().method() === 'POST'
|
||||
&& new URL(response.url()).pathname.endsWith(`/console/api/agent/${agent.id}/copy`)
|
||||
))
|
||||
await dialog.getByRole('button', { name: 'Duplicate' }).click()
|
||||
|
||||
const copyResponse = await copyResponsePromise
|
||||
expect(copyResponse.status()).toBe(201)
|
||||
const copiedAgent = (await copyResponse.json()) as PostAgentByAgentIdCopyResponse
|
||||
if (!copiedAgent.id)
|
||||
throw new Error('Agent v2 duplicate response did not include a copied Agent ID.')
|
||||
|
||||
this.createdAgentIds.push(copiedAgent.id)
|
||||
this.lastCreatedAgentName = copiedAgent.name
|
||||
this.lastCreatedAgentRole = copiedAgent.role ?? undefined
|
||||
|
||||
await expect(page.getByText('Agent duplicated.')).toBeVisible()
|
||||
},
|
||||
)
|
||||
|
||||
Then('I should see the Agent v2 full-config fixture sections', async function (this: DifyWorld) {
|
||||
const page = this.getPage()
|
||||
const stableModel = this.agentBuilder.preflight.stableModel
|
||||
if (!stableModel)
|
||||
throw new Error('Stable chat model preflight must run before asserting the full-config Agent.')
|
||||
|
||||
await expect(page.getByRole('heading', { name: 'Configure' })).toBeVisible({ timeout: 30_000 })
|
||||
await expect(page.getByText(agentBuilderPreseededResources.fullConfigAgent, { exact: true }))
|
||||
.toBeVisible()
|
||||
await expect(page.getByText(stableModel.name, { exact: true })).toBeVisible()
|
||||
|
||||
const promptSection = page.getByRole('region', { name: 'Prompt' })
|
||||
await expect(promptSection).toBeVisible()
|
||||
await expect(promptSection).toContainText(agentBuilderExpectedTokens.agentReply)
|
||||
|
||||
const skillsSection = page.getByRole('region', { name: 'Skills' })
|
||||
await expect(skillsSection).toBeVisible()
|
||||
await expect(skillsSection.getByRole('button', {
|
||||
exact: true,
|
||||
name: agentBuilderPreseededResources.summarySkill,
|
||||
})).toBeVisible()
|
||||
|
||||
const filesSection = page.getByRole('region', { name: 'Files' })
|
||||
await expect(filesSection).toBeVisible()
|
||||
await expect(filesSection.getByRole('button', {
|
||||
exact: true,
|
||||
name: agentBuilderTestMaterials.smallFile,
|
||||
})).toBeVisible()
|
||||
await expect(filesSection.getByRole('button', {
|
||||
exact: true,
|
||||
name: agentBuilderTestMaterials.specialFilename,
|
||||
})).toBeVisible()
|
||||
|
||||
const toolsSection = page.getByRole('region', { name: 'Tools' })
|
||||
await expect(toolsSection).toBeVisible()
|
||||
await expectProviderToolActionVisible(
|
||||
toolsSection,
|
||||
agentBuilderPreseededResources.jsonReplaceTool,
|
||||
)
|
||||
|
||||
const knowledgeSection = page.getByRole('region', { name: 'Knowledge Retrieval' })
|
||||
await expect(knowledgeSection).toBeVisible()
|
||||
await expect(knowledgeSection.getByText('Retrieval 1', { exact: true })).toBeVisible()
|
||||
|
||||
const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' })
|
||||
await expect(advancedSettings).toBeVisible()
|
||||
await expect(
|
||||
advancedSettings.getByText('For power users. Env vars, sandbox & memory.'),
|
||||
).toBeVisible()
|
||||
})
|
||||
|
||||
Then(
|
||||
'the duplicated Agent v2 should inherit the full-config fixture from {string}',
|
||||
async function (this: DifyWorld, agentName: string) {
|
||||
const sourceAgent = getPreseededAgent(this, agentName)
|
||||
const duplicatedAgentId = getCurrentAgentId(this)
|
||||
const stableModel = this.agentBuilder.preflight.stableModel
|
||||
if (!stableModel)
|
||||
throw new Error('Stable chat model preflight must run before asserting the duplicated Agent.')
|
||||
|
||||
const [sourceDetail, duplicatedDetail, sourceSnapshot, duplicatedSnapshot] = await Promise.all([
|
||||
getTestAgent(sourceAgent.id),
|
||||
getTestAgent(duplicatedAgentId),
|
||||
getComposerInheritanceSnapshot(sourceAgent.id),
|
||||
getComposerInheritanceSnapshot(duplicatedAgentId),
|
||||
])
|
||||
|
||||
expect(duplicatedDetail.id).toBe(duplicatedAgentId)
|
||||
expect(duplicatedDetail.name).toBe(this.lastCreatedAgentName)
|
||||
expect(duplicatedDetail.active_config_is_published).toBe(sourceDetail.active_config_is_published)
|
||||
expect(duplicatedSnapshot.model).toEqual({
|
||||
name: stableModel.name,
|
||||
provider: stableModel.provider,
|
||||
})
|
||||
expect(duplicatedSnapshot.model).toEqual(sourceSnapshot.model)
|
||||
expect(duplicatedSnapshot.prompt).toBe(sourceSnapshot.prompt)
|
||||
expect(duplicatedSnapshot.fileNames).toEqual(expect.arrayContaining([
|
||||
agentBuilderTestMaterials.smallFile,
|
||||
agentBuilderTestMaterials.specialFilename,
|
||||
]))
|
||||
expect(duplicatedSnapshot.skillNames).toEqual(expect.arrayContaining([
|
||||
agentBuilderPreseededResources.summarySkill,
|
||||
]))
|
||||
expect(duplicatedSnapshot.skillNames).toEqual(sourceSnapshot.skillNames)
|
||||
expect(duplicatedSnapshot.toolSignatures).toEqual(sourceSnapshot.toolSignatures)
|
||||
expect(duplicatedSnapshot.knowledgeDatasetNames).toEqual(expect.arrayContaining([
|
||||
agentBuilderPreseededResources.agentKnowledgeBase,
|
||||
]))
|
||||
},
|
||||
)
|
||||
|
||||
Then(
|
||||
'the preseeded Agent v2 {string} should still use the normal E2E prompt',
|
||||
async function (this: DifyWorld, agentName: string) {
|
||||
const sourceAgent = getPreseededAgent(this, agentName)
|
||||
|
||||
await expect.poll(
|
||||
async () => {
|
||||
const draft = await getAgentComposerDraft(sourceAgent.id)
|
||||
|
||||
return asString(asRecord(draft.agent_soul?.prompt).system_prompt)
|
||||
},
|
||||
{ timeout: 30_000 },
|
||||
).toBe(normalAgentPrompt)
|
||||
},
|
||||
)
|
||||
|
||||
Then('I should see the Agent v2 tool state fixture tools', async function (this: DifyWorld) {
|
||||
const page = this.getPage()
|
||||
const toolsSection = page.getByRole('region', { name: 'Tools' })
|
||||
|
||||
await expect(toolsSection).toBeVisible({ timeout: 30_000 })
|
||||
await expect(toolsSection.getByRole('button', { exact: true, name: 'Not authorized' })).toBeVisible()
|
||||
|
||||
const { action: jsonReplaceAction, tool: jsonTool } = await expectProviderToolActionVisible(
|
||||
toolsSection,
|
||||
agentBuilderPreseededResources.jsonReplaceTool,
|
||||
)
|
||||
await jsonReplaceAction.hover()
|
||||
await expect(toolsSection.getByRole('button', {
|
||||
exact: true,
|
||||
name: `Edit ${jsonTool.actionName}`,
|
||||
})).toBeVisible()
|
||||
await expect(toolsSection.getByRole('button', {
|
||||
exact: true,
|
||||
name: `Remove ${jsonTool.actionName}`,
|
||||
})).toBeVisible()
|
||||
|
||||
await expectProviderToolActionVisible(
|
||||
toolsSection,
|
||||
agentBuilderPreseededResources.tavilySearchTool,
|
||||
)
|
||||
})
|
||||
|
||||
async function skipToolCredentialErrorState(world: DifyWorld) {
|
||||
return skipBlockedPrecondition(
|
||||
world,
|
||||
'Agent v2 Tool credential error state is not covered: the current fixture only proves usable and not-authorized tool states.',
|
||||
{
|
||||
owner: 'seed/product',
|
||||
remediation: 'Define a stable invalid credential fixture and the expected user-visible error label before enabling this scenario.',
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
Given('Agent v2 Tool credential error state is available', async function (this: DifyWorld) {
|
||||
return skipToolCredentialErrorState(this)
|
||||
})
|
||||
|
||||
Then('Agent v2 Tool credential error state should be available', async function (this: DifyWorld) {
|
||||
return skipToolCredentialErrorState(this)
|
||||
})
|
||||
|
||||
Then('I should see the Agent v2 dual retrieval fixture settings', async function (this: DifyWorld) {
|
||||
const page = this.getPage()
|
||||
const knowledgeSection = page.getByRole('region', { name: 'Knowledge Retrieval' })
|
||||
|
||||
await expect(knowledgeSection).toBeVisible({ timeout: 30_000 })
|
||||
await expect(knowledgeSection.getByText('Retrieval 1', { exact: true })).toBeVisible()
|
||||
await expect(knowledgeSection.getByText('Retrieval 2', { exact: true })).toBeVisible()
|
||||
|
||||
const agentDecideDialog = await openAgentKnowledgeRetrievalDialog(knowledgeSection, 'Retrieval 1')
|
||||
await expect(agentDecideDialog.getByText(agentBuilderPreseededResources.agentKnowledgeBase, {
|
||||
exact: true,
|
||||
})).toBeVisible()
|
||||
await expect(agentDecideDialog.getByRole('radio', {
|
||||
exact: true,
|
||||
name: 'Agent decide',
|
||||
})).toBeChecked()
|
||||
await agentDecideDialog.getByRole('button', { name: 'Close' }).click()
|
||||
await expect(agentDecideDialog).not.toBeVisible()
|
||||
|
||||
const customQueryDialog = await openAgentKnowledgeRetrievalDialog(knowledgeSection, 'Retrieval 2')
|
||||
await expect(customQueryDialog.getByText(agentBuilderPreseededResources.agentKnowledgeBase, {
|
||||
exact: true,
|
||||
})).toBeVisible()
|
||||
await expect(customQueryDialog.getByRole('radio', {
|
||||
exact: true,
|
||||
name: 'Custom query',
|
||||
})).toBeChecked()
|
||||
await expect(customQueryDialog.getByRole('textbox', {
|
||||
exact: true,
|
||||
name: 'Custom query text',
|
||||
})).toHaveValue(agentBuilderFixedInputs.customKnowledgeQuery)
|
||||
})
|
||||
48
e2e/features/step-definitions/agent-v2/agent-roster.steps.ts
Normal file
48
e2e/features/step-definitions/agent-v2/agent-roster.steps.ts
Normal file
@ -0,0 +1,48 @@
|
||||
import type { AgentAppDetailWithSite } from '@dify/contracts/api/console/agent/types.gen'
|
||||
import type { DifyWorld } from '../../support/world'
|
||||
import { Then, When } from '@cucumber/cucumber'
|
||||
import { expect } from '@playwright/test'
|
||||
import { createE2EResourceName } from '../../../support/naming'
|
||||
|
||||
When('I create an Agent v2 test agent from the Agent Roster', async function (this: DifyWorld) {
|
||||
const page = this.getPage()
|
||||
const agentName = createE2EResourceName('Agent', 'roster-ui')
|
||||
const agentRole = 'E2E roster-created assistant'
|
||||
const agentDescription = 'Created by Dify E2E through the Agent Roster UI.'
|
||||
|
||||
await page.goto('/roster')
|
||||
await page.getByRole('button', { name: 'Create agent' }).click()
|
||||
|
||||
const dialog = page.getByRole('dialog', { name: 'Create agent' })
|
||||
await expect(dialog).toBeVisible()
|
||||
await dialog.getByRole('textbox', { name: 'Name' }).fill(agentName)
|
||||
await dialog.getByRole('textbox', { name: 'Role' }).fill(agentRole)
|
||||
await dialog.getByRole('textbox', { name: 'Description' }).fill(agentDescription)
|
||||
|
||||
const createResponsePromise = page.waitForResponse(response => (
|
||||
response.request().method() === 'POST'
|
||||
&& new URL(response.url()).pathname.endsWith('/console/api/agent')
|
||||
))
|
||||
|
||||
await dialog.getByRole('button', { name: 'Create' }).click()
|
||||
const createResponse = await createResponsePromise
|
||||
expect(createResponse.ok()).toBe(true)
|
||||
|
||||
const createdAgent = await createResponse.json() as AgentAppDetailWithSite
|
||||
this.createdAgentIds.push(createdAgent.id)
|
||||
this.lastCreatedAgentName = createdAgent.name
|
||||
this.lastCreatedAgentRole = createdAgent.role ?? undefined
|
||||
})
|
||||
|
||||
Then('the created Agent v2 should open in Configure', async function (this: DifyWorld) {
|
||||
const agentId = this.createdAgentIds.at(-1)
|
||||
if (!agentId)
|
||||
throw new Error('No Agent v2 ID found. Create an Agent v2 test agent first.')
|
||||
|
||||
await expect(this.getPage()).toHaveURL(
|
||||
new RegExp(`/roster/agent/${agentId}/configure(?:\\?.*)?$`),
|
||||
{ timeout: 30_000 },
|
||||
)
|
||||
await expect(this.getPage().getByRole('heading', { name: 'Configure' }))
|
||||
.toBeVisible({ timeout: 30_000 })
|
||||
})
|
||||
367
e2e/features/step-definitions/agent-v2/build-draft.steps.ts
Normal file
367
e2e/features/step-definitions/agent-v2/build-draft.steps.ts
Normal file
@ -0,0 +1,367 @@
|
||||
import type { Response } from '@playwright/test'
|
||||
import type { DifyWorld } from '../../support/world'
|
||||
import { readFile } from 'node:fs/promises'
|
||||
import { Given, Then, When } from '@cucumber/cucumber'
|
||||
import { expect } from '@playwright/test'
|
||||
import {
|
||||
getAgentComposerDraft,
|
||||
saveAgentComposerDraft,
|
||||
} from '../../agent-v2/support/agent'
|
||||
import { saveAgentBuildDraft } from '../../agent-v2/support/agent-build-draft'
|
||||
import { agentBuilderFixedInputs, agentBuilderPreseededResources } from '../../agent-v2/support/agent-builder-resources'
|
||||
import { uploadAgentConfigFileToDraft } from '../../agent-v2/support/agent-drive'
|
||||
import {
|
||||
createAgentSoulConfigWithModel,
|
||||
normalAgentPrompt,
|
||||
normalAgentSoulConfig,
|
||||
updatedAgentPrompt,
|
||||
updatedAgentSoulConfig,
|
||||
} from '../../agent-v2/support/agent-soul'
|
||||
import { asArray, asRecord, skipBlockedPrecondition } from '../../agent-v2/support/preflight/common'
|
||||
import { hasToolEntry } from '../../agent-v2/support/preflight/tools'
|
||||
import { agentBuilderTestMaterials, getAgentBuilderTestMaterialPath } from '../../agent-v2/support/test-materials'
|
||||
import { getPreseededToolContract } from '../../agent-v2/support/tools'
|
||||
import {
|
||||
getAgentEnvVariableValue,
|
||||
getCurrentAgentId,
|
||||
uploadSummaryConfigSkillForBuildDraft,
|
||||
} from './configure-helpers'
|
||||
|
||||
Given(
|
||||
'an Agent v2 Build draft adds the supported E2E files, skills, and env',
|
||||
async function (this: DifyWorld) {
|
||||
const agentId = getCurrentAgentId(this)
|
||||
const configFile = await uploadAgentConfigFileToDraft({
|
||||
agentId,
|
||||
fileName: agentBuilderTestMaterials.smallFile,
|
||||
filePath: getAgentBuilderTestMaterialPath('smallFile'),
|
||||
})
|
||||
const skill = await uploadSummaryConfigSkillForBuildDraft(this)
|
||||
|
||||
if (!configFile.file_id)
|
||||
throw new Error('Agent v2 build draft config file fixture did not return a file_id.')
|
||||
this.createdAgentConfigFiles.push({ agentId, name: configFile.name })
|
||||
|
||||
const normalConfig = this.agentBuilder.preflight.stableModel
|
||||
? createAgentSoulConfigWithModel(normalAgentSoulConfig, this.agentBuilder.preflight.stableModel)
|
||||
: normalAgentSoulConfig
|
||||
const updatedConfig = this.agentBuilder.preflight.stableModel
|
||||
? createAgentSoulConfigWithModel(updatedAgentSoulConfig, this.agentBuilder.preflight.stableModel)
|
||||
: updatedAgentSoulConfig
|
||||
|
||||
await saveAgentComposerDraft(agentId, normalConfig)
|
||||
await saveAgentBuildDraft(agentId, {
|
||||
...updatedConfig,
|
||||
config_files: [configFile],
|
||||
config_skills: [skill],
|
||||
env: {
|
||||
secret_refs: [],
|
||||
variables: [{
|
||||
id: agentBuilderFixedInputs.envPlainKey,
|
||||
key: agentBuilderFixedInputs.envPlainKey,
|
||||
name: agentBuilderFixedInputs.envPlainKey,
|
||||
value: agentBuilderFixedInputs.envPlainValue,
|
||||
variable: agentBuilderFixedInputs.envPlainKey,
|
||||
}],
|
||||
},
|
||||
})
|
||||
},
|
||||
)
|
||||
|
||||
Given(
|
||||
'an Agent v2 Build draft includes the existing e2e-summary-skill Skill',
|
||||
async function (this: DifyWorld) {
|
||||
const skill = await uploadSummaryConfigSkillForBuildDraft(this)
|
||||
const normalConfig = this.agentBuilder.preflight.stableModel
|
||||
? createAgentSoulConfigWithModel(normalAgentSoulConfig, this.agentBuilder.preflight.stableModel)
|
||||
: normalAgentSoulConfig
|
||||
const updatedConfig = this.agentBuilder.preflight.stableModel
|
||||
? createAgentSoulConfigWithModel(updatedAgentSoulConfig, this.agentBuilder.preflight.stableModel)
|
||||
: updatedAgentSoulConfig
|
||||
const configSkills = [skill]
|
||||
|
||||
await saveAgentComposerDraft(getCurrentAgentId(this), {
|
||||
...normalConfig,
|
||||
config_skills: configSkills,
|
||||
})
|
||||
await saveAgentBuildDraft(getCurrentAgentId(this), {
|
||||
...updatedConfig,
|
||||
config_skills: configSkills,
|
||||
})
|
||||
},
|
||||
)
|
||||
|
||||
Given('an Agent v2 Build draft uses the updated E2E prompt', async function (this: DifyWorld) {
|
||||
await saveAgentBuildDraft(getCurrentAgentId(this), updatedAgentSoulConfig)
|
||||
})
|
||||
|
||||
Given(
|
||||
'an Agent v2 Build draft uses the updated E2E prompt with the stable E2E model',
|
||||
async function (this: DifyWorld) {
|
||||
if (!this.agentBuilder.preflight.stableModel)
|
||||
throw new Error('Create an Agent v2 Build draft with a stable model after stable model preflight.')
|
||||
|
||||
await saveAgentBuildDraft(
|
||||
getCurrentAgentId(this),
|
||||
createAgentSoulConfigWithModel(updatedAgentSoulConfig, this.agentBuilder.preflight.stableModel),
|
||||
)
|
||||
},
|
||||
)
|
||||
|
||||
When('I generate an Agent v2 Build draft from the fixed instruction', async function (this: DifyWorld) {
|
||||
const page = this.getPage()
|
||||
const agentId = getCurrentAgentId(this)
|
||||
const instruction = (await readFile(getAgentBuilderTestMaterialPath('buildInstruction'), 'utf8')).trim()
|
||||
|
||||
await page.getByRole('button', { name: 'Build' }).click()
|
||||
await page.getByPlaceholder('Describe what your agent should do').fill(instruction)
|
||||
|
||||
const checkoutResponsePromise = page.waitForResponse(response => (
|
||||
response.request().method() === 'POST'
|
||||
&& new URL(response.url()).pathname.endsWith(`/console/api/agent/${agentId}/build-draft/checkout`)
|
||||
))
|
||||
const chatResponsePromise = page.waitForResponse(response => (
|
||||
response.request().method() === 'POST'
|
||||
&& new URL(response.url()).pathname.endsWith(`/console/api/agent/${agentId}/chat-messages`)
|
||||
))
|
||||
|
||||
await page.getByRole('button', { name: 'Start build' }).click()
|
||||
expect((await checkoutResponsePromise).ok()).toBe(true)
|
||||
expect((await chatResponsePromise).ok()).toBe(true)
|
||||
await expect(page.getByText('Build draft')).toBeVisible({ timeout: 120_000 })
|
||||
await expect(page.getByRole('button', { name: 'Apply' })).toBeEnabled({ timeout: 120_000 })
|
||||
await expect(page.getByRole('button', { name: 'Discard' })).toBeEnabled()
|
||||
})
|
||||
|
||||
const expectPageResponseOK = async (response: Response, action: string) => {
|
||||
if (response.ok())
|
||||
return
|
||||
|
||||
let body = ''
|
||||
try {
|
||||
body = await response.text()
|
||||
}
|
||||
catch {
|
||||
body = '<response body unavailable>'
|
||||
}
|
||||
|
||||
const trimmedBody = body.length > 1000 ? `${body.slice(0, 1000)}...` : body
|
||||
throw new Error(`${action} failed with ${response.status()} ${response.statusText()} at ${response.url()}: ${trimmedBody}`)
|
||||
}
|
||||
|
||||
When('I discard the Agent v2 Build draft', async function (this: DifyWorld) {
|
||||
await this.getPage().getByRole('button', { name: 'Discard' }).click()
|
||||
})
|
||||
|
||||
When('I apply the Agent v2 Build draft', async function (this: DifyWorld) {
|
||||
const page = this.getPage()
|
||||
const agentId = getCurrentAgentId(this)
|
||||
const applyButton = page.getByRole('button', { name: 'Apply' })
|
||||
|
||||
await expect(applyButton).toBeEnabled({ timeout: 30_000 })
|
||||
const finalizeResponsePromise = page.waitForResponse(response => (
|
||||
response.request().method() === 'POST'
|
||||
&& new URL(response.url()).pathname.endsWith(`/console/api/agent/${agentId}/build-chat/finalize`)
|
||||
), { timeout: 120_000 })
|
||||
const applyResponsePromise = page.waitForResponse(response => (
|
||||
response.request().method() === 'POST'
|
||||
&& new URL(response.url()).pathname.endsWith(`/console/api/agent/${agentId}/build-draft/apply`)
|
||||
), { timeout: 120_000 })
|
||||
|
||||
await applyButton.click()
|
||||
await expectPageResponseOK(await finalizeResponsePromise, 'Finalize Agent v2 Build draft')
|
||||
await expectPageResponseOK(await applyResponsePromise, 'Apply Agent v2 Build draft')
|
||||
await expect(page.getByText('Action succeeded')).toBeVisible()
|
||||
})
|
||||
|
||||
async function skipBuildDraftToolWriteback(world: DifyWorld) {
|
||||
return skipBlockedPrecondition(
|
||||
world,
|
||||
'Build draft Dify Tool writeback is not available: Build draft currently supports files, skills, and env only.',
|
||||
{
|
||||
owner: 'product',
|
||||
remediation: 'Define and implement Build draft Tool writeback before enabling this scenario.',
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
Given('Agent v2 Build chat Dify Tool writeback is available', async function (this: DifyWorld) {
|
||||
return skipBuildDraftToolWriteback(this)
|
||||
})
|
||||
|
||||
Then('Agent v2 Build chat Dify Tool writeback should be available', async function (this: DifyWorld) {
|
||||
return skipBuildDraftToolWriteback(this)
|
||||
})
|
||||
|
||||
async function skipBuildDraftUnavailableResourceRecovery(world: DifyWorld) {
|
||||
return skipBlockedPrecondition(
|
||||
world,
|
||||
'Build chat unavailable Skill/Tool recovery is not covered: the product needs a stable user-visible failure state and deterministic request fixture before this can be automated.',
|
||||
{
|
||||
owner: 'product/seed',
|
||||
remediation: 'Define the unavailable-resource UX contract, then seed a stable model-backed prompt that requests a missing Skill and Tool without mutating the saved Agent config.',
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
Given('Agent v2 Build chat unavailable Skill and Tool recovery is available', async function (this: DifyWorld) {
|
||||
return skipBuildDraftUnavailableResourceRecovery(this)
|
||||
})
|
||||
|
||||
Then('Agent v2 Build chat unavailable Skill and Tool recovery should be available', async function (this: DifyWorld) {
|
||||
return skipBuildDraftUnavailableResourceRecovery(this)
|
||||
})
|
||||
|
||||
Then('I should see the Agent v2 Build draft pending changes', async function (this: DifyWorld) {
|
||||
const page = this.getPage()
|
||||
|
||||
await expect(page.getByText('Build draft')).toBeVisible({ timeout: 30_000 })
|
||||
await expect(page.getByRole('button', { name: 'Apply' })).toBeEnabled()
|
||||
await expect(page.getByRole('button', { name: 'Discard' })).toBeEnabled()
|
||||
})
|
||||
|
||||
Then('I should see the Agent v2 Build mode confirmation state', async function (this: DifyWorld) {
|
||||
const page = this.getPage()
|
||||
|
||||
await expect(page.getByText('Build mode', { exact: true })).toBeVisible()
|
||||
await expect(
|
||||
page.getByText('You\'re in build mode. Shape this setup through the chat on the right, then Apply.'),
|
||||
).toBeVisible()
|
||||
})
|
||||
|
||||
Then('I should see the e2e-summary-skill Skill in the Skills section', async function (this: DifyWorld) {
|
||||
const skillsSection = this.getPage().getByRole('region', { name: 'Skills' })
|
||||
|
||||
await expect(skillsSection).toBeVisible({ timeout: 30_000 })
|
||||
await expect(skillsSection.getByRole('button', {
|
||||
exact: true,
|
||||
name: agentBuilderPreseededResources.summarySkill,
|
||||
})).toBeVisible()
|
||||
})
|
||||
|
||||
Then('I should not see the e2e-summary-skill Skill in the Skills section', async function (this: DifyWorld) {
|
||||
const skillsSection = this.getPage().getByRole('region', { name: 'Skills' })
|
||||
|
||||
await expect(skillsSection).toBeVisible({ timeout: 30_000 })
|
||||
await expect(skillsSection.getByRole('button', {
|
||||
exact: true,
|
||||
name: agentBuilderPreseededResources.summarySkill,
|
||||
})).toHaveCount(0)
|
||||
})
|
||||
|
||||
Then('I should see one e2e-summary-skill Skill in the Skills section', async function (this: DifyWorld) {
|
||||
const skillsSection = this.getPage().getByRole('region', { name: 'Skills' })
|
||||
|
||||
await expect(skillsSection).toBeVisible({ timeout: 30_000 })
|
||||
await expect(skillsSection.getByRole('button', {
|
||||
exact: true,
|
||||
name: agentBuilderPreseededResources.summarySkill,
|
||||
})).toHaveCount(1)
|
||||
})
|
||||
|
||||
Then(
|
||||
'the normal Agent v2 draft should not include the e2e-summary-skill Skill',
|
||||
async function (this: DifyWorld) {
|
||||
await expect.poll(
|
||||
async () => {
|
||||
const agentSoul = (await getAgentComposerDraft(getCurrentAgentId(this))).agent_soul
|
||||
|
||||
return agentSoul?.config_skills?.some(
|
||||
skill => skill.name === agentBuilderPreseededResources.summarySkill,
|
||||
) ?? false
|
||||
},
|
||||
{ timeout: 30_000 },
|
||||
).toBe(false)
|
||||
},
|
||||
)
|
||||
|
||||
Then(
|
||||
'the normal Agent v2 draft should not include the Agent Builder JSON Replace tool',
|
||||
async function (this: DifyWorld) {
|
||||
const agentId = getCurrentAgentId(this)
|
||||
const tool = getPreseededToolContract(this, agentBuilderPreseededResources.jsonReplaceTool)
|
||||
|
||||
await expect.poll(
|
||||
async () => {
|
||||
const draft = await getAgentComposerDraft(agentId)
|
||||
const tools = asArray(asRecord(draft.agent_soul?.tools).dify_tools)
|
||||
|
||||
return hasToolEntry(tools, tool)
|
||||
},
|
||||
{ timeout: 30_000 },
|
||||
).toBe(false)
|
||||
},
|
||||
)
|
||||
|
||||
Then(
|
||||
'the Agent v2 draft should include the supported Build draft config',
|
||||
async function (this: DifyWorld) {
|
||||
await expect.poll(
|
||||
async () => {
|
||||
const agentSoul = (await getAgentComposerDraft(getCurrentAgentId(this))).agent_soul
|
||||
const variables = agentSoul?.env?.variables ?? []
|
||||
|
||||
return {
|
||||
envValue: getAgentEnvVariableValue(variables, agentBuilderFixedInputs.envPlainKey),
|
||||
fileNames: agentSoul?.config_files?.map(file => file.name) ?? [],
|
||||
prompt: agentSoul?.prompt,
|
||||
skillNames: agentSoul?.config_skills?.map(skill => skill.name) ?? [],
|
||||
}
|
||||
},
|
||||
{ timeout: 30_000 },
|
||||
).toEqual({
|
||||
envValue: agentBuilderFixedInputs.envPlainValue,
|
||||
fileNames: expect.arrayContaining([agentBuilderTestMaterials.smallFile]),
|
||||
prompt: { system_prompt: updatedAgentPrompt },
|
||||
skillNames: expect.arrayContaining([agentBuilderPreseededResources.summarySkill]),
|
||||
})
|
||||
},
|
||||
)
|
||||
|
||||
Then(
|
||||
'the Agent v2 draft should not include the supported Build draft config',
|
||||
async function (this: DifyWorld) {
|
||||
await expect.poll(
|
||||
async () => {
|
||||
const agentSoul = (await getAgentComposerDraft(getCurrentAgentId(this))).agent_soul
|
||||
const variables = agentSoul?.env?.variables ?? []
|
||||
|
||||
return {
|
||||
envValue: getAgentEnvVariableValue(variables, agentBuilderFixedInputs.envPlainKey),
|
||||
fileNames: agentSoul?.config_files?.map(file => file.name) ?? [],
|
||||
prompt: agentSoul?.prompt,
|
||||
skillNames: agentSoul?.config_skills?.map(skill => skill.name) ?? [],
|
||||
}
|
||||
},
|
||||
{ timeout: 30_000 },
|
||||
).toEqual({
|
||||
envValue: undefined,
|
||||
fileNames: expect.not.arrayContaining([agentBuilderTestMaterials.smallFile]),
|
||||
prompt: { system_prompt: normalAgentPrompt },
|
||||
skillNames: expect.not.arrayContaining([agentBuilderPreseededResources.summarySkill]),
|
||||
})
|
||||
},
|
||||
)
|
||||
|
||||
Then(
|
||||
'the Agent v2 draft should include one e2e-summary-skill Skill',
|
||||
async function (this: DifyWorld) {
|
||||
await expect.poll(
|
||||
async () => {
|
||||
const agentSoul = (await getAgentComposerDraft(getCurrentAgentId(this))).agent_soul
|
||||
return agentSoul?.config_skills?.filter(
|
||||
skill => skill.name === agentBuilderPreseededResources.summarySkill,
|
||||
).length ?? 0
|
||||
},
|
||||
{ timeout: 30_000 },
|
||||
).toBe(1)
|
||||
},
|
||||
)
|
||||
|
||||
Then('the Agent v2 Build draft should no longer be active', async function (this: DifyWorld) {
|
||||
const page = this.getPage()
|
||||
|
||||
await expect(page.getByText('Build draft')).not.toBeVisible()
|
||||
await expect(page.getByRole('button', { name: 'Apply' })).not.toBeVisible()
|
||||
await expect(page.getByRole('button', { name: 'Discard' })).not.toBeVisible()
|
||||
})
|
||||
254
e2e/features/step-definitions/agent-v2/configure-helpers.ts
Normal file
254
e2e/features/step-definitions/agent-v2/configure-helpers.ts
Normal file
@ -0,0 +1,254 @@
|
||||
import type { Locator } from '@playwright/test'
|
||||
import type { AgentComposerEnvVariable } from '../../agent-v2/support/agent-soul'
|
||||
import type { DifyWorld } from '../../support/world'
|
||||
import { expect } from '@playwright/test'
|
||||
import { getAgentComposerDraft } from '../../agent-v2/support/agent'
|
||||
import { uploadAgentConfigSkillToDraft } from '../../agent-v2/support/agent-drive'
|
||||
import { normalAgentPrompt } from '../../agent-v2/support/agent-soul'
|
||||
import {
|
||||
agentBuilderTestMaterials,
|
||||
getAgentBuilderTestMaterialPath,
|
||||
} from '../../agent-v2/support/test-materials'
|
||||
|
||||
export const getCurrentAgentId = (world: DifyWorld) => {
|
||||
const agentId = world.createdAgentIds.at(-1)
|
||||
if (!agentId)
|
||||
throw new Error('No Agent v2 ID found. Create an Agent v2 test agent first.')
|
||||
|
||||
return agentId
|
||||
}
|
||||
|
||||
export const getPreseededAgent = (world: DifyWorld, name: string) => {
|
||||
const resource = world.agentBuilder.preflight.preseededResources[name]
|
||||
if (!resource || resource.kind !== 'agent') {
|
||||
throw new Error(
|
||||
`Preseeded Agent "${name}" is not available. Run the matching preflight step first.`,
|
||||
)
|
||||
}
|
||||
|
||||
return resource
|
||||
}
|
||||
|
||||
const getPreseededToolDisplayParts = (displayName: string) => {
|
||||
const [providerName, actionName] = displayName.split(' / ')
|
||||
if (!providerName || !actionName)
|
||||
throw new Error(`Preseeded tool display name must use "Provider / Action": ${displayName}`)
|
||||
|
||||
return { actionName, providerName }
|
||||
}
|
||||
|
||||
export const getEnvVariableKey = (variable: AgentComposerEnvVariable) =>
|
||||
variable.key ?? variable.name ?? variable.variable
|
||||
|
||||
export const getAgentEnvVariableValue = (
|
||||
variables: AgentComposerEnvVariable[],
|
||||
key: string,
|
||||
) => variables.find(variable => getEnvVariableKey(variable) === key)?.value
|
||||
|
||||
export const getAgentEnvVariables = async (agentId: string) =>
|
||||
(await getAgentComposerDraft(agentId)).agent_soul?.env?.variables ?? []
|
||||
|
||||
export const uploadAgentConfigFile = async (
|
||||
world: DifyWorld,
|
||||
material: keyof typeof agentBuilderTestMaterials,
|
||||
) => {
|
||||
const page = world.getPage()
|
||||
const agentId = getCurrentAgentId(world)
|
||||
const fileName = agentBuilderTestMaterials[material]
|
||||
const filePath = getAgentBuilderTestMaterialPath(material)
|
||||
|
||||
await page.getByRole('button', { name: 'Add file' }).click()
|
||||
const dialog = page.getByRole('dialog', { name: 'Upload file' })
|
||||
await expect(dialog).toBeVisible()
|
||||
|
||||
const fileChooserPromise = page.waitForEvent('filechooser')
|
||||
await dialog.getByRole('button', { name: 'browse' }).click()
|
||||
await (await fileChooserPromise).setFiles(filePath)
|
||||
await expect(dialog.getByText(fileName)).toBeVisible()
|
||||
|
||||
const commitResponsePromise = page.waitForResponse(response => (
|
||||
response.request().method() === 'POST'
|
||||
&& new URL(response.url()).pathname.endsWith(`/console/api/agent/${agentId}/config/files`)
|
||||
))
|
||||
await dialog.getByRole('button', { name: 'Upload' }).click()
|
||||
const commitResponse = await commitResponsePromise
|
||||
expect(commitResponse.status()).toBe(201)
|
||||
const committed = await commitResponse.json() as { file?: { name?: string } }
|
||||
await expect(dialog).not.toBeVisible({ timeout: 30_000 })
|
||||
|
||||
const committedName = committed.file?.name
|
||||
if (!committedName)
|
||||
throw new Error('Agent config file upload response did not include a file name.')
|
||||
|
||||
world.createdAgentConfigFiles.push({ agentId, name: committedName })
|
||||
}
|
||||
|
||||
export const expectAgentConfigFileVisible = async (
|
||||
world: DifyWorld,
|
||||
material: keyof typeof agentBuilderTestMaterials,
|
||||
) => {
|
||||
await expect(
|
||||
world.getPage().getByRole('button', {
|
||||
exact: true,
|
||||
name: agentBuilderTestMaterials[material],
|
||||
}),
|
||||
).toBeVisible({ timeout: 30_000 })
|
||||
}
|
||||
|
||||
export const expectAgentConfigFileHidden = async (
|
||||
world: DifyWorld,
|
||||
material: keyof typeof agentBuilderTestMaterials,
|
||||
) => {
|
||||
await expect(
|
||||
world.getPage().getByRole('button', {
|
||||
exact: true,
|
||||
name: agentBuilderTestMaterials[material],
|
||||
}),
|
||||
).not.toBeVisible()
|
||||
}
|
||||
|
||||
export const expectAgentConfigFileSaved = async (
|
||||
world: DifyWorld,
|
||||
material: keyof typeof agentBuilderTestMaterials,
|
||||
options?: {
|
||||
size?: number
|
||||
},
|
||||
) => {
|
||||
const agentId = getCurrentAgentId(world)
|
||||
const fileName = agentBuilderTestMaterials[material]
|
||||
|
||||
await expect
|
||||
.poll(async () => {
|
||||
const file = (await getAgentComposerDraft(agentId)).agent_soul?.config_files?.find(
|
||||
file => file.name === fileName,
|
||||
)
|
||||
|
||||
return file
|
||||
? {
|
||||
name: file.name,
|
||||
size: file.size,
|
||||
}
|
||||
: undefined
|
||||
}, {
|
||||
timeout: 30_000,
|
||||
})
|
||||
.toEqual({
|
||||
name: fileName,
|
||||
size: options?.size ?? expect.anything(),
|
||||
})
|
||||
}
|
||||
|
||||
export const uploadSummaryConfigSkillForBuildDraft = async (world: DifyWorld) => {
|
||||
const agentId = getCurrentAgentId(world)
|
||||
const skill = await uploadAgentConfigSkillToDraft({
|
||||
agentId,
|
||||
fileName: agentBuilderTestMaterials.summarySkill,
|
||||
filePath: getAgentBuilderTestMaterialPath('summarySkill'),
|
||||
})
|
||||
|
||||
if (!skill.file_id)
|
||||
throw new Error('Agent v2 build draft Skill fixture did not return a file_id.')
|
||||
|
||||
world.createdAgentConfigSkills.push({ agentId, name: skill.name })
|
||||
|
||||
return skill
|
||||
}
|
||||
|
||||
export const openAgentAdvancedSettings = async (page: ReturnType<DifyWorld['getPage']>) => {
|
||||
const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' })
|
||||
const envEditorHeading = advancedSettings.getByRole('heading', { name: 'Env Editor' })
|
||||
|
||||
if (!await envEditorHeading.isVisible().catch(() => false))
|
||||
await page.getByRole('button', { name: 'Advanced Settings' }).first().click()
|
||||
|
||||
await expect(envEditorHeading).toBeVisible()
|
||||
|
||||
return advancedSettings
|
||||
}
|
||||
|
||||
export const expectAgentEnvVariableVisible = async (
|
||||
world: DifyWorld,
|
||||
key: string,
|
||||
value: string,
|
||||
) => {
|
||||
const advancedSettings = await openAgentAdvancedSettings(world.getPage())
|
||||
|
||||
await expect.poll(
|
||||
async () => {
|
||||
const text = await advancedSettings.textContent()
|
||||
const inputValues = await advancedSettings.getByRole('textbox').evaluateAll(inputs =>
|
||||
inputs.map(input => (input as HTMLInputElement).value),
|
||||
)
|
||||
|
||||
return {
|
||||
hasKey: inputValues.includes(key) || !!text?.includes(key),
|
||||
hasValue: inputValues.includes(value) || !!text?.includes(value),
|
||||
}
|
||||
},
|
||||
{ timeout: 30_000 },
|
||||
).toEqual({
|
||||
hasKey: true,
|
||||
hasValue: true,
|
||||
})
|
||||
await expect(advancedSettings.getByText('Plain', { exact: true })).toBeVisible()
|
||||
}
|
||||
|
||||
export const expectAgentEnvVariableHidden = async (
|
||||
world: DifyWorld,
|
||||
key: string,
|
||||
) => {
|
||||
const advancedSettings = await openAgentAdvancedSettings(world.getPage())
|
||||
|
||||
await expect.poll(
|
||||
async () => {
|
||||
const text = await advancedSettings.textContent()
|
||||
const inputValues = await advancedSettings.getByRole('textbox').evaluateAll(inputs =>
|
||||
inputs.map(input => (input as HTMLInputElement).value),
|
||||
)
|
||||
|
||||
return inputValues.includes(key) || !!text?.includes(key)
|
||||
},
|
||||
{ timeout: 30_000 },
|
||||
).toBe(false)
|
||||
}
|
||||
|
||||
export const expectNormalAgentPromptDraft = async (world: DifyWorld) => {
|
||||
await expect.poll(
|
||||
async () => (await getAgentComposerDraft(getCurrentAgentId(world))).agent_soul?.prompt,
|
||||
{ timeout: 30_000 },
|
||||
).toEqual({ system_prompt: normalAgentPrompt })
|
||||
}
|
||||
|
||||
export const expectProviderToolActionVisible = async (
|
||||
toolsSection: Locator,
|
||||
displayName: string,
|
||||
) => {
|
||||
const tool = getPreseededToolDisplayParts(displayName)
|
||||
const provider = toolsSection.getByRole('button', {
|
||||
exact: true,
|
||||
name: tool.providerName,
|
||||
})
|
||||
await expect(provider).toBeVisible()
|
||||
|
||||
const action = toolsSection.getByText(tool.actionName, { exact: true })
|
||||
if (!await action.isVisible())
|
||||
await provider.click()
|
||||
await expect(action).toBeVisible()
|
||||
|
||||
return { action, tool }
|
||||
}
|
||||
|
||||
export const openAgentKnowledgeRetrievalDialog = async (knowledgeSection: Locator, name: string) => {
|
||||
await knowledgeSection.getByText(name, { exact: true }).hover()
|
||||
await knowledgeSection.getByRole('button', {
|
||||
exact: true,
|
||||
name: `Edit ${name}`,
|
||||
}).click()
|
||||
|
||||
const dialog = knowledgeSection.page().getByRole('dialog', {
|
||||
name: 'Knowledge Retrieval · Agent decide',
|
||||
})
|
||||
await expect(dialog).toBeVisible()
|
||||
|
||||
return dialog
|
||||
}
|
||||
@ -1,39 +1,247 @@
|
||||
import type { Page } from '@playwright/test'
|
||||
import type { DifyWorld } from '../../support/world'
|
||||
import { Given, Then, When } from '@cucumber/cucumber'
|
||||
import { expect } from '@playwright/test'
|
||||
import { waitForAgentConfigureAutosaved } from '../../../support/agent-configure'
|
||||
import {
|
||||
createConfiguredTestAgent,
|
||||
createTestAgent,
|
||||
getAgentComposerDraft,
|
||||
getAgentConfigurePath,
|
||||
saveAgentComposerDraft,
|
||||
} from '../../../support/agent'
|
||||
} from '../../agent-v2/support/agent'
|
||||
import { getAgentDriveSkills, uploadAgentDriveSkill } from '../../agent-v2/support/agent-drive'
|
||||
import {
|
||||
concurrentFirstAgentPrompt,
|
||||
concurrentSecondAgentPrompt,
|
||||
createAgentSoulConfigWithModel,
|
||||
normalAgentPrompt,
|
||||
normalAgentSoulConfig,
|
||||
updatedAgentPrompt,
|
||||
} from '../../agent-v2/support/agent-soul'
|
||||
import { agentBuilderTestMaterials, getAgentBuilderTestMaterialPath } from '../../agent-v2/support/test-materials'
|
||||
import {
|
||||
expectNormalAgentPromptDraft,
|
||||
getCurrentAgentId,
|
||||
getPreseededAgent,
|
||||
} from './configure-helpers'
|
||||
|
||||
const concurrentAgentPrompts = [
|
||||
concurrentFirstAgentPrompt,
|
||||
concurrentSecondAgentPrompt,
|
||||
]
|
||||
|
||||
function attachPageDiagnostics(world: DifyWorld, page: Page) {
|
||||
page.setDefaultTimeout(30_000)
|
||||
page.on('console', (message) => {
|
||||
if (message.type() === 'error')
|
||||
world.consoleErrors.push(message.text())
|
||||
})
|
||||
page.on('pageerror', (error) => {
|
||||
world.pageErrors.push(error.message)
|
||||
})
|
||||
page.on('download', (dl) => {
|
||||
world.capturedDownloads.push(dl)
|
||||
})
|
||||
}
|
||||
|
||||
const getPromptEditor = (page: Page) =>
|
||||
page.getByRole('region', { name: 'Prompt' }).getByRole('textbox', { name: 'Prompt' })
|
||||
|
||||
async function fillAgentPromptEditor(page: Page, prompt: string) {
|
||||
const promptSection = page.getByRole('region', { name: 'Prompt' })
|
||||
|
||||
await expect(promptSection).toBeVisible({ timeout: 30_000 })
|
||||
await getPromptEditor(page).fill(prompt)
|
||||
}
|
||||
|
||||
async function selectAgentModel(page: Page, modelName: string) {
|
||||
await page.getByRole('combobox', { name: 'Configure model' }).click()
|
||||
await page.getByLabel('Search model').fill(modelName)
|
||||
await page.getByRole('option', { name: modelName }).click()
|
||||
}
|
||||
|
||||
async function expectAgentComposerPrompt(agentId: string, prompt: string) {
|
||||
await expect.poll(
|
||||
async () => (await getAgentComposerDraft(agentId)).agent_soul?.prompt?.system_prompt,
|
||||
{ timeout: 30_000 },
|
||||
).toBe(prompt)
|
||||
}
|
||||
|
||||
Given('an Agent v2 test agent has been created via API', async function (this: DifyWorld) {
|
||||
const agent = await createTestAgent()
|
||||
this.createdAgentIds.push(agent.id)
|
||||
this.lastCreatedAgentName = agent.name
|
||||
this.lastCreatedAgentRole = agent.role
|
||||
this.lastCreatedAgentRole = agent.role ?? undefined
|
||||
})
|
||||
|
||||
Given(
|
||||
'a basic configured Agent v2 test agent has been created via API',
|
||||
async function (this: DifyWorld) {
|
||||
const agent = await createConfiguredTestAgent()
|
||||
this.createdAgentIds.push(agent.id)
|
||||
this.lastCreatedAgentName = agent.name
|
||||
this.lastCreatedAgentRole = agent.role ?? undefined
|
||||
},
|
||||
)
|
||||
|
||||
Given('a runnable Agent v2 test agent has been created via API', async function (this: DifyWorld) {
|
||||
if (!this.agentBuilder.preflight.stableModel)
|
||||
throw new Error('Create a runnable Agent v2 test agent after stable model preflight.')
|
||||
|
||||
const agent = await createConfiguredTestAgent({
|
||||
agentSoul: createAgentSoulConfigWithModel(
|
||||
normalAgentSoulConfig,
|
||||
this.agentBuilder.preflight.stableModel,
|
||||
),
|
||||
})
|
||||
this.createdAgentIds.push(agent.id)
|
||||
this.lastCreatedAgentName = agent.name
|
||||
this.lastCreatedAgentRole = agent.role ?? undefined
|
||||
})
|
||||
|
||||
Given('a minimal Agent v2 composer draft has been synced', async function (this: DifyWorld) {
|
||||
const agentId = this.createdAgentIds.at(-1)
|
||||
if (!agentId)
|
||||
throw new Error('No Agent v2 ID found. Create an Agent v2 test agent first.')
|
||||
const agentId = getCurrentAgentId(this)
|
||||
|
||||
await saveAgentComposerDraft(agentId)
|
||||
})
|
||||
|
||||
When('I open the Agent v2 configure page', async function (this: DifyWorld) {
|
||||
const agentId = this.createdAgentIds.at(-1)
|
||||
if (!agentId)
|
||||
throw new Error('No Agent v2 ID found. Create an Agent v2 test agent first.')
|
||||
Given('the Agent v2 composer draft uses the normal E2E prompt', async function (this: DifyWorld) {
|
||||
await saveAgentComposerDraft(getCurrentAgentId(this), normalAgentSoulConfig)
|
||||
})
|
||||
|
||||
await this.getPage().goto(getAgentConfigurePath(agentId))
|
||||
Given('the e2e-summary-skill Skill is available to the Agent v2 test agent', async function (this: DifyWorld) {
|
||||
const agentId = getCurrentAgentId(this)
|
||||
const upload = await uploadAgentDriveSkill({
|
||||
agentId,
|
||||
fileName: agentBuilderTestMaterials.summarySkill,
|
||||
filePath: getAgentBuilderTestMaterialPath('summarySkill'),
|
||||
})
|
||||
this.createdAgentDriveFiles.push({ agentId, key: upload.skill.skill_md_key })
|
||||
if (upload.skill.archive_key)
|
||||
this.createdAgentDriveFiles.push({ agentId, key: upload.skill.archive_key })
|
||||
})
|
||||
|
||||
Then('the Agent v2 test agent should include drive skill {string}', async function (this: DifyWorld, skillName: string) {
|
||||
const skills = await getAgentDriveSkills(getCurrentAgentId(this))
|
||||
|
||||
expect(skills.map(skill => skill.name)).toContain(skillName)
|
||||
})
|
||||
|
||||
When('I open the Agent v2 configure page', async function (this: DifyWorld) {
|
||||
await this.getPage().goto(getAgentConfigurePath(getCurrentAgentId(this)))
|
||||
})
|
||||
|
||||
When(
|
||||
'I select the stable E2E model in the Agent v2 model selector',
|
||||
async function (this: DifyWorld) {
|
||||
const stableModel = this.agentBuilder.preflight.stableModel
|
||||
if (!stableModel)
|
||||
throw new Error('Stable chat model preflight must run before selecting the Agent model.')
|
||||
|
||||
await selectAgentModel(this.getPage(), stableModel.name)
|
||||
},
|
||||
)
|
||||
|
||||
When('I switch to the Agent v2 Configure section', async function (this: DifyWorld) {
|
||||
const page = this.getPage()
|
||||
const agentId = getCurrentAgentId(this)
|
||||
|
||||
await page.getByRole('link', { name: 'Configure' }).click()
|
||||
await expect(page).toHaveURL(new RegExp(`/roster/agent/${agentId}/configure(?:\\?.*)?$`))
|
||||
await expect(page.getByRole('heading', { name: 'Configure' })).toBeVisible({ timeout: 30_000 })
|
||||
})
|
||||
|
||||
When('I leave the Agent v2 configure page immediately after editing', async function (this: DifyWorld) {
|
||||
const page = this.getPage()
|
||||
|
||||
await page.goto('/roster')
|
||||
await expect(page).toHaveURL(/\/roster(?:\?.*)?$/)
|
||||
})
|
||||
|
||||
When('I open the Agent v2 configure page from the Agent Roster', async function (this: DifyWorld) {
|
||||
const page = this.getPage()
|
||||
const agentId = getCurrentAgentId(this)
|
||||
const agentName = this.lastCreatedAgentName
|
||||
if (!agentName)
|
||||
throw new Error('No Agent v2 name found. Create an Agent v2 test agent first.')
|
||||
|
||||
await page.goto('/roster')
|
||||
await page.getByRole('link', { name: agentName }).click()
|
||||
await expect(page).toHaveURL(new RegExp(`/roster/agent/${agentId}/configure(?:\\?.*)?$`))
|
||||
await expect(page.getByRole('heading', { name: 'Configure' })).toBeVisible({ timeout: 30_000 })
|
||||
})
|
||||
|
||||
When(
|
||||
'I open the preseeded Agent v2 configure page for {string} from the Agent Roster',
|
||||
async function (this: DifyWorld, agentName: string) {
|
||||
const page = this.getPage()
|
||||
const agent = getPreseededAgent(this, agentName)
|
||||
|
||||
await page.goto('/roster')
|
||||
await page.getByRole('link', { name: agentName }).click()
|
||||
await expect(page).toHaveURL(new RegExp(`/roster/agent/${agent.id}/configure(?:\\?.*)?$`))
|
||||
await expect(page.getByRole('heading', { name: 'Configure' })).toBeVisible({ timeout: 30_000 })
|
||||
},
|
||||
)
|
||||
|
||||
When('I fill the Agent v2 prompt editor with the normal E2E prompt', async function (this: DifyWorld) {
|
||||
await fillAgentPromptEditor(this.getPage(), normalAgentPrompt)
|
||||
})
|
||||
|
||||
When('I fill the Agent v2 prompt editor with the updated E2E prompt', async function (this: DifyWorld) {
|
||||
await fillAgentPromptEditor(this.getPage(), updatedAgentPrompt)
|
||||
})
|
||||
|
||||
When('I open the same Agent v2 configure page in another tab', async function (this: DifyWorld) {
|
||||
if (!this.context)
|
||||
throw new Error('Playwright context has not been initialized for this scenario.')
|
||||
|
||||
const agentId = getCurrentAgentId(this)
|
||||
const concurrentPage = await this.context.newPage()
|
||||
attachPageDiagnostics(this, concurrentPage)
|
||||
this.agentBuilder.configure.concurrentPage = concurrentPage
|
||||
|
||||
await concurrentPage.goto(getAgentConfigurePath(agentId))
|
||||
await expect(concurrentPage).toHaveURL(new RegExp(`/roster/agent/${agentId}/configure(?:\\?.*)?$`))
|
||||
await expect(concurrentPage.getByRole('heading', { name: 'Configure' })).toBeVisible({ timeout: 30_000 })
|
||||
})
|
||||
|
||||
When('I save the Agent v2 prompt from the first configure tab', async function (this: DifyWorld) {
|
||||
const agentId = getCurrentAgentId(this)
|
||||
|
||||
await fillAgentPromptEditor(this.getPage(), concurrentFirstAgentPrompt)
|
||||
await waitForAgentConfigureAutosaved(this.getPage())
|
||||
await expectAgentComposerPrompt(agentId, concurrentFirstAgentPrompt)
|
||||
})
|
||||
|
||||
When('I save the Agent v2 prompt from the second configure tab', async function (this: DifyWorld) {
|
||||
const agentId = getCurrentAgentId(this)
|
||||
const concurrentPage = this.agentBuilder.configure.concurrentPage
|
||||
if (!concurrentPage)
|
||||
throw new Error('Open the same Agent v2 configure page in another tab before editing it.')
|
||||
|
||||
await fillAgentPromptEditor(concurrentPage, concurrentSecondAgentPrompt)
|
||||
await waitForAgentConfigureAutosaved(concurrentPage)
|
||||
await expectAgentComposerPrompt(agentId, concurrentSecondAgentPrompt)
|
||||
})
|
||||
|
||||
When('I refresh both Agent v2 configure tabs', async function (this: DifyWorld) {
|
||||
const page = this.getPage()
|
||||
const concurrentPage = this.agentBuilder.configure.concurrentPage
|
||||
if (!concurrentPage)
|
||||
throw new Error('Open the same Agent v2 configure page in another tab before refreshing it.')
|
||||
|
||||
await Promise.all([
|
||||
page.reload(),
|
||||
concurrentPage.reload(),
|
||||
])
|
||||
await expect(page.getByRole('heading', { name: 'Configure' })).toBeVisible({ timeout: 30_000 })
|
||||
await expect(concurrentPage.getByRole('heading', { name: 'Configure' })).toBeVisible({ timeout: 30_000 })
|
||||
})
|
||||
|
||||
Then('I should be on the Agent v2 configure page', async function (this: DifyWorld) {
|
||||
const agentId = this.createdAgentIds.at(-1)
|
||||
if (!agentId)
|
||||
throw new Error('No Agent v2 ID found. Create an Agent v2 test agent first.')
|
||||
const agentId = getCurrentAgentId(this)
|
||||
|
||||
await expect(this.getPage()).toHaveURL(
|
||||
new RegExp(`/roster/agent/${agentId}/configure(?:\\?.*)?$`),
|
||||
@ -47,3 +255,119 @@ Then('I should see the Agent v2 configure workspace', async function (this: Dify
|
||||
await expect(page.getByRole('heading', { name: 'Configure' })).toBeVisible()
|
||||
await expect(page.getByText(this.lastCreatedAgentName!)).toBeVisible()
|
||||
})
|
||||
|
||||
Then(
|
||||
'I should see the normal E2E prompt in the Agent v2 prompt editor',
|
||||
async function (this: DifyWorld) {
|
||||
const page = this.getPage()
|
||||
|
||||
await expect(getPromptEditor(page)).toContainText(normalAgentPrompt, { timeout: 30_000 })
|
||||
},
|
||||
)
|
||||
|
||||
Then(
|
||||
'I should see the stable E2E model in the Agent v2 model selector',
|
||||
async function (this: DifyWorld) {
|
||||
const stableModel = this.agentBuilder.preflight.stableModel
|
||||
if (!stableModel)
|
||||
throw new Error('Stable chat model preflight must run before asserting the Agent model.')
|
||||
|
||||
await expect(this.getPage().getByText(stableModel.name, { exact: true }))
|
||||
.toBeVisible({ timeout: 30_000 })
|
||||
},
|
||||
)
|
||||
|
||||
Then(
|
||||
'I should see the updated E2E prompt in the Agent v2 prompt editor',
|
||||
async function (this: DifyWorld) {
|
||||
const page = this.getPage()
|
||||
|
||||
await expect(getPromptEditor(page)).toContainText(updatedAgentPrompt, { timeout: 30_000 })
|
||||
},
|
||||
)
|
||||
|
||||
Then(
|
||||
'both Agent v2 configure tabs and the Agent v2 draft should show one saved concurrent prompt',
|
||||
async function (this: DifyWorld) {
|
||||
const agentId = getCurrentAgentId(this)
|
||||
const concurrentPage = this.agentBuilder.configure.concurrentPage
|
||||
if (!concurrentPage)
|
||||
throw new Error('Open the same Agent v2 configure page in another tab before asserting convergence.')
|
||||
|
||||
let savedPrompt = ''
|
||||
await expect.poll(
|
||||
async () => {
|
||||
const prompt = (await getAgentComposerDraft(agentId)).agent_soul?.prompt?.system_prompt
|
||||
if (prompt && concurrentAgentPrompts.includes(prompt))
|
||||
savedPrompt = prompt
|
||||
|
||||
return !!savedPrompt
|
||||
},
|
||||
{ timeout: 30_000 },
|
||||
).toBe(true)
|
||||
|
||||
await expect(getPromptEditor(this.getPage())).toContainText(savedPrompt)
|
||||
await expect(getPromptEditor(concurrentPage)).toContainText(savedPrompt)
|
||||
},
|
||||
)
|
||||
|
||||
Then(
|
||||
'Agent v2 Preview should be unavailable until a model is configured',
|
||||
async function (this: DifyWorld) {
|
||||
const page = this.getPage()
|
||||
|
||||
await expect(page.getByRole('heading', { name: 'Configure' })).toBeVisible({ timeout: 30_000 })
|
||||
await expect(page.getByRole('button', { name: /^Preview$/i })).toBeDisabled()
|
||||
},
|
||||
)
|
||||
|
||||
Then(
|
||||
'the normal Agent v2 draft should still use the normal E2E prompt',
|
||||
async function (this: DifyWorld) {
|
||||
await expectNormalAgentPromptDraft(this)
|
||||
},
|
||||
)
|
||||
|
||||
Then(
|
||||
'the normal Agent v2 draft should use the normal E2E prompt',
|
||||
async function (this: DifyWorld) {
|
||||
await expectNormalAgentPromptDraft(this)
|
||||
},
|
||||
)
|
||||
|
||||
Then(
|
||||
'the normal Agent v2 draft should use the updated E2E prompt',
|
||||
async function (this: DifyWorld) {
|
||||
await expect.poll(
|
||||
async () => (await getAgentComposerDraft(getCurrentAgentId(this))).agent_soul?.prompt,
|
||||
{ timeout: 30_000 },
|
||||
).toEqual({ system_prompt: updatedAgentPrompt })
|
||||
},
|
||||
)
|
||||
|
||||
Then(
|
||||
'the Agent v2 draft should use the stable E2E model',
|
||||
async function (this: DifyWorld) {
|
||||
const stableModel = this.agentBuilder.preflight.stableModel
|
||||
if (!stableModel)
|
||||
throw new Error('Stable chat model preflight must run before asserting the Agent model.')
|
||||
|
||||
await expect.poll(
|
||||
async () => {
|
||||
const model = (await getAgentComposerDraft(getCurrentAgentId(this))).agent_soul?.model
|
||||
const modelConfig = typeof model === 'object' && model !== null && !Array.isArray(model)
|
||||
? model as Record<string, unknown>
|
||||
: undefined
|
||||
|
||||
return {
|
||||
model: modelConfig?.model,
|
||||
provider: modelConfig?.model_provider,
|
||||
}
|
||||
},
|
||||
{ timeout: 30_000 },
|
||||
).toEqual({
|
||||
model: stableModel.name,
|
||||
provider: stableModel.provider,
|
||||
})
|
||||
},
|
||||
)
|
||||
|
||||
@ -0,0 +1,139 @@
|
||||
import type { Locator } from '@playwright/test'
|
||||
import type { DifyWorld } from '../../support/world'
|
||||
import { Then, When } from '@cucumber/cucumber'
|
||||
import { expect } from '@playwright/test'
|
||||
import { getAgentComposerDraft } from '../../agent-v2/support/agent'
|
||||
import { agentBuilderFixedInputs } from '../../agent-v2/support/agent-builder-resources'
|
||||
import { skipBlockedPrecondition } from '../../agent-v2/support/preflight/common'
|
||||
import { getCurrentAgentId } from './configure-helpers'
|
||||
|
||||
const getModerationSettingsDialog = (world: DifyWorld) =>
|
||||
world.getPage().getByRole('dialog').filter({ hasText: 'Content moderation settings' })
|
||||
|
||||
const getContentModerationRegion = (world: DifyWorld) =>
|
||||
world.getPage().getByRole('region', { name: 'Content moderation' })
|
||||
|
||||
const ensureSwitchChecked = async (switchLocator: Locator) => {
|
||||
if (await switchLocator.getAttribute('aria-checked') !== 'true')
|
||||
await switchLocator.click()
|
||||
}
|
||||
|
||||
When(
|
||||
'I configure Agent v2 Content Moderation keyword preset replies',
|
||||
async function (this: DifyWorld) {
|
||||
const page = this.getPage()
|
||||
const contentModeration = getContentModerationRegion(this)
|
||||
const enabledSwitch = contentModeration.getByRole('switch', { name: 'Content moderation' })
|
||||
|
||||
if (await enabledSwitch.getAttribute('aria-checked') === 'true')
|
||||
await contentModeration.getByRole('button', { name: 'Settings' }).click()
|
||||
else
|
||||
await enabledSwitch.click()
|
||||
|
||||
const dialog = getModerationSettingsDialog(this)
|
||||
await expect(dialog).toBeVisible()
|
||||
await dialog.getByRole('button', { name: 'Keywords' }).click()
|
||||
await dialog
|
||||
.getByRole('textbox', { name: 'Keywords' })
|
||||
.fill(agentBuilderFixedInputs.moderationKeyword)
|
||||
|
||||
const inputModeration = dialog.getByRole('region', { name: 'Moderate INPUT Content' })
|
||||
const outputModeration = dialog.getByRole('region', { name: 'Moderate OUTPUT Content' })
|
||||
await ensureSwitchChecked(inputModeration.getByRole('switch', { name: 'Moderate INPUT Content' }))
|
||||
|
||||
await dialog.getByRole('button', { name: 'Save' }).click()
|
||||
await expect(page.getByText('Preset replies cannot be empty')).toBeVisible()
|
||||
await expect(dialog).toBeVisible()
|
||||
|
||||
await inputModeration
|
||||
.getByRole('textbox', { name: 'Preset replies' })
|
||||
.fill(agentBuilderFixedInputs.inputModerationReply)
|
||||
await ensureSwitchChecked(outputModeration.getByRole('switch', { name: 'Moderate OUTPUT Content' }))
|
||||
await outputModeration
|
||||
.getByRole('textbox', { name: 'Preset replies' })
|
||||
.fill(agentBuilderFixedInputs.outputModerationReply)
|
||||
await dialog.getByRole('button', { name: 'Save' }).click()
|
||||
await expect(dialog).not.toBeVisible()
|
||||
},
|
||||
)
|
||||
|
||||
Then('Agent v2 Content Moderation Settings should be available', async function (this: DifyWorld) {
|
||||
const advancedSettings = this.getPage().getByRole('region', { name: 'Advanced Settings' })
|
||||
const contentModeration = advancedSettings.getByRole('region', { name: 'Content moderation' })
|
||||
|
||||
try {
|
||||
await expect(contentModeration).toBeVisible({ timeout: 3_000 })
|
||||
}
|
||||
catch {
|
||||
return skipBlockedPrecondition(
|
||||
this,
|
||||
'Agent v2 Content Moderation Settings is not available in this build.',
|
||||
{
|
||||
owner: 'product',
|
||||
remediation: 'Enable the Agent v2 Content Moderation feature flag in the product or keep this scenario feature-gated.',
|
||||
},
|
||||
)
|
||||
}
|
||||
})
|
||||
|
||||
Then(
|
||||
'Agent v2 Content Moderation keyword preset replies should be saved in the Agent v2 draft',
|
||||
async function (this: DifyWorld) {
|
||||
const agentId = getCurrentAgentId(this)
|
||||
|
||||
await expect
|
||||
.poll(async () => {
|
||||
const draft = await getAgentComposerDraft(agentId)
|
||||
const appFeatures = draft.agent_soul?.app_features as Record<string, unknown> | undefined
|
||||
const moderation = appFeatures?.sensitive_word_avoidance as Record<string, unknown> | undefined
|
||||
const config = moderation?.config as Record<string, unknown> | undefined
|
||||
const inputsConfig = config?.inputs_config as Record<string, unknown> | undefined
|
||||
const outputsConfig = config?.outputs_config as Record<string, unknown> | undefined
|
||||
|
||||
return {
|
||||
enabled: moderation?.enabled,
|
||||
inputEnabled: inputsConfig?.enabled,
|
||||
inputPreset: inputsConfig?.preset_response,
|
||||
keywords: config?.keywords,
|
||||
outputEnabled: outputsConfig?.enabled,
|
||||
outputPreset: outputsConfig?.preset_response,
|
||||
type: moderation?.type,
|
||||
}
|
||||
}, {
|
||||
timeout: 30_000,
|
||||
})
|
||||
.toEqual({
|
||||
enabled: true,
|
||||
inputEnabled: true,
|
||||
inputPreset: agentBuilderFixedInputs.inputModerationReply,
|
||||
keywords: agentBuilderFixedInputs.moderationKeyword,
|
||||
outputEnabled: true,
|
||||
outputPreset: agentBuilderFixedInputs.outputModerationReply,
|
||||
type: 'keywords',
|
||||
})
|
||||
},
|
||||
)
|
||||
|
||||
Then(
|
||||
'I should see the Agent v2 Content Moderation keyword preset replies in Advanced Settings',
|
||||
async function (this: DifyWorld) {
|
||||
const contentModeration = getContentModerationRegion(this)
|
||||
|
||||
await expect(contentModeration).toContainText('Keywords')
|
||||
await expect(contentModeration).toContainText('INPUT & OUTPUT')
|
||||
await contentModeration.getByRole('button', { name: 'Settings' }).click()
|
||||
|
||||
const dialog = getModerationSettingsDialog(this)
|
||||
await expect(dialog).toBeVisible()
|
||||
await expect(dialog.getByRole('textbox', { name: 'Keywords' }))
|
||||
.toHaveValue(agentBuilderFixedInputs.moderationKeyword)
|
||||
await expect(dialog.getByRole('region', { name: 'Moderate INPUT Content' })
|
||||
.getByRole('textbox', { name: 'Preset replies' }))
|
||||
.toHaveValue(agentBuilderFixedInputs.inputModerationReply)
|
||||
await expect(dialog.getByRole('region', { name: 'Moderate OUTPUT Content' })
|
||||
.getByRole('textbox', { name: 'Preset replies' }))
|
||||
.toHaveValue(agentBuilderFixedInputs.outputModerationReply)
|
||||
await dialog.getByRole('button', { name: 'Cancel' }).click()
|
||||
await expect(dialog).not.toBeVisible()
|
||||
},
|
||||
)
|
||||
310
e2e/features/step-definitions/agent-v2/env-editor.steps.ts
Normal file
310
e2e/features/step-definitions/agent-v2/env-editor.steps.ts
Normal file
@ -0,0 +1,310 @@
|
||||
import type { DifyWorld } from '../../support/world'
|
||||
import { Then, When } from '@cucumber/cucumber'
|
||||
import { expect } from '@playwright/test'
|
||||
import { getAgentComposerDraft } from '../../agent-v2/support/agent'
|
||||
import { agentBuilderFixedInputs } from '../../agent-v2/support/agent-builder-resources'
|
||||
import { getAgentBuilderTestMaterialPath } from '../../agent-v2/support/test-materials'
|
||||
import {
|
||||
expectAgentEnvVariableHidden,
|
||||
expectAgentEnvVariableVisible,
|
||||
getAgentEnvVariables,
|
||||
getAgentEnvVariableValue,
|
||||
getCurrentAgentId,
|
||||
getEnvVariableKey,
|
||||
openAgentAdvancedSettings,
|
||||
} from './configure-helpers'
|
||||
|
||||
When(
|
||||
'I add the plain Agent v2 environment variable from Advanced Settings',
|
||||
async function (this: DifyWorld) {
|
||||
const advancedSettings = await openAgentAdvancedSettings(this.getPage())
|
||||
|
||||
await advancedSettings
|
||||
.getByRole('textbox', { name: 'Key' })
|
||||
.fill(agentBuilderFixedInputs.envPlainKey)
|
||||
await advancedSettings
|
||||
.getByRole('textbox', { name: 'Value' })
|
||||
.fill(agentBuilderFixedInputs.envPlainValue)
|
||||
await expect(advancedSettings.getByText('Plain', { exact: true })).toBeVisible()
|
||||
},
|
||||
)
|
||||
|
||||
When(
|
||||
'I import the invalid Agent v2 environment file from Advanced Settings',
|
||||
async function (this: DifyWorld) {
|
||||
const page = this.getPage()
|
||||
const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' })
|
||||
|
||||
const fileChooserPromise = page.waitForEvent('filechooser')
|
||||
await advancedSettings.getByRole('button', { name: 'Import .env' }).click()
|
||||
await (await fileChooserPromise).setFiles(getAgentBuilderTestMaterialPath('invalidEnv'))
|
||||
},
|
||||
)
|
||||
|
||||
When(
|
||||
'I add the secondary plain Agent v2 environment variable from Advanced Settings',
|
||||
async function (this: DifyWorld) {
|
||||
const page = this.getPage()
|
||||
const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' })
|
||||
|
||||
await advancedSettings.getByRole('button', { name: 'Add environment variable' }).click()
|
||||
await advancedSettings
|
||||
.getByRole('textbox', { name: 'Key' })
|
||||
.last()
|
||||
.fill(agentBuilderFixedInputs.envModeKey)
|
||||
await advancedSettings
|
||||
.getByRole('textbox', { name: 'Value' })
|
||||
.last()
|
||||
.fill(agentBuilderFixedInputs.envModeValue)
|
||||
await expect(advancedSettings.getByText('Plain', { exact: true })).toHaveCount(2)
|
||||
},
|
||||
)
|
||||
|
||||
When(
|
||||
'I delete the plain Agent v2 environment variable from Advanced Settings',
|
||||
async function (this: DifyWorld) {
|
||||
const page = this.getPage()
|
||||
const advancedSettings = page.getByRole('region', { name: 'Advanced Settings' })
|
||||
|
||||
await advancedSettings
|
||||
.getByRole('button', { name: `Delete ${agentBuilderFixedInputs.envPlainKey}` })
|
||||
.click()
|
||||
},
|
||||
)
|
||||
|
||||
When(
|
||||
'I import the valid Agent v2 environment file from Advanced Settings',
|
||||
async function (this: DifyWorld) {
|
||||
const advancedSettings = await openAgentAdvancedSettings(this.getPage())
|
||||
|
||||
const fileChooserPromise = this.getPage().waitForEvent('filechooser')
|
||||
await advancedSettings.getByRole('button', { name: 'Import .env' }).click()
|
||||
await (await fileChooserPromise).setFiles(getAgentBuilderTestMaterialPath('validEnv'))
|
||||
},
|
||||
)
|
||||
|
||||
Then(
|
||||
'the plain Agent v2 environment variable should be saved in the Agent v2 draft',
|
||||
async function (this: DifyWorld) {
|
||||
const agentId = getCurrentAgentId(this)
|
||||
|
||||
await expect
|
||||
.poll(async () => {
|
||||
const env = (await getAgentComposerDraft(agentId)).agent_soul?.env
|
||||
const variable = env?.variables?.find(item =>
|
||||
getEnvVariableKey(item) === agentBuilderFixedInputs.envPlainKey,
|
||||
)
|
||||
|
||||
return {
|
||||
secretCount: env?.secret_refs?.length ?? 0,
|
||||
value: variable?.value,
|
||||
}
|
||||
}, {
|
||||
timeout: 30_000,
|
||||
})
|
||||
.toEqual({
|
||||
secretCount: 0,
|
||||
value: agentBuilderFixedInputs.envPlainValue,
|
||||
})
|
||||
},
|
||||
)
|
||||
|
||||
Then(
|
||||
'the Agent v2 environment variables for deletion should be saved in the Agent v2 draft',
|
||||
async function (this: DifyWorld) {
|
||||
const agentId = getCurrentAgentId(this)
|
||||
|
||||
await expect
|
||||
.poll(async () => {
|
||||
const variables = await getAgentEnvVariables(agentId)
|
||||
|
||||
return {
|
||||
modeValue: getAgentEnvVariableValue(variables, agentBuilderFixedInputs.envModeKey),
|
||||
plainValue: getAgentEnvVariableValue(variables, agentBuilderFixedInputs.envPlainKey),
|
||||
}
|
||||
}, {
|
||||
timeout: 30_000,
|
||||
})
|
||||
.toEqual({
|
||||
modeValue: agentBuilderFixedInputs.envModeValue,
|
||||
plainValue: agentBuilderFixedInputs.envPlainValue,
|
||||
})
|
||||
},
|
||||
)
|
||||
|
||||
Then(
|
||||
'the plain Agent v2 environment variable should be removed from the Agent v2 draft',
|
||||
async function (this: DifyWorld) {
|
||||
const agentId = getCurrentAgentId(this)
|
||||
|
||||
await expect
|
||||
.poll(async () => {
|
||||
const variables = await getAgentEnvVariables(agentId)
|
||||
|
||||
return {
|
||||
modeValue: getAgentEnvVariableValue(variables, agentBuilderFixedInputs.envModeKey),
|
||||
plainValue: getAgentEnvVariableValue(variables, agentBuilderFixedInputs.envPlainKey),
|
||||
}
|
||||
}, {
|
||||
timeout: 30_000,
|
||||
})
|
||||
.toEqual({
|
||||
modeValue: agentBuilderFixedInputs.envModeValue,
|
||||
plainValue: undefined,
|
||||
})
|
||||
},
|
||||
)
|
||||
|
||||
Then(
|
||||
'the valid Agent v2 environment import should be saved in the Agent v2 draft',
|
||||
async function (this: DifyWorld) {
|
||||
const agentId = getCurrentAgentId(this)
|
||||
|
||||
await expect
|
||||
.poll(async () => {
|
||||
const variables = await getAgentEnvVariables(agentId)
|
||||
|
||||
return {
|
||||
modeValue: getAgentEnvVariableValue(variables, agentBuilderFixedInputs.envModeKey),
|
||||
plainValue: getAgentEnvVariableValue(variables, agentBuilderFixedInputs.envPlainKey),
|
||||
}
|
||||
}, {
|
||||
timeout: 30_000,
|
||||
})
|
||||
.toEqual({
|
||||
modeValue: agentBuilderFixedInputs.envModeValue,
|
||||
plainValue: agentBuilderFixedInputs.envPlainValue,
|
||||
})
|
||||
},
|
||||
)
|
||||
|
||||
Then(
|
||||
'the invalid Agent v2 environment import should report skipped lines',
|
||||
async function (this: DifyWorld) {
|
||||
await expect(this.getPage().getByText('2 invalid .env lines were skipped.')).toBeVisible()
|
||||
},
|
||||
)
|
||||
|
||||
Then(
|
||||
'the Agent v2 environment variables from the invalid import should be saved in the Agent v2 draft',
|
||||
async function (this: DifyWorld) {
|
||||
const agentId = getCurrentAgentId(this)
|
||||
|
||||
await expect
|
||||
.poll(async () => {
|
||||
const variables = await getAgentEnvVariables(agentId)
|
||||
|
||||
return {
|
||||
importedValue: getAgentEnvVariableValue(
|
||||
variables,
|
||||
agentBuilderFixedInputs.envAfterInvalidImportKey,
|
||||
),
|
||||
plainValue: getAgentEnvVariableValue(variables, agentBuilderFixedInputs.envPlainKey),
|
||||
}
|
||||
}, {
|
||||
timeout: 30_000,
|
||||
})
|
||||
.toEqual({
|
||||
importedValue: agentBuilderFixedInputs.envAfterInvalidImportValue,
|
||||
plainValue: agentBuilderFixedInputs.envPlainValue,
|
||||
})
|
||||
},
|
||||
)
|
||||
|
||||
Then(
|
||||
'I should see the Agent v2 environment variables from the valid import in Advanced Settings',
|
||||
async function (this: DifyWorld) {
|
||||
const advancedSettings = await openAgentAdvancedSettings(this.getPage())
|
||||
|
||||
await expect.poll(
|
||||
async () => advancedSettings.getByRole('textbox').evaluateAll(inputs =>
|
||||
inputs.map(input => (input as HTMLInputElement).value),
|
||||
),
|
||||
{ timeout: 30_000 },
|
||||
).toEqual(expect.arrayContaining([
|
||||
agentBuilderFixedInputs.envPlainKey,
|
||||
agentBuilderFixedInputs.envPlainValue,
|
||||
agentBuilderFixedInputs.envModeKey,
|
||||
agentBuilderFixedInputs.envModeValue,
|
||||
]))
|
||||
await expect(advancedSettings.getByText('Plain', { exact: true })).toHaveCount(2)
|
||||
},
|
||||
)
|
||||
|
||||
Then(
|
||||
'I should not see the deleted Agent v2 environment variable in Advanced Settings',
|
||||
async function (this: DifyWorld) {
|
||||
const advancedSettings = await openAgentAdvancedSettings(this.getPage())
|
||||
|
||||
await expect.poll(
|
||||
async () => advancedSettings.getByRole('textbox').evaluateAll(inputs =>
|
||||
inputs.map(input => (input as HTMLInputElement).value),
|
||||
),
|
||||
{ timeout: 30_000 },
|
||||
).toEqual(expect.arrayContaining([
|
||||
agentBuilderFixedInputs.envModeKey,
|
||||
agentBuilderFixedInputs.envModeValue,
|
||||
]))
|
||||
await expect.poll(
|
||||
async () => advancedSettings.getByRole('textbox').evaluateAll(inputs =>
|
||||
inputs.map(input => (input as HTMLInputElement).value),
|
||||
),
|
||||
{ timeout: 30_000 },
|
||||
).not.toContain(agentBuilderFixedInputs.envPlainKey)
|
||||
await expect(advancedSettings.getByText('Plain', { exact: true })).toHaveCount(1)
|
||||
},
|
||||
)
|
||||
|
||||
Then(
|
||||
'I should see the plain Agent v2 environment variable in Advanced Settings',
|
||||
async function (this: DifyWorld) {
|
||||
const page = this.getPage()
|
||||
const advancedSettings = await openAgentAdvancedSettings(page)
|
||||
|
||||
await expect(advancedSettings.getByRole('textbox', { name: 'Key' }))
|
||||
.toHaveValue(agentBuilderFixedInputs.envPlainKey)
|
||||
await expect(advancedSettings.getByRole('textbox', { name: 'Value' }))
|
||||
.toHaveValue(agentBuilderFixedInputs.envPlainValue)
|
||||
await expect(advancedSettings.getByText('Plain', { exact: true })).toBeVisible()
|
||||
await expect(page.getByRole('button', { name: /^Build$/i })).toBeVisible()
|
||||
},
|
||||
)
|
||||
|
||||
Then(
|
||||
'I should see the supported E2E environment variable in Advanced Settings',
|
||||
async function (this: DifyWorld) {
|
||||
await expectAgentEnvVariableVisible(
|
||||
this,
|
||||
agentBuilderFixedInputs.envPlainKey,
|
||||
agentBuilderFixedInputs.envPlainValue,
|
||||
)
|
||||
},
|
||||
)
|
||||
|
||||
Then(
|
||||
'I should not see the supported E2E environment variable in Advanced Settings',
|
||||
async function (this: DifyWorld) {
|
||||
await expectAgentEnvVariableHidden(this, agentBuilderFixedInputs.envPlainKey)
|
||||
},
|
||||
)
|
||||
|
||||
Then(
|
||||
'I should see the Agent v2 environment variables from the invalid import in Advanced Settings',
|
||||
async function (this: DifyWorld) {
|
||||
const page = this.getPage()
|
||||
const advancedSettings = await openAgentAdvancedSettings(page)
|
||||
|
||||
await expect.poll(
|
||||
async () => advancedSettings.getByRole('textbox').evaluateAll(inputs =>
|
||||
inputs.map(input => (input as HTMLInputElement).value),
|
||||
),
|
||||
{ timeout: 30_000 },
|
||||
).toEqual(expect.arrayContaining([
|
||||
agentBuilderFixedInputs.envPlainKey,
|
||||
agentBuilderFixedInputs.envPlainValue,
|
||||
agentBuilderFixedInputs.envAfterInvalidImportKey,
|
||||
agentBuilderFixedInputs.envAfterInvalidImportValue,
|
||||
]))
|
||||
await expect(page.getByRole('button', { name: /^Build$/i })).toBeVisible()
|
||||
},
|
||||
)
|
||||
178
e2e/features/step-definitions/agent-v2/files.steps.ts
Normal file
178
e2e/features/step-definitions/agent-v2/files.steps.ts
Normal file
@ -0,0 +1,178 @@
|
||||
import type { DifyWorld } from '../../support/world'
|
||||
import { Given, Then, When } from '@cucumber/cucumber'
|
||||
import { expect } from '@playwright/test'
|
||||
import { skipBlockedPrecondition } from '../../agent-v2/support/preflight/common'
|
||||
import { agentBuilderFileTreeFixtureFileNames } from '../../agent-v2/support/test-materials'
|
||||
import {
|
||||
expectAgentConfigFileHidden,
|
||||
expectAgentConfigFileSaved,
|
||||
expectAgentConfigFileVisible,
|
||||
uploadAgentConfigFile,
|
||||
} from './configure-helpers'
|
||||
|
||||
When('I upload the small Agent v2 file from the Files section', async function (this: DifyWorld) {
|
||||
await uploadAgentConfigFile(this, 'smallFile')
|
||||
})
|
||||
|
||||
When('I upload the empty Agent v2 file from the Files section', async function (this: DifyWorld) {
|
||||
await uploadAgentConfigFile(this, 'emptyFile')
|
||||
})
|
||||
|
||||
When('I upload the special-name Agent v2 file from the Files section', async function (this: DifyWorld) {
|
||||
await uploadAgentConfigFile(this, 'specialFilename')
|
||||
})
|
||||
|
||||
Then(
|
||||
'I should see the Agent v2 file fixture entries in the current flat Files list',
|
||||
async function (this: DifyWorld) {
|
||||
const page = this.getPage()
|
||||
const filesSection = page.getByRole('region', { name: 'Files' })
|
||||
const filesList = filesSection.getByLabel('Agent files')
|
||||
|
||||
await expect(filesSection).toBeVisible({ timeout: 30_000 })
|
||||
await expect(filesList).toBeVisible()
|
||||
|
||||
for (const fileName of agentBuilderFileTreeFixtureFileNames) {
|
||||
await expect(filesList.getByRole('button', {
|
||||
exact: true,
|
||||
name: fileName,
|
||||
})).toBeVisible()
|
||||
}
|
||||
|
||||
await expect(filesList.getByRole('button', { exact: true, name: 'assets' })).toHaveCount(0)
|
||||
await expect(filesList.getByRole('button', { exact: true, name: 'docs' })).toHaveCount(0)
|
||||
await expect(filesList.getByRole('button', { exact: true, name: 'public' })).toHaveCount(0)
|
||||
await expect(filesList.getByRole('button', { exact: true, name: 'src' })).toHaveCount(0)
|
||||
await expect(filesList.getByRole('button', { exact: true, name: 'web-game' })).toHaveCount(0)
|
||||
},
|
||||
)
|
||||
Then('I should see the small Agent v2 file in the Files section', async function (this: DifyWorld) {
|
||||
await expectAgentConfigFileVisible(this, 'smallFile')
|
||||
})
|
||||
|
||||
Then('I should see the empty Agent v2 file in the Files section', async function (this: DifyWorld) {
|
||||
await expectAgentConfigFileVisible(this, 'emptyFile')
|
||||
})
|
||||
|
||||
Then('I should not see the small Agent v2 file in the Files section', async function (this: DifyWorld) {
|
||||
await expectAgentConfigFileHidden(this, 'smallFile')
|
||||
})
|
||||
|
||||
Then('I should see the special-name Agent v2 file in the Files section', async function (this: DifyWorld) {
|
||||
await expectAgentConfigFileVisible(this, 'specialFilename')
|
||||
})
|
||||
Then(
|
||||
'the small Agent v2 file should be saved in the Agent v2 draft',
|
||||
async function (this: DifyWorld) {
|
||||
await expectAgentConfigFileSaved(this, 'smallFile')
|
||||
},
|
||||
)
|
||||
|
||||
Then(
|
||||
'the empty Agent v2 file should be saved as a zero-byte file in the Agent v2 draft',
|
||||
async function (this: DifyWorld) {
|
||||
await expectAgentConfigFileSaved(this, 'emptyFile', { size: 0 })
|
||||
},
|
||||
)
|
||||
|
||||
Then(
|
||||
'the special-name Agent v2 file should be saved in the Agent v2 draft',
|
||||
async function (this: DifyWorld) {
|
||||
await expectAgentConfigFileSaved(this, 'specialFilename')
|
||||
},
|
||||
)
|
||||
|
||||
async function skipUnsupportedFileFormatRejection(world: DifyWorld) {
|
||||
return skipBlockedPrecondition(
|
||||
world,
|
||||
'Agent v2 unsupported file format rejection is not stable: default upload configuration allows arbitrary extensions unless UPLOAD_FILE_EXTENSION_BLACKLIST is seeded.',
|
||||
{
|
||||
owner: 'product/seed',
|
||||
remediation: 'Define Agent config file type restrictions or seed UPLOAD_FILE_EXTENSION_BLACKLIST before enabling this scenario.',
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
Given('Agent v2 unsupported file format rejection is available', async function (this: DifyWorld) {
|
||||
return skipUnsupportedFileFormatRejection(this)
|
||||
})
|
||||
|
||||
Then('Agent v2 unsupported file format rejection should be available', async function (this: DifyWorld) {
|
||||
return skipUnsupportedFileFormatRejection(this)
|
||||
})
|
||||
|
||||
async function skipOversizedFileRejection(world: DifyWorld) {
|
||||
return skipBlockedPrecondition(
|
||||
world,
|
||||
'Agent v2 oversized file rejection lacks a clear user-visible reason: the current upload dialog collapses upload and commit failures into a generic failure toast.',
|
||||
{
|
||||
owner: 'product',
|
||||
remediation: 'Expose a stable user-visible file-size error before enabling this scenario.',
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
Given('Agent v2 oversized file rejection is available', async function (this: DifyWorld) {
|
||||
return skipOversizedFileRejection(this)
|
||||
})
|
||||
|
||||
Then('Agent v2 oversized file rejection should be available', async function (this: DifyWorld) {
|
||||
return skipOversizedFileRejection(this)
|
||||
})
|
||||
|
||||
async function skipSingleBatchFileCountLimits(world: DifyWorld) {
|
||||
return skipBlockedPrecondition(
|
||||
world,
|
||||
'Agent v2 single-batch file count limits are not reachable: the current Agent config file upload dialog accepts one file per upload.',
|
||||
{
|
||||
owner: 'product',
|
||||
remediation: 'Define multi-file upload behavior and its count-limit error before enabling this scenario.',
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
Given('Agent v2 single-batch file count limits are available', async function (this: DifyWorld) {
|
||||
return skipSingleBatchFileCountLimits(this)
|
||||
})
|
||||
|
||||
Then('Agent v2 single-batch file count limits should be available', async function (this: DifyWorld) {
|
||||
return skipSingleBatchFileCountLimits(this)
|
||||
})
|
||||
|
||||
async function skipTotalFileCountLimits(world: DifyWorld) {
|
||||
return skipBlockedPrecondition(
|
||||
world,
|
||||
'Agent v2 total file count limits are not defined for Agent config files in the current product contract.',
|
||||
{
|
||||
owner: 'product',
|
||||
remediation: 'Define the Agent config file total-count limit and user-visible error before enabling this scenario.',
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
Given('Agent v2 total file count limits are available', async function (this: DifyWorld) {
|
||||
return skipTotalFileCountLimits(this)
|
||||
})
|
||||
|
||||
Then('Agent v2 total file count limits should be available', async function (this: DifyWorld) {
|
||||
return skipTotalFileCountLimits(this)
|
||||
})
|
||||
|
||||
async function skipInProgressFileUploadRecovery(world: DifyWorld) {
|
||||
return skipBlockedPrecondition(
|
||||
world,
|
||||
'Agent v2 in-progress file upload recovery is not stable: the current dialog has no deterministic slow-upload fixture or user-visible navigation guard contract.',
|
||||
{
|
||||
owner: 'product/test-infra',
|
||||
remediation: 'Define upload-in-progress navigation behavior and provide a deterministic slow upload fixture before enabling this scenario.',
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
Given('Agent v2 in-progress file upload recovery is available', async function (this: DifyWorld) {
|
||||
return skipInProgressFileUploadRecovery(this)
|
||||
})
|
||||
|
||||
Then('Agent v2 in-progress file upload recovery should be available', async function (this: DifyWorld) {
|
||||
return skipInProgressFileUploadRecovery(this)
|
||||
})
|
||||
259
e2e/features/step-definitions/agent-v2/knowledge.steps.ts
Normal file
259
e2e/features/step-definitions/agent-v2/knowledge.steps.ts
Normal file
@ -0,0 +1,259 @@
|
||||
import type { Locator } from '@playwright/test'
|
||||
import type { DifyWorld } from '../../support/world'
|
||||
import { Given, Then, When } from '@cucumber/cucumber'
|
||||
import { expect } from '@playwright/test'
|
||||
import {
|
||||
createConfiguredTestAgent,
|
||||
getAgentComposerDraft,
|
||||
} from '../../agent-v2/support/agent'
|
||||
import { agentBuilderFixedInputs, agentBuilderPreseededResources } from '../../agent-v2/support/agent-builder-resources'
|
||||
import {
|
||||
createAgentSoulConfigWithKnowledgeDataset,
|
||||
normalAgentSoulConfig,
|
||||
} from '../../agent-v2/support/agent-soul'
|
||||
import { asArray, asRecord } from '../../agent-v2/support/preflight/common'
|
||||
import { getCurrentAgentId } from './configure-helpers'
|
||||
|
||||
const getPreseededKnowledgeBase = (world: DifyWorld) => {
|
||||
const resource = world.agentBuilder.preflight.preseededResources[
|
||||
agentBuilderPreseededResources.agentKnowledgeBase
|
||||
]
|
||||
if (!resource || resource.kind !== 'dataset') {
|
||||
throw new Error(
|
||||
`Preseeded dataset "${agentBuilderPreseededResources.agentKnowledgeBase}" is not available. Run the matching preflight step first.`,
|
||||
)
|
||||
}
|
||||
|
||||
return resource
|
||||
}
|
||||
|
||||
const getKnowledgeSection = (world: DifyWorld) =>
|
||||
world.getPage().getByRole('region', { name: 'Knowledge Retrieval' })
|
||||
|
||||
const getKnowledgeSets = async (agentId: string) => {
|
||||
const draft = await getAgentComposerDraft(agentId)
|
||||
|
||||
return asArray(asRecord(draft.agent_soul?.knowledge).sets)
|
||||
}
|
||||
|
||||
const openNewKnowledgeRetrievalDialog = async (world: DifyWorld) => {
|
||||
const knowledgeSection = getKnowledgeSection(world)
|
||||
|
||||
await expect(knowledgeSection).toBeVisible({ timeout: 30_000 })
|
||||
await knowledgeSection.getByRole('button', { name: 'Add knowledge retrieval' }).click()
|
||||
|
||||
const dialog = world.getPage().getByRole('dialog', {
|
||||
name: 'Knowledge Retrieval · Agent decide',
|
||||
})
|
||||
await expect(dialog).toBeVisible()
|
||||
|
||||
return dialog
|
||||
}
|
||||
|
||||
const selectPreseededKnowledgeBase = async (
|
||||
world: DifyWorld,
|
||||
dialog: Locator,
|
||||
) => {
|
||||
const knowledgeBase = getPreseededKnowledgeBase(world)
|
||||
const page = world.getPage()
|
||||
|
||||
await dialog.getByRole('button', { name: 'Add Knowledge' }).click()
|
||||
|
||||
const selectorDialog = page.getByRole('dialog', { name: 'Select reference Knowledge' })
|
||||
await expect(selectorDialog).toBeVisible()
|
||||
await selectorDialog.getByRole('button').filter({ hasText: knowledgeBase.name }).click()
|
||||
await selectorDialog.getByRole('button', { name: 'Add' }).click()
|
||||
}
|
||||
|
||||
const openKnowledgeRetrievalSettings = async (world: DifyWorld, name: string) => {
|
||||
const knowledgeSection = getKnowledgeSection(world)
|
||||
|
||||
await expect(knowledgeSection.getByText(name, { exact: true }))
|
||||
.toBeVisible({ timeout: 30_000 })
|
||||
await knowledgeSection.getByText(name, { exact: true }).hover()
|
||||
await knowledgeSection.getByRole('button', {
|
||||
exact: true,
|
||||
name: `Edit ${name}`,
|
||||
}).click()
|
||||
|
||||
const dialog = world.getPage().getByRole('dialog', {
|
||||
name: 'Knowledge Retrieval · Agent decide',
|
||||
})
|
||||
await expect(dialog).toBeVisible()
|
||||
|
||||
return dialog
|
||||
}
|
||||
|
||||
const expectKnowledgeRetrievalDraft = async (
|
||||
world: DifyWorld,
|
||||
expected: {
|
||||
mode: 'generated_query' | 'user_query'
|
||||
value?: string
|
||||
},
|
||||
) => {
|
||||
const agentId = getCurrentAgentId(world)
|
||||
const knowledgeBase = getPreseededKnowledgeBase(world)
|
||||
|
||||
await expect.poll(
|
||||
async () => {
|
||||
const knowledgeSets = await getKnowledgeSets(agentId)
|
||||
const knowledgeSet = asRecord(knowledgeSets[0])
|
||||
const datasets = asArray(knowledgeSet.datasets)
|
||||
const query = asRecord(knowledgeSet.query)
|
||||
const retrieval = asRecord(knowledgeSet.retrieval)
|
||||
|
||||
return {
|
||||
datasetNames: datasets.map(dataset => asRecord(dataset).name),
|
||||
mode: query.mode,
|
||||
name: knowledgeSet.name,
|
||||
retrievalMode: retrieval.mode,
|
||||
value: query.value,
|
||||
}
|
||||
},
|
||||
{ timeout: 30_000 },
|
||||
).toEqual({
|
||||
datasetNames: expect.arrayContaining([knowledgeBase.name]),
|
||||
mode: expected.mode,
|
||||
name: 'Retrieval 1',
|
||||
retrievalMode: 'multiple',
|
||||
value: expected.value,
|
||||
})
|
||||
}
|
||||
|
||||
Given(
|
||||
'a knowledge-backed Agent v2 test agent has been created via API',
|
||||
async function (this: DifyWorld) {
|
||||
const knowledgeBase = getPreseededKnowledgeBase(this)
|
||||
const agent = await createConfiguredTestAgent({
|
||||
agentSoul: createAgentSoulConfigWithKnowledgeDataset(
|
||||
normalAgentSoulConfig,
|
||||
{
|
||||
id: knowledgeBase.id,
|
||||
name: knowledgeBase.name,
|
||||
},
|
||||
),
|
||||
})
|
||||
|
||||
this.createdAgentIds.push(agent.id)
|
||||
this.lastCreatedAgentName = agent.name
|
||||
this.lastCreatedAgentRole = agent.role ?? undefined
|
||||
},
|
||||
)
|
||||
|
||||
When(
|
||||
'I add the Agent Builder knowledge base as an Agent decide Knowledge Retrieval',
|
||||
async function (this: DifyWorld) {
|
||||
const dialog = await openNewKnowledgeRetrievalDialog(this)
|
||||
|
||||
await expect(dialog.getByRole('radio', { name: 'Agent decide' })).toBeChecked()
|
||||
await selectPreseededKnowledgeBase(this, dialog)
|
||||
await expect(dialog.getByText(getPreseededKnowledgeBase(this).name, { exact: true }))
|
||||
.toBeVisible()
|
||||
},
|
||||
)
|
||||
|
||||
When(
|
||||
'I add the Agent Builder knowledge base as a Custom query Knowledge Retrieval',
|
||||
async function (this: DifyWorld) {
|
||||
const dialog = await openNewKnowledgeRetrievalDialog(this)
|
||||
|
||||
await dialog.getByRole('radio', { name: 'Custom query' }).click()
|
||||
await dialog
|
||||
.getByRole('textbox', { name: 'Custom query text' })
|
||||
.fill(agentBuilderFixedInputs.customKnowledgeQuery)
|
||||
await selectPreseededKnowledgeBase(this, dialog)
|
||||
await expect(dialog.getByText(getPreseededKnowledgeBase(this).name, { exact: true }))
|
||||
.toBeVisible()
|
||||
},
|
||||
)
|
||||
|
||||
Then('I should see the Agent v2 Knowledge Retrieval {string}', async function (this: DifyWorld, name: string) {
|
||||
const knowledgeSection = getKnowledgeSection(this)
|
||||
|
||||
await expect(knowledgeSection).toBeVisible({ timeout: 30_000 })
|
||||
await expect(knowledgeSection.getByText(name, { exact: true })).toBeVisible()
|
||||
})
|
||||
|
||||
Then(
|
||||
'the Agent v2 Agent decide Knowledge Retrieval should be saved in the Agent v2 draft',
|
||||
async function (this: DifyWorld) {
|
||||
await expectKnowledgeRetrievalDraft(this, {
|
||||
mode: 'generated_query',
|
||||
})
|
||||
},
|
||||
)
|
||||
|
||||
Then(
|
||||
'the Agent v2 Custom query Knowledge Retrieval should be saved in the Agent v2 draft',
|
||||
async function (this: DifyWorld) {
|
||||
await expectKnowledgeRetrievalDraft(this, {
|
||||
mode: 'user_query',
|
||||
value: agentBuilderFixedInputs.customKnowledgeQuery,
|
||||
})
|
||||
},
|
||||
)
|
||||
|
||||
Then(
|
||||
'I should see the Agent v2 Agent decide Knowledge Retrieval settings',
|
||||
async function (this: DifyWorld) {
|
||||
const dialog = await openKnowledgeRetrievalSettings(this, 'Retrieval 1')
|
||||
|
||||
await expect(dialog.getByRole('radio', { name: 'Agent decide' })).toBeChecked()
|
||||
await expect(dialog.getByText(getPreseededKnowledgeBase(this).name, { exact: true }))
|
||||
.toBeVisible()
|
||||
await expect(dialog.getByRole('button', { name: 'Disabled' })).toBeVisible()
|
||||
},
|
||||
)
|
||||
|
||||
Then(
|
||||
'I should see the Agent v2 Custom query Knowledge Retrieval settings',
|
||||
async function (this: DifyWorld) {
|
||||
const dialog = await openKnowledgeRetrievalSettings(this, 'Retrieval 1')
|
||||
|
||||
await expect(dialog.getByRole('radio', { name: 'Custom query' })).toBeChecked()
|
||||
await expect(dialog.getByRole('textbox', { name: 'Custom query text' }))
|
||||
.toHaveValue(agentBuilderFixedInputs.customKnowledgeQuery)
|
||||
await expect(dialog.getByText(getPreseededKnowledgeBase(this).name, { exact: true }))
|
||||
.toBeVisible()
|
||||
await expect(dialog.getByRole('button', { name: 'Disabled' })).toBeVisible()
|
||||
},
|
||||
)
|
||||
|
||||
When('I remove the Agent v2 Knowledge Retrieval {string}', async function (this: DifyWorld, name: string) {
|
||||
const knowledgeSection = getKnowledgeSection(this)
|
||||
|
||||
await expect(knowledgeSection).toBeVisible({ timeout: 30_000 })
|
||||
await knowledgeSection.getByText(name, { exact: true }).hover()
|
||||
await knowledgeSection.getByRole('button', {
|
||||
exact: true,
|
||||
name: `Remove ${name}`,
|
||||
}).click()
|
||||
})
|
||||
|
||||
Then(
|
||||
'the Agent v2 draft should no longer reference the Agent Builder knowledge base',
|
||||
async function (this: DifyWorld) {
|
||||
const agentId = getCurrentAgentId(this)
|
||||
const knowledgeBase = getPreseededKnowledgeBase(this)
|
||||
|
||||
await expect.poll(
|
||||
async () => {
|
||||
const knowledgeSets = await getKnowledgeSets(agentId)
|
||||
|
||||
return knowledgeSets.some(set => asArray(asRecord(set).datasets).some((dataset) => {
|
||||
const record = asRecord(dataset)
|
||||
|
||||
return record.id === knowledgeBase.id || record.name === knowledgeBase.name
|
||||
}))
|
||||
},
|
||||
{ timeout: 30_000 },
|
||||
).toBe(false)
|
||||
},
|
||||
)
|
||||
|
||||
Then('I should not see the Agent v2 Knowledge Retrieval {string}', async function (this: DifyWorld, name: string) {
|
||||
const knowledgeSection = getKnowledgeSection(this)
|
||||
|
||||
await expect(knowledgeSection).toBeVisible({ timeout: 30_000 })
|
||||
await expect(knowledgeSection.getByText(name, { exact: true })).not.toBeVisible()
|
||||
})
|
||||
413
e2e/features/step-definitions/agent-v2/output-variables.steps.ts
Normal file
413
e2e/features/step-definitions/agent-v2/output-variables.steps.ts
Normal file
@ -0,0 +1,413 @@
|
||||
import type { DataTable } from '@cucumber/cucumber'
|
||||
import type { DeclaredOutputConfig } from '@dify/contracts/api/console/apps/types.gen'
|
||||
import type { AgentV2WorkflowOutputVariable, DifyWorld } from '../../support/world'
|
||||
import { Given, Then, When } from '@cucumber/cucumber'
|
||||
import { expect } from '@playwright/test'
|
||||
import { getWorkflowDraft } from '../../../support/api'
|
||||
import { skipBlockedPrecondition } from '../../agent-v2/support/preflight/common'
|
||||
|
||||
const agentV2WorkflowNodeId = 'agent-v2'
|
||||
const taskFileOutputName = 'e2e_report.pdf'
|
||||
const renamedTaskFileOutputName = 'e2e_final_report.pdf'
|
||||
|
||||
const getAgentOutputToken = (name: string) => `[§output:${name}:${name}§]`
|
||||
|
||||
const getCurrentAppId = (world: DifyWorld) => {
|
||||
const appId = world.createdAppIds.at(-1)
|
||||
if (!appId)
|
||||
throw new Error('No app ID found. Create a workflow app first.')
|
||||
|
||||
return appId
|
||||
}
|
||||
|
||||
const getAgentV2WorkflowNodeData = async (appId: string) => {
|
||||
const draft = await getWorkflowDraft(appId)
|
||||
const agentNode = draft.graph.nodes.find(node => node.id === agentV2WorkflowNodeId)
|
||||
if (!agentNode)
|
||||
throw new Error(`Workflow draft ${appId} does not include Agent v2 node ${agentV2WorkflowNodeId}.`)
|
||||
|
||||
return agentNode.data ?? {}
|
||||
}
|
||||
|
||||
const getDeclaredOutputsFromDraft = async (appId: string): Promise<DeclaredOutputConfig[]> => {
|
||||
const data = await getAgentV2WorkflowNodeData(appId)
|
||||
const outputs = data.agent_declared_outputs
|
||||
if (!Array.isArray(outputs))
|
||||
return []
|
||||
|
||||
return outputs as DeclaredOutputConfig[]
|
||||
}
|
||||
|
||||
const getOutputVariablesFromDraft = async (appId: string) => getDeclaredOutputsFromDraft(appId)
|
||||
|
||||
const waitForWorkflowDraftSave = (world: DifyWorld, appId: string) =>
|
||||
world.getPage().waitForResponse(response => (
|
||||
response.request().method() === 'POST'
|
||||
&& new URL(response.url()).pathname.endsWith(`/console/api/apps/${appId}/workflows/draft`)
|
||||
))
|
||||
|
||||
const openWorkflowOutputVariablesPanel = async (world: DifyWorld) => {
|
||||
const page = world.getPage()
|
||||
const newOutputButton = page.getByRole('button', { name: 'New output' })
|
||||
|
||||
if (!await newOutputButton.isVisible().catch(() => false))
|
||||
await page.getByRole('button', { name: 'Output Variables' }).click()
|
||||
|
||||
await expect(newOutputButton).toBeVisible()
|
||||
}
|
||||
|
||||
const fillOutputVariableEditor = async (
|
||||
world: DifyWorld,
|
||||
{
|
||||
name,
|
||||
required = false,
|
||||
type = 'string',
|
||||
}: {
|
||||
name: string
|
||||
required?: boolean
|
||||
type?: string
|
||||
},
|
||||
) => {
|
||||
const page = world.getPage()
|
||||
const editor = page.getByRole('form', { name: 'Output variable editor' })
|
||||
|
||||
await expect(editor).toBeVisible()
|
||||
await editor.getByRole('textbox', { name: 'Field name' }).fill(name)
|
||||
if (type !== 'string') {
|
||||
await editor.getByRole('button', { name: 'Output type' }).click()
|
||||
await page.getByRole('option', { name: type, exact: true }).click()
|
||||
}
|
||||
if (required)
|
||||
await editor.getByRole('switch', { name: 'Required' }).click()
|
||||
}
|
||||
|
||||
When(
|
||||
'I insert a file output reference from the Agent v2 workflow node task editor',
|
||||
async function (this: DifyWorld) {
|
||||
const page = this.getPage()
|
||||
const appId = getCurrentAppId(this)
|
||||
const taskEditor = page.getByRole('textbox', { name: 'Agent task' })
|
||||
|
||||
await expect(taskEditor).toBeVisible()
|
||||
await taskEditor.click()
|
||||
await page.getByRole('button', { name: 'Insert' }).click()
|
||||
await page.getByRole('button', { name: 'New output' }).click()
|
||||
|
||||
const nameInput = page.getByRole('textbox', { name: 'Field name' })
|
||||
await expect(nameInput).toBeVisible()
|
||||
await nameInput.fill(taskFileOutputName)
|
||||
|
||||
const saveResponse = waitForWorkflowDraftSave(this, appId)
|
||||
await nameInput.press('Enter')
|
||||
expect((await saveResponse).ok()).toBe(true)
|
||||
},
|
||||
)
|
||||
|
||||
When(
|
||||
'I rename the Agent v2 workflow node task output reference',
|
||||
async function (this: DifyWorld) {
|
||||
const page = this.getPage()
|
||||
const appId = getCurrentAppId(this)
|
||||
|
||||
await page.getByText(taskFileOutputName, { exact: true }).hover()
|
||||
const editor = page.getByRole('form', { name: 'Output variable editor' })
|
||||
await expect(editor).toBeVisible()
|
||||
await editor.getByRole('textbox', { name: 'Field name' }).fill(renamedTaskFileOutputName)
|
||||
|
||||
const saveResponse = waitForWorkflowDraftSave(this, appId)
|
||||
await editor.getByRole('button', { name: 'Confirm' }).click()
|
||||
expect((await saveResponse).ok()).toBe(true)
|
||||
await expect(editor).not.toBeVisible()
|
||||
},
|
||||
)
|
||||
|
||||
When(
|
||||
'I add these Agent v2 workflow node output variables',
|
||||
async function (this: DifyWorld, table: DataTable) {
|
||||
const page = this.getPage()
|
||||
const appId = getCurrentAppId(this)
|
||||
const rows = table.hashes() as AgentV2WorkflowOutputVariable[]
|
||||
this.agentBuilder.workflow.outputVariables = rows
|
||||
|
||||
await openWorkflowOutputVariablesPanel(this)
|
||||
|
||||
for (const row of rows) {
|
||||
await page.getByRole('button', { name: 'New output' }).click()
|
||||
await fillOutputVariableEditor(this, row)
|
||||
|
||||
const editor = page.getByRole('form', { name: 'Output variable editor' })
|
||||
const saveResponse = waitForWorkflowDraftSave(this, appId)
|
||||
await editor.getByRole('button', { name: 'Confirm' }).click()
|
||||
expect((await saveResponse).ok()).toBe(true)
|
||||
await expect(editor).not.toBeVisible()
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
When(
|
||||
'I add a required Agent v2 workflow node object output variable with text and analysis fields',
|
||||
async function (this: DifyWorld) {
|
||||
const page = this.getPage()
|
||||
const appId = getCurrentAppId(this)
|
||||
|
||||
await openWorkflowOutputVariablesPanel(this)
|
||||
await page.getByRole('button', { name: 'New output' }).click()
|
||||
await fillOutputVariableEditor(this, {
|
||||
name: 'response',
|
||||
required: true,
|
||||
type: 'object',
|
||||
})
|
||||
|
||||
let saveResponse = waitForWorkflowDraftSave(this, appId)
|
||||
await page.getByRole('form', { name: 'Output variable editor' }).getByRole('button', { name: 'Confirm' }).click()
|
||||
expect((await saveResponse).ok()).toBe(true)
|
||||
|
||||
for (const fieldName of ['text', 'analysis']) {
|
||||
await page.getByText('response', { exact: true }).hover()
|
||||
await page.getByRole('button', { name: 'Add response' }).click()
|
||||
await fillOutputVariableEditor(this, { name: fieldName })
|
||||
|
||||
saveResponse = waitForWorkflowDraftSave(this, appId)
|
||||
await page.getByRole('form', { name: 'Output variable editor' }).getByRole('button', { name: 'Confirm' }).click()
|
||||
expect((await saveResponse).ok()).toBe(true)
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
Then(
|
||||
'the Agent v2 workflow node output variables should be saved in the workflow draft',
|
||||
async function (this: DifyWorld) {
|
||||
const appId = getCurrentAppId(this)
|
||||
const expectedOutputVariables = this.agentBuilder.workflow.outputVariables
|
||||
if (expectedOutputVariables.length === 0)
|
||||
throw new Error('No Agent v2 workflow output variables were recorded for this scenario.')
|
||||
|
||||
await expect
|
||||
.poll(async () => {
|
||||
const outputs = await getOutputVariablesFromDraft(appId)
|
||||
|
||||
return expectedOutputVariables.map((expected) => {
|
||||
const output = outputs.find(item => item.name === expected.name)
|
||||
return {
|
||||
name: output?.name,
|
||||
type: output?.type === 'array'
|
||||
? `array[${output.array_item?.type ?? 'object'}]`
|
||||
: output?.type,
|
||||
}
|
||||
})
|
||||
}, {
|
||||
timeout: 30_000,
|
||||
})
|
||||
.toEqual(expectedOutputVariables)
|
||||
},
|
||||
)
|
||||
|
||||
Then('I should see the Agent v2 workflow node output variables', async function (this: DifyWorld) {
|
||||
const page = this.getPage()
|
||||
const expectedOutputVariables = this.agentBuilder.workflow.outputVariables
|
||||
if (expectedOutputVariables.length === 0)
|
||||
throw new Error('No Agent v2 workflow output variables were recorded for this scenario.')
|
||||
|
||||
await openWorkflowOutputVariablesPanel(this)
|
||||
|
||||
for (const output of expectedOutputVariables) {
|
||||
await expect(page.getByText(output.name, { exact: true })).toBeVisible()
|
||||
await expect(page.getByText(output.type, { exact: true })).toBeVisible()
|
||||
}
|
||||
})
|
||||
|
||||
Then(
|
||||
'the Agent v2 workflow node nested object output variable should be saved in the workflow draft',
|
||||
async function (this: DifyWorld) {
|
||||
const appId = getCurrentAppId(this)
|
||||
|
||||
await expect
|
||||
.poll(async () => {
|
||||
const outputs = await getDeclaredOutputsFromDraft(appId)
|
||||
const response = outputs.find(output => output.name === 'response')
|
||||
|
||||
return {
|
||||
children: response?.children?.map(child => ({
|
||||
name: child.name,
|
||||
required: child.required,
|
||||
type: child.type,
|
||||
})),
|
||||
name: response?.name,
|
||||
required: response?.required,
|
||||
type: response?.type,
|
||||
}
|
||||
}, {
|
||||
timeout: 30_000,
|
||||
})
|
||||
.toEqual({
|
||||
children: [
|
||||
{
|
||||
name: 'text',
|
||||
required: false,
|
||||
type: 'string',
|
||||
},
|
||||
{
|
||||
name: 'analysis',
|
||||
required: false,
|
||||
type: 'string',
|
||||
},
|
||||
],
|
||||
name: 'response',
|
||||
required: true,
|
||||
type: 'object',
|
||||
})
|
||||
},
|
||||
)
|
||||
|
||||
Then(
|
||||
'the Agent v2 workflow node task should reference the file output',
|
||||
async function (this: DifyWorld) {
|
||||
await expectAgentTaskOutputReference(this, taskFileOutputName)
|
||||
},
|
||||
)
|
||||
|
||||
Then(
|
||||
'the Agent v2 workflow node task should reference the renamed file output',
|
||||
async function (this: DifyWorld) {
|
||||
await expectAgentTaskOutputReference(this, renamedTaskFileOutputName, taskFileOutputName)
|
||||
},
|
||||
)
|
||||
|
||||
Then('I should see the Agent v2 workflow node nested object output variable', async function (this: DifyWorld) {
|
||||
const page = this.getPage()
|
||||
|
||||
await openWorkflowOutputVariablesPanel(this)
|
||||
await expect(page.getByText('response', { exact: true })).toBeVisible()
|
||||
await expect(page.getByText('object', { exact: true })).toBeVisible()
|
||||
await expect(page.getByText('Required', { exact: true })).toBeVisible()
|
||||
await expect(page.getByText('text', { exact: true })).toBeVisible()
|
||||
await expect(page.getByText('analysis', { exact: true })).toBeVisible()
|
||||
await expect(page.getByText('string', { exact: true })).toBeVisible()
|
||||
})
|
||||
|
||||
async function expectAgentTaskOutputReference(
|
||||
world: DifyWorld,
|
||||
expectedName: string,
|
||||
unexpectedName?: string,
|
||||
) {
|
||||
const page = world.getPage()
|
||||
const appId = getCurrentAppId(world)
|
||||
|
||||
await expect.poll(
|
||||
async () => {
|
||||
const data = await getAgentV2WorkflowNodeData(appId)
|
||||
const outputs = Array.isArray(data.agent_declared_outputs)
|
||||
? data.agent_declared_outputs as DeclaredOutputConfig[]
|
||||
: []
|
||||
const expectedOutput = outputs.find(output => output.name === expectedName)
|
||||
|
||||
return {
|
||||
agentTask: data.agent_task,
|
||||
expectedOutput: expectedOutput
|
||||
? {
|
||||
name: expectedOutput.name,
|
||||
type: expectedOutput.type,
|
||||
}
|
||||
: undefined,
|
||||
unexpectedOutput: unexpectedName
|
||||
? outputs.some(output => output.name === unexpectedName)
|
||||
: false,
|
||||
}
|
||||
},
|
||||
{ timeout: 30_000 },
|
||||
).toEqual({
|
||||
agentTask: expect.stringContaining(getAgentOutputToken(expectedName)),
|
||||
expectedOutput: {
|
||||
name: expectedName,
|
||||
type: 'file',
|
||||
},
|
||||
unexpectedOutput: false,
|
||||
})
|
||||
|
||||
await expect(page.getByText(expectedName, { exact: true })).toBeVisible()
|
||||
await expect(page.getByText('file', { exact: true })).toBeVisible()
|
||||
if (unexpectedName)
|
||||
await expect(page.getByText(unexpectedName, { exact: true })).toHaveCount(0)
|
||||
}
|
||||
|
||||
async function skipStandaloneOutputVariables(world: DifyWorld) {
|
||||
return skipBlockedPrecondition(
|
||||
world,
|
||||
'Standalone Agent Output Variables are not available: output variables currently belong to Workflow Agent v2 nodes.',
|
||||
{
|
||||
owner: 'product',
|
||||
remediation: 'Expose standalone Agent Output Variables or keep this scenario excluded until the product path exists.',
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
Given('Agent v2 standalone Output Variables are available', async function (this: DifyWorld) {
|
||||
return skipStandaloneOutputVariables(this)
|
||||
})
|
||||
|
||||
Then('Agent v2 standalone Output Variables should be available', async function (this: DifyWorld) {
|
||||
return skipStandaloneOutputVariables(this)
|
||||
})
|
||||
|
||||
async function skipWorkflowOutputRetryStrategy(world: DifyWorld) {
|
||||
return skipBlockedPrecondition(
|
||||
world,
|
||||
'Agent v2 workflow Output Variables retry strategy is not available in the current editor UI.',
|
||||
{
|
||||
owner: 'product',
|
||||
remediation: 'Expose user-visible retry strategy controls before enabling this scenario.',
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
Given('Agent v2 workflow output retry strategy is available', async function (this: DifyWorld) {
|
||||
return skipWorkflowOutputRetryStrategy(this)
|
||||
})
|
||||
|
||||
Then('Agent v2 workflow output retry strategy should be available', async function (this: DifyWorld) {
|
||||
return skipWorkflowOutputRetryStrategy(this)
|
||||
})
|
||||
|
||||
async function skipWorkflowTaskOutputReferenceDeletionConsistency(world: DifyWorld) {
|
||||
return skipBlockedPrecondition(
|
||||
world,
|
||||
'Agent v2 workflow task output deletion consistency is not available: deleting an output from the list currently leaves the Prompt token without a stable user-visible invalid-reference state.',
|
||||
{
|
||||
owner: 'product',
|
||||
remediation: 'Define whether deletion should sync the Prompt token, block deletion, or expose an invalid-reference state before enabling this scenario.',
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
Given(
|
||||
'Agent v2 workflow task output reference deletion consistency is available',
|
||||
async function (this: DifyWorld) {
|
||||
return skipWorkflowTaskOutputReferenceDeletionConsistency(this)
|
||||
},
|
||||
)
|
||||
|
||||
Then(
|
||||
'Agent v2 workflow task output reference deletion consistency should be available',
|
||||
async function (this: DifyWorld) {
|
||||
return skipWorkflowTaskOutputReferenceDeletionConsistency(this)
|
||||
},
|
||||
)
|
||||
|
||||
async function skipWorkflowOutputRetryCountValidation(world: DifyWorld) {
|
||||
return skipBlockedPrecondition(
|
||||
world,
|
||||
'Agent v2 workflow Output Variables retry count validation is not reachable because retry strategy controls are not available in the current editor UI.',
|
||||
{
|
||||
owner: 'product',
|
||||
remediation: 'Expose retry count controls and validation states before enabling this scenario.',
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
Given('Agent v2 workflow output retry count validation is available', async function (this: DifyWorld) {
|
||||
return skipWorkflowOutputRetryCountValidation(this)
|
||||
})
|
||||
|
||||
Then('Agent v2 workflow output retry count validation should be available', async function (this: DifyWorld) {
|
||||
return skipWorkflowOutputRetryCountValidation(this)
|
||||
})
|
||||
211
e2e/features/step-definitions/agent-v2/preflight.steps.ts
Normal file
211
e2e/features/step-definitions/agent-v2/preflight.steps.ts
Normal file
@ -0,0 +1,211 @@
|
||||
import type { DifyWorld } from '../../support/world'
|
||||
import { Given } from '@cucumber/cucumber'
|
||||
import {
|
||||
skipMissingPreseededAgentBackendApiKey,
|
||||
skipMissingPreseededAgentPublishedWebApp,
|
||||
skipMissingPreseededAgentWorkflowReference,
|
||||
} from '../../agent-v2/support/preflight/access'
|
||||
import {
|
||||
skipMissingPreseededAgent,
|
||||
skipMissingPreseededAgentDriveSkill,
|
||||
skipMissingPreseededAgentFileTreeFixture,
|
||||
skipMissingPreseededAgentFlatFileFixtureConfiguration,
|
||||
skipMissingPreseededDualRetrievalAgentConfiguration,
|
||||
skipMissingPreseededFullConfigAgentCoreConfiguration,
|
||||
skipMissingPreseededToolStatesAgentConfiguration,
|
||||
skipMissingPreseededWorkflow,
|
||||
} from '../../agent-v2/support/preflight/agents'
|
||||
import {
|
||||
skipMissingIndexingPreseededDataset,
|
||||
skipMissingPreseededDataset,
|
||||
skipMissingReadyPreseededDataset,
|
||||
} from '../../agent-v2/support/preflight/datasets'
|
||||
import {
|
||||
skipMissingAgentBuilderBrokenChatModel,
|
||||
skipMissingAgentBuilderStableChatModel,
|
||||
} from '../../agent-v2/support/preflight/models'
|
||||
import { skipMissingPreseededTool } from '../../agent-v2/support/preflight/tools'
|
||||
|
||||
Given('the Agent Builder stable chat model is available', async function (this: DifyWorld) {
|
||||
const stableModel = await skipMissingAgentBuilderStableChatModel(this)
|
||||
if (stableModel === 'skipped')
|
||||
return stableModel
|
||||
|
||||
this.agentBuilder.preflight.stableModel = stableModel
|
||||
})
|
||||
|
||||
Given('the Agent Builder broken chat model is available', async function (this: DifyWorld) {
|
||||
const brokenModel = await skipMissingAgentBuilderBrokenChatModel(this)
|
||||
if (brokenModel === 'skipped')
|
||||
return brokenModel
|
||||
|
||||
this.agentBuilder.preflight.brokenModel = brokenModel
|
||||
})
|
||||
|
||||
Given(
|
||||
'the Agent Builder preseeded Agent {string} is available',
|
||||
async function (this: DifyWorld, resourceName: string) {
|
||||
const resource = await skipMissingPreseededAgent(this, resourceName)
|
||||
if (resource === 'skipped')
|
||||
return resource
|
||||
|
||||
this.agentBuilder.preflight.preseededResources[resourceName] = resource
|
||||
},
|
||||
)
|
||||
|
||||
Given(
|
||||
'the Agent Builder preseeded workflow {string} is available',
|
||||
async function (this: DifyWorld, resourceName: string) {
|
||||
const resource = await skipMissingPreseededWorkflow(this, resourceName)
|
||||
if (resource === 'skipped')
|
||||
return resource
|
||||
|
||||
this.agentBuilder.preflight.preseededResources[resourceName] = resource
|
||||
},
|
||||
)
|
||||
|
||||
Given(
|
||||
'the Agent Builder preseeded dataset {string} is available',
|
||||
async function (this: DifyWorld, resourceName: string) {
|
||||
const resource = await skipMissingPreseededDataset(this, resourceName)
|
||||
if (resource === 'skipped')
|
||||
return resource
|
||||
|
||||
this.agentBuilder.preflight.preseededResources[resourceName] = resource
|
||||
},
|
||||
)
|
||||
|
||||
Given(
|
||||
'the Agent Builder preseeded dataset {string} is indexed and ready',
|
||||
async function (this: DifyWorld, resourceName: string) {
|
||||
const resource = await skipMissingReadyPreseededDataset(this, resourceName)
|
||||
if (resource === 'skipped')
|
||||
return resource
|
||||
|
||||
this.agentBuilder.preflight.preseededResources[resourceName] = resource
|
||||
},
|
||||
)
|
||||
|
||||
Given(
|
||||
'the Agent Builder preseeded dataset {string} is indexing',
|
||||
async function (this: DifyWorld, resourceName: string) {
|
||||
const resource = await skipMissingIndexingPreseededDataset(this, resourceName)
|
||||
if (resource === 'skipped')
|
||||
return resource
|
||||
|
||||
this.agentBuilder.preflight.preseededResources[resourceName] = resource
|
||||
},
|
||||
)
|
||||
|
||||
Given(
|
||||
'the Agent Builder preseeded tool {string} is available',
|
||||
async function (this: DifyWorld, resourceName: string) {
|
||||
const resource = await skipMissingPreseededTool(this, resourceName)
|
||||
if (resource === 'skipped')
|
||||
return resource
|
||||
|
||||
this.agentBuilder.preflight.preseededResources[resourceName] = resource
|
||||
},
|
||||
)
|
||||
|
||||
Given(
|
||||
'the Agent Builder preseeded Agent {string} includes drive skill {string}',
|
||||
async function (this: DifyWorld, agentName: string, skillName: string) {
|
||||
const resource = await skipMissingPreseededAgentDriveSkill(this, agentName, skillName)
|
||||
if (resource === 'skipped')
|
||||
return resource
|
||||
|
||||
this.agentBuilder.preflight.preseededResources[`${agentName} / ${skillName}`] = resource
|
||||
},
|
||||
)
|
||||
|
||||
Given(
|
||||
'the Agent Builder preseeded Agent {string} includes the core fixture configuration',
|
||||
async function (this: DifyWorld, agentName: string) {
|
||||
const resource = await skipMissingPreseededFullConfigAgentCoreConfiguration(this, agentName)
|
||||
if (resource === 'skipped')
|
||||
return resource
|
||||
|
||||
this.agentBuilder.preflight.preseededResources[`${agentName} / core fixture configuration`] = resource
|
||||
},
|
||||
)
|
||||
|
||||
Given(
|
||||
'the Agent Builder preseeded Agent {string} includes the tool state fixture configuration',
|
||||
async function (this: DifyWorld, agentName: string) {
|
||||
const resource = await skipMissingPreseededToolStatesAgentConfiguration(this, agentName)
|
||||
if (resource === 'skipped')
|
||||
return resource
|
||||
|
||||
this.agentBuilder.preflight.preseededResources[`${agentName} / tool state fixture configuration`]
|
||||
= resource
|
||||
},
|
||||
)
|
||||
|
||||
Given(
|
||||
'the Agent Builder preseeded Agent {string} includes the dual retrieval fixture configuration',
|
||||
async function (this: DifyWorld, agentName: string) {
|
||||
const resource = await skipMissingPreseededDualRetrievalAgentConfiguration(this, agentName)
|
||||
if (resource === 'skipped')
|
||||
return resource
|
||||
|
||||
this.agentBuilder.preflight.preseededResources[`${agentName} / dual retrieval fixture configuration`]
|
||||
= resource
|
||||
},
|
||||
)
|
||||
|
||||
Given(
|
||||
'the Agent Builder preseeded Agent {string} includes the file tree fixture files',
|
||||
async function (this: DifyWorld, agentName: string) {
|
||||
const resource = await skipMissingPreseededAgentFileTreeFixture(this, agentName)
|
||||
if (resource === 'skipped')
|
||||
return resource
|
||||
|
||||
this.agentBuilder.preflight.preseededResources[`${agentName} / file tree fixture`] = resource
|
||||
},
|
||||
)
|
||||
|
||||
Given(
|
||||
'the Agent Builder preseeded Agent {string} includes the current flat file fixture configuration',
|
||||
async function (this: DifyWorld, agentName: string) {
|
||||
const resource = await skipMissingPreseededAgentFlatFileFixtureConfiguration(this, agentName)
|
||||
if (resource === 'skipped')
|
||||
return resource
|
||||
|
||||
this.agentBuilder.preflight.preseededResources[`${agentName} / flat file fixture configuration`]
|
||||
= resource
|
||||
},
|
||||
)
|
||||
|
||||
Given(
|
||||
'the Agent Builder preseeded Agent {string} has Backend service API access with an API key',
|
||||
async function (this: DifyWorld, agentName: string) {
|
||||
const resource = await skipMissingPreseededAgentBackendApiKey(this, agentName)
|
||||
if (resource === 'skipped')
|
||||
return resource
|
||||
|
||||
this.agentBuilder.preflight.preseededResources[`${agentName} / Backend service API key`] = resource
|
||||
},
|
||||
)
|
||||
|
||||
Given(
|
||||
'the Agent Builder preseeded Agent {string} has published Web app access',
|
||||
async function (this: DifyWorld, agentName: string) {
|
||||
const resource = await skipMissingPreseededAgentPublishedWebApp(this, agentName)
|
||||
if (resource === 'skipped')
|
||||
return resource
|
||||
|
||||
this.agentBuilder.preflight.preseededResources[`${agentName} / Web app`] = resource
|
||||
},
|
||||
)
|
||||
|
||||
Given(
|
||||
'the Agent Builder preseeded Agent {string} is referenced by workflow {string}',
|
||||
async function (this: DifyWorld, agentName: string, workflowName: string) {
|
||||
const resource = await skipMissingPreseededAgentWorkflowReference(this, agentName, workflowName)
|
||||
if (resource === 'skipped')
|
||||
return resource
|
||||
|
||||
this.agentBuilder.preflight.preseededResources[`${agentName} / ${workflowName}`] = resource
|
||||
},
|
||||
)
|
||||
107
e2e/features/step-definitions/agent-v2/publish.steps.ts
Normal file
107
e2e/features/step-definitions/agent-v2/publish.steps.ts
Normal file
@ -0,0 +1,107 @@
|
||||
import type { DifyWorld } from '../../support/world'
|
||||
import { Then, When } from '@cucumber/cucumber'
|
||||
import { expect } from '@playwright/test'
|
||||
import { waitForAgentConfigureAutosaved } from '../../../support/agent-configure'
|
||||
import { getAgentVersionDetail, getTestAgent } from '../../agent-v2/support/agent'
|
||||
import { normalAgentPrompt } from '../../agent-v2/support/agent-soul'
|
||||
import { getCurrentAgentId } from './configure-helpers'
|
||||
|
||||
When('I publish the Agent v2 draft', async function (this: DifyWorld) {
|
||||
const page = this.getPage()
|
||||
const publishButton = page.getByRole('button', { name: /^Publish(?: update)?$/ })
|
||||
|
||||
await expect(publishButton).toBeEnabled({ timeout: 30_000 })
|
||||
await publishButton.click()
|
||||
})
|
||||
|
||||
Then('the Agent v2 configuration should be saved automatically', async function (this: DifyWorld) {
|
||||
await waitForAgentConfigureAutosaved(this.getPage())
|
||||
})
|
||||
|
||||
Then('the Agent v2 draft should be published and up to date', async function (this: DifyWorld) {
|
||||
const page = this.getPage()
|
||||
const agentId = getCurrentAgentId(this)
|
||||
|
||||
await expect(page.getByRole('button', { name: 'Published' })).toBeVisible({ timeout: 30_000 })
|
||||
await expect(page.getByRole('status', { name: /^Up to date\./ })).toBeVisible()
|
||||
await expect(page.getByText('Up to date')).toBeVisible()
|
||||
await expect.poll(async () => (await getTestAgent(agentId)).active_config_is_published).toBe(true)
|
||||
})
|
||||
|
||||
Then(
|
||||
'the Agent v2 publish action should be available for unpublished changes',
|
||||
async function (this: DifyWorld) {
|
||||
const page = this.getPage()
|
||||
const agentId = getCurrentAgentId(this)
|
||||
|
||||
await expect(page.getByRole('status', { name: /^(?:Draft|Unpublished changes)\./ }))
|
||||
.toBeVisible({ timeout: 30_000 })
|
||||
await expect(page.getByRole('button', { name: /^Publish(?: update)?$/ }))
|
||||
.toBeEnabled()
|
||||
await expect.poll(async () => (await getTestAgent(agentId)).active_config_is_published)
|
||||
.toBe(false)
|
||||
},
|
||||
)
|
||||
|
||||
Then('the Agent v2 publish action should be unavailable while up to date', async function (this: DifyWorld) {
|
||||
const page = this.getPage()
|
||||
|
||||
await expect(page.getByRole('status', { name: /^Up to date\./ })).toBeVisible({ timeout: 30_000 })
|
||||
await expect(page.getByRole('button', { name: 'Published' })).toBeDisabled()
|
||||
})
|
||||
|
||||
When('I open the Agent v2 version history', async function (this: DifyWorld) {
|
||||
const page = this.getPage()
|
||||
|
||||
await page.getByRole('button', { name: 'Open version history' }).click()
|
||||
await expect(page.getByRole('heading', { name: 'Versions' })).toBeVisible({ timeout: 30_000 })
|
||||
})
|
||||
|
||||
When('I select Agent v2 published version {int}', async function (this: DifyWorld, versionNumber: number) {
|
||||
const page = this.getPage()
|
||||
const versionButton = page.getByRole('button', { name: new RegExp(`\\bVersion ${versionNumber}\\b`) })
|
||||
|
||||
await expect(versionButton).toBeVisible({ timeout: 30_000 })
|
||||
await versionButton.click()
|
||||
})
|
||||
|
||||
Then('the selected Agent v2 version should be displayed in view-only mode', async function (this: DifyWorld) {
|
||||
const page = this.getPage()
|
||||
|
||||
await expect(page.getByText('View Only')).toBeVisible({ timeout: 30_000 })
|
||||
await expect(page.getByRole('button', { name: 'Restore' })).toBeEnabled()
|
||||
})
|
||||
|
||||
When('I restore the selected Agent v2 version', async function (this: DifyWorld) {
|
||||
const page = this.getPage()
|
||||
const agentId = getCurrentAgentId(this)
|
||||
const restoreResponse = page.waitForResponse(response =>
|
||||
response.request().method() === 'POST'
|
||||
&& response.url().includes(`/console/api/agent/${agentId}/versions/`)
|
||||
&& response.url().endsWith('/restore'),
|
||||
)
|
||||
|
||||
await page.getByRole('button', { name: 'Restore' }).click()
|
||||
const response = await restoreResponse
|
||||
expect(response.ok()).toBe(true)
|
||||
})
|
||||
|
||||
Then(
|
||||
'the active published Agent v2 version should still use the normal E2E prompt',
|
||||
async function (this: DifyWorld) {
|
||||
const agentId = getCurrentAgentId(this)
|
||||
|
||||
await expect.poll(async () => (await getTestAgent(agentId)).active_config_is_published, {
|
||||
timeout: 30_000,
|
||||
}).toBe(false)
|
||||
|
||||
const agent = await getTestAgent(agentId)
|
||||
const activeSnapshotId = agent?.active_config_snapshot_id
|
||||
if (!activeSnapshotId)
|
||||
throw new Error(`Agent v2 ${agentId} does not have an active published snapshot.`)
|
||||
|
||||
const version = await getAgentVersionDetail(agentId, activeSnapshotId)
|
||||
|
||||
expect(version.config_snapshot.prompt).toEqual({ system_prompt: normalAgentPrompt })
|
||||
},
|
||||
)
|
||||
127
e2e/features/step-definitions/agent-v2/tools.steps.ts
Normal file
127
e2e/features/step-definitions/agent-v2/tools.steps.ts
Normal file
@ -0,0 +1,127 @@
|
||||
import type { DifyWorld } from '../../support/world'
|
||||
import { Given, Then, When } from '@cucumber/cucumber'
|
||||
import { expect } from '@playwright/test'
|
||||
import { getAgentComposerDraft } from '../../agent-v2/support/agent'
|
||||
import { agentBuilderFixedInputs, agentBuilderPreseededResources } from '../../agent-v2/support/agent-builder-resources'
|
||||
import { asArray, asRecord, skipBlockedPrecondition } from '../../agent-v2/support/preflight/common'
|
||||
import { hasToolEntry } from '../../agent-v2/support/preflight/tools'
|
||||
import { getPreseededToolContract } from '../../agent-v2/support/tools'
|
||||
import { expectProviderToolActionVisible, getCurrentAgentId } from './configure-helpers'
|
||||
|
||||
const getToolsSection = (world: DifyWorld) =>
|
||||
world.getPage().getByRole('region', { name: 'Tools' })
|
||||
|
||||
const getToolSelectorSearch = (world: DifyWorld) =>
|
||||
world.getPage().getByRole('textbox', { name: 'Search integrations...' })
|
||||
|
||||
const expectJsonReplaceToolDraft = async (world: DifyWorld) => {
|
||||
const agentId = getCurrentAgentId(world)
|
||||
const tool = getPreseededToolContract(world, agentBuilderPreseededResources.jsonReplaceTool)
|
||||
|
||||
await expect.poll(
|
||||
async () => {
|
||||
const draft = await getAgentComposerDraft(agentId)
|
||||
const tools = asArray(asRecord(draft.agent_soul?.tools).dify_tools)
|
||||
|
||||
return hasToolEntry(tools, tool)
|
||||
},
|
||||
{ timeout: 30_000 },
|
||||
).toBe(true)
|
||||
}
|
||||
|
||||
async function skipJsonReplaceRuntimeVerification(world: DifyWorld) {
|
||||
return skipBlockedPrecondition(
|
||||
world,
|
||||
'Agent v2 JSON Replace runtime verification is blocked: the suite needs the JSON Process / JSON Replace runtime parameter contract and a deterministic published-runtime prompt before asserting tool execution.',
|
||||
{
|
||||
owner: 'test/seed',
|
||||
remediation: 'Seed the JSON Replace tool runtime contract, then verify execution through published Web app or Backend service API instead of Builder Preview.',
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
When(
|
||||
'I add the Agent Builder JSON Replace tool from the Tools selector',
|
||||
async function (this: DifyWorld) {
|
||||
const page = this.getPage()
|
||||
const toolsSection = getToolsSection(this)
|
||||
|
||||
await expect(toolsSection).toBeVisible({ timeout: 30_000 })
|
||||
await toolsSection.getByRole('button', { name: 'Add tool' }).click()
|
||||
await page.getByRole('button', { name: /^Tool\b/ }).click()
|
||||
|
||||
const search = getToolSelectorSearch(this)
|
||||
await expect(search).toBeVisible()
|
||||
await search.fill('JSON Replace')
|
||||
|
||||
await page.getByRole('button', { exact: true, name: 'JSON Replace' }).click()
|
||||
await expectProviderToolActionVisible(
|
||||
toolsSection,
|
||||
agentBuilderPreseededResources.jsonReplaceTool,
|
||||
)
|
||||
},
|
||||
)
|
||||
|
||||
When(
|
||||
'I search for the missing Agent v2 tool from the Tools selector',
|
||||
async function (this: DifyWorld) {
|
||||
const toolsSection = getToolsSection(this)
|
||||
|
||||
await expect(toolsSection).toBeVisible({ timeout: 30_000 })
|
||||
await toolsSection.getByRole('button', { name: 'Add tool' }).click()
|
||||
await this.getPage().getByRole('button', { name: /^Tool\b/ }).click()
|
||||
|
||||
const search = getToolSelectorSearch(this)
|
||||
await expect(search).toBeVisible()
|
||||
await search.fill(agentBuilderFixedInputs.missingToolSearchWithSuffix)
|
||||
},
|
||||
)
|
||||
|
||||
When('I clear the Agent v2 tool selector search', async function (this: DifyWorld) {
|
||||
const search = getToolSelectorSearch(this)
|
||||
|
||||
await search.fill('')
|
||||
})
|
||||
|
||||
Given('Agent v2 JSON Replace runtime verification is available', async function (this: DifyWorld) {
|
||||
return skipJsonReplaceRuntimeVerification(this)
|
||||
})
|
||||
|
||||
Then('Agent v2 JSON Replace runtime verification should be available', async function (this: DifyWorld) {
|
||||
return skipJsonReplaceRuntimeVerification(this)
|
||||
})
|
||||
|
||||
Then(
|
||||
'the Agent v2 JSON Replace tool should be saved in the Agent v2 draft',
|
||||
async function (this: DifyWorld) {
|
||||
await expectJsonReplaceToolDraft(this)
|
||||
},
|
||||
)
|
||||
|
||||
Then(
|
||||
'I should see the Agent v2 JSON Replace tool in the Tools section',
|
||||
async function (this: DifyWorld) {
|
||||
await expectProviderToolActionVisible(
|
||||
getToolsSection(this),
|
||||
agentBuilderPreseededResources.jsonReplaceTool,
|
||||
)
|
||||
await expectJsonReplaceToolDraft(this)
|
||||
},
|
||||
)
|
||||
|
||||
Then('I should see the Agent v2 tool selector empty state', async function (this: DifyWorld) {
|
||||
const page = this.getPage()
|
||||
|
||||
await expect(page.getByText('No integrations were found')).toBeVisible({ timeout: 30_000 })
|
||||
await expect(page.getByRole('link', { name: 'Requests to the community' })).toBeVisible()
|
||||
await expect(page.getByText(agentBuilderFixedInputs.missingToolSearchWithSuffix)).not.toBeVisible()
|
||||
})
|
||||
|
||||
Then('I should see the Agent v2 tool selector ready for another search', async function (this: DifyWorld) {
|
||||
const page = this.getPage()
|
||||
const search = getToolSelectorSearch(this)
|
||||
|
||||
await expect(search).toHaveValue('')
|
||||
await expect(page.getByText('No integrations were found')).not.toBeVisible()
|
||||
await expect(page.getByText('All tools')).toBeVisible()
|
||||
})
|
||||
129
e2e/features/step-definitions/agent-v2/workflow-node.steps.ts
Normal file
129
e2e/features/step-definitions/agent-v2/workflow-node.steps.ts
Normal file
@ -0,0 +1,129 @@
|
||||
import type { DifyWorld } from '../../support/world'
|
||||
import { Given, Then, When } from '@cucumber/cucumber'
|
||||
import { expect } from '@playwright/test'
|
||||
import {
|
||||
createTestApp,
|
||||
syncAgentV2WorkflowDraft,
|
||||
} from '../../../support/api'
|
||||
import { createE2EResourceName } from '../../../support/naming'
|
||||
import { createConfiguredTestAgent } from '../../agent-v2/support/agent'
|
||||
import {
|
||||
createAgentSoulConfigWithModel,
|
||||
normalAgentPrompt,
|
||||
normalAgentSoulConfig,
|
||||
} from '../../agent-v2/support/agent-soul'
|
||||
|
||||
Given(
|
||||
'a workflow app with an Agent v2 node has been created via API',
|
||||
async function (this: DifyWorld) {
|
||||
if (!this.agentBuilder.preflight.stableModel)
|
||||
throw new Error('Create an Agent v2 workflow node after stable model preflight.')
|
||||
|
||||
const agent = await createConfiguredTestAgent({
|
||||
agentSoul: createAgentSoulConfigWithModel(
|
||||
normalAgentSoulConfig,
|
||||
this.agentBuilder.preflight.stableModel,
|
||||
),
|
||||
})
|
||||
this.createdAgentIds.push(agent.id)
|
||||
this.lastCreatedAgentName = agent.name
|
||||
this.lastCreatedAgentRole = agent.role ?? undefined
|
||||
|
||||
const app = await createTestApp(createE2EResourceName('App', 'workflow-agent-v2'), 'workflow')
|
||||
this.createdAppIds.push(app.id)
|
||||
this.lastCreatedAppName = app.name
|
||||
|
||||
await syncAgentV2WorkflowDraft(app.id, agent.id)
|
||||
},
|
||||
)
|
||||
|
||||
When('I open the Agent v2 workflow node panel', async function (this: DifyWorld) {
|
||||
const page = this.getPage()
|
||||
const workflowCanvas = page.locator('#workflow-container')
|
||||
const agentNode = workflowCanvas.getByRole('button', { name: 'Agent' }).first()
|
||||
|
||||
await expect(agentNode).toBeVisible({ timeout: 30_000 })
|
||||
await agentNode.click()
|
||||
await expect(page.getByRole('button', { name: 'Output Variables' })).toBeVisible()
|
||||
})
|
||||
|
||||
When('I open the Agent v2 workflow Agent details', async function (this: DifyWorld) {
|
||||
const page = this.getPage()
|
||||
const agentName = this.lastCreatedAgentName
|
||||
if (!agentName)
|
||||
throw new Error('No Agent v2 name found. Create a workflow Agent v2 node first.')
|
||||
|
||||
await page.getByRole('button', { name: `Open ${agentName} details` }).click()
|
||||
await expect(page.getByRole('dialog', { name: `${agentName} details` })).toBeVisible()
|
||||
})
|
||||
|
||||
When('I open the Agent v2 workflow Agent in Agent Console', async function (this: DifyWorld) {
|
||||
const page = this.getPage()
|
||||
const agentName = this.lastCreatedAgentName
|
||||
if (!agentName)
|
||||
throw new Error('No Agent v2 name found. Create a workflow Agent v2 node first.')
|
||||
|
||||
const detailsDialog = page.getByRole('dialog', { name: `${agentName} details` })
|
||||
const [agentConsolePage] = await Promise.all([
|
||||
page.waitForEvent('popup'),
|
||||
detailsDialog.getByRole('link', { name: 'Edit in Agent Console' }).click(),
|
||||
])
|
||||
|
||||
this.agentBuilder.workflow.agentConsolePage = agentConsolePage
|
||||
})
|
||||
|
||||
Then(
|
||||
'I should see the Agent v2 workflow Agent details for the created Agent',
|
||||
async function (this: DifyWorld) {
|
||||
const page = this.getPage()
|
||||
const agentName = this.lastCreatedAgentName
|
||||
const agentRole = this.lastCreatedAgentRole
|
||||
const stableModel = this.agentBuilder.preflight.stableModel
|
||||
if (!agentName)
|
||||
throw new Error('No Agent v2 name found. Create a workflow Agent v2 node first.')
|
||||
if (!stableModel)
|
||||
throw new Error('Stable chat model preflight must run before asserting workflow Agent details.')
|
||||
|
||||
const detailsDialog = page.getByRole('dialog', { name: `${agentName} details` })
|
||||
|
||||
await expect(detailsDialog).toBeVisible()
|
||||
await expect(detailsDialog.getByText(agentName, { exact: true })).toBeVisible()
|
||||
if (agentRole)
|
||||
await expect(detailsDialog.getByText(agentRole, { exact: true })).toBeVisible()
|
||||
await expect(detailsDialog.getByText(stableModel.name, { exact: true })).toBeVisible()
|
||||
await expect(detailsDialog.getByText(normalAgentPrompt)).toBeVisible()
|
||||
await expect(detailsDialog.getByRole('link', { name: 'Edit in Agent Console' })).toHaveAttribute(
|
||||
'href',
|
||||
`/roster/agent/${this.createdAgentIds.at(-1)}/configure`,
|
||||
)
|
||||
},
|
||||
)
|
||||
|
||||
Then(
|
||||
'the Agent v2 Agent Console should open for the same workflow Agent',
|
||||
async function (this: DifyWorld) {
|
||||
const agentConsolePage = this.agentBuilder.workflow.agentConsolePage
|
||||
const agentId = this.createdAgentIds.at(-1)
|
||||
const agentName = this.lastCreatedAgentName
|
||||
const stableModel = this.agentBuilder.preflight.stableModel
|
||||
if (!agentConsolePage)
|
||||
throw new Error('Agent Console page was not opened.')
|
||||
if (!agentId || !agentName)
|
||||
throw new Error('No Agent v2 ID or name found. Create a workflow Agent v2 node first.')
|
||||
if (!stableModel)
|
||||
throw new Error('Stable chat model preflight must run before asserting Agent Console.')
|
||||
|
||||
await expect(agentConsolePage).toHaveURL(
|
||||
new RegExp(`/roster/agent/${agentId}/configure(?:\\?.*)?$`),
|
||||
)
|
||||
await expect(agentConsolePage.getByRole('heading', { name: 'Configure' })).toBeVisible({
|
||||
timeout: 30_000,
|
||||
})
|
||||
await expect(agentConsolePage.getByText(agentName, { exact: true })).toBeVisible()
|
||||
await expect(agentConsolePage.getByText(stableModel.name, { exact: true })).toBeVisible()
|
||||
await expect(agentConsolePage.getByText(normalAgentPrompt)).toBeVisible()
|
||||
|
||||
await agentConsolePage.close()
|
||||
this.agentBuilder.workflow.agentConsolePage = undefined
|
||||
},
|
||||
)
|
||||
@ -2,13 +2,14 @@ import type { DifyWorld } from '../../support/world'
|
||||
import { Then, When } from '@cucumber/cucumber'
|
||||
import { expect } from '@playwright/test'
|
||||
import { openBlankAppCreation } from '../../../support/apps'
|
||||
import { createE2EResourceName } from '../../../support/naming'
|
||||
|
||||
When('I start creating a blank app', async function (this: DifyWorld) {
|
||||
await openBlankAppCreation(this.getPage())
|
||||
})
|
||||
|
||||
When('I enter a unique E2E app name', async function (this: DifyWorld) {
|
||||
const appName = `E2E App ${Date.now()}`
|
||||
const appName = createE2EResourceName('App')
|
||||
this.lastCreatedAppName = appName
|
||||
await this.getPage().getByPlaceholder('Give your app a name').fill(appName)
|
||||
})
|
||||
|
||||
@ -2,9 +2,10 @@ import type { DifyWorld } from '../../support/world'
|
||||
import { Given, When } from '@cucumber/cucumber'
|
||||
import { expect } from '@playwright/test'
|
||||
import { createTestApp } from '../../../support/api'
|
||||
import { createE2EResourceName } from '../../../support/naming'
|
||||
|
||||
Given('there is an existing E2E app available for testing', async function (this: DifyWorld) {
|
||||
const name = `E2E Test App ${Date.now()}`
|
||||
const name = createE2EResourceName('App', 'Test')
|
||||
const app = await createTestApp(name, 'completion')
|
||||
this.lastCreatedAppName = app.name
|
||||
this.createdAppIds.push(app.id)
|
||||
|
||||
@ -7,6 +7,7 @@ import {
|
||||
publishWorkflowApp,
|
||||
syncRunnableWorkflowDraft,
|
||||
} from '../../../support/api'
|
||||
import { createE2EResourceName } from '../../../support/naming'
|
||||
|
||||
When('I enable the Web App share', async function (this: DifyWorld) {
|
||||
const page = this.getPage()
|
||||
@ -27,7 +28,7 @@ Then('the Web App should be in service', async function (this: DifyWorld) {
|
||||
})
|
||||
|
||||
Given('a workflow app has been published and shared via API', async function (this: DifyWorld) {
|
||||
const app = await createTestApp(`E2E Share ${Date.now()}`, 'workflow')
|
||||
const app = await createTestApp(createE2EResourceName('App', 'Share'), 'workflow')
|
||||
this.createdAppIds.push(app.id)
|
||||
this.lastCreatedAppName = app.name
|
||||
await syncRunnableWorkflowDraft(app.id)
|
||||
|
||||
@ -2,11 +2,12 @@ import type { DifyWorld } from '../../support/world'
|
||||
import { Given, Then, When } from '@cucumber/cucumber'
|
||||
import { expect } from '@playwright/test'
|
||||
import { createTestApp } from '../../../support/api'
|
||||
import { createE2EResourceName } from '../../../support/naming'
|
||||
|
||||
Given(
|
||||
'there is an existing E2E completion app available for testing',
|
||||
async function (this: DifyWorld) {
|
||||
const name = `E2E Test App ${Date.now()}`
|
||||
const name = createE2EResourceName('App', 'Test')
|
||||
const app = await createTestApp(name, 'completion')
|
||||
this.lastCreatedAppName = app.name
|
||||
this.createdAppIds.push(app.id)
|
||||
|
||||
@ -3,9 +3,10 @@ import { Given, When } from '@cucumber/cucumber'
|
||||
import { expect } from '@playwright/test'
|
||||
import { createTestApp, syncMinimalWorkflowDraft } from '../../../support/api'
|
||||
import { waitForAppsConsole } from '../../../support/apps'
|
||||
import { createE2EResourceName } from '../../../support/naming'
|
||||
|
||||
Given('a {string} app has been created via API', async function (this: DifyWorld, mode: string) {
|
||||
const app = await createTestApp(`E2E ${Date.now()}`, mode)
|
||||
const app = await createTestApp(createE2EResourceName('App', mode), mode)
|
||||
this.createdAppIds.push(app.id)
|
||||
this.lastCreatedAppName = app.name
|
||||
})
|
||||
|
||||
@ -7,6 +7,10 @@ When('I open the apps console', async function (this: DifyWorld) {
|
||||
await this.getPage().goto('/apps')
|
||||
})
|
||||
|
||||
When('I refresh the current page', async function (this: DifyWorld) {
|
||||
await this.getPage().reload()
|
||||
})
|
||||
|
||||
Then('I should stay on the apps console', async function (this: DifyWorld) {
|
||||
await waitForAppsConsole(this.getPage())
|
||||
})
|
||||
|
||||
@ -7,9 +7,12 @@ import { fileURLToPath } from 'node:url'
|
||||
import { After, AfterAll, Before, BeforeAll, setDefaultTimeout, Status } from '@cucumber/cucumber'
|
||||
import { chromium } from '@playwright/test'
|
||||
import { AUTH_BOOTSTRAP_TIMEOUT_MS, ensureAuthenticatedState } from '../../fixtures/auth'
|
||||
import { deleteTestAgent } from '../../support/agent'
|
||||
import { deleteTestApp } from '../../support/api'
|
||||
import { deleteTestDataset } from '../../support/datasets'
|
||||
import { deleteBuiltinToolCredential } from '../../support/tools'
|
||||
import { baseURL, cucumberHeadless, cucumberSlowMo } from '../../test-env'
|
||||
import { deleteTestAgent } from '../agent-v2/support/agent'
|
||||
import { deleteAgentConfigFile, deleteAgentConfigSkill, deleteAgentDriveFile } from '../agent-v2/support/agent-drive'
|
||||
|
||||
const e2eRoot = fileURLToPath(new URL('../..', import.meta.url))
|
||||
const artifactsDir = path.join(e2eRoot, 'cucumber-report', 'artifacts')
|
||||
@ -18,6 +21,14 @@ let browser: Browser | undefined
|
||||
|
||||
setDefaultTimeout(60_000)
|
||||
|
||||
const diagnosticArtifactStatuses = new Set([
|
||||
Status.FAILED,
|
||||
Status.AMBIGUOUS,
|
||||
Status.PENDING,
|
||||
Status.UNDEFINED,
|
||||
Status.UNKNOWN,
|
||||
])
|
||||
|
||||
const sanitizeForPath = (value: string) =>
|
||||
value.replaceAll(/[^\w-]+/g, '-').replaceAll(/^-+|-+$/g, '')
|
||||
|
||||
@ -35,6 +46,19 @@ const writeArtifact = async (
|
||||
return artifactPath
|
||||
}
|
||||
|
||||
const recordCleanup = async (
|
||||
errors: string[],
|
||||
label: string,
|
||||
cleanup: () => Promise<void>,
|
||||
) => {
|
||||
try {
|
||||
await cleanup()
|
||||
}
|
||||
catch (error) {
|
||||
errors.push(`${label}: ${error instanceof Error ? error.message : String(error)}`)
|
||||
}
|
||||
}
|
||||
|
||||
BeforeAll({ timeout: AUTH_BOOTSTRAP_TIMEOUT_MS }, async () => {
|
||||
await mkdir(artifactsDir, { recursive: true })
|
||||
|
||||
@ -65,8 +89,9 @@ Before(async function (this: DifyWorld, { pickle }) {
|
||||
|
||||
After(async function (this: DifyWorld, { pickle, result }) {
|
||||
const elapsedMs = this.scenarioStartedAt ? Date.now() - this.scenarioStartedAt : undefined
|
||||
const status = result?.status || Status.UNKNOWN
|
||||
|
||||
if (result?.status !== Status.PASSED && this.page) {
|
||||
if (diagnosticArtifactStatuses.has(status) && this.page) {
|
||||
const screenshot = await this.page.screenshot({
|
||||
fullPage: true,
|
||||
})
|
||||
@ -86,13 +111,40 @@ After(async function (this: DifyWorld, { pickle, result }) {
|
||||
this.attach(`Artifacts:\n${[screenshotPath, htmlPath].join('\n')}`, 'text/plain')
|
||||
}
|
||||
|
||||
const status = result?.status || 'UNKNOWN'
|
||||
console.warn(
|
||||
`[e2e] end ${pickle.name} status=${status}${elapsedMs ? ` durationMs=${elapsedMs}` : ''}`,
|
||||
)
|
||||
|
||||
for (const id of this.createdAgentIds) await deleteTestAgent(id).catch(() => {})
|
||||
for (const id of this.createdAppIds) await deleteTestApp(id).catch(() => {})
|
||||
const cleanupErrors: string[] = []
|
||||
|
||||
for (const skill of this.createdAgentConfigSkills.toReversed()) {
|
||||
await recordCleanup(cleanupErrors, `Delete Agent config skill ${skill.name}`, () =>
|
||||
deleteAgentConfigSkill(skill.agentId, skill.name))
|
||||
}
|
||||
for (const file of this.createdAgentConfigFiles.toReversed()) {
|
||||
await recordCleanup(cleanupErrors, `Delete Agent config file ${file.name}`, () =>
|
||||
deleteAgentConfigFile(file.agentId, file.name))
|
||||
}
|
||||
for (const file of this.createdAgentDriveFiles.toReversed()) {
|
||||
await recordCleanup(cleanupErrors, `Delete Agent drive file ${file.key}`, () =>
|
||||
deleteAgentDriveFile(file.agentId, file.key))
|
||||
}
|
||||
for (const id of this.createdAppIds)
|
||||
await recordCleanup(cleanupErrors, `Delete app ${id}`, () => deleteTestApp(id))
|
||||
for (const id of this.createdAgentIds)
|
||||
await recordCleanup(cleanupErrors, `Delete Agent ${id}`, () => deleteTestAgent(id))
|
||||
for (const id of this.createdDatasetIds)
|
||||
await recordCleanup(cleanupErrors, `Delete dataset ${id}`, () => deleteTestDataset(id))
|
||||
for (const credential of this.createdBuiltinToolCredentials.toReversed()) {
|
||||
await recordCleanup(
|
||||
cleanupErrors,
|
||||
`Delete builtin tool credential ${credential.provider}/${credential.credentialId}`,
|
||||
() => deleteBuiltinToolCredential(credential.provider, credential.credentialId),
|
||||
)
|
||||
}
|
||||
if (cleanupErrors.length > 0)
|
||||
this.attach(`Typed cleanup errors:\n${cleanupErrors.join('\n')}`, 'text/plain')
|
||||
await this.runRegisteredCleanups()
|
||||
|
||||
await this.closeSession()
|
||||
})
|
||||
|
||||
@ -5,6 +5,65 @@ import { setWorldConstructor, World } from '@cucumber/cucumber'
|
||||
import { authStatePath, readAuthSessionMetadata } from '../../fixtures/auth'
|
||||
import { baseURL, defaultLocale } from '../../test-env'
|
||||
|
||||
export type ScenarioCleanup = () => Promise<void> | void
|
||||
export type CreatedAgentDriveFile = {
|
||||
agentId: string
|
||||
key: string
|
||||
}
|
||||
export type CreatedAgentConfigFile = {
|
||||
agentId: string
|
||||
name: string
|
||||
}
|
||||
export type CreatedAgentConfigSkill = {
|
||||
agentId: string
|
||||
name: string
|
||||
}
|
||||
export type CreatedBuiltinToolCredential = {
|
||||
credentialId: string
|
||||
provider: string
|
||||
}
|
||||
export type AgentBuilderStableChatModel = {
|
||||
name: string
|
||||
provider: string
|
||||
type: string
|
||||
}
|
||||
export type AgentBuilderPreseededResource = {
|
||||
id: string
|
||||
kind: 'agent' | 'api-key' | 'dataset' | 'skill' | 'tool' | 'workflow'
|
||||
name: string
|
||||
}
|
||||
export type AgentV2WorkflowOutputVariable = {
|
||||
name: string
|
||||
type: string
|
||||
}
|
||||
|
||||
export const createAgentBuilderWorldState = () => ({
|
||||
preflight: {
|
||||
brokenModel: undefined as AgentBuilderStableChatModel | undefined,
|
||||
preseededResources: {} as Record<string, AgentBuilderPreseededResource>,
|
||||
stableModel: undefined as AgentBuilderStableChatModel | undefined,
|
||||
},
|
||||
accessPoint: {
|
||||
apiReferencePage: undefined as Page | undefined,
|
||||
composerDraftSnapshot: undefined as string | undefined,
|
||||
generatedApiKey: undefined as string | undefined,
|
||||
serviceApiResponse: undefined as { body: unknown, ok: boolean, status: number } | undefined,
|
||||
serviceApiBaseURL: undefined as string | undefined,
|
||||
webAppPage: undefined as Page | undefined,
|
||||
webAppURL: undefined as string | undefined,
|
||||
workflowReferencePage: undefined as Page | undefined,
|
||||
},
|
||||
configure: {
|
||||
concurrentPage: undefined as Page | undefined,
|
||||
},
|
||||
workflow: {
|
||||
agentConsolePage: undefined as Page | undefined,
|
||||
outputVariables: [] as AgentV2WorkflowOutputVariable[],
|
||||
},
|
||||
})
|
||||
|
||||
export type AgentBuilderWorldState = ReturnType<typeof createAgentBuilderWorldState>
|
||||
|
||||
export class DifyWorld extends World {
|
||||
context: BrowserContext | undefined
|
||||
page: Page | undefined
|
||||
@ -17,6 +76,13 @@ export class DifyWorld extends World {
|
||||
lastCreatedAgentRole: string | undefined
|
||||
createdAppIds: string[] = []
|
||||
createdAgentIds: string[] = []
|
||||
createdDatasetIds: string[] = []
|
||||
createdAgentConfigFiles: CreatedAgentConfigFile[] = []
|
||||
createdAgentConfigSkills: CreatedAgentConfigSkill[] = []
|
||||
createdAgentDriveFiles: CreatedAgentDriveFile[] = []
|
||||
createdBuiltinToolCredentials: CreatedBuiltinToolCredential[] = []
|
||||
agentBuilder: AgentBuilderWorldState = createAgentBuilderWorldState()
|
||||
scenarioCleanups: ScenarioCleanup[] = []
|
||||
capturedDownloads: Download[] = []
|
||||
shareURL: string | undefined
|
||||
|
||||
@ -33,6 +99,13 @@ export class DifyWorld extends World {
|
||||
this.lastCreatedAgentRole = undefined
|
||||
this.createdAppIds = []
|
||||
this.createdAgentIds = []
|
||||
this.createdDatasetIds = []
|
||||
this.createdAgentConfigFiles = []
|
||||
this.createdAgentConfigSkills = []
|
||||
this.createdAgentDriveFiles = []
|
||||
this.createdBuiltinToolCredentials = []
|
||||
this.agentBuilder = createAgentBuilderWorldState()
|
||||
this.scenarioCleanups = []
|
||||
this.capturedDownloads = []
|
||||
this.shareURL = undefined
|
||||
}
|
||||
@ -80,6 +153,26 @@ export class DifyWorld extends World {
|
||||
return this.session
|
||||
}
|
||||
|
||||
registerCleanup(cleanup: ScenarioCleanup) {
|
||||
this.scenarioCleanups.push(cleanup)
|
||||
}
|
||||
|
||||
async runRegisteredCleanups() {
|
||||
const errors: string[] = []
|
||||
|
||||
for (const cleanup of this.scenarioCleanups.toReversed()) {
|
||||
try {
|
||||
await cleanup()
|
||||
}
|
||||
catch (error) {
|
||||
errors.push(error instanceof Error ? error.message : String(error))
|
||||
}
|
||||
}
|
||||
|
||||
if (errors.length > 0)
|
||||
this.attach(`Cleanup errors:\n${errors.join('\n')}`, 'text/plain')
|
||||
}
|
||||
|
||||
async closeSession() {
|
||||
await this.context?.close()
|
||||
this.context = undefined
|
||||
|
||||
3
e2e/fixtures/test-materials/agent-build-instruction.txt
Normal file
3
e2e/fixtures/test-materials/agent-build-instruction.txt
Normal file
@ -0,0 +1,3 @@
|
||||
Ask the Agent to use e2e-summary-skill to summarize user input.
|
||||
When replacing JSON strings, use JSON Process / JSON Replace.
|
||||
After applying, include these capabilities in the Agent instructions.
|
||||
0
e2e/fixtures/test-materials/agent-empty-file.txt
Normal file
0
e2e/fixtures/test-materials/agent-empty-file.txt
Normal file
4
e2e/fixtures/test-materials/agent-invalid.env
Normal file
4
e2e/fixtures/test-materials/agent-invalid.env
Normal file
@ -0,0 +1,4 @@
|
||||
E2E_AGENT_FLAG=enabled
|
||||
INVALID ENV LINE WITHOUT EQUALS
|
||||
=missing_key
|
||||
E2E_AGENT_AFTER_INVALID=still-valid
|
||||
2
e2e/fixtures/test-materials/agent-knowledge-source.txt
Normal file
2
e2e/fixtures/test-materials/agent-knowledge-source.txt
Normal file
@ -0,0 +1,2 @@
|
||||
Dify Agent E2E knowledge source.
|
||||
Dify Agent E2E test passphrase is AGENT_KNOWLEDGE_PASS.
|
||||
2
e2e/fixtures/test-materials/agent-small-file.txt
Normal file
2
e2e/fixtures/test-materials/agent-small-file.txt
Normal file
@ -0,0 +1,2 @@
|
||||
Dify Agent E2E small file fixture.
|
||||
Expected token: AGENT_FILE_PASS
|
||||
@ -0,0 +1,2 @@
|
||||
Special filename fixture.
|
||||
Expected token: AGENT_SPECIAL_FILENAME_PASS
|
||||
1
e2e/fixtures/test-materials/agent-unsupported-file.exe
Normal file
1
e2e/fixtures/test-materials/agent-unsupported-file.exe
Normal file
@ -0,0 +1 @@
|
||||
This file intentionally uses an unsupported extension for upload validation.
|
||||
2
e2e/fixtures/test-materials/agent-valid.env
Normal file
2
e2e/fixtures/test-materials/agent-valid.env
Normal file
@ -0,0 +1,2 @@
|
||||
E2E_AGENT_FLAG=enabled
|
||||
E2E_AGENT_MODE=plain
|
||||
@ -0,0 +1 @@
|
||||
Batch 5 valid file 1 token E2E_BATCH_5_1
|
||||
@ -0,0 +1 @@
|
||||
Batch 5 valid file 2 token E2E_BATCH_5_2
|
||||
@ -0,0 +1 @@
|
||||
Batch 5 valid file 3 token E2E_BATCH_5_3
|
||||
@ -0,0 +1 @@
|
||||
Batch 5 valid file 4 token E2E_BATCH_5_4
|
||||
@ -0,0 +1 @@
|
||||
Batch 5 valid file 5 token E2E_BATCH_5_5
|
||||
@ -0,0 +1 @@
|
||||
Batch 6 valid file 1 token E2E_BATCH_6_1
|
||||
@ -0,0 +1 @@
|
||||
Batch 6 valid file 2 token E2E_BATCH_6_2
|
||||
@ -0,0 +1 @@
|
||||
Batch 6 valid file 3 token E2E_BATCH_6_3
|
||||
@ -0,0 +1 @@
|
||||
Batch 6 valid file 4 token E2E_BATCH_6_4
|
||||
@ -0,0 +1 @@
|
||||
Batch 6 valid file 5 token E2E_BATCH_6_5
|
||||
@ -0,0 +1 @@
|
||||
Batch 6 valid file 6 token E2E_BATCH_6_6
|
||||
@ -0,0 +1 @@
|
||||
Total 50 valid file 01 token E2E_TOTAL_50_01
|
||||
@ -0,0 +1 @@
|
||||
Total 50 valid file 02 token E2E_TOTAL_50_02
|
||||
@ -0,0 +1 @@
|
||||
Total 50 valid file 03 token E2E_TOTAL_50_03
|
||||
@ -0,0 +1 @@
|
||||
Total 50 valid file 04 token E2E_TOTAL_50_04
|
||||
@ -0,0 +1 @@
|
||||
Total 50 valid file 05 token E2E_TOTAL_50_05
|
||||
@ -0,0 +1 @@
|
||||
Total 50 valid file 06 token E2E_TOTAL_50_06
|
||||
@ -0,0 +1 @@
|
||||
Total 50 valid file 07 token E2E_TOTAL_50_07
|
||||
@ -0,0 +1 @@
|
||||
Total 50 valid file 08 token E2E_TOTAL_50_08
|
||||
@ -0,0 +1 @@
|
||||
Total 50 valid file 09 token E2E_TOTAL_50_09
|
||||
@ -0,0 +1 @@
|
||||
Total 50 valid file 10 token E2E_TOTAL_50_10
|
||||
@ -0,0 +1 @@
|
||||
Total 50 valid file 11 token E2E_TOTAL_50_11
|
||||
@ -0,0 +1 @@
|
||||
Total 50 valid file 12 token E2E_TOTAL_50_12
|
||||
@ -0,0 +1 @@
|
||||
Total 50 valid file 13 token E2E_TOTAL_50_13
|
||||
@ -0,0 +1 @@
|
||||
Total 50 valid file 14 token E2E_TOTAL_50_14
|
||||
@ -0,0 +1 @@
|
||||
Total 50 valid file 15 token E2E_TOTAL_50_15
|
||||
@ -0,0 +1 @@
|
||||
Total 50 valid file 16 token E2E_TOTAL_50_16
|
||||
@ -0,0 +1 @@
|
||||
Total 50 valid file 17 token E2E_TOTAL_50_17
|
||||
@ -0,0 +1 @@
|
||||
Total 50 valid file 18 token E2E_TOTAL_50_18
|
||||
@ -0,0 +1 @@
|
||||
Total 50 valid file 19 token E2E_TOTAL_50_19
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user