mirror of
https://github.com/langgenius/dify.git
synced 2026-06-08 09:27:39 +08:00
447 lines
18 KiB
TypeScript
447 lines
18 KiB
TypeScript
/**
|
|
* E2E: difyctl run app — basic app execution
|
|
*
|
|
* Test cases sourced from: Dify CLI Enhanced spec — Dify CLI/Run/Basic App Execution (4.1)
|
|
*
|
|
* Streaming output cases → run-app-streaming.e2e.ts
|
|
* Conversation mode cases → run-app-conversation.e2e.ts
|
|
*
|
|
* Staging app prerequisites (specified via DIFY_E2E_* env vars):
|
|
* echo-chat — mode=chat, query variable, outputs "echo: {query}"
|
|
* echo-workflow — mode=workflow, x variable (required), outputs "echo: {x}"
|
|
*/
|
|
|
|
import type { AuthFixture } from '../../helpers/cli.js'
|
|
import { mkdir, writeFile } from 'node:fs/promises'
|
|
import { join } from 'node:path'
|
|
import { afterEach, beforeEach, describe, expect, it } from 'vitest'
|
|
import {
|
|
assertErrorEnvelope,
|
|
assertExitCode,
|
|
assertJson,
|
|
assertNoAnsi,
|
|
assertPipeFriendlyJson,
|
|
assertStdoutContains,
|
|
} from '../../helpers/assert.js'
|
|
import { run, withAuthFixture, withTempConfig } from '../../helpers/cli.js'
|
|
import { withRetry } from '../../helpers/retry.js'
|
|
import { optionalIt } from '../../helpers/skip.js'
|
|
import { loadE2EEnv } from '../../setup/env.js'
|
|
|
|
const E = loadE2EEnv()
|
|
const itWithSso = optionalIt(Boolean(E.ssoToken))
|
|
|
|
// ── Suite ──────────────────────────────────────────────────────────────────
|
|
|
|
describe('E2E / difyctl run app', () => {
|
|
let fx: AuthFixture
|
|
|
|
beforeEach(async () => {
|
|
fx = await withAuthFixture(E)
|
|
})
|
|
afterEach(async () => {
|
|
await fx.cleanup()
|
|
})
|
|
|
|
// =========================================================================
|
|
// Basic execution
|
|
// =========================================================================
|
|
|
|
describe('Basic execution', () => {
|
|
it('[P0] logged-in internal user can run app — stdout contains the app result', async () => {
|
|
// Spec: logged-in internal user can run app / default output shows execution result
|
|
// withRetry: staging LLM inference may have transient 5xx on cold start
|
|
const result = await withRetry(() => fx.r(['run', 'app', E.chatAppId, 'hello']), {
|
|
attempts: 3,
|
|
delayMs: 2000,
|
|
shouldRetry: err => err instanceof Error && /5\d{2}|ECONNRESET|timeout/i.test(err.message),
|
|
})
|
|
assertExitCode(result, 0)
|
|
assertStdoutContains(result, 'echo:hello')
|
|
// Spec 4.1.4: default output has no ANSI colour codes (non-TTY; run() sets NO_COLOR=1)
|
|
assertNoAnsi(result.stdout, 'stdout')
|
|
})
|
|
|
|
it('[P0] run app invokes the execute endpoint (stdout has actual content)', async () => {
|
|
// Spec: run app invokes the execute endpoint
|
|
const result = await fx.r(['run', 'app', E.chatAppId, 'e2e-smoke'])
|
|
assertExitCode(result, 0)
|
|
expect(result.stdout.length).toBeGreaterThan(0)
|
|
})
|
|
|
|
it('[P1] text output preserves newlines (stdout ends with \\n)', async () => {
|
|
// Spec: text output preserves newlines
|
|
const result = await fx.r(['run', 'app', E.chatAppId, 'newline'])
|
|
assertExitCode(result, 0)
|
|
expect(result.stdout).toMatch(/\n$/)
|
|
})
|
|
|
|
it('[P1] repeated run app calls each complete independently (3 iterations)', async () => {
|
|
// Spec: repeated run app calls do not affect historical state
|
|
for (let i = 0; i < 3; i++) {
|
|
const result = await fx.r(['run', 'app', E.chatAppId, `repeat-${i}`])
|
|
assertExitCode(result, 0)
|
|
assertStdoutContains(result, `echo:repeat-${i}`)
|
|
}
|
|
})
|
|
})
|
|
|
|
// =========================================================================
|
|
// Output format
|
|
// =========================================================================
|
|
|
|
describe('Output format (-o)', () => {
|
|
it('[P0] -o json outputs valid JSON', async () => {
|
|
// Spec: -o json produces valid JSON
|
|
const result = await fx.r(['run', 'app', E.chatAppId, 'json-test', '-o', 'json'])
|
|
assertExitCode(result, 0)
|
|
const parsed = assertJson<{ answer: string, mode: string }>(result)
|
|
expect(parsed).toHaveProperty('answer')
|
|
expect(parsed.mode).toMatch(/chat/)
|
|
})
|
|
|
|
it('[P1] JSON output includes execution metadata (message_id / conversation_id)', async () => {
|
|
// Spec: JSON output includes execution metadata
|
|
const result = await fx.r(['run', 'app', E.chatAppId, 'meta', '-o', 'json'])
|
|
assertExitCode(result, 0)
|
|
const parsed = assertJson<Record<string, unknown>>(result)
|
|
expect(parsed).toHaveProperty('message_id')
|
|
expect(parsed).toHaveProperty('conversation_id')
|
|
})
|
|
|
|
it('[P1] JSON output supports piping (no ANSI, starts with {, ends with \\n)', async () => {
|
|
// Spec: JSON output supports piping
|
|
const result = await fx.r(['run', 'app', E.chatAppId, 'pipe', '-o', 'json'])
|
|
assertExitCode(result, 0)
|
|
assertPipeFriendlyJson(result)
|
|
})
|
|
|
|
it('[P1] JSON mode outputs a JSON error envelope to stderr', async () => {
|
|
// Spec: JSON mode outputs a JSON error envelope
|
|
const result = await fx.r(['run', 'app', 'app-nonexistent-xyz-e2e', 'hello', '-o', 'json'])
|
|
assertNonZeroExit(result)
|
|
assertErrorEnvelope(result, 'server_4xx_other')
|
|
})
|
|
})
|
|
|
|
// =========================================================================
|
|
// --inputs flag
|
|
// =========================================================================
|
|
|
|
describe('--inputs flag', () => {
|
|
it('[P0] run app supports --inputs (workflow app)', async () => {
|
|
// Spec: run app supports --inputs
|
|
// withRetry: staging workflow execution may have transient 5xx
|
|
const result = await withRetry(
|
|
() => fx.r(['run', 'app', E.workflowAppId, '--inputs', JSON.stringify({ x: 'workflow-val', num: 42, enum_var: 'A', paragraph: 'short text' })]),
|
|
{ attempts: 3, delayMs: 2000, shouldRetry: err => err instanceof Error && /5\d{2}|ECONNRESET|timeout/i.test(err.message) },
|
|
)
|
|
assertExitCode(result, 0)
|
|
assertStdoutContains(result, 'workflow-val')
|
|
})
|
|
|
|
it('[P0] multiple inputs take effect simultaneously', async () => {
|
|
// Spec: multiple --inputs entries take effect simultaneously
|
|
const result = await fx.r([
|
|
'run',
|
|
'app',
|
|
E.workflowAppId,
|
|
'--inputs',
|
|
JSON.stringify({ x: 'multi-test', num: 42, enum_var: 'A', paragraph: 'short text' }),
|
|
])
|
|
assertExitCode(result, 0)
|
|
})
|
|
|
|
it('[P0] invalid JSON for --inputs returns usage error (exit code 2)', async () => {
|
|
// Spec: missing required parameter / invalid input
|
|
const result = await fx.r(['run', 'app', E.workflowAppId, '--inputs', 'not-json'])
|
|
assertExitCode(result, 2)
|
|
expect(result.stderr).toMatch(/valid JSON/i)
|
|
})
|
|
|
|
it('[P0] JSON array for --inputs returns usage error', async () => {
|
|
const result = await fx.r(['run', 'app', E.workflowAppId, '--inputs', '[1,2,3]'])
|
|
assertExitCode(result, 2)
|
|
expect(result.stderr).toMatch(/JSON object/i)
|
|
})
|
|
|
|
it('[P0] --inputs and --inputs-file are mutually exclusive — returns usage error', async () => {
|
|
// Spec: mutually exclusive flags return a usage error
|
|
const inputsFile = join(fx.configDir, 'inputs.json')
|
|
await writeFile(inputsFile, JSON.stringify({ x: 'file-val' }))
|
|
const result = await fx.r([
|
|
'run',
|
|
'app',
|
|
E.workflowAppId,
|
|
'--inputs',
|
|
'{"x":"flag-val"}',
|
|
'--inputs-file',
|
|
inputsFile,
|
|
])
|
|
assertExitCode(result, 2)
|
|
expect(result.stderr).toMatch(/mutually exclusive/i)
|
|
})
|
|
|
|
it('[P0] positional message passed to workflow app returns usage error', async () => {
|
|
// Spec: execution fails when required positional parameter is missing (workflow)
|
|
const result = await fx.r(['run', 'app', E.workflowAppId, 'positional-msg'])
|
|
assertExitCode(result, 2)
|
|
expect(result.stderr).toMatch(/workflow apps do not accept a positional message/i)
|
|
})
|
|
|
|
it('[P0] --inputs-file reads JSON inputs from a file', async () => {
|
|
const inputsFile = join(fx.configDir, 'wf-inputs.json')
|
|
await writeFile(inputsFile, JSON.stringify({ x: 'from-file', num: 42, enum_var: 'A', paragraph: 'short text' }))
|
|
const result = await fx.r(['run', 'app', E.workflowAppId, '--inputs-file', inputsFile])
|
|
assertExitCode(result, 0)
|
|
assertStdoutContains(result, 'from-file')
|
|
})
|
|
|
|
it('[P0] required inputs missing causes execution failure (exit code non-zero)', async () => {
|
|
// Spec 4.1.11: workflow app fails when required inputs are not provided.
|
|
// Passing an empty object omits the required "x" field; the server
|
|
// returns a validation error and the CLI exits with a non-zero code.
|
|
const result = await fx.r(['run', 'app', E.workflowAppId, '--inputs', '{}'])
|
|
expect(result.exitCode).not.toBe(0)
|
|
expect(result.stderr.length).toBeGreaterThan(0)
|
|
})
|
|
|
|
it('[P0] paragraph input within limit succeeds; exceeding max_length returns error', async () => {
|
|
// Spec 4.1.19: paragraph input exceeding max_length (100) returns validation error
|
|
// App: basic_auto_test — variable "paragraph" (text-input, max_length=100, optional)
|
|
|
|
// ── Within limit: 50 chars ──────────────────────────────────────────
|
|
const shortResult = await fx.r([
|
|
'run',
|
|
'app',
|
|
E.workflowAppId,
|
|
'--inputs',
|
|
JSON.stringify({
|
|
x: 'hello',
|
|
num: 42,
|
|
enum_var: 'A',
|
|
paragraph: 'A'.repeat(50),
|
|
}),
|
|
])
|
|
assertExitCode(shortResult, 0)
|
|
|
|
// ── Exceeding limit: 101 chars ──────────────────────────────────────
|
|
const longResult = await fx.r([
|
|
'run',
|
|
'app',
|
|
E.workflowAppId,
|
|
'--inputs',
|
|
JSON.stringify({
|
|
x: 'hello',
|
|
num: 42,
|
|
enum_var: 'A',
|
|
paragraph: 'A'.repeat(101),
|
|
}),
|
|
])
|
|
expect(longResult.exitCode).not.toBe(0)
|
|
expect(longResult.stderr).toMatch(/paragraph.*less than 100|paragraph.*100 characters/i)
|
|
})
|
|
|
|
it('[P0] valid inputs of all types execute successfully; invalid typed/enum inputs return errors', async () => {
|
|
// Spec 4.1.17: non-typed input value returns a validation error
|
|
// Spec 4.1.18: invalid enum value returns a validation error
|
|
//
|
|
// App: basic_auto_test (DIFY_E2E_WORKFLOW_APP_ID)
|
|
// Input schema:
|
|
// x — text-input (required)
|
|
// num — number (required, Spec 4.1.17)
|
|
// enum_var — select (required, options: A/B/C, Spec 4.1.18)
|
|
|
|
// ── Happy path: all correct values ──────────────────────────────────
|
|
const happyResult = await fx.r([
|
|
'run',
|
|
'app',
|
|
E.workflowAppId,
|
|
'--inputs',
|
|
JSON.stringify({ x: 'hello', num: 42, enum_var: 'A', paragraph: 'short text' }),
|
|
])
|
|
assertExitCode(happyResult, 0)
|
|
assertStdoutContains(happyResult, 'echo:hello')
|
|
|
|
// ── 4.1.17: number field receives a string value ─────────────────────
|
|
const typedResult = await fx.r([
|
|
'run',
|
|
'app',
|
|
E.workflowAppId,
|
|
'--inputs',
|
|
JSON.stringify({ x: 'hello', num: 'not-a-number', enum_var: 'A' }),
|
|
])
|
|
expect(typedResult.exitCode).not.toBe(0)
|
|
expect(typedResult.stderr).toMatch(/num.*number|must be a valid number/i)
|
|
|
|
// ── 4.1.18: enum field receives a value outside the allowed options ──
|
|
const enumResult = await fx.r([
|
|
'run',
|
|
'app',
|
|
E.workflowAppId,
|
|
'--inputs',
|
|
JSON.stringify({ x: 'hello', num: 42, enum_var: 'invalid' }),
|
|
])
|
|
expect(enumResult.exitCode).not.toBe(0)
|
|
expect(enumResult.stderr).toMatch(/enum_var.*must be one of|one of the following/i)
|
|
})
|
|
})
|
|
|
|
// =========================================================================
|
|
// Error scenarios
|
|
// =========================================================================
|
|
|
|
describe('Error scenarios', () => {
|
|
it('[P0] non-existent app returns error — exit code 1', async () => {
|
|
// Spec 4.1.20: non-existent app returns an error with not-found message
|
|
// Spec 4.1.21: exit code is exactly 1
|
|
const result = await fx.r(['run', 'app', 'app-id-does-not-exist-e2e-xyz', 'hello'])
|
|
assertExitCode(result, 1)
|
|
expect(result.stderr).toMatch(/not.?found/i)
|
|
})
|
|
|
|
it('[P0] missing app id returns error (exit code 1 — CLI returns 1 for missing required arg)', async () => {
|
|
// Spec: missing app id returns a usage error
|
|
// Actual behaviour: CLI framework returns exit 1 (not 2) for missing required argument
|
|
const result = await fx.r(['run', 'app'])
|
|
assertExitCode(result, 1)
|
|
expect(result.stderr).toMatch(/missing required argument/i)
|
|
})
|
|
|
|
it('[P0] unauthenticated run app returns auth error (exit code 4)', async () => {
|
|
// Spec 4.1.22: unauthenticated run app returns auth error message
|
|
// Spec 4.1.23: exit code is exactly 4
|
|
const unauthTmp = await withTempConfig()
|
|
try {
|
|
const result = await run(['run', 'app', E.chatAppId, 'hello'], {
|
|
configDir: unauthTmp.configDir,
|
|
})
|
|
assertExitCode(result, 4)
|
|
expect(result.stderr).toMatch(/not.?logged.?in|auth.?login/i)
|
|
}
|
|
finally {
|
|
await unauthTmp.cleanup()
|
|
}
|
|
})
|
|
|
|
it('[P1] network error returns non-zero exit code and error message', async () => {
|
|
// Spec 4.1.26: when the host is unreachable the CLI returns a network error.
|
|
// Uses a local port that has nothing listening (127.0.0.1:19999) so the
|
|
// connection is refused immediately without waiting for DNS.
|
|
const networkTmp = await withTempConfig()
|
|
try {
|
|
await mkdir(networkTmp.configDir, { recursive: true })
|
|
const hostsYml = `${[
|
|
`current_host: http://127.0.0.1:19999`,
|
|
`token_storage: file`,
|
|
`tokens:`,
|
|
` bearer: dfoa_fake_token_network_test`,
|
|
`workspace:`,
|
|
` id: ${E.workspaceId}`,
|
|
` name: "E2E Test Workspace"`,
|
|
` role: owner`,
|
|
`available_workspaces:`,
|
|
` - id: ${E.workspaceId}`,
|
|
` name: "E2E Test Workspace"`,
|
|
` role: owner`,
|
|
].join('\n')}\n`
|
|
await writeFile(join(networkTmp.configDir, 'hosts.yml'), hostsYml, { mode: 0o600 })
|
|
const result = await run(
|
|
['run', 'app', E.chatAppId, 'hello'],
|
|
{ configDir: networkTmp.configDir, timeout: 15_000 },
|
|
)
|
|
expect(result.exitCode).not.toBe(0)
|
|
expect(result.stderr.length).toBeGreaterThan(0)
|
|
}
|
|
finally {
|
|
await networkTmp.cleanup()
|
|
}
|
|
})
|
|
})
|
|
|
|
// =========================================================================
|
|
// Non-interactive mode / CI environment
|
|
// =========================================================================
|
|
|
|
describe('Non-interactive mode (CI)', () => {
|
|
it('[P0] CI=1 environment has no spinner — stdout has no ANSI colour', async () => {
|
|
// Spec: ANSI colour is disabled in non-TTY environment; spinner is suppressed in non-interactive mode
|
|
const result = await fx.r(['run', 'app', E.chatAppId, 'ci-test'], { CI: '1', NO_COLOR: '1' })
|
|
assertExitCode(result, 0)
|
|
assertNoAnsi(result.stdout, 'stdout')
|
|
assertNoAnsi(result.stderr, 'stderr')
|
|
})
|
|
|
|
it('[P0] non-interactive mode exit code is correctly propagated', async () => {
|
|
// Spec: non-interactive mode exit code is correct
|
|
const result = await fx.r(['run', 'app', E.chatAppId, 'code'])
|
|
expect(typeof result.exitCode).toBe('number')
|
|
expect(result.exitCode).toBe(0)
|
|
})
|
|
})
|
|
|
|
// =========================================================================
|
|
// Workspace override
|
|
// =========================================================================
|
|
|
|
describe('workspace override', () => {
|
|
it('[P1] --workspace flag overrides the default workspace', async () => {
|
|
// Spec: workspace override takes effect
|
|
// run app uses --workspace (no -w short form)
|
|
const result = await fx.r([
|
|
'run',
|
|
'app',
|
|
E.chatAppId,
|
|
'ws-override',
|
|
'--workspace',
|
|
E.workspaceId,
|
|
])
|
|
assertExitCode(result, 0)
|
|
})
|
|
|
|
itWithSso('[P1] external SSO user: --workspace parameter is silently ignored', async () => {
|
|
// Spec 4.1.25: SSO subjects operate without workspace scoping.
|
|
// Passing --workspace must not change the outcome — the parameter
|
|
// should be ignored, so both calls produce the same exit code.
|
|
const ssoTmp = await withTempConfig()
|
|
try {
|
|
await mkdir(ssoTmp.configDir, { recursive: true })
|
|
const hostsYml = `${[
|
|
`current_host: ${E.host}`,
|
|
`token_storage: file`,
|
|
`tokens:`,
|
|
` bearer: ${E.ssoToken}`,
|
|
`external_subject:`,
|
|
` email: sso@example.com`,
|
|
` issuer: https://issuer.example.com`,
|
|
].join('\n')}\n`
|
|
await writeFile(join(ssoTmp.configDir, 'hosts.yml'), hostsYml, { mode: 0o600 })
|
|
|
|
// Run WITHOUT --workspace
|
|
const resultWithout = await run(
|
|
['run', 'app', E.chatAppId, 'hello'],
|
|
{ configDir: ssoTmp.configDir },
|
|
)
|
|
|
|
// Run WITH --workspace (should be ignored → same exit code)
|
|
const resultWith = await run(
|
|
['run', 'app', E.chatAppId, 'hello', '--workspace', E.workspaceId],
|
|
{ configDir: ssoTmp.configDir },
|
|
)
|
|
|
|
// If --workspace were honoured for SSO users it would change behaviour;
|
|
// identical exit codes confirm the parameter is silently ignored.
|
|
expect(resultWith.exitCode).toBe(resultWithout.exitCode)
|
|
}
|
|
finally {
|
|
await ssoTmp.cleanup()
|
|
}
|
|
})
|
|
})
|
|
})
|
|
|
|
// ── local helper (avoids import confusion) ─────────────────────────────────
|
|
function assertNonZeroExit(result: import('../../helpers/cli.js').RunResult): void {
|
|
expect(result.exitCode, 'exit code should be non-zero').not.toBe(0)
|
|
}
|