diff --git a/cli/test/e2e/.env.e2e.example b/cli/test/e2e/.env.e2e.example index 53fbedadf31..85f2219daeb 100644 --- a/cli/test/e2e/.env.e2e.example +++ b/cli/test/e2e/.env.e2e.example @@ -67,3 +67,15 @@ DIFY_E2E_PASSWORD= # DIFY_E2E_HITL_SINGLE_ACTION_APP_ID= # DIFY_E2E_HITL_MULTI_NODE_APP_ID= # DIFY_E2E_WS2_APP_ID= + +# ── Separated-mode reasoning suite (opt-in) ───────────────────────────────── +# run-app-reasoning.e2e.ts is skipped unless DIFY_E2E_REASONING_APP_ID resolves. +# It needs a chatflow whose LLM node uses reasoning_format=separated AND a +# workspace with a default chat model configured. +# +# Either point at an existing app: +# DIFY_E2E_REASONING_APP_ID= +# +# …or auto-provision reasoning-chat.yml (→ app name "reasoning-bot"). Off by +# default so the shared bootstrap stays free of any model dependency. +# DIFY_E2E_REASONING_PROVISION=1 diff --git a/cli/test/e2e/README.md b/cli/test/e2e/README.md index fd3e9182507..21635b0f6f9 100644 --- a/cli/test/e2e/README.md +++ b/cli/test/e2e/README.md @@ -39,11 +39,12 @@ test/e2e/ │ ├── describe-app.e2e.ts — describe app │ └── get-app-all-workspaces.e2e.ts — get app -A ([EE] multi-workspace cases) └── run/ - ├── run-app-basic.e2e.ts — basic run, -o json, --inputs, streaming, - │ conversation, CI mode - ├── run-app-streaming.e2e.ts — Ctrl+C / error-event / chunk timing - ├── run-app-file.e2e.ts — --file upload (local + remote URL) - └── run-app-hitl.e2e.ts — HITL pause + resume + ├── run-app-basic.e2e.ts — basic run, -o json, --inputs, streaming, + │ conversation, CI mode + ├── run-app-streaming.e2e.ts — Ctrl+C / error-event / chunk timing + ├── run-app-file.e2e.ts — --file upload (local + remote URL) + ├── run-app-reasoning.e2e.ts — separated-mode reasoning (--think); opt-in + └── run-app-hitl.e2e.ts — HITL pause + resume ``` ## Edition support @@ -137,6 +138,24 @@ global-setup will: | `DIFY_E2E_HITL_SINGLE_ACTION_APP_ID` | | | `DIFY_E2E_HITL_MULTI_NODE_APP_ID` | | | `DIFY_E2E_WS2_APP_ID` | Override secondary workspace app ID (EE) | +| `DIFY_E2E_REASONING_APP_ID` | separated-reasoning chatflow app ID (opt-in) | +| `DIFY_E2E_REASONING_PROVISION` | `1` → auto-provision `reasoning-chat.yml` | + +### Separated-mode reasoning suite (opt-in) + +`run-app-reasoning.e2e.ts` verifies the out-of-band `reasoning_chunk` channel +(PR #37460): `--think` surfaces the chain-of-thought to stderr framed as +`…`, the answer stays clean, and `-o json` persists it under +`metadata.reasoning`. It is **skipped** unless `DIFY_E2E_REASONING_APP_ID` +resolves, because it runs a real LLM node and needs: + +1. a chatflow whose LLM node uses `reasoning_format: separated`, and +1. a workspace with a default chat model configured. + +Point `DIFY_E2E_REASONING_APP_ID` at such an app, or set +`DIFY_E2E_REASONING_PROVISION=1` to import the `reasoning-chat.yml` fixture +(its system prompt forces a `` block, so any chat model triggers the +separated path — no dedicated reasoning model required). ## Running tests diff --git a/cli/test/e2e/fixtures/apps/reasoning-chat.yml b/cli/test/e2e/fixtures/apps/reasoning-chat.yml new file mode 100644 index 00000000000..ba24be57ef5 --- /dev/null +++ b/cli/test/e2e/fixtures/apps/reasoning-chat.yml @@ -0,0 +1,120 @@ +# Chatflow that exercises separated-mode reasoning (PR #37460): the LLM node sets +# reasoning_format=separated, so the server strips ... from the +# answer and streams the chain-of-thought on the out-of-band `reasoning_chunk` +# channel instead. The system prompt forces a block, so the separated +# path triggers with any chat model — no dedicated reasoning model required. +# +# NOTE: the LLM node leaves model.provider/name empty and relies on the target +# workspace's configured default chat model. The run-app-reasoning E2E suite is +# gated on DIFY_E2E_REASONING_APP_ID, so it is skipped unless a server with a +# working model is wired up. +app: + description: e2e-test reasoning (separated mode) + icon: 🧠 + icon_background: '#FFEAD5' + icon_type: emoji + mode: advanced-chat + name: reasoning-bot + use_icon_as_answer_icon: false +dependencies: [] +kind: app +version: 0.6.0 +workflow: + conversation_variables: [] + environment_variables: [] + features: + file_upload: {} + opening_statement: '' + retriever_resource: + enabled: true + sensitive_word_avoidance: + enabled: false + speech_to_text: + enabled: false + suggested_questions: [] + suggested_questions_after_answer: + enabled: false + text_to_speech: + enabled: false + language: '' + voice: '' + graph: + edges: + - id: start-llm + source: '1755189262236' + sourceHandle: source + target: llm + targetHandle: target + - id: llm-answer + source: llm + sourceHandle: source + target: answer + targetHandle: target + nodes: + - data: + desc: '' + title: Start + type: start + variables: [] + id: '1755189262236' + position: + x: 80 + y: 282 + sourcePosition: right + targetPosition: left + type: custom + - data: + context: + enabled: false + variable_selector: [] + desc: '' + memory: + query_prompt_template: '{{#sys.query#}}' + window: + enabled: false + size: 10 + model: + completion_params: + temperature: 0.7 + mode: chat + name: '' + provider: '' + prompt_template: + - role: system + text: >- + You are a helpful assistant. Always reason step by step INSIDE a + single ... block first, then write the final + answer AFTER the closing tag. The final answer must not + contain any tags. + reasoning_format: separated + selected: false + title: LLM + type: llm + variables: [] + vision: + enabled: false + id: llm + position: + x: 380 + y: 282 + sourcePosition: right + targetPosition: left + type: custom + - data: + answer: '{{#llm.text#}}' + desc: '' + title: Answer + type: answer + variables: [] + id: answer + position: + x: 680 + y: 282 + sourcePosition: right + targetPosition: left + type: custom + viewport: + x: 0 + y: 0 + zoom: 1 + rag_pipeline_variables: [] diff --git a/cli/test/e2e/setup/env.ts b/cli/test/e2e/setup/env.ts index 3f71b514426..33f8361d46b 100644 --- a/cli/test/e2e/setup/env.ts +++ b/cli/test/e2e/setup/env.ts @@ -37,6 +37,9 @@ * DIFY_E2E_HITL_EXTERNAL_APP_ID * DIFY_E2E_HITL_SINGLE_ACTION_APP_ID * DIFY_E2E_HITL_MULTI_NODE_APP_ID + * DIFY_E2E_REASONING_APP_ID Override separated-reasoning chatflow app ID + * DIFY_E2E_REASONING_PROVISION=1 Opt in to auto-provisioning reasoning-chat.yml + * (needs a workspace default chat model) */ /** Supported edition values. */ @@ -74,6 +77,12 @@ export type E2EEnv = { fileAppId: string /** Chat app (advanced-chat) with a file input variable */ fileChatAppId: string + /** + * Chatflow whose LLM node uses reasoning_format=separated. Empty unless + * DIFY_E2E_REASONING_APP_ID is set or the fixture is auto-provisioned; the + * run-app-reasoning suite is skipped when empty. + */ + reasoningAppId: string /** * Secondary workspace ID — EE only ("auto_test1"). * Empty in CE mode (CE has a single workspace). @@ -118,6 +127,7 @@ export type E2ECapabilities = { workflowAppId: string fileAppId: string fileChatAppId: string + reasoningAppId: string hitlAppId: string hitlExternalAppId: string hitlSingleActionAppId: string @@ -171,6 +181,7 @@ export function loadE2EEnv(): E2EEnv { hitlMultiNodeAppId: process.env.DIFY_E2E_HITL_MULTI_NODE_APP_ID ?? '', fileAppId: process.env.DIFY_E2E_FILE_APP_ID ?? '', fileChatAppId: process.env.DIFY_E2E_FILE_CHAT_APP_ID ?? '', + reasoningAppId: process.env.DIFY_E2E_REASONING_APP_ID ?? '', ws2Id: process.env.DIFY_E2E_WS2_ID ?? '', ws2AppId: process.env.DIFY_E2E_WS2_APP_ID ?? '', email: process.env.DIFY_E2E_EMAIL!, @@ -206,6 +217,7 @@ export function resolveEnv(caps: E2ECapabilities | undefined): E2EEnv { workflowAppId: caps.workflowAppId || env.workflowAppId, fileAppId: caps.fileAppId || env.fileAppId, fileChatAppId: caps.fileChatAppId || env.fileChatAppId, + reasoningAppId: caps.reasoningAppId || env.reasoningAppId, hitlAppId: caps.hitlAppId || env.hitlAppId, hitlExternalAppId: caps.hitlExternalAppId || env.hitlExternalAppId, hitlSingleActionAppId: caps.hitlSingleActionAppId || env.hitlSingleActionAppId, diff --git a/cli/test/e2e/setup/global-setup.ts b/cli/test/e2e/setup/global-setup.ts index 20b23295acb..92cc29fc1f2 100644 --- a/cli/test/e2e/setup/global-setup.ts +++ b/cli/test/e2e/setup/global-setup.ts @@ -182,6 +182,7 @@ export async function setup(project: TestProject): Promise { workflowAppId: '', fileAppId: '', fileChatAppId: '', + reasoningAppId: '', hitlAppId: '', hitlExternalAppId: '', hitlSingleActionAppId: '', @@ -288,6 +289,7 @@ export async function setup(project: TestProject): Promise { workflowAppId: provisionedIds.DIFY_E2E_WORKFLOW_APP_ID || E.workflowAppId, fileAppId: provisionedIds.DIFY_E2E_FILE_APP_ID || E.fileAppId, fileChatAppId: provisionedIds.DIFY_E2E_FILE_CHAT_APP_ID || E.fileChatAppId, + reasoningAppId: provisionedIds.DIFY_E2E_REASONING_APP_ID || E.reasoningAppId, hitlAppId: provisionedIds.DIFY_E2E_HITL_APP_ID || E.hitlAppId, hitlExternalAppId: provisionedIds.DIFY_E2E_HITL_EXTERNAL_APP_ID || E.hitlExternalAppId, hitlSingleActionAppId: provisionedIds.DIFY_E2E_HITL_SINGLE_ACTION_APP_ID || E.hitlSingleActionAppId, @@ -503,6 +505,12 @@ async function provisionApps( ['hitl-single-action.yml', 'DIFY_E2E_HITL_SINGLE_ACTION_APP_ID', primaryWsId], ['hitl-multi-node.yml', 'DIFY_E2E_HITL_MULTI_NODE_APP_ID', primaryWsId], ['file-chat.yml', 'DIFY_E2E_FILE_CHAT_APP_ID', primaryWsId], + // reasoning-chat.yml runs a real LLM node, so it is opt-in: provisioning it + // requires the workspace to have a default chat model configured. Off by + // default to keep the shared bootstrap free of any model dependency. + ...(process.env.DIFY_E2E_REASONING_PROVISION === '1' + ? [['reasoning-chat.yml', 'DIFY_E2E_REASONING_APP_ID', primaryWsId] as [string, string, string]] + : []), ...(edition === 'ee' ? [['ws2-workflow.yml', 'DIFY_E2E_WS2_APP_ID', secondaryWsId] as [string, string, string]] : []), diff --git a/cli/test/e2e/suites/run/run-app-reasoning.e2e.ts b/cli/test/e2e/suites/run/run-app-reasoning.e2e.ts new file mode 100644 index 00000000000..fd5f12e0b57 --- /dev/null +++ b/cli/test/e2e/suites/run/run-app-reasoning.e2e.ts @@ -0,0 +1,91 @@ +/** + * E2E: difyctl run app — separated-mode reasoning (PR #37460) + * + * Exercises the out-of-band `reasoning_chunk` SSE channel against a real server. + * Requires a chatflow whose LLM node uses reasoning_format=separated AND a + * workspace with a configured chat model. The whole suite is skipped unless + * DIFY_E2E_REASONING_APP_ID resolves (set it directly, or provision the + * reasoning-chat.yml fixture with DIFY_E2E_REASONING_PROVISION=1). + * + * Verifies the client adaptation: + * - --think surfaces the separated reasoning to stderr, framed as … + * - the answer (stdout) stays free of + * - -o json persists the reasoning under metadata.reasoning + * - without --think, reasoning stays hidden + */ + +import type { AuthFixture } from '../../helpers/cli.js' +import { afterEach, beforeEach, describe, expect, inject } from 'vitest' +import { assertExitCode, assertJson, assertStderrContains } from '../../helpers/assert.js' +import { registerConversation } from '../../helpers/cleanup-registry.js' +import { withAuthFixture } from '../../helpers/cli.js' +import { withRetry } from '../../helpers/retry.js' +import { optionalIt } from '../../helpers/skip.js' +import { resolveEnv } from '../../setup/env.js' + +// @ts-expect-error — see test/e2e/helpers/vitest-context.ts for explanation +const caps = inject('e2eCapabilities') as import('../../setup/env.js').E2ECapabilities +const E = resolveEnv(caps) + +// Skipped unless a separated-reasoning chatflow is wired up (needs a real model). +const reasoningIt = optionalIt(Boolean(E.reasoningAppId)) + +const QUERY = 'In one short sentence, why is the sky blue?' + +describe('E2E / difyctl run app — separated reasoning', () => { + let fx: AuthFixture + + beforeEach(async () => { + fx = await withAuthFixture(E) + }) + afterEach(async () => { + await fx.cleanup() + }) + + reasoningIt('[P1] --think --stream surfaces reasoning on stderr, clean answer on stdout', async () => { + const result = await withRetry( + () => fx.r(['run', 'app', E.reasoningAppId, QUERY, '--think', '--stream']), + { attempts: 3, delayMs: 1000 }, + ) + + assertExitCode(result, 0) + expect(result.stdout.trim().length).toBeGreaterThan(0) + // Separated mode keeps the answer free of ; reasoning is framed on stderr. + expect(result.stdout).not.toContain('') + assertStderrContains(result, '') + }) + + reasoningIt('[P1] --think -o json persists reasoning under metadata.reasoning', async () => { + const result = await withRetry( + () => fx.r(['run', 'app', E.reasoningAppId, QUERY, '--think', '-o', 'json']), + { attempts: 3, delayMs: 1000 }, + ) + + assertExitCode(result, 0) + const parsed = assertJson<{ + conversation_id?: string + answer: string + metadata?: { reasoning?: Record } + }>(result) + + if (parsed.conversation_id) + registerConversation(E.host, E.token, E.reasoningAppId, parsed.conversation_id) + + const reasoning = parsed.metadata?.reasoning ?? {} + expect(Object.keys(reasoning).length).toBeGreaterThan(0) + expect(Object.values(reasoning).join('').length).toBeGreaterThan(0) + // --think also echoes the separated reasoning to stderr. + assertStderrContains(result, '') + }) + + reasoningIt('[P1] without --think, reasoning stays hidden', async () => { + const result = await withRetry( + () => fx.r(['run', 'app', E.reasoningAppId, QUERY, '--stream']), + { attempts: 3, delayMs: 1000 }, + ) + + assertExitCode(result, 0) + expect(result.stdout.trim().length).toBeGreaterThan(0) + expect(result.stderr).not.toContain('') + }) +})