refactor: how to add a node type in workflow (#558)
This commit is contained in:
shentongmartin
GitHub
@ -33,6 +33,8 @@ import (
|
||||
"github.com/coze-dev/coze-studio/backend/domain/workflow/entity/vo"
|
||||
"github.com/coze-dev/coze-studio/backend/domain/workflow/internal/execute"
|
||||
"github.com/coze-dev/coze-studio/backend/domain/workflow/internal/nodes"
|
||||
schema2 "github.com/coze-dev/coze-studio/backend/domain/workflow/internal/schema"
|
||||
"github.com/coze-dev/coze-studio/backend/pkg/ctxcache"
|
||||
"github.com/coze-dev/coze-studio/backend/pkg/errorx"
|
||||
"github.com/coze-dev/coze-studio/backend/pkg/logs"
|
||||
"github.com/coze-dev/coze-studio/backend/pkg/safego"
|
||||
@ -48,7 +50,6 @@ type nodeRunConfig[O any] struct {
|
||||
maxRetry int64
|
||||
errProcessType vo.ErrorProcessType
|
||||
dataOnErr func(ctx context.Context) map[string]any
|
||||
callbackEnabled bool
|
||||
preProcessors []func(ctx context.Context, input map[string]any) (map[string]any, error)
|
||||
postProcessors []func(ctx context.Context, input map[string]any) (map[string]any, error)
|
||||
streamPreProcessors []func(ctx context.Context,
|
||||
@ -58,12 +59,14 @@ type nodeRunConfig[O any] struct {
|
||||
init []func(context.Context) (context.Context, error)
|
||||
i compose.Invoke[map[string]any, map[string]any, O]
|
||||
s compose.Stream[map[string]any, map[string]any, O]
|
||||
c compose.Collect[map[string]any, map[string]any, O]
|
||||
t compose.Transform[map[string]any, map[string]any, O]
|
||||
}
|
||||
|
||||
func newNodeRunConfig[O any](ns *NodeSchema,
|
||||
func newNodeRunConfig[O any](ns *schema2.NodeSchema,
|
||||
i compose.Invoke[map[string]any, map[string]any, O],
|
||||
s compose.Stream[map[string]any, map[string]any, O],
|
||||
c compose.Collect[map[string]any, map[string]any, O],
|
||||
t compose.Transform[map[string]any, map[string]any, O],
|
||||
opts *newNodeOptions) *nodeRunConfig[O] {
|
||||
meta := entity.NodeMetaByNodeType(ns.Type)
|
||||
@ -92,12 +95,12 @@ func newNodeRunConfig[O any](ns *NodeSchema,
|
||||
keyFinishedMarkerTrimmer(),
|
||||
}
|
||||
if meta.PreFillZero {
|
||||
preProcessors = append(preProcessors, ns.inputValueFiller())
|
||||
preProcessors = append(preProcessors, inputValueFiller(ns))
|
||||
}
|
||||
|
||||
var postProcessors []func(ctx context.Context, input map[string]any) (map[string]any, error)
|
||||
if meta.PostFillNil {
|
||||
postProcessors = append(postProcessors, ns.outputValueFiller())
|
||||
postProcessors = append(postProcessors, outputValueFiller(ns))
|
||||
}
|
||||
|
||||
streamPreProcessors := []func(ctx context.Context,
|
||||
@ -110,7 +113,15 @@ func newNodeRunConfig[O any](ns *NodeSchema,
|
||||
},
|
||||
}
|
||||
if meta.PreFillZero {
|
||||
streamPreProcessors = append(streamPreProcessors, ns.streamInputValueFiller())
|
||||
streamPreProcessors = append(streamPreProcessors, streamInputValueFiller(ns))
|
||||
}
|
||||
|
||||
if meta.UseCtxCache {
|
||||
opts.init = append([]func(ctx context.Context) (context.Context, error){
|
||||
func(ctx context.Context) (context.Context, error) {
|
||||
return ctxcache.Init(ctx), nil
|
||||
},
|
||||
}, opts.init...)
|
||||
}
|
||||
|
||||
opts.init = append(opts.init, func(ctx context.Context) (context.Context, error) {
|
||||
@ -129,7 +140,6 @@ func newNodeRunConfig[O any](ns *NodeSchema,
|
||||
maxRetry: maxRetry,
|
||||
errProcessType: errProcessType,
|
||||
dataOnErr: dataOnErr,
|
||||
callbackEnabled: meta.CallbackEnabled,
|
||||
preProcessors: preProcessors,
|
||||
postProcessors: postProcessors,
|
||||
streamPreProcessors: streamPreProcessors,
|
||||
@ -138,18 +148,21 @@ func newNodeRunConfig[O any](ns *NodeSchema,
|
||||
init: opts.init,
|
||||
i: i,
|
||||
s: s,
|
||||
c: c,
|
||||
t: t,
|
||||
}
|
||||
}
|
||||
|
||||
func newNodeRunConfigWOOpt(ns *NodeSchema,
|
||||
func newNodeRunConfigWOOpt(ns *schema2.NodeSchema,
|
||||
i compose.InvokeWOOpt[map[string]any, map[string]any],
|
||||
s compose.StreamWOOpt[map[string]any, map[string]any],
|
||||
c compose.CollectWOOpt[map[string]any, map[string]any],
|
||||
t compose.TransformWOOpts[map[string]any, map[string]any],
|
||||
opts *newNodeOptions) *nodeRunConfig[any] {
|
||||
var (
|
||||
iWO compose.Invoke[map[string]any, map[string]any, any]
|
||||
sWO compose.Stream[map[string]any, map[string]any, any]
|
||||
cWO compose.Collect[map[string]any, map[string]any, any]
|
||||
tWO compose.Transform[map[string]any, map[string]any, any]
|
||||
)
|
||||
|
||||
@ -165,13 +178,19 @@ func newNodeRunConfigWOOpt(ns *NodeSchema,
|
||||
}
|
||||
}
|
||||
|
||||
if c != nil {
|
||||
cWO = func(ctx context.Context, in *schema.StreamReader[map[string]any], _ ...any) (out map[string]any, err error) {
|
||||
return c(ctx, in)
|
||||
}
|
||||
}
|
||||
|
||||
if t != nil {
|
||||
tWO = func(ctx context.Context, input *schema.StreamReader[map[string]any], opts ...any) (output *schema.StreamReader[map[string]any], err error) {
|
||||
return t(ctx, input)
|
||||
}
|
||||
}
|
||||
|
||||
return newNodeRunConfig[any](ns, iWO, sWO, tWO, opts)
|
||||
return newNodeRunConfig[any](ns, iWO, sWO, cWO, tWO, opts)
|
||||
}
|
||||
|
||||
type newNodeOptions struct {
|
||||
@ -180,57 +199,100 @@ type newNodeOptions struct {
|
||||
init []func(context.Context) (context.Context, error)
|
||||
}
|
||||
|
||||
type newNodeOption func(*newNodeOptions)
|
||||
func toNode(ns *schema2.NodeSchema, r any) *Node {
|
||||
iWOpt, _ := r.(nodes.InvokableNodeWOpt)
|
||||
sWOpt, _ := r.(nodes.StreamableNodeWOpt)
|
||||
cWOpt, _ := r.(nodes.CollectableNodeWOpt)
|
||||
tWOpt, _ := r.(nodes.TransformableNodeWOpt)
|
||||
iWOOpt, _ := r.(nodes.InvokableNode)
|
||||
sWOOpt, _ := r.(nodes.StreamableNode)
|
||||
cWOOpt, _ := r.(nodes.CollectableNode)
|
||||
tWOOpt, _ := r.(nodes.TransformableNode)
|
||||
|
||||
func withCallbackInputConverter(f func(context.Context, map[string]any) (map[string]any, error)) newNodeOption {
|
||||
return func(opts *newNodeOptions) {
|
||||
opts.callbackInputConverter = f
|
||||
var wOpt, wOOpt bool
|
||||
if iWOpt != nil || sWOpt != nil || cWOpt != nil || tWOpt != nil {
|
||||
wOpt = true
|
||||
}
|
||||
}
|
||||
func withCallbackOutputConverter(f func(context.Context, map[string]any) (*nodes.StructuredCallbackOutput, error)) newNodeOption {
|
||||
return func(opts *newNodeOptions) {
|
||||
opts.callbackOutputConverter = f
|
||||
if iWOOpt != nil || sWOOpt != nil || cWOOpt != nil || tWOOpt != nil {
|
||||
wOOpt = true
|
||||
}
|
||||
|
||||
if wOpt && wOOpt {
|
||||
panic("a node's different streaming methods needs to be consistent: " +
|
||||
"they should ALL have NodeOption or None should have them")
|
||||
}
|
||||
|
||||
if !wOpt && !wOOpt {
|
||||
panic("a node should implement at least one interface among: InvokableNodeWOpt, StreamableNodeWOpt, CollectableNodeWOpt, TransformableNodeWOpt, InvokableNode, StreamableNode, CollectableNode, TransformableNode")
|
||||
}
|
||||
}
|
||||
func withInit(f func(context.Context) (context.Context, error)) newNodeOption {
|
||||
return func(opts *newNodeOptions) {
|
||||
opts.init = append(opts.init, f)
|
||||
}
|
||||
}
|
||||
|
||||
func invokableNode(ns *NodeSchema, i compose.InvokeWOOpt[map[string]any, map[string]any], opts ...newNodeOption) *Node {
|
||||
options := &newNodeOptions{}
|
||||
for _, opt := range opts {
|
||||
opt(options)
|
||||
ci, ok := r.(nodes.CallbackInputConverted)
|
||||
if ok {
|
||||
options.callbackInputConverter = ci.ToCallbackInput
|
||||
}
|
||||
|
||||
return newNodeRunConfigWOOpt(ns, i, nil, nil, options).toNode()
|
||||
}
|
||||
|
||||
func invokableNodeWO[O any](ns *NodeSchema, i compose.Invoke[map[string]any, map[string]any, O], opts ...newNodeOption) *Node {
|
||||
options := &newNodeOptions{}
|
||||
for _, opt := range opts {
|
||||
opt(options)
|
||||
co, ok := r.(nodes.CallbackOutputConverted)
|
||||
if ok {
|
||||
options.callbackOutputConverter = co.ToCallbackOutput
|
||||
}
|
||||
|
||||
return newNodeRunConfig(ns, i, nil, nil, options).toNode()
|
||||
}
|
||||
|
||||
func invokableTransformableNode(ns *NodeSchema, i compose.InvokeWOOpt[map[string]any, map[string]any],
|
||||
t compose.TransformWOOpts[map[string]any, map[string]any], opts ...newNodeOption) *Node {
|
||||
options := &newNodeOptions{}
|
||||
for _, opt := range opts {
|
||||
opt(options)
|
||||
init, ok := r.(nodes.Initializer)
|
||||
if ok {
|
||||
options.init = append(options.init, init.Init)
|
||||
}
|
||||
return newNodeRunConfigWOOpt(ns, i, nil, t, options).toNode()
|
||||
}
|
||||
|
||||
func invokableStreamableNodeWO[O any](ns *NodeSchema, i compose.Invoke[map[string]any, map[string]any, O], s compose.Stream[map[string]any, map[string]any, O], opts ...newNodeOption) *Node {
|
||||
options := &newNodeOptions{}
|
||||
for _, opt := range opts {
|
||||
opt(options)
|
||||
if wOpt {
|
||||
var (
|
||||
i compose.Invoke[map[string]any, map[string]any, nodes.NodeOption]
|
||||
s compose.Stream[map[string]any, map[string]any, nodes.NodeOption]
|
||||
c compose.Collect[map[string]any, map[string]any, nodes.NodeOption]
|
||||
t compose.Transform[map[string]any, map[string]any, nodes.NodeOption]
|
||||
)
|
||||
|
||||
if iWOpt != nil {
|
||||
i = iWOpt.Invoke
|
||||
}
|
||||
|
||||
if sWOpt != nil {
|
||||
s = sWOpt.Stream
|
||||
}
|
||||
|
||||
if cWOpt != nil {
|
||||
c = cWOpt.Collect
|
||||
}
|
||||
|
||||
if tWOpt != nil {
|
||||
t = tWOpt.Transform
|
||||
}
|
||||
|
||||
return newNodeRunConfig(ns, i, s, c, t, options).toNode()
|
||||
}
|
||||
return newNodeRunConfig(ns, i, s, nil, options).toNode()
|
||||
|
||||
var (
|
||||
i compose.InvokeWOOpt[map[string]any, map[string]any]
|
||||
s compose.StreamWOOpt[map[string]any, map[string]any]
|
||||
c compose.CollectWOOpt[map[string]any, map[string]any]
|
||||
t compose.TransformWOOpts[map[string]any, map[string]any]
|
||||
)
|
||||
|
||||
if iWOOpt != nil {
|
||||
i = iWOOpt.Invoke
|
||||
}
|
||||
|
||||
if sWOOpt != nil {
|
||||
s = sWOOpt.Stream
|
||||
}
|
||||
|
||||
if cWOOpt != nil {
|
||||
c = cWOOpt.Collect
|
||||
}
|
||||
|
||||
if tWOOpt != nil {
|
||||
t = tWOOpt.Transform
|
||||
}
|
||||
|
||||
return newNodeRunConfigWOOpt(ns, i, s, c, t, options).toNode()
|
||||
}
|
||||
|
||||
func (nc *nodeRunConfig[O]) invoke() func(ctx context.Context, input map[string]any, opts ...O) (output map[string]any, err error) {
|
||||
@ -375,10 +437,8 @@ func (nc *nodeRunConfig[O]) transform() func(ctx context.Context, input *schema.
|
||||
func (nc *nodeRunConfig[O]) toNode() *Node {
|
||||
var opts []compose.LambdaOpt
|
||||
opts = append(opts, compose.WithLambdaType(string(nc.nodeType)))
|
||||
opts = append(opts, compose.WithLambdaCallbackEnable(true))
|
||||
|
||||
if nc.callbackEnabled {
|
||||
opts = append(opts, compose.WithLambdaCallbackEnable(true))
|
||||
}
|
||||
l, err := compose.AnyLambda(nc.invoke(), nc.stream(), nil, nc.transform(), opts...)
|
||||
if err != nil {
|
||||
panic(fmt.Sprintf("failed to create lambda for node %s, err: %v", nc.nodeName, err))
|
||||
@ -406,9 +466,6 @@ func newNodeRunner[O any](ctx context.Context, cfg *nodeRunConfig[O]) (context.C
|
||||
}
|
||||
|
||||
func (r *nodeRunner[O]) onStart(ctx context.Context, input map[string]any) (context.Context, error) {
|
||||
if !r.callbackEnabled {
|
||||
return ctx, nil
|
||||
}
|
||||
if r.callbackInputConverter != nil {
|
||||
convertedInput, err := r.callbackInputConverter(ctx, input)
|
||||
if err != nil {
|
||||
@ -425,10 +482,6 @@ func (r *nodeRunner[O]) onStart(ctx context.Context, input map[string]any) (cont
|
||||
|
||||
func (r *nodeRunner[O]) onStartStream(ctx context.Context, input *schema.StreamReader[map[string]any]) (
|
||||
context.Context, *schema.StreamReader[map[string]any], error) {
|
||||
if !r.callbackEnabled {
|
||||
return ctx, input, nil
|
||||
}
|
||||
|
||||
if r.callbackInputConverter != nil {
|
||||
copied := input.Copy(2)
|
||||
realConverter := func(ctx context.Context) func(map[string]any) (map[string]any, error) {
|
||||
@ -580,14 +633,10 @@ func (r *nodeRunner[O]) transform(ctx context.Context, input *schema.StreamReade
|
||||
}
|
||||
|
||||
func (r *nodeRunner[O]) onEnd(ctx context.Context, output map[string]any) error {
|
||||
if r.errProcessType == vo.ErrorProcessTypeExceptionBranch || r.errProcessType == vo.ErrorProcessTypeDefault {
|
||||
if r.errProcessType == vo.ErrorProcessTypeExceptionBranch || r.errProcessType == vo.ErrorProcessTypeReturnDefaultData {
|
||||
output["isSuccess"] = true
|
||||
}
|
||||
|
||||
if !r.callbackEnabled {
|
||||
return nil
|
||||
}
|
||||
|
||||
if r.callbackOutputConverter != nil {
|
||||
convertedOutput, err := r.callbackOutputConverter(ctx, output)
|
||||
if err != nil {
|
||||
@ -603,15 +652,11 @@ func (r *nodeRunner[O]) onEnd(ctx context.Context, output map[string]any) error
|
||||
|
||||
func (r *nodeRunner[O]) onEndStream(ctx context.Context, output *schema.StreamReader[map[string]any]) (
|
||||
*schema.StreamReader[map[string]any], error) {
|
||||
if r.errProcessType == vo.ErrorProcessTypeExceptionBranch || r.errProcessType == vo.ErrorProcessTypeDefault {
|
||||
if r.errProcessType == vo.ErrorProcessTypeExceptionBranch || r.errProcessType == vo.ErrorProcessTypeReturnDefaultData {
|
||||
flag := schema.StreamReaderFromArray([]map[string]any{{"isSuccess": true}})
|
||||
output = schema.MergeStreamReaders([]*schema.StreamReader[map[string]any]{flag, output})
|
||||
}
|
||||
|
||||
if !r.callbackEnabled {
|
||||
return output, nil
|
||||
}
|
||||
|
||||
if r.callbackOutputConverter != nil {
|
||||
copied := output.Copy(2)
|
||||
realConverter := func(ctx context.Context) func(map[string]any) (*nodes.StructuredCallbackOutput, error) {
|
||||
@ -632,9 +677,7 @@ func (r *nodeRunner[O]) onEndStream(ctx context.Context, output *schema.StreamRe
|
||||
|
||||
func (r *nodeRunner[O]) onError(ctx context.Context, err error) (map[string]any, bool) {
|
||||
if r.interrupted {
|
||||
if r.callbackEnabled {
|
||||
_ = callbacks.OnError(ctx, err)
|
||||
}
|
||||
_ = callbacks.OnError(ctx, err)
|
||||
return nil, false
|
||||
}
|
||||
|
||||
@ -653,22 +696,20 @@ func (r *nodeRunner[O]) onError(ctx context.Context, err error) (map[string]any,
|
||||
msg := sErr.Msg()
|
||||
|
||||
switch r.errProcessType {
|
||||
case vo.ErrorProcessTypeDefault:
|
||||
case vo.ErrorProcessTypeReturnDefaultData:
|
||||
d := r.dataOnErr(ctx)
|
||||
d["errorBody"] = map[string]any{
|
||||
"errorMessage": msg,
|
||||
"errorCode": code,
|
||||
}
|
||||
d["isSuccess"] = false
|
||||
if r.callbackEnabled {
|
||||
sErr = sErr.ChangeErrLevel(vo.LevelWarn)
|
||||
sOutput := &nodes.StructuredCallbackOutput{
|
||||
Output: d,
|
||||
RawOutput: d,
|
||||
Error: sErr,
|
||||
}
|
||||
_ = callbacks.OnEnd(ctx, sOutput)
|
||||
sErr = sErr.ChangeErrLevel(vo.LevelWarn)
|
||||
sOutput := &nodes.StructuredCallbackOutput{
|
||||
Output: d,
|
||||
RawOutput: d,
|
||||
Error: sErr,
|
||||
}
|
||||
_ = callbacks.OnEnd(ctx, sOutput)
|
||||
return d, true
|
||||
case vo.ErrorProcessTypeExceptionBranch:
|
||||
s := make(map[string]any)
|
||||
@ -677,20 +718,16 @@ func (r *nodeRunner[O]) onError(ctx context.Context, err error) (map[string]any,
|
||||
"errorCode": code,
|
||||
}
|
||||
s["isSuccess"] = false
|
||||
if r.callbackEnabled {
|
||||
sErr = sErr.ChangeErrLevel(vo.LevelWarn)
|
||||
sOutput := &nodes.StructuredCallbackOutput{
|
||||
Output: s,
|
||||
RawOutput: s,
|
||||
Error: sErr,
|
||||
}
|
||||
_ = callbacks.OnEnd(ctx, sOutput)
|
||||
sErr = sErr.ChangeErrLevel(vo.LevelWarn)
|
||||
sOutput := &nodes.StructuredCallbackOutput{
|
||||
Output: s,
|
||||
RawOutput: s,
|
||||
Error: sErr,
|
||||
}
|
||||
_ = callbacks.OnEnd(ctx, sOutput)
|
||||
return s, true
|
||||
default:
|
||||
if r.callbackEnabled {
|
||||
_ = callbacks.OnError(ctx, sErr)
|
||||
}
|
||||
_ = callbacks.OnError(ctx, sErr)
|
||||
return nil, false
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user