feat(workflow): Filter content from card-type messages to streamline … (#2221)

backend/crossdomain/impl/message/message.go

@@ -174,98 +174,161 @@ func (c *impl) BatchCreate(ctx context.Context, msgs []*entity.Message) ([]*enti
 	return c.DomainSVC.BatchCreate(ctx, msgs)
 }
 
-func convertToConvAndSchemaMessage(ctx context.Context, msgs []*entity.Message) ([]*crossmessage.WfMessage, []*schema.Message, error) {
+func extractContentFromCard(content string) *schema.Message {
-	messages := make([]*schema.Message, 0)
+	type inputCard struct {
-	convMessages := make([]*crossmessage.WfMessage, 0)
+		CardType     int64             `json:"card_type"`
-	for _, m := range msgs {
+		ContentType  int64             `json:"content_type"`
-		msg := &schema.Message{}
+		ResponseType string            `json:"response_type"`
-		err := sonic.UnmarshalString(m.ModelContent, msg)
+		TemplateId   int64             `json:"template_id"`
-		if err != nil {
+		TemplateURL  string            `json:"template_url"`
-			return nil, nil, err
+		Data         string            `json:"data"`
+		XProperties  map[string]string `json:"x_properties"`
+	}
+	type qaField struct {
+		Name string `json:"name"`
+	}
+	type qaProps struct {
+		CardType         string `json:"card_type"`
+		QuestionCardData struct {
+			Title   string     `json:"Title"`
+			Options []*qaField `json:"Options"`
+		} `json:"question_card_data"`
+	}
+
+	card := &inputCard{}
+	if err := sonic.UnmarshalString(content, card); err != nil {
+		return nil
+	}
+
+	prop, ok := card.XProperties["workflow_card_info"]
+	if !ok || prop == "" {
+		return nil
+	}
+
+	qaCard := &qaProps{}
+	if err := sonic.UnmarshalString(prop, qaCard); err != nil {
+		return nil
+	}
+
+	if qaCard.QuestionCardData.Title != "" {
+		return &schema.Message{
+			Content: qaCard.QuestionCardData.Title,
 		}
-		msg.Role = m.Role
+	}
 
-		covMsg := &crossmessage.WfMessage{
+	return nil
-			ID:          m.ID,
+}
-			Role:        m.Role,
-			ContentType: string(m.ContentType),
-			SectionID:   m.SectionID,
-		}
 
-		if len(msg.MultiContent) == 0 {
+func buildConvMessage(ctx context.Context, m *entity.Message, multiContent []schema.ChatMessagePart) (*crossmessage.WfMessage, error) {
-			covMsg.Text = ptr.Of(msg.Content)
+	covMsg := &crossmessage.WfMessage{
-		} else {
+		ID:          m.ID,
-			covMsg.MultiContent = make([]*crossmessage.Content, 0, len(msg.MultiContent))
+		Role:        m.Role,
-			for _, part := range msg.MultiContent {
+		ContentType: string(m.ContentType),
-				switch part.Type {
+		SectionID:   m.SectionID,
-				case schema.ChatMessagePartTypeText:
+	}
-					covMsg.MultiContent = append(covMsg.MultiContent, &crossmessage.Content{
-						Type: model.InputTypeText,
-						Text: ptr.Of(part.Text),
-					})
 
-				case schema.ChatMessagePartTypeImageURL:
+	if len(multiContent) == 0 {
-					if part.ImageURL != nil {
+		covMsg.Text = ptr.Of(m.Content)
-						part.ImageURL.URL, err = workflow.GetRepository().GetObjectUrl(ctx, part.ImageURL.URI)
+		return covMsg, nil
-						if err != nil {
+	}
-							return nil, nil, err
-						}
-						covMsg.MultiContent = append(covMsg.MultiContent, &crossmessage.Content{
-							Uri:  ptr.Of(part.ImageURL.URI),
-							Type: model.InputTypeImage,
-							Url:  ptr.Of(part.ImageURL.URL),
-						})
-					}
 
-				case schema.ChatMessagePartTypeFileURL:
+	covMsg.MultiContent = make([]*crossmessage.Content, 0, len(multiContent))
-
+	for _, part := range multiContent {
-					if part.FileURL != nil {
+		var err error
-						part.FileURL.URL, err = workflow.GetRepository().GetObjectUrl(ctx, part.FileURL.URI)
+		switch part.Type {
-						if err != nil {
+		case schema.ChatMessagePartTypeText:
-							return nil, nil, err
+			covMsg.MultiContent = append(covMsg.MultiContent, &crossmessage.Content{
-						}
+				Type: model.InputTypeText,
-
+				Text: ptr.Of(part.Text),
-						covMsg.MultiContent = append(covMsg.MultiContent, &crossmessage.Content{
+			})
-							Uri:  ptr.Of(part.FileURL.URI),
+		case schema.ChatMessagePartTypeImageURL:
-							Type: model.InputTypeFile,
+			if part.ImageURL != nil {
-							Url:  ptr.Of(part.FileURL.URL),
+				part.ImageURL.URL, err = workflow.GetRepository().GetObjectUrl(ctx, part.ImageURL.URI)
-						})
+				if err != nil {
-
+					return nil, err
-					}
-
-				case schema.ChatMessagePartTypeAudioURL:
-					if part.AudioURL != nil {
-						part.AudioURL.URL, err = workflow.GetRepository().GetObjectUrl(ctx, part.AudioURL.URI)
-						if err != nil {
-							return nil, nil, err
-						}
-						covMsg.MultiContent = append(covMsg.MultiContent, &crossmessage.Content{
-							Uri:  ptr.Of(part.AudioURL.URI),
-							Type: model.InputTypeAudio,
-							Url:  ptr.Of(part.AudioURL.URL),
-						})
-
-					}
-				case schema.ChatMessagePartTypeVideoURL:
-					if part.VideoURL != nil {
-						part.VideoURL.URL, err = workflow.GetRepository().GetObjectUrl(ctx, part.VideoURL.URI)
-						if err != nil {
-							return nil, nil, err
-						}
-						covMsg.MultiContent = append(covMsg.MultiContent, &crossmessage.Content{
-							Uri:  ptr.Of(part.VideoURL.URI),
-							Type: model.InputTypeVideo,
-							Url:  ptr.Of(part.VideoURL.URL),
-						})
-					}
-				default:
-					return nil, nil, fmt.Errorf("unknown part type: %s", part.Type)
 				}
+				covMsg.MultiContent = append(covMsg.MultiContent, &crossmessage.Content{
+					Uri:  ptr.Of(part.ImageURL.URI),
+					Type: model.InputTypeImage,
+					Url:  ptr.Of(part.ImageURL.URL),
+				})
+			}
+		case schema.ChatMessagePartTypeFileURL:
+			if part.FileURL != nil {
+				part.FileURL.URL, err = workflow.GetRepository().GetObjectUrl(ctx, part.FileURL.URI)
+				if err != nil {
+					return nil, err
+				}
+				covMsg.MultiContent = append(covMsg.MultiContent, &crossmessage.Content{
+					Uri:  ptr.Of(part.FileURL.URI),
+					Type: model.InputTypeFile,
+					Url:  ptr.Of(part.FileURL.URL),
+				})
+			}
+		case schema.ChatMessagePartTypeAudioURL:
+			if part.AudioURL != nil {
+				part.AudioURL.URL, err = workflow.GetRepository().GetObjectUrl(ctx, part.AudioURL.URI)
+				if err != nil {
+					return nil, err
+				}
+				covMsg.MultiContent = append(covMsg.MultiContent, &crossmessage.Content{
+					Uri:  ptr.Of(part.AudioURL.URI),
+					Type: model.InputTypeAudio,
+					Url:  ptr.Of(part.AudioURL.URL),
+				})
+			}
+		case schema.ChatMessagePartTypeVideoURL:
+			if part.VideoURL != nil {
+				part.VideoURL.URL, err = workflow.GetRepository().GetObjectUrl(ctx, part.VideoURL.URI)
+				if err != nil {
+					return nil, err
+				}
+				covMsg.MultiContent = append(covMsg.MultiContent, &crossmessage.Content{
+					Uri:  ptr.Of(part.VideoURL.URI),
+					Type: model.InputTypeVideo,
+					Url:  ptr.Of(part.VideoURL.URL),
+				})
+			}
+		default:
+			return nil, fmt.Errorf("unknown part type: %s", part.Type)
+		}
+	}
+	return covMsg, nil
+}
+
+func convertToConvAndSchemaMessage(ctx context.Context, msgs []*entity.Message) ([]*crossmessage.WfMessage, []*schema.Message, error) {
+	messages := make([]*schema.Message, 0, len(msgs))
+	convMessages := make([]*crossmessage.WfMessage, 0, len(msgs))
+
+	for _, m := range msgs {
+		var schemaMsg *schema.Message
+		var err error
+
+		if m.ContentType == model.ContentTypeCard {
+			schemaMsg = extractContentFromCard(m.Content)
+		} else {
+			schemaMsg = &schema.Message{}
+			if err = sonic.UnmarshalString(m.ModelContent, schemaMsg); err != nil {
+				return nil, nil, fmt.Errorf("failed to unmarshal message content: %w", err)
 			}
 		}
 
-		messages = append(messages, msg)
+		var multiContentForUI []schema.ChatMessagePart
+		if schemaMsg != nil {
+			multiContentForUI = schemaMsg.MultiContent
+		}
+
+		covMsg, err := buildConvMessage(ctx, m, multiContentForUI)
+		if err != nil {
+			return nil, nil, fmt.Errorf("failed to build conversation message: %w", err)
+		}
 		convMessages = append(convMessages, covMsg)
+
+		if schemaMsg != nil && (schemaMsg.Content != "" || len(schemaMsg.MultiContent) > 0) {
+			schemaMsg.Role = m.Role
+			messages = append(messages, schemaMsg)
+		}
 	}
+
 	return convMessages, messages, nil
 }

backend/crossdomain/impl/message/message_test.go

@@ -68,6 +68,26 @@ func Test_convertToConvAndSchemaMessage(t *testing.T) {
 	sm8, err := sonic.MarshalString(&schema.Message{MultiContent: []schema.ChatMessagePart{{Type: schema.ChatMessagePartTypeText, Text: "hello"}, {Type: schema.ChatMessagePartTypeImageURL, ImageURL: &schema.ChatMessageImageURL{URI: "file_id_8"}}, {Type: schema.ChatMessagePartTypeFileURL, FileURL: &schema.ChatMessageFileURL{URI: "file_id_9"}}}})
 	require.NoError(t, err)
 
+	qaCardData := map[string]interface{}{
+		"question_card_data": map[string]interface{}{
+			"Title": "card title",
+		},
+	}
+	prop, err := sonic.MarshalString(qaCardData)
+	require.NoError(t, err)
+	cardContent, err := sonic.MarshalString(map[string]interface{}{
+		"x_properties": map[string]string{
+			"workflow_card_info": prop,
+		},
+	})
+	require.NoError(t, err)
+
+	smAudio, err := sonic.MarshalString(&schema.Message{MultiContent: []schema.ChatMessagePart{{Type: schema.ChatMessagePartTypeAudioURL, AudioURL: &schema.ChatMessageAudioURL{URI: "audio_uri_1"}}}})
+	require.NoError(t, err)
+
+	smVideo, err := sonic.MarshalString(&schema.Message{MultiContent: []schema.ChatMessagePart{{Type: schema.ChatMessagePartTypeVideoURL, VideoURL: &schema.ChatMessageVideoURL{URI: "video_uri_1"}}}})
+	require.NoError(t, err)
+
 	type args struct {
 		msgs []*entity.Message
 	}
@@ -87,6 +107,7 @@ func Test_convertToConvAndSchemaMessage(t *testing.T) {
 				msgs: []*entity.Message{
 					{
 						ID:           1,
+						Content:      "hello",
 						Role:         schema.User,
 						ContentType:  "text",
 						ModelContent: sm1,
@@ -234,12 +255,7 @@ func Test_convertToConvAndSchemaMessage(t *testing.T) {
 						Text:        ptr.Of(""),
 					},
 				},
-				schemaMsgs: []*schema.Message{
+				schemaMsgs: []*schema.Message{},
-					{
-						Role:    schema.User,
-						Content: "",
-					},
-				},
 			},
 		},
 		{
@@ -316,6 +332,7 @@ func Test_convertToConvAndSchemaMessage(t *testing.T) {
 				msgs: []*entity.Message{
 					{
 						ID:           8,
+						Content:      "hello",
 						Role:         schema.User,
 						ContentType:  "mix",
 						ModelContent: sm8,
@@ -347,6 +364,101 @@ func Test_convertToConvAndSchemaMessage(t *testing.T) {
 				},
 			},
 		},
+		{
+			name: "card",
+			args: args{
+				msgs: []*entity.Message{
+					{
+						ID:          9,
+						Role:        schema.User,
+						ContentType: "card",
+						Content:     cardContent,
+					},
+				},
+			},
+			want: want{
+				convMsgs: []*crossmessage.WfMessage{
+					{
+						ID:          9,
+						Role:        schema.User,
+						ContentType: "card",
+						Text:        ptr.Of(cardContent),
+					},
+				},
+				schemaMsgs: []*schema.Message{
+					{
+						Role:    schema.User,
+						Content: "card title",
+					},
+				},
+			},
+		},
+		{
+			name: "audio",
+			args: args{
+				msgs: []*entity.Message{
+					{
+						ID:           10,
+						Role:         schema.User,
+						ContentType:  "audio",
+						ModelContent: smAudio,
+					},
+				},
+			},
+			want: want{
+				convMsgs: []*crossmessage.WfMessage{
+					{
+						ID:          10,
+						Role:        schema.User,
+						ContentType: "audio",
+						MultiContent: []*crossmessage.Content{
+							{Type: message.InputTypeAudio, Uri: ptr.Of("audio_uri_1"), Url: ptr.Of("audio_uri_1")},
+						},
+					},
+				},
+				schemaMsgs: []*schema.Message{
+					{
+						Role: schema.User,
+						MultiContent: []schema.ChatMessagePart{
+							{Type: schema.ChatMessagePartTypeAudioURL, AudioURL: &schema.ChatMessageAudioURL{URI: "audio_uri_1", URL: "audio_uri_1"}},
+						},
+					},
+				},
+			},
+		},
+		{
+			name: "video",
+			args: args{
+				msgs: []*entity.Message{
+					{
+						ID:           11,
+						Role:         schema.User,
+						ContentType:  "video",
+						ModelContent: smVideo,
+					},
+				},
+			},
+			want: want{
+				convMsgs: []*crossmessage.WfMessage{
+					{
+						ID:          11,
+						Role:        schema.User,
+						ContentType: "video",
+						MultiContent: []*crossmessage.Content{
+							{Type: message.InputTypeVideo, Uri: ptr.Of("video_uri_1"), Url: ptr.Of("video_uri_1")},
+						},
+					},
+				},
+				schemaMsgs: []*schema.Message{
+					{
+						Role: schema.User,
+						MultiContent: []schema.ChatMessagePart{
+							{Type: schema.ChatMessagePartTypeVideoURL, VideoURL: &schema.ChatMessageVideoURL{URI: "video_uri_1", URL: "video_uri_1"}},
+						},
+					},
+				},
+			},
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {