diff --git a/backend/crossdomain/impl/message/message.go b/backend/crossdomain/impl/message/message.go index eafe75a36..935ac9576 100644 --- a/backend/crossdomain/impl/message/message.go +++ b/backend/crossdomain/impl/message/message.go @@ -174,98 +174,161 @@ func (c *impl) BatchCreate(ctx context.Context, msgs []*entity.Message) ([]*enti return c.DomainSVC.BatchCreate(ctx, msgs) } -func convertToConvAndSchemaMessage(ctx context.Context, msgs []*entity.Message) ([]*crossmessage.WfMessage, []*schema.Message, error) { - messages := make([]*schema.Message, 0) - convMessages := make([]*crossmessage.WfMessage, 0) - for _, m := range msgs { - msg := &schema.Message{} - err := sonic.UnmarshalString(m.ModelContent, msg) - if err != nil { - return nil, nil, err +func extractContentFromCard(content string) *schema.Message { + type inputCard struct { + CardType int64 `json:"card_type"` + ContentType int64 `json:"content_type"` + ResponseType string `json:"response_type"` + TemplateId int64 `json:"template_id"` + TemplateURL string `json:"template_url"` + Data string `json:"data"` + XProperties map[string]string `json:"x_properties"` + } + type qaField struct { + Name string `json:"name"` + } + type qaProps struct { + CardType string `json:"card_type"` + QuestionCardData struct { + Title string `json:"Title"` + Options []*qaField `json:"Options"` + } `json:"question_card_data"` + } + + card := &inputCard{} + if err := sonic.UnmarshalString(content, card); err != nil { + return nil + } + + prop, ok := card.XProperties["workflow_card_info"] + if !ok || prop == "" { + return nil + } + + qaCard := &qaProps{} + if err := sonic.UnmarshalString(prop, qaCard); err != nil { + return nil + } + + if qaCard.QuestionCardData.Title != "" { + return &schema.Message{ + Content: qaCard.QuestionCardData.Title, } - msg.Role = m.Role + } - covMsg := &crossmessage.WfMessage{ - ID: m.ID, - Role: m.Role, - ContentType: string(m.ContentType), - SectionID: m.SectionID, - } + return nil +} - if len(msg.MultiContent) == 0 { - covMsg.Text = ptr.Of(msg.Content) - } else { - covMsg.MultiContent = make([]*crossmessage.Content, 0, len(msg.MultiContent)) - for _, part := range msg.MultiContent { - switch part.Type { - case schema.ChatMessagePartTypeText: - covMsg.MultiContent = append(covMsg.MultiContent, &crossmessage.Content{ - Type: model.InputTypeText, - Text: ptr.Of(part.Text), - }) +func buildConvMessage(ctx context.Context, m *entity.Message, multiContent []schema.ChatMessagePart) (*crossmessage.WfMessage, error) { + covMsg := &crossmessage.WfMessage{ + ID: m.ID, + Role: m.Role, + ContentType: string(m.ContentType), + SectionID: m.SectionID, + } - case schema.ChatMessagePartTypeImageURL: - if part.ImageURL != nil { - part.ImageURL.URL, err = workflow.GetRepository().GetObjectUrl(ctx, part.ImageURL.URI) - if err != nil { - return nil, nil, err - } - covMsg.MultiContent = append(covMsg.MultiContent, &crossmessage.Content{ - Uri: ptr.Of(part.ImageURL.URI), - Type: model.InputTypeImage, - Url: ptr.Of(part.ImageURL.URL), - }) - } + if len(multiContent) == 0 { + covMsg.Text = ptr.Of(m.Content) + return covMsg, nil + } - case schema.ChatMessagePartTypeFileURL: - - if part.FileURL != nil { - part.FileURL.URL, err = workflow.GetRepository().GetObjectUrl(ctx, part.FileURL.URI) - if err != nil { - return nil, nil, err - } - - covMsg.MultiContent = append(covMsg.MultiContent, &crossmessage.Content{ - Uri: ptr.Of(part.FileURL.URI), - Type: model.InputTypeFile, - Url: ptr.Of(part.FileURL.URL), - }) - - } - - case schema.ChatMessagePartTypeAudioURL: - if part.AudioURL != nil { - part.AudioURL.URL, err = workflow.GetRepository().GetObjectUrl(ctx, part.AudioURL.URI) - if err != nil { - return nil, nil, err - } - covMsg.MultiContent = append(covMsg.MultiContent, &crossmessage.Content{ - Uri: ptr.Of(part.AudioURL.URI), - Type: model.InputTypeAudio, - Url: ptr.Of(part.AudioURL.URL), - }) - - } - case schema.ChatMessagePartTypeVideoURL: - if part.VideoURL != nil { - part.VideoURL.URL, err = workflow.GetRepository().GetObjectUrl(ctx, part.VideoURL.URI) - if err != nil { - return nil, nil, err - } - covMsg.MultiContent = append(covMsg.MultiContent, &crossmessage.Content{ - Uri: ptr.Of(part.VideoURL.URI), - Type: model.InputTypeVideo, - Url: ptr.Of(part.VideoURL.URL), - }) - } - default: - return nil, nil, fmt.Errorf("unknown part type: %s", part.Type) + covMsg.MultiContent = make([]*crossmessage.Content, 0, len(multiContent)) + for _, part := range multiContent { + var err error + switch part.Type { + case schema.ChatMessagePartTypeText: + covMsg.MultiContent = append(covMsg.MultiContent, &crossmessage.Content{ + Type: model.InputTypeText, + Text: ptr.Of(part.Text), + }) + case schema.ChatMessagePartTypeImageURL: + if part.ImageURL != nil { + part.ImageURL.URL, err = workflow.GetRepository().GetObjectUrl(ctx, part.ImageURL.URI) + if err != nil { + return nil, err } + covMsg.MultiContent = append(covMsg.MultiContent, &crossmessage.Content{ + Uri: ptr.Of(part.ImageURL.URI), + Type: model.InputTypeImage, + Url: ptr.Of(part.ImageURL.URL), + }) + } + case schema.ChatMessagePartTypeFileURL: + if part.FileURL != nil { + part.FileURL.URL, err = workflow.GetRepository().GetObjectUrl(ctx, part.FileURL.URI) + if err != nil { + return nil, err + } + covMsg.MultiContent = append(covMsg.MultiContent, &crossmessage.Content{ + Uri: ptr.Of(part.FileURL.URI), + Type: model.InputTypeFile, + Url: ptr.Of(part.FileURL.URL), + }) + } + case schema.ChatMessagePartTypeAudioURL: + if part.AudioURL != nil { + part.AudioURL.URL, err = workflow.GetRepository().GetObjectUrl(ctx, part.AudioURL.URI) + if err != nil { + return nil, err + } + covMsg.MultiContent = append(covMsg.MultiContent, &crossmessage.Content{ + Uri: ptr.Of(part.AudioURL.URI), + Type: model.InputTypeAudio, + Url: ptr.Of(part.AudioURL.URL), + }) + } + case schema.ChatMessagePartTypeVideoURL: + if part.VideoURL != nil { + part.VideoURL.URL, err = workflow.GetRepository().GetObjectUrl(ctx, part.VideoURL.URI) + if err != nil { + return nil, err + } + covMsg.MultiContent = append(covMsg.MultiContent, &crossmessage.Content{ + Uri: ptr.Of(part.VideoURL.URI), + Type: model.InputTypeVideo, + Url: ptr.Of(part.VideoURL.URL), + }) + } + default: + return nil, fmt.Errorf("unknown part type: %s", part.Type) + } + } + return covMsg, nil +} + +func convertToConvAndSchemaMessage(ctx context.Context, msgs []*entity.Message) ([]*crossmessage.WfMessage, []*schema.Message, error) { + messages := make([]*schema.Message, 0, len(msgs)) + convMessages := make([]*crossmessage.WfMessage, 0, len(msgs)) + + for _, m := range msgs { + var schemaMsg *schema.Message + var err error + + if m.ContentType == model.ContentTypeCard { + schemaMsg = extractContentFromCard(m.Content) + } else { + schemaMsg = &schema.Message{} + if err = sonic.UnmarshalString(m.ModelContent, schemaMsg); err != nil { + return nil, nil, fmt.Errorf("failed to unmarshal message content: %w", err) } } - messages = append(messages, msg) + var multiContentForUI []schema.ChatMessagePart + if schemaMsg != nil { + multiContentForUI = schemaMsg.MultiContent + } + + covMsg, err := buildConvMessage(ctx, m, multiContentForUI) + if err != nil { + return nil, nil, fmt.Errorf("failed to build conversation message: %w", err) + } convMessages = append(convMessages, covMsg) + + if schemaMsg != nil && (schemaMsg.Content != "" || len(schemaMsg.MultiContent) > 0) { + schemaMsg.Role = m.Role + messages = append(messages, schemaMsg) + } } + return convMessages, messages, nil } diff --git a/backend/crossdomain/impl/message/message_test.go b/backend/crossdomain/impl/message/message_test.go index 01e9310c5..b1125aac9 100644 --- a/backend/crossdomain/impl/message/message_test.go +++ b/backend/crossdomain/impl/message/message_test.go @@ -68,6 +68,26 @@ func Test_convertToConvAndSchemaMessage(t *testing.T) { sm8, err := sonic.MarshalString(&schema.Message{MultiContent: []schema.ChatMessagePart{{Type: schema.ChatMessagePartTypeText, Text: "hello"}, {Type: schema.ChatMessagePartTypeImageURL, ImageURL: &schema.ChatMessageImageURL{URI: "file_id_8"}}, {Type: schema.ChatMessagePartTypeFileURL, FileURL: &schema.ChatMessageFileURL{URI: "file_id_9"}}}}) require.NoError(t, err) + qaCardData := map[string]interface{}{ + "question_card_data": map[string]interface{}{ + "Title": "card title", + }, + } + prop, err := sonic.MarshalString(qaCardData) + require.NoError(t, err) + cardContent, err := sonic.MarshalString(map[string]interface{}{ + "x_properties": map[string]string{ + "workflow_card_info": prop, + }, + }) + require.NoError(t, err) + + smAudio, err := sonic.MarshalString(&schema.Message{MultiContent: []schema.ChatMessagePart{{Type: schema.ChatMessagePartTypeAudioURL, AudioURL: &schema.ChatMessageAudioURL{URI: "audio_uri_1"}}}}) + require.NoError(t, err) + + smVideo, err := sonic.MarshalString(&schema.Message{MultiContent: []schema.ChatMessagePart{{Type: schema.ChatMessagePartTypeVideoURL, VideoURL: &schema.ChatMessageVideoURL{URI: "video_uri_1"}}}}) + require.NoError(t, err) + type args struct { msgs []*entity.Message } @@ -87,6 +107,7 @@ func Test_convertToConvAndSchemaMessage(t *testing.T) { msgs: []*entity.Message{ { ID: 1, + Content: "hello", Role: schema.User, ContentType: "text", ModelContent: sm1, @@ -234,12 +255,7 @@ func Test_convertToConvAndSchemaMessage(t *testing.T) { Text: ptr.Of(""), }, }, - schemaMsgs: []*schema.Message{ - { - Role: schema.User, - Content: "", - }, - }, + schemaMsgs: []*schema.Message{}, }, }, { @@ -316,6 +332,7 @@ func Test_convertToConvAndSchemaMessage(t *testing.T) { msgs: []*entity.Message{ { ID: 8, + Content: "hello", Role: schema.User, ContentType: "mix", ModelContent: sm8, @@ -347,6 +364,101 @@ func Test_convertToConvAndSchemaMessage(t *testing.T) { }, }, }, + { + name: "card", + args: args{ + msgs: []*entity.Message{ + { + ID: 9, + Role: schema.User, + ContentType: "card", + Content: cardContent, + }, + }, + }, + want: want{ + convMsgs: []*crossmessage.WfMessage{ + { + ID: 9, + Role: schema.User, + ContentType: "card", + Text: ptr.Of(cardContent), + }, + }, + schemaMsgs: []*schema.Message{ + { + Role: schema.User, + Content: "card title", + }, + }, + }, + }, + { + name: "audio", + args: args{ + msgs: []*entity.Message{ + { + ID: 10, + Role: schema.User, + ContentType: "audio", + ModelContent: smAudio, + }, + }, + }, + want: want{ + convMsgs: []*crossmessage.WfMessage{ + { + ID: 10, + Role: schema.User, + ContentType: "audio", + MultiContent: []*crossmessage.Content{ + {Type: message.InputTypeAudio, Uri: ptr.Of("audio_uri_1"), Url: ptr.Of("audio_uri_1")}, + }, + }, + }, + schemaMsgs: []*schema.Message{ + { + Role: schema.User, + MultiContent: []schema.ChatMessagePart{ + {Type: schema.ChatMessagePartTypeAudioURL, AudioURL: &schema.ChatMessageAudioURL{URI: "audio_uri_1", URL: "audio_uri_1"}}, + }, + }, + }, + }, + }, + { + name: "video", + args: args{ + msgs: []*entity.Message{ + { + ID: 11, + Role: schema.User, + ContentType: "video", + ModelContent: smVideo, + }, + }, + }, + want: want{ + convMsgs: []*crossmessage.WfMessage{ + { + ID: 11, + Role: schema.User, + ContentType: "video", + MultiContent: []*crossmessage.Content{ + {Type: message.InputTypeVideo, Uri: ptr.Of("video_uri_1"), Url: ptr.Of("video_uri_1")}, + }, + }, + }, + schemaMsgs: []*schema.Message{ + { + Role: schema.User, + MultiContent: []schema.ChatMessagePart{ + {Type: schema.ChatMessagePartTypeVideoURL, VideoURL: &schema.ChatMessageVideoURL{URI: "video_uri_1", URL: "video_uri_1"}}, + }, + }, + }, + }, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) {