youngkingdom/coze-studio
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: modify record info api and mini os tagging logic

Browse Source
This commit is contained in:
zhuangjie.1125
2025-09-03 11:48:17 +08:00
parent 009f310cc0
commit edf47b96dd
7 changed files with 107 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files

60
backend/application/upload/icon.go
View File

@ -39,7 +39,6 @@ import (
"strings"
"time"
"github.com/redis/go-redis/v9"
_ "golang.org/x/image/tiff"
_ "golang.org/x/image/webp"
"gorm.io/gorm"
@ -289,14 +288,6 @@ func (u *UploadService) UploadFileCommon(ctx context.Context, req *upload.Common
return nil, errorx.New(errno.ErrUploadInvalidParamCode, errorx.KV("msg", "tos key not found"))
}
objKey := match[1]
fInfo, existed, err := u.getFileInfo(ctx, objKey)
if err != nil {
return nil, err
}
if !existed {
logs.CtxErrorf(ctx, "file uri %s not record file info frist", objKey)
}
if strings.Contains(fullPath, "?uploads") {
uploadID, err := u.PartUploadFileInit(ctx, objKey)
if err != nil {
@ -338,12 +329,6 @@ func (u *UploadService) UploadFileCommon(ctx context.Context, req *upload.Common
if len(contentType) != 0 {
opts = append(opts, storage.WithContentType(contentType))
}
if fInfo != nil {
opts = append(opts, storage.WithTagging(map[string]string{
"filename": fInfo.FileName,
"file_ext": fInfo.FileExtension,
}))
}
err = u.oss.PutObject(ctx, objKey, req.ByteData, opts...)
if err != nil {
return resp, errorx.New(errno.ErrUploadSystemErrorCode, errorx.KV("msg", err.Error()))
@ -756,48 +741,15 @@ func (u *UploadService) CommitImageUpload(ctx context.Context, req *upload.Apply
return &resp, nil
}
const (
fileInfoFormat = "record_file_info:%s"
)
type fileInfo struct {
FileURI string `json:"file_uri"`
FileName string `json:"file_name"`
FileSize string `json:"file_size"`
FileExtension string `json:"file_extension"`
}
func (u *UploadService) RecordFileInfo(ctx context.Context, req *upload.RecordFileInfoRequest) (*upload.RecordFileInfoResponse, error) {
fInfo := &fileInfo{
FileURI: req.GetFileURI(),
FileName: req.GetFileName(),
FileSize: req.GetFileSize(),
FileExtension: req.GetFileExtension(),
}
fInfoBs, _ := sonic.Marshal(fInfo)
err := u.cache.Set(ctx, fmt.Sprintf(fileInfoFormat, req.GetFileURI()), fInfoBs, time.Minute*30).Err()
err := u.oss.PutObjectTagging(ctx, req.GetFileURI(), map[string]string{
"filename": req.GetFileName(),
"file_ext": req.GetFileExtension(),
})
if err != nil {
return nil, err
}
resp := &upload.RecordFileInfoResponse{}
return resp, nil
}
func (u *UploadService) getFileInfo(ctx context.Context, fileURI string) (*fileInfo, bool, error) {
bs, err := u.cache.Get(ctx, fmt.Sprintf(fileInfoFormat, fileURI)).Bytes()
if err != nil {
if errors.Is(err, redis.Nil) {
return nil, false, nil
}
return nil, false, err
}
fInfo := &fileInfo{}
err = sonic.Unmarshal(bs, fInfo)
if err != nil {
return nil, false, err
}
return fInfo, true, nil
return &upload.RecordFileInfoResponse{}, nil
}

4
backend/domain/workflow/internal/nodes/knowledge/knowledge_indexer.go
View File

@ -196,7 +196,9 @@ func parseToFileNameAndFileExtension(ctx context.Context, fileURL string) (strin
if !ok {
return "", "", fmt.Errorf("failed to get file tagging file_ext")
}
if len(fileExt) == 0 {
fileExt = strings.TrimPrefix(filepath.Ext(fileURI), ".")
}
ext, support := parser.ValidateFileExtension(fileExt)
if !support {
return "", "", fmt.Errorf("unsupported file type: %s", fileExt)

2
backend/infra/contract/storage/storage.go
View File

@ -32,6 +32,8 @@ type Storage interface {
GetObject(ctx context.Context, objectKey string) ([]byte, error)
// GetObjectTagging returns object tagging
GetObjectTagging(ctx context.Context, objectKey string) (map[string]string, error)
// PutObjectTagging update the tag of the object in full
PutObjectTagging(ctx context.Context, objectKey string, ts map[string]string) error
// DeleteObject deletes the object with the specified key.
DeleteObject(ctx context.Context, objectKey string) error
// GetObjectUrl returns a presigned URL for the object.

47
backend/infra/impl/storage/minio/minio.go
View File

@ -19,6 +19,7 @@ package minio
import (
"bytes"
"context"
"encoding/base64"
"fmt"
"io"
@ -28,6 +29,7 @@ import (
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
"github.com/minio/minio-go/v7/pkg/tags"
"github.com/coze-dev/coze-studio/backend/infra/contract/storage"
"github.com/coze-dev/coze-studio/backend/infra/impl/storage/internal/fileutil"
@ -184,8 +186,12 @@ func (m *minioClient) PutObjectWithReader(ctx context.Context, objectKey string,
minioOpts.Expires = *option.Expires
}
tagging := make(map[string]string, len(option.Tagging))
for k, v := range option.Tagging {
tagging[k] = base64.StdEncoding.EncodeToString([]byte(v))
}
if option.Tagging != nil {
minioOpts.UserTags = option.Tagging
minioOpts.UserTags = tagging
}
_, err := m.client.PutObject(ctx, m.bucketName, objectKey,
@ -294,12 +300,20 @@ func (m *minioClient) ListAllObjects(ctx context.Context, prefix string, opts ..
return nil, object.Err
}
tagging := make(map[string]string, len(object.UserTags))
for k, v := range object.UserTags {
valBs, err := base64.StdEncoding.DecodeString(v)
if err != nil {
return nil, err
}
tagging[k] = string(valBs)
}
files = append(files, &storage.FileInfo{
Key: object.Key,
LastModified: object.LastModified,
ETag: object.ETag,
Size: object.Size,
Tagging: object.UserTags,
Tagging: tagging,
})
logs.CtxDebugf(ctx, "key = %s, lastModified = %s, eTag = %s, size = %d, tagging = %v",
@ -332,12 +346,11 @@ func (m *minioClient) HeadObject(ctx context.Context, objectKey string, opts ...
}
if option.WithTagging {
tags, err := m.client.GetObjectTagging(ctx, m.bucketName, objectKey, minio.GetObjectTaggingOptions{})
objTags, err := m.GetObjectTagging(ctx, objectKey)
if err != nil {
return nil, err
}
f.Tagging = tags.ToMap()
f.Tagging = objTags
}
if option.WithURL {
@ -355,5 +368,27 @@ func (m *minioClient) GetObjectTagging(ctx context.Context, objectKey string) (m
if err != nil {
return nil, fmt.Errorf("GetObjectTagging failed: %v", err)
}
return response.ToMap(), nil
rst := make(map[string]string)
for k, v := range response.ToMap() {
valBs, err := base64.StdEncoding.DecodeString(v)
if err != nil {
return nil, fmt.Errorf("QueryUnescape failed: %v", err)
}
rst[k] = string(valBs)
}
return rst, nil
}
func (m *minioClient) PutObjectTagging(ctx context.Context, objectKey string, ts map[string]string) error {
saveTags := make(map[string]string)
for k, v := range ts {
saveTags[k] = base64.StdEncoding.EncodeToString([]byte(v))
}
t, err := tags.NewTags(saveTags, false)
if err != nil {
return err
}
return m.client.PutObjectTagging(ctx, m.bucketName, objectKey, t, minio.PutObjectTaggingOptions{})
}

22
backend/infra/impl/storage/s3/s3.go
View File

@ -454,3 +454,25 @@ func (t *s3Client) GetObjectTagging(ctx context.Context, objectKey string) (map[
return *e.Key, *e.Value
}), err
}
func (t *s3Client) PutObjectTagging(ctx context.Context, objectKey string, ts map[string]string) error {
tags := make([]types.Tag, 0, len(ts))
for k, v := range ts {
tags = append(tags, types.Tag{
Key: aws.String(k),
Value: aws.String(v),
})
}
_, err := t.client.PutObjectTagging(ctx, &s3.PutObjectTaggingInput{
Bucket: aws.String(t.bucketName),
Key: aws.String(objectKey),
Tagging: &types.Tagging{
TagSet: tags,
},
})
if err != nil {
return err
}
return nil
}

20
backend/infra/impl/storage/tos/tos.go
View File

@ -20,7 +20,6 @@ import (
"bytes"
"context"
"fmt"
"io"
"net/http"
"strings"
@ -457,3 +456,22 @@ func (t *tosClient) GetObjectTagging(ctx context.Context, objectKey string) (map
return e.Key, e.Value
}), nil
}
func (t *tosClient) PutObjectTagging(ctx context.Context, objectKey string, ts map[string]string) error {
tags := make([]tos.Tag, 0, len(ts))
for k, v := range ts {
tags = append(tags, tos.Tag{Key: k, Value: v})
}
_, err := t.client.PutObjectTagging(ctx, &tos.PutObjectTaggingInput{
Bucket: t.bucketName,
Key: objectKey,
TagSet: tos.TagSet{
Tags: tags,
},
})
if err != nil {
return err
}
return nil
}

14
backend/internal/mock/infra/contract/storage/storage_mock.go
View File

@ -185,6 +185,20 @@ func (mr *MockStorageMockRecorder) PutObject(ctx, objectKey, content any, opts .
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PutObject", reflect.TypeOf((*MockStorage)(nil).PutObject), varargs...)
}
// PutObjectTagging mocks base method.
func (m *MockStorage) PutObjectTagging(ctx context.Context, objectKey string, ts map[string]string) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "PutObjectTagging", ctx, objectKey, ts)
ret0, _ := ret[0].(error)
return ret0
}
// PutObjectTagging indicates an expected call of PutObjectTagging.
func (mr *MockStorageMockRecorder) PutObjectTagging(ctx, objectKey, ts any) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PutObjectTagging", reflect.TypeOf((*MockStorage)(nil).PutObjectTagging), ctx, objectKey, ts)
}
// PutObjectWithReader mocks base method.
func (m *MockStorage) PutObjectWithReader(ctx context.Context, objectKey string, content io.Reader, opts ...storage.PutOptFn) error {
m.ctrl.T.Helper()