youngkingdom/coze-studio
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(infra): remove minio local proxy (#2059)

Browse Source
This commit is contained in:
Ryo
2025-09-10 10:34:16 +08:00
committed by GitHub
parent 77ebc297f9
commit 878a7ef15b
6 changed files with 6 additions and 147 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
backend/infra/contract/storage/storage.go
View File

@ -68,10 +68,10 @@ type ListObjectsPaginatedOutput struct {
}
type FileInfo struct {
Key string
LastModified time.Time
ETag string
Size int64
URL string
Tagging map[string]string
Key string `json:"key"`
LastModified time.Time `json:"last_modified"`
ETag string `json:"etag"`
Size int64 `json:"size"`
URL string `json:"url"`
Tagging map[string]string `json:"tagging"`
}

63
backend/infra/impl/storage/internal/proxy/proxy.go
View File

@ -1,63 +0,0 @@
/*
* Copyright 2025 coze-dev Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package proxy
import (
"context"
"net"
"net/url"
"os"
"github.com/coze-dev/coze-studio/backend/pkg/ctxcache"
"github.com/coze-dev/coze-studio/backend/pkg/logs"
"github.com/coze-dev/coze-studio/backend/types/consts"
)
func CheckIfNeedReplaceHost(ctx context.Context, originURLStr string) (ok bool, proxyURL string) {
// url parse
originURL, err := url.Parse(originURLStr)
if err != nil {
logs.CtxWarnf(ctx, "[CheckIfNeedReplaceHost] url parse failed, err: %v", err)
return false, ""
}
proxyPort := os.Getenv(consts.MinIOProxyEndpoint) // :8889
if proxyPort == "" {
return false, ""
}
currentHost, ok := ctxcache.Get[string](ctx, consts.HostKeyInCtx)
if !ok {
return false, ""
}
currentScheme, ok := ctxcache.Get[string](ctx, consts.RequestSchemeKeyInCtx)
if !ok {
return false, ""
}
host, _, err := net.SplitHostPort(currentHost)
if err != nil {
host = currentHost
}
minioProxyHost := host + proxyPort
originURL.Host = minioProxyHost
originURL.Scheme = currentScheme
logs.CtxDebugf(ctx, "[CheckIfNeedReplaceHost] reset originURL.String = %s", originURL.String())
return true, originURL.String()
}

7
backend/infra/impl/storage/minio/minio.go
View File

@ -30,7 +30,6 @@ import (
"github.com/coze-dev/coze-studio/backend/infra/contract/storage"
"github.com/coze-dev/coze-studio/backend/infra/impl/storage/internal/fileutil"
"github.com/coze-dev/coze-studio/backend/infra/impl/storage/internal/proxy"
"github.com/coze-dev/coze-studio/backend/pkg/logs"
)
@ -232,12 +231,6 @@ func (m *minioClient) GetObjectUrl(ctx context.Context, objectKey string, opts .
return "", fmt.Errorf("GetObjectUrl failed: %v", err)
}
// logs.CtxDebugf(ctx, "[GetObjectUrl] origin presignedURL.String = %s", presignedURL.String())
ok, proxyURL := proxy.CheckIfNeedReplaceHost(ctx, presignedURL.String())
if ok {
return proxyURL, nil
}
return presignedURL.String(), nil
}

6
backend/infra/impl/storage/s3/s3.go
View File

@ -32,7 +32,6 @@ import (
"github.com/coze-dev/coze-studio/backend/infra/contract/storage"
"github.com/coze-dev/coze-studio/backend/infra/impl/storage/internal/fileutil"
"github.com/coze-dev/coze-studio/backend/infra/impl/storage/internal/proxy"
"github.com/coze-dev/coze-studio/backend/pkg/goutil"
"github.com/coze-dev/coze-studio/backend/pkg/logs"
"github.com/coze-dev/coze-studio/backend/pkg/taskgroup"
@ -240,11 +239,6 @@ func (t *s3Client) GetObjectUrl(ctx context.Context, objectKey string, opts ...s
return "", fmt.Errorf("get object presigned url failed: %v", err)
}
ok, proxyURL := proxy.CheckIfNeedReplaceHost(ctx, req.URL)
if ok {
return proxyURL, nil
}
return req.URL, nil
}

6
backend/infra/impl/storage/tos/tos.go
View File

@ -30,7 +30,6 @@ import (
"github.com/coze-dev/coze-studio/backend/infra/contract/storage"
"github.com/coze-dev/coze-studio/backend/infra/impl/storage/internal/fileutil"
"github.com/coze-dev/coze-studio/backend/infra/impl/storage/internal/proxy"
"github.com/coze-dev/coze-studio/backend/pkg/goutil"
"github.com/coze-dev/coze-studio/backend/pkg/lang/conv"
"github.com/coze-dev/coze-studio/backend/pkg/logs"
@ -257,11 +256,6 @@ func (t *tosClient) GetObjectUrl(ctx context.Context, objectKey string, opts ...
return "", err
}
ok, proxyURL := proxy.CheckIfNeedReplaceHost(ctx, output.SignedUrl)
if ok {
return proxyURL, nil
}
return output.SignedUrl, nil
}

59
backend/main.go
View File

@ -22,10 +22,6 @@ import (
"context"
"crypto/tls"
"fmt"
"log"
"net/http"
"net/http/httputil"
"net/url"
"os"
"runtime/debug"
"strings"
@ -41,7 +37,6 @@ import (
"github.com/coze-dev/coze-studio/backend/pkg/lang/conv"
"github.com/coze-dev/coze-studio/backend/pkg/lang/ternary"
"github.com/coze-dev/coze-studio/backend/pkg/logs"
"github.com/coze-dev/coze-studio/backend/pkg/safego"
"github.com/coze-dev/coze-studio/backend/types/consts"
)
@ -60,7 +55,6 @@ func main() {
panic("InitializeInfra failed, err=" + err.Error())
}
asyncStartMinioProxyServer(ctx)
startHttpServer()
}
@ -160,56 +154,3 @@ func setCrashOutput() {
crashFile, _ := os.Create("crash.log")
debug.SetCrashOutput(crashFile, debug.CrashOptions{})
}
// TODO: remove me later
func asyncStartMinioProxyServer(ctx context.Context) {
storageType := getEnv(consts.StorageType, "minio")
proxyURL := getEnv(consts.MinIOAPIHost, "http://localhost:9000")
if storageType == "tos" {
proxyURL = getEnv(consts.TOSBucketEndpoint, "https://opencoze.tos-cn-beijing.volces.com")
}
if storageType == "s3" {
proxyURL = getEnv(consts.S3BucketEndpoint, "")
}
minioProxyEndpoint := getEnv(consts.MinIOProxyEndpoint, "")
if len(minioProxyEndpoint) == 0 {
return
}
safego.Go(ctx, func() {
target, err := url.Parse(proxyURL)
if err != nil {
log.Fatal(err)
}
proxy := httputil.NewSingleHostReverseProxy(target)
originDirector := proxy.Director
proxy.Director = func(req *http.Request) {
q := req.URL.Query()
q.Del("x-wf-file_name")
req.URL.RawQuery = q.Encode()
originDirector(req)
req.Host = req.URL.Host
}
useSSL := getEnv(consts.UseSSL, "0")
if useSSL == "1" {
logs.Infof("Minio proxy server is listening on %s with SSL", minioProxyEndpoint)
err := http.ListenAndServeTLS(minioProxyEndpoint,
getEnv(consts.SSLCertFile, ""),
getEnv(consts.SSLKeyFile, ""), proxy)
if err != nil {
log.Fatal(err)
}
} else {
logs.Infof("Minio proxy server is listening on %s", minioProxyEndpoint)
err := http.ListenAndServe(minioProxyEndpoint, proxy)
if err != nil {
log.Fatal(err)
}
}
})
}