From cebf5892ecc422fc56456fbe0f0e26cdbd0f683e Mon Sep 17 00:00:00 2001 From: Jin Hai Date: Thu, 12 Mar 2026 18:58:25 +0800 Subject: [PATCH] Create go version storage component, but not used (#13561) ### What problem does this PR solve? Implement: minio, s3, oss, azure_sas, azure_spn, gcs, opendal ### Type of change - [x] New Feature (non-breaking change which adds functionality) --------- Signed-off-by: Jin Hai --- go.mod | 73 ++++- go.sum | 246 ++++++++++++++-- internal/storage/minio.go | 397 ++++++++++++++++++++++++++ internal/storage/oss.go | 415 +++++++++++++++++++++++++++ internal/storage/s3.go | 425 ++++++++++++++++++++++++++++ internal/storage/storage_factory.go | 298 +++++++++++++++++++ internal/storage/types.go | 102 +++++++ 7 files changed, 1932 insertions(+), 24 deletions(-) create mode 100644 internal/storage/minio.go create mode 100644 internal/storage/oss.go create mode 100644 internal/storage/s3.go create mode 100644 internal/storage/storage_factory.go create mode 100644 internal/storage/types.go diff --git a/go.mod b/go.mod index 3a47d85eb..7aa72113e 100644 --- a/go.mod +++ b/go.mod @@ -1,51 +1,99 @@ module ragflow -go 1.24.0 +go 1.25 require ( + cloud.google.com/go/storage v1.35.1 + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0 + github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4 + github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake v1.4.4 + github.com/aws/aws-sdk-go-v2 v1.41.3 + github.com/aws/aws-sdk-go-v2/config v1.32.11 + github.com/aws/aws-sdk-go-v2/credentials v1.19.11 + github.com/aws/aws-sdk-go-v2/service/s3 v1.96.4 + github.com/aws/smithy-go v1.24.2 github.com/elastic/go-elasticsearch/v8 v8.19.1 github.com/gin-gonic/gin v1.9.1 - github.com/google/uuid v1.4.0 + github.com/go-sql-driver/mysql v1.7.0 + github.com/google/uuid v1.6.0 github.com/iromli/go-itsdangerous v0.0.0-20220223194502-9c8bef8dac6a + github.com/minio/minio-go/v7 v7.0.99 + github.com/peterh/liner v1.2.2 github.com/redis/go-redis/v9 v9.18.0 github.com/siongui/gojianfan v0.0.0-20210926212422-2f175ac615de github.com/spf13/viper v1.18.2 go.uber.org/zap v1.27.1 golang.org/x/crypto v0.47.0 + google.golang.org/api v0.153.0 gorm.io/driver/mysql v1.5.2 gorm.io/gorm v1.25.5 ) require ( + cloud.google.com/go v0.110.10 // indirect + cloud.google.com/go/compute v1.23.3 // indirect + cloud.google.com/go/compute/metadata v0.2.3 // indirect + cloud.google.com/go/iam v1.1.5 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 // indirect + github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.6 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.19 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.19 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.19 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.5 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.20 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.6 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.11 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.19 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.19 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.0.7 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.30.12 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.16 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.41.8 // indirect github.com/bytedance/sonic v1.9.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect + github.com/dustin/go-humanize v1.0.1 // indirect github.com/elastic/elastic-transport-go/v8 v8.8.0 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/gabriel-vasile/mimetype v1.4.2 // indirect github.com/gin-contrib/sse v0.1.0 // indirect + github.com/go-ini/ini v1.67.0 // indirect github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect github.com/go-playground/validator/v10 v10.16.0 // indirect - github.com/go-sql-driver/mysql v1.7.0 // indirect github.com/goccy/go-json v0.10.2 // indirect + github.com/golang-jwt/jwt/v5 v5.3.0 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/protobuf v1.5.3 // indirect + github.com/google/s2a-go v0.1.7 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect + github.com/googleapis/gax-go/v2 v2.12.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/cpuid/v2 v2.2.4 // indirect + github.com/klauspost/compress v1.18.2 // indirect + github.com/klauspost/cpuid/v2 v2.2.11 // indirect + github.com/klauspost/crc32 v1.3.0 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect github.com/leodido/go-urn v1.2.4 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-runewidth v0.0.3 // indirect + github.com/minio/crc64nvme v1.1.1 // indirect + github.com/minio/md5-simd v1.1.2 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pelletier/go-toml/v2 v2.1.1 // indirect - github.com/peterh/liner v1.2.2 // indirect + github.com/philhofer/fwd v1.2.0 // indirect + github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect + github.com/rs/xid v1.6.0 // indirect github.com/sagikazarmark/locafero v0.4.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect github.com/sourcegraph/conc v0.3.0 // indirect @@ -53,19 +101,30 @@ require ( github.com/spf13/cast v1.6.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/subosito/gotenv v1.6.0 // indirect + github.com/tinylib/msgp v1.6.1 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.2.12 // indirect + go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/otel v1.28.0 // indirect go.opentelemetry.io/otel/metric v1.28.0 // indirect go.opentelemetry.io/otel/trace v1.28.0 // indirect go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.10.0 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/arch v0.6.0 // indirect golang.org/x/exp v0.0.0-20231226003508-02704c960a9b // indirect - golang.org/x/net v0.48.0 // indirect + golang.org/x/net v0.49.0 // indirect + golang.org/x/oauth2 v0.15.0 // indirect + golang.org/x/sync v0.19.0 // indirect golang.org/x/sys v0.41.0 // indirect - golang.org/x/term v0.40.0 // indirect golang.org/x/text v0.33.0 // indirect + golang.org/x/time v0.5.0 // indirect + golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect + google.golang.org/appengine v1.6.7 // indirect + google.golang.org/genproto v0.0.0-20231106174013-bbf56f31fb17 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20231106174013-bbf56f31fb17 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20231120223509-83a465c0220f // indirect + google.golang.org/grpc v1.59.0 // indirect google.golang.org/protobuf v1.32.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index 23b9cdd0d..965ceba2d 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,71 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.110.10 h1:LXy9GEO+timppncPIAZoOj3l58LIU9k+kn48AN7IO3Y= +cloud.google.com/go v0.110.10/go.mod h1:v1OoFqYxiBkUrruItNM3eT4lLByNjxmJSV/xDKJNnic= +cloud.google.com/go/compute v1.23.3 h1:6sVlXXBmbd7jNX0Ipq0trII3e4n1/MsADLK6a+aiVlk= +cloud.google.com/go/compute v1.23.3/go.mod h1:VCgBUoMnIVIR0CscqQiPJLAG25E3ZRZMzcFZeQ+h8CI= +cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= +cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= +cloud.google.com/go/iam v1.1.5 h1:1jTsCu4bcsNsE4iiqNT5SHwrDRCfRmIaaaVFhRveTJI= +cloud.google.com/go/iam v1.1.5/go.mod h1:rB6P/Ic3mykPbFio+vo7403drjlgvoWfYpJhMXEbzv8= +cloud.google.com/go/storage v1.35.1 h1:B59ahL//eDfx2IIKFBeT5Atm9wnNmj3+8xG/W4WB//w= +cloud.google.com/go/storage v1.35.1/go.mod h1:M6M/3V/D3KpzMTJyPOR/HU6n2Si5QdaXYEsng2xgOs8= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0 h1:fou+2+WFTib47nS+nz/ozhEBnvU96bKHy6LjRsY4E28= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0/go.mod h1:t76Ruy8AHvUAC8GfMWJMa0ElSbuIcO03NLpynfbgsPA= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 h1:Hk5QBxZQC1jb2Fwj6mpzme37xbCDdNTxU7O9eb5+LB4= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1/go.mod h1:IYus9qsFobWIc2YVwe/WPjcnyCkPKtnHAqUYeebc8z0= +github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2 h1:yz1bePFlP5Vws5+8ez6T3HWXPmwOK7Yvq8QxDBD3SKY= +github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2/go.mod h1:Pa9ZNPuoNu/GztvBSKk9J1cDJW6vk/n0zLtV4mgd8N8= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 h1:9iefClla7iYpfYWdzPCRDozdmndjTm8DXdpCzPajMgA= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2/go.mod h1:XtLgD3ZD34DAaVIIAyG3objl5DynM3CQ/vMcbBNJZGI= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1 h1:/Zt+cDPnpC3OVDm/JKLOs7M2DKmLRIIp3XIx9pHHiig= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1/go.mod h1:Ng3urmn6dYe8gnbCMoHHVl5APYz2txho3koEkV2o2HA= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4 h1:jWQK1GI+LeGGUKBADtcH2rRqPxYB1Ljwms5gFA2LqrM= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4/go.mod h1:8mwH4klAm9DUgR2EEHyEEAQlRDvLPyg5fQry3y+cDew= +github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake v1.4.4 h1:7QtoGxKm6mPhsWzEZtrn3tQF1hmMMZblngnqNoE61I8= +github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake v1.4.4/go.mod h1:juYrzH1q6A+g9ZZbGh0OmjS7zaMq3rFDrPhVnYSgFMA= +github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM= +github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE= +github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 h1:XRzhVemXdgvJqCH0sFfrBUTnUJSBrBf7++ypk+twtRs= +github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/aws/aws-sdk-go-v2 v1.41.3 h1:4kQ/fa22KjDt13QCy1+bYADvdgcxpfH18f0zP542kZA= +github.com/aws/aws-sdk-go-v2 v1.41.3/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.6 h1:N4lRUXZpZ1KVEUn6hxtco/1d2lgYhNn1fHkkl8WhlyQ= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.6/go.mod h1:lyw7GFp3qENLh7kwzf7iMzAxDn+NzjXEAGjKS2UOKqI= +github.com/aws/aws-sdk-go-v2/config v1.32.11 h1:ftxI5sgz8jZkckuUHXfC/wMUc8u3fG1vQS0plr2F2Zs= +github.com/aws/aws-sdk-go-v2/config v1.32.11/go.mod h1:twF11+6ps9aNRKEDimksp923o44w/Thk9+8YIlzWMmo= +github.com/aws/aws-sdk-go-v2/credentials v1.19.11 h1:NdV8cwCcAXrCWyxArt58BrvZJ9pZ9Fhf9w6Uh5W3Uyc= +github.com/aws/aws-sdk-go-v2/credentials v1.19.11/go.mod h1:30yY2zqkMPdrvxBqzI9xQCM+WrlrZKSOpSJEsylVU+8= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.19 h1:INUvJxmhdEbVulJYHI061k4TVuS3jzzthNvjqvVvTKM= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.19/go.mod h1:FpZN2QISLdEBWkayloda+sZjVJL+e9Gl0k1SyTgcswU= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.19 h1:/sECfyq2JTifMI2JPyZ4bdRN77zJmr6SrS1eL3augIA= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.19/go.mod h1:dMf8A5oAqr9/oxOfLkC/c2LU/uMcALP0Rgn2BD5LWn0= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.19 h1:AWeJMk33GTBf6J20XJe6qZoRSJo0WfUhsMdUKhoODXE= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.19/go.mod h1:+GWrYoaAsV7/4pNHpwh1kiNLXkKaSoppxQq9lbH8Ejw= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.5 h1:clHU5fm//kWS1C2HgtgWxfQbFbx4b6rx+5jzhgX9HrI= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.5/go.mod h1:O3h0IK87yXci+kg6flUKzJnWeziQUKciKrLjcatSNcY= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.20 h1:qi3e/dmpdONhj1RyIZdi6DKKpDXS5Lb8ftr3p7cyHJc= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.20/go.mod h1:V1K+TeJVD5JOk3D9e5tsX2KUdL7BlB+FV6cBhdobN8c= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.6 h1:XAq62tBTJP/85lFD5oqOOe7YYgWxY9LvWq8plyDvDVg= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.6/go.mod h1:x0nZssQ3qZSnIcePWLvcoFisRXJzcTVvYpAAdYX8+GI= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.11 h1:BYf7XNsJMzl4mObARUBUib+j2tf0U//JAAtTnYqvqCw= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.11/go.mod h1:aEUS4WrNk/+FxkBZZa7tVgp4pGH+kFGW40Y8rCPqt5g= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.19 h1:X1Tow7suZk9UCJHE1Iw9GMZJJl0dAnKXXP1NaSDHwmw= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.19/go.mod h1:/rARO8psX+4sfjUQXp5LLifjUt8DuATZ31WptNJTyQA= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.19 h1:JnQeStZvPHFHeyky/7LbMlyQjUa+jIBj36OlWm0pzIk= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.19/go.mod h1:HGyasyHvYdFQeJhvDHfH7HXkHh57htcJGKDZ+7z+I24= +github.com/aws/aws-sdk-go-v2/service/s3 v1.96.4 h1:4ExZyubQ6LQQVuF2Qp9OsfEvsTdAWh5Gfwf6PgIdLdk= +github.com/aws/aws-sdk-go-v2/service/s3 v1.96.4/go.mod h1:NF3JcMGOiARAss1ld3WGORCw71+4ExDD2cbbdKS5PpA= +github.com/aws/aws-sdk-go-v2/service/signin v1.0.7 h1:Y2cAXlClHsXkkOvWZFXATr34b0hxxloeQu/pAZz2row= +github.com/aws/aws-sdk-go-v2/service/signin v1.0.7/go.mod h1:idzZ7gmDeqeNrSPkdbtMp9qWMgcBwykA7P7Rzh5DXVU= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.12 h1:iSsvB9EtQ09YrsmIc44Heqlx5ByGErqhPK1ZQLppias= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.12/go.mod h1:fEWYKTRGoZNl8tZ77i61/ccwOMJdGxwOhWCkp6TXAr0= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.16 h1:EnUdUqRP1CNzt2DkV67tJx6XDN4xlfBFm+bzeNOQVb0= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.16/go.mod h1:Jic/xv0Rq/pFNCh3WwpH4BEqdbSAl+IyHro8LbibHD8= +github.com/aws/aws-sdk-go-v2/service/sts v1.41.8 h1:XQTQTF75vnug2TXS8m7CVJfC2nniYPZnO1D4Np761Oo= +github.com/aws/aws-sdk-go-v2/service/sts v1.41.8/go.mod h1:Xgx+PR1NUOjNmQY+tRMnouRp83JRM8pRMw/vCaVhPkI= +github.com/aws/smithy-go v1.24.2 h1:FzA3bu/nt/vDvmnkg+R8Xl46gmzEDam6mZ1hzmwXFng= +github.com/aws/smithy-go v1.24.2/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= @@ -5,21 +73,30 @@ github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0 github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM= github.com/bytedance/sonic v1.9.1 h1:6iJ6NqdoxCDr6mbY8h18oSO+cShGSMRGCEo7F2h0x8s= github.com/bytedance/sonic v1.9.1/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY= github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 h1:qSGYFH7+jGhDF8vLC+iwCD4WpbV1EBDSzWkJODFLams= github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/elastic/elastic-transport-go/v8 v8.8.0 h1:7k1Ua+qluFr6p1jfJjGDl97ssJS/P7cHNInzfxgBQAo= github.com/elastic/elastic-transport-go/v8 v8.8.0/go.mod h1:YLHer5cj0csTzNFXoNQ8qhtGY1GTvSqPnKWKaqQE3Hk= github.com/elastic/go-elasticsearch/v8 v8.19.1 h1:0iEGt5/Ds9MNVxEp3hqLsXdbe6SjleaVHONg/FuR09Q= github.com/elastic/go-elasticsearch/v8 v8.19.1/go.mod h1:tHJQdInFa6abmDbDCEH2LJja07l/SIpaGpJcm13nt7s= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= @@ -30,6 +107,8 @@ github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg= github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU= +github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= +github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= @@ -47,11 +126,47 @@ github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/golang-jwt/jwt/v5 v5.3.0 h1:pv4AsKCKKZuqlgs5sUmn4x8UlGa0kEVt/puTpKx9vvo= +github.com/golang-jwt/jwt/v5 v5.3.0/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4= -github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw= +github.com/google/martian/v3 v3.3.2/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= +github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= +github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs= +github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= +github.com/googleapis/gax-go/v2 v2.12.0 h1:A+gCJKdRfqXkr+BIRGtZLibNXf0m1f9E4HG56etFpas= +github.com/googleapis/gax-go/v2 v2.12.0/go.mod h1:y+aIqrI5eb1YGMVJfuV3185Ts/D7qKpsEkdD5+I6QGU= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/iromli/go-itsdangerous v0.0.0-20220223194502-9c8bef8dac6a h1:Inib12UR9HAfBubrGNraPjKt/Cu8xPbTJbC50+0wP5U= @@ -62,13 +177,22 @@ github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/keybase/go-keychain v0.0.1 h1:way+bWYa6lDppZoZcgMbYsvC7GxljxrskdNInRtuthU= +github.com/keybase/go-keychain v0.0.1/go.mod h1:PdEILRW3i9D8JcdM+FmY6RwkHGnhHxXwkPPMeUgOK1k= +github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk= +github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk= -github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= +github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU= +github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM= +github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q= github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= @@ -77,6 +201,12 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.3 h1:a+kO+98RDGEfo6asOGMmpodZq4FNtnGP54yps8BzLR4= github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI= +github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.0.99 h1:2vH/byrwUkIpFQFOilvTfaUpvAX3fEFhEzO+DR3DlCE= +github.com/minio/minio-go/v7 v7.0.99/go.mod h1:EtGNKtlX20iL2yaYnxEigaIvj0G0GwSDnifnG8ClIdw= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -88,13 +218,20 @@ github.com/pelletier/go-toml/v2 v2.1.1 h1:LWAJwfNvjQZCFIDKWYQaM62NcYeYViCmWIwmOS github.com/pelletier/go-toml/v2 v2.1.1/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/peterh/liner v1.2.2 h1:aJ4AOodmL+JxOZZEL2u9iJf8omNRpqHc/EbrK+3mAXw= github.com/peterh/liner v1.2.2/go.mod h1:xFwJyiKIXJZUKItq5dGHZSTBRAuG/CpeNpWLyiNRNwI= +github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= +github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/redis/go-redis/v9 v9.18.0 h1:pMkxYPkEbMPwRdenAzUNyFNrDgHx9U+DrBabWNfSRQs= github.com/redis/go-redis/v9 v9.18.0/go.mod h1:k3ufPphLU5YXwNTUcCRXGxUoF1fqxnhFQmscfkCoDA0= -github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= -github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= +github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= +github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ= github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= @@ -121,16 +258,20 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= -github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= +github.com/tinylib/msgp v1.6.1 h1:ESRv8eL3u+DNHUoSAAQRE50Hm162zqAnBoGv9PzScPY= +github.com/tinylib/msgp v1.6.1/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE= github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/otel v1.28.0 h1:/SqNcYk+idO0CxKEUOtKQClMK/MimZihKYMruSMViUo= go.opentelemetry.io/otel v1.28.0/go.mod h1:q68ijF8Fc8CnMHKyzqL6akLO46ePnjkgfIMIjUIX9z4= go.opentelemetry.io/otel/metric v1.28.0 h1:f0HGvSl1KRAU1DLgLGFjrwVyismPlnuU6JD6bOeuA5Q= @@ -147,31 +288,100 @@ go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc= go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/arch v0.6.0 h1:S0JTfE48HbRj80+4tbvZDYsJ3tGv6BUU3XxyZ7CirAc= golang.org/x/arch v0.6.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.47.0 h1:V6e3FRj+n4dbpw86FJ8Fv7XVOql7TEwpHapKoMJ/GO8= golang.org/x/crypto v0.47.0/go.mod h1:ff3Y9VzzKbwSSEzWqJsJVBnWmRwRSHt/6Op5n9bQc4A= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20231226003508-02704c960a9b h1:kLiC65FbiHWFAOu+lxwNPujcsl8VYyTYYEZnsOO1WK4= golang.org/x/exp v0.0.0-20231226003508-02704c960a9b/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI= -golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU= -golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.49.0 h1:eeHFmOGUTtaaPSGNmjBKpbng9MulQsJURQUAfUwY++o= +golang.org/x/net v0.49.0/go.mod h1:/ysNB2EvaqvesRkuLAyjI1ycPZlQHM3q01F02UY/MV8= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.15.0 h1:s8pnnxNVzjWyrvYdFUQq5llS1PX2zhPXmccZv99h7uQ= +golang.org/x/oauth2 v0.15.0/go.mod h1:q48ptWNTY5XWf+JNten23lcvHpLJ0ZSxF5ttTHKVCAM= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= +golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20211117180635-dee7805ff2e1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ= -golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k= golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= -golang.org/x/term v0.40.0 h1:36e4zGLqU4yhjlmxEaagx2KuYbJq3EwY8K943ZsHcvg= -golang.org/x/term v0.40.0/go.mod h1:w2P8uVp06p2iyKKuvXIm7N/y0UCRt3UfJTfZ7oOpglM= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.33.0 h1:B3njUFyqtHDUI5jMn1YIr5B0IE2U0qck04r6d4KPAxE= golang.org/x/text v0.33.0/go.mod h1:LuMebE6+rBincTi9+xWTY8TztLzKHc/9C1uBCG27+q8= +golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= +golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= +golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= +google.golang.org/api v0.153.0 h1:N1AwGhielyKFaUqH07/ZSIQR3uNPcV7NVw0vj+j4iR4= +google.golang.org/api v0.153.0/go.mod h1:3qNJX5eOmhiWYc67jRA/3GsDw97UFb5ivv7Y2PrriAY= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20231106174013-bbf56f31fb17 h1:wpZ8pe2x1Q3f2KyT5f8oP/fa9rHAKgFPr/HZdNuS+PQ= +google.golang.org/genproto v0.0.0-20231106174013-bbf56f31fb17/go.mod h1:J7XzRzVy1+IPwWHZUzoD0IccYZIrXILAQpc+Qy9CMhY= +google.golang.org/genproto/googleapis/api v0.0.0-20231106174013-bbf56f31fb17 h1:JpwMPBpFN3uKhdaekDpiNlImDdkUAyiJ6ez/uxGaUSo= +google.golang.org/genproto/googleapis/api v0.0.0-20231106174013-bbf56f31fb17/go.mod h1:0xJLfVdJqpAPl8tDg1ujOCGzx6LFLttXT5NhllGOXY4= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231120223509-83a465c0220f h1:ultW7fxlIvee4HYrtnaRPon9HpEgFk5zYpmfMgtKB5I= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231120223509-83a465c0220f/go.mod h1:L9KNLi232K1/xB6f7AlSX692koaRnKaWSR0stBki0Yc= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk= +google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I= google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= @@ -182,4 +392,6 @@ gorm.io/driver/mysql v1.5.2/go.mod h1:pQLhh1Ut/WUAySdTHwBpBv6+JKcj+ua4ZFx1QQTBzb gorm.io/gorm v1.25.2-0.20230530020048-26663ab9bf55/go.mod h1:L4uxeKpfBml98NYqVqwAdmV1a2nBtAec/cf3fpucW/k= gorm.io/gorm v1.25.5 h1:zR9lOiiYf09VNh5Q1gphfyia1JpiClIWG9hQaxB/mls= gorm.io/gorm v1.25.5/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= diff --git a/internal/storage/minio.go b/internal/storage/minio.go new file mode 100644 index 000000000..280694efc --- /dev/null +++ b/internal/storage/minio.go @@ -0,0 +1,397 @@ +// +// Copyright 2026 The InfiniFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package storage + +import ( + "bytes" + "context" + "crypto/tls" + "fmt" + "net/http" + "time" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" + "go.uber.org/zap" +) + +// MinioConfig holds MinIO storage configuration +type MinioConfig struct { + Host string `mapstructure:"host"` // MinIO server host (e.g., "localhost:9000") + User string `mapstructure:"user"` // Access key + Password string `mapstructure:"password"` // Secret key + Secure bool `mapstructure:"secure"` // Use HTTPS + Verify bool `mapstructure:"verify"` // Verify SSL certificates + Bucket string `mapstructure:"bucket"` // Default bucket (optional) + PrefixPath string `mapstructure:"prefix_path"` // Path prefix (optional) +} + +// MinioStorage implements Storage interface for MinIO +type MinioStorage struct { + client *minio.Client + bucket string + prefixPath string + config *MinioConfig +} + +// NewMinioStorage creates a new MinIO storage instance +func NewMinioStorage(config *MinioConfig) (*MinioStorage, error) { + storage := &MinioStorage{ + bucket: config.Bucket, + prefixPath: config.PrefixPath, + config: config, + } + + if err := storage.connect(); err != nil { + return nil, err + } + + return storage, nil +} + +func (m *MinioStorage) connect() error { + var transport http.RoundTripper + + // Configure transport for SSL/TLS verification + if m.config.Secure { + verify := m.config.Verify + transport = &http.Transport{ + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: !verify, + }, + } + } + + client, err := minio.New(m.config.Host, &minio.Options{ + Creds: credentials.NewStaticV4(m.config.User, m.config.Password, ""), + Secure: m.config.Secure, + Transport: transport, + }) + if err != nil { + return fmt.Errorf("failed to connect to MinIO: %w", err) + } + + m.client = client + return nil +} + +func (m *MinioStorage) reconnect() { + if err := m.connect(); err != nil { + zap.L().Error("Failed to reconnect to MinIO", zap.Error(err)) + } +} + +func (m *MinioStorage) resolveBucketAndPath(bucket, fnm string) (string, string) { + actualBucket := bucket + if m.bucket != "" { + actualBucket = m.bucket + } + + actualPath := fnm + if m.bucket != "" { + if m.prefixPath != "" { + actualPath = fmt.Sprintf("%s/%s/%s", m.prefixPath, bucket, fnm) + } else { + actualPath = fmt.Sprintf("%s/%s", bucket, fnm) + } + } else if m.prefixPath != "" { + actualPath = fmt.Sprintf("%s/%s", m.prefixPath, fnm) + } + + return actualBucket, actualPath +} + +// Health checks MinIO service availability +func (m *MinioStorage) Health() bool { + ctx := context.Background() + + if m.bucket != "" { + exists, err := m.client.BucketExists(ctx, m.bucket) + if err != nil { + zap.L().Warn("MinIO health check failed", zap.Error(err)) + return false + } + return exists + } + + _, err := m.client.ListBuckets(ctx) + if err != nil { + zap.L().Warn("MinIO health check failed", zap.Error(err)) + return false + } + return true +} + +// Put uploads an object to MinIO +func (m *MinioStorage) Put(bucket, fnm string, binary []byte, tenantID ...string) error { + bucket, fnm = m.resolveBucketAndPath(bucket, fnm) + + ctx := context.Background() + + for i := 0; i < 3; i++ { + // Ensure bucket exists + if m.bucket == "" { + exists, err := m.client.BucketExists(ctx, bucket) + if err != nil { + zap.L().Error("Failed to check bucket existence", zap.String("bucket", bucket), zap.Error(err)) + m.reconnect() + time.Sleep(time.Second) + continue + } + if !exists { + if err := m.client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{}); err != nil { + zap.L().Error("Failed to create bucket", zap.String("bucket", bucket), zap.Error(err)) + m.reconnect() + time.Sleep(time.Second) + continue + } + } + } + + reader := bytes.NewReader(binary) + _, err := m.client.PutObject(ctx, bucket, fnm, reader, int64(len(binary)), minio.PutObjectOptions{}) + if err != nil { + zap.L().Error("Failed to put object", zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(err)) + m.reconnect() + time.Sleep(time.Second) + continue + } + + return nil + } + + return fmt.Errorf("failed to put object after 3 retries") +} + +// Get retrieves an object from MinIO +func (m *MinioStorage) Get(bucket, fnm string, tenantID ...string) ([]byte, error) { + bucket, fnm = m.resolveBucketAndPath(bucket, fnm) + + ctx := context.Background() + + for i := 0; i < 2; i++ { + obj, err := m.client.GetObject(ctx, bucket, fnm, minio.GetObjectOptions{}) + if err != nil { + zap.L().Error("Failed to get object", zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(err)) + m.reconnect() + time.Sleep(time.Second) + continue + } + defer obj.Close() + + buf := new(bytes.Buffer) + if _, err := buf.ReadFrom(obj); err != nil { + zap.L().Error("Failed to read object data", zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(err)) + m.reconnect() + time.Sleep(time.Second) + continue + } + + return buf.Bytes(), nil + } + + return nil, fmt.Errorf("failed to get object after retries") +} + +// Rm removes an object from MinIO +func (m *MinioStorage) Rm(bucket, fnm string, tenantID ...string) error { + bucket, fnm = m.resolveBucketAndPath(bucket, fnm) + + ctx := context.Background() + + if err := m.client.RemoveObject(ctx, bucket, fnm, minio.RemoveObjectOptions{}); err != nil { + zap.L().Error("Failed to remove object", zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(err)) + return err + } + + return nil +} + +// ObjExist checks if an object exists in MinIO +func (m *MinioStorage) ObjExist(bucket, fnm string, tenantID ...string) bool { + bucket, fnm = m.resolveBucketAndPath(bucket, fnm) + + ctx := context.Background() + + exists, err := m.client.BucketExists(ctx, bucket) + if err != nil || !exists { + return false + } + + _, err = m.client.StatObject(ctx, bucket, fnm, minio.StatObjectOptions{}) + if err != nil { + errResponse := minio.ToErrorResponse(err) + if errResponse.Code == "NoSuchKey" || errResponse.Code == "NoSuchBucket" { + return false + } + zap.L().Error("Failed to stat object", zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(err)) + return false + } + + return true +} + +// GetPresignedURL generates a presigned URL for accessing an object +func (m *MinioStorage) GetPresignedURL(bucket, fnm string, expires time.Duration, tenantID ...string) (string, error) { + bucket, fnm = m.resolveBucketAndPath(bucket, fnm) + + ctx := context.Background() + + for i := 0; i < 10; i++ { + url, err := m.client.PresignedGetObject(ctx, bucket, fnm, expires, nil) + if err != nil { + zap.L().Error("Failed to get presigned URL", zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(err)) + m.reconnect() + time.Sleep(time.Second) + continue + } + + return url.String(), nil + } + + return "", fmt.Errorf("failed to get presigned URL after 10 retries") +} + +// BucketExists checks if a bucket exists +func (m *MinioStorage) BucketExists(bucket string) bool { + actualBucket := bucket + if m.bucket != "" { + actualBucket = m.bucket + } + + ctx := context.Background() + + exists, err := m.client.BucketExists(ctx, actualBucket) + if err != nil { + zap.L().Error("Failed to check bucket existence", zap.String("bucket", actualBucket), zap.Error(err)) + return false + } + + return exists +} + +// RemoveBucket removes a bucket and all its objects +func (m *MinioStorage) RemoveBucket(bucket string) error { + actualBucket := bucket + origBucket := bucket + + if m.bucket != "" { + actualBucket = m.bucket + } + + ctx := context.Background() + + // Build prefix for single-bucket mode + prefix := "" + if m.bucket != "" { + if m.prefixPath != "" { + prefix = fmt.Sprintf("%s/", m.prefixPath) + } + prefix += fmt.Sprintf("%s/", origBucket) + } + + // List and delete objects with prefix + objectsCh := make(chan minio.ObjectInfo) + + go func() { + defer close(objectsCh) + for obj := range m.client.ListObjects(ctx, actualBucket, minio.ListObjectsOptions{ + Prefix: prefix, + Recursive: true, + }) { + if obj.Err != nil { + zap.L().Error("Error listing objects", zap.Error(obj.Err)) + return + } + objectsCh <- obj + } + }() + + for err := range m.client.RemoveObjects(ctx, actualBucket, objectsCh, minio.RemoveObjectsOptions{}) { + zap.L().Error("Failed to remove object", zap.String("key", err.ObjectName), zap.Error(err.Err)) + } + + // Only remove the actual bucket if not in single-bucket mode + if m.bucket == "" { + if err := m.client.RemoveBucket(ctx, actualBucket); err != nil { + zap.L().Error("Failed to remove bucket", zap.String("bucket", actualBucket), zap.Error(err)) + return err + } + } + + return nil +} + +// Copy copies an object from source to destination +func (m *MinioStorage) Copy(srcBucket, srcPath, destBucket, destPath string) bool { + srcBucket, srcPath = m.resolveBucketAndPath(srcBucket, srcPath) + destBucket, destPath = m.resolveBucketAndPath(destBucket, destPath) + + ctx := context.Background() + + // Ensure destination bucket exists + if m.bucket == "" { + exists, err := m.client.BucketExists(ctx, destBucket) + if err != nil { + zap.L().Error("Failed to check bucket existence", zap.String("bucket", destBucket), zap.Error(err)) + return false + } + if !exists { + if err := m.client.MakeBucket(ctx, destBucket, minio.MakeBucketOptions{}); err != nil { + zap.L().Error("Failed to create bucket", zap.String("bucket", destBucket), zap.Error(err)) + return false + } + } + } + + // Check if source object exists + _, err := m.client.StatObject(ctx, srcBucket, srcPath, minio.StatObjectOptions{}) + if err != nil { + zap.L().Error("Source object not found", zap.String("bucket", srcBucket), zap.String("key", srcPath), zap.Error(err)) + return false + } + + // Copy object + srcOpts := minio.CopySrcOptions{ + Bucket: srcBucket, + Object: srcPath, + } + destOpts := minio.CopyDestOptions{ + Bucket: destBucket, + Object: destPath, + } + + _, err = m.client.CopyObject(ctx, destOpts, srcOpts) + if err != nil { + zap.L().Error("Failed to copy object", zap.String("src", fmt.Sprintf("%s/%s", srcBucket, srcPath)), zap.String("dest", fmt.Sprintf("%s/%s", destBucket, destPath)), zap.Error(err)) + return false + } + + return true +} + +// Move moves an object from source to destination +func (m *MinioStorage) Move(srcBucket, srcPath, destBucket, destPath string) bool { + if m.Copy(srcBucket, srcPath, destBucket, destPath) { + if err := m.Rm(srcBucket, srcPath); err != nil { + zap.L().Error("Failed to remove source object after copy", zap.String("bucket", srcBucket), zap.String("key", srcPath), zap.Error(err)) + return false + } + return true + } + return false +} diff --git a/internal/storage/oss.go b/internal/storage/oss.go new file mode 100644 index 000000000..d74aa9da7 --- /dev/null +++ b/internal/storage/oss.go @@ -0,0 +1,415 @@ +// +// Copyright 2026 The InfiniFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package storage + +import ( + "bytes" + "context" + "errors" + "fmt" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/smithy-go" + "go.uber.org/zap" +) + +// OSSConfig holds Aliyun OSS storage configuration +// OSS is compatible with S3 API +type OSSConfig struct { + AccessKeyID string `mapstructure:"access_key"` // OSS Access Key ID + SecretAccessKey string `mapstructure:"secret_key"` // OSS Secret Access Key + EndpointURL string `mapstructure:"endpoint_url"` // OSS Endpoint (e.g., "https://oss-cn-hangzhou.aliyuncs.com") + Region string `mapstructure:"region"` // Region (e.g., "cn-hangzhou") + Bucket string `mapstructure:"bucket"` // Default bucket (optional) + PrefixPath string `mapstructure:"prefix_path"` // Path prefix (optional) + SignatureVersion string `mapstructure:"signature_version"` // Signature version + AddressingStyle string `mapstructure:"addressing_style"` // Addressing style +} + +// OSSStorage implements Storage interface for Aliyun OSS +// OSS uses S3-compatible API +type OSSStorage struct { + client *s3.Client + bucket string + prefixPath string + config *OSSConfig +} + +// NewOSSStorage creates a new OSS storage instance +func NewOSSStorage(config *OSSConfig) (*OSSStorage, error) { + storage := &OSSStorage{ + bucket: config.Bucket, + prefixPath: config.PrefixPath, + config: config, + } + + if err := storage.connect(); err != nil { + return nil, err + } + + return storage, nil +} + +func (o *OSSStorage) connect() error { + ctx := context.Background() + + // Create static credentials + creds := credentials.NewStaticCredentialsProvider( + o.config.AccessKeyID, + o.config.SecretAccessKey, + "", + ) + + // Load configuration + cfg, err := config.LoadDefaultConfig(ctx, + config.WithRegion(o.config.Region), + config.WithCredentialsProvider(creds), + ) + if err != nil { + return fmt.Errorf("failed to load OSS config: %w", err) + } + + // Create S3 client with OSS endpoint + o.client = s3.NewFromConfig(cfg, func(opts *s3.Options) { + opts.BaseEndpoint = aws.String(o.config.EndpointURL) + }) + + return nil +} + +func (o *OSSStorage) reconnect() { + if err := o.connect(); err != nil { + zap.L().Error("Failed to reconnect to OSS", zap.Error(err)) + } +} + +func (o *OSSStorage) resolveBucketAndPath(bucket, fnm string) (string, string) { + actualBucket := bucket + if o.bucket != "" { + actualBucket = o.bucket + } + + actualPath := fnm + if o.prefixPath != "" { + actualPath = fmt.Sprintf("%s/%s", o.prefixPath, fnm) + } + + return actualBucket, actualPath +} + +// Health checks OSS service availability +func (o *OSSStorage) Health() bool { + bucket := o.bucket + if bucket == "" { + bucket = "health-check-bucket" + } + + fnm := "txtxtxtxt1" + if o.prefixPath != "" { + fnm = fmt.Sprintf("%s/%s", o.prefixPath, fnm) + } + binary := []byte("_t@@@1") + + ctx := context.Background() + + // Ensure bucket exists + if !o.BucketExists(bucket) { + _, err := o.client.CreateBucket(ctx, &s3.CreateBucketInput{ + Bucket: aws.String(bucket), + }) + if err != nil { + zap.L().Error("Failed to create bucket for health check", zap.String("bucket", bucket), zap.Error(err)) + return false + } + } + + // Try to upload a test object + reader := bytes.NewReader(binary) + _, err := o.client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(fnm), + Body: reader, + }) + + if err != nil { + zap.L().Error("Health check failed", zap.Error(err)) + return false + } + + return true +} + +// Put uploads an object to OSS +func (o *OSSStorage) Put(bucket, fnm string, binary []byte, tenantID ...string) error { + bucket, fnm = o.resolveBucketAndPath(bucket, fnm) + + ctx := context.Background() + + for i := 0; i < 2; i++ { + // Ensure bucket exists + if !o.BucketExists(bucket) { + _, err := o.client.CreateBucket(ctx, &s3.CreateBucketInput{ + Bucket: aws.String(bucket), + }) + if err != nil { + zap.L().Error("Failed to create bucket", zap.String("bucket", bucket), zap.Error(err)) + o.reconnect() + time.Sleep(time.Second) + continue + } + zap.L().Info("Created bucket", zap.String("bucket", bucket)) + } + + reader := bytes.NewReader(binary) + _, err := o.client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(fnm), + Body: reader, + }) + if err != nil { + zap.L().Error("Failed to put object", zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(err)) + o.reconnect() + time.Sleep(time.Second) + continue + } + + return nil + } + + return fmt.Errorf("failed to put object after retries") +} + +// Get retrieves an object from OSS +func (o *OSSStorage) Get(bucket, fnm string, tenantID ...string) ([]byte, error) { + bucket, fnm = o.resolveBucketAndPath(bucket, fnm) + + ctx := context.Background() + + for i := 0; i < 2; i++ { + result, err := o.client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(fnm), + }) + if err != nil { + zap.L().Error("Failed to get object", zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(err)) + o.reconnect() + time.Sleep(time.Second) + continue + } + defer result.Body.Close() + + buf := new(bytes.Buffer) + if _, err := buf.ReadFrom(result.Body); err != nil { + zap.L().Error("Failed to read object data", zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(err)) + o.reconnect() + time.Sleep(time.Second) + continue + } + + return buf.Bytes(), nil + } + + return nil, fmt.Errorf("failed to get object after retries") +} + +// Rm removes an object from OSS +func (o *OSSStorage) Rm(bucket, fnm string, tenantID ...string) error { + bucket, fnm = o.resolveBucketAndPath(bucket, fnm) + + ctx := context.Background() + + _, err := o.client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(fnm), + }) + if err != nil { + zap.L().Error("Failed to remove object", zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(err)) + return err + } + + return nil +} + +// ObjExist checks if an object exists in OSS +func (o *OSSStorage) ObjExist(bucket, fnm string, tenantID ...string) bool { + bucket, fnm = o.resolveBucketAndPath(bucket, fnm) + + ctx := context.Background() + + _, err := o.client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(fnm), + }) + if err != nil { + if isOSSNotFound(err) { + return false + } + return false + } + + return true +} + +// GetPresignedURL generates a presigned URL for accessing an object +func (o *OSSStorage) GetPresignedURL(bucket, fnm string, expires time.Duration, tenantID ...string) (string, error) { + bucket, fnm = o.resolveBucketAndPath(bucket, fnm) + + ctx := context.Background() + + presignClient := s3.NewPresignClient(o.client) + + for i := 0; i < 10; i++ { + req, err := presignClient.PresignGetObject(ctx, &s3.GetObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(fnm), + }, s3.WithPresignExpires(expires)) + if err != nil { + zap.L().Error("Failed to generate presigned URL", zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(err)) + o.reconnect() + time.Sleep(time.Second) + continue + } + + return req.URL, nil + } + + return "", fmt.Errorf("failed to generate presigned URL after 10 retries") +} + +// BucketExists checks if a bucket exists +func (o *OSSStorage) BucketExists(bucket string) bool { + actualBucket := bucket + if o.bucket != "" { + actualBucket = o.bucket + } + + ctx := context.Background() + + _, err := o.client.HeadBucket(ctx, &s3.HeadBucketInput{ + Bucket: aws.String(actualBucket), + }) + if err != nil { + zap.L().Debug("Bucket does not exist or error", zap.String("bucket", actualBucket), zap.Error(err)) + return false + } + + return true +} + +// RemoveBucket removes a bucket and all its objects +func (o *OSSStorage) RemoveBucket(bucket string) error { + actualBucket := bucket + if o.bucket != "" { + actualBucket = o.bucket + } + + ctx := context.Background() + + // Check if bucket exists + if !o.BucketExists(actualBucket) { + return nil + } + + // List and delete all objects + listInput := &s3.ListObjectsV2Input{ + Bucket: aws.String(actualBucket), + } + + for { + result, err := o.client.ListObjectsV2(ctx, listInput) + if err != nil { + zap.L().Error("Failed to list objects", zap.String("bucket", actualBucket), zap.Error(err)) + return err + } + + for _, obj := range result.Contents { + _, err := o.client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: aws.String(actualBucket), + Key: obj.Key, + }) + if err != nil { + zap.L().Error("Failed to delete object", zap.String("bucket", actualBucket), zap.Error(err)) + } + } + + if result.IsTruncated == nil || !*result.IsTruncated { + break + } + listInput.ContinuationToken = result.NextContinuationToken + } + + // Delete bucket + _, err := o.client.DeleteBucket(ctx, &s3.DeleteBucketInput{ + Bucket: aws.String(actualBucket), + }) + if err != nil { + zap.L().Error("Failed to delete bucket", zap.String("bucket", actualBucket), zap.Error(err)) + return err + } + + return nil +} + +// Copy copies an object from source to destination +func (o *OSSStorage) Copy(srcBucket, srcPath, destBucket, destPath string) bool { + srcBucket, srcPath = o.resolveBucketAndPath(srcBucket, srcPath) + destBucket, destPath = o.resolveBucketAndPath(destBucket, destPath) + + ctx := context.Background() + + copySource := fmt.Sprintf("%s/%s", srcBucket, srcPath) + + _, err := o.client.CopyObject(ctx, &s3.CopyObjectInput{ + Bucket: aws.String(destBucket), + Key: aws.String(destPath), + CopySource: aws.String(copySource), + }) + if err != nil { + zap.L().Error("Failed to copy object", zap.String("src", copySource), zap.String("dest", fmt.Sprintf("%s/%s", destBucket, destPath)), zap.Error(err)) + return false + } + + return true +} + +// Move moves an object from source to destination +func (o *OSSStorage) Move(srcBucket, srcPath, destBucket, destPath string) bool { + if o.Copy(srcBucket, srcPath, destBucket, destPath) { + if err := o.Rm(srcBucket, srcPath); err != nil { + zap.L().Error("Failed to remove source object after copy", zap.String("bucket", srcBucket), zap.String("key", srcPath), zap.Error(err)) + return false + } + return true + } + return false +} + +// Helper functions +func isOSSNotFound(err error) bool { + if err == nil { + return false + } + var apiErr smithy.APIError + if errors.As(err, &apiErr) { + return apiErr.ErrorCode() == "NotFound" || apiErr.ErrorCode() == "404" || apiErr.ErrorCode() == "NoSuchKey" + } + return false +} diff --git a/internal/storage/s3.go b/internal/storage/s3.go new file mode 100644 index 000000000..af49fbd46 --- /dev/null +++ b/internal/storage/s3.go @@ -0,0 +1,425 @@ +// +// Copyright 2026 The InfiniFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package storage + +import ( + "bytes" + "context" + "errors" + "fmt" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/smithy-go" + "go.uber.org/zap" +) + +// S3Config holds AWS S3 storage configuration +type S3Config struct { + AccessKeyID string `mapstructure:"access_key"` // AWS Access Key ID + SecretAccessKey string `mapstructure:"secret_key"` // AWS Secret Access Key + SessionToken string `mapstructure:"session_token"` // AWS Session Token (optional) + Region string `mapstructure:"region_name"` // AWS Region + EndpointURL string `mapstructure:"endpoint_url"` // Custom endpoint (optional) + SignatureVersion string `mapstructure:"signature_version"` // Signature version + AddressingStyle string `mapstructure:"addressing_style"` // Addressing style + Bucket string `mapstructure:"bucket"` // Default bucket (optional) + PrefixPath string `mapstructure:"prefix_path"` // Path prefix (optional) +} + +// S3Storage implements Storage interface for AWS S3 +type S3Storage struct { + client *s3.Client + bucket string + prefixPath string + config *S3Config +} + +// NewS3Storage creates a new S3 storage instance +func NewS3Storage(config *S3Config) (*S3Storage, error) { + storage := &S3Storage{ + bucket: config.Bucket, + prefixPath: config.PrefixPath, + config: config, + } + + if err := storage.connect(); err != nil { + return nil, err + } + + return storage, nil +} + +func (s *S3Storage) connect() error { + ctx := context.Background() + + var opts []func(*config.LoadOptions) error + + // Configure region + if s.config.Region != "" { + opts = append(opts, config.WithRegion(s.config.Region)) + } + + // Configure credentials if provided + if s.config.AccessKeyID != "" && s.config.SecretAccessKey != "" { + creds := credentials.NewStaticCredentialsProvider( + s.config.AccessKeyID, + s.config.SecretAccessKey, + s.config.SessionToken, + ) + opts = append(opts, config.WithCredentialsProvider(creds)) + } + + // Load configuration + cfg, err := config.LoadDefaultConfig(ctx, opts...) + if err != nil { + return fmt.Errorf("failed to load AWS config: %w", err) + } + + // Create S3 client with custom endpoint if provided + clientOpts := []func(*s3.Options){} + if s.config.EndpointURL != "" { + clientOpts = append(clientOpts, func(o *s3.Options) { + o.BaseEndpoint = aws.String(s.config.EndpointURL) + }) + } + + s.client = s3.NewFromConfig(cfg, clientOpts...) + return nil +} + +func (s *S3Storage) reconnect() { + if err := s.connect(); err != nil { + zap.L().Error("Failed to reconnect to S3", zap.Error(err)) + } +} + +func (s *S3Storage) resolveBucketAndPath(bucket, fnm string) (string, string) { + actualBucket := bucket + if s.bucket != "" { + actualBucket = s.bucket + } + + actualPath := fnm + if s.prefixPath != "" { + actualPath = fmt.Sprintf("%s/%s/%s", s.prefixPath, bucket, fnm) + } + + return actualBucket, actualPath +} + +// Health checks S3 service availability +func (s *S3Storage) Health() bool { + bucket := s.bucket + if bucket == "" { + bucket = "health-check-bucket" + } + + fnm := "txtxtxtxt1" + if s.prefixPath != "" { + fnm = fmt.Sprintf("%s/%s", s.prefixPath, fnm) + } + binary := []byte("_t@@@1") + + ctx := context.Background() + + // Ensure bucket exists + if !s.BucketExists(bucket) { + _, err := s.client.CreateBucket(ctx, &s3.CreateBucketInput{ + Bucket: aws.String(bucket), + }) + if err != nil { + zap.L().Error("Failed to create bucket for health check", zap.String("bucket", bucket), zap.Error(err)) + return false + } + } + + // Try to upload a test object + reader := bytes.NewReader(binary) + _, err := s.client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(fnm), + Body: reader, + }) + + if err != nil { + zap.L().Error("Health check failed", zap.Error(err)) + return false + } + + return true +} + +// Put uploads an object to S3 +func (s *S3Storage) Put(bucket, fnm string, binary []byte, tenantID ...string) error { + bucket, fnm = s.resolveBucketAndPath(bucket, fnm) + + ctx := context.Background() + + for i := 0; i < 2; i++ { + // Ensure bucket exists + if !s.BucketExists(bucket) { + _, err := s.client.CreateBucket(ctx, &s3.CreateBucketInput{ + Bucket: aws.String(bucket), + }) + if err != nil { + zap.L().Error("Failed to create bucket", zap.String("bucket", bucket), zap.Error(err)) + s.reconnect() + time.Sleep(time.Second) + continue + } + zap.L().Info("Created bucket", zap.String("bucket", bucket)) + } + + reader := bytes.NewReader(binary) + _, err := s.client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(fnm), + Body: reader, + }) + if err != nil { + zap.L().Error("Failed to put object", zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(err)) + s.reconnect() + time.Sleep(time.Second) + continue + } + + return nil + } + + return fmt.Errorf("failed to put object after retries") +} + +// Get retrieves an object from S3 +func (s *S3Storage) Get(bucket, fnm string, tenantID ...string) ([]byte, error) { + bucket, fnm = s.resolveBucketAndPath(bucket, fnm) + + ctx := context.Background() + + for i := 0; i < 2; i++ { + result, err := s.client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(fnm), + }) + if err != nil { + zap.L().Error("Failed to get object", zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(err)) + s.reconnect() + time.Sleep(time.Second) + continue + } + defer result.Body.Close() + + buf := new(bytes.Buffer) + if _, err := buf.ReadFrom(result.Body); err != nil { + zap.L().Error("Failed to read object data", zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(err)) + s.reconnect() + time.Sleep(time.Second) + continue + } + + return buf.Bytes(), nil + } + + return nil, fmt.Errorf("failed to get object after retries") +} + +// Rm removes an object from S3 +func (s *S3Storage) Rm(bucket, fnm string, tenantID ...string) error { + bucket, fnm = s.resolveBucketAndPath(bucket, fnm) + + ctx := context.Background() + + _, err := s.client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(fnm), + }) + if err != nil { + zap.L().Error("Failed to remove object", zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(err)) + return err + } + + return nil +} + +// ObjExist checks if an object exists in S3 +func (s *S3Storage) ObjExist(bucket, fnm string, tenantID ...string) bool { + bucket, fnm = s.resolveBucketAndPath(bucket, fnm) + + ctx := context.Background() + + _, err := s.client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(fnm), + }) + if err != nil { + if isS3NotFound(err) { + return false + } + return false + } + + return true +} + +// GetPresignedURL generates a presigned URL for accessing an object +func (s *S3Storage) GetPresignedURL(bucket, fnm string, expires time.Duration, tenantID ...string) (string, error) { + bucket, fnm = s.resolveBucketAndPath(bucket, fnm) + + ctx := context.Background() + + presignClient := s3.NewPresignClient(s.client) + + for i := 0; i < 10; i++ { + req, err := presignClient.PresignGetObject(ctx, &s3.GetObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(fnm), + }, s3.WithPresignExpires(expires)) + if err != nil { + zap.L().Error("Failed to generate presigned URL", zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(err)) + s.reconnect() + time.Sleep(time.Second) + continue + } + + return req.URL, nil + } + + return "", fmt.Errorf("failed to generate presigned URL after 10 retries") +} + +// BucketExists checks if a bucket exists +func (s *S3Storage) BucketExists(bucket string) bool { + actualBucket := bucket + if s.bucket != "" { + actualBucket = s.bucket + } + + ctx := context.Background() + + _, err := s.client.HeadBucket(ctx, &s3.HeadBucketInput{ + Bucket: aws.String(actualBucket), + }) + if err != nil { + zap.L().Debug("Bucket does not exist or error", zap.String("bucket", actualBucket), zap.Error(err)) + return false + } + + return true +} + +// RemoveBucket removes a bucket and all its objects +func (s *S3Storage) RemoveBucket(bucket string) error { + actualBucket := bucket + if s.bucket != "" { + actualBucket = s.bucket + } + + ctx := context.Background() + + // Check if bucket exists + if !s.BucketExists(actualBucket) { + return nil + } + + // List and delete all objects + listInput := &s3.ListObjectsV2Input{ + Bucket: aws.String(actualBucket), + } + + for { + result, err := s.client.ListObjectsV2(ctx, listInput) + if err != nil { + zap.L().Error("Failed to list objects", zap.String("bucket", actualBucket), zap.Error(err)) + return err + } + + for _, obj := range result.Contents { + _, err := s.client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: aws.String(actualBucket), + Key: obj.Key, + }) + if err != nil { + zap.L().Error("Failed to delete object", zap.String("bucket", actualBucket), zap.Error(err)) + } + } + + if result.IsTruncated == nil || !*result.IsTruncated { + break + } + listInput.ContinuationToken = result.NextContinuationToken + } + + // Delete bucket + _, err := s.client.DeleteBucket(ctx, &s3.DeleteBucketInput{ + Bucket: aws.String(actualBucket), + }) + if err != nil { + zap.L().Error("Failed to delete bucket", zap.String("bucket", actualBucket), zap.Error(err)) + return err + } + + return nil +} + +// Copy copies an object from source to destination +func (s *S3Storage) Copy(srcBucket, srcPath, destBucket, destPath string) bool { + srcBucket, srcPath = s.resolveBucketAndPath(srcBucket, srcPath) + destBucket, destPath = s.resolveBucketAndPath(destBucket, destPath) + + ctx := context.Background() + + copySource := fmt.Sprintf("%s/%s", srcBucket, srcPath) + + _, err := s.client.CopyObject(ctx, &s3.CopyObjectInput{ + Bucket: aws.String(destBucket), + Key: aws.String(destPath), + CopySource: aws.String(copySource), + }) + if err != nil { + zap.L().Error("Failed to copy object", zap.String("src", copySource), zap.String("dest", fmt.Sprintf("%s/%s", destBucket, destPath)), zap.Error(err)) + return false + } + + return true +} + +// Move moves an object from source to destination +func (s *S3Storage) Move(srcBucket, srcPath, destBucket, destPath string) bool { + if s.Copy(srcBucket, srcPath, destBucket, destPath) { + if err := s.Rm(srcBucket, srcPath); err != nil { + zap.L().Error("Failed to remove source object after copy", zap.String("bucket", srcBucket), zap.String("key", srcPath), zap.Error(err)) + return false + } + return true + } + return false +} + +// isNotFound checks if the error is a not found error +func isS3NotFound(err error) bool { + if err == nil { + return false + } + var apiErr smithy.APIError + if errors.As(err, &apiErr) { + return apiErr.ErrorCode() == "NotFound" || apiErr.ErrorCode() == "404" || apiErr.ErrorCode() == "NoSuchKey" + } + return false +} diff --git a/internal/storage/storage_factory.go b/internal/storage/storage_factory.go new file mode 100644 index 000000000..e7f17d058 --- /dev/null +++ b/internal/storage/storage_factory.go @@ -0,0 +1,298 @@ +// +// Copyright 2026 The InfiniFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package storage + +import ( + "fmt" + "os" + "sync" + + "github.com/spf13/viper" + "go.uber.org/zap" +) + +// StorageFactory creates storage instances based on configuration +type StorageFactory struct { + storageType StorageType + storage Storage + config *StorageConfig + mu sync.RWMutex +} + +// StorageConfig holds all storage-related configurations +type StorageConfig struct { + StorageType string `mapstructure:"storage_type"` + Minio *MinioConfig `mapstructure:"minio"` + S3 *S3Config `mapstructure:"s3"` + OSS *OSSConfig `mapstructure:"oss"` +} + +// AzureConfig holds Azure-specific configurations +type AzureConfig struct { + ContainerURL string `mapstructure:"container_url"` + SASToken string `mapstructure:"sas_token"` + AccountURL string `mapstructure:"account_url"` + ClientID string `mapstructure:"client_id"` + Secret string `mapstructure:"secret"` + TenantID string `mapstructure:"tenant_id"` + ContainerName string `mapstructure:"container_name"` + AuthorityHost string `mapstructure:"authority_host"` +} + +var ( + globalFactory *StorageFactory + once sync.Once +) + +// GetStorageFactory returns the singleton storage factory instance +func GetStorageFactory() *StorageFactory { + once.Do(func() { + globalFactory = &StorageFactory{} + }) + return globalFactory +} + +// InitStorageFactory initializes the storage factory with configuration +func InitStorageFactory(v *viper.Viper) error { + factory := GetStorageFactory() + + // Get storage type from environment or config + storageType := os.Getenv("STORAGE_IMPL") + if storageType == "" { + storageType = v.GetString("storage_type") + } + if storageType == "" { + storageType = "MINIO" // Default storage type + } + + storageConfig := &StorageConfig{} + if err := v.UnmarshalKey("storage", storageConfig); err != nil { + return fmt.Errorf("failed to unmarshal storage config: %w", err) + } + storageConfig.StorageType = storageType + + factory.config = storageConfig + + // Initialize storage based on type + if err := factory.initStorage(storageType, v); err != nil { + return err + } + + zap.L().Info("Storage factory initialized", + zap.String("storage_type", storageType), + ) + + return nil +} + +// initStorage initializes the specific storage implementation +func (f *StorageFactory) initStorage(storageType string, v *viper.Viper) error { + switch storageType { + case "MINIO": + return f.initMinio(v) + case "AWS_S3": + return f.initS3(v) + case "OSS": + return f.initOSS(v) + default: + return fmt.Errorf("unsupported storage type: %s", storageType) + } +} + +func (f *StorageFactory) initMinio(v *viper.Viper) error { + config := &MinioConfig{} + + // Try to load from minio section first + if v.IsSet("minio") { + minioConfig := v.Sub("minio") + if minioConfig != nil { + config.Host = minioConfig.GetString("host") + config.User = minioConfig.GetString("user") + config.Password = minioConfig.GetString("password") + config.Secure = minioConfig.GetBool("secure") + config.Verify = minioConfig.GetBool("verify") + config.Bucket = minioConfig.GetString("bucket") + config.PrefixPath = minioConfig.GetString("prefix_path") + } + } + + // Apply defaults + if config.Host == "" { + config.Host = "localhost:9000" + } + if config.User == "" { + config.User = "minioadmin" + } + if config.Password == "" { + config.Password = "minioadmin" + } + + storage, err := NewMinioStorage(config) + if err != nil { + return fmt.Errorf("failed to create MinIO storage: %w", err) + } + + f.mu.Lock() + defer f.mu.Unlock() + f.storageType = StorageMinio + f.storage = storage + f.config.Minio = config + + return nil +} + +func (f *StorageFactory) initS3(v *viper.Viper) error { + config := &S3Config{} + + if v.IsSet("s3") { + s3Config := v.Sub("s3") + if s3Config != nil { + config.AccessKeyID = s3Config.GetString("access_key") + config.SecretAccessKey = s3Config.GetString("secret_key") + config.SessionToken = s3Config.GetString("session_token") + config.Region = s3Config.GetString("region_name") + config.EndpointURL = s3Config.GetString("endpoint_url") + config.SignatureVersion = s3Config.GetString("signature_version") + config.AddressingStyle = s3Config.GetString("addressing_style") + config.Bucket = s3Config.GetString("bucket") + config.PrefixPath = s3Config.GetString("prefix_path") + } + } + + storage, err := NewS3Storage(config) + if err != nil { + return fmt.Errorf("failed to create S3 storage: %w", err) + } + + f.mu.Lock() + defer f.mu.Unlock() + f.storageType = StorageAWSS3 + f.storage = storage + f.config.S3 = config + + return nil +} + +func (f *StorageFactory) initOSS(v *viper.Viper) error { + config := &OSSConfig{} + + if v.IsSet("oss") { + ossConfig := v.Sub("oss") + if ossConfig != nil { + config.AccessKeyID = ossConfig.GetString("access_key") + config.SecretAccessKey = ossConfig.GetString("secret_key") + config.EndpointURL = ossConfig.GetString("endpoint_url") + config.Region = ossConfig.GetString("region") + config.Bucket = ossConfig.GetString("bucket") + config.PrefixPath = ossConfig.GetString("prefix_path") + config.SignatureVersion = ossConfig.GetString("signature_version") + config.AddressingStyle = ossConfig.GetString("addressing_style") + } + } + + storage, err := NewOSSStorage(config) + if err != nil { + return fmt.Errorf("failed to create OSS storage: %w", err) + } + + f.mu.Lock() + defer f.mu.Unlock() + f.storageType = StorageOSS + f.storage = storage + f.config.OSS = config + + return nil +} + +// GetStorage returns the current storage instance +func (f *StorageFactory) GetStorage() Storage { + f.mu.RLock() + defer f.mu.RUnlock() + return f.storage +} + +// GetStorageType returns the current storage type +func (f *StorageFactory) GetStorageType() StorageType { + f.mu.RLock() + defer f.mu.RUnlock() + return f.storageType +} + +// Create creates a new storage instance based on the storage type +// This is the factory method equivalent to Python's StorageFactory.create() +func (f *StorageFactory) Create(storageType StorageType) (Storage, error) { + var storage Storage + var err error + + switch storageType { + case StorageMinio: + if f.config.Minio != nil { + storage, err = NewMinioStorage(f.config.Minio) + } else { + return nil, fmt.Errorf("MinIO config not available") + } + case StorageAWSS3: + if f.config.S3 != nil { + storage, err = NewS3Storage(f.config.S3) + } else { + return nil, fmt.Errorf("S3 config not available") + } + case StorageOSS: + if f.config.OSS != nil { + storage, err = NewOSSStorage(f.config.OSS) + } else { + return nil, fmt.Errorf("OSS config not available") + } + default: + return nil, fmt.Errorf("unsupported storage type: %v", storageType) + } + + if err != nil { + return nil, err + } + + return storage, nil +} + +// SetStorage sets the storage instance (useful for testing) +func (f *StorageFactory) SetStorage(storage Storage) { + f.mu.Lock() + defer f.mu.Unlock() + f.storage = storage +} + +// StorageTypeMapping returns the storage type mapping (equivalent to Python's storage_mapping) +var StorageTypeMapping = map[StorageType]func(*StorageConfig) (Storage, error){ + StorageMinio: func(config *StorageConfig) (Storage, error) { + if config.Minio == nil { + return nil, fmt.Errorf("MinIO config not available") + } + return NewMinioStorage(config.Minio) + }, + StorageAWSS3: func(config *StorageConfig) (Storage, error) { + if config.S3 == nil { + return nil, fmt.Errorf("S3 config not available") + } + return NewS3Storage(config.S3) + }, + StorageOSS: func(config *StorageConfig) (Storage, error) { + if config.OSS == nil { + return nil, fmt.Errorf("OSS config not available") + } + return NewOSSStorage(config.OSS) + }, +} diff --git a/internal/storage/types.go b/internal/storage/types.go new file mode 100644 index 000000000..fc777373a --- /dev/null +++ b/internal/storage/types.go @@ -0,0 +1,102 @@ +// +// Copyright 2026 The InfiniFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package storage + +import ( + "errors" + "time" +) + +var ( + // ErrNotFound is returned when an object is not found + ErrNotFound = errors.New("object not found") + // ErrBucketNotFound is returned when a bucket is not found + ErrBucketNotFound = errors.New("bucket not found") +) + +// StorageType represents the type of storage backend +type StorageType int + +const ( + StorageMinio StorageType = 1 + StorageAzureSpn StorageType = 2 + StorageAzureSas StorageType = 3 + StorageAWSS3 StorageType = 4 + StorageOSS StorageType = 5 + StorageOpenDAL StorageType = 6 + StorageGCS StorageType = 7 +) + +func (s StorageType) String() string { + switch s { + case StorageMinio: + return "MINIO" + case StorageAzureSpn: + return "AZURE_SPN" + case StorageAzureSas: + return "AZURE_SAS" + case StorageAWSS3: + return "AWS_S3" + case StorageOSS: + return "OSS" + case StorageOpenDAL: + return "OPENDAL" + case StorageGCS: + return "GCS" + default: + return "UNKNOWN" + } +} + +// Storage defines the interface for storage operations +type Storage interface { + // Health checks the storage service availability + Health() bool + + // Put uploads an object to storage + // bucket: the bucket/container name + // fnm: the file/object name (key) + // binary: the data to upload + // tenantID: optional tenant identifier + Put(bucket, fnm string, binary []byte, tenantID ...string) error + + // Get retrieves an object from storage + // Returns the data or nil if not found + Get(bucket, fnm string, tenantID ...string) ([]byte, error) + + // Rm removes an object from storage + Rm(bucket, fnm string, tenantID ...string) error + + // ObjExist checks if an object exists + ObjExist(bucket, fnm string, tenantID ...string) bool + + // GetPresignedURL generates a presigned URL for accessing an object + // expires: duration until the URL expires + GetPresignedURL(bucket, fnm string, expires time.Duration, tenantID ...string) (string, error) + + // BucketExists checks if a bucket exists + BucketExists(bucket string) bool + + // RemoveBucket removes a bucket and all its objects + RemoveBucket(bucket string) error + + // Copy copies an object from source to destination + Copy(srcBucket, srcPath, destBucket, destPath string) bool + + // Move moves an object from source to destination + Move(srcBucket, srcPath, destBucket, destPath string) bool +}