mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-03-23 23:47:37 +08:00
Create go version storage component, but not used (#13561)
### What problem does this PR solve? Implement: minio, s3, oss, azure_sas, azure_spn, gcs, opendal ### Type of change - [x] New Feature (non-breaking change which adds functionality) --------- Signed-off-by: Jin Hai <haijin.chn@gmail.com>
This commit is contained in:
Jin Hai
GitHub
73
go.mod
73
go.mod
@ -1,51 +1,99 @@
|
||||
module ragflow
|
||||
|
||||
go 1.24.0
|
||||
go 1.25
|
||||
|
||||
require (
|
||||
cloud.google.com/go/storage v1.35.1
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake v1.4.4
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.3
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.11
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.11
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.96.4
|
||||
github.com/aws/smithy-go v1.24.2
|
||||
github.com/elastic/go-elasticsearch/v8 v8.19.1
|
||||
github.com/gin-gonic/gin v1.9.1
|
||||
github.com/google/uuid v1.4.0
|
||||
github.com/go-sql-driver/mysql v1.7.0
|
||||
github.com/google/uuid v1.6.0
|
||||
github.com/iromli/go-itsdangerous v0.0.0-20220223194502-9c8bef8dac6a
|
||||
github.com/minio/minio-go/v7 v7.0.99
|
||||
github.com/peterh/liner v1.2.2
|
||||
github.com/redis/go-redis/v9 v9.18.0
|
||||
github.com/siongui/gojianfan v0.0.0-20210926212422-2f175ac615de
|
||||
github.com/spf13/viper v1.18.2
|
||||
go.uber.org/zap v1.27.1
|
||||
golang.org/x/crypto v0.47.0
|
||||
google.golang.org/api v0.153.0
|
||||
gorm.io/driver/mysql v1.5.2
|
||||
gorm.io/gorm v1.25.5
|
||||
)
|
||||
|
||||
require (
|
||||
cloud.google.com/go v0.110.10 // indirect
|
||||
cloud.google.com/go/compute v1.23.3 // indirect
|
||||
cloud.google.com/go/compute/metadata v0.2.3 // indirect
|
||||
cloud.google.com/go/iam v1.1.5 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 // indirect
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.6 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.19 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.19 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.19 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.5 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.20 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.6 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.11 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.19 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.19 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.7 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.12 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.16 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.8 // indirect
|
||||
github.com/bytedance/sonic v1.9.1 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||
github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
|
||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||
github.com/elastic/elastic-transport-go/v8 v8.8.0 // indirect
|
||||
github.com/fsnotify/fsnotify v1.7.0 // indirect
|
||||
github.com/gabriel-vasile/mimetype v1.4.2 // indirect
|
||||
github.com/gin-contrib/sse v0.1.0 // indirect
|
||||
github.com/go-ini/ini v1.67.0 // indirect
|
||||
github.com/go-logr/logr v1.4.2 // indirect
|
||||
github.com/go-logr/stdr v1.2.2 // indirect
|
||||
github.com/go-playground/locales v0.14.1 // indirect
|
||||
github.com/go-playground/universal-translator v0.18.1 // indirect
|
||||
github.com/go-playground/validator/v10 v10.16.0 // indirect
|
||||
github.com/go-sql-driver/mysql v1.7.0 // indirect
|
||||
github.com/goccy/go-json v0.10.2 // indirect
|
||||
github.com/golang-jwt/jwt/v5 v5.3.0 // indirect
|
||||
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
|
||||
github.com/golang/protobuf v1.5.3 // indirect
|
||||
github.com/google/s2a-go v0.1.7 // indirect
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect
|
||||
github.com/googleapis/gax-go/v2 v2.12.0 // indirect
|
||||
github.com/hashicorp/hcl v1.0.0 // indirect
|
||||
github.com/jinzhu/inflection v1.0.0 // indirect
|
||||
github.com/jinzhu/now v1.1.5 // indirect
|
||||
github.com/json-iterator/go v1.1.12 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.2.4 // indirect
|
||||
github.com/klauspost/compress v1.18.2 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.2.11 // indirect
|
||||
github.com/klauspost/crc32 v1.3.0 // indirect
|
||||
github.com/kylelemons/godebug v1.1.0 // indirect
|
||||
github.com/leodido/go-urn v1.2.4 // indirect
|
||||
github.com/magiconair/properties v1.8.7 // indirect
|
||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.3 // indirect
|
||||
github.com/minio/crc64nvme v1.1.1 // indirect
|
||||
github.com/minio/md5-simd v1.1.2 // indirect
|
||||
github.com/mitchellh/mapstructure v1.5.0 // indirect
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
|
||||
github.com/modern-go/reflect2 v1.0.2 // indirect
|
||||
github.com/pelletier/go-toml/v2 v2.1.1 // indirect
|
||||
github.com/peterh/liner v1.2.2 // indirect
|
||||
github.com/philhofer/fwd v1.2.0 // indirect
|
||||
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
|
||||
github.com/rs/xid v1.6.0 // indirect
|
||||
github.com/sagikazarmark/locafero v0.4.0 // indirect
|
||||
github.com/sagikazarmark/slog-shim v0.1.0 // indirect
|
||||
github.com/sourcegraph/conc v0.3.0 // indirect
|
||||
@ -53,19 +101,30 @@ require (
|
||||
github.com/spf13/cast v1.6.0 // indirect
|
||||
github.com/spf13/pflag v1.0.5 // indirect
|
||||
github.com/subosito/gotenv v1.6.0 // indirect
|
||||
github.com/tinylib/msgp v1.6.1 // indirect
|
||||
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
|
||||
github.com/ugorji/go/codec v1.2.12 // indirect
|
||||
go.opencensus.io v0.24.0 // indirect
|
||||
go.opentelemetry.io/otel v1.28.0 // indirect
|
||||
go.opentelemetry.io/otel/metric v1.28.0 // indirect
|
||||
go.opentelemetry.io/otel/trace v1.28.0 // indirect
|
||||
go.uber.org/atomic v1.11.0 // indirect
|
||||
go.uber.org/multierr v1.10.0 // indirect
|
||||
go.yaml.in/yaml/v3 v3.0.4 // indirect
|
||||
golang.org/x/arch v0.6.0 // indirect
|
||||
golang.org/x/exp v0.0.0-20231226003508-02704c960a9b // indirect
|
||||
golang.org/x/net v0.48.0 // indirect
|
||||
golang.org/x/net v0.49.0 // indirect
|
||||
golang.org/x/oauth2 v0.15.0 // indirect
|
||||
golang.org/x/sync v0.19.0 // indirect
|
||||
golang.org/x/sys v0.41.0 // indirect
|
||||
golang.org/x/term v0.40.0 // indirect
|
||||
golang.org/x/text v0.33.0 // indirect
|
||||
golang.org/x/time v0.5.0 // indirect
|
||||
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
|
||||
google.golang.org/appengine v1.6.7 // indirect
|
||||
google.golang.org/genproto v0.0.0-20231106174013-bbf56f31fb17 // indirect
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20231106174013-bbf56f31fb17 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20231120223509-83a465c0220f // indirect
|
||||
google.golang.org/grpc v1.59.0 // indirect
|
||||
google.golang.org/protobuf v1.32.0 // indirect
|
||||
gopkg.in/ini.v1 v1.67.0 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
|
||||
246
go.sum
246
go.sum
@ -1,3 +1,71 @@
|
||||
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
|
||||
cloud.google.com/go v0.110.10 h1:LXy9GEO+timppncPIAZoOj3l58LIU9k+kn48AN7IO3Y=
|
||||
cloud.google.com/go v0.110.10/go.mod h1:v1OoFqYxiBkUrruItNM3eT4lLByNjxmJSV/xDKJNnic=
|
||||
cloud.google.com/go/compute v1.23.3 h1:6sVlXXBmbd7jNX0Ipq0trII3e4n1/MsADLK6a+aiVlk=
|
||||
cloud.google.com/go/compute v1.23.3/go.mod h1:VCgBUoMnIVIR0CscqQiPJLAG25E3ZRZMzcFZeQ+h8CI=
|
||||
cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY=
|
||||
cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA=
|
||||
cloud.google.com/go/iam v1.1.5 h1:1jTsCu4bcsNsE4iiqNT5SHwrDRCfRmIaaaVFhRveTJI=
|
||||
cloud.google.com/go/iam v1.1.5/go.mod h1:rB6P/Ic3mykPbFio+vo7403drjlgvoWfYpJhMXEbzv8=
|
||||
cloud.google.com/go/storage v1.35.1 h1:B59ahL//eDfx2IIKFBeT5Atm9wnNmj3+8xG/W4WB//w=
|
||||
cloud.google.com/go/storage v1.35.1/go.mod h1:M6M/3V/D3KpzMTJyPOR/HU6n2Si5QdaXYEsng2xgOs8=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0 h1:fou+2+WFTib47nS+nz/ozhEBnvU96bKHy6LjRsY4E28=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0/go.mod h1:t76Ruy8AHvUAC8GfMWJMa0ElSbuIcO03NLpynfbgsPA=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 h1:Hk5QBxZQC1jb2Fwj6mpzme37xbCDdNTxU7O9eb5+LB4=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1/go.mod h1:IYus9qsFobWIc2YVwe/WPjcnyCkPKtnHAqUYeebc8z0=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2 h1:yz1bePFlP5Vws5+8ez6T3HWXPmwOK7Yvq8QxDBD3SKY=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2/go.mod h1:Pa9ZNPuoNu/GztvBSKk9J1cDJW6vk/n0zLtV4mgd8N8=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 h1:9iefClla7iYpfYWdzPCRDozdmndjTm8DXdpCzPajMgA=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2/go.mod h1:XtLgD3ZD34DAaVIIAyG3objl5DynM3CQ/vMcbBNJZGI=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1 h1:/Zt+cDPnpC3OVDm/JKLOs7M2DKmLRIIp3XIx9pHHiig=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1/go.mod h1:Ng3urmn6dYe8gnbCMoHHVl5APYz2txho3koEkV2o2HA=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4 h1:jWQK1GI+LeGGUKBADtcH2rRqPxYB1Ljwms5gFA2LqrM=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4/go.mod h1:8mwH4klAm9DUgR2EEHyEEAQlRDvLPyg5fQry3y+cDew=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake v1.4.4 h1:7QtoGxKm6mPhsWzEZtrn3tQF1hmMMZblngnqNoE61I8=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake v1.4.4/go.mod h1:juYrzH1q6A+g9ZZbGh0OmjS7zaMq3rFDrPhVnYSgFMA=
|
||||
github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM=
|
||||
github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE=
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 h1:XRzhVemXdgvJqCH0sFfrBUTnUJSBrBf7++ypk+twtRs=
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk=
|
||||
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.3 h1:4kQ/fa22KjDt13QCy1+bYADvdgcxpfH18f0zP542kZA=
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.3/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.6 h1:N4lRUXZpZ1KVEUn6hxtco/1d2lgYhNn1fHkkl8WhlyQ=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.6/go.mod h1:lyw7GFp3qENLh7kwzf7iMzAxDn+NzjXEAGjKS2UOKqI=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.11 h1:ftxI5sgz8jZkckuUHXfC/wMUc8u3fG1vQS0plr2F2Zs=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.11/go.mod h1:twF11+6ps9aNRKEDimksp923o44w/Thk9+8YIlzWMmo=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.11 h1:NdV8cwCcAXrCWyxArt58BrvZJ9pZ9Fhf9w6Uh5W3Uyc=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.11/go.mod h1:30yY2zqkMPdrvxBqzI9xQCM+WrlrZKSOpSJEsylVU+8=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.19 h1:INUvJxmhdEbVulJYHI061k4TVuS3jzzthNvjqvVvTKM=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.19/go.mod h1:FpZN2QISLdEBWkayloda+sZjVJL+e9Gl0k1SyTgcswU=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.19 h1:/sECfyq2JTifMI2JPyZ4bdRN77zJmr6SrS1eL3augIA=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.19/go.mod h1:dMf8A5oAqr9/oxOfLkC/c2LU/uMcALP0Rgn2BD5LWn0=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.19 h1:AWeJMk33GTBf6J20XJe6qZoRSJo0WfUhsMdUKhoODXE=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.19/go.mod h1:+GWrYoaAsV7/4pNHpwh1kiNLXkKaSoppxQq9lbH8Ejw=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.5 h1:clHU5fm//kWS1C2HgtgWxfQbFbx4b6rx+5jzhgX9HrI=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.5/go.mod h1:O3h0IK87yXci+kg6flUKzJnWeziQUKciKrLjcatSNcY=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.20 h1:qi3e/dmpdONhj1RyIZdi6DKKpDXS5Lb8ftr3p7cyHJc=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.20/go.mod h1:V1K+TeJVD5JOk3D9e5tsX2KUdL7BlB+FV6cBhdobN8c=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.6 h1:XAq62tBTJP/85lFD5oqOOe7YYgWxY9LvWq8plyDvDVg=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.6/go.mod h1:x0nZssQ3qZSnIcePWLvcoFisRXJzcTVvYpAAdYX8+GI=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.11 h1:BYf7XNsJMzl4mObARUBUib+j2tf0U//JAAtTnYqvqCw=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.11/go.mod h1:aEUS4WrNk/+FxkBZZa7tVgp4pGH+kFGW40Y8rCPqt5g=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.19 h1:X1Tow7suZk9UCJHE1Iw9GMZJJl0dAnKXXP1NaSDHwmw=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.19/go.mod h1:/rARO8psX+4sfjUQXp5LLifjUt8DuATZ31WptNJTyQA=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.19 h1:JnQeStZvPHFHeyky/7LbMlyQjUa+jIBj36OlWm0pzIk=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.19/go.mod h1:HGyasyHvYdFQeJhvDHfH7HXkHh57htcJGKDZ+7z+I24=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.96.4 h1:4ExZyubQ6LQQVuF2Qp9OsfEvsTdAWh5Gfwf6PgIdLdk=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.96.4/go.mod h1:NF3JcMGOiARAss1ld3WGORCw71+4ExDD2cbbdKS5PpA=
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.7 h1:Y2cAXlClHsXkkOvWZFXATr34b0hxxloeQu/pAZz2row=
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.7/go.mod h1:idzZ7gmDeqeNrSPkdbtMp9qWMgcBwykA7P7Rzh5DXVU=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.12 h1:iSsvB9EtQ09YrsmIc44Heqlx5ByGErqhPK1ZQLppias=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.12/go.mod h1:fEWYKTRGoZNl8tZ77i61/ccwOMJdGxwOhWCkp6TXAr0=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.16 h1:EnUdUqRP1CNzt2DkV67tJx6XDN4xlfBFm+bzeNOQVb0=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.16/go.mod h1:Jic/xv0Rq/pFNCh3WwpH4BEqdbSAl+IyHro8LbibHD8=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.8 h1:XQTQTF75vnug2TXS8m7CVJfC2nniYPZnO1D4Np761Oo=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.8/go.mod h1:Xgx+PR1NUOjNmQY+tRMnouRp83JRM8pRMw/vCaVhPkI=
|
||||
github.com/aws/smithy-go v1.24.2 h1:FzA3bu/nt/vDvmnkg+R8Xl46gmzEDam6mZ1hzmwXFng=
|
||||
github.com/aws/smithy-go v1.24.2/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
|
||||
github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
|
||||
github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
|
||||
github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
|
||||
@ -5,21 +73,30 @@ github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0
|
||||
github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM=
|
||||
github.com/bytedance/sonic v1.9.1 h1:6iJ6NqdoxCDr6mbY8h18oSO+cShGSMRGCEo7F2h0x8s=
|
||||
github.com/bytedance/sonic v1.9.1/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U=
|
||||
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
|
||||
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
|
||||
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY=
|
||||
github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 h1:qSGYFH7+jGhDF8vLC+iwCD4WpbV1EBDSzWkJODFLams=
|
||||
github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk=
|
||||
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
|
||||
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
|
||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
|
||||
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
||||
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
||||
github.com/elastic/elastic-transport-go/v8 v8.8.0 h1:7k1Ua+qluFr6p1jfJjGDl97ssJS/P7cHNInzfxgBQAo=
|
||||
github.com/elastic/elastic-transport-go/v8 v8.8.0/go.mod h1:YLHer5cj0csTzNFXoNQ8qhtGY1GTvSqPnKWKaqQE3Hk=
|
||||
github.com/elastic/go-elasticsearch/v8 v8.19.1 h1:0iEGt5/Ds9MNVxEp3hqLsXdbe6SjleaVHONg/FuR09Q=
|
||||
github.com/elastic/go-elasticsearch/v8 v8.19.1/go.mod h1:tHJQdInFa6abmDbDCEH2LJja07l/SIpaGpJcm13nt7s=
|
||||
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
|
||||
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
|
||||
github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
|
||||
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
|
||||
github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
|
||||
github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
|
||||
github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=
|
||||
@ -30,6 +107,8 @@ github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE
|
||||
github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
|
||||
github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg=
|
||||
github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU=
|
||||
github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A=
|
||||
github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
|
||||
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
|
||||
github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
|
||||
github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
|
||||
@ -47,11 +126,47 @@ github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ
|
||||
github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI=
|
||||
github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
|
||||
github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
|
||||
github.com/golang-jwt/jwt/v5 v5.3.0 h1:pv4AsKCKKZuqlgs5sUmn4x8UlGa0kEVt/puTpKx9vvo=
|
||||
github.com/golang-jwt/jwt/v5 v5.3.0/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE=
|
||||
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
|
||||
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
|
||||
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
|
||||
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
|
||||
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
|
||||
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
|
||||
github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
|
||||
github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
|
||||
github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
|
||||
github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
|
||||
github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
|
||||
github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
|
||||
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
|
||||
github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
|
||||
github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
|
||||
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
|
||||
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
|
||||
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
|
||||
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
|
||||
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
||||
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
|
||||
github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4=
|
||||
github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw=
|
||||
github.com/google/martian/v3 v3.3.2/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk=
|
||||
github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o=
|
||||
github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw=
|
||||
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs=
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0=
|
||||
github.com/googleapis/gax-go/v2 v2.12.0 h1:A+gCJKdRfqXkr+BIRGtZLibNXf0m1f9E4HG56etFpas=
|
||||
github.com/googleapis/gax-go/v2 v2.12.0/go.mod h1:y+aIqrI5eb1YGMVJfuV3185Ts/D7qKpsEkdD5+I6QGU=
|
||||
github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
|
||||
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
|
||||
github.com/iromli/go-itsdangerous v0.0.0-20220223194502-9c8bef8dac6a h1:Inib12UR9HAfBubrGNraPjKt/Cu8xPbTJbC50+0wP5U=
|
||||
@ -62,13 +177,22 @@ github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
|
||||
github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
|
||||
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
|
||||
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
|
||||
github.com/keybase/go-keychain v0.0.1 h1:way+bWYa6lDppZoZcgMbYsvC7GxljxrskdNInRtuthU=
|
||||
github.com/keybase/go-keychain v0.0.1/go.mod h1:PdEILRW3i9D8JcdM+FmY6RwkHGnhHxXwkPPMeUgOK1k=
|
||||
github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk=
|
||||
github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
|
||||
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||
github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||
github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk=
|
||||
github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY=
|
||||
github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU=
|
||||
github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
|
||||
github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM=
|
||||
github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw=
|
||||
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
||||
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
|
||||
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
|
||||
github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q=
|
||||
github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4=
|
||||
github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY=
|
||||
@ -77,6 +201,12 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE
|
||||
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||
github.com/mattn/go-runewidth v0.0.3 h1:a+kO+98RDGEfo6asOGMmpodZq4FNtnGP54yps8BzLR4=
|
||||
github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
|
||||
github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI=
|
||||
github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg=
|
||||
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
|
||||
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
|
||||
github.com/minio/minio-go/v7 v7.0.99 h1:2vH/byrwUkIpFQFOilvTfaUpvAX3fEFhEzO+DR3DlCE=
|
||||
github.com/minio/minio-go/v7 v7.0.99/go.mod h1:EtGNKtlX20iL2yaYnxEigaIvj0G0GwSDnifnG8ClIdw=
|
||||
github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
|
||||
github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
|
||||
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
@ -88,13 +218,20 @@ github.com/pelletier/go-toml/v2 v2.1.1 h1:LWAJwfNvjQZCFIDKWYQaM62NcYeYViCmWIwmOS
|
||||
github.com/pelletier/go-toml/v2 v2.1.1/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc=
|
||||
github.com/peterh/liner v1.2.2 h1:aJ4AOodmL+JxOZZEL2u9iJf8omNRpqHc/EbrK+3mAXw=
|
||||
github.com/peterh/liner v1.2.2/go.mod h1:xFwJyiKIXJZUKItq5dGHZSTBRAuG/CpeNpWLyiNRNwI=
|
||||
github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM=
|
||||
github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM=
|
||||
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ=
|
||||
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
|
||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
|
||||
github.com/redis/go-redis/v9 v9.18.0 h1:pMkxYPkEbMPwRdenAzUNyFNrDgHx9U+DrBabWNfSRQs=
|
||||
github.com/redis/go-redis/v9 v9.18.0/go.mod h1:k3ufPphLU5YXwNTUcCRXGxUoF1fqxnhFQmscfkCoDA0=
|
||||
github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
|
||||
github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
|
||||
github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
|
||||
github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
|
||||
github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
|
||||
github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
|
||||
github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ=
|
||||
github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4=
|
||||
github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE=
|
||||
@ -121,16 +258,20 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO
|
||||
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
|
||||
github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
|
||||
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
|
||||
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
|
||||
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
|
||||
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
||||
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||
github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
|
||||
github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
|
||||
github.com/tinylib/msgp v1.6.1 h1:ESRv8eL3u+DNHUoSAAQRE50Hm162zqAnBoGv9PzScPY=
|
||||
github.com/tinylib/msgp v1.6.1/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA=
|
||||
github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI=
|
||||
github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
|
||||
github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE=
|
||||
github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg=
|
||||
github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0=
|
||||
github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA=
|
||||
go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
|
||||
go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
|
||||
go.opentelemetry.io/otel v1.28.0 h1:/SqNcYk+idO0CxKEUOtKQClMK/MimZihKYMruSMViUo=
|
||||
go.opentelemetry.io/otel v1.28.0/go.mod h1:q68ijF8Fc8CnMHKyzqL6akLO46ePnjkgfIMIjUIX9z4=
|
||||
go.opentelemetry.io/otel/metric v1.28.0 h1:f0HGvSl1KRAU1DLgLGFjrwVyismPlnuU6JD6bOeuA5Q=
|
||||
@ -147,31 +288,100 @@ go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ=
|
||||
go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
|
||||
go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc=
|
||||
go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
|
||||
go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
|
||||
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
|
||||
golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
|
||||
golang.org/x/arch v0.6.0 h1:S0JTfE48HbRj80+4tbvZDYsJ3tGv6BUU3XxyZ7CirAc=
|
||||
golang.org/x/arch v0.6.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.47.0 h1:V6e3FRj+n4dbpw86FJ8Fv7XVOql7TEwpHapKoMJ/GO8=
|
||||
golang.org/x/crypto v0.47.0/go.mod h1:ff3Y9VzzKbwSSEzWqJsJVBnWmRwRSHt/6Op5n9bQc4A=
|
||||
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20231226003508-02704c960a9b h1:kLiC65FbiHWFAOu+lxwNPujcsl8VYyTYYEZnsOO1WK4=
|
||||
golang.org/x/exp v0.0.0-20231226003508-02704c960a9b/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI=
|
||||
golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU=
|
||||
golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY=
|
||||
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
|
||||
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
|
||||
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
|
||||
golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
|
||||
golang.org/x/net v0.49.0 h1:eeHFmOGUTtaaPSGNmjBKpbng9MulQsJURQUAfUwY++o=
|
||||
golang.org/x/net v0.49.0/go.mod h1:/ysNB2EvaqvesRkuLAyjI1ycPZlQHM3q01F02UY/MV8=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.15.0 h1:s8pnnxNVzjWyrvYdFUQq5llS1PX2zhPXmccZv99h7uQ=
|
||||
golang.org/x/oauth2 v0.15.0/go.mod h1:q48ptWNTY5XWf+JNten23lcvHpLJ0ZSxF5ttTHKVCAM=
|
||||
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4=
|
||||
golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
|
||||
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20211117180635-dee7805ff2e1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ=
|
||||
golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
|
||||
golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k=
|
||||
golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
|
||||
golang.org/x/term v0.40.0 h1:36e4zGLqU4yhjlmxEaagx2KuYbJq3EwY8K943ZsHcvg=
|
||||
golang.org/x/term v0.40.0/go.mod h1:w2P8uVp06p2iyKKuvXIm7N/y0UCRt3UfJTfZ7oOpglM=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.33.0 h1:B3njUFyqtHDUI5jMn1YIr5B0IE2U0qck04r6d4KPAxE=
|
||||
golang.org/x/text v0.33.0/go.mod h1:LuMebE6+rBincTi9+xWTY8TztLzKHc/9C1uBCG27+q8=
|
||||
golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
|
||||
golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
|
||||
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||
golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk=
|
||||
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=
|
||||
google.golang.org/api v0.153.0 h1:N1AwGhielyKFaUqH07/ZSIQR3uNPcV7NVw0vj+j4iR4=
|
||||
google.golang.org/api v0.153.0/go.mod h1:3qNJX5eOmhiWYc67jRA/3GsDw97UFb5ivv7Y2PrriAY=
|
||||
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
|
||||
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
|
||||
google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
|
||||
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
|
||||
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
|
||||
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
|
||||
google.golang.org/genproto v0.0.0-20231106174013-bbf56f31fb17 h1:wpZ8pe2x1Q3f2KyT5f8oP/fa9rHAKgFPr/HZdNuS+PQ=
|
||||
google.golang.org/genproto v0.0.0-20231106174013-bbf56f31fb17/go.mod h1:J7XzRzVy1+IPwWHZUzoD0IccYZIrXILAQpc+Qy9CMhY=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20231106174013-bbf56f31fb17 h1:JpwMPBpFN3uKhdaekDpiNlImDdkUAyiJ6ez/uxGaUSo=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20231106174013-bbf56f31fb17/go.mod h1:0xJLfVdJqpAPl8tDg1ujOCGzx6LFLttXT5NhllGOXY4=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20231120223509-83a465c0220f h1:ultW7fxlIvee4HYrtnaRPon9HpEgFk5zYpmfMgtKB5I=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20231120223509-83a465c0220f/go.mod h1:L9KNLi232K1/xB6f7AlSX692koaRnKaWSR0stBki0Yc=
|
||||
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
|
||||
google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
|
||||
google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
|
||||
google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
|
||||
google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
|
||||
google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk=
|
||||
google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98=
|
||||
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
|
||||
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
|
||||
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
|
||||
google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
|
||||
google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
|
||||
google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
|
||||
google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
|
||||
google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
|
||||
google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
|
||||
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
|
||||
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
|
||||
google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I=
|
||||
google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
|
||||
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
|
||||
gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
|
||||
gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
@ -182,4 +392,6 @@ gorm.io/driver/mysql v1.5.2/go.mod h1:pQLhh1Ut/WUAySdTHwBpBv6+JKcj+ua4ZFx1QQTBzb
|
||||
gorm.io/gorm v1.25.2-0.20230530020048-26663ab9bf55/go.mod h1:L4uxeKpfBml98NYqVqwAdmV1a2nBtAec/cf3fpucW/k=
|
||||
gorm.io/gorm v1.25.5 h1:zR9lOiiYf09VNh5Q1gphfyia1JpiClIWG9hQaxB/mls=
|
||||
gorm.io/gorm v1.25.5/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8=
|
||||
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=
|
||||
|
||||
397
internal/storage/minio.go
Normal file
397
internal/storage/minio.go
Normal file
@ -0,0 +1,397 @@
|
||||
//
|
||||
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
|
||||
package storage
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
// MinioConfig holds MinIO storage configuration
|
||||
type MinioConfig struct {
|
||||
Host string `mapstructure:"host"` // MinIO server host (e.g., "localhost:9000")
|
||||
User string `mapstructure:"user"` // Access key
|
||||
Password string `mapstructure:"password"` // Secret key
|
||||
Secure bool `mapstructure:"secure"` // Use HTTPS
|
||||
Verify bool `mapstructure:"verify"` // Verify SSL certificates
|
||||
Bucket string `mapstructure:"bucket"` // Default bucket (optional)
|
||||
PrefixPath string `mapstructure:"prefix_path"` // Path prefix (optional)
|
||||
}
|
||||
|
||||
// MinioStorage implements Storage interface for MinIO
|
||||
type MinioStorage struct {
|
||||
client *minio.Client
|
||||
bucket string
|
||||
prefixPath string
|
||||
config *MinioConfig
|
||||
}
|
||||
|
||||
// NewMinioStorage creates a new MinIO storage instance
|
||||
func NewMinioStorage(config *MinioConfig) (*MinioStorage, error) {
|
||||
storage := &MinioStorage{
|
||||
bucket: config.Bucket,
|
||||
prefixPath: config.PrefixPath,
|
||||
config: config,
|
||||
}
|
||||
|
||||
if err := storage.connect(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return storage, nil
|
||||
}
|
||||
|
||||
func (m *MinioStorage) connect() error {
|
||||
var transport http.RoundTripper
|
||||
|
||||
// Configure transport for SSL/TLS verification
|
||||
if m.config.Secure {
|
||||
verify := m.config.Verify
|
||||
transport = &http.Transport{
|
||||
TLSClientConfig: &tls.Config{
|
||||
InsecureSkipVerify: !verify,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
client, err := minio.New(m.config.Host, &minio.Options{
|
||||
Creds: credentials.NewStaticV4(m.config.User, m.config.Password, ""),
|
||||
Secure: m.config.Secure,
|
||||
Transport: transport,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to connect to MinIO: %w", err)
|
||||
}
|
||||
|
||||
m.client = client
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *MinioStorage) reconnect() {
|
||||
if err := m.connect(); err != nil {
|
||||
zap.L().Error("Failed to reconnect to MinIO", zap.Error(err))
|
||||
}
|
||||
}
|
||||
|
||||
func (m *MinioStorage) resolveBucketAndPath(bucket, fnm string) (string, string) {
|
||||
actualBucket := bucket
|
||||
if m.bucket != "" {
|
||||
actualBucket = m.bucket
|
||||
}
|
||||
|
||||
actualPath := fnm
|
||||
if m.bucket != "" {
|
||||
if m.prefixPath != "" {
|
||||
actualPath = fmt.Sprintf("%s/%s/%s", m.prefixPath, bucket, fnm)
|
||||
} else {
|
||||
actualPath = fmt.Sprintf("%s/%s", bucket, fnm)
|
||||
}
|
||||
} else if m.prefixPath != "" {
|
||||
actualPath = fmt.Sprintf("%s/%s", m.prefixPath, fnm)
|
||||
}
|
||||
|
||||
return actualBucket, actualPath
|
||||
}
|
||||
|
||||
// Health checks MinIO service availability
|
||||
func (m *MinioStorage) Health() bool {
|
||||
ctx := context.Background()
|
||||
|
||||
if m.bucket != "" {
|
||||
exists, err := m.client.BucketExists(ctx, m.bucket)
|
||||
if err != nil {
|
||||
zap.L().Warn("MinIO health check failed", zap.Error(err))
|
||||
return false
|
||||
}
|
||||
return exists
|
||||
}
|
||||
|
||||
_, err := m.client.ListBuckets(ctx)
|
||||
if err != nil {
|
||||
zap.L().Warn("MinIO health check failed", zap.Error(err))
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// Put uploads an object to MinIO
|
||||
func (m *MinioStorage) Put(bucket, fnm string, binary []byte, tenantID ...string) error {
|
||||
bucket, fnm = m.resolveBucketAndPath(bucket, fnm)
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
for i := 0; i < 3; i++ {
|
||||
// Ensure bucket exists
|
||||
if m.bucket == "" {
|
||||
exists, err := m.client.BucketExists(ctx, bucket)
|
||||
if err != nil {
|
||||
zap.L().Error("Failed to check bucket existence", zap.String("bucket", bucket), zap.Error(err))
|
||||
m.reconnect()
|
||||
time.Sleep(time.Second)
|
||||
continue
|
||||
}
|
||||
if !exists {
|
||||
if err := m.client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{}); err != nil {
|
||||
zap.L().Error("Failed to create bucket", zap.String("bucket", bucket), zap.Error(err))
|
||||
m.reconnect()
|
||||
time.Sleep(time.Second)
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
reader := bytes.NewReader(binary)
|
||||
_, err := m.client.PutObject(ctx, bucket, fnm, reader, int64(len(binary)), minio.PutObjectOptions{})
|
||||
if err != nil {
|
||||
zap.L().Error("Failed to put object", zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(err))
|
||||
m.reconnect()
|
||||
time.Sleep(time.Second)
|
||||
continue
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
return fmt.Errorf("failed to put object after 3 retries")
|
||||
}
|
||||
|
||||
// Get retrieves an object from MinIO
|
||||
func (m *MinioStorage) Get(bucket, fnm string, tenantID ...string) ([]byte, error) {
|
||||
bucket, fnm = m.resolveBucketAndPath(bucket, fnm)
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
for i := 0; i < 2; i++ {
|
||||
obj, err := m.client.GetObject(ctx, bucket, fnm, minio.GetObjectOptions{})
|
||||
if err != nil {
|
||||
zap.L().Error("Failed to get object", zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(err))
|
||||
m.reconnect()
|
||||
time.Sleep(time.Second)
|
||||
continue
|
||||
}
|
||||
defer obj.Close()
|
||||
|
||||
buf := new(bytes.Buffer)
|
||||
if _, err := buf.ReadFrom(obj); err != nil {
|
||||
zap.L().Error("Failed to read object data", zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(err))
|
||||
m.reconnect()
|
||||
time.Sleep(time.Second)
|
||||
continue
|
||||
}
|
||||
|
||||
return buf.Bytes(), nil
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("failed to get object after retries")
|
||||
}
|
||||
|
||||
// Rm removes an object from MinIO
|
||||
func (m *MinioStorage) Rm(bucket, fnm string, tenantID ...string) error {
|
||||
bucket, fnm = m.resolveBucketAndPath(bucket, fnm)
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
if err := m.client.RemoveObject(ctx, bucket, fnm, minio.RemoveObjectOptions{}); err != nil {
|
||||
zap.L().Error("Failed to remove object", zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(err))
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// ObjExist checks if an object exists in MinIO
|
||||
func (m *MinioStorage) ObjExist(bucket, fnm string, tenantID ...string) bool {
|
||||
bucket, fnm = m.resolveBucketAndPath(bucket, fnm)
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
exists, err := m.client.BucketExists(ctx, bucket)
|
||||
if err != nil || !exists {
|
||||
return false
|
||||
}
|
||||
|
||||
_, err = m.client.StatObject(ctx, bucket, fnm, minio.StatObjectOptions{})
|
||||
if err != nil {
|
||||
errResponse := minio.ToErrorResponse(err)
|
||||
if errResponse.Code == "NoSuchKey" || errResponse.Code == "NoSuchBucket" {
|
||||
return false
|
||||
}
|
||||
zap.L().Error("Failed to stat object", zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(err))
|
||||
return false
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
// GetPresignedURL generates a presigned URL for accessing an object
|
||||
func (m *MinioStorage) GetPresignedURL(bucket, fnm string, expires time.Duration, tenantID ...string) (string, error) {
|
||||
bucket, fnm = m.resolveBucketAndPath(bucket, fnm)
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
for i := 0; i < 10; i++ {
|
||||
url, err := m.client.PresignedGetObject(ctx, bucket, fnm, expires, nil)
|
||||
if err != nil {
|
||||
zap.L().Error("Failed to get presigned URL", zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(err))
|
||||
m.reconnect()
|
||||
time.Sleep(time.Second)
|
||||
continue
|
||||
}
|
||||
|
||||
return url.String(), nil
|
||||
}
|
||||
|
||||
return "", fmt.Errorf("failed to get presigned URL after 10 retries")
|
||||
}
|
||||
|
||||
// BucketExists checks if a bucket exists
|
||||
func (m *MinioStorage) BucketExists(bucket string) bool {
|
||||
actualBucket := bucket
|
||||
if m.bucket != "" {
|
||||
actualBucket = m.bucket
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
exists, err := m.client.BucketExists(ctx, actualBucket)
|
||||
if err != nil {
|
||||
zap.L().Error("Failed to check bucket existence", zap.String("bucket", actualBucket), zap.Error(err))
|
||||
return false
|
||||
}
|
||||
|
||||
return exists
|
||||
}
|
||||
|
||||
// RemoveBucket removes a bucket and all its objects
|
||||
func (m *MinioStorage) RemoveBucket(bucket string) error {
|
||||
actualBucket := bucket
|
||||
origBucket := bucket
|
||||
|
||||
if m.bucket != "" {
|
||||
actualBucket = m.bucket
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Build prefix for single-bucket mode
|
||||
prefix := ""
|
||||
if m.bucket != "" {
|
||||
if m.prefixPath != "" {
|
||||
prefix = fmt.Sprintf("%s/", m.prefixPath)
|
||||
}
|
||||
prefix += fmt.Sprintf("%s/", origBucket)
|
||||
}
|
||||
|
||||
// List and delete objects with prefix
|
||||
objectsCh := make(chan minio.ObjectInfo)
|
||||
|
||||
go func() {
|
||||
defer close(objectsCh)
|
||||
for obj := range m.client.ListObjects(ctx, actualBucket, minio.ListObjectsOptions{
|
||||
Prefix: prefix,
|
||||
Recursive: true,
|
||||
}) {
|
||||
if obj.Err != nil {
|
||||
zap.L().Error("Error listing objects", zap.Error(obj.Err))
|
||||
return
|
||||
}
|
||||
objectsCh <- obj
|
||||
}
|
||||
}()
|
||||
|
||||
for err := range m.client.RemoveObjects(ctx, actualBucket, objectsCh, minio.RemoveObjectsOptions{}) {
|
||||
zap.L().Error("Failed to remove object", zap.String("key", err.ObjectName), zap.Error(err.Err))
|
||||
}
|
||||
|
||||
// Only remove the actual bucket if not in single-bucket mode
|
||||
if m.bucket == "" {
|
||||
if err := m.client.RemoveBucket(ctx, actualBucket); err != nil {
|
||||
zap.L().Error("Failed to remove bucket", zap.String("bucket", actualBucket), zap.Error(err))
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Copy copies an object from source to destination
|
||||
func (m *MinioStorage) Copy(srcBucket, srcPath, destBucket, destPath string) bool {
|
||||
srcBucket, srcPath = m.resolveBucketAndPath(srcBucket, srcPath)
|
||||
destBucket, destPath = m.resolveBucketAndPath(destBucket, destPath)
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Ensure destination bucket exists
|
||||
if m.bucket == "" {
|
||||
exists, err := m.client.BucketExists(ctx, destBucket)
|
||||
if err != nil {
|
||||
zap.L().Error("Failed to check bucket existence", zap.String("bucket", destBucket), zap.Error(err))
|
||||
return false
|
||||
}
|
||||
if !exists {
|
||||
if err := m.client.MakeBucket(ctx, destBucket, minio.MakeBucketOptions{}); err != nil {
|
||||
zap.L().Error("Failed to create bucket", zap.String("bucket", destBucket), zap.Error(err))
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Check if source object exists
|
||||
_, err := m.client.StatObject(ctx, srcBucket, srcPath, minio.StatObjectOptions{})
|
||||
if err != nil {
|
||||
zap.L().Error("Source object not found", zap.String("bucket", srcBucket), zap.String("key", srcPath), zap.Error(err))
|
||||
return false
|
||||
}
|
||||
|
||||
// Copy object
|
||||
srcOpts := minio.CopySrcOptions{
|
||||
Bucket: srcBucket,
|
||||
Object: srcPath,
|
||||
}
|
||||
destOpts := minio.CopyDestOptions{
|
||||
Bucket: destBucket,
|
||||
Object: destPath,
|
||||
}
|
||||
|
||||
_, err = m.client.CopyObject(ctx, destOpts, srcOpts)
|
||||
if err != nil {
|
||||
zap.L().Error("Failed to copy object", zap.String("src", fmt.Sprintf("%s/%s", srcBucket, srcPath)), zap.String("dest", fmt.Sprintf("%s/%s", destBucket, destPath)), zap.Error(err))
|
||||
return false
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
// Move moves an object from source to destination
|
||||
func (m *MinioStorage) Move(srcBucket, srcPath, destBucket, destPath string) bool {
|
||||
if m.Copy(srcBucket, srcPath, destBucket, destPath) {
|
||||
if err := m.Rm(srcBucket, srcPath); err != nil {
|
||||
zap.L().Error("Failed to remove source object after copy", zap.String("bucket", srcBucket), zap.String("key", srcPath), zap.Error(err))
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
415
internal/storage/oss.go
Normal file
415
internal/storage/oss.go
Normal file
@ -0,0 +1,415 @@
|
||||
//
|
||||
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
|
||||
package storage
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/config"
|
||||
"github.com/aws/aws-sdk-go-v2/credentials"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/aws/smithy-go"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
// OSSConfig holds Aliyun OSS storage configuration
|
||||
// OSS is compatible with S3 API
|
||||
type OSSConfig struct {
|
||||
AccessKeyID string `mapstructure:"access_key"` // OSS Access Key ID
|
||||
SecretAccessKey string `mapstructure:"secret_key"` // OSS Secret Access Key
|
||||
EndpointURL string `mapstructure:"endpoint_url"` // OSS Endpoint (e.g., "https://oss-cn-hangzhou.aliyuncs.com")
|
||||
Region string `mapstructure:"region"` // Region (e.g., "cn-hangzhou")
|
||||
Bucket string `mapstructure:"bucket"` // Default bucket (optional)
|
||||
PrefixPath string `mapstructure:"prefix_path"` // Path prefix (optional)
|
||||
SignatureVersion string `mapstructure:"signature_version"` // Signature version
|
||||
AddressingStyle string `mapstructure:"addressing_style"` // Addressing style
|
||||
}
|
||||
|
||||
// OSSStorage implements Storage interface for Aliyun OSS
|
||||
// OSS uses S3-compatible API
|
||||
type OSSStorage struct {
|
||||
client *s3.Client
|
||||
bucket string
|
||||
prefixPath string
|
||||
config *OSSConfig
|
||||
}
|
||||
|
||||
// NewOSSStorage creates a new OSS storage instance
|
||||
func NewOSSStorage(config *OSSConfig) (*OSSStorage, error) {
|
||||
storage := &OSSStorage{
|
||||
bucket: config.Bucket,
|
||||
prefixPath: config.PrefixPath,
|
||||
config: config,
|
||||
}
|
||||
|
||||
if err := storage.connect(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return storage, nil
|
||||
}
|
||||
|
||||
func (o *OSSStorage) connect() error {
|
||||
ctx := context.Background()
|
||||
|
||||
// Create static credentials
|
||||
creds := credentials.NewStaticCredentialsProvider(
|
||||
o.config.AccessKeyID,
|
||||
o.config.SecretAccessKey,
|
||||
"",
|
||||
)
|
||||
|
||||
// Load configuration
|
||||
cfg, err := config.LoadDefaultConfig(ctx,
|
||||
config.WithRegion(o.config.Region),
|
||||
config.WithCredentialsProvider(creds),
|
||||
)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to load OSS config: %w", err)
|
||||
}
|
||||
|
||||
// Create S3 client with OSS endpoint
|
||||
o.client = s3.NewFromConfig(cfg, func(opts *s3.Options) {
|
||||
opts.BaseEndpoint = aws.String(o.config.EndpointURL)
|
||||
})
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (o *OSSStorage) reconnect() {
|
||||
if err := o.connect(); err != nil {
|
||||
zap.L().Error("Failed to reconnect to OSS", zap.Error(err))
|
||||
}
|
||||
}
|
||||
|
||||
func (o *OSSStorage) resolveBucketAndPath(bucket, fnm string) (string, string) {
|
||||
actualBucket := bucket
|
||||
if o.bucket != "" {
|
||||
actualBucket = o.bucket
|
||||
}
|
||||
|
||||
actualPath := fnm
|
||||
if o.prefixPath != "" {
|
||||
actualPath = fmt.Sprintf("%s/%s", o.prefixPath, fnm)
|
||||
}
|
||||
|
||||
return actualBucket, actualPath
|
||||
}
|
||||
|
||||
// Health checks OSS service availability
|
||||
func (o *OSSStorage) Health() bool {
|
||||
bucket := o.bucket
|
||||
if bucket == "" {
|
||||
bucket = "health-check-bucket"
|
||||
}
|
||||
|
||||
fnm := "txtxtxtxt1"
|
||||
if o.prefixPath != "" {
|
||||
fnm = fmt.Sprintf("%s/%s", o.prefixPath, fnm)
|
||||
}
|
||||
binary := []byte("_t@@@1")
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Ensure bucket exists
|
||||
if !o.BucketExists(bucket) {
|
||||
_, err := o.client.CreateBucket(ctx, &s3.CreateBucketInput{
|
||||
Bucket: aws.String(bucket),
|
||||
})
|
||||
if err != nil {
|
||||
zap.L().Error("Failed to create bucket for health check", zap.String("bucket", bucket), zap.Error(err))
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// Try to upload a test object
|
||||
reader := bytes.NewReader(binary)
|
||||
_, err := o.client.PutObject(ctx, &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Key: aws.String(fnm),
|
||||
Body: reader,
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
zap.L().Error("Health check failed", zap.Error(err))
|
||||
return false
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
// Put uploads an object to OSS
|
||||
func (o *OSSStorage) Put(bucket, fnm string, binary []byte, tenantID ...string) error {
|
||||
bucket, fnm = o.resolveBucketAndPath(bucket, fnm)
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
for i := 0; i < 2; i++ {
|
||||
// Ensure bucket exists
|
||||
if !o.BucketExists(bucket) {
|
||||
_, err := o.client.CreateBucket(ctx, &s3.CreateBucketInput{
|
||||
Bucket: aws.String(bucket),
|
||||
})
|
||||
if err != nil {
|
||||
zap.L().Error("Failed to create bucket", zap.String("bucket", bucket), zap.Error(err))
|
||||
o.reconnect()
|
||||
time.Sleep(time.Second)
|
||||
continue
|
||||
}
|
||||
zap.L().Info("Created bucket", zap.String("bucket", bucket))
|
||||
}
|
||||
|
||||
reader := bytes.NewReader(binary)
|
||||
_, err := o.client.PutObject(ctx, &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Key: aws.String(fnm),
|
||||
Body: reader,
|
||||
})
|
||||
if err != nil {
|
||||
zap.L().Error("Failed to put object", zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(err))
|
||||
o.reconnect()
|
||||
time.Sleep(time.Second)
|
||||
continue
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
return fmt.Errorf("failed to put object after retries")
|
||||
}
|
||||
|
||||
// Get retrieves an object from OSS
|
||||
func (o *OSSStorage) Get(bucket, fnm string, tenantID ...string) ([]byte, error) {
|
||||
bucket, fnm = o.resolveBucketAndPath(bucket, fnm)
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
for i := 0; i < 2; i++ {
|
||||
result, err := o.client.GetObject(ctx, &s3.GetObjectInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Key: aws.String(fnm),
|
||||
})
|
||||
if err != nil {
|
||||
zap.L().Error("Failed to get object", zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(err))
|
||||
o.reconnect()
|
||||
time.Sleep(time.Second)
|
||||
continue
|
||||
}
|
||||
defer result.Body.Close()
|
||||
|
||||
buf := new(bytes.Buffer)
|
||||
if _, err := buf.ReadFrom(result.Body); err != nil {
|
||||
zap.L().Error("Failed to read object data", zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(err))
|
||||
o.reconnect()
|
||||
time.Sleep(time.Second)
|
||||
continue
|
||||
}
|
||||
|
||||
return buf.Bytes(), nil
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("failed to get object after retries")
|
||||
}
|
||||
|
||||
// Rm removes an object from OSS
|
||||
func (o *OSSStorage) Rm(bucket, fnm string, tenantID ...string) error {
|
||||
bucket, fnm = o.resolveBucketAndPath(bucket, fnm)
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
_, err := o.client.DeleteObject(ctx, &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Key: aws.String(fnm),
|
||||
})
|
||||
if err != nil {
|
||||
zap.L().Error("Failed to remove object", zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(err))
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// ObjExist checks if an object exists in OSS
|
||||
func (o *OSSStorage) ObjExist(bucket, fnm string, tenantID ...string) bool {
|
||||
bucket, fnm = o.resolveBucketAndPath(bucket, fnm)
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
_, err := o.client.HeadObject(ctx, &s3.HeadObjectInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Key: aws.String(fnm),
|
||||
})
|
||||
if err != nil {
|
||||
if isOSSNotFound(err) {
|
||||
return false
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
// GetPresignedURL generates a presigned URL for accessing an object
|
||||
func (o *OSSStorage) GetPresignedURL(bucket, fnm string, expires time.Duration, tenantID ...string) (string, error) {
|
||||
bucket, fnm = o.resolveBucketAndPath(bucket, fnm)
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
presignClient := s3.NewPresignClient(o.client)
|
||||
|
||||
for i := 0; i < 10; i++ {
|
||||
req, err := presignClient.PresignGetObject(ctx, &s3.GetObjectInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Key: aws.String(fnm),
|
||||
}, s3.WithPresignExpires(expires))
|
||||
if err != nil {
|
||||
zap.L().Error("Failed to generate presigned URL", zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(err))
|
||||
o.reconnect()
|
||||
time.Sleep(time.Second)
|
||||
continue
|
||||
}
|
||||
|
||||
return req.URL, nil
|
||||
}
|
||||
|
||||
return "", fmt.Errorf("failed to generate presigned URL after 10 retries")
|
||||
}
|
||||
|
||||
// BucketExists checks if a bucket exists
|
||||
func (o *OSSStorage) BucketExists(bucket string) bool {
|
||||
actualBucket := bucket
|
||||
if o.bucket != "" {
|
||||
actualBucket = o.bucket
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
_, err := o.client.HeadBucket(ctx, &s3.HeadBucketInput{
|
||||
Bucket: aws.String(actualBucket),
|
||||
})
|
||||
if err != nil {
|
||||
zap.L().Debug("Bucket does not exist or error", zap.String("bucket", actualBucket), zap.Error(err))
|
||||
return false
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
// RemoveBucket removes a bucket and all its objects
|
||||
func (o *OSSStorage) RemoveBucket(bucket string) error {
|
||||
actualBucket := bucket
|
||||
if o.bucket != "" {
|
||||
actualBucket = o.bucket
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Check if bucket exists
|
||||
if !o.BucketExists(actualBucket) {
|
||||
return nil
|
||||
}
|
||||
|
||||
// List and delete all objects
|
||||
listInput := &s3.ListObjectsV2Input{
|
||||
Bucket: aws.String(actualBucket),
|
||||
}
|
||||
|
||||
for {
|
||||
result, err := o.client.ListObjectsV2(ctx, listInput)
|
||||
if err != nil {
|
||||
zap.L().Error("Failed to list objects", zap.String("bucket", actualBucket), zap.Error(err))
|
||||
return err
|
||||
}
|
||||
|
||||
for _, obj := range result.Contents {
|
||||
_, err := o.client.DeleteObject(ctx, &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(actualBucket),
|
||||
Key: obj.Key,
|
||||
})
|
||||
if err != nil {
|
||||
zap.L().Error("Failed to delete object", zap.String("bucket", actualBucket), zap.Error(err))
|
||||
}
|
||||
}
|
||||
|
||||
if result.IsTruncated == nil || !*result.IsTruncated {
|
||||
break
|
||||
}
|
||||
listInput.ContinuationToken = result.NextContinuationToken
|
||||
}
|
||||
|
||||
// Delete bucket
|
||||
_, err := o.client.DeleteBucket(ctx, &s3.DeleteBucketInput{
|
||||
Bucket: aws.String(actualBucket),
|
||||
})
|
||||
if err != nil {
|
||||
zap.L().Error("Failed to delete bucket", zap.String("bucket", actualBucket), zap.Error(err))
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Copy copies an object from source to destination
|
||||
func (o *OSSStorage) Copy(srcBucket, srcPath, destBucket, destPath string) bool {
|
||||
srcBucket, srcPath = o.resolveBucketAndPath(srcBucket, srcPath)
|
||||
destBucket, destPath = o.resolveBucketAndPath(destBucket, destPath)
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
copySource := fmt.Sprintf("%s/%s", srcBucket, srcPath)
|
||||
|
||||
_, err := o.client.CopyObject(ctx, &s3.CopyObjectInput{
|
||||
Bucket: aws.String(destBucket),
|
||||
Key: aws.String(destPath),
|
||||
CopySource: aws.String(copySource),
|
||||
})
|
||||
if err != nil {
|
||||
zap.L().Error("Failed to copy object", zap.String("src", copySource), zap.String("dest", fmt.Sprintf("%s/%s", destBucket, destPath)), zap.Error(err))
|
||||
return false
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
// Move moves an object from source to destination
|
||||
func (o *OSSStorage) Move(srcBucket, srcPath, destBucket, destPath string) bool {
|
||||
if o.Copy(srcBucket, srcPath, destBucket, destPath) {
|
||||
if err := o.Rm(srcBucket, srcPath); err != nil {
|
||||
zap.L().Error("Failed to remove source object after copy", zap.String("bucket", srcBucket), zap.String("key", srcPath), zap.Error(err))
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Helper functions
|
||||
func isOSSNotFound(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
var apiErr smithy.APIError
|
||||
if errors.As(err, &apiErr) {
|
||||
return apiErr.ErrorCode() == "NotFound" || apiErr.ErrorCode() == "404" || apiErr.ErrorCode() == "NoSuchKey"
|
||||
}
|
||||
return false
|
||||
}
|
||||
425
internal/storage/s3.go
Normal file
425
internal/storage/s3.go
Normal file
@ -0,0 +1,425 @@
|
||||
//
|
||||
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
|
||||
package storage
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/config"
|
||||
"github.com/aws/aws-sdk-go-v2/credentials"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/aws/smithy-go"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
// S3Config holds AWS S3 storage configuration
|
||||
type S3Config struct {
|
||||
AccessKeyID string `mapstructure:"access_key"` // AWS Access Key ID
|
||||
SecretAccessKey string `mapstructure:"secret_key"` // AWS Secret Access Key
|
||||
SessionToken string `mapstructure:"session_token"` // AWS Session Token (optional)
|
||||
Region string `mapstructure:"region_name"` // AWS Region
|
||||
EndpointURL string `mapstructure:"endpoint_url"` // Custom endpoint (optional)
|
||||
SignatureVersion string `mapstructure:"signature_version"` // Signature version
|
||||
AddressingStyle string `mapstructure:"addressing_style"` // Addressing style
|
||||
Bucket string `mapstructure:"bucket"` // Default bucket (optional)
|
||||
PrefixPath string `mapstructure:"prefix_path"` // Path prefix (optional)
|
||||
}
|
||||
|
||||
// S3Storage implements Storage interface for AWS S3
|
||||
type S3Storage struct {
|
||||
client *s3.Client
|
||||
bucket string
|
||||
prefixPath string
|
||||
config *S3Config
|
||||
}
|
||||
|
||||
// NewS3Storage creates a new S3 storage instance
|
||||
func NewS3Storage(config *S3Config) (*S3Storage, error) {
|
||||
storage := &S3Storage{
|
||||
bucket: config.Bucket,
|
||||
prefixPath: config.PrefixPath,
|
||||
config: config,
|
||||
}
|
||||
|
||||
if err := storage.connect(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return storage, nil
|
||||
}
|
||||
|
||||
func (s *S3Storage) connect() error {
|
||||
ctx := context.Background()
|
||||
|
||||
var opts []func(*config.LoadOptions) error
|
||||
|
||||
// Configure region
|
||||
if s.config.Region != "" {
|
||||
opts = append(opts, config.WithRegion(s.config.Region))
|
||||
}
|
||||
|
||||
// Configure credentials if provided
|
||||
if s.config.AccessKeyID != "" && s.config.SecretAccessKey != "" {
|
||||
creds := credentials.NewStaticCredentialsProvider(
|
||||
s.config.AccessKeyID,
|
||||
s.config.SecretAccessKey,
|
||||
s.config.SessionToken,
|
||||
)
|
||||
opts = append(opts, config.WithCredentialsProvider(creds))
|
||||
}
|
||||
|
||||
// Load configuration
|
||||
cfg, err := config.LoadDefaultConfig(ctx, opts...)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to load AWS config: %w", err)
|
||||
}
|
||||
|
||||
// Create S3 client with custom endpoint if provided
|
||||
clientOpts := []func(*s3.Options){}
|
||||
if s.config.EndpointURL != "" {
|
||||
clientOpts = append(clientOpts, func(o *s3.Options) {
|
||||
o.BaseEndpoint = aws.String(s.config.EndpointURL)
|
||||
})
|
||||
}
|
||||
|
||||
s.client = s3.NewFromConfig(cfg, clientOpts...)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *S3Storage) reconnect() {
|
||||
if err := s.connect(); err != nil {
|
||||
zap.L().Error("Failed to reconnect to S3", zap.Error(err))
|
||||
}
|
||||
}
|
||||
|
||||
func (s *S3Storage) resolveBucketAndPath(bucket, fnm string) (string, string) {
|
||||
actualBucket := bucket
|
||||
if s.bucket != "" {
|
||||
actualBucket = s.bucket
|
||||
}
|
||||
|
||||
actualPath := fnm
|
||||
if s.prefixPath != "" {
|
||||
actualPath = fmt.Sprintf("%s/%s/%s", s.prefixPath, bucket, fnm)
|
||||
}
|
||||
|
||||
return actualBucket, actualPath
|
||||
}
|
||||
|
||||
// Health checks S3 service availability
|
||||
func (s *S3Storage) Health() bool {
|
||||
bucket := s.bucket
|
||||
if bucket == "" {
|
||||
bucket = "health-check-bucket"
|
||||
}
|
||||
|
||||
fnm := "txtxtxtxt1"
|
||||
if s.prefixPath != "" {
|
||||
fnm = fmt.Sprintf("%s/%s", s.prefixPath, fnm)
|
||||
}
|
||||
binary := []byte("_t@@@1")
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Ensure bucket exists
|
||||
if !s.BucketExists(bucket) {
|
||||
_, err := s.client.CreateBucket(ctx, &s3.CreateBucketInput{
|
||||
Bucket: aws.String(bucket),
|
||||
})
|
||||
if err != nil {
|
||||
zap.L().Error("Failed to create bucket for health check", zap.String("bucket", bucket), zap.Error(err))
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// Try to upload a test object
|
||||
reader := bytes.NewReader(binary)
|
||||
_, err := s.client.PutObject(ctx, &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Key: aws.String(fnm),
|
||||
Body: reader,
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
zap.L().Error("Health check failed", zap.Error(err))
|
||||
return false
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
// Put uploads an object to S3
|
||||
func (s *S3Storage) Put(bucket, fnm string, binary []byte, tenantID ...string) error {
|
||||
bucket, fnm = s.resolveBucketAndPath(bucket, fnm)
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
for i := 0; i < 2; i++ {
|
||||
// Ensure bucket exists
|
||||
if !s.BucketExists(bucket) {
|
||||
_, err := s.client.CreateBucket(ctx, &s3.CreateBucketInput{
|
||||
Bucket: aws.String(bucket),
|
||||
})
|
||||
if err != nil {
|
||||
zap.L().Error("Failed to create bucket", zap.String("bucket", bucket), zap.Error(err))
|
||||
s.reconnect()
|
||||
time.Sleep(time.Second)
|
||||
continue
|
||||
}
|
||||
zap.L().Info("Created bucket", zap.String("bucket", bucket))
|
||||
}
|
||||
|
||||
reader := bytes.NewReader(binary)
|
||||
_, err := s.client.PutObject(ctx, &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Key: aws.String(fnm),
|
||||
Body: reader,
|
||||
})
|
||||
if err != nil {
|
||||
zap.L().Error("Failed to put object", zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(err))
|
||||
s.reconnect()
|
||||
time.Sleep(time.Second)
|
||||
continue
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
return fmt.Errorf("failed to put object after retries")
|
||||
}
|
||||
|
||||
// Get retrieves an object from S3
|
||||
func (s *S3Storage) Get(bucket, fnm string, tenantID ...string) ([]byte, error) {
|
||||
bucket, fnm = s.resolveBucketAndPath(bucket, fnm)
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
for i := 0; i < 2; i++ {
|
||||
result, err := s.client.GetObject(ctx, &s3.GetObjectInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Key: aws.String(fnm),
|
||||
})
|
||||
if err != nil {
|
||||
zap.L().Error("Failed to get object", zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(err))
|
||||
s.reconnect()
|
||||
time.Sleep(time.Second)
|
||||
continue
|
||||
}
|
||||
defer result.Body.Close()
|
||||
|
||||
buf := new(bytes.Buffer)
|
||||
if _, err := buf.ReadFrom(result.Body); err != nil {
|
||||
zap.L().Error("Failed to read object data", zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(err))
|
||||
s.reconnect()
|
||||
time.Sleep(time.Second)
|
||||
continue
|
||||
}
|
||||
|
||||
return buf.Bytes(), nil
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("failed to get object after retries")
|
||||
}
|
||||
|
||||
// Rm removes an object from S3
|
||||
func (s *S3Storage) Rm(bucket, fnm string, tenantID ...string) error {
|
||||
bucket, fnm = s.resolveBucketAndPath(bucket, fnm)
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
_, err := s.client.DeleteObject(ctx, &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Key: aws.String(fnm),
|
||||
})
|
||||
if err != nil {
|
||||
zap.L().Error("Failed to remove object", zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(err))
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// ObjExist checks if an object exists in S3
|
||||
func (s *S3Storage) ObjExist(bucket, fnm string, tenantID ...string) bool {
|
||||
bucket, fnm = s.resolveBucketAndPath(bucket, fnm)
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
_, err := s.client.HeadObject(ctx, &s3.HeadObjectInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Key: aws.String(fnm),
|
||||
})
|
||||
if err != nil {
|
||||
if isS3NotFound(err) {
|
||||
return false
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
// GetPresignedURL generates a presigned URL for accessing an object
|
||||
func (s *S3Storage) GetPresignedURL(bucket, fnm string, expires time.Duration, tenantID ...string) (string, error) {
|
||||
bucket, fnm = s.resolveBucketAndPath(bucket, fnm)
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
presignClient := s3.NewPresignClient(s.client)
|
||||
|
||||
for i := 0; i < 10; i++ {
|
||||
req, err := presignClient.PresignGetObject(ctx, &s3.GetObjectInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Key: aws.String(fnm),
|
||||
}, s3.WithPresignExpires(expires))
|
||||
if err != nil {
|
||||
zap.L().Error("Failed to generate presigned URL", zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(err))
|
||||
s.reconnect()
|
||||
time.Sleep(time.Second)
|
||||
continue
|
||||
}
|
||||
|
||||
return req.URL, nil
|
||||
}
|
||||
|
||||
return "", fmt.Errorf("failed to generate presigned URL after 10 retries")
|
||||
}
|
||||
|
||||
// BucketExists checks if a bucket exists
|
||||
func (s *S3Storage) BucketExists(bucket string) bool {
|
||||
actualBucket := bucket
|
||||
if s.bucket != "" {
|
||||
actualBucket = s.bucket
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
_, err := s.client.HeadBucket(ctx, &s3.HeadBucketInput{
|
||||
Bucket: aws.String(actualBucket),
|
||||
})
|
||||
if err != nil {
|
||||
zap.L().Debug("Bucket does not exist or error", zap.String("bucket", actualBucket), zap.Error(err))
|
||||
return false
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
// RemoveBucket removes a bucket and all its objects
|
||||
func (s *S3Storage) RemoveBucket(bucket string) error {
|
||||
actualBucket := bucket
|
||||
if s.bucket != "" {
|
||||
actualBucket = s.bucket
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Check if bucket exists
|
||||
if !s.BucketExists(actualBucket) {
|
||||
return nil
|
||||
}
|
||||
|
||||
// List and delete all objects
|
||||
listInput := &s3.ListObjectsV2Input{
|
||||
Bucket: aws.String(actualBucket),
|
||||
}
|
||||
|
||||
for {
|
||||
result, err := s.client.ListObjectsV2(ctx, listInput)
|
||||
if err != nil {
|
||||
zap.L().Error("Failed to list objects", zap.String("bucket", actualBucket), zap.Error(err))
|
||||
return err
|
||||
}
|
||||
|
||||
for _, obj := range result.Contents {
|
||||
_, err := s.client.DeleteObject(ctx, &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(actualBucket),
|
||||
Key: obj.Key,
|
||||
})
|
||||
if err != nil {
|
||||
zap.L().Error("Failed to delete object", zap.String("bucket", actualBucket), zap.Error(err))
|
||||
}
|
||||
}
|
||||
|
||||
if result.IsTruncated == nil || !*result.IsTruncated {
|
||||
break
|
||||
}
|
||||
listInput.ContinuationToken = result.NextContinuationToken
|
||||
}
|
||||
|
||||
// Delete bucket
|
||||
_, err := s.client.DeleteBucket(ctx, &s3.DeleteBucketInput{
|
||||
Bucket: aws.String(actualBucket),
|
||||
})
|
||||
if err != nil {
|
||||
zap.L().Error("Failed to delete bucket", zap.String("bucket", actualBucket), zap.Error(err))
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Copy copies an object from source to destination
|
||||
func (s *S3Storage) Copy(srcBucket, srcPath, destBucket, destPath string) bool {
|
||||
srcBucket, srcPath = s.resolveBucketAndPath(srcBucket, srcPath)
|
||||
destBucket, destPath = s.resolveBucketAndPath(destBucket, destPath)
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
copySource := fmt.Sprintf("%s/%s", srcBucket, srcPath)
|
||||
|
||||
_, err := s.client.CopyObject(ctx, &s3.CopyObjectInput{
|
||||
Bucket: aws.String(destBucket),
|
||||
Key: aws.String(destPath),
|
||||
CopySource: aws.String(copySource),
|
||||
})
|
||||
if err != nil {
|
||||
zap.L().Error("Failed to copy object", zap.String("src", copySource), zap.String("dest", fmt.Sprintf("%s/%s", destBucket, destPath)), zap.Error(err))
|
||||
return false
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
// Move moves an object from source to destination
|
||||
func (s *S3Storage) Move(srcBucket, srcPath, destBucket, destPath string) bool {
|
||||
if s.Copy(srcBucket, srcPath, destBucket, destPath) {
|
||||
if err := s.Rm(srcBucket, srcPath); err != nil {
|
||||
zap.L().Error("Failed to remove source object after copy", zap.String("bucket", srcBucket), zap.String("key", srcPath), zap.Error(err))
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// isNotFound checks if the error is a not found error
|
||||
func isS3NotFound(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
var apiErr smithy.APIError
|
||||
if errors.As(err, &apiErr) {
|
||||
return apiErr.ErrorCode() == "NotFound" || apiErr.ErrorCode() == "404" || apiErr.ErrorCode() == "NoSuchKey"
|
||||
}
|
||||
return false
|
||||
}
|
||||
298
internal/storage/storage_factory.go
Normal file
298
internal/storage/storage_factory.go
Normal file
@ -0,0 +1,298 @@
|
||||
//
|
||||
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
|
||||
package storage
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"sync"
|
||||
|
||||
"github.com/spf13/viper"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
// StorageFactory creates storage instances based on configuration
|
||||
type StorageFactory struct {
|
||||
storageType StorageType
|
||||
storage Storage
|
||||
config *StorageConfig
|
||||
mu sync.RWMutex
|
||||
}
|
||||
|
||||
// StorageConfig holds all storage-related configurations
|
||||
type StorageConfig struct {
|
||||
StorageType string `mapstructure:"storage_type"`
|
||||
Minio *MinioConfig `mapstructure:"minio"`
|
||||
S3 *S3Config `mapstructure:"s3"`
|
||||
OSS *OSSConfig `mapstructure:"oss"`
|
||||
}
|
||||
|
||||
// AzureConfig holds Azure-specific configurations
|
||||
type AzureConfig struct {
|
||||
ContainerURL string `mapstructure:"container_url"`
|
||||
SASToken string `mapstructure:"sas_token"`
|
||||
AccountURL string `mapstructure:"account_url"`
|
||||
ClientID string `mapstructure:"client_id"`
|
||||
Secret string `mapstructure:"secret"`
|
||||
TenantID string `mapstructure:"tenant_id"`
|
||||
ContainerName string `mapstructure:"container_name"`
|
||||
AuthorityHost string `mapstructure:"authority_host"`
|
||||
}
|
||||
|
||||
var (
|
||||
globalFactory *StorageFactory
|
||||
once sync.Once
|
||||
)
|
||||
|
||||
// GetStorageFactory returns the singleton storage factory instance
|
||||
func GetStorageFactory() *StorageFactory {
|
||||
once.Do(func() {
|
||||
globalFactory = &StorageFactory{}
|
||||
})
|
||||
return globalFactory
|
||||
}
|
||||
|
||||
// InitStorageFactory initializes the storage factory with configuration
|
||||
func InitStorageFactory(v *viper.Viper) error {
|
||||
factory := GetStorageFactory()
|
||||
|
||||
// Get storage type from environment or config
|
||||
storageType := os.Getenv("STORAGE_IMPL")
|
||||
if storageType == "" {
|
||||
storageType = v.GetString("storage_type")
|
||||
}
|
||||
if storageType == "" {
|
||||
storageType = "MINIO" // Default storage type
|
||||
}
|
||||
|
||||
storageConfig := &StorageConfig{}
|
||||
if err := v.UnmarshalKey("storage", storageConfig); err != nil {
|
||||
return fmt.Errorf("failed to unmarshal storage config: %w", err)
|
||||
}
|
||||
storageConfig.StorageType = storageType
|
||||
|
||||
factory.config = storageConfig
|
||||
|
||||
// Initialize storage based on type
|
||||
if err := factory.initStorage(storageType, v); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
zap.L().Info("Storage factory initialized",
|
||||
zap.String("storage_type", storageType),
|
||||
)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// initStorage initializes the specific storage implementation
|
||||
func (f *StorageFactory) initStorage(storageType string, v *viper.Viper) error {
|
||||
switch storageType {
|
||||
case "MINIO":
|
||||
return f.initMinio(v)
|
||||
case "AWS_S3":
|
||||
return f.initS3(v)
|
||||
case "OSS":
|
||||
return f.initOSS(v)
|
||||
default:
|
||||
return fmt.Errorf("unsupported storage type: %s", storageType)
|
||||
}
|
||||
}
|
||||
|
||||
func (f *StorageFactory) initMinio(v *viper.Viper) error {
|
||||
config := &MinioConfig{}
|
||||
|
||||
// Try to load from minio section first
|
||||
if v.IsSet("minio") {
|
||||
minioConfig := v.Sub("minio")
|
||||
if minioConfig != nil {
|
||||
config.Host = minioConfig.GetString("host")
|
||||
config.User = minioConfig.GetString("user")
|
||||
config.Password = minioConfig.GetString("password")
|
||||
config.Secure = minioConfig.GetBool("secure")
|
||||
config.Verify = minioConfig.GetBool("verify")
|
||||
config.Bucket = minioConfig.GetString("bucket")
|
||||
config.PrefixPath = minioConfig.GetString("prefix_path")
|
||||
}
|
||||
}
|
||||
|
||||
// Apply defaults
|
||||
if config.Host == "" {
|
||||
config.Host = "localhost:9000"
|
||||
}
|
||||
if config.User == "" {
|
||||
config.User = "minioadmin"
|
||||
}
|
||||
if config.Password == "" {
|
||||
config.Password = "minioadmin"
|
||||
}
|
||||
|
||||
storage, err := NewMinioStorage(config)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create MinIO storage: %w", err)
|
||||
}
|
||||
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.storageType = StorageMinio
|
||||
f.storage = storage
|
||||
f.config.Minio = config
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *StorageFactory) initS3(v *viper.Viper) error {
|
||||
config := &S3Config{}
|
||||
|
||||
if v.IsSet("s3") {
|
||||
s3Config := v.Sub("s3")
|
||||
if s3Config != nil {
|
||||
config.AccessKeyID = s3Config.GetString("access_key")
|
||||
config.SecretAccessKey = s3Config.GetString("secret_key")
|
||||
config.SessionToken = s3Config.GetString("session_token")
|
||||
config.Region = s3Config.GetString("region_name")
|
||||
config.EndpointURL = s3Config.GetString("endpoint_url")
|
||||
config.SignatureVersion = s3Config.GetString("signature_version")
|
||||
config.AddressingStyle = s3Config.GetString("addressing_style")
|
||||
config.Bucket = s3Config.GetString("bucket")
|
||||
config.PrefixPath = s3Config.GetString("prefix_path")
|
||||
}
|
||||
}
|
||||
|
||||
storage, err := NewS3Storage(config)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create S3 storage: %w", err)
|
||||
}
|
||||
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.storageType = StorageAWSS3
|
||||
f.storage = storage
|
||||
f.config.S3 = config
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *StorageFactory) initOSS(v *viper.Viper) error {
|
||||
config := &OSSConfig{}
|
||||
|
||||
if v.IsSet("oss") {
|
||||
ossConfig := v.Sub("oss")
|
||||
if ossConfig != nil {
|
||||
config.AccessKeyID = ossConfig.GetString("access_key")
|
||||
config.SecretAccessKey = ossConfig.GetString("secret_key")
|
||||
config.EndpointURL = ossConfig.GetString("endpoint_url")
|
||||
config.Region = ossConfig.GetString("region")
|
||||
config.Bucket = ossConfig.GetString("bucket")
|
||||
config.PrefixPath = ossConfig.GetString("prefix_path")
|
||||
config.SignatureVersion = ossConfig.GetString("signature_version")
|
||||
config.AddressingStyle = ossConfig.GetString("addressing_style")
|
||||
}
|
||||
}
|
||||
|
||||
storage, err := NewOSSStorage(config)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create OSS storage: %w", err)
|
||||
}
|
||||
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.storageType = StorageOSS
|
||||
f.storage = storage
|
||||
f.config.OSS = config
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetStorage returns the current storage instance
|
||||
func (f *StorageFactory) GetStorage() Storage {
|
||||
f.mu.RLock()
|
||||
defer f.mu.RUnlock()
|
||||
return f.storage
|
||||
}
|
||||
|
||||
// GetStorageType returns the current storage type
|
||||
func (f *StorageFactory) GetStorageType() StorageType {
|
||||
f.mu.RLock()
|
||||
defer f.mu.RUnlock()
|
||||
return f.storageType
|
||||
}
|
||||
|
||||
// Create creates a new storage instance based on the storage type
|
||||
// This is the factory method equivalent to Python's StorageFactory.create()
|
||||
func (f *StorageFactory) Create(storageType StorageType) (Storage, error) {
|
||||
var storage Storage
|
||||
var err error
|
||||
|
||||
switch storageType {
|
||||
case StorageMinio:
|
||||
if f.config.Minio != nil {
|
||||
storage, err = NewMinioStorage(f.config.Minio)
|
||||
} else {
|
||||
return nil, fmt.Errorf("MinIO config not available")
|
||||
}
|
||||
case StorageAWSS3:
|
||||
if f.config.S3 != nil {
|
||||
storage, err = NewS3Storage(f.config.S3)
|
||||
} else {
|
||||
return nil, fmt.Errorf("S3 config not available")
|
||||
}
|
||||
case StorageOSS:
|
||||
if f.config.OSS != nil {
|
||||
storage, err = NewOSSStorage(f.config.OSS)
|
||||
} else {
|
||||
return nil, fmt.Errorf("OSS config not available")
|
||||
}
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported storage type: %v", storageType)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return storage, nil
|
||||
}
|
||||
|
||||
// SetStorage sets the storage instance (useful for testing)
|
||||
func (f *StorageFactory) SetStorage(storage Storage) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.storage = storage
|
||||
}
|
||||
|
||||
// StorageTypeMapping returns the storage type mapping (equivalent to Python's storage_mapping)
|
||||
var StorageTypeMapping = map[StorageType]func(*StorageConfig) (Storage, error){
|
||||
StorageMinio: func(config *StorageConfig) (Storage, error) {
|
||||
if config.Minio == nil {
|
||||
return nil, fmt.Errorf("MinIO config not available")
|
||||
}
|
||||
return NewMinioStorage(config.Minio)
|
||||
},
|
||||
StorageAWSS3: func(config *StorageConfig) (Storage, error) {
|
||||
if config.S3 == nil {
|
||||
return nil, fmt.Errorf("S3 config not available")
|
||||
}
|
||||
return NewS3Storage(config.S3)
|
||||
},
|
||||
StorageOSS: func(config *StorageConfig) (Storage, error) {
|
||||
if config.OSS == nil {
|
||||
return nil, fmt.Errorf("OSS config not available")
|
||||
}
|
||||
return NewOSSStorage(config.OSS)
|
||||
},
|
||||
}
|
||||
102
internal/storage/types.go
Normal file
102
internal/storage/types.go
Normal file
@ -0,0 +1,102 @@
|
||||
//
|
||||
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
|
||||
package storage
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"time"
|
||||
)
|
||||
|
||||
var (
|
||||
// ErrNotFound is returned when an object is not found
|
||||
ErrNotFound = errors.New("object not found")
|
||||
// ErrBucketNotFound is returned when a bucket is not found
|
||||
ErrBucketNotFound = errors.New("bucket not found")
|
||||
)
|
||||
|
||||
// StorageType represents the type of storage backend
|
||||
type StorageType int
|
||||
|
||||
const (
|
||||
StorageMinio StorageType = 1
|
||||
StorageAzureSpn StorageType = 2
|
||||
StorageAzureSas StorageType = 3
|
||||
StorageAWSS3 StorageType = 4
|
||||
StorageOSS StorageType = 5
|
||||
StorageOpenDAL StorageType = 6
|
||||
StorageGCS StorageType = 7
|
||||
)
|
||||
|
||||
func (s StorageType) String() string {
|
||||
switch s {
|
||||
case StorageMinio:
|
||||
return "MINIO"
|
||||
case StorageAzureSpn:
|
||||
return "AZURE_SPN"
|
||||
case StorageAzureSas:
|
||||
return "AZURE_SAS"
|
||||
case StorageAWSS3:
|
||||
return "AWS_S3"
|
||||
case StorageOSS:
|
||||
return "OSS"
|
||||
case StorageOpenDAL:
|
||||
return "OPENDAL"
|
||||
case StorageGCS:
|
||||
return "GCS"
|
||||
default:
|
||||
return "UNKNOWN"
|
||||
}
|
||||
}
|
||||
|
||||
// Storage defines the interface for storage operations
|
||||
type Storage interface {
|
||||
// Health checks the storage service availability
|
||||
Health() bool
|
||||
|
||||
// Put uploads an object to storage
|
||||
// bucket: the bucket/container name
|
||||
// fnm: the file/object name (key)
|
||||
// binary: the data to upload
|
||||
// tenantID: optional tenant identifier
|
||||
Put(bucket, fnm string, binary []byte, tenantID ...string) error
|
||||
|
||||
// Get retrieves an object from storage
|
||||
// Returns the data or nil if not found
|
||||
Get(bucket, fnm string, tenantID ...string) ([]byte, error)
|
||||
|
||||
// Rm removes an object from storage
|
||||
Rm(bucket, fnm string, tenantID ...string) error
|
||||
|
||||
// ObjExist checks if an object exists
|
||||
ObjExist(bucket, fnm string, tenantID ...string) bool
|
||||
|
||||
// GetPresignedURL generates a presigned URL for accessing an object
|
||||
// expires: duration until the URL expires
|
||||
GetPresignedURL(bucket, fnm string, expires time.Duration, tenantID ...string) (string, error)
|
||||
|
||||
// BucketExists checks if a bucket exists
|
||||
BucketExists(bucket string) bool
|
||||
|
||||
// RemoveBucket removes a bucket and all its objects
|
||||
RemoveBucket(bucket string) error
|
||||
|
||||
// Copy copies an object from source to destination
|
||||
Copy(srcBucket, srcPath, destBucket, destPath string) bool
|
||||
|
||||
// Move moves an object from source to destination
|
||||
Move(srcBucket, srcPath, destBucket, destPath string) bool
|
||||
}
|
||||
Reference in New Issue
Block a user