youngkingdom/ragflow
1
0
Fork 0
mirror of https://github.com/infiniflow/ragflow.git synced 2026-04-20 10:47:28 +08:00
Code Issues Packages Projects Releases Wiki Activity

Add minio go test (#13800)

Browse Source 
### What problem does this PR solve?

1. Add go test
2. Update CI process

### Type of change

- [x] New Feature (non-breaking change which adds functionality)

---------

Signed-off-by: Jin Hai <haijin.chn@gmail.com>
This commit is contained in:
Jin Hai
2026-03-27 18:12:56 +08:00
committed by GitHub
parent 2240fc778c
commit 1fff48b656
10 changed files with 775 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
internal/admin/service.go
View File

@ -1137,7 +1137,7 @@ func (s *Service) getMySQLStatus(name string) (map[string]interface{}, error) {
return map[string]interface{}{
"service_name": name,
"status": "timeout",
"elapsed": fmt.Sprintf("%.1f", time.Since(startTime).Milliseconds()),
"elapsed": fmt.Sprintf("%.1d", time.Since(startTime).Milliseconds()),
"message": err.Error(),
}, nil
}
@ -1148,7 +1148,7 @@ func (s *Service) getMySQLStatus(name string) (map[string]interface{}, error) {
return map[string]interface{}{
"service_name": name,
"status": "timeout",
"elapsed": fmt.Sprintf("%.1f", time.Since(startTime).Milliseconds()),
"elapsed": fmt.Sprintf("%.1d", time.Since(startTime).Milliseconds()),
"message": err.Error(),
}, nil
}
@ -1156,7 +1156,7 @@ func (s *Service) getMySQLStatus(name string) (map[string]interface{}, error) {
return map[string]interface{}{
"service_name": name,
"status": "alive",
"elapsed": fmt.Sprintf("%.1f", time.Since(startTime).Milliseconds()),
"elapsed": fmt.Sprintf("%.1d", time.Since(startTime).Milliseconds()),
"message": "MySQL connection successful",
}, nil
}
@ -1170,7 +1170,7 @@ func (s *Service) getRedisInfo(name string) (map[string]interface{}, error) {
return map[string]interface{}{
"service_name": name,
"status": "timeout",
"elapsed": fmt.Sprintf("%.1f", time.Since(startTime).Milliseconds()),
"elapsed": fmt.Sprintf("%.1d", time.Since(startTime).Milliseconds()),
"error": "Redis client not initialized",
}, nil
}
@ -1180,7 +1180,7 @@ func (s *Service) getRedisInfo(name string) (map[string]interface{}, error) {
return map[string]interface{}{
"service_name": name,
"status": "timeout",
"elapsed": fmt.Sprintf("%.1f", time.Since(startTime).Milliseconds()),
"elapsed": fmt.Sprintf("%.1d", time.Since(startTime).Milliseconds()),
"error": "Redis health check failed",
}, nil
}
@ -1188,7 +1188,7 @@ func (s *Service) getRedisInfo(name string) (map[string]interface{}, error) {
return map[string]interface{}{
"service_name": name,
"status": "alive",
"elapsed": fmt.Sprintf("%.1f", time.Since(startTime).Milliseconds()),
"elapsed": fmt.Sprintf("%.1d", time.Since(startTime).Milliseconds()),
"message": "Redis connection successful",
}, nil
}

20
internal/cli/cli.go
View File

@ -59,9 +59,9 @@ type ConnectionArgs struct {
Password string
APIToken string
UserName string
Command string // Original command string (for SQL mode)
CommandArgs []string // Split command arguments (for ContextEngine mode)
IsSQLMode bool // true=SQL mode (quoted), false=ContextEngine mode (unquoted)
Command string // Original command string (for SQL mode)
CommandArgs []string // Split command arguments (for ContextEngine mode)
IsSQLMode bool // true=SQL mode (quoted), false=ContextEngine mode (unquoted)
ShowHelp bool
AdminMode bool
OutputFormat OutputFormat // Output format: table, plain, json
@ -384,8 +384,7 @@ Configuration File:
Commands:
SQL commands (use quotes): "LIST USERS", "CREATE USER 'email' 'password'", etc.
Context Engine commands (no quotes): ls datasets, search "keyword", cat path, etc.
If no command is provided, CLI runs in interactive mode.
`)
If no command is provided, CLI runs in interactive mode.`)
}
// HistoryFile returns the path to the history file
@ -921,10 +920,10 @@ func (c *CLI) printContextEngineResult(result *contextengine.Result, cmdType con
break
}
}
fmt.Println(sep)
fmt.Printf("Total: %d\n", result.Total)
}
case contextengine.CommandCat:
fmt.Println(sep)
fmt.Printf("Total: %d\n", result.Total)
}
case contextengine.CommandCat:
// Cat output is handled differently - it returns []byte, not *Result
// This case should not be reached in normal flow since Cat returns []byte directly
fmt.Println("Content retrieved")
@ -1138,7 +1137,8 @@ type ListCommandOptions struct {
// parseSearchCommandArgs parses search command arguments
// Format: search [-d dir1] [-d dir2] ... -q query [-k top_k] [-t threshold]
// search -h|--help (shows help)
//
// search -h|--help (shows help)
func parseSearchCommandArgs(args []string) (*SearchCommandOptions, error) {
opts := &SearchCommandOptions{
TopK: 10,

4
internal/cli/user_parser.go
View File

@ -446,7 +446,7 @@ func (p *Parser) parseCreateCommand() (*Command, error) {
case TokenToken:
return p.parseCreateToken()
case TokenIndex:
return p.parseCreateIndex()
return p.parseCreateIndex()
default:
return nil, fmt.Errorf("unknown CREATE target: %s", p.curToken.Value)
}
@ -691,7 +691,7 @@ func (p *Parser) parseDropCommand() (*Command, error) {
case TokenToken:
return p.parseDropToken()
case TokenIndex:
return p.parseDropIndex()
return p.parseDropIndex()
default:
return nil, fmt.Errorf("unknown DROP target: %s", p.curToken.Value)
}

25
internal/server/config.go
View File

@ -18,6 +18,7 @@ package server
import (
"fmt"
"net"
"net/mail"
"net/url"
"os"
@ -213,7 +214,7 @@ var (
// Init initialize configuration
func Init(configPath string) error {
err := FromConfigFile("")
err := FromConfigFile(configPath)
if err != nil {
return err
}
@ -444,6 +445,26 @@ func FromEnvironments() error {
return fmt.Errorf("invalid storage type: %s", storageType)
}
// Minio
minioIP := strings.ToLower(os.Getenv("MINIO_IP"))
if minioIP != "" {
_, port, err := net.SplitHostPort(globalConfig.StorageEngine.Minio.Host)
if err != nil {
return fmt.Errorf("Error parsing host address %s: %v\n", globalConfig.StorageEngine.Minio.Host, err)
}
globalConfig.StorageEngine.Minio.Host = fmt.Sprintf("%s:%s", minioIP, port)
}
minioPort := strings.ToLower(os.Getenv("MINIO_PORT"))
println(fmt.Sprintf("MINIO ip and port from env: %s:%s", minioIP, minioPort))
if minioPort != "" {
ip, _, err := net.SplitHostPort(globalConfig.StorageEngine.Minio.Host)
if err != nil {
return fmt.Errorf("Error parsing host address %s: %v\n", globalConfig.StorageEngine.Minio.Host, err)
}
globalConfig.StorageEngine.Minio.Host = fmt.Sprintf("%s:%s", ip, minioPort)
}
// Language
if globalConfig.Language == "" {
globalConfig.Language = GetLanguage()
@ -464,8 +485,6 @@ func FromConfigFile(configPath string) error {
v.SetConfigType("yaml")
v.AddConfigPath("./conf")
v.AddConfigPath(".")
v.AddConfigPath("./config")
v.AddConfigPath("./internal/config")
v.AddConfigPath("/etc/ragflow/")
}

78
internal/storage/minio.go
View File

@ -22,6 +22,7 @@ import (
"crypto/tls"
"fmt"
"net/http"
"ragflow/internal/logger"
"ragflow/internal/server"
"time"
@ -33,8 +34,8 @@ import (
// MinioStorage implements Storage interface for MinIO
type MinioStorage struct {
client *minio.Client
bucket string
prefixPath string
bucket string // default bucket
prefixPath string // default prefix path
config *server.MinioConfig
}
@ -81,7 +82,7 @@ func (m *MinioStorage) connect() error {
func (m *MinioStorage) reconnect() {
if err := m.connect(); err != nil {
zap.L().Error("Failed to reconnect to MinIO", zap.Error(err))
logger.Fatal(fmt.Sprintf("Failed to reconnect to MinIO, %s", err.Error()))
}
}
@ -107,23 +108,17 @@ func (m *MinioStorage) resolveBucketAndPath(bucket, fnm string) (string, string)
// Health checks MinIO service availability
func (m *MinioStorage) Health() bool {
ctx := context.Background()
if m.bucket != "" {
exists, err := m.client.BucketExists(ctx, m.bucket)
if err != nil {
zap.L().Warn("MinIO health check failed", zap.Error(err))
return false
}
return exists
cancelFunction, err := m.client.HealthCheck(time.Second * 5)
if cancelFunction != nil {
defer cancelFunction()
}
_, err := m.client.ListBuckets(ctx)
if err != nil {
zap.L().Warn("MinIO health check failed", zap.Error(err))
logger.Warn("Failed to check MinIO health", zap.Error(err))
return false
}
return true
return m.client.IsOnline()
}
// Put uploads an object to MinIO
@ -132,19 +127,22 @@ func (m *MinioStorage) Put(bucket, fnm string, binary []byte, tenantID ...string
ctx := context.Background()
var err error
for i := 0; i < 3; i++ {
var exists bool
// Ensure bucket exists
if m.bucket == "" {
exists, err := m.client.BucketExists(ctx, bucket)
exists, err = m.client.BucketExists(ctx, bucket)
if err != nil {
zap.L().Error("Failed to check bucket existence", zap.String("bucket", bucket), zap.Error(err))
logger.Warn("Failed to check bucket existence", zap.String("bucket", bucket), zap.Error(err))
m.reconnect()
time.Sleep(time.Second)
continue
}
if !exists {
if err := m.client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{}); err != nil {
zap.L().Error("Failed to create bucket", zap.String("bucket", bucket), zap.Error(err))
if err = m.client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{}); err != nil {
logger.Warn("Failed to create bucket", zap.String("bucket", bucket), zap.Error(err))
m.reconnect()
time.Sleep(time.Second)
continue
@ -153,9 +151,9 @@ func (m *MinioStorage) Put(bucket, fnm string, binary []byte, tenantID ...string
}
reader := bytes.NewReader(binary)
_, err := m.client.PutObject(ctx, bucket, fnm, reader, int64(len(binary)), minio.PutObjectOptions{})
_, err = m.client.PutObject(ctx, bucket, fnm, reader, int64(len(binary)), minio.PutObjectOptions{})
if err != nil {
zap.L().Error("Failed to put object", zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(err))
logger.Warn("Failed to put object", zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(err))
m.reconnect()
time.Sleep(time.Second)
continue
@ -164,7 +162,7 @@ func (m *MinioStorage) Put(bucket, fnm string, binary []byte, tenantID ...string
return nil
}
return fmt.Errorf("failed to put object after 3 retries")
return err
}
// Get retrieves an object from MinIO
@ -176,7 +174,7 @@ func (m *MinioStorage) Get(bucket, fnm string, tenantID ...string) ([]byte, erro
for i := 0; i < 2; i++ {
obj, err := m.client.GetObject(ctx, bucket, fnm, minio.GetObjectOptions{})
if err != nil {
zap.L().Error("Failed to get object", zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(err))
logger.Warn("Failed to get object", zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(err))
m.reconnect()
time.Sleep(time.Second)
continue
@ -185,7 +183,7 @@ func (m *MinioStorage) Get(bucket, fnm string, tenantID ...string) ([]byte, erro
buf := new(bytes.Buffer)
if _, err := buf.ReadFrom(obj); err != nil {
zap.L().Error("Failed to read object data", zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(err))
logger.Warn("Failed to read object data", zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(err))
m.reconnect()
time.Sleep(time.Second)
continue
@ -197,14 +195,14 @@ func (m *MinioStorage) Get(bucket, fnm string, tenantID ...string) ([]byte, erro
return nil, fmt.Errorf("failed to get object after retries")
}
// Rm removes an object from MinIO
func (m *MinioStorage) Rm(bucket, fnm string, tenantID ...string) error {
// Remove removes an object from MinIO
func (m *MinioStorage) Remove(bucket, fnm string, tenantID ...string) error {
bucket, fnm = m.resolveBucketAndPath(bucket, fnm)
ctx := context.Background()
if err := m.client.RemoveObject(ctx, bucket, fnm, minio.RemoveObjectOptions{}); err != nil {
zap.L().Error("Failed to remove object", zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(err))
logger.Warn("Failed to remove object", zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(err))
return err
}
@ -228,7 +226,7 @@ func (m *MinioStorage) ObjExist(bucket, fnm string, tenantID ...string) bool {
if errResponse.Code == "NoSuchKey" || errResponse.Code == "NoSuchBucket" {
return false
}
zap.L().Error("Failed to stat object", zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(err))
logger.Warn("Failed to stat object", zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(err))
return false
}
@ -244,7 +242,7 @@ func (m *MinioStorage) GetPresignedURL(bucket, fnm string, expires time.Duration
for i := 0; i < 10; i++ {
url, err := m.client.PresignedGetObject(ctx, bucket, fnm, expires, nil)
if err != nil {
zap.L().Error("Failed to get presigned URL", zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(err))
logger.Warn("Failed to get presigned URL", zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(err))
m.reconnect()
time.Sleep(time.Second)
continue
@ -267,7 +265,7 @@ func (m *MinioStorage) BucketExists(bucket string) bool {
exists, err := m.client.BucketExists(ctx, actualBucket)
if err != nil {
zap.L().Error("Failed to check bucket existence", zap.String("bucket", actualBucket), zap.Error(err))
logger.Warn("Failed to check bucket existence", zap.String("bucket", actualBucket), zap.Error(err))
return false
}
@ -304,7 +302,7 @@ func (m *MinioStorage) RemoveBucket(bucket string) error {
Recursive: true,
}) {
if obj.Err != nil {
zap.L().Error("Error listing objects", zap.Error(obj.Err))
logger.Warn("Failed to list objects", zap.Error(obj.Err))
return
}
objectsCh <- obj
@ -312,13 +310,13 @@ func (m *MinioStorage) RemoveBucket(bucket string) error {
}()
for err := range m.client.RemoveObjects(ctx, actualBucket, objectsCh, minio.RemoveObjectsOptions{}) {
zap.L().Error("Failed to remove object", zap.String("key", err.ObjectName), zap.Error(err.Err))
logger.Warn(fmt.Sprintf("Failed to remove object, key: %s", err.ObjectName), zap.Error(err.Err))
}
// Only remove the actual bucket if not in single-bucket mode
if m.bucket == "" {
if err := m.client.RemoveBucket(ctx, actualBucket); err != nil {
zap.L().Error("Failed to remove bucket", zap.String("bucket", actualBucket), zap.Error(err))
logger.Warn("Failed to remove bucket", zap.String("bucket", actualBucket), zap.Error(err))
return err
}
}
@ -337,12 +335,12 @@ func (m *MinioStorage) Copy(srcBucket, srcPath, destBucket, destPath string) boo
if m.bucket == "" {
exists, err := m.client.BucketExists(ctx, destBucket)
if err != nil {
zap.L().Error("Failed to check bucket existence", zap.String("bucket", destBucket), zap.Error(err))
logger.Warn("Failed to check bucket existence", zap.String("bucket", destBucket), zap.Error(err))
return false
}
if !exists {
if err := m.client.MakeBucket(ctx, destBucket, minio.MakeBucketOptions{}); err != nil {
zap.L().Error("Failed to create bucket", zap.String("bucket", destBucket), zap.Error(err))
if err = m.client.MakeBucket(ctx, destBucket, minio.MakeBucketOptions{}); err != nil {
logger.Warn("Failed to create bucket", zap.String("bucket", destBucket), zap.Error(err))
return false
}
}
@ -351,7 +349,7 @@ func (m *MinioStorage) Copy(srcBucket, srcPath, destBucket, destPath string) boo
// Check if source object exists
_, err := m.client.StatObject(ctx, srcBucket, srcPath, minio.StatObjectOptions{})
if err != nil {
zap.L().Error("Source object not found", zap.String("bucket", srcBucket), zap.String("key", srcPath), zap.Error(err))
logger.Warn("Failed to stat source object", zap.String("bucket", srcBucket), zap.String("key", srcPath), zap.Error(err))
return false
}
@ -367,7 +365,7 @@ func (m *MinioStorage) Copy(srcBucket, srcPath, destBucket, destPath string) boo
_, err = m.client.CopyObject(ctx, destOpts, srcOpts)
if err != nil {
zap.L().Error("Failed to copy object", zap.String("src", fmt.Sprintf("%s/%s", srcBucket, srcPath)), zap.String("dest", fmt.Sprintf("%s/%s", destBucket, destPath)), zap.Error(err))
logger.Warn("Failed to copy object", zap.String("src", fmt.Sprintf("%s/%s", srcBucket, srcPath)), zap.String("dest", fmt.Sprintf("%s/%s", destBucket, destPath)), zap.Error(err))
return false
}
@ -377,8 +375,8 @@ func (m *MinioStorage) Copy(srcBucket, srcPath, destBucket, destPath string) boo
// Move moves an object from source to destination
func (m *MinioStorage) Move(srcBucket, srcPath, destBucket, destPath string) bool {
if m.Copy(srcBucket, srcPath, destBucket, destPath) {
if err := m.Rm(srcBucket, srcPath); err != nil {
zap.L().Error("Failed to remove source object after copy", zap.String("bucket", srcBucket), zap.String("key", srcPath), zap.Error(err))
if err := m.Remove(srcBucket, srcPath); err != nil {
logger.Warn("Failed to remove source object after copy", zap.String("bucket", srcBucket), zap.String("key", srcPath), zap.Error(err))
return false
}
return true

658
internal/storage/minio_test.go Normal file
View File

@ -0,0 +1,658 @@
//
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package storage
import (
"bytes"
"fmt"
"log"
"os"
"ragflow/internal/utility"
"testing"
"time"
"ragflow/internal/server"
)
// getMinioConfig returns MinIO configuration for testing
// Configuration can be loaded from environment variables or config file
func getMinioConfig() (*server.MinioConfig, error) {
// Initialize configuration
if err := server.Init(""); err != nil {
return nil, err
}
// Try to get configuration from environment variables first
config := server.GetConfig().StorageEngine.Minio
log.Printf("MinioConfig: %+v", config)
return config, nil
}
// getEnv gets environment variable or returns default value
func getEnv(key, defaultValue string) string {
if value := os.Getenv(key); value != "" {
return value
}
return defaultValue
}
// getEnvBool gets environment variable as bool or returns default value
func getEnvBool(key string, defaultValue bool) bool {
if value := os.Getenv(key); value != "" {
return value == "true" || value == "1" || value == "yes"
}
return defaultValue
}
// newTestMinioStorage creates a new MinIO storage instance for testing
func newTestMinioStorage(t *testing.T) *MinioStorage {
rootDir := utility.GetProjectRoot()
t.Chdir(rootDir)
t.Chdir(rootDir)
config, err := getMinioConfig()
if err != nil {
t.Skipf("Skipping test: failed to get MinIO configuration: %v", err)
return nil
}
storage, err := NewMinioStorage(config)
if err != nil {
t.Skipf("Skipping test: failed to connect to MinIO: %v", err)
}
return storage
}
func TestNewMinioStorage(t *testing.T) {
rootDir := utility.GetProjectRoot()
t.Chdir(rootDir)
config, err := getMinioConfig()
if err != nil {
t.Skipf("Skipping test: failed to get MinIO configuration: %v", err)
return
}
storage, err := NewMinioStorage(config)
if err != nil {
t.Skipf("Skipping test: failed to connect to MinIO: %v", err)
}
if storage == nil {
t.Error("Expected storage to be non-nil")
}
if storage.client == nil {
t.Error("Expected client to be non-nil")
}
if storage.config == nil {
t.Error("Expected config to be non-nil")
}
}
func TestNewMinioStorage_InvalidConfig(t *testing.T) {
// Test with invalid host
config := &server.MinioConfig{
Host: "invalid-host:99999",
User: "test",
Password: "test",
Secure: false,
}
_, err := NewMinioStorage(config)
// Should return an error for invalid connection
if err == nil {
t.Log("Note: Connection may succeed but fail later depending on network timeout")
}
}
func TestMinioStorage_Health(t *testing.T) {
storage := newTestMinioStorage(t)
healthy := storage.Health()
// Health check should return true if connection is working
// Note: This depends on whether a default bucket is configured
t.Logf("Health check result: %v", healthy)
if !healthy {
t.Error("Expected storage to be healthy")
}
}
func TestMinioStorage_PutAndGet(t *testing.T) {
storage := newTestMinioStorage(t)
bucket := "test-bucket"
key := "test-file.txt"
content := []byte("Hello, MinIO Test!")
// Test Put
err := storage.Put(bucket, key, content)
if err != nil {
t.Fatalf("Failed to put object: %v", err)
}
// Test Get
retrieved, err := storage.Get(bucket, key)
if err != nil {
t.Fatalf("Failed to get object: %v", err)
}
if !bytes.Equal(retrieved, content) {
t.Errorf("Retrieved content does not match. Expected %s, got %s", content, retrieved)
}
// Cleanup
err = storage.Remove(bucket, key)
if err != nil {
t.Logf("Warning: failed to cleanup test object: %v", err)
}
}
func TestMinioStorage_Put_EmptyData(t *testing.T) {
storage := newTestMinioStorage(t)
bucket := "test-bucket"
key := "empty-file.txt"
content := []byte{}
err := storage.Put(bucket, key, content)
if err != nil {
t.Fatalf("Failed to put empty object: %v", err)
}
// Verify object exists
exists := storage.ObjExist(bucket, key)
if !exists {
t.Error("Expected empty object to exist")
}
// Cleanup
storage.Remove(bucket, key)
}
func TestMinioStorage_Put_LargeData(t *testing.T) {
storage := newTestMinioStorage(t)
bucket := "test-bucket"
key := "large-file.bin"
// Create 1MB of data
content := make([]byte, 1024*1024)
for i := range content {
content[i] = byte(i % 256)
}
err := storage.Put(bucket, key, content)
if err != nil {
t.Fatalf("Failed to put large object: %v", err)
}
retrieved, err := storage.Get(bucket, key)
if err != nil {
t.Fatalf("Failed to get large object: %v", err)
}
if !bytes.Equal(retrieved, content) {
t.Error("Retrieved large content does not match original")
}
// Cleanup
storage.Remove(bucket, key)
}
func TestMinioStorage_Get_NonExistent(t *testing.T) {
storage := newTestMinioStorage(t)
bucket := "test-bucket"
key := "non-existent-file.txt"
_, err := storage.Get(bucket, key)
if err == nil {
t.Error("Expected error when getting non-existent object")
}
}
func TestMinioStorage_Remove(t *testing.T) {
storage := newTestMinioStorage(t)
bucket := "test-bucket"
key := "file-to-delete.txt"
content := []byte("Delete me")
// First, put an object
err := storage.Put(bucket, key, content)
if err != nil {
t.Fatalf("Failed to put object: %v", err)
}
// Verify it exists
exists := storage.ObjExist(bucket, key)
if !exists {
t.Fatal("Expected object to exist before removal")
}
// Remove it
err = storage.Remove(bucket, key)
if err != nil {
t.Fatalf("Failed to remove object: %v", err)
}
// Verify it's gone
exists = storage.ObjExist(bucket, key)
if exists {
t.Error("Expected object to not exist after removal")
}
}
func TestMinioStorage_Remove_NonExistent(t *testing.T) {
storage := newTestMinioStorage(t)
bucket := "test-bucket"
key := "non-existent-file.txt"
// Removing a non-existent object should not error
err := storage.Remove(bucket, key)
if err != nil {
t.Logf("Remove non-existent object returned error (may be acceptable): %v", err)
}
}
func TestMinioStorage_ObjExist(t *testing.T) {
storage := newTestMinioStorage(t)
bucket := "test-bucket"
key := "existence-test.txt"
content := []byte("Test content")
// Check non-existent object
exists := storage.ObjExist(bucket, key)
if exists {
t.Error("Expected non-existent object to return false")
}
// Create object
err := storage.Put(bucket, key, content)
if err != nil {
t.Fatalf("Failed to put object: %v", err)
}
// Check existing object
exists = storage.ObjExist(bucket, key)
if !exists {
t.Error("Expected existing object to return true")
}
// Cleanup
storage.Remove(bucket, key)
}
func TestMinioStorage_GetPresignedURL(t *testing.T) {
storage := newTestMinioStorage(t)
bucket := "test-bucket"
key := "presigned-test.txt"
content := []byte("Presigned URL test content")
// Create object first
err := storage.Put(bucket, key, content)
if err != nil {
t.Fatalf("Failed to put object: %v", err)
}
// Get presigned URL
url, err := storage.GetPresignedURL(bucket, key, 5*time.Minute)
if err != nil {
t.Fatalf("Failed to get presigned URL: %v", err)
}
if url == "" {
t.Error("Expected presigned URL to be non-empty")
}
// Verify URL contains expected components
if len(url) > 0 {
t.Logf("Generated presigned URL (first 100 chars): %s...", url[:min(100, len(url))])
}
// Cleanup
storage.Remove(bucket, key)
}
func TestMinioStorage_GetPresignedURL_NonExistent(t *testing.T) {
storage := newTestMinioStorage(t)
bucket := "test-bucket"
key := "non-existent-presigned.txt"
_, err := storage.GetPresignedURL(bucket, key, 5*time.Minute)
if err == nil {
t.Log("Note: Some MinIO versions may allow presigned URLs for non-existent objects")
}
}
func TestMinioStorage_BucketExists(t *testing.T) {
storage := newTestMinioStorage(t)
bucket := fmt.Sprintf("test-bucket-exists-%d", time.Now().Unix())
// Check non-existent bucket
exists := storage.BucketExists(bucket)
if exists {
t.Error("Expected non-existent bucket to return false")
}
// Create bucket by putting an object
err := storage.Put(bucket, "test.txt", []byte("test"))
if err != nil {
t.Fatalf("Failed to create bucket: %v", err)
}
// Check existing bucket
exists = storage.BucketExists(bucket)
if !exists {
t.Error("Expected existing bucket to return true")
}
// Cleanup
storage.RemoveBucket(bucket)
}
func TestMinioStorage_RemoveBucket(t *testing.T) {
storage := newTestMinioStorage(t)
bucket := fmt.Sprintf("test-bucket-remove-%d", time.Now().Unix())
// Create bucket with some objects
err := storage.Put(bucket, "file1.txt", []byte("content1"))
if err != nil {
t.Fatalf("Failed to put object: %v", err)
}
err = storage.Put(bucket, "file2.txt", []byte("content2"))
if err != nil {
t.Fatalf("Failed to put object: %v", err)
}
// Verify bucket exists
exists := storage.BucketExists(bucket)
if !exists {
t.Fatal("Expected bucket to exist before removal")
}
// Remove bucket
err = storage.RemoveBucket(bucket)
if err != nil {
t.Fatalf("Failed to remove bucket: %v", err)
}
// Verify bucket is gone
exists = storage.BucketExists(bucket)
if exists {
t.Error("Expected bucket to not exist after removal")
}
}
func TestMinioStorage_Copy(t *testing.T) {
storage := newTestMinioStorage(t)
srcBucket := "test-bucket-src"
srcKey := "source-file.txt"
destBucket := "test-bucket-dest"
destKey := "copied-file.txt"
content := []byte("Content to copy")
// Create source object
err := storage.Put(srcBucket, srcKey, content)
if err != nil {
t.Fatalf("Failed to put source object: %v", err)
}
// Copy object
success := storage.Copy(srcBucket, srcKey, destBucket, destKey)
if !success {
t.Fatal("Failed to copy object")
}
// Verify destination exists
exists := storage.ObjExist(destBucket, destKey)
if !exists {
t.Error("Expected copied object to exist")
}
// Verify content matches
retrieved, err := storage.Get(destBucket, destKey)
if err != nil {
t.Fatalf("Failed to get copied object: %v", err)
}
if !bytes.Equal(retrieved, content) {
t.Error("Copied content does not match original")
}
// Cleanup
storage.Remove(srcBucket, srcKey)
storage.Remove(destBucket, destKey)
}
func TestMinioStorage_Copy_NonExistentSource(t *testing.T) {
storage := newTestMinioStorage(t)
srcBucket := "test-bucket-src"
srcKey := "non-existent-source.txt"
destBucket := "test-bucket-dest"
destKey := "should-not-exist.txt"
success := storage.Copy(srcBucket, srcKey, destBucket, destKey)
if success {
t.Error("Expected copy of non-existent object to fail")
}
// Verify destination does not exist
exists := storage.ObjExist(destBucket, destKey)
if exists {
t.Error("Expected destination object to not exist after failed copy")
storage.Remove(destBucket, destKey)
}
}
func TestMinioStorage_Move(t *testing.T) {
storage := newTestMinioStorage(t)
srcBucket := "test-bucket-src"
srcKey := "file-to-move.txt"
destBucket := "test-bucket-dest"
destKey := "moved-file.txt"
content := []byte("Content to move")
// Create source object
err := storage.Put(srcBucket, srcKey, content)
if err != nil {
t.Fatalf("Failed to put source object: %v", err)
}
// Move object
success := storage.Move(srcBucket, srcKey, destBucket, destKey)
if !success {
t.Fatal("Failed to move object")
}
// Verify source is gone
exists := storage.ObjExist(srcBucket, srcKey)
if exists {
t.Error("Expected source object to not exist after move")
}
// Verify destination exists
exists = storage.ObjExist(destBucket, destKey)
if !exists {
t.Error("Expected moved object to exist")
}
// Verify content matches
retrieved, err := storage.Get(destBucket, destKey)
if err != nil {
t.Fatalf("Failed to get moved object: %v", err)
}
if !bytes.Equal(retrieved, content) {
t.Error("Moved content does not match original")
}
// Cleanup
storage.Remove(destBucket, destKey)
}
func TestMinioStorage_Move_NonExistentSource(t *testing.T) {
storage := newTestMinioStorage(t)
srcBucket := "test-bucket-src"
srcKey := "non-existent-source.txt"
destBucket := "test-bucket-dest"
destKey := "should-not-exist.txt"
success := storage.Move(srcBucket, srcKey, destBucket, destKey)
if success {
t.Error("Expected move of non-existent object to fail")
}
}
func TestMinioStorage_MultipleObjectsInBucket(t *testing.T) {
storage := newTestMinioStorage(t)
bucket := fmt.Sprintf("test-multi-%d", time.Now().Unix())
numObjects := 10
// Create multiple objects
for i := 0; i < numObjects; i++ {
key := fmt.Sprintf("file-%d.txt", i)
content := []byte(fmt.Sprintf("Content %d", i))
err := storage.Put(bucket, key, content)
if err != nil {
t.Fatalf("Failed to put object %d: %v", i, err)
}
}
// Verify all objects exist
for i := 0; i < numObjects; i++ {
key := fmt.Sprintf("file-%d.txt", i)
exists := storage.ObjExist(bucket, key)
if !exists {
t.Errorf("Expected object %s to exist", key)
}
}
// Verify content
for i := 0; i < numObjects; i++ {
key := fmt.Sprintf("file-%d.txt", i)
expectedContent := []byte(fmt.Sprintf("Content %d", i))
retrieved, err := storage.Get(bucket, key)
if err != nil {
t.Errorf("Failed to get object %s: %v", key, err)
continue
}
if !bytes.Equal(retrieved, expectedContent) {
t.Errorf("Content mismatch for object %s", key)
}
}
// Cleanup - remove bucket with all objects
err := storage.RemoveBucket(bucket)
if err != nil {
t.Logf("Warning: failed to cleanup bucket: %v", err)
}
}
func TestMinioStorage_SpecialCharactersInKey(t *testing.T) {
storage := newTestMinioStorage(t)
bucket := "test-bucket"
specialKeys := []string{
"file with spaces.txt",
"file-with-dashes.txt",
"file_with_underscores.txt",
"file.multiple.dots.txt",
"path/to/nested/file.txt",
"unicode-文件.txt",
}
for _, key := range specialKeys {
content := []byte(fmt.Sprintf("Content for %s", key))
err := storage.Put(bucket, key, content)
if err != nil {
t.Errorf("Failed to put object with key '%s': %v", key, err)
continue
}
retrieved, err := storage.Get(bucket, key)
if err != nil {
t.Errorf("Failed to get object with key '%s': %v", key, err)
continue
}
if !bytes.Equal(retrieved, content) {
t.Errorf("Content mismatch for key '%s'", key)
}
// Cleanup
storage.Remove(bucket, key)
}
}
func TestMinioStorage_TenantID(t *testing.T) {
storage := newTestMinioStorage(t)
bucket := "test-bucket"
key := "tenant-test.txt"
content := []byte("Tenant test content")
tenantID := "tenant-123"
// Put with tenant ID
err := storage.Put(bucket, key, content, tenantID)
if err != nil {
t.Fatalf("Failed to put object with tenant ID: %v", err)
}
// Get with tenant ID
retrieved, err := storage.Get(bucket, key, tenantID)
if err != nil {
t.Fatalf("Failed to get object with tenant ID: %v", err)
}
if !bytes.Equal(retrieved, content) {
t.Error("Content mismatch for tenant-specific object")
}
// Check existence with tenant ID
exists := storage.ObjExist(bucket, key, tenantID)
if !exists {
t.Error("Expected object to exist with tenant ID")
}
// Cleanup
storage.Remove(bucket, key, tenantID)
}
// min is a helper function to get the minimum of two integers
func min(a, b int) int {
if a < b {
return a
}
return b
}

6
internal/storage/oss.go
View File

@ -218,8 +218,8 @@ func (o *OSSStorage) Get(bucket, fnm string, tenantID ...string) ([]byte, error)
return nil, fmt.Errorf("failed to get object after retries")
}
// Rm removes an object from OSS
func (o *OSSStorage) Rm(bucket, fnm string, tenantID ...string) error {
// Remove removes an object from OSS
func (o *OSSStorage) Remove(bucket, fnm string, tenantID ...string) error {
bucket, fnm = o.resolveBucketAndPath(bucket, fnm)
ctx := context.Background()
@ -381,7 +381,7 @@ func (o *OSSStorage) Copy(srcBucket, srcPath, destBucket, destPath string) bool
// Move moves an object from source to destination
func (o *OSSStorage) Move(srcBucket, srcPath, destBucket, destPath string) bool {
if o.Copy(srcBucket, srcPath, destBucket, destPath) {
if err := o.Rm(srcBucket, srcPath); err != nil {
if err := o.Remove(srcBucket, srcPath); err != nil {
zap.L().Error("Failed to remove source object after copy", zap.String("bucket", srcBucket), zap.String("key", srcPath), zap.Error(err))
return false
}

6
internal/storage/s3.go
View File

@ -226,8 +226,8 @@ func (s *S3Storage) Get(bucket, fnm string, tenantID ...string) ([]byte, error)
return nil, fmt.Errorf("failed to get object after retries")
}
// Rm removes an object from S3
func (s *S3Storage) Rm(bucket, fnm string, tenantID ...string) error {
// Remove removes an object from S3
func (s *S3Storage) Remove(bucket, fnm string, tenantID ...string) error {
bucket, fnm = s.resolveBucketAndPath(bucket, fnm)
ctx := context.Background()
@ -389,7 +389,7 @@ func (s *S3Storage) Copy(srcBucket, srcPath, destBucket, destPath string) bool {
// Move moves an object from source to destination
func (s *S3Storage) Move(srcBucket, srcPath, destBucket, destPath string) bool {
if s.Copy(srcBucket, srcPath, destBucket, destPath) {
if err := s.Rm(srcBucket, srcPath); err != nil {
if err := s.Remove(srcBucket, srcPath); err != nil {
zap.L().Error("Failed to remove source object after copy", zap.String("bucket", srcBucket), zap.String("key", srcPath), zap.Error(err))
return false
}

4
internal/storage/types.go
View File

@ -78,8 +78,8 @@ type Storage interface {
// Returns the data or nil if not found
Get(bucket, fnm string, tenantID ...string) ([]byte, error)
// Rm removes an object from storage
Rm(bucket, fnm string, tenantID ...string) error
// Remove removes an object from storage
Remove(bucket, fnm string, tenantID ...string) error
// ObjExist checks if an object exists
ObjExist(bucket, fnm string, tenantID ...string) bool

31
run_go_tests.sh Executable file
View File

@ -0,0 +1,31 @@
#!/bin/bash
set -e
PACKAGES=(
"./internal/admin/..."
# "./internal/binding/..."
"./internal/cache/..."
"./internal/cli/..."
"./internal/common/..."
"./internal/dao/..."
"./internal/engine/..."
"./internal/handler/..."
"./internal/logger/..."
"./internal/model/..."
"./internal/router/..."
"./internal/server/..."
# "./internal/service/..."
"./internal/storage/..."
"./internal/tokenizer/..."
# "./internal/utility/..."
)
echo "Running tests for specific packages..."
for pkg in "${PACKAGES[@]}"; do
echo "=== Testing $pkg ==="
go test $pkg -v -cover -test.v
echo ""
done
#echo "Running all tests except failed packages..."
#go test $(go list ./internal/... | grep -v -E '(cli|service|binding)$') -v