youngkingdom/ragflow
1
0
Fork 0
mirror of https://github.com/infiniflow/ragflow.git synced 2026-03-23 15:37:37 +08:00
Code Issues Packages Projects Releases Wiki Activity

Add environments reading (#13701)

Browse Source 
### What problem does this PR solve?

environment variable > config file

### Type of change

- [x] New Feature (non-breaking change which adds functionality)

---------

Signed-off-by: Jin Hai <haijin.chn@gmail.com>
This commit is contained in:
Jin Hai
2026-03-19 18:50:28 +08:00
committed by GitHub
parent 757d8d42dd
commit 8d50ee632d
6 changed files with 239 additions and 119 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
internal/admin/handler.go
View File

@ -473,7 +473,14 @@ func (h *Handler) ListRoles(c *gin.Context) {
return
}
success(c, roles, "")
if roles == nil {
roles = []map[string]interface{}{}
}
success(c, gin.H{
"roles": roles,
"total": len(roles),
}, "")
}
// CreateRoleHTTPRequest create role request

259
internal/server/config.go
View File

@ -18,6 +18,7 @@ package server
import (
"fmt"
"net/mail"
"net/url"
"os"
"strconv"
@ -33,15 +34,18 @@ const DefaultConnectTimeout = 5 * time.Second
// Config application configuration
type Config struct {
Server ServerConfig `mapstructure:"server"`
Database DatabaseConfig `mapstructure:"database"`
Redis RedisConfig `mapstructure:"redis"`
Log LogConfig `mapstructure:"log"`
DocEngine DocEngineConfig `mapstructure:"doc_engine"`
RegisterEnabled int `mapstructure:"register_enabled"`
OAuth map[string]OAuthConfig `mapstructure:"oauth"`
Admin AdminConfig `mapstructure:"admin"`
UserDefaultLLM UserDefaultLLMConfig `mapstructure:"user_default_llm"`
Server ServerConfig `mapstructure:"server"`
Database DatabaseConfig `mapstructure:"database"`
Redis RedisConfig `mapstructure:"redis"`
Log LogConfig `mapstructure:"log"`
DocEngine DocEngineConfig `mapstructure:"doc_engine"`
StorageEngine StorageConfig `mapstructure:"storage_engine"`
RegisterEnabled int `mapstructure:"register_enabled"`
OAuth map[string]OAuthConfig `mapstructure:"oauth"`
Admin AdminConfig `mapstructure:"admin"`
UserDefaultLLM UserDefaultLLMConfig `mapstructure:"user_default_llm"`
DefaultSuperUser DefaultSuperUser `mapstructure:"default_super_user"`
Language string `mapstructure:"language"`
}
// AdminConfig admin server configuration
@ -50,6 +54,12 @@ type AdminConfig struct {
Port int `mapstructure:"http_port"`
}
type DefaultSuperUser struct {
Email string `mapstructure:"email"`
Password string `mapstructure:"password"`
Nickname string `mapstructure:"nickname"`
}
// UserDefaultLLMConfig user default LLM configuration
type UserDefaultLLMConfig struct {
DefaultModels DefaultModelsConfig `mapstructure:"default_models"`
@ -130,6 +140,59 @@ type InfinityConfig struct {
DBName string `mapstructure:"db_name"`
}
type StorageType string
// StorageConfig holds all storage-related configurations
type StorageConfig struct {
Type StorageType `mapstructure:"type"`
Minio *MinioConfig `mapstructure:"minio"`
S3 *S3Config `mapstructure:"s3"`
OSS *OSSConfig `mapstructure:"oss"`
}
const (
StorageOSS StorageType = "oss"
StorageS3 StorageType = "s3"
StorageMinio StorageType = "minio"
)
// OSSConfig holds Aliyun OSS storage configuration
// OSS is compatible with S3 API
type OSSConfig struct {
AccessKeyID string `mapstructure:"access_key"` // OSS Access Key ID
SecretAccessKey string `mapstructure:"secret_key"` // OSS Secret Access Key
EndpointURL string `mapstructure:"endpoint_url"` // OSS Endpoint (e.g., "https://oss-cn-hangzhou.aliyuncs.com")
Region string `mapstructure:"region"` // Region (e.g., "cn-hangzhou")
Bucket string `mapstructure:"bucket"` // Default bucket (optional)
PrefixPath string `mapstructure:"prefix_path"` // Path prefix (optional)
SignatureVersion string `mapstructure:"signature_version"` // Signature version
AddressingStyle string `mapstructure:"addressing_style"` // Addressing style
}
// MinioConfig holds MinIO storage configuration
type MinioConfig struct {
Host string `mapstructure:"host"` // MinIO server host (e.g., "localhost:9000")
User string `mapstructure:"user"` // Access key
Password string `mapstructure:"password"` // Secret key
Secure bool `mapstructure:"secure"` // Use HTTPS
Verify bool `mapstructure:"verify"` // Verify SSL certificates
Bucket string `mapstructure:"bucket"` // Default bucket (optional)
PrefixPath string `mapstructure:"prefix_path"` // Path prefix (optional)
}
// S3Config holds AWS S3 storage configuration
type S3Config struct {
AccessKeyID string `mapstructure:"access_key"` // AWS Access Key ID
SecretAccessKey string `mapstructure:"secret_key"` // AWS Secret Access Key
SessionToken string `mapstructure:"session_token"` // AWS Session Token (optional)
Region string `mapstructure:"region_name"` // AWS Region
EndpointURL string `mapstructure:"endpoint_url"` // Custom endpoint (optional)
SignatureVersion string `mapstructure:"signature_version"` // Signature version
AddressingStyle string `mapstructure:"addressing_style"` // Addressing style
Bucket string `mapstructure:"bucket"` // Default bucket (optional)
PrefixPath string `mapstructure:"prefix_path"` // Path prefix (optional)
}
// RedisConfig Redis configuration
type RedisConfig struct {
Host string `mapstructure:"host"`
@ -147,42 +210,17 @@ var (
// Init initialize configuration
func Init(configPath string) error {
v := viper.New()
// Set configuration file path
if configPath != "" {
v.SetConfigFile(configPath)
} else {
// Try to load service_conf.yaml from conf directory first
v.SetConfigName("service_conf")
v.SetConfigType("yaml")
v.AddConfigPath("./conf")
v.AddConfigPath(".")
v.AddConfigPath("./config")
v.AddConfigPath("./internal/config")
v.AddConfigPath("/etc/ragflow/")
err := FromConfigFile("")
if err != nil {
return err
}
// Read environment variables
v.SetEnvPrefix("RAGFLOW")
v.SetEnvKeyReplacer(strings.NewReplacer(".", "_"))
v.AutomaticEnv()
// Read configuration file
if err := v.ReadInConfig(); err != nil {
if _, ok := err.(viper.ConfigFileNotFoundError); !ok {
return fmt.Errorf("read config file error: %w", err)
}
zapLogger.Info("Config file not found, using environment variables only")
err = FromEnvironments()
if err != nil {
return err
}
// Save viper instance
globalViper = v
docEngine := os.Getenv("DOC_ENGINE")
if docEngine == "" {
docEngine = "elasticsearch"
}
id := 0
for k, v := range globalViper.AllSettings() {
configDict, ok := v.(map[string]interface{})
@ -200,7 +238,7 @@ func Init(configPath string) error {
delete(configDict, "http_port")
case "es":
// Skip if retrieval_type doesn't match doc_engine
if docEngine != "elasticsearch" {
if globalConfig.DocEngine.Type != "elasticsearch" {
continue
}
hosts := getString(configDict, "hosts")
@ -222,7 +260,7 @@ func Init(configPath string) error {
delete(configDict, "password")
case "infinity":
// Skip if retrieval_type doesn't match doc_engine
if docEngine != "infinity" {
if globalConfig.DocEngine.Type != "infinity" {
continue
}
uri := getString(configDict, "uri")
@ -326,6 +364,120 @@ func Init(configPath string) error {
id++
}
return nil
}
func FromEnvironments() error {
// Doc engine
docEngine := strings.ToLower(os.Getenv("DOC_ENGINE"))
switch docEngine {
case "infinity":
globalConfig.DocEngine.Type = EngineInfinity
case "":
// Default
if globalConfig.DocEngine.Type == "" {
globalConfig.DocEngine.Type = EngineElasticsearch
}
case "elasticsearch":
globalConfig.DocEngine.Type = EngineElasticsearch
case "opensearch":
case "oceanbase":
return fmt.Errorf("not implemented: %s", docEngine)
default:
return fmt.Errorf("invalid doc engine: %s", docEngine)
}
// Default super user email
globalConfig.DefaultSuperUser.Email = "admin@ragflow.io"
superUserEmail := os.Getenv("DEFAULT_SUPERUSER_EMAIL")
if superUserEmail != "" {
_, err := mail.ParseAddress(superUserEmail)
if err != nil {
return fmt.Errorf("invalid super user email: %s", superUserEmail)
}
globalConfig.DefaultSuperUser.Email = superUserEmail
}
globalConfig.DefaultSuperUser.Password = "admin"
superUserPassword := os.Getenv("DEFAULT_SUPERUSER_PASSWORD")
if superUserPassword != "" {
globalConfig.DefaultSuperUser.Password = superUserPassword
}
globalConfig.DefaultSuperUser.Nickname = "admin"
superUserNickname := os.Getenv("DEFAULT_SUPERUSER_NICKNAME")
if superUserNickname != "" {
globalConfig.DefaultSuperUser.Nickname = superUserNickname
}
// Meta database
databaseType := strings.ToLower(os.Getenv("DB_TYPE"))
switch databaseType {
case "mysql":
globalConfig.Database.Driver = "mysql"
case "":
// Default
if globalConfig.Database.Driver == "" {
globalConfig.Database.Driver = "mysql"
}
default:
return fmt.Errorf("invalid database type: %s", databaseType)
}
// Storage
storageType := strings.ToLower(os.Getenv("STORAGE_IMPL"))
switch storageType {
case "minio":
globalConfig.StorageEngine.Type = StorageMinio
case "s3":
globalConfig.StorageEngine.Type = StorageS3
case "oss":
globalConfig.StorageEngine.Type = StorageOSS
default:
return fmt.Errorf("invalid storage type: %s", storageType)
}
// Language
if globalConfig.Language == "" {
globalConfig.Language = GetLanguage()
}
return nil
}
func FromConfigFile(configPath string) error {
v := viper.New()
// Set configuration file path
if configPath != "" {
v.SetConfigFile(configPath)
} else {
// Try to load service_conf.yaml from conf directory first
v.SetConfigName("service_conf")
v.SetConfigType("yaml")
v.AddConfigPath("./conf")
v.AddConfigPath(".")
v.AddConfigPath("./config")
v.AddConfigPath("./internal/config")
v.AddConfigPath("/etc/ragflow/")
}
// Read environment variables
v.SetEnvPrefix("RAGFLOW")
v.SetEnvKeyReplacer(strings.NewReplacer(".", "_"))
v.AutomaticEnv()
// Read configuration file
if err := v.ReadInConfig(); err != nil {
if _, ok := err.(viper.ConfigFileNotFoundError); !ok {
return fmt.Errorf("read config file error: %w", err)
}
zapLogger.Info("Config file not found, using environment variables only")
}
// Save viper instance
globalViper = v
// Unmarshal configuration to globalConfig
// Note: This will only unmarshal fields that match the Config struct
if err := v.Unmarshal(&globalConfig); err != nil {
@ -415,9 +567,9 @@ func Init(configPath string) error {
// Map doc_engine section to DocEngineConfig
if globalConfig != nil && globalConfig.DocEngine.Type == "" {
// Use DOC_ENGINE env var if set
if docEngine != "" {
globalConfig.DocEngine.Type = EngineType(docEngine)
}
//if docEngine != "" {
// globalConfig.DocEngine.Type = EngineType(docEngine)
//}
// Try to map from doc_engine section (overrides env var if present)
if v.IsSet("doc_engine") {
docEngineConfig := v.Sub("doc_engine")
@ -583,3 +735,20 @@ func getInt(m map[string]interface{}, key string) int {
}
return 0
}
func GetLanguage() string {
lang := os.Getenv("LANG")
if lang == "" {
lang = os.Getenv("LANGUAGE")
}
lang = strings.ToLower(lang)
if strings.Contains(lang, "zh_") ||
strings.Contains(lang, "zh-") ||
strings.HasPrefix(lang, "zh") {
return "Chinese"
}
return "English"
}

11
internal/storage/minio.go
View File

@ -29,17 +29,6 @@ import (
"go.uber.org/zap"
)
// MinioConfig holds MinIO storage configuration
type MinioConfig struct {
Host string `mapstructure:"host"` // MinIO server host (e.g., "localhost:9000")
User string `mapstructure:"user"` // Access key
Password string `mapstructure:"password"` // Secret key
Secure bool `mapstructure:"secure"` // Use HTTPS
Verify bool `mapstructure:"verify"` // Verify SSL certificates
Bucket string `mapstructure:"bucket"` // Default bucket (optional)
PrefixPath string `mapstructure:"prefix_path"` // Path prefix (optional)
}
// MinioStorage implements Storage interface for MinIO
type MinioStorage struct {
client *minio.Client

13
internal/storage/oss.go
View File

@ -31,19 +31,6 @@ import (
"go.uber.org/zap"
)
// OSSConfig holds Aliyun OSS storage configuration
// OSS is compatible with S3 API
type OSSConfig struct {
AccessKeyID string `mapstructure:"access_key"` // OSS Access Key ID
SecretAccessKey string `mapstructure:"secret_key"` // OSS Secret Access Key
EndpointURL string `mapstructure:"endpoint_url"` // OSS Endpoint (e.g., "https://oss-cn-hangzhou.aliyuncs.com")
Region string `mapstructure:"region"` // Region (e.g., "cn-hangzhou")
Bucket string `mapstructure:"bucket"` // Default bucket (optional)
PrefixPath string `mapstructure:"prefix_path"` // Path prefix (optional)
SignatureVersion string `mapstructure:"signature_version"` // Signature version
AddressingStyle string `mapstructure:"addressing_style"` // Addressing style
}
// OSSStorage implements Storage interface for Aliyun OSS
// OSS uses S3-compatible API
type OSSStorage struct {

13
internal/storage/s3.go
View File

@ -31,19 +31,6 @@ import (
"go.uber.org/zap"
)
// S3Config holds AWS S3 storage configuration
type S3Config struct {
AccessKeyID string `mapstructure:"access_key"` // AWS Access Key ID
SecretAccessKey string `mapstructure:"secret_key"` // AWS Secret Access Key
SessionToken string `mapstructure:"session_token"` // AWS Session Token (optional)
Region string `mapstructure:"region_name"` // AWS Region
EndpointURL string `mapstructure:"endpoint_url"` // Custom endpoint (optional)
SignatureVersion string `mapstructure:"signature_version"` // Signature version
AddressingStyle string `mapstructure:"addressing_style"` // Addressing style
Bucket string `mapstructure:"bucket"` // Default bucket (optional)
PrefixPath string `mapstructure:"prefix_path"` // Path prefix (optional)
}
// S3Storage implements Storage interface for AWS S3
type S3Storage struct {
client *s3.Client

53
internal/storage/storage_factory.go
View File

@ -19,45 +19,26 @@ package storage
import (
"fmt"
"os"
"ragflow/internal/server"
"sync"
"github.com/spf13/viper"
"go.uber.org/zap"
)
// StorageFactory creates storage instances based on configuration
type StorageFactory struct {
storageType StorageType
storage Storage
config *StorageConfig
mu sync.RWMutex
}
// StorageConfig holds all storage-related configurations
type StorageConfig struct {
StorageType string `mapstructure:"storage_type"`
Minio *MinioConfig `mapstructure:"minio"`
S3 *S3Config `mapstructure:"s3"`
OSS *OSSConfig `mapstructure:"oss"`
}
// AzureConfig holds Azure-specific configurations
type AzureConfig struct {
ContainerURL string `mapstructure:"container_url"`
SASToken string `mapstructure:"sas_token"`
AccountURL string `mapstructure:"account_url"`
ClientID string `mapstructure:"client_id"`
Secret string `mapstructure:"secret"`
TenantID string `mapstructure:"tenant_id"`
ContainerName string `mapstructure:"container_name"`
AuthorityHost string `mapstructure:"authority_host"`
}
var (
globalFactory *StorageFactory
once sync.Once
)
// StorageFactory creates storage instances based on configuration
type StorageFactory struct {
storageType StorageType
storage Storage
config *server.StorageConfig
mu sync.RWMutex
}
// GetStorageFactory returns the singleton storage factory instance
func GetStorageFactory() *StorageFactory {
once.Do(func() {
@ -79,7 +60,7 @@ func InitStorageFactory(v *viper.Viper) error {
storageType = "MINIO" // Default storage type
}
storageConfig := &StorageConfig{}
storageConfig := &server.StorageConfig{}
if err := v.UnmarshalKey("storage", storageConfig); err != nil {
return fmt.Errorf("failed to unmarshal storage config: %w", err)
}
@ -114,7 +95,7 @@ func (f *StorageFactory) initStorage(storageType string, v *viper.Viper) error {
}
func (f *StorageFactory) initMinio(v *viper.Viper) error {
config := &MinioConfig{}
config := &server.MinioConfig{}
// Try to load from minio section first
if v.IsSet("minio") {
@ -156,7 +137,7 @@ func (f *StorageFactory) initMinio(v *viper.Viper) error {
}
func (f *StorageFactory) initS3(v *viper.Viper) error {
config := &S3Config{}
config := &server.S3Config{}
if v.IsSet("s3") {
s3Config := v.Sub("s3")
@ -188,7 +169,7 @@ func (f *StorageFactory) initS3(v *viper.Viper) error {
}
func (f *StorageFactory) initOSS(v *viper.Viper) error {
config := &OSSConfig{}
config := &server.OSSConfig{}
if v.IsSet("oss") {
ossConfig := v.Sub("oss")
@ -276,20 +257,20 @@ func (f *StorageFactory) SetStorage(storage Storage) {
}
// StorageTypeMapping returns the storage type mapping (equivalent to Python's storage_mapping)
var StorageTypeMapping = map[StorageType]func(*StorageConfig) (Storage, error){
StorageMinio: func(config *StorageConfig) (Storage, error) {
var StorageTypeMapping = map[StorageType]func(*server.StorageConfig) (Storage, error){
StorageMinio: func(config *server.StorageConfig) (Storage, error) {
if config.Minio == nil {
return nil, fmt.Errorf("MinIO config not available")
}
return NewMinioStorage(config.Minio)
},
StorageAWSS3: func(config *StorageConfig) (Storage, error) {
StorageAWSS3: func(config *server.StorageConfig) (Storage, error) {
if config.S3 == nil {
return nil, fmt.Errorf("S3 config not available")
}
return NewS3Storage(config.S3)
},
StorageOSS: func(config *StorageConfig) (Storage, error) {
StorageOSS: func(config *server.StorageConfig) (Storage, error) {
if config.OSS == nil {
return nil, fmt.Errorf("OSS config not available")
}