mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-05-28 19:53:06 +08:00
### What problem does this PR solve? 1. Go ingestion server will connected with admin server with gRPC stream 2. Go ingestion server will be responsible for ingestion tasks ``` RAGFlow(admin)> list ingestors; +-----------------+-----------+----------------------------------+---------------------------+----------+------------+--------------+--------+------------+---------------+ | address | cpu_usage | id | last_heartbeat | name | process_id | rss_usage | status | task_count | vms_usage | +-----------------+-----------+----------------------------------+---------------------------+----------+------------+--------------+--------+------------+---------------+ | 127.0.0.1:58564 | 0 | bdd1870eea2646e0aacb8a2cd3307aa2 | 2026-05-24T18:16:17+08:00 | ingestor | 680152 | 212.72265625 | active | 0 | 2589.12109375 | +-----------------+-----------+----------------------------------+---------------------------+----------+------------+--------------+--------+------------+---------------+ RAGFlow(admin)> start ingestion 'abc'; +----------------------------------+ | task_id | +----------------------------------+ | e714777639ca4760ab427b5f211e81ad | +----------------------------------+ RAGFlow(admin)> stop ingestion 'f7bd39d0a724457eb5fdce6d81699776'; +----------------------------------+ | task_id | +----------------------------------+ | f7bd39d0a724457eb5fdce6d81699776 | +----------------------------------+ RAGFlow(admin)> list tasks; +-----+----------------------------------+-------+------+----------------------------------+---------------------------+------------+------------+ | ETA | assign_to | error | from | id | last_update | start_time | status | +-----+----------------------------------+-------+------+----------------------------------+---------------------------+------------+------------+ | 0 | 17937da188b84f23a5c10bb87588944b | | CLI | eae6431da72a40e796cff3a03008091b | 2026-05-24T19:46:03+08:00 | | COMPLETED | | 0 | 17937da188b84f23a5c10bb87588944b | | CLI | 6cccdd174bd049ecb05a774bbb47593f | 2026-05-24T19:46:03+08:00 | | COMPLETED | | 0 | 17937da188b84f23a5c10bb87588944b | | CLI | ef360d777e57485799adb96b30f2b4b8 | 2026-05-24T19:46:03+08:00 | | CANCELED | | 0 | 17937da188b84f23a5c10bb87588944b | | CLI | bcc5c5448cb64de48b6b6171c36fb790 | 2026-05-24T19:46:03+08:00 | | CANCELED | | 0 | 17937da188b84f23a5c10bb87588944b | | CLI | bfc25384c43a443294fe2da979a38ac2 | 2026-05-24T19:46:03+08:00 | | DISPATCHED | | 0 | 17937da188b84f23a5c10bb87588944b | | CLI | 84960537b85d413b8990a9efd5952d67 | 2026-05-24T19:46:04+08:00 | | DISPATCHED | | 0 | 17937da188b84f23a5c10bb87588944b | | CLI | 3d223c1b51e24b36861a3bfb2f1d58d4 | 2026-05-24T19:46:03+08:00 | | CANCELED | | 0 | 17937da188b84f23a5c10bb87588944b | | CLI | e433b0e356b846c89c301621a3c54494 | 2026-05-24T19:46:03+08:00 | | COMPLETED | | 0 | 17937da188b84f23a5c10bb87588944b | | CLI | 7c93a3880f074ebd8eca14e6b51bb7ef | 2026-05-24T19:46:03+08:00 | | COMPLETED | | 0 | 17937da188b84f23a5c10bb87588944b | | CLI | df2e4ef51aaf4390bff9a23f2692486e | 2026-05-24T19:46:04+08:00 | | DISPATCHED | | 0 | 17937da188b84f23a5c10bb87588944b | | CLI | 7377c53010194ef7a83aa206698d66ff | 2026-05-24T19:46:05+08:00 | | DISPATCHED | | 0 | 17937da188b84f23a5c10bb87588944b | | CLI | df64d1a1f9d348e3a2f174c4d7d69e73 | 2026-05-24T19:46:05+08:00 | | DISPATCHED | | 0 | 17937da188b84f23a5c10bb87588944b | | CLI | b59834512e2847e1bdf13ace04b8a456 | 2026-05-24T19:46:06+08:00 | | DISPATCHED | | 0 | 17937da188b84f23a5c10bb87588944b | | CLI | 0064bb0ab69344028d1ecfda053826f4 | 2026-05-24T19:46:03+08:00 | | QUEUED | +-----+----------------------------------+-------+------+----------------------------------+---------------------------+------------+------------+ ``` ### Type of change - [x] New Feature (non-breaking change which adds functionality) --------- Signed-off-by: Jin Hai <haijin.chn@gmail.com>
115 lines
2.9 KiB
Go
115 lines
2.9 KiB
Go
//
|
|
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
//
|
|
|
|
package admin
|
|
|
|
import (
|
|
"errors"
|
|
"ragflow/internal/common"
|
|
"sync"
|
|
"time"
|
|
)
|
|
|
|
// Service errors
|
|
var (
|
|
ErrInvalidToken = errors.New("invalid token")
|
|
ErrNotAdmin = errors.New("user is not admin")
|
|
ErrUserInactive = errors.New("user is inactive")
|
|
ErrUserNotFound = errors.New("user not found")
|
|
)
|
|
|
|
// API server state
|
|
|
|
// ServerStore is a thread-safe global server status storage
|
|
type ServerStore struct {
|
|
mu sync.RWMutex
|
|
servers map[string]*common.BaseMessage // key: server_id
|
|
}
|
|
|
|
// GlobalServerStore is the global instance
|
|
var GlobalServerStore = &ServerStore{
|
|
servers: make(map[string]*common.BaseMessage),
|
|
}
|
|
|
|
// UpdateServerInfo updates or adds a server status
|
|
func (s *ServerStore) UpdateServerInfo(serverName string, status *common.BaseMessage) {
|
|
|
|
//switch serviceType {
|
|
//case "meta_data":
|
|
// return s.getMySQLStatus(name)
|
|
|
|
switch status.ServerType {
|
|
case common.ServerTypeAPI:
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
s.servers[serverName] = status
|
|
return
|
|
case common.ServerTypeIngestion:
|
|
return
|
|
}
|
|
}
|
|
|
|
// GetServerInfo gets a single server status
|
|
func (s *ServerStore) GetServerInfo(serverName string) (*common.BaseMessage, bool) {
|
|
s.mu.RLock()
|
|
defer s.mu.RUnlock()
|
|
status, ok := s.servers[serverName]
|
|
return status, ok
|
|
}
|
|
|
|
// ListInfos gets all server infos
|
|
func (s *ServerStore) ListInfos() []*common.BaseMessage {
|
|
s.mu.RLock()
|
|
defer s.mu.RUnlock()
|
|
result := make([]*common.BaseMessage, 0, len(s.servers))
|
|
for _, status := range s.servers {
|
|
result = append(result, status)
|
|
}
|
|
return result
|
|
}
|
|
|
|
// ListInfosByType gets server infos by type
|
|
func (s *ServerStore) ListInfosByType(serverType common.ServerType) []*common.BaseMessage {
|
|
s.mu.RLock()
|
|
defer s.mu.RUnlock()
|
|
result := make([]*common.BaseMessage, 0)
|
|
for _, status := range s.servers {
|
|
if status.ServerType == serverType {
|
|
result = append(result, status)
|
|
}
|
|
}
|
|
return result
|
|
}
|
|
|
|
// RemoveStatus removes a server status
|
|
func (s *ServerStore) RemoveStatus(serverID string) {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
delete(s.servers, serverID)
|
|
}
|
|
|
|
// CleanupStaleStatuses cleans up servers that haven't reported for a specified duration
|
|
func (s *ServerStore) CleanupStaleStatuses(maxAge time.Duration) {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
now := time.Now()
|
|
for id, status := range s.servers {
|
|
if now.Sub(status.Timestamp) > maxAge {
|
|
delete(s.servers, id)
|
|
}
|
|
}
|
|
}
|