Files
ragflow/internal/cli/admin_parser.go
Jin Hai f8c626bbc8 Go: add ingestion server (#15094)
### What problem does this PR solve?

1. Go ingestion server will connected with admin server with gRPC stream
2. Go ingestion server will be responsible for ingestion tasks
```

RAGFlow(admin)> list ingestors;
+-----------------+-----------+----------------------------------+---------------------------+----------+------------+--------------+--------+------------+---------------+
| address         | cpu_usage | id                               | last_heartbeat            | name     | process_id | rss_usage    | status | task_count | vms_usage     |
+-----------------+-----------+----------------------------------+---------------------------+----------+------------+--------------+--------+------------+---------------+
| 127.0.0.1:58564 | 0         | bdd1870eea2646e0aacb8a2cd3307aa2 | 2026-05-24T18:16:17+08:00 | ingestor | 680152     | 212.72265625 | active | 0          | 2589.12109375 |
+-----------------+-----------+----------------------------------+---------------------------+----------+------------+--------------+--------+------------+---------------+

RAGFlow(admin)> start ingestion 'abc';
+----------------------------------+
| task_id                          |
+----------------------------------+
| e714777639ca4760ab427b5f211e81ad |
+----------------------------------+

RAGFlow(admin)> stop ingestion 'f7bd39d0a724457eb5fdce6d81699776';
+----------------------------------+
| task_id                          |
+----------------------------------+
| f7bd39d0a724457eb5fdce6d81699776 |
+----------------------------------+

RAGFlow(admin)> list tasks;
+-----+----------------------------------+-------+------+----------------------------------+---------------------------+------------+------------+
| ETA | assign_to                        | error | from | id                               | last_update               | start_time | status     |
+-----+----------------------------------+-------+------+----------------------------------+---------------------------+------------+------------+
| 0   | 17937da188b84f23a5c10bb87588944b |       | CLI  | eae6431da72a40e796cff3a03008091b | 2026-05-24T19:46:03+08:00 |            | COMPLETED  |
| 0   | 17937da188b84f23a5c10bb87588944b |       | CLI  | 6cccdd174bd049ecb05a774bbb47593f | 2026-05-24T19:46:03+08:00 |            | COMPLETED  |
| 0   | 17937da188b84f23a5c10bb87588944b |       | CLI  | ef360d777e57485799adb96b30f2b4b8 | 2026-05-24T19:46:03+08:00 |            | CANCELED   |
| 0   | 17937da188b84f23a5c10bb87588944b |       | CLI  | bcc5c5448cb64de48b6b6171c36fb790 | 2026-05-24T19:46:03+08:00 |            | CANCELED   |
| 0   | 17937da188b84f23a5c10bb87588944b |       | CLI  | bfc25384c43a443294fe2da979a38ac2 | 2026-05-24T19:46:03+08:00 |            | DISPATCHED |
| 0   | 17937da188b84f23a5c10bb87588944b |       | CLI  | 84960537b85d413b8990a9efd5952d67 | 2026-05-24T19:46:04+08:00 |            | DISPATCHED |
| 0   | 17937da188b84f23a5c10bb87588944b |       | CLI  | 3d223c1b51e24b36861a3bfb2f1d58d4 | 2026-05-24T19:46:03+08:00 |            | CANCELED   |
| 0   | 17937da188b84f23a5c10bb87588944b |       | CLI  | e433b0e356b846c89c301621a3c54494 | 2026-05-24T19:46:03+08:00 |            | COMPLETED  |
| 0   | 17937da188b84f23a5c10bb87588944b |       | CLI  | 7c93a3880f074ebd8eca14e6b51bb7ef | 2026-05-24T19:46:03+08:00 |            | COMPLETED  |
| 0   | 17937da188b84f23a5c10bb87588944b |       | CLI  | df2e4ef51aaf4390bff9a23f2692486e | 2026-05-24T19:46:04+08:00 |            | DISPATCHED |
| 0   | 17937da188b84f23a5c10bb87588944b |       | CLI  | 7377c53010194ef7a83aa206698d66ff | 2026-05-24T19:46:05+08:00 |            | DISPATCHED |
| 0   | 17937da188b84f23a5c10bb87588944b |       | CLI  | df64d1a1f9d348e3a2f174c4d7d69e73 | 2026-05-24T19:46:05+08:00 |            | DISPATCHED |
| 0   | 17937da188b84f23a5c10bb87588944b |       | CLI  | b59834512e2847e1bdf13ace04b8a456 | 2026-05-24T19:46:06+08:00 |            | DISPATCHED |
| 0   | 17937da188b84f23a5c10bb87588944b |       | CLI  | 0064bb0ab69344028d1ecfda053826f4 | 2026-05-24T19:46:03+08:00 |            | QUEUED     |
+-----+----------------------------------+-------+------+----------------------------------+---------------------------+------------+------------+


```


### Type of change

- [x] New Feature (non-breaking change which adds functionality)

---------

Signed-off-by: Jin Hai <haijin.chn@gmail.com>
2026-05-25 14:00:08 +08:00

1721 lines
37 KiB
Go

//
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package cli
import "fmt"
// Command parsers
func (p *Parser) parseAdminLoginUser() (*Command, error) {
cmd := NewCommand("login_user")
p.nextToken() // consume LOGIN
if p.curToken.Type != TokenUser {
return nil, fmt.Errorf("expected USER after LOGIN")
}
p.nextToken()
email, err := p.parseQuotedString()
if err != nil {
return nil, err
}
cmd.Params["email"] = email
p.nextToken()
// Optional: PASSWORD 'password'
if p.curToken.Type == TokenPassword {
p.nextToken()
password, err := p.parseQuotedString()
if err != nil {
return nil, err
}
cmd.Params["password"] = password
p.nextToken()
}
// Semicolon is optional for UNSET TOKEN
if p.curToken.Type == TokenSemicolon {
p.nextToken()
}
return cmd, nil
}
func (p *Parser) parseAdminLogout() (*Command, error) {
cmd := NewCommand("logout")
p.nextToken()
// Semicolon is optional for UNSET TOKEN
if p.curToken.Type == TokenSemicolon {
p.nextToken()
}
return cmd, nil
}
func (p *Parser) parseAdminPingServer() (*Command, error) {
cmd := NewCommand("ping")
p.nextToken()
// Semicolon is optional for UNSET TOKEN
if p.curToken.Type == TokenSemicolon {
p.nextToken()
}
return cmd, nil
}
func (p *Parser) parseAdminRegisterCommand() (*Command, error) {
cmd := NewCommand("register_user")
if err := p.expectPeek(TokenUser); err != nil {
return nil, err
}
p.nextToken()
userName, err := p.parseQuotedString()
if err != nil {
return nil, err
}
cmd.Params["user_name"] = userName
p.nextToken()
if p.curToken.Type != TokenAs {
return nil, fmt.Errorf("expected AS")
}
p.nextToken()
nickname, err := p.parseQuotedString()
if err != nil {
return nil, err
}
cmd.Params["nickname"] = nickname
p.nextToken()
if p.curToken.Type != TokenPassword {
return nil, fmt.Errorf("expected PASSWORD")
}
p.nextToken()
password, err := p.parseQuotedString()
if err != nil {
return nil, err
}
cmd.Params["password"] = password
p.nextToken()
// Semicolon is optional for UNSET TOKEN
if p.curToken.Type == TokenSemicolon {
p.nextToken()
}
return cmd, nil
}
func (p *Parser) parseAdminListCommand() (*Command, error) {
p.nextToken() // consume LIST
switch p.curToken.Type {
case TokenServices:
p.nextToken()
// Semicolon is optional for SHOW TOKEN
if p.curToken.Type == TokenSemicolon {
p.nextToken()
}
return NewCommand("list_services"), nil
case TokenUsers:
p.nextToken()
// Semicolon is optional for SHOW TOKEN
if p.curToken.Type == TokenSemicolon {
p.nextToken()
}
return NewCommand("list_users"), nil
case TokenRoles:
p.nextToken()
// Semicolon is optional for SHOW TOKEN
if p.curToken.Type == TokenSemicolon {
p.nextToken()
}
return NewCommand("list_roles"), nil
case TokenVars:
p.nextToken()
// Semicolon is optional for SHOW TOKEN
if p.curToken.Type == TokenSemicolon {
p.nextToken()
}
return NewCommand("list_variables"), nil
case TokenConfigs:
p.nextToken()
// Semicolon is optional for SHOW TOKEN
if p.curToken.Type == TokenSemicolon {
p.nextToken()
}
return NewCommand("list_configs"), nil
case TokenEnvs:
p.nextToken()
// Semicolon is optional for SHOW TOKEN
if p.curToken.Type == TokenSemicolon {
p.nextToken()
}
return NewCommand("list_environments"), nil
case TokenDatasets:
return p.parseAdminListDatasets()
case TokenAgents:
return p.parseAdminListAgents()
case TokenTokens:
return p.parseAdminListTokens()
case TokenModel:
return p.parseAdminListModelProviders()
case TokenDefault:
return p.parseAdminListDefaultModels()
case TokenAvailable:
return p.parseCommonListProviders()
case TokenModels:
return p.parseListModelsOfProvider()
case TokenChats:
p.nextToken()
// Semicolon is optional for SHOW TOKEN
if p.curToken.Type == TokenSemicolon {
p.nextToken()
}
return NewCommand("list_user_chats"), nil
case TokenFiles:
return p.parseAdminListFiles()
case TokenTasks:
return p.parseAdminListTasks()
case TokenIngestors:
return p.parseAdminListIngestors()
case TokenIngestion:
return p.parseAdminListIngestionTasks()
default:
return nil, fmt.Errorf("unknown LIST target: %s", p.curToken.Value)
}
}
func (p *Parser) parseAdminListDatasets() (*Command, error) {
cmd := NewCommand("list_user_datasets")
p.nextToken() // consume DATASETS
if p.curToken.Type == TokenSemicolon {
return cmd, nil
}
if p.curToken.Type == TokenOf {
p.nextToken()
userName, err := p.parseQuotedString()
if err != nil {
return nil, err
}
cmd = NewCommand("list_datasets")
cmd.Params["user_name"] = userName
p.nextToken()
}
// Semicolon is optional for UNSET TOKEN
if p.curToken.Type == TokenSemicolon {
p.nextToken()
}
return cmd, nil
}
func (p *Parser) parseAdminListAgents() (*Command, error) {
p.nextToken() // consume AGENTS
if p.curToken.Type == TokenSemicolon {
return NewCommand("list_user_agents"), nil
}
if p.curToken.Type != TokenOf {
return nil, fmt.Errorf("expected OF")
}
p.nextToken()
userName, err := p.parseQuotedString()
if err != nil {
return nil, err
}
cmd := NewCommand("list_agents")
cmd.Params["user_name"] = userName
p.nextToken()
// Semicolon is optional for UNSET TOKEN
if p.curToken.Type == TokenSemicolon {
p.nextToken()
}
return cmd, nil
}
func (p *Parser) parseAdminListTokens() (*Command, error) {
p.nextToken() // consume TOKENS
cmd := NewCommand("list_tokens")
// Semicolon is optional for UNSET TOKEN
if p.curToken.Type == TokenSemicolon {
p.nextToken()
}
return cmd, nil
}
func (p *Parser) parseAdminListModelProviders() (*Command, error) {
p.nextToken() // consume MODEL
if p.curToken.Type != TokenProviders {
return nil, fmt.Errorf("expected PROVIDERS")
}
p.nextToken()
// Semicolon is optional for UNSET TOKEN
if p.curToken.Type == TokenSemicolon {
p.nextToken()
}
return NewCommand("list_user_model_providers"), nil
}
func (p *Parser) parseAdminListDefaultModels() (*Command, error) {
p.nextToken() // consume DEFAULT
if p.curToken.Type != TokenModels {
return nil, fmt.Errorf("expected MODELS")
}
p.nextToken()
// Semicolon is optional for UNSET TOKEN
if p.curToken.Type == TokenSemicolon {
p.nextToken()
}
return NewCommand("list_user_default_models"), nil
}
func (p *Parser) parseCommonListProviders() (*Command, error) {
p.nextToken() // consume AVAILABLE
if p.curToken.Type != TokenProviders {
return nil, fmt.Errorf("expected PROVIDERS")
}
return NewCommand("list_available_providers"), nil
}
func (p *Parser) parseCommonShowPoolModel() (*Command, error) {
p.nextToken() // consume POOL
if p.curToken.Type == TokenProvider {
p.nextToken()
providerName, err := p.parseQuotedString()
if err != nil {
return nil, err
}
cmd := NewCommand("show_pool_provider")
cmd.Params["provider_name"] = providerName
p.nextToken()
// Semicolon is optional for UNSET TOKEN
if p.curToken.Type == TokenSemicolon {
p.nextToken()
}
return cmd, nil
} else if p.curToken.Type == TokenModel {
p.nextToken() // skip model
modelName, err := p.parseQuotedString()
if err != nil {
return nil, err
}
p.nextToken() // skip model name
if p.curToken.Type != TokenFrom {
return nil, fmt.Errorf("expected FROM")
}
p.nextToken() // skip from
providerName, err := p.parseQuotedString()
if err != nil {
return nil, err
}
p.nextToken() // skip provider name
cmd := NewCommand("show_pool_model")
cmd.Params["provider_name"] = providerName
cmd.Params["model_name"] = modelName
// Semicolon is optional for UNSET TOKEN
if p.curToken.Type == TokenSemicolon {
p.nextToken()
}
return cmd, nil
} else {
return nil, fmt.Errorf("expected PROVIDERS or MODELS")
}
}
func (p *Parser) parseAdminListFiles() (*Command, error) {
p.nextToken() // consume FILES
if p.curToken.Type != TokenOf {
return nil, fmt.Errorf("expected OF")
}
p.nextToken()
if p.curToken.Type != TokenDataset {
return nil, fmt.Errorf("expected DATASET")
}
p.nextToken()
datasetName, err := p.parseQuotedString()
if err != nil {
return nil, err
}
cmd := NewCommand("list_user_dataset_files")
cmd.Params["dataset_name"] = datasetName
p.nextToken()
// Semicolon is optional for UNSET TOKEN
if p.curToken.Type == TokenSemicolon {
p.nextToken()
}
return cmd, nil
}
func (p *Parser) parseAdminListTasks() (*Command, error) {
p.nextToken() // consume TASKS
cmd := NewCommand("list_admin_tasks")
return cmd, nil
}
func (p *Parser) parseAdminListIngestors() (*Command, error) {
p.nextToken() // consume TASKS
cmd := NewCommand("admin_list_ingestors")
return cmd, nil
}
func (p *Parser) parseAdminListIngestionTasks() (*Command, error) {
p.nextToken() // consume Ingestion
if p.curToken.Type != TokenTasks {
return nil, fmt.Errorf("expected TASKS")
}
cmd := NewCommand("list_admin_ingestion_tasks")
return cmd, nil
}
func (p *Parser) parseAdminShowCommand() (*Command, error) {
p.nextToken() // consume SHOW
switch p.curToken.Type {
case TokenVersion:
p.nextToken()
// Semicolon is optional for SHOW TOKEN
if p.curToken.Type == TokenSemicolon {
p.nextToken()
}
return NewCommand("show_version"), nil
case TokenToken:
p.nextToken()
// Semicolon is optional for SHOW TOKEN
if p.curToken.Type == TokenSemicolon {
p.nextToken()
}
return NewCommand("show_token"), nil
case TokenCurrent:
p.nextToken()
if p.curToken.Type != TokenUser {
return nil, fmt.Errorf("expected USER after CURRENT")
}
p.nextToken()
// Semicolon is optional for SHOW TOKEN
if p.curToken.Type == TokenSemicolon {
p.nextToken()
}
return NewCommand("show_current_user"), nil
case TokenUser:
return p.parseShowUser()
case TokenRole:
return p.parseShowRole()
case TokenVar:
return p.parseShowVariable()
case TokenService:
return p.parseShowService()
case TokenProvider:
return p.parseShowProvider()
case TokenModel:
return p.parseShowModel()
default:
return nil, fmt.Errorf("unknown SHOW target: %s", p.curToken.Value)
}
}
func (p *Parser) parseAdminShowUser() (*Command, error) {
p.nextToken() // consume USER
// Check for PERMISSION
if p.curToken.Type == TokenPermission {
p.nextToken()
userName, err := p.parseQuotedString()
if err != nil {
return nil, err
}
cmd := NewCommand("show_user_permission")
cmd.Params["user_name"] = userName
p.nextToken()
// Semicolon is optional for SHOW TOKEN
if p.curToken.Type == TokenSemicolon {
p.nextToken()
}
return cmd, nil
}
userName, err := p.parseQuotedString()
if err != nil {
return nil, err
}
cmd := NewCommand("show_user")
cmd.Params["user_name"] = userName
p.nextToken()
// Semicolon is optional for UNSET TOKEN
if p.curToken.Type == TokenSemicolon {
p.nextToken()
}
return cmd, nil
}
func (p *Parser) parseAdminShowRole() (*Command, error) {
p.nextToken() // consume ROLE
roleName, err := p.parseIdentifier()
if err != nil {
return nil, err
}
cmd := NewCommand("show_role")
cmd.Params["role_name"] = roleName
p.nextToken()
// Semicolon is optional for UNSET TOKEN
if p.curToken.Type == TokenSemicolon {
p.nextToken()
}
return cmd, nil
}
func (p *Parser) parseAdminShowVariable() (*Command, error) {
p.nextToken() // consume VAR
varName, err := p.parseIdentifier()
if err != nil {
return nil, err
}
cmd := NewCommand("show_variable")
cmd.Params["var_name"] = varName
p.nextToken()
// Semicolon is optional for UNSET TOKEN
if p.curToken.Type == TokenSemicolon {
p.nextToken()
}
return cmd, nil
}
func (p *Parser) parseAdminShowService() (*Command, error) {
p.nextToken() // consume SERVICE
serviceNum, err := p.parseNumber()
if err != nil {
return nil, err
}
cmd := NewCommand("show_service")
cmd.Params["number"] = serviceNum
p.nextToken()
// Semicolon is optional for UNSET TOKEN
if p.curToken.Type == TokenSemicolon {
p.nextToken()
}
return cmd, nil
}
func (p *Parser) parseAdminCreateCommand() (*Command, error) {
p.nextToken() // consume CREATE
switch p.curToken.Type {
case TokenUser:
return p.parseCreateUser()
case TokenRole:
return p.parseCreateRole()
case TokenModel:
return p.parseCreateModelProvider()
case TokenDataset:
return p.parseCreateDataset()
case TokenChat:
return p.parseCreateChat()
case TokenToken:
return p.parseCreateToken()
default:
return nil, fmt.Errorf("unknown CREATE target: %s", p.curToken.Value)
}
}
func (p *Parser) parseAdminCreateToken() (*Command, error) {
p.nextToken() // consume TOKEN
// Semicolon is optional for UNSET TOKEN
if p.curToken.Type == TokenSemicolon {
p.nextToken()
}
return NewCommand("create_token"), nil
}
func (p *Parser) parseAdminCreateUser() (*Command, error) {
p.nextToken() // consume USER
userName, err := p.parseQuotedString()
if err != nil {
return nil, err
}
p.nextToken()
password, err := p.parseQuotedString()
if err != nil {
return nil, err
}
cmd := NewCommand("create_user")
cmd.Params["user_name"] = userName
cmd.Params["password"] = password
cmd.Params["role"] = "user"
p.nextToken()
// Semicolon is optional for UNSET TOKEN
if p.curToken.Type == TokenSemicolon {
p.nextToken()
}
return cmd, nil
}
func (p *Parser) parseAdminCreateRole() (*Command, error) {
p.nextToken() // consume ROLE
roleName, err := p.parseIdentifier()
if err != nil {
return nil, err
}
cmd := NewCommand("create_role")
cmd.Params["role_name"] = roleName
p.nextToken()
if p.curToken.Type == TokenDescription {
p.nextToken()
description, err := p.parseQuotedString()
if err != nil {
return nil, err
}
cmd.Params["description"] = description
p.nextToken()
}
// Semicolon is optional for UNSET TOKEN
if p.curToken.Type == TokenSemicolon {
p.nextToken()
}
return cmd, nil
}
func (p *Parser) parseAdminCreateModelProvider() (*Command, error) {
p.nextToken() // consume MODEL
if p.curToken.Type != TokenProvider {
return nil, fmt.Errorf("expected PROVIDER")
}
p.nextToken()
providerName, err := p.parseQuotedString()
if err != nil {
return nil, err
}
p.nextToken()
providerKey, err := p.parseQuotedString()
if err != nil {
return nil, err
}
cmd := NewCommand("create_model_provider")
cmd.Params["provider_name"] = providerName
cmd.Params["provider_key"] = providerKey
p.nextToken()
// Semicolon is optional for UNSET TOKEN
if p.curToken.Type == TokenSemicolon {
p.nextToken()
}
return cmd, nil
}
func (p *Parser) parseAdminCreateDataset() (*Command, error) {
p.nextToken() // consume DATASET
datasetName, err := p.parseQuotedString()
if err != nil {
return nil, err
}
p.nextToken()
if p.curToken.Type != TokenWith {
return nil, fmt.Errorf("expected WITH")
}
p.nextToken()
if p.curToken.Type != TokenEmbedding {
return nil, fmt.Errorf("expected EMBEDDING")
}
p.nextToken()
embedding, err := p.parseQuotedString()
if err != nil {
return nil, err
}
p.nextToken()
cmd := NewCommand("create_user_dataset")
cmd.Params["dataset_name"] = datasetName
cmd.Params["embedding"] = embedding
if p.curToken.Type == TokenParser {
p.nextToken()
parserType, err := p.parseQuotedString()
if err != nil {
return nil, err
}
cmd.Params["parser_type"] = parserType
p.nextToken()
} else if p.curToken.Type == TokenPipeline {
p.nextToken()
pipeline, err := p.parseQuotedString()
if err != nil {
return nil, err
}
cmd.Params["pipeline"] = pipeline
p.nextToken()
} else {
return nil, fmt.Errorf("expected PARSER or PIPELINE")
}
// Semicolon is optional for UNSET TOKEN
if p.curToken.Type == TokenSemicolon {
p.nextToken()
}
return cmd, nil
}
func (p *Parser) parseAdminCreateChat() (*Command, error) {
p.nextToken() // consume CHAT
chatName, err := p.parseQuotedString()
if err != nil {
return nil, err
}
cmd := NewCommand("create_user_chat")
cmd.Params["chat_name"] = chatName
p.nextToken()
// Semicolon is optional for UNSET TOKEN
if p.curToken.Type == TokenSemicolon {
p.nextToken()
}
return cmd, nil
}
func (p *Parser) parseAdminDropCommand() (*Command, error) {
p.nextToken() // consume DROP
switch p.curToken.Type {
case TokenUser:
return p.parseDropUser()
case TokenRole:
return p.parseDropRole()
case TokenDataset:
return p.parseDropDataset()
case TokenChat:
return p.parseDropChat()
case TokenToken:
return p.parseDropToken()
default:
return nil, fmt.Errorf("unknown DROP target: %s", p.curToken.Value)
}
}
func (p *Parser) parseAdminDropToken() (*Command, error) {
p.nextToken() // consume TOKEN
tokenValue, err := p.parseQuotedString()
if err != nil {
return nil, err
}
p.nextToken()
if p.curToken.Type != TokenOf {
return nil, fmt.Errorf("expected OF")
}
p.nextToken()
userName, err := p.parseQuotedString()
if err != nil {
return nil, err
}
cmd := NewCommand("drop_token")
cmd.Params["token"] = tokenValue
cmd.Params["user_name"] = userName
p.nextToken()
// Semicolon is optional
if p.curToken.Type == TokenSemicolon {
p.nextToken()
}
return cmd, nil
}
func (p *Parser) parseAdminDropUser() (*Command, error) {
p.nextToken() // consume USER
userName, err := p.parseQuotedString()
if err != nil {
return nil, err
}
cmd := NewCommand("drop_user")
cmd.Params["user_name"] = userName
p.nextToken()
// Semicolon is optional for UNSET TOKEN
if p.curToken.Type == TokenSemicolon {
p.nextToken()
}
return cmd, nil
}
func (p *Parser) parseAdminDropRole() (*Command, error) {
p.nextToken() // consume ROLE
roleName, err := p.parseIdentifier()
if err != nil {
return nil, err
}
cmd := NewCommand("drop_role")
cmd.Params["role_name"] = roleName
p.nextToken()
// Semicolon is optional for UNSET TOKEN
if p.curToken.Type == TokenSemicolon {
p.nextToken()
}
return cmd, nil
}
func (p *Parser) parseAdminDropModelProvider() (*Command, error) {
p.nextToken() // consume MODEL
if p.curToken.Type != TokenProvider {
return nil, fmt.Errorf("expected PROVIDER")
}
p.nextToken()
providerName, err := p.parseQuotedString()
if err != nil {
return nil, err
}
cmd := NewCommand("drop_model_provider")
cmd.Params["provider_name"] = providerName
p.nextToken()
// Semicolon is optional for UNSET TOKEN
if p.curToken.Type == TokenSemicolon {
p.nextToken()
}
return cmd, nil
}
func (p *Parser) parseAdminDropDataset() (*Command, error) {
p.nextToken() // consume DATASET
datasetName, err := p.parseQuotedString()
if err != nil {
return nil, err
}
cmd := NewCommand("drop_user_dataset")
cmd.Params["dataset_name"] = datasetName
p.nextToken()
// Semicolon is optional for UNSET TOKEN
if p.curToken.Type == TokenSemicolon {
p.nextToken()
}
return cmd, nil
}
func (p *Parser) parseAdminDropChat() (*Command, error) {
p.nextToken() // consume CHAT
chatName, err := p.parseQuotedString()
if err != nil {
return nil, err
}
cmd := NewCommand("drop_user_chat")
cmd.Params["chat_name"] = chatName
p.nextToken()
// Semicolon is optional for UNSET TOKEN
if p.curToken.Type == TokenSemicolon {
p.nextToken()
}
return cmd, nil
}
func (p *Parser) parseAdminAlterCommand() (*Command, error) {
p.nextToken() // consume ALTER
switch p.curToken.Type {
case TokenUser:
return p.parseAlterUser()
case TokenRole:
return p.parseAlterRole()
default:
return nil, fmt.Errorf("unknown ALTER target: %s", p.curToken.Value)
}
}
func (p *Parser) parseAdminAlterUser() (*Command, error) {
p.nextToken() // consume USER
if p.curToken.Type == TokenActive {
return p.parseActivateUser()
}
if p.curToken.Type == TokenPassword {
p.nextToken()
userName, err := p.parseQuotedString()
if err != nil {
return nil, err
}
p.nextToken()
password, err := p.parseQuotedString()
if err != nil {
return nil, err
}
cmd := NewCommand("alter_user")
cmd.Params["user_name"] = userName
cmd.Params["password"] = password
p.nextToken()
// Semicolon is optional for SHOW TOKEN
if p.curToken.Type == TokenSemicolon {
p.nextToken()
}
return cmd, nil
}
userName, err := p.parseQuotedString()
if err != nil {
return nil, err
}
p.nextToken()
if p.curToken.Type != TokenSet {
return nil, fmt.Errorf("expected SET")
}
p.nextToken()
if p.curToken.Type != TokenRole {
return nil, fmt.Errorf("expected ROLE")
}
p.nextToken()
roleName, err := p.parseIdentifier()
if err != nil {
return nil, err
}
cmd := NewCommand("alter_user_role")
cmd.Params["user_name"] = userName
cmd.Params["role_name"] = roleName
p.nextToken()
// Semicolon is optional for UNSET TOKEN
if p.curToken.Type == TokenSemicolon {
p.nextToken()
}
return cmd, nil
}
func (p *Parser) parseAdminActivateUser() (*Command, error) {
p.nextToken() // consume ACTIVE
userName, err := p.parseQuotedString()
if err != nil {
return nil, err
}
p.nextToken()
// Accept 'on' or 'off' as identifier
status := p.curToken.Value
if status != "on" && status != "off" {
return nil, fmt.Errorf("expected 'on' or 'off', got %s", p.curToken.Value)
}
cmd := NewCommand("activate_user")
cmd.Params["user_name"] = userName
cmd.Params["activate_status"] = status
p.nextToken()
// Semicolon is optional for UNSET TOKEN
if p.curToken.Type == TokenSemicolon {
p.nextToken()
}
return cmd, nil
}
func (p *Parser) parseAdminAlterRole() (*Command, error) {
p.nextToken() // consume ROLE
roleName, err := p.parseIdentifier()
if err != nil {
return nil, err
}
p.nextToken()
if p.curToken.Type != TokenSet {
return nil, fmt.Errorf("expected SET")
}
p.nextToken()
if p.curToken.Type != TokenDescription {
return nil, fmt.Errorf("expected DESCRIPTION")
}
p.nextToken()
description, err := p.parseQuotedString()
if err != nil {
return nil, err
}
cmd := NewCommand("alter_role")
cmd.Params["role_name"] = roleName
cmd.Params["description"] = description
p.nextToken()
// Semicolon is optional for UNSET TOKEN
if p.curToken.Type == TokenSemicolon {
p.nextToken()
}
return cmd, nil
}
func (p *Parser) parseAdminGrantCommand() (*Command, error) {
p.nextToken() // consume GRANT
if p.curToken.Type == TokenAdmin {
return p.parseGrantAdmin()
}
return p.parseGrantPermission()
}
func (p *Parser) parseAdminGrantAdmin() (*Command, error) {
p.nextToken() // consume ADMIN
userName, err := p.parseQuotedString()
if err != nil {
return nil, err
}
cmd := NewCommand("grant_admin")
cmd.Params["user_name"] = userName
p.nextToken()
// Semicolon is optional for UNSET TOKEN
if p.curToken.Type == TokenSemicolon {
p.nextToken()
}
return cmd, nil
}
func (p *Parser) parseAdminGrantPermission() (*Command, error) {
actions, err := p.parseIdentifierList()
if err != nil {
return nil, err
}
if p.curToken.Type != TokenOn {
return nil, fmt.Errorf("expected ON")
}
p.nextToken()
resource, err := p.parseIdentifier()
if err != nil {
return nil, err
}
p.nextToken()
if p.curToken.Type != TokenTo {
return nil, fmt.Errorf("expected TO")
}
p.nextToken()
if p.curToken.Type != TokenRole {
return nil, fmt.Errorf("expected ROLE")
}
p.nextToken()
roleName, err := p.parseIdentifier()
if err != nil {
return nil, err
}
cmd := NewCommand("grant_permission")
cmd.Params["actions"] = actions
cmd.Params["resource"] = resource
cmd.Params["role_name"] = roleName
p.nextToken()
// Semicolon is optional for UNSET TOKEN
if p.curToken.Type == TokenSemicolon {
p.nextToken()
}
return cmd, nil
}
func (p *Parser) parseAdminRevokeCommand() (*Command, error) {
p.nextToken() // consume REVOKE
if p.curToken.Type == TokenAdmin {
return p.parseRevokeAdmin()
}
return p.parseRevokePermission()
}
func (p *Parser) parseAdminRevokeAdmin() (*Command, error) {
p.nextToken() // consume ADMIN
userName, err := p.parseQuotedString()
if err != nil {
return nil, err
}
cmd := NewCommand("revoke_admin")
cmd.Params["user_name"] = userName
p.nextToken()
// Semicolon is optional for UNSET TOKEN
if p.curToken.Type == TokenSemicolon {
p.nextToken()
}
return cmd, nil
}
func (p *Parser) parseAdminRevokePermission() (*Command, error) {
actions, err := p.parseIdentifierList()
if err != nil {
return nil, err
}
if p.curToken.Type != TokenOn {
return nil, fmt.Errorf("expected ON")
}
p.nextToken()
resource, err := p.parseIdentifier()
if err != nil {
return nil, err
}
p.nextToken()
if p.curToken.Type != TokenFrom {
return nil, fmt.Errorf("expected FROM")
}
p.nextToken()
if p.curToken.Type != TokenRole {
return nil, fmt.Errorf("expected ROLE")
}
p.nextToken()
roleName, err := p.parseIdentifier()
if err != nil {
return nil, err
}
cmd := NewCommand("revoke_permission")
cmd.Params["actions"] = actions
cmd.Params["resource"] = resource
cmd.Params["role_name"] = roleName
p.nextToken()
// Semicolon is optional for UNSET TOKEN
if p.curToken.Type == TokenSemicolon {
p.nextToken()
}
return cmd, nil
}
func (p *Parser) parseAdminIdentifierList() ([]string, error) {
var list []string
ident, err := p.parseIdentifier()
if err != nil {
return nil, err
}
list = append(list, ident)
p.nextToken()
for p.curToken.Type == TokenComma {
p.nextToken()
ident, err := p.parseIdentifier()
if err != nil {
return nil, err
}
list = append(list, ident)
p.nextToken()
}
return list, nil
}
func (p *Parser) parseAdminSetCommand() (*Command, error) {
p.nextToken() // consume SET
if p.curToken.Type == TokenVar {
return p.parseSetVariable()
}
if p.curToken.Type == TokenDefault {
return p.parseSetDefault()
}
if p.curToken.Type == TokenToken {
return p.parseSetToken()
}
return nil, fmt.Errorf("unknown SET target: %s", p.curToken.Value)
}
func (p *Parser) parseAdminSetVariable() (*Command, error) {
p.nextToken() // consume VAR
varName, err := p.parseIdentifier()
if err != nil {
return nil, err
}
p.nextToken()
varValue, err := p.parseVariableValue()
if err != nil {
return nil, err
}
cmd := NewCommand("set_variable")
cmd.Params["var_name"] = varName
cmd.Params["var_value"] = varValue
p.nextToken()
// Semicolon is optional for UNSET TOKEN
if p.curToken.Type == TokenSemicolon {
p.nextToken()
}
return cmd, nil
}
func (p *Parser) parseAdminSetDefault() (*Command, error) {
p.nextToken() // consume DEFAULT
var modelType string
switch p.curToken.Type {
case TokenChat:
modelType = "chat"
case TokenVision:
modelType = "vision"
case TokenEmbedding:
modelType = "embedding"
case TokenRerank:
modelType = "rerank"
case TokenASR:
modelType = "asr"
case TokenTTS:
modelType = "tts"
case TokenOCR:
modelType = "ocr"
default:
return nil, fmt.Errorf("unknown model type: %s", p.curToken.Value)
}
p.nextToken()
compositeModelName, err := p.parseQuotedString()
if err != nil {
return nil, err
}
cmd := NewCommand("set_default_model")
cmd.Params["model_type"] = modelType
cmd.Params["composite_model_name"] = compositeModelName
p.nextToken()
// Semicolon is optional for UNSET TOKEN
if p.curToken.Type == TokenSemicolon {
p.nextToken()
}
return cmd, nil
}
func (p *Parser) parseAdminSetToken() (*Command, error) {
p.nextToken() // consume TOKEN
tokenValue, err := p.parseQuotedString()
if err != nil {
return nil, err
}
cmd := NewCommand("set_token")
cmd.Params["token"] = tokenValue
p.nextToken()
// Semicolon is optional for UNSET TOKEN
if p.curToken.Type == TokenSemicolon {
p.nextToken()
}
return cmd, nil
}
func (p *Parser) parseAdminResetCommand() (*Command, error) {
p.nextToken() // consume RESET
if p.curToken.Type != TokenDefault {
return nil, fmt.Errorf("expected DEFAULT")
}
p.nextToken()
var modelType string
switch p.curToken.Type {
case TokenChat:
modelType = "chat"
case TokenVision:
modelType = "vision"
case TokenEmbedding:
modelType = "embedding"
case TokenRerank:
modelType = "rerank"
case TokenASR:
modelType = "asr"
case TokenTTS:
modelType = "tts"
case TokenOCR:
modelType = "ocr"
default:
return nil, fmt.Errorf("unknown model type: %s", p.curToken.Value)
}
cmd := NewCommand("reset_default_model")
cmd.Params["model_type"] = modelType
p.nextToken()
// Semicolon is optional for UNSET TOKEN
if p.curToken.Type == TokenSemicolon {
p.nextToken()
}
return cmd, nil
}
func (p *Parser) parseAdminGenerateCommand() (*Command, error) {
p.nextToken() // consume GENERATE
if p.curToken.Type != TokenToken {
return nil, fmt.Errorf("expected TOKEN")
}
p.nextToken()
if p.curToken.Type != TokenFor {
return nil, fmt.Errorf("expected FOR")
}
p.nextToken()
if p.curToken.Type != TokenUser {
return nil, fmt.Errorf("expected USER")
}
p.nextToken()
userName, err := p.parseQuotedString()
if err != nil {
return nil, err
}
cmd := NewCommand("generate_token")
cmd.Params["user_name"] = userName
p.nextToken()
// Semicolon is optional
if p.curToken.Type == TokenSemicolon {
p.nextToken()
}
return cmd, nil
}
func (p *Parser) parseAdminImportCommand() (*Command, error) {
p.nextToken() // consume IMPORT
documentPaths, err := p.parseQuotedString()
if err != nil {
return nil, err
}
p.nextToken()
if p.curToken.Type != TokenInto {
return nil, fmt.Errorf("expected INTO")
}
p.nextToken()
if p.curToken.Type != TokenDataset {
return nil, fmt.Errorf("expected DATASET")
}
p.nextToken()
datasetName, err := p.parseQuotedString()
if err != nil {
return nil, err
}
cmd := NewCommand("import_docs_into_dataset")
cmd.Params["document_paths"] = documentPaths
cmd.Params["dataset_name"] = datasetName
p.nextToken()
// Semicolon is optional for UNSET TOKEN
if p.curToken.Type == TokenSemicolon {
p.nextToken()
}
return cmd, nil
}
func (p *Parser) parseAdminSearchCommand() (*Command, error) {
p.nextToken() // consume SEARCH
question, err := p.parseQuotedString()
if err != nil {
return nil, err
}
p.nextToken()
if p.curToken.Type != TokenOn {
return nil, fmt.Errorf("expected ON")
}
p.nextToken()
if p.curToken.Type != TokenDatasets {
return nil, fmt.Errorf("expected DATASETS")
}
p.nextToken()
datasets, err := p.parseQuotedString()
if err != nil {
return nil, err
}
cmd := NewCommand("search_on_datasets")
cmd.Params["question"] = question
cmd.Params["datasets"] = datasets
p.nextToken()
// Semicolon is optional for UNSET TOKEN
if p.curToken.Type == TokenSemicolon {
p.nextToken()
}
return cmd, nil
}
func (p *Parser) parseAdminParseCommand() (*Command, error) {
p.nextToken() // consume PARSE
if p.curToken.Type == TokenDataset {
return p.parseParseDataset()
}
return p.parseParseDocs()
}
func (p *Parser) parseAdminParseDataset() (*Command, error) {
p.nextToken() // consume DATASET
datasetName, err := p.parseQuotedString()
if err != nil {
return nil, err
}
p.nextToken()
var method string
if p.curToken.Type == TokenSync {
method = "sync"
} else if p.curToken.Type == TokenAsync {
method = "async"
} else {
return nil, fmt.Errorf("expected SYNC or ASYNC")
}
cmd := NewCommand("parse_dataset")
cmd.Params["dataset_name"] = datasetName
cmd.Params["method"] = method
p.nextToken()
// Semicolon is optional for UNSET TOKEN
if p.curToken.Type == TokenSemicolon {
p.nextToken()
}
return cmd, nil
}
func (p *Parser) parseAdminParseDocs() (*Command, error) {
documentNames, err := p.parseQuotedString()
if err != nil {
return nil, err
}
p.nextToken()
if p.curToken.Type != TokenOf {
return nil, fmt.Errorf("expected OF")
}
p.nextToken()
if p.curToken.Type != TokenDataset {
return nil, fmt.Errorf("expected DATASET")
}
p.nextToken()
datasetName, err := p.parseQuotedString()
if err != nil {
return nil, err
}
cmd := NewCommand("parse_dataset_docs")
cmd.Params["document_names"] = documentNames
cmd.Params["dataset_name"] = datasetName
p.nextToken()
// Semicolon is optional for UNSET TOKEN
if p.curToken.Type == TokenSemicolon {
p.nextToken()
}
return cmd, nil
}
func (p *Parser) parseAdminBenchmarkCommand() (*Command, error) {
cmd := NewCommand("benchmark")
p.nextToken() // consume BENCHMARK
concurrency, err := p.parseNumber()
if err != nil {
return nil, err
}
cmd.Params["concurrency"] = concurrency
p.nextToken()
iterations, err := p.parseNumber()
if err != nil {
return nil, err
}
cmd.Params["iterations"] = iterations
p.nextToken()
// Parse user_statement
nestedCmd, err := p.parseUserStatement() // Not only user statement
if err != nil {
return nil, err
}
cmd.Params["command"] = nestedCmd
return cmd, nil
}
func (p *Parser) parseAdminUserStatement() (*Command, error) {
switch p.curToken.Type {
case TokenPing:
return p.parsePingServer()
case TokenShow:
return p.parseShowCommand()
case TokenCreate:
return p.parseCreateCommand()
case TokenDrop:
return p.parseDropCommand()
case TokenSet:
return p.parseSetCommand()
case TokenUnset:
return p.parseUnsetCommand()
case TokenReset:
return p.parseResetCommand()
case TokenList:
return p.parseListCommand()
case TokenParse:
return p.parseParseCommand()
case TokenImport:
return p.parseImportCommand()
case TokenSearch:
return p.parseSearchCommand()
default:
return nil, fmt.Errorf("invalid user statement: %s", p.curToken.Value)
}
}
func (p *Parser) parseAdminStartupCommand() (*Command, error) {
p.nextToken() // consume STARTUP
if p.curToken.Type != TokenService {
return nil, fmt.Errorf("expected SERVICE")
}
p.nextToken()
serviceNum, err := p.parseNumber()
if err != nil {
return nil, err
}
cmd := NewCommand("startup_service")
cmd.Params["number"] = serviceNum
p.nextToken()
// Semicolon is optional for UNSET TOKEN
if p.curToken.Type == TokenSemicolon {
p.nextToken()
}
return cmd, nil
}
func (p *Parser) parseAdminShutdownCommand() (*Command, error) {
p.nextToken() // consume SHUTDOWN
switch p.curToken.Type {
case TokenService:
return p.parseAdminShutdownServiceCommand()
case TokenIngestor:
return p.parseAdminShutdownIngestorCommand()
default:
return nil, fmt.Errorf("expected SERVICE or INGESTOR")
}
}
func (p *Parser) parseAdminShutdownServiceCommand() (*Command, error) {
p.nextToken() // consume SERVICE
serviceNum, err := p.parseNumber()
if err != nil {
return nil, err
}
cmd := NewCommand("shutdown_service")
cmd.Params["number"] = serviceNum
p.nextToken()
// Semicolon is optional for UNSET TOKEN
if p.curToken.Type == TokenSemicolon {
p.nextToken()
}
return cmd, nil
}
func (p *Parser) parseAdminShutdownIngestorCommand() (*Command, error) {
p.nextToken() // consume INGESTOR
ingestorName, err := p.parseQuotedString()
if err != nil {
return nil, err
}
cmd := NewCommand("admin_shutdown_ingestor_command")
cmd.Params["ingestor_name"] = ingestorName
p.nextToken()
// Semicolon is optional for UNSET TOKEN
if p.curToken.Type == TokenSemicolon {
p.nextToken()
}
return cmd, nil
}
func (p *Parser) parseAdminRestartCommand() (*Command, error) {
p.nextToken() // consume RESTART
if p.curToken.Type != TokenService {
return nil, fmt.Errorf("expected SERVICE")
}
p.nextToken()
serviceNum, err := p.parseNumber()
if err != nil {
return nil, err
}
cmd := NewCommand("restart_service")
cmd.Params["number"] = serviceNum
p.nextToken()
// Semicolon is optional for UNSET TOKEN
if p.curToken.Type == TokenSemicolon {
p.nextToken()
}
return cmd, nil
}
func (p *Parser) parseStartIngestion() (*Command, error) {
p.nextToken() // consume Start
if p.curToken.Type != TokenIngestion {
return nil, fmt.Errorf("expect INGESTION")
}
p.nextToken() // consume Ingest
uri, err := p.parseQuotedString()
if err != nil {
return nil, err
}
cmd := NewCommand("admin_start_ingestion_command")
cmd.Params["uri"] = uri
p.nextToken()
// Semicolon is optional for UNSET TOKEN
if p.curToken.Type == TokenSemicolon {
p.nextToken()
}
return cmd, nil
}
func (p *Parser) parseStopIngestion() (*Command, error) {
p.nextToken() // consume Stop
if p.curToken.Type != TokenIngestion {
return nil, fmt.Errorf("expect INGESTION")
}
p.nextToken() // consume Ingest
taskID, err := p.parseQuotedString()
if err != nil {
return nil, err
}
cmd := NewCommand("admin_stop_ingestion_command")
cmd.Params["task_id"] = taskID
p.nextToken()
// Semicolon is optional for UNSET TOKEN
if p.curToken.Type == TokenSemicolon {
p.nextToken()
}
return cmd, nil
}
func (p *Parser) parseAdminIngestCommand() (*Command, error) {
p.nextToken() // consume Ingest
uri, err := p.parseQuotedString()
if err != nil {
return nil, err
}
cmd := NewCommand("admin_ingest_command")
cmd.Params["uri"] = uri
p.nextToken()
// Semicolon is optional for UNSET TOKEN
if p.curToken.Type == TokenSemicolon {
p.nextToken()
}
return cmd, nil
}
func (p *Parser) parseAdminUnsetCommand() (*Command, error) {
p.nextToken() // consume UNSET
if p.curToken.Type != TokenToken {
return nil, fmt.Errorf("expected TOKEN after UNSET")
}
p.nextToken()
// Semicolon is optional for UNSET TOKEN
if p.curToken.Type == TokenSemicolon {
p.nextToken()
}
return NewCommand("unset_token"), nil
}