youngkingdom/ragflow
1
0
Fork 0
mirror of https://github.com/infiniflow/ragflow.git synced 2026-05-03 16:57:48 +08:00
Code Issues Packages Projects Releases Wiki Activity

Implement Create/Drop Index/Metadata index in GO (#13791)

Browse Source 
### What problem does this PR solve?

Implement Create/Drop Index/Metadata index in GO

New API handling in GO:
POST/kb/index 
DELETE /kb/index
POST /tenant/doc_meta_index
DELETE /tenant/doc_meta_index

CREATE INDEX FOR DATASET 'dataset_name' VECTOR_SIZE 1024;
DROP INDEX FOR DATASET 'dataset_name';
CREATE INDEX DOC_META;
DROP INDEX DOC_META;

### Type of change

- [x] Refactoring
This commit is contained in:
qinling0210
2026-03-26 11:54:10 +08:00
committed by GitHub
parent d19ca71b43
commit ebf36950e4
20 changed files with 1165 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
internal/cli/cli.go
View File

@ -564,6 +564,10 @@ Commands (User Mode):
SET TOKEN 'token_value'; - Set and validate API token
SHOW TOKEN; - Show current API token
UNSET TOKEN; - Remove current API token
CREATE INDEX FOR DATASET 'name' VECTOR_SIZE N; - Create index for dataset
DROP INDEX FOR DATASET 'name'; - Drop index for dataset
CREATE INDEX DOC_META; - Create doc meta index
DROP INDEX DOC_META; - Drop doc meta index
Commands (Admin Mode):
LIST USERS; - List all users

8
internal/cli/client.go
View File

@ -372,6 +372,14 @@ func (c *RAGFlowClient) ExecuteUserCommand(cmd *Command) (ResponseIf, error) {
return c.UnsetToken(cmd)
case "show_version":
return c.ShowServerVersion(cmd)
case "create_index":
return c.CreateIndex(cmd)
case "drop_index":
return c.DropIndex(cmd)
case "create_doc_meta_index":
return c.CreateDocMetaIndex(cmd)
case "drop_doc_meta_index":
return c.DropDocMetaIndex(cmd)
// TODO: Implement other commands
default:
return nil, fmt.Errorf("command '%s' would be executed with API", cmd.Type)

6
internal/cli/lexer.go
View File

@ -293,6 +293,12 @@ func (l *Lexer) lookupIdent(ident string) Token {
return Token{Type: TokenToken, Value: ident}
case "TOKENS":
return Token{Type: TokenTokens, Value: ident}
case "INDEX":
return Token{Type: TokenIndex, Value: ident}
case "VECTOR_SIZE":
return Token{Type: TokenVectorSize, Value: ident}
case "DOC_META":
return Token{Type: TokenDocMeta, Value: ident}
default:
return Token{Type: TokenIdentifier, Value: ident}
}

2
internal/cli/parser.go
View File

@ -208,7 +208,7 @@ func (p *Parser) expectSemicolon() error {
}
func isKeyword(tokenType int) bool {
return tokenType >= TokenLogin && tokenType <= TokenPing
return tokenType >= TokenLogin && tokenType <= TokenDocMeta
}
// Helper functions for parsing

3
internal/cli/types.go
View File

@ -98,6 +98,9 @@ const (
TokenToken
TokenTokens
TokenUnset
TokenIndex
TokenVectorSize
TokenDocMeta
// Literals
TokenIdentifier

180
internal/cli/user_command.go
View File

@ -546,3 +546,183 @@ func (c *RAGFlowClient) UnsetToken(cmd *Command) (ResponseIf, error) {
result.Duration = 0
return &result, nil
}
// CreateIndex creates an index for a dataset
func (c *RAGFlowClient) CreateIndex(cmd *Command) (ResponseIf, error) {
if c.ServerType != "user" {
return nil, fmt.Errorf("this command is only allowed in USER mode")
}
datasetName, ok := cmd.Params["dataset_name"].(string)
if !ok {
return nil, fmt.Errorf("dataset_name not provided")
}
vectorSize, ok := cmd.Params["vector_size"].(int)
if !ok {
return nil, fmt.Errorf("vector_size not provided")
}
// Get dataset ID by name
datasetID, err := c.getDatasetID(datasetName)
if err != nil {
return nil, err
}
payload := map[string]interface{}{
"kb_id": datasetID,
"vector_size": vectorSize,
}
resp, err := c.HTTPClient.Request("POST", "/kb/index", false, "web", nil, payload)
if err != nil {
return nil, fmt.Errorf("failed to create index: %w", err)
}
if resp.StatusCode != 200 {
return nil, fmt.Errorf("failed to create index: HTTP %d, body: %s", resp.StatusCode, string(resp.Body))
}
resJSON, err := resp.JSON()
if err != nil {
return nil, fmt.Errorf("invalid JSON response: %w", err)
}
code, ok := resJSON["code"].(float64)
if !ok {
return nil, fmt.Errorf("invalid response format: code is not a number")
}
var result SimpleResponse
result.Code = int(code)
if result.Code == 0 {
result.Message = fmt.Sprintf("Success to create index for dataset: %s", datasetName)
} else {
result.Message = fmt.Sprintf("Failed to create index: %v", resJSON)
}
result.Duration = 0
return &result, nil
}
// DropIndex drops an index for a dataset
func (c *RAGFlowClient) DropIndex(cmd *Command) (ResponseIf, error) {
if c.ServerType != "user" {
return nil, fmt.Errorf("this command is only allowed in USER mode")
}
datasetName, ok := cmd.Params["dataset_name"].(string)
if !ok {
return nil, fmt.Errorf("dataset_name not provided")
}
// Get dataset ID by name
datasetID, err := c.getDatasetID(datasetName)
if err != nil {
return nil, err
}
payload := map[string]interface{}{
"kb_id": datasetID,
}
resp, err := c.HTTPClient.Request("DELETE", "/kb/index", false, "web", nil, payload)
if err != nil {
return nil, fmt.Errorf("failed to drop index: %w", err)
}
if resp.StatusCode != 200 {
return nil, fmt.Errorf("failed to drop index: HTTP %d, body: %s", resp.StatusCode, string(resp.Body))
}
resJSON, err := resp.JSON()
if err != nil {
return nil, fmt.Errorf("invalid JSON response: %w", err)
}
code, ok := resJSON["code"].(float64)
if !ok {
return nil, fmt.Errorf("invalid response format: code is not a number")
}
var result SimpleResponse
result.Code = int(code)
if result.Code == 0 {
result.Message = fmt.Sprintf("Success to drop index for dataset: %s", datasetName)
} else {
result.Message = fmt.Sprintf("Failed to drop index: %v", resJSON)
}
result.Duration = 0
return &result, nil
}
// CreateDocMetaIndex creates the document metadata index for the tenant
func (c *RAGFlowClient) CreateDocMetaIndex(cmd *Command) (ResponseIf, error) {
if c.ServerType != "user" {
return nil, fmt.Errorf("this command is only allowed in USER mode")
}
resp, err := c.HTTPClient.Request("POST", "/tenant/doc_meta_index", false, "web", nil, nil)
if err != nil {
return nil, fmt.Errorf("failed to create doc meta index: %w", err)
}
if resp.StatusCode != 200 {
return nil, fmt.Errorf("failed to create doc meta index: HTTP %d, body: %s", resp.StatusCode, string(resp.Body))
}
resJSON, err := resp.JSON()
if err != nil {
return nil, fmt.Errorf("invalid JSON response: %w", err)
}
code, ok := resJSON["code"].(float64)
if !ok {
return nil, fmt.Errorf("invalid response format: code is not a number")
}
var result SimpleResponse
result.Code = int(code)
if result.Code == 0 {
result.Message = "Success to create doc meta index"
} else {
result.Message = fmt.Sprintf("Failed to create doc meta index: %v", resJSON)
}
result.Duration = 0
return &result, nil
}
// DropDocMetaIndex drops the document metadata index for the tenant
func (c *RAGFlowClient) DropDocMetaIndex(cmd *Command) (ResponseIf, error) {
if c.ServerType != "user" {
return nil, fmt.Errorf("this command is only allowed in USER mode")
}
resp, err := c.HTTPClient.Request("DELETE", "/tenant/doc_meta_index", false, "web", nil, nil)
if err != nil {
return nil, fmt.Errorf("failed to drop doc meta index: %w", err)
}
if resp.StatusCode != 200 {
return nil, fmt.Errorf("failed to drop doc meta index: HTTP %d, body: %s", resp.StatusCode, string(resp.Body))
}
resJSON, err := resp.JSON()
if err != nil {
return nil, fmt.Errorf("invalid JSON response: %w", err)
}
code, ok := resJSON["code"].(float64)
if !ok {
return nil, fmt.Errorf("invalid response format: code is not a number")
}
var result SimpleResponse
result.Code = int(code)
if result.Code == 0 {
result.Message = "Success to drop doc meta index"
} else {
result.Message = fmt.Sprintf("Failed to drop doc meta index: %v", resJSON)
}
result.Duration = 0
return &result, nil
}

99
internal/cli/user_parser.go
View File

@ -432,6 +432,8 @@ func (p *Parser) parseCreateCommand() (*Command, error) {
return p.parseCreateChat()
case TokenToken:
return p.parseCreateToken()
case TokenIndex:
return p.parseCreateIndex()
default:
return nil, fmt.Errorf("unknown CREATE target: %s", p.curToken.Value)
}
@ -448,6 +450,61 @@ func (p *Parser) parseCreateToken() (*Command, error) {
return NewCommand("create_token"), nil
}
func (p *Parser) parseCreateIndex() (*Command, error) {
// CREATE INDEX FOR DATASET 'name' VECTOR_SIZE N
// CREATE INDEX DOC_META
p.nextToken() // consume INDEX
// Check if creating doc meta index
if p.curToken.Type == TokenDocMeta {
p.nextToken()
if p.curToken.Type == TokenSemicolon {
p.nextToken()
}
return NewCommand("create_doc_meta_index"), nil
}
// Otherwise, must be CREATE INDEX FOR DATASET 'name' VECTOR_SIZE N
if p.curToken.Type != TokenFor {
return nil, fmt.Errorf("expected FOR or DOC_META after INDEX, got %s", p.curToken.Value)
}
p.nextToken()
if p.curToken.Type != TokenDataset {
return nil, fmt.Errorf("expected DATASET after FOR, got %s", p.curToken.Value)
}
p.nextToken()
datasetName, err := p.parseQuotedString()
if err != nil {
return nil, fmt.Errorf("expected dataset name, got %s", p.curToken.Value)
}
p.nextToken()
if p.curToken.Type != TokenVectorSize {
return nil, fmt.Errorf("expected VECTOR_SIZE after dataset name, got %s", p.curToken.Value)
}
p.nextToken()
if p.curToken.Type != TokenNumber {
return nil, fmt.Errorf("expected vector size number, got %s", p.curToken.Value)
}
vectorSize, err := strconv.Atoi(p.curToken.Value)
if err != nil {
return nil, fmt.Errorf("invalid vector size: %s", p.curToken.Value)
}
p.nextToken()
if p.curToken.Type == TokenSemicolon {
p.nextToken()
}
cmd := NewCommand("create_index")
cmd.Params["dataset_name"] = datasetName
cmd.Params["vector_size"] = vectorSize
return cmd, nil
}
func (p *Parser) parseCreateUser() (*Command, error) {
p.nextToken() // consume USER
userName, err := p.parseQuotedString()
@ -620,6 +677,8 @@ func (p *Parser) parseDropCommand() (*Command, error) {
return p.parseDropChat()
case TokenToken:
return p.parseDropToken()
case TokenIndex:
return p.parseDropIndex()
default:
return nil, fmt.Errorf("unknown DROP target: %s", p.curToken.Value)
}
@ -656,6 +715,46 @@ func (p *Parser) parseDropToken() (*Command, error) {
return cmd, nil
}
func (p *Parser) parseDropIndex() (*Command, error) {
// DROP INDEX FOR DATASET 'name' OR DROP INDEX DOC_META
p.nextToken() // consume INDEX
// Check if dropping doc meta index
if p.curToken.Type == TokenDocMeta {
p.nextToken()
if p.curToken.Type == TokenSemicolon {
p.nextToken()
}
cmd := NewCommand("drop_doc_meta_index")
return cmd, nil
}
// Otherwise, must be DROP INDEX FOR DATASET 'name'
if p.curToken.Type != TokenFor {
return nil, fmt.Errorf("expected FOR or DOC_META after INDEX, got %s", p.curToken.Value)
}
p.nextToken()
if p.curToken.Type != TokenDataset {
return nil, fmt.Errorf("expected DATASET after FOR, got %s", p.curToken.Value)
}
p.nextToken()
datasetName, err := p.parseQuotedString()
if err != nil {
return nil, fmt.Errorf("expected dataset name, got %s", p.curToken.Value)
}
p.nextToken()
if p.curToken.Type == TokenSemicolon {
p.nextToken()
}
cmd := NewCommand("drop_index")
cmd.Params["dataset_name"] = datasetName
return cmd, nil
}
func (p *Parser) parseDropUser() (*Command, error) {
p.nextToken() // consume USER
userName, err := p.parseQuotedString()