Refactor: dataset / kb API to RESTFul style (#13619)

### What problem does this PR solve?

1. Split dataset api to gateway and service, and modify web UI to use
restful http api.
2. Old KB releated APIs are commented.

### Type of change

- [x] Refactoring

web/src/components/chunk-method-dialog/index.tsx

@@ -181,7 +181,7 @@ export function ChunkMethodDialog({
   });
 
   const selectedTag = useWatch({
-    name: 'parser_id',
+    name: 'chunk_method',
     control: form.control,
   });
   const isMineruSelected =

web/src/components/knowledge-base-item.tsx

@@ -23,7 +23,7 @@ export function useDisableDifferenceEmbeddingDataset() {
 
   useEffect(() => {
     const datasetListMap = datasetListOrigin
-      .filter((x) => x.parser_id !== DocumentParserType.Tag)
+      .filter((x) => x.chunk_method !== DocumentParserType.Tag)
       .map((item: IKnowledge) => {
         return {
           label: item.name,
@@ -36,12 +36,12 @@ export function useDisableDifferenceEmbeddingDataset() {
           ),
           suffix: (
             <div className="text-xs px-4 p-1 bg-bg-card text-text-secondary rounded-lg border border-bg-card">
-              {item.embd_id}
+              {item.embedding_model}
             </div>
           ),
           value: item.id,
           disabled:
-            item.embd_id !== datasetSelectEmbedId &&
+            item.embedding_model !== datasetSelectEmbedId &&
             datasetSelectEmbedId !== '',
         };
       });
@@ -54,7 +54,7 @@ export function useDisableDifferenceEmbeddingDataset() {
   ) => {
     if (value.length) {
       const data = datasetListOrigin?.find((item) => item.id === value[0]);
-      setDatasetSelectEmbedId(data?.embd_id ?? '');
+      setDatasetSelectEmbedId(data?.embedding_model ?? '');
     } else {
       setDatasetSelectEmbedId('');
     }

web/src/components/ui/multi-select.tsx

@@ -242,7 +242,9 @@ export const MultiSelect = React.forwardRef<
 
     const disabledValueSet = React.useMemo(() => {
       return new Set(
-        flatOptions.filter((option) => option.disabled).map((option) => option.value),
+        flatOptions
+          .filter((option) => option.disabled)
+          .map((option) => option.value),
       );
     }, [flatOptions]);
 

web/src/hooks/use-knowledge-request.ts

@@ -18,6 +18,7 @@ import kbService, {
   listTag,
   removeTag,
   renameTag,
+  updateKb,
 } from '@/services/knowledge-service';
 import {
   useIsMutating,
@@ -137,22 +138,20 @@ export const useFetchNextKnowledgeListByPage = () => {
     ],
     initialData: {
       kbs: [],
-      total: 0,
+      total_datasets: 0,
     },
     gcTime: 0,
     queryFn: async () => {
-      const { data } = await listDataset(
-        {
+      const { data } = await listDataset({
+        page_size: pagination.pageSize,
+        page: pagination.current,
+        ext: {
           keywords: debouncedSearchString,
-          page_size: pagination.pageSize,
-          page: pagination.current,
-        },
-        {
           owner_ids: filterValue.owner,
         },
-      );
+      });
 
-      return data?.data;
+      return { kbs: data?.data, total_datasets: data?.total_datasets };
     },
   });
 
@@ -168,7 +167,7 @@ export const useFetchNextKnowledgeListByPage = () => {
     ...data,
     searchString,
     handleInputChange: onInputChange,
-    pagination: { ...pagination, total: data?.total },
+    pagination: { ...pagination, total: data?.total_datasets },
     setPagination,
     loading,
     filterValue,
@@ -184,7 +183,18 @@ export const useCreateKnowledge = () => {
     mutateAsync,
   } = useMutation({
     mutationKey: [KnowledgeApiAction.CreateKnowledge],
-    mutationFn: async (params: { id?: string; name: string }) => {
+    mutationFn: async (params: {
+      id?: string;
+      name: string;
+      embedding_model?: string;
+      chunk_method?: string;
+      parseType?: number;
+      pipeline_id?: string | null;
+      ext?: {
+        language?: string;
+        [key: string]: any;
+      };
+    }) => {
       const { data = {} } = await kbService.createKb(params);
       if (data.code === 0) {
         message.success(
@@ -208,7 +218,7 @@ export const useDeleteKnowledge = () => {
   } = useMutation({
     mutationKey: [KnowledgeApiAction.DeleteKnowledge],
     mutationFn: async (id: string) => {
-      const { data } = await kbService.rmKb({ kb_id: id });
+      const { data } = await kbService.rmKb({ ids: [id] });
       if (data.code === 0) {
         message.success(i18n.t(`message.deleted`));
         queryClient.invalidateQueries({
@@ -225,17 +235,119 @@ export const useDeleteKnowledge = () => {
 export const useUpdateKnowledge = (shouldFetchList = false) => {
   const knowledgeBaseId = useKnowledgeBaseId();
   const queryClient = useQueryClient();
+
+  const extractRaptorConfigExt = (
+    raptorConfig: Record<string, any> | undefined,
+  ) => {
+    if (!raptorConfig) return raptorConfig;
+    const {
+      use_raptor,
+      prompt,
+      max_token,
+      threshold,
+      max_cluster,
+      random_seed,
+      auto_disable_for_structured_data,
+      ext,
+      ...raptorExt
+    } = raptorConfig;
+    return {
+      use_raptor,
+      prompt,
+      max_token,
+      threshold,
+      max_cluster,
+      random_seed,
+      auto_disable_for_structured_data,
+      ext: { ...ext, ...raptorExt },
+    };
+  };
+
+  const extractParserConfigExt = (
+    parserConfig: Record<string, any> | undefined,
+  ) => {
+    if (!parserConfig) return parserConfig;
+    const {
+      auto_keywords,
+      auto_questions,
+      chunk_token_num,
+      delimiter,
+      graphrag,
+      html4excel,
+      layout_recognize,
+      raptor,
+      tag_kb_ids,
+      topn_tags,
+      filename_embd_weight,
+      task_page_size,
+      pages,
+      ext,
+      ...parserExt
+    } = parserConfig;
+    return {
+      auto_keywords,
+      auto_questions,
+      chunk_token_num,
+      delimiter,
+      graphrag,
+      html4excel,
+      layout_recognize,
+      raptor: extractRaptorConfigExt(raptor),
+      tag_kb_ids,
+      topn_tags,
+      filename_embd_weight,
+      task_page_size,
+      pages,
+      ext: { ...ext, ...parserExt },
+    };
+  };
+
   const {
     data,
     isPending: loading,
     mutateAsync,
   } = useMutation({
     mutationKey: [KnowledgeApiAction.SaveKnowledge],
-    mutationFn: async (params: Record<string, any>) => {
-      const { data = {} } = await kbService.updateKb({
-        kb_id: params?.kb_id ? params?.kb_id : knowledgeBaseId,
-        ...params,
-      });
+    mutationFn: async (params: {
+      kb_id?: string;
+      name?: string;
+      embedding_model?: string;
+      chunk_method?: string;
+      pipeline_id?: string | null;
+      avatar?: string | null;
+      description?: string;
+      permission?: string;
+      pagerank?: number;
+      parser_config?: Record<string, any>;
+      [key: string]: any;
+    }) => {
+      const kbId = params?.kb_id || knowledgeBaseId;
+      const {
+        kb_id,
+        name,
+        embedding_model,
+        chunk_method,
+        pipeline_id,
+        avatar,
+        description,
+        permission,
+        pagerank,
+        parser_config,
+        ...ext
+      } = params;
+      const requestBody: Record<string, any> = {
+        name,
+        embedding_model,
+        chunk_method,
+        pipeline_id,
+        avatar,
+        description,
+        permission,
+        pagerank,
+        parser_config: extractParserConfigExt(parser_config),
+        ext,
+      };
+      const { data = {} } = await updateKb(kbId, requestBody);
       if (data.code === 0) {
         message.success(i18n.t(`message.updated`));
         if (shouldFetchList) {
@@ -359,9 +471,9 @@ export const useFetchKnowledgeList = (
     gcTime: 0, // https://tanstack.com/query/latest/docs/framework/react/guides/caching?from=reactQueryV3
     queryFn: async () => {
       const { data } = await listDataset();
-      const list = data?.data?.kbs ?? [];
+      const list = data?.data ?? [];
       return shouldFilterListWithoutDocument
-        ? list.filter((x: IKnowledge) => x.chunk_num > 0)
+        ? list.filter((x: IKnowledge) => x.chunk_count > 0)
         : list;
     },
   });

web/src/interfaces/database/knowledge.ts

@@ -11,16 +11,16 @@ export interface IConnector {
 // knowledge base
 export interface IKnowledge {
   avatar?: any;
-  chunk_num: number;
+  chunk_count: number;
   create_date: string;
   create_time: number;
   created_by: string;
   description: string;
-  doc_num: number;
+  document_count: number;
   id: string;
   name: string;
   parser_config: ParserConfig;
-  parser_id: string;
+  chunk_method: string;
   pipeline_id: string;
   pipeline_name: string;
   pipeline_avatar: string;
@@ -32,7 +32,7 @@ export interface IKnowledge {
   update_date: string;
   update_time: number;
   vector_similarity_weight: number;
-  embd_id: string;
+  embedding_model: string;
   nickname: string;
   operator_permission: number;
   size: number;
@@ -47,7 +47,7 @@ export interface IKnowledge {
 
 export interface IKnowledgeResult {
   kbs: IKnowledge[];
-  total: number;
+  total_datasets: number;
 }
 
 export interface Raptor {

web/src/interfaces/request/knowledge.ts

@@ -24,10 +24,14 @@ export interface IFetchKnowledgeListRequestBody {
 }
 
 export interface IFetchKnowledgeListRequestParams {
-  kb_id?: string;
-  keywords?: string;
+  id?: string;
   page?: number;
   page_size?: number;
+  ext?: {
+    keywords?: string;
+    owner_ids?: string[];
+    parser_id?: string;
+  };
 }
 
 export interface IFetchDocumentListRequestBody {

web/src/pages/dataset/dataset-setting/chunk-method-form.tsx

@@ -45,7 +45,7 @@ export function ChunkMethodForm() {
 
   const finalParserId: DocumentParserType = useWatch({
     control: form.control,
-    name: 'parser_id',
+    name: 'chunk_method',
   });
 
   const ConfigurationComponent = useMemo(() => {

web/src/pages/dataset/dataset-setting/configuration/common-item.tsx

@@ -69,7 +69,7 @@ export function ChunkMethodItem(props: IProps) {
   return (
     <FormField
       control={form.control}
-      name={'parser_id'}
+      name={'chunk_method'}
       render={({ field }) => (
         <FormItem className=" items-center space-y-1">
           <div className={line === 1 ? 'flex items-center' : ''}>
@@ -121,7 +121,7 @@ export const EmbeddingSelect = ({
   const { handleChange } = useHandleKbEmbedding();
 
   const oldValue = useMemo(() => {
-    const embdStr = form.getValues(name || 'embd_id');
+    const embdStr = form.getValues(name || 'embedding_model');
     return embdStr || '';
   }, [form]);
   const [loading, setLoading] = useState(false);
@@ -165,7 +165,7 @@ export function EmbeddingModelItem({ line = 1, isEdit }: IProps) {
     <>
       <FormField
         control={form.control}
-        name={'embd_id'}
+        name={'embedding_model'}
         render={({ field }) => (
           <FormItem className={cn(' items-center space-y-0 ')}>
             <div

web/src/pages/dataset/dataset-setting/form-schema.ts

@@ -14,11 +14,11 @@ export const formSchema = z
     avatar: z.any().nullish(),
     permission: z.string().optional(),
     language: z.string().optional(),
-    parser_id: z.string(),
+    chunk_method: z.string(),
     pipeline_id: z.string().optional(),
     pipeline_name: z.string().optional(),
     pipeline_avatar: z.string().optional(),
-    embd_id: z.string(),
+    embedding_model: z.string(),
     parser_config: z
       .object({
         layout_recognize: z.string(),

web/src/pages/dataset/dataset-setting/hooks.ts

@@ -31,7 +31,7 @@ export function useHasParsedDocument(isEdit?: boolean) {
   const { data: knowledgeDetails } = useFetchKnowledgeBaseConfiguration({
     isEdit,
   });
-  return knowledgeDetails.chunk_num > 0;
+  return knowledgeDetails.chunk_count > 0;
 }
 
 export const useFetchKnowledgeConfigurationOnMount = (
@@ -60,14 +60,14 @@ export const useFetchKnowledgeConfigurationOnMount = (
         'description',
         'name',
         'permission',
-        'embd_id',
-        'parser_id',
         'language',
         'parser_config',
         'connectors',
         'pagerank',
         'avatar',
       ]),
+      embedding_model: knowledgeDetails.embd_id,
+      chunk_method: knowledgeDetails.parser_id,
     } as z.infer<typeof formSchema>;
     form.reset(formValues);
   }, [form, knowledgeDetails]);

web/src/pages/dataset/dataset-setting/index.tsx

@@ -219,7 +219,7 @@ export default function DatasetSettings() {
     defaultValue: knowledgeDetails.pipeline_id ? 2 : 1,
   });
   const selectedTag = useWatch({
-    name: 'parser_id',
+    name: 'chunk_method',
     control: form.control,
   });
   useEffect(() => {

web/src/pages/dataset/dataset-setting/saving-button.tsx

@@ -16,7 +16,7 @@ export function GeneralSavingButton() {
     () => form.formState.defaultValues ?? {},
     [form.formState.defaultValues],
   );
-  const parser_id = defaultValues['parser_id'];
+  const chunk_method = defaultValues['chunk_method'];
 
   return (
     <ButtonLoading
@@ -31,7 +31,7 @@ export function GeneralSavingButton() {
           if (isValidate) {
             saveKnowledgeConfiguration({
               kb_id,
-              parser_id,
+              chunk_method,
               name,
               description,
               avatar,

web/src/pages/dataset/dataset/generate-button/hook.ts

@@ -1,6 +1,12 @@
 import message from '@/components/ui/message';
 import agentService from '@/services/agent-service';
-import kbService, { deletePipelineTask } from '@/services/knowledge-service';
+import {
+  deletePipelineTask,
+  runGraphRag,
+  runRaptor,
+  traceGraphRag,
+  traceRaptor,
+} from '@/services/knowledge-service';
 import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query';
 import { t } from 'i18next';
 import { useEffect, useState } from 'react';
@@ -53,9 +59,7 @@ export const useTraceGenerate = ({ open }: { open: boolean }) => {
       retryDelay: 1000,
       enabled: open,
       queryFn: async () => {
-        const { data } = await kbService.traceGraphRag({
-          kb_id: id,
-        });
+        const { data } = await traceGraphRag(id);
         return data?.data || {};
       },
     });
@@ -70,9 +74,7 @@ export const useTraceGenerate = ({ open }: { open: boolean }) => {
       retryDelay: 1000,
       enabled: open,
       queryFn: async () => {
-        const { data } = await kbService.traceRaptor({
-          kb_id: id,
-        });
+        const { data } = await traceRaptor(id);
         return data?.data || {};
       },
     });
@@ -133,12 +135,8 @@ export const useDatasetGenerate = () => {
     mutationKey: [DatasetKey.generate],
     mutationFn: async ({ type }: { type: GenerateType }) => {
       const func =
-        type === GenerateType.KnowledgeGraph
-          ? kbService.runGraphRag
-          : kbService.runRaptor;
-      const { data } = await func({
-        kb_id: id,
-      });
+        type === GenerateType.KnowledgeGraph ? runGraphRag : runRaptor;
+      const { data } = await func(id);
       if (data.code === 0) {
         message.success(t('message.operated'));
         queryClient.invalidateQueries({

web/src/pages/datasets/dataset-card.tsx

@@ -23,7 +23,7 @@ export function DatasetCard({
     <HomeCard
       data={{
         ...dataset,
-        description: `${dataset.doc_num} ${t('knowledgeDetails.files')}`,
+        description: `${dataset.document_count} ${t('knowledgeDetails.files')}`,
       }}
       moreDropdown={
         <DatasetDropdown

web/src/pages/datasets/dataset-creating-dialog.tsx

@@ -46,20 +46,20 @@ export function InputForm({ onOk }: IModalProps<any>) {
         })
         .trim(),
       parseType: z.number().optional(),
-      embd_id: z
+      embedding_model: z
         .string()
         .min(1, {
           message: t('knowledgeConfiguration.embeddingModelPlaceholder'),
         })
         .trim(),
-      parser_id: z.string().optional(),
+      chunk_method: z.string().optional(),
       pipeline_id: z.string().optional(),
     })
     .superRefine((data, ctx) => {
-      // When parseType === 1, parser_id is required
+      // When parseType === 1, chunk_method is required
       if (
         data.parseType === 1 &&
-        (!data.parser_id || data.parser_id.trim() === '')
+        (!data.chunk_method || data.chunk_method.trim() === '')
       ) {
         ctx.addIssue({
           code: z.ZodIssueCode.custom,
@@ -82,8 +82,8 @@ export function InputForm({ onOk }: IModalProps<any>) {
     defaultValues: {
       name: '',
       parseType: 1,
-      parser_id: '',
-      embd_id: tenantInfo?.embd_id,
+      chunk_method: '',
+      embedding_model: tenantInfo?.embd_id,
     },
   });
 

web/src/pages/datasets/hooks.ts

@@ -16,8 +16,14 @@ export const useSearchKnowledge = () => {
 
 export interface Iknowledge {
   name: string;
-  embd_id: string;
-  parser_id: string;
+  embedding_model?: string;
+  chunk_method?: string;
+  parseType?: number;
+  pipeline_id?: string | null;
+  ext?: {
+    language?: string;
+    [key: string]: any;
+  };
 }
 export const useSaveKnowledge = () => {
   const { visible: visible, hideModal, showModal } = useSetModalState();
@@ -30,7 +36,7 @@ export const useSaveKnowledge = () => {
 
       if (ret?.code === 0) {
         hideModal();
-        navigateToDataset(ret.data.kb_id)();
+        navigateToDataset(ret.data.id)();
       }
     },
     [createKnowledge, hideModal, navigateToDataset],

web/src/pages/datasets/index.tsx

@@ -30,7 +30,7 @@ export default function Datasets() {
 
   const {
     kbs,
-    total,
+    total_datasets,
     pagination,
     setPagination,
     handleInputChange,
@@ -107,7 +107,7 @@ export default function Datasets() {
               <footer className="mt-4 px-5 pb-5">
                 <RAGFlowPagination
                   {...pick(pagination, 'current', 'pageSize')}
-                  total={total}
+                  total={total_datasets}
                   onChange={handlePageChange}
                 />
               </footer>

web/src/services/knowledge-service.ts

@@ -1,7 +1,6 @@
 import { IRenameTag } from '@/interfaces/database/knowledge';
 import {
   IFetchDocumentListRequestBody,
-  IFetchKnowledgeListRequestBody,
   IFetchKnowledgeListRequestParams,
 } from '@/interfaces/request/knowledge';
 import { ProcessingType } from '@/pages/dataset/dataset-overview/dataset-common';
@@ -11,7 +10,6 @@ import request, { post } from '@/utils/request';
 
 const {
   create_kb,
-  update_kb,
   rm_kb,
   get_kb_detail,
   kb_list,
@@ -42,10 +40,6 @@ const {
   getKnowledgeBasicInfo,
   fetchDataPipelineLog,
   fetchPipelineDatasetLogs,
-  runGraphRag,
-  traceGraphRag,
-  runRaptor,
-  traceRaptor,
   check_embedding,
   kbUpdateMetaData,
   documentUpdateMetaData,
@@ -56,13 +50,9 @@ const methods = {
     url: create_kb,
     method: 'post',
   },
-  updateKb: {
-    url: update_kb,
-    method: 'post',
-  },
   rmKb: {
     url: rm_kb,
-    method: 'post',
+    method: 'delete',
   },
   get_kb_detail: {
     url: get_kb_detail,
@@ -70,7 +60,7 @@ const methods = {
   },
   getList: {
     url: kb_list,
-    method: 'post',
+    method: 'get',
   },
   // document manager
   get_document_list: {
@@ -191,22 +181,6 @@ const methods = {
     method: 'get',
   },
 
-  runGraphRag: {
-    url: runGraphRag,
-    method: 'post',
-  },
-  traceGraphRag: {
-    url: traceGraphRag,
-    method: 'get',
-  },
-  runRaptor: {
-    url: runRaptor,
-    method: 'post',
-  },
-  traceRaptor: {
-    url: traceRaptor,
-    method: 'get',
-  },
   pipelineRerun: {
     url: api.pipelineRerun,
     method: 'post',
@@ -251,10 +225,23 @@ export function deleteKnowledgeGraph(knowledgeId: string) {
   return request.delete(api.getKnowledgeGraph(knowledgeId));
 }
 
-export const listDataset = (
-  params?: IFetchKnowledgeListRequestParams,
-  body?: IFetchKnowledgeListRequestBody,
-) => request.post(api.kb_list, { data: body || {}, params });
+export const listDataset = (params?: IFetchKnowledgeListRequestParams) =>
+  request.get(api.kb_list, { params });
+
+export const updateKb = (datasetId: string, data: Record<string, any>) =>
+  request.put(api.update_kb(datasetId), { data });
+
+export const runGraphRag = (datasetId: string) =>
+  request.post(api.runGraphRag(datasetId));
+
+export const traceGraphRag = (datasetId: string) =>
+  request.get(api.traceGraphRag(datasetId));
+
+export const runRaptor = (datasetId: string) =>
+  request.post(api.runRaptor(datasetId));
+
+export const traceRaptor = (datasetId: string) =>
+  request.get(api.traceRaptor(datasetId));
 
 export const listDocument = (
   params?: IFetchKnowledgeListRequestParams,

web/src/utils/api.ts

@@ -57,23 +57,30 @@ export default {
   // knowledge base
 
   check_embedding: `${api_host}/kb/check_embedding`,
-  kb_list: `${api_host}/kb/list`,
-  create_kb: `${api_host}/kb/create`,
-  update_kb: `${api_host}/kb/update`,
-  rm_kb: `${api_host}/kb/rm`,
+  kb_list: `${ExternalApi}${api_host}/datasets`,
+  create_kb: `${ExternalApi}${api_host}/datasets`,
+  update_kb: (datasetId: string) =>
+    `${ExternalApi}${api_host}/datasets/${datasetId}`,
+  rm_kb: `${ExternalApi}${api_host}/datasets`,
   get_kb_detail: `${api_host}/kb/detail`,
   getKnowledgeGraph: (knowledgeId: string) =>
-    `${api_host}/kb/${knowledgeId}/knowledge_graph`,
+    `${ExternalApi}${api_host}/datasets/${knowledgeId}/knowledge_graph`,
+  deleteKnowledgeGraph: (knowledgeId: string) =>
+    `${ExternalApi}${api_host}/datasets/${knowledgeId}/knowledge_graph`,
   getMeta: `${api_host}/kb/get_meta`,
   getKnowledgeBasicInfo: `${api_host}/kb/basic_info`,
   // data pipeline log
   fetchDataPipelineLog: `${api_host}/kb/list_pipeline_logs`,
   get_pipeline_detail: `${api_host}/kb/pipeline_log_detail`,
   fetchPipelineDatasetLogs: `${api_host}/kb/list_pipeline_dataset_logs`,
-  runGraphRag: `${api_host}/kb/run_graphrag`,
-  traceGraphRag: `${api_host}/kb/trace_graphrag`,
-  runRaptor: `${api_host}/kb/run_raptor`,
-  traceRaptor: `${api_host}/kb/trace_raptor`,
+  runGraphRag: (datasetId: string) =>
+    `${ExternalApi}${api_host}/datasets/${datasetId}/run_graphrag`,
+  traceGraphRag: (datasetId: string) =>
+    `${ExternalApi}${api_host}/datasets/${datasetId}/trace_graphrag`,
+  runRaptor: (datasetId: string) =>
+    `${ExternalApi}${api_host}/datasets/${datasetId}/run_raptor`,
+  traceRaptor: (datasetId: string) =>
+    `${ExternalApi}${api_host}/datasets/${datasetId}/trace_raptor`,
   unbindPipelineTask: ({ kb_id, type }: { kb_id: string; type: string }) =>
     `${api_host}/kb/unbind_task?kb_id=${kb_id}&pipeline_task_type=${type}`,
   pipelineRerun: `${api_host}/canvas/rerun`,