diff --git a/api/apps/evaluation_app.py b/api/apps/evaluation_app.py deleted file mode 100644 index b33db26da..000000000 --- a/api/apps/evaluation_app.py +++ /dev/null @@ -1,479 +0,0 @@ -# -# Copyright 2025 The InfiniFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -""" -RAG Evaluation API Endpoints - -Provides REST API for RAG evaluation functionality including: -- Dataset management -- Test case management -- Evaluation execution -- Results retrieval -- Configuration recommendations -""" - -from quart import request -from api.apps import login_required, current_user -from api.db.services.evaluation_service import EvaluationService -from api.utils.api_utils import ( - get_data_error_result, - get_json_result, - get_request_json, - server_error_response, - validate_request -) -from common.constants import RetCode - - -# ==================== Dataset Management ==================== - -@manager.route('/dataset/create', methods=['POST']) # noqa: F821 -@login_required -@validate_request("name", "kb_ids") -async def create_dataset(): - """ - Create a new evaluation dataset. - - Request body: - { - "name": "Dataset name", - "description": "Optional description", - "kb_ids": ["kb_id1", "kb_id2"] - } - """ - try: - req = await get_request_json() - name = req.get("name", "").strip() - description = req.get("description", "") - kb_ids = req.get("kb_ids", []) - - if not name: - return get_data_error_result(message="Dataset name cannot be empty") - - if not kb_ids or not isinstance(kb_ids, list): - return get_data_error_result(message="kb_ids must be a non-empty list") - - success, result = EvaluationService.create_dataset( - name=name, - description=description, - kb_ids=kb_ids, - tenant_id=current_user.id, - user_id=current_user.id - ) - - if not success: - return get_data_error_result(message=result) - - return get_json_result(data={"dataset_id": result}) - except Exception as e: - return server_error_response(e) - - -@manager.route('/dataset/list', methods=['GET']) # noqa: F821 -@login_required -async def list_datasets(): - """ - List evaluation datasets for current tenant. - - Query params: - - page: Page number (default: 1) - - page_size: Items per page (default: 20) - """ - try: - page = int(request.args.get("page", 1)) - page_size = int(request.args.get("page_size", 20)) - - result = EvaluationService.list_datasets( - tenant_id=current_user.id, - user_id=current_user.id, - page=page, - page_size=page_size - ) - - return get_json_result(data=result) - except Exception as e: - return server_error_response(e) - - -@manager.route('/dataset/', methods=['GET']) # noqa: F821 -@login_required -async def get_dataset(dataset_id): - """Get dataset details by ID""" - try: - dataset = EvaluationService.get_dataset(dataset_id) - if not dataset: - return get_data_error_result( - message="Dataset not found", - code=RetCode.DATA_ERROR - ) - - return get_json_result(data=dataset) - except Exception as e: - return server_error_response(e) - - -@manager.route('/dataset/', methods=['PUT']) # noqa: F821 -@login_required -async def update_dataset(dataset_id): - """ - Update dataset. - - Request body: - { - "name": "New name", - "description": "New description", - "kb_ids": ["kb_id1", "kb_id2"] - } - """ - try: - req = await get_request_json() - - # Remove fields that shouldn't be updated - req.pop("id", None) - req.pop("tenant_id", None) - req.pop("created_by", None) - req.pop("create_time", None) - - success = EvaluationService.update_dataset(dataset_id, **req) - - if not success: - return get_data_error_result(message="Failed to update dataset") - - return get_json_result(data={"dataset_id": dataset_id}) - except Exception as e: - return server_error_response(e) - - -@manager.route('/dataset/', methods=['DELETE']) # noqa: F821 -@login_required -async def delete_dataset(dataset_id): - """Delete dataset (soft delete)""" - try: - success = EvaluationService.delete_dataset(dataset_id) - - if not success: - return get_data_error_result(message="Failed to delete dataset") - - return get_json_result(data={"dataset_id": dataset_id}) - except Exception as e: - return server_error_response(e) - - -# ==================== Test Case Management ==================== - -@manager.route('/dataset//case/add', methods=['POST']) # noqa: F821 -@login_required -@validate_request("question") -async def add_test_case(dataset_id): - """ - Add a test case to a dataset. - - Request body: - { - "question": "Test question", - "reference_answer": "Optional ground truth answer", - "relevant_doc_ids": ["doc_id1", "doc_id2"], - "relevant_chunk_ids": ["chunk_id1", "chunk_id2"], - "metadata": {"key": "value"} - } - """ - try: - req = await get_request_json() - question = req.get("question", "").strip() - - if not question: - return get_data_error_result(message="Question cannot be empty") - - success, result = EvaluationService.add_test_case( - dataset_id=dataset_id, - question=question, - reference_answer=req.get("reference_answer"), - relevant_doc_ids=req.get("relevant_doc_ids"), - relevant_chunk_ids=req.get("relevant_chunk_ids"), - metadata=req.get("metadata") - ) - - if not success: - return get_data_error_result(message=result) - - return get_json_result(data={"case_id": result}) - except Exception as e: - return server_error_response(e) - - -@manager.route('/dataset//case/import', methods=['POST']) # noqa: F821 -@login_required -@validate_request("cases") -async def import_test_cases(dataset_id): - """ - Bulk import test cases. - - Request body: - { - "cases": [ - { - "question": "Question 1", - "reference_answer": "Answer 1", - ... - }, - { - "question": "Question 2", - ... - } - ] - } - """ - try: - req = await get_request_json() - cases = req.get("cases", []) - - if not cases or not isinstance(cases, list): - return get_data_error_result(message="cases must be a non-empty list") - - success_count, failure_count = EvaluationService.import_test_cases( - dataset_id=dataset_id, - cases=cases - ) - - return get_json_result(data={ - "success_count": success_count, - "failure_count": failure_count, - "total": len(cases) - }) - except Exception as e: - return server_error_response(e) - - -@manager.route('/dataset//cases', methods=['GET']) # noqa: F821 -@login_required -async def get_test_cases(dataset_id): - """Get all test cases for a dataset""" - try: - cases = EvaluationService.get_test_cases(dataset_id) - return get_json_result(data={"cases": cases, "total": len(cases)}) - except Exception as e: - return server_error_response(e) - - -@manager.route('/case/', methods=['DELETE']) # noqa: F821 -@login_required -async def delete_test_case(case_id): - """Delete a test case""" - try: - success = EvaluationService.delete_test_case(case_id) - - if not success: - return get_data_error_result(message="Failed to delete test case") - - return get_json_result(data={"case_id": case_id}) - except Exception as e: - return server_error_response(e) - - -# ==================== Evaluation Execution ==================== - -@manager.route('/run/start', methods=['POST']) # noqa: F821 -@login_required -@validate_request("dataset_id", "dialog_id") -async def start_evaluation(): - """ - Start an evaluation run. - - Request body: - { - "dataset_id": "dataset_id", - "dialog_id": "dialog_id", - "name": "Optional run name" - } - """ - try: - req = await get_request_json() - dataset_id = req.get("dataset_id") - dialog_id = req.get("dialog_id") - name = req.get("name") - - success, result = EvaluationService.start_evaluation( - dataset_id=dataset_id, - dialog_id=dialog_id, - user_id=current_user.id, - name=name - ) - - if not success: - return get_data_error_result(message=result) - - return get_json_result(data={"run_id": result}) - except Exception as e: - return server_error_response(e) - - -@manager.route('/run/', methods=['GET']) # noqa: F821 -@login_required -async def get_evaluation_run(run_id): - """Get evaluation run details""" - try: - result = EvaluationService.get_run_results(run_id) - - if not result: - return get_data_error_result( - message="Evaluation run not found", - code=RetCode.DATA_ERROR - ) - - return get_json_result(data=result) - except Exception as e: - return server_error_response(e) - - -@manager.route('/run//results', methods=['GET']) # noqa: F821 -@login_required -async def get_run_results(run_id): - """Get detailed results for an evaluation run""" - try: - result = EvaluationService.get_run_results(run_id) - - if not result: - return get_data_error_result( - message="Evaluation run not found", - code=RetCode.DATA_ERROR - ) - - return get_json_result(data=result) - except Exception as e: - return server_error_response(e) - - -@manager.route('/run/list', methods=['GET']) # noqa: F821 -@login_required -async def list_evaluation_runs(): - """ - List evaluation runs. - - Query params: - - dataset_id: Filter by dataset (optional) - - dialog_id: Filter by dialog (optional) - - page: Page number (default: 1) - - page_size: Items per page (default: 20) - """ - try: - # TODO: Implement list_runs in EvaluationService - return get_json_result(data={"runs": [], "total": 0}) - except Exception as e: - return server_error_response(e) - - -@manager.route('/run/', methods=['DELETE']) # noqa: F821 -@login_required -async def delete_evaluation_run(run_id): - """Delete an evaluation run""" - try: - # TODO: Implement delete_run in EvaluationService - return get_json_result(data={"run_id": run_id}) - except Exception as e: - return server_error_response(e) - - -# ==================== Analysis & Recommendations ==================== - -@manager.route('/run//recommendations', methods=['GET']) # noqa: F821 -@login_required -async def get_recommendations(run_id): - """Get configuration recommendations based on evaluation results""" - try: - recommendations = EvaluationService.get_recommendations(run_id) - return get_json_result(data={"recommendations": recommendations}) - except Exception as e: - return server_error_response(e) - - -@manager.route('/compare', methods=['POST']) # noqa: F821 -@login_required -@validate_request("run_ids") -async def compare_runs(): - """ - Compare multiple evaluation runs. - - Request body: - { - "run_ids": ["run_id1", "run_id2", "run_id3"] - } - """ - try: - req = await get_request_json() - run_ids = req.get("run_ids", []) - - if not run_ids or not isinstance(run_ids, list) or len(run_ids) < 2: - return get_data_error_result( - message="run_ids must be a list with at least 2 run IDs" - ) - - # TODO: Implement compare_runs in EvaluationService - return get_json_result(data={"comparison": {}}) - except Exception as e: - return server_error_response(e) - - -@manager.route('/run//export', methods=['GET']) # noqa: F821 -@login_required -async def export_results(run_id): - """Export evaluation results as JSON/CSV""" - try: - # format_type = request.args.get("format", "json") # TODO: Use for CSV export - - result = EvaluationService.get_run_results(run_id) - - if not result: - return get_data_error_result( - message="Evaluation run not found", - code=RetCode.DATA_ERROR - ) - - # TODO: Implement CSV export - return get_json_result(data=result) - except Exception as e: - return server_error_response(e) - - -# ==================== Real-time Evaluation ==================== - -@manager.route('/evaluate_single', methods=['POST']) # noqa: F821 -@login_required -@validate_request("question", "dialog_id") -async def evaluate_single(): - """ - Evaluate a single question-answer pair in real-time. - - Request body: - { - "question": "Test question", - "dialog_id": "dialog_id", - "reference_answer": "Optional ground truth", - "relevant_chunk_ids": ["chunk_id1", "chunk_id2"] - } - """ - try: - # req = await get_request_json() # TODO: Use for single evaluation implementation - - # TODO: Implement single evaluation - # This would execute the RAG pipeline and return metrics immediately - - return get_json_result(data={ - "answer": "", - "metrics": {}, - "retrieved_chunks": [] - }) - except Exception as e: - return server_error_response(e) diff --git a/api/apps/kb_app.py b/api/apps/kb_app.py deleted file mode 100644 index b8551c2a9..000000000 --- a/api/apps/kb_app.py +++ /dev/null @@ -1,446 +0,0 @@ -# -# Copyright 2024 The InfiniFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -""" -Deprecated, todo delete -@manager.route('/create', methods=['post']) # noqa: F821 -@login_required -@validate_request("name") -async def create(): - req = await get_request_json() - create_dict = ensure_tenant_model_id_for_params(current_user.id, req) - e, res = KnowledgebaseService.create_with_name( - name = create_dict.pop("name", None), - tenant_id = current_user.id, - parser_id = create_dict.pop("parser_id", None), - **create_dict - ) - - if not e: - return res - - try: - if not KnowledgebaseService.save(**res): - return get_data_error_result() - return get_json_result(data={"kb_id":res["id"]}) - except Exception as e: - return server_error_response(e) - - -@manager.route('/update', methods=['post']) # noqa: F821 -@login_required -@validate_request("kb_id", "name", "description", "parser_id") -@not_allowed_parameters("id", "tenant_id", "created_by", "create_time", "update_time", "create_date", "update_date", "created_by") -async def update(): - req = await get_request_json() - update_dict = ensure_tenant_model_id_for_params(current_user.id, req) - if not isinstance(update_dict["name"], str): - return get_data_error_result(message="Dataset name must be string.") - if update_dict["name"].strip() == "": - return get_data_error_result(message="Dataset name can't be empty.") - if len(update_dict["name"].encode("utf-8")) > DATASET_NAME_LIMIT: - return get_data_error_result( - message=f"Dataset name length is {len(update_dict['name'])} which is large than {DATASET_NAME_LIMIT}") - update_dict["name"] = update_dict["name"].strip() - if settings.DOC_ENGINE_INFINITY: - parser_id = update_dict.get("parser_id") - if isinstance(parser_id, str) and parser_id.lower() == "tag": - return get_json_result( - code=RetCode.OPERATING_ERROR, - message="The chunking method Tag has not been supported by Infinity yet.", - data=False, - ) - if "pagerank" in update_dict and update_dict["pagerank"] > 0: - return get_json_result( - code=RetCode.DATA_ERROR, - message="'pagerank' can only be set when doc_engine is elasticsearch", - data=False, - ) - - if not KnowledgebaseService.accessible4deletion(update_dict["kb_id"], current_user.id): - return get_json_result( - data=False, - message='No authorization.', - code=RetCode.AUTHENTICATION_ERROR - ) - try: - if not KnowledgebaseService.query( - created_by=current_user.id, id=update_dict["kb_id"]): - return get_json_result( - data=False, message='Only owner of dataset authorized for this operation.', - code=RetCode.OPERATING_ERROR) - - e, kb = KnowledgebaseService.get_by_id(update_dict["kb_id"]) - - # Rename folder in FileService - if e and update_dict["name"].lower() != kb.name.lower(): - FileService.filter_update( - [ - File.tenant_id == kb.tenant_id, - File.source_type == FileSource.KNOWLEDGEBASE, - File.type == "folder", - File.name == kb.name, - ], - {"name": update_dict["name"]}, - ) - - if not e: - return get_data_error_result( - message="Can't find this dataset!") - - if update_dict["name"].lower() != kb.name.lower() \ - and len( - KnowledgebaseService.query(name=update_dict["name"], tenant_id=current_user.id, status=StatusEnum.VALID.value)) >= 1: - return get_data_error_result( - message="Duplicated dataset name.") - - del update_dict["kb_id"] - connectors = [] - if "connectors" in update_dict: - connectors = update_dict["connectors"] - del update_dict["connectors"] - if not KnowledgebaseService.update_by_id(kb.id, update_dict): - return get_data_error_result() - - if kb.pagerank != update_dict.get("pagerank", 0): - if update_dict.get("pagerank", 0) > 0: - await thread_pool_exec( - settings.docStoreConn.update, - {"kb_id": kb.id}, - {PAGERANK_FLD: update_dict["pagerank"]}, - search.index_name(kb.tenant_id), - kb.id, - ) - else: - # Elasticsearch requires PAGERANK_FLD be non-zero! - await thread_pool_exec( - settings.docStoreConn.update, - {"exists": PAGERANK_FLD}, - {"remove": PAGERANK_FLD}, - search.index_name(kb.tenant_id), - kb.id, - ) - - e, kb = KnowledgebaseService.get_by_id(kb.id) - if not e: - return get_data_error_result( - message="Database error (Knowledgebase rename)!") - errors = Connector2KbService.link_connectors(kb.id, [conn for conn in connectors], current_user.id) - if errors: - logging.error("Link KB errors: ", errors) - kb = kb.to_dict() - kb.update(update_dict) - kb["connectors"] = connectors - - return get_json_result(data=kb) - except Exception as e: - return server_error_response(e) -""" - -""" -Deprecated, todo delete -@manager.route('/list', methods=['POST']) # noqa: F821 -@login_required -async def list_kbs(): - args = request.args - keywords = args.get("keywords", "") - page_number = int(args.get("page", 0)) - items_per_page = int(args.get("page_size", 0)) - parser_id = args.get("parser_id") - orderby = args.get("orderby", "create_time") - if args.get("desc", "true").lower() == "false": - desc = False - else: - desc = True - - req = await get_request_json() - owner_ids = req.get("owner_ids", []) - try: - if not owner_ids: - tenants = TenantService.get_joined_tenants_by_user_id(current_user.id) - tenants = [m["tenant_id"] for m in tenants] - kbs, total = KnowledgebaseService.get_by_tenant_ids( - tenants, current_user.id, page_number, - items_per_page, orderby, desc, keywords, parser_id) - else: - tenants = owner_ids - kbs, total = KnowledgebaseService.get_by_tenant_ids( - tenants, current_user.id, 0, - 0, orderby, desc, keywords, parser_id) - kbs = [kb for kb in kbs if kb["tenant_id"] in tenants] - total = len(kbs) - if page_number and items_per_page: - kbs = kbs[(page_number-1)*items_per_page:page_number*items_per_page] - return get_json_result(data={"kbs": kbs, "total": total}) - except Exception as e: - return server_error_response(e) - - -@manager.route('/rm', methods=['post']) # noqa: F821 -@login_required -@validate_request("kb_id") -async def rm(): - req = await get_request_json() - uid = current_user.id - if not KnowledgebaseService.accessible4deletion(req["kb_id"], uid): - return get_json_result( - data=False, - message='No authorization.', - code=RetCode.AUTHENTICATION_ERROR - ) - try: - kbs = KnowledgebaseService.query( - created_by=uid, id=req["kb_id"]) - if not kbs: - return get_json_result( - data=False, message='Only owner of dataset authorized for this operation.', - code=RetCode.OPERATING_ERROR) - - def _rm_sync(): - for doc in DocumentService.query(kb_id=req["kb_id"]): - if not DocumentService.remove_document(doc, kbs[0].tenant_id): - return get_data_error_result( - message="Database error (Document removal)!") - f2d = File2DocumentService.get_by_document_id(doc.id) - if f2d: - FileService.filter_delete([File.source_type == FileSource.KNOWLEDGEBASE, File.id == f2d[0].file_id]) - File2DocumentService.delete_by_document_id(doc.id) - FileService.filter_delete( - [ - File.tenant_id == kbs[0].tenant_id, - File.source_type == FileSource.KNOWLEDGEBASE, - File.type == "folder", - File.name == kbs[0].name, - ] - ) - # Delete the table BEFORE deleting the database record - for kb in kbs: - try: - settings.docStoreConn.delete({"kb_id": kb.id}, search.index_name(kb.tenant_id), kb.id) - settings.docStoreConn.delete_idx(search.index_name(kb.tenant_id), kb.id) - logging.info(f"Dropped index for dataset {kb.id}") - except Exception as e: - logging.error(f"Failed to drop index for dataset {kb.id}: {e}") - - if not KnowledgebaseService.delete_by_id(req["kb_id"]): - return get_data_error_result( - message="Database error (Knowledgebase removal)!") - for kb in kbs: - if hasattr(settings.STORAGE_IMPL, 'remove_bucket'): - settings.STORAGE_IMPL.remove_bucket(kb.id) - return get_json_result(data=True) - - return await thread_pool_exec(_rm_sync) - except Exception as e: - return server_error_response(e) -""" - -""" -Deprecated, todo delete -@manager.route('//knowledge_graph', methods=['GET']) # noqa: F821 -@login_required -async def knowledge_graph(kb_id): - if not KnowledgebaseService.accessible(kb_id, current_user.id): - return get_json_result( - data=False, - message='No authorization.', - code=RetCode.AUTHENTICATION_ERROR - ) - _, kb = KnowledgebaseService.get_by_id(kb_id) - req = { - "kb_id": [kb_id], - "knowledge_graph_kwd": ["graph"] - } - - obj = {"graph": {}, "mind_map": {}} - if not settings.docStoreConn.index_exist(search.index_name(kb.tenant_id), kb_id): - return get_json_result(data=obj) - sres = await settings.retriever.search(req, search.index_name(kb.tenant_id), [kb_id]) - if not len(sres.ids): - return get_json_result(data=obj) - - for id in sres.ids[:1]: - ty = sres.field[id]["knowledge_graph_kwd"] - try: - content_json = json.loads(sres.field[id]["content_with_weight"]) - except Exception: - continue - - obj[ty] = content_json - - if "nodes" in obj["graph"]: - obj["graph"]["nodes"] = sorted(obj["graph"]["nodes"], key=lambda x: x.get("pagerank", 0), reverse=True)[:256] - if "edges" in obj["graph"]: - node_id_set = { o["id"] for o in obj["graph"]["nodes"] } - filtered_edges = [o for o in obj["graph"]["edges"] if o["source"] != o["target"] and o["source"] in node_id_set and o["target"] in node_id_set] - obj["graph"]["edges"] = sorted(filtered_edges, key=lambda x: x.get("weight", 0), reverse=True)[:128] - return get_json_result(data=obj) - - -@manager.route('//knowledge_graph', methods=['DELETE']) # noqa: F821 -@login_required -def delete_knowledge_graph(kb_id): - if not KnowledgebaseService.accessible(kb_id, current_user.id): - return get_json_result( - data=False, - message='No authorization.', - code=RetCode.AUTHENTICATION_ERROR - ) - _, kb = KnowledgebaseService.get_by_id(kb_id) - settings.docStoreConn.delete({"knowledge_graph_kwd": ["graph", "subgraph", "entity", "relation"]}, search.index_name(kb.tenant_id), kb_id) - - return get_json_result(data=True) -""" - -""" -Deprecated, todo delete -@manager.route("/run_graphrag", methods=["POST"]) # noqa: F821 -@login_required -async def run_graphrag(): - req = await get_request_json() - - kb_id = req.get("kb_id", "") - if not kb_id: - return get_error_data_result(message='Lack of "KB ID"') - - ok, kb = KnowledgebaseService.get_by_id(kb_id) - if not ok: - return get_error_data_result(message="Invalid Knowledgebase ID") - - task_id = kb.graphrag_task_id - if task_id: - ok, task = TaskService.get_by_id(task_id) - if not ok: - logging.warning(f"A valid GraphRAG task id is expected for kb {kb_id}") - - if task and task.progress not in [-1, 1]: - return get_error_data_result(message=f"Task {task_id} in progress with status {task.progress}. A Graph Task is already running.") - - documents, _ = DocumentService.get_by_kb_id( - kb_id=kb_id, - page_number=0, - items_per_page=0, - orderby="create_time", - desc=False, - keywords="", - run_status=[], - types=[], - suffix=[], - ) - if not documents: - return get_error_data_result(message=f"No documents in Knowledgebase {kb_id}") - - sample_document = documents[0] - document_ids = [document["id"] for document in documents] - - task_id = queue_raptor_o_graphrag_tasks(sample_doc_id=sample_document, ty="graphrag", priority=0, fake_doc_id=GRAPH_RAPTOR_FAKE_DOC_ID, doc_ids=list(document_ids)) - - if not KnowledgebaseService.update_by_id(kb.id, {"graphrag_task_id": task_id}): - logging.warning(f"Cannot save graphrag_task_id for kb {kb_id}") - - return get_json_result(data={"graphrag_task_id": task_id}) - - -@manager.route("/trace_graphrag", methods=["GET"]) # noqa: F821 -@login_required -def trace_graphrag(): - kb_id = request.args.get("kb_id", "") - if not kb_id: - return get_error_data_result(message='Lack of "KB ID"') - - ok, kb = KnowledgebaseService.get_by_id(kb_id) - if not ok: - return get_error_data_result(message="Invalid Knowledgebase ID") - - task_id = kb.graphrag_task_id - if not task_id: - return get_json_result(data={}) - - ok, task = TaskService.get_by_id(task_id) - if not ok: - return get_json_result(data={}) - - return get_json_result(data=task.to_dict()) - - -@manager.route("/run_raptor", methods=["POST"]) # noqa: F821 -@login_required -async def run_raptor(): - req = await get_request_json() - - kb_id = req.get("kb_id", "") - if not kb_id: - return get_error_data_result(message='Lack of "KB ID"') - - ok, kb = KnowledgebaseService.get_by_id(kb_id) - if not ok: - return get_error_data_result(message="Invalid Knowledgebase ID") - - task_id = kb.raptor_task_id - if task_id: - ok, task = TaskService.get_by_id(task_id) - if not ok: - logging.warning(f"A valid RAPTOR task id is expected for kb {kb_id}") - - if task and task.progress not in [-1, 1]: - return get_error_data_result(message=f"Task {task_id} in progress with status {task.progress}. A RAPTOR Task is already running.") - - documents, _ = DocumentService.get_by_kb_id( - kb_id=kb_id, - page_number=0, - items_per_page=0, - orderby="create_time", - desc=False, - keywords="", - run_status=[], - types=[], - suffix=[], - ) - if not documents: - return get_error_data_result(message=f"No documents in Knowledgebase {kb_id}") - - sample_document = documents[0] - document_ids = [document["id"] for document in documents] - - task_id = queue_raptor_o_graphrag_tasks(sample_doc_id=sample_document, ty="raptor", priority=0, fake_doc_id=GRAPH_RAPTOR_FAKE_DOC_ID, doc_ids=list(document_ids)) - - if not KnowledgebaseService.update_by_id(kb.id, {"raptor_task_id": task_id}): - logging.warning(f"Cannot save raptor_task_id for kb {kb_id}") - - return get_json_result(data={"raptor_task_id": task_id}) - - -@manager.route("/trace_raptor", methods=["GET"]) # noqa: F821 -@login_required -def trace_raptor(): - kb_id = request.args.get("kb_id", "") - if not kb_id: - return get_error_data_result(message='Lack of "KB ID"') - - ok, kb = KnowledgebaseService.get_by_id(kb_id) - if not ok: - return get_error_data_result(message="Invalid Knowledgebase ID") - - task_id = kb.raptor_task_id - if not task_id: - return get_json_result(data={}) - - ok, task = TaskService.get_by_id(task_id) - if not ok: - return get_error_data_result(message="RAPTOR Task Not Found or Error Occurred") - - return get_json_result(data=task.to_dict()) -""" diff --git a/test/testcases/test_web_api/test_evaluation_app/test_evaluation_routes_unit.py b/test/testcases/test_web_api/test_evaluation_app/test_evaluation_routes_unit.py deleted file mode 100644 index 938d82d3d..000000000 --- a/test/testcases/test_web_api/test_evaluation_app/test_evaluation_routes_unit.py +++ /dev/null @@ -1,575 +0,0 @@ -# -# Copyright 2026 The InfiniFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import asyncio -import importlib.util -import sys -from pathlib import Path -from types import ModuleType, SimpleNamespace - -import pytest - - -class _DummyManager: - def route(self, *_args, **_kwargs): - def decorator(func): - return func - - return decorator - - -class _Args(dict): - def get(self, key, default=None): - return super().get(key, default) - - -class _DummyRetCode: - SUCCESS = 0 - EXCEPTION_ERROR = 100 - ARGUMENT_ERROR = 101 - DATA_ERROR = 102 - OPERATING_ERROR = 103 - AUTHENTICATION_ERROR = 109 - - -def _run(coro): - return asyncio.run(coro) - - -def _set_request_json(monkeypatch, module, payload): - async def _request_json(): - return payload - - monkeypatch.setattr(module, "get_request_json", _request_json) - - -def _set_request_args(monkeypatch, module, args=None): - monkeypatch.setattr(module, "request", SimpleNamespace(args=_Args(args or {}))) - - -@pytest.fixture(scope="session") -def auth(): - return "unit-auth" - - -@pytest.fixture(scope="session", autouse=True) -def set_tenant_info(): - return None - - -def _load_evaluation_app(monkeypatch): - repo_root = Path(__file__).resolve().parents[4] - - quart_mod = ModuleType("quart") - quart_mod.request = SimpleNamespace(args=_Args()) - monkeypatch.setitem(sys.modules, "quart", quart_mod) - - common_pkg = ModuleType("common") - common_pkg.__path__ = [str(repo_root / "common")] - monkeypatch.setitem(sys.modules, "common", common_pkg) - - constants_mod = ModuleType("common.constants") - constants_mod.RetCode = _DummyRetCode - monkeypatch.setitem(sys.modules, "common.constants", constants_mod) - common_pkg.constants = constants_mod - - api_pkg = ModuleType("api") - api_pkg.__path__ = [str(repo_root / "api")] - monkeypatch.setitem(sys.modules, "api", api_pkg) - - apps_mod = ModuleType("api.apps") - apps_mod.__path__ = [str(repo_root / "api" / "apps")] - apps_mod.current_user = SimpleNamespace(id="tenant-1") - apps_mod.login_required = lambda func: func - monkeypatch.setitem(sys.modules, "api.apps", apps_mod) - api_pkg.apps = apps_mod - - db_pkg = ModuleType("api.db") - db_pkg.__path__ = [] - monkeypatch.setitem(sys.modules, "api.db", db_pkg) - api_pkg.db = db_pkg - - services_pkg = ModuleType("api.db.services") - services_pkg.__path__ = [] - monkeypatch.setitem(sys.modules, "api.db.services", services_pkg) - - evaluation_service_mod = ModuleType("api.db.services.evaluation_service") - - class _EvaluationService: - @staticmethod - def create_dataset(**_kwargs): - return True, "dataset-1" - - @staticmethod - def list_datasets(**_kwargs): - return {"datasets": [], "total": 0} - - @staticmethod - def get_dataset(_dataset_id): - return {"id": _dataset_id} - - @staticmethod - def update_dataset(_dataset_id, **_kwargs): - return True - - @staticmethod - def delete_dataset(_dataset_id): - return True - - @staticmethod - def add_test_case(**_kwargs): - return True, "case-1" - - @staticmethod - def import_test_cases(**_kwargs): - return 0, 0 - - @staticmethod - def get_test_cases(_dataset_id): - return [] - - @staticmethod - def delete_test_case(_case_id): - return True - - @staticmethod - def start_evaluation(**_kwargs): - return True, "run-1" - - @staticmethod - def get_run_results(_run_id): - return {"id": _run_id} - - @staticmethod - def get_recommendations(_run_id): - return [] - - evaluation_service_mod.EvaluationService = _EvaluationService - monkeypatch.setitem(sys.modules, "api.db.services.evaluation_service", evaluation_service_mod) - - utils_pkg = ModuleType("api.utils") - utils_pkg.__path__ = [] - monkeypatch.setitem(sys.modules, "api.utils", utils_pkg) - - api_utils_mod = ModuleType("api.utils.api_utils") - - async def _default_request_json(): - return {} - - def _get_data_error_result(code=_DummyRetCode.DATA_ERROR, message="Sorry! Data missing!"): - return {"code": code, "message": message} - - def _get_json_result(code=_DummyRetCode.SUCCESS, message="success", data=None): - return {"code": code, "message": message, "data": data} - - def _server_error_response(error): - return {"code": _DummyRetCode.EXCEPTION_ERROR, "message": repr(error)} - - def _validate_request(*_args, **_kwargs): - def _decorator(func): - return func - - return _decorator - - api_utils_mod.get_data_error_result = _get_data_error_result - api_utils_mod.get_json_result = _get_json_result - api_utils_mod.get_request_json = _default_request_json - api_utils_mod.server_error_response = _server_error_response - api_utils_mod.validate_request = _validate_request - monkeypatch.setitem(sys.modules, "api.utils.api_utils", api_utils_mod) - utils_pkg.api_utils = api_utils_mod - - module_name = "test_evaluation_routes_unit_module" - module_path = repo_root / "api" / "apps" / "evaluation_app.py" - spec = importlib.util.spec_from_file_location(module_name, module_path) - module = importlib.util.module_from_spec(spec) - module.manager = _DummyManager() - monkeypatch.setitem(sys.modules, module_name, module) - spec.loader.exec_module(module) - return module - - -@pytest.mark.p2 -def test_dataset_routes_matrix_unit(monkeypatch): - module = _load_evaluation_app(monkeypatch) - - _set_request_json(monkeypatch, module, {"name": " data-1 ", "description": "desc", "kb_ids": ["kb-1"]}) - monkeypatch.setattr(module.EvaluationService, "create_dataset", lambda **_kwargs: (True, "dataset-ok")) - res = _run(module.create_dataset()) - assert res["code"] == 0 - assert res["data"]["dataset_id"] == "dataset-ok" - - _set_request_json(monkeypatch, module, {"name": " ", "kb_ids": ["kb-1"]}) - res = _run(module.create_dataset()) - assert res["code"] == module.RetCode.DATA_ERROR - assert "empty" in res["message"].lower() - - _set_request_json(monkeypatch, module, {"name": "data-2", "kb_ids": "kb-1"}) - res = _run(module.create_dataset()) - assert res["code"] == module.RetCode.DATA_ERROR - assert "kb_ids" in res["message"] - - _set_request_json(monkeypatch, module, {"name": "data-3", "kb_ids": ["kb-1"]}) - monkeypatch.setattr(module.EvaluationService, "create_dataset", lambda **_kwargs: (False, "create failed")) - res = _run(module.create_dataset()) - assert res["code"] == module.RetCode.DATA_ERROR - assert res["message"] == "create failed" - - def _raise_create(**_kwargs): - raise RuntimeError("create boom") - - monkeypatch.setattr(module.EvaluationService, "create_dataset", _raise_create) - res = _run(module.create_dataset()) - assert res["code"] == module.RetCode.EXCEPTION_ERROR - assert "create boom" in res["message"] - - _set_request_args(monkeypatch, module, {"page": "2", "page_size": "3"}) - monkeypatch.setattr(module.EvaluationService, "list_datasets", lambda **_kwargs: {"datasets": [{"id": "a"}], "total": 1}) - res = _run(module.list_datasets()) - assert res["code"] == 0 - assert res["data"]["total"] == 1 - - _set_request_args(monkeypatch, module, {"page": "x"}) - res = _run(module.list_datasets()) - assert res["code"] == module.RetCode.EXCEPTION_ERROR - - monkeypatch.setattr(module.EvaluationService, "get_dataset", lambda _dataset_id: None) - res = _run(module.get_dataset("dataset-1")) - assert res["code"] == module.RetCode.DATA_ERROR - assert "not found" in res["message"].lower() - - monkeypatch.setattr(module.EvaluationService, "get_dataset", lambda _dataset_id: {"id": _dataset_id}) - res = _run(module.get_dataset("dataset-2")) - assert res["code"] == 0 - assert res["data"]["id"] == "dataset-2" - - def _raise_get(_dataset_id): - raise RuntimeError("get dataset boom") - - monkeypatch.setattr(module.EvaluationService, "get_dataset", _raise_get) - res = _run(module.get_dataset("dataset-3")) - assert res["code"] == module.RetCode.EXCEPTION_ERROR - assert "get dataset boom" in res["message"] - - captured = {} - - def _update(dataset_id, **kwargs): - captured["dataset_id"] = dataset_id - captured["kwargs"] = kwargs - return True - - _set_request_json( - monkeypatch, - module, - { - "id": "forbidden", - "tenant_id": "forbidden", - "created_by": "forbidden", - "create_time": 123, - "name": "new-name", - }, - ) - monkeypatch.setattr(module.EvaluationService, "update_dataset", _update) - res = _run(module.update_dataset("dataset-4")) - assert res["code"] == 0 - assert res["data"]["dataset_id"] == "dataset-4" - assert captured["dataset_id"] == "dataset-4" - assert "id" not in captured["kwargs"] - assert "tenant_id" not in captured["kwargs"] - assert "created_by" not in captured["kwargs"] - assert "create_time" not in captured["kwargs"] - - _set_request_json(monkeypatch, module, {"name": "new-name"}) - monkeypatch.setattr(module.EvaluationService, "update_dataset", lambda _dataset_id, **_kwargs: False) - res = _run(module.update_dataset("dataset-5")) - assert res["code"] == module.RetCode.DATA_ERROR - assert "failed" in res["message"].lower() - - def _raise_update(_dataset_id, **_kwargs): - raise RuntimeError("update boom") - - monkeypatch.setattr(module.EvaluationService, "update_dataset", _raise_update) - res = _run(module.update_dataset("dataset-6")) - assert res["code"] == module.RetCode.EXCEPTION_ERROR - assert "update boom" in res["message"] - - monkeypatch.setattr(module.EvaluationService, "delete_dataset", lambda _dataset_id: False) - res = _run(module.delete_dataset("dataset-7")) - assert res["code"] == module.RetCode.DATA_ERROR - assert "failed" in res["message"].lower() - - monkeypatch.setattr(module.EvaluationService, "delete_dataset", lambda _dataset_id: True) - res = _run(module.delete_dataset("dataset-8")) - assert res["code"] == 0 - assert res["data"]["dataset_id"] == "dataset-8" - - def _raise_delete(_dataset_id): - raise RuntimeError("delete dataset boom") - - monkeypatch.setattr(module.EvaluationService, "delete_dataset", _raise_delete) - res = _run(module.delete_dataset("dataset-9")) - assert res["code"] == module.RetCode.EXCEPTION_ERROR - assert "delete dataset boom" in res["message"] - - -@pytest.mark.p2 -def test_test_case_routes_matrix_unit(monkeypatch): - module = _load_evaluation_app(monkeypatch) - - _set_request_json(monkeypatch, module, {"question": " "}) - res = _run(module.add_test_case("dataset-1")) - assert res["code"] == module.RetCode.DATA_ERROR - assert "question" in res["message"].lower() - - _set_request_json(monkeypatch, module, {"question": "q1"}) - monkeypatch.setattr(module.EvaluationService, "add_test_case", lambda **_kwargs: (False, "add failed")) - res = _run(module.add_test_case("dataset-2")) - assert res["code"] == module.RetCode.DATA_ERROR - assert "add failed" in res["message"] - - _set_request_json( - monkeypatch, - module, - { - "question": "q2", - "reference_answer": "a2", - "relevant_doc_ids": ["doc-1"], - "relevant_chunk_ids": ["chunk-1"], - "metadata": {"k": "v"}, - }, - ) - monkeypatch.setattr(module.EvaluationService, "add_test_case", lambda **_kwargs: (True, "case-ok")) - res = _run(module.add_test_case("dataset-3")) - assert res["code"] == 0 - assert res["data"]["case_id"] == "case-ok" - - def _raise_add(**_kwargs): - raise RuntimeError("add case boom") - - monkeypatch.setattr(module.EvaluationService, "add_test_case", _raise_add) - res = _run(module.add_test_case("dataset-4")) - assert res["code"] == module.RetCode.EXCEPTION_ERROR - assert "add case boom" in res["message"] - - _set_request_json(monkeypatch, module, {"cases": {}}) - res = _run(module.import_test_cases("dataset-5")) - assert res["code"] == module.RetCode.DATA_ERROR - assert "cases" in res["message"] - - _set_request_json(monkeypatch, module, {"cases": [{"question": "q1"}, {"question": "q2"}]}) - monkeypatch.setattr(module.EvaluationService, "import_test_cases", lambda **_kwargs: (2, 0)) - res = _run(module.import_test_cases("dataset-6")) - assert res["code"] == 0 - assert res["data"]["success_count"] == 2 - assert res["data"]["failure_count"] == 0 - assert res["data"]["total"] == 2 - - def _raise_import(**_kwargs): - raise RuntimeError("import boom") - - monkeypatch.setattr(module.EvaluationService, "import_test_cases", _raise_import) - res = _run(module.import_test_cases("dataset-7")) - assert res["code"] == module.RetCode.EXCEPTION_ERROR - assert "import boom" in res["message"] - - monkeypatch.setattr(module.EvaluationService, "get_test_cases", lambda _dataset_id: [{"id": "case-1"}]) - res = _run(module.get_test_cases("dataset-8")) - assert res["code"] == 0 - assert res["data"]["total"] == 1 - assert res["data"]["cases"][0]["id"] == "case-1" - - def _raise_get_cases(_dataset_id): - raise RuntimeError("get cases boom") - - monkeypatch.setattr(module.EvaluationService, "get_test_cases", _raise_get_cases) - res = _run(module.get_test_cases("dataset-9")) - assert res["code"] == module.RetCode.EXCEPTION_ERROR - assert "get cases boom" in res["message"] - - monkeypatch.setattr(module.EvaluationService, "delete_test_case", lambda _case_id: False) - res = _run(module.delete_test_case("case-1")) - assert res["code"] == module.RetCode.DATA_ERROR - assert "failed" in res["message"].lower() - - monkeypatch.setattr(module.EvaluationService, "delete_test_case", lambda _case_id: True) - res = _run(module.delete_test_case("case-2")) - assert res["code"] == 0 - assert res["data"]["case_id"] == "case-2" - - def _raise_delete_case(_case_id): - raise RuntimeError("delete case boom") - - monkeypatch.setattr(module.EvaluationService, "delete_test_case", _raise_delete_case) - res = _run(module.delete_test_case("case-3")) - assert res["code"] == module.RetCode.EXCEPTION_ERROR - assert "delete case boom" in res["message"] - - -@pytest.mark.p2 -def test_run_and_recommendation_routes_matrix_unit(monkeypatch): - module = _load_evaluation_app(monkeypatch) - - _set_request_json(monkeypatch, module, {"dataset_id": "d1", "dialog_id": "dialog-1", "name": "run 1"}) - monkeypatch.setattr(module.EvaluationService, "start_evaluation", lambda **_kwargs: (False, "start failed")) - res = _run(module.start_evaluation()) - assert res["code"] == module.RetCode.DATA_ERROR - assert "start failed" in res["message"] - - monkeypatch.setattr(module.EvaluationService, "start_evaluation", lambda **_kwargs: (True, "run-ok")) - res = _run(module.start_evaluation()) - assert res["code"] == 0 - assert res["data"]["run_id"] == "run-ok" - - def _raise_start(**_kwargs): - raise RuntimeError("start boom") - - monkeypatch.setattr(module.EvaluationService, "start_evaluation", _raise_start) - res = _run(module.start_evaluation()) - assert res["code"] == module.RetCode.EXCEPTION_ERROR - assert "start boom" in res["message"] - - monkeypatch.setattr(module.EvaluationService, "get_run_results", lambda _run_id: None) - res = _run(module.get_evaluation_run("run-1")) - assert res["code"] == module.RetCode.DATA_ERROR - assert "not found" in res["message"].lower() - - monkeypatch.setattr(module.EvaluationService, "get_run_results", lambda _run_id: {"id": _run_id}) - res = _run(module.get_evaluation_run("run-2")) - assert res["code"] == 0 - assert res["data"]["id"] == "run-2" - - def _raise_get_run(_run_id): - raise RuntimeError("get run boom") - - monkeypatch.setattr(module.EvaluationService, "get_run_results", _raise_get_run) - res = _run(module.get_evaluation_run("run-3")) - assert res["code"] == module.RetCode.EXCEPTION_ERROR - assert "get run boom" in res["message"] - - monkeypatch.setattr(module.EvaluationService, "get_run_results", lambda _run_id: None) - res = _run(module.get_run_results("run-4")) - assert res["code"] == module.RetCode.DATA_ERROR - assert "not found" in res["message"].lower() - - monkeypatch.setattr(module.EvaluationService, "get_run_results", lambda _run_id: {"id": _run_id, "score": 0.9}) - res = _run(module.get_run_results("run-5")) - assert res["code"] == 0 - assert res["data"]["id"] == "run-5" - - def _raise_results(_run_id): - raise RuntimeError("get results boom") - - monkeypatch.setattr(module.EvaluationService, "get_run_results", _raise_results) - res = _run(module.get_run_results("run-6")) - assert res["code"] == module.RetCode.EXCEPTION_ERROR - assert "get results boom" in res["message"] - - res = _run(module.list_evaluation_runs()) - assert res["code"] == 0 - assert res["data"]["total"] == 0 - - def _raise_json_list(*_args, **_kwargs): - raise RuntimeError("list runs boom") - - monkeypatch.setattr(module, "get_json_result", _raise_json_list) - res = _run(module.list_evaluation_runs()) - assert res["code"] == module.RetCode.EXCEPTION_ERROR - assert "list runs boom" in res["message"] - - monkeypatch.setattr(module, "get_json_result", lambda code=0, message="success", data=None: {"code": code, "message": message, "data": data}) - res = _run(module.delete_evaluation_run("run-7")) - assert res["code"] == 0 - assert res["data"]["run_id"] == "run-7" - - def _raise_json_delete(*_args, **_kwargs): - raise RuntimeError("delete run boom") - - monkeypatch.setattr(module, "get_json_result", _raise_json_delete) - res = _run(module.delete_evaluation_run("run-8")) - assert res["code"] == module.RetCode.EXCEPTION_ERROR - assert "delete run boom" in res["message"] - - monkeypatch.setattr(module, "get_json_result", lambda code=0, message="success", data=None: {"code": code, "message": message, "data": data}) - monkeypatch.setattr(module.EvaluationService, "get_recommendations", lambda _run_id: [{"name": "cfg-1"}]) - res = _run(module.get_recommendations("run-9")) - assert res["code"] == 0 - assert res["data"]["recommendations"][0]["name"] == "cfg-1" - - def _raise_recommend(_run_id): - raise RuntimeError("recommend boom") - - monkeypatch.setattr(module.EvaluationService, "get_recommendations", _raise_recommend) - res = _run(module.get_recommendations("run-10")) - assert res["code"] == module.RetCode.EXCEPTION_ERROR - assert "recommend boom" in res["message"] - - -@pytest.mark.p2 -def test_compare_export_and_evaluate_single_matrix_unit(monkeypatch): - module = _load_evaluation_app(monkeypatch) - - _set_request_json(monkeypatch, module, {"run_ids": ["run-1"]}) - res = _run(module.compare_runs()) - assert res["code"] == module.RetCode.DATA_ERROR - assert "at least 2" in res["message"] - - _set_request_json(monkeypatch, module, {"run_ids": ["run-1", "run-2"]}) - res = _run(module.compare_runs()) - assert res["code"] == 0 - assert res["data"]["comparison"] == {} - - def _raise_json_compare(*_args, **_kwargs): - raise RuntimeError("compare boom") - - monkeypatch.setattr(module, "get_json_result", _raise_json_compare) - _set_request_json(monkeypatch, module, {"run_ids": ["run-1", "run-2", "run-3"]}) - res = _run(module.compare_runs()) - assert res["code"] == module.RetCode.EXCEPTION_ERROR - assert "compare boom" in res["message"] - - monkeypatch.setattr(module, "get_json_result", lambda code=0, message="success", data=None: {"code": code, "message": message, "data": data}) - monkeypatch.setattr(module.EvaluationService, "get_run_results", lambda _run_id: None) - res = _run(module.export_results("run-11")) - assert res["code"] == module.RetCode.DATA_ERROR - assert "not found" in res["message"].lower() - - monkeypatch.setattr(module.EvaluationService, "get_run_results", lambda _run_id: {"id": _run_id, "rows": []}) - res = _run(module.export_results("run-12")) - assert res["code"] == 0 - assert res["data"]["id"] == "run-12" - - def _raise_export(_run_id): - raise RuntimeError("export boom") - - monkeypatch.setattr(module.EvaluationService, "get_run_results", _raise_export) - res = _run(module.export_results("run-13")) - assert res["code"] == module.RetCode.EXCEPTION_ERROR - assert "export boom" in res["message"] - - monkeypatch.setattr(module, "get_json_result", lambda code=0, message="success", data=None: {"code": code, "message": message, "data": data}) - res = _run(module.evaluate_single()) - assert res["code"] == 0 - assert res["data"]["answer"] == "" - assert res["data"]["metrics"] == {} - assert res["data"]["retrieved_chunks"] == [] - - def _raise_json_single(*_args, **_kwargs): - raise RuntimeError("single boom") - - monkeypatch.setattr(module, "get_json_result", _raise_json_single) - res = _run(module.evaluate_single()) - assert res["code"] == module.RetCode.EXCEPTION_ERROR - assert "single boom" in res["message"]