Merge branch 'main' into fix/chore-fix

api/core/rag/cleaner/clean_processor.py

@@ -12,7 +12,7 @@ class CleanProcessor:
         # Unicode  U+FFFE
         text = re.sub("\ufffe", "", text)
 
-        rules = process_rule["rules"] if process_rule else None
+        rules = process_rule["rules"] if process_rule else {}
         if "pre_processing_rules" in rules:
             pre_processing_rules = rules["pre_processing_rules"]
             for pre_processing_rule in pre_processing_rules:

api/core/rag/datasource/keyword/keyword_type.py

@@ -1,5 +1,5 @@
-from enum import Enum
+from enum import StrEnum
 
 
-class KeyWordType(str, Enum):
+class KeyWordType(StrEnum):
     JIEBA = "jieba"

api/core/rag/datasource/vdb/vector_type.py

@@ -1,7 +1,7 @@
-from enum import Enum
+from enum import StrEnum
 
 
-class VectorType(str, Enum):
+class VectorType(StrEnum):
     ANALYTICDB = "analyticdb"
     CHROMA = "chroma"
     MILVUS = "milvus"

api/core/rag/extractor/word_extractor.py

@@ -114,10 +114,10 @@ class WordExtractor(BaseExtractor):
                     mime_type=mime_type or "",
                     created_by=self.user_id,
                     created_by_role=CreatedByRole.ACCOUNT,
-                    created_at=datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None),
+                    created_at=datetime.datetime.now(datetime.UTC).replace(tzinfo=None),
                     used=True,
                     used_by=self.user_id,
-                    used_at=datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None),
+                    used_at=datetime.datetime.now(datetime.UTC).replace(tzinfo=None),
                 )
 
                 db.session.add(upload_file)

api/core/rag/rerank/rerank_model.py

@@ -27,11 +27,11 @@ class RerankModelRunner(BaseRerankRunner):
         :return:
         """
         docs = []
-        doc_id = set()
+        doc_ids = set()
         unique_documents = []
         for document in documents:
-            if document.provider == "dify" and document.metadata["doc_id"] not in doc_id:
-                doc_id.add(document.metadata["doc_id"])
+            if document.provider == "dify" and document.metadata["doc_id"] not in doc_ids:
+                doc_ids.add(document.metadata["doc_id"])
                 docs.append(document.page_content)
                 unique_documents.append(document)
             elif document.provider == "external":

api/core/rag/rerank/rerank_type.py

@@ -1,6 +1,6 @@
-from enum import Enum
+from enum import StrEnum
 
 
-class RerankMode(str, Enum):
+class RerankMode(StrEnum):
     RERANKING_MODEL = "reranking_model"
     WEIGHTED_SCORE = "weighted_score"

api/core/rag/rerank/weight_rerank.py

@@ -37,11 +37,10 @@ class WeightRerankRunner(BaseRerankRunner):
         :return:
         """
         unique_documents = []
-        doc_id = set()
+        doc_ids = set()
         for document in documents:
-            doc_id = document.metadata.get("doc_id")
-            if doc_id not in doc_id:
-                doc_id.add(doc_id)
+            if document.metadata["doc_id"] not in doc_ids:
+                doc_ids.add(document.metadata["doc_id"])
                 unique_documents.append(document)
 
         documents = unique_documents