youngkingdom/dify
1
0
Fork 0
mirror of https://github.com/langgenius/dify.git synced 2026-05-05 09:58:04 +08:00
Code Issues Packages Projects Releases Wiki Activity

chore: apply ruff's pyupgrade linter rules to modernize Python code with targeted version (#2419)

Browse Source
This commit is contained in:
Bowen Liang
2024-02-09 15:21:33 +08:00
committed by GitHub
parent 589099a005
commit 063191889d
246 changed files with 912 additions and 937 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
api/core/vector_store/vector/milvus.py
View File

@ -2,7 +2,8 @@
from __future__ import annotations
import logging
from typing import Any, Iterable, List, Optional, Sequence, Tuple, Union
from collections.abc import Iterable, Sequence
from typing import Any, Optional, Union
from uuid import uuid4
import numpy as np
@ -381,11 +382,11 @@ class Milvus(VectorStore):
def add_texts(
self,
texts: Iterable[str],
metadatas: Optional[List[dict]] = None,
metadatas: Optional[list[dict]] = None,
timeout: Optional[int] = None,
batch_size: int = 1000,
**kwargs: Any,
) -> List[str]:
) -> list[str]:
"""Insert text data into Milvus.
Inserting data when the collection has not be made yet will result
@ -476,7 +477,7 @@ class Milvus(VectorStore):
expr: Optional[str] = None,
timeout: Optional[int] = None,
**kwargs: Any,
) -> List[Document]:
) -> list[Document]:
"""Perform a similarity search against the query string.
Args:
@ -502,13 +503,13 @@ class Milvus(VectorStore):
def similarity_search_by_vector(
self,
embedding: List[float],
embedding: list[float],
k: int = 4,
param: Optional[dict] = None,
expr: Optional[str] = None,
timeout: Optional[int] = None,
**kwargs: Any,
) -> List[Document]:
) -> list[Document]:
"""Perform a similarity search against the query string.
Args:
@ -540,7 +541,7 @@ class Milvus(VectorStore):
expr: Optional[str] = None,
timeout: Optional[int] = None,
**kwargs: Any,
) -> List[Tuple[Document, float]]:
) -> list[tuple[Document, float]]:
"""Perform a search on a query string and return results with score.
For more information about the search parameters, take a look at the pymilvus
@ -577,7 +578,7 @@ class Milvus(VectorStore):
query: str,
k: int = 4,
**kwargs: Any,
) -> List[Tuple[Document, float]]:
) -> list[tuple[Document, float]]:
"""Return docs and relevance scores in the range [0, 1].
0 is dissimilar, 1 is most similar.
@ -596,13 +597,13 @@ class Milvus(VectorStore):
def similarity_search_with_score_by_vector(
self,
embedding: List[float],
embedding: list[float],
k: int = 4,
param: Optional[dict] = None,
expr: Optional[str] = None,
timeout: Optional[int] = None,
**kwargs: Any,
) -> List[Tuple[Document, float]]:
) -> list[tuple[Document, float]]:
"""Perform a search on a query string and return results with score.
For more information about the search parameters, take a look at the pymilvus
@ -664,7 +665,7 @@ class Milvus(VectorStore):
expr: Optional[str] = None,
timeout: Optional[int] = None,
**kwargs: Any,
) -> List[Document]:
) -> list[Document]:
"""Perform a search and return results that are reordered by MMR.
Args:
@ -714,7 +715,7 @@ class Milvus(VectorStore):
expr: Optional[str] = None,
timeout: Optional[int] = None,
**kwargs: Any,
) -> List[Document]:
) -> list[Document]:
"""Perform a search and return results that are reordered by MMR.
Args:
@ -797,9 +798,9 @@ class Milvus(VectorStore):
@classmethod
def from_texts(
cls,
texts: List[str],
texts: list[str],
embedding: Embeddings,
metadatas: Optional[List[dict]] = None,
metadatas: Optional[list[dict]] = None,
collection_name: str = "LangChainCollection",
connection_args: dict[str, Any] = DEFAULT_MILVUS_CONNECTION,
consistency_level: str = "Session",

99
api/core/vector_store/vector/qdrant.py
View File

@ -5,9 +5,10 @@ import asyncio
import functools
import uuid
import warnings
from collections.abc import Callable, Generator, Iterable, Sequence
from itertools import islice
from operator import itemgetter
from typing import TYPE_CHECKING, Any, Callable, Dict, Generator, Iterable, List, Optional, Sequence, Tuple, Type, Union
from typing import TYPE_CHECKING, Any, Optional, Union
import numpy as np
from langchain.docstore.document import Document
@ -21,7 +22,7 @@ if TYPE_CHECKING:
from qdrant_client.conversions import common_types
from qdrant_client.http import models as rest
DictFilter = Dict[str, Union[str, int, bool, dict, list]]
DictFilter = dict[str, Union[str, int, bool, dict, list]]
MetadataFilter = Union[DictFilter, common_types.Filter]
@ -148,11 +149,11 @@ class Qdrant(VectorStore):
def add_texts(
self,
texts: Iterable[str],
metadatas: Optional[List[dict]] = None,
metadatas: Optional[list[dict]] = None,
ids: Optional[Sequence[str]] = None,
batch_size: int = 64,
**kwargs: Any,
) -> List[str]:
) -> list[str]:
"""Run more texts through the embeddings and add to the vectorstore.
Args:
@ -200,11 +201,11 @@ class Qdrant(VectorStore):
async def aadd_texts(
self,
texts: Iterable[str],
metadatas: Optional[List[dict]] = None,
metadatas: Optional[list[dict]] = None,
ids: Optional[Sequence[str]] = None,
batch_size: int = 64,
**kwargs: Any,
) -> List[str]:
) -> list[str]:
"""Run more texts through the embeddings and add to the vectorstore.
Args:
@ -247,7 +248,7 @@ class Qdrant(VectorStore):
score_threshold: Optional[float] = None,
consistency: Optional[common_types.ReadConsistency] = None,
**kwargs: Any,
) -> List[Tuple[Document, float]]:
) -> list[tuple[Document, float]]:
"""Return docs most similar to query.
Args:
@ -299,7 +300,7 @@ class Qdrant(VectorStore):
k: int = 4,
filter: Optional[MetadataFilter] = None,
**kwargs: Any,
) -> List[Document]:
) -> list[Document]:
"""Return docs most similar to query.
Args:
query: Text to look up documents similar to.
@ -321,7 +322,7 @@ class Qdrant(VectorStore):
score_threshold: Optional[float] = None,
consistency: Optional[common_types.ReadConsistency] = None,
**kwargs: Any,
) -> List[Tuple[Document, float]]:
) -> list[tuple[Document, float]]:
"""Return docs most similar to query.
Args:
@ -376,7 +377,7 @@ class Qdrant(VectorStore):
score_threshold: Optional[float] = None,
consistency: Optional[common_types.ReadConsistency] = None,
**kwargs: Any,
) -> List[Tuple[Document, float]]:
) -> list[tuple[Document, float]]:
"""Return docs most similar to query.
Args:
@ -422,7 +423,7 @@ class Qdrant(VectorStore):
def similarity_search_by_vector(
self,
embedding: List[float],
embedding: list[float],
k: int = 4,
filter: Optional[MetadataFilter] = None,
search_params: Optional[common_types.SearchParams] = None,
@ -430,7 +431,7 @@ class Qdrant(VectorStore):
score_threshold: Optional[float] = None,
consistency: Optional[common_types.ReadConsistency] = None,
**kwargs: Any,
) -> List[Document]:
) -> list[Document]:
"""Return docs most similar to embedding vector.
Args:
@ -478,7 +479,7 @@ class Qdrant(VectorStore):
@sync_call_fallback
async def asimilarity_search_by_vector(
self,
embedding: List[float],
embedding: list[float],
k: int = 4,
filter: Optional[MetadataFilter] = None,
search_params: Optional[common_types.SearchParams] = None,
@ -486,7 +487,7 @@ class Qdrant(VectorStore):
score_threshold: Optional[float] = None,
consistency: Optional[common_types.ReadConsistency] = None,
**kwargs: Any,
) -> List[Document]:
) -> list[Document]:
"""Return docs most similar to embedding vector.
Args:
@ -533,7 +534,7 @@ class Qdrant(VectorStore):
def similarity_search_with_score_by_vector(
self,
embedding: List[float],
embedding: list[float],
k: int = 4,
filter: Optional[MetadataFilter] = None,
search_params: Optional[common_types.SearchParams] = None,
@ -541,7 +542,7 @@ class Qdrant(VectorStore):
score_threshold: Optional[float] = None,
consistency: Optional[common_types.ReadConsistency] = None,
**kwargs: Any,
) -> List[Tuple[Document, float]]:
) -> list[tuple[Document, float]]:
"""Return docs most similar to embedding vector.
Args:
@ -616,7 +617,7 @@ class Qdrant(VectorStore):
self,
filter: Optional[MetadataFilter] = None,
k: int = 4
) -> List[Document]:
) -> list[Document]:
"""Return docs most similar by bm25.
Args:
@ -648,7 +649,7 @@ class Qdrant(VectorStore):
@sync_call_fallback
async def asimilarity_search_with_score_by_vector(
self,
embedding: List[float],
embedding: list[float],
k: int = 4,
filter: Optional[MetadataFilter] = None,
search_params: Optional[common_types.SearchParams] = None,
@ -656,7 +657,7 @@ class Qdrant(VectorStore):
score_threshold: Optional[float] = None,
consistency: Optional[common_types.ReadConsistency] = None,
**kwargs: Any,
) -> List[Tuple[Document, float]]:
) -> list[tuple[Document, float]]:
"""Return docs most similar to embedding vector.
Args:
@ -741,7 +742,7 @@ class Qdrant(VectorStore):
fetch_k: int = 20,
lambda_mult: float = 0.5,
**kwargs: Any,
) -> List[Document]:
) -> list[Document]:
"""Return docs selected using the maximal marginal relevance.
Maximal marginal relevance optimizes for similarity to query AND diversity
@ -772,7 +773,7 @@ class Qdrant(VectorStore):
fetch_k: int = 20,
lambda_mult: float = 0.5,
**kwargs: Any,
) -> List[Document]:
) -> list[Document]:
"""Return docs selected using the maximal marginal relevance.
Maximal marginal relevance optimizes for similarity to query AND diversity
@ -797,12 +798,12 @@ class Qdrant(VectorStore):
def max_marginal_relevance_search_by_vector(
self,
embedding: List[float],
embedding: list[float],
k: int = 4,
fetch_k: int = 20,
lambda_mult: float = 0.5,
**kwargs: Any,
) -> List[Document]:
) -> list[Document]:
"""Return docs selected using the maximal marginal relevance.
Maximal marginal relevance optimizes for similarity to query AND diversity
@ -827,12 +828,12 @@ class Qdrant(VectorStore):
@sync_call_fallback
async def amax_marginal_relevance_search_by_vector(
self,
embedding: List[float],
embedding: list[float],
k: int = 4,
fetch_k: int = 20,
lambda_mult: float = 0.5,
**kwargs: Any,
) -> List[Document]:
) -> list[Document]:
"""Return docs selected using the maximal marginal relevance.
Maximal marginal relevance optimizes for similarity to query AND diversity
among selected documents.
@ -856,12 +857,12 @@ class Qdrant(VectorStore):
def max_marginal_relevance_search_with_score_by_vector(
self,
embedding: List[float],
embedding: list[float],
k: int = 4,
fetch_k: int = 20,
lambda_mult: float = 0.5,
**kwargs: Any,
) -> List[Tuple[Document, float]]:
) -> list[tuple[Document, float]]:
"""Return docs selected using the maximal marginal relevance.
Maximal marginal relevance optimizes for similarity to query AND diversity
among selected documents.
@ -911,12 +912,12 @@ class Qdrant(VectorStore):
@sync_call_fallback
async def amax_marginal_relevance_search_with_score_by_vector(
self,
embedding: List[float],
embedding: list[float],
k: int = 4,
fetch_k: int = 20,
lambda_mult: float = 0.5,
**kwargs: Any,
) -> List[Tuple[Document, float]]:
) -> list[tuple[Document, float]]:
"""Return docs selected using the maximal marginal relevance.
Maximal marginal relevance optimizes for similarity to query AND diversity
among selected documents.
@ -949,13 +950,13 @@ class Qdrant(VectorStore):
results = [
GrpcToRest.convert_vectors(result.vectors) for result in response.result
]
embeddings: List[List[float]] = [
embeddings: list[list[float]] = [
result.get(self.vector_name) # type: ignore
if isinstance(result, dict)
else result
for result in results
]
mmr_selected: List[int] = maximal_marginal_relevance(
mmr_selected: list[int] = maximal_marginal_relevance(
np.array(embedding),
embeddings,
k=k,
@ -973,7 +974,7 @@ class Qdrant(VectorStore):
for i in mmr_selected
]
def delete(self, ids: Optional[List[str]] = None, **kwargs: Any) -> Optional[bool]:
def delete(self, ids: Optional[list[str]] = None, **kwargs: Any) -> Optional[bool]:
"""Delete by vector ID or other criteria.
Args:
@ -994,10 +995,10 @@ class Qdrant(VectorStore):
@classmethod
def from_texts(
cls: Type[Qdrant],
texts: List[str],
cls: type[Qdrant],
texts: list[str],
embedding: Embeddings,
metadatas: Optional[List[dict]] = None,
metadatas: Optional[list[dict]] = None,
ids: Optional[Sequence[str]] = None,
location: Optional[str] = None,
url: Optional[str] = None,
@ -1179,10 +1180,10 @@ class Qdrant(VectorStore):
@classmethod
@sync_call_fallback
async def afrom_texts(
cls: Type[Qdrant],
texts: List[str],
cls: type[Qdrant],
texts: list[str],
embedding: Embeddings,
metadatas: Optional[List[dict]] = None,
metadatas: Optional[list[dict]] = None,
ids: Optional[Sequence[str]] = None,
location: Optional[str] = None,
url: Optional[str] = None,
@ -1354,10 +1355,10 @@ class Qdrant(VectorStore):
@classmethod
def _construct_instance(
cls: Type[Qdrant],
texts: List[str],
cls: type[Qdrant],
texts: list[str],
embedding: Embeddings,
metadatas: Optional[List[dict]] = None,
metadatas: Optional[list[dict]] = None,
ids: Optional[Sequence[str]] = None,
location: Optional[str] = None,
url: Optional[str] = None,
@ -1553,7 +1554,7 @@ class Qdrant(VectorStore):
query: str,
k: int = 4,
**kwargs: Any,
) -> List[Tuple[Document, float]]:
) -> list[tuple[Document, float]]:
"""Return docs and relevance scores in the range [0, 1].
0 is dissimilar, 1 is most similar.
@ -1574,12 +1575,12 @@ class Qdrant(VectorStore):
def _build_payloads(
cls,
texts: Iterable[str],
metadatas: Optional[List[dict]],
metadatas: Optional[list[dict]],
content_payload_key: str,
metadata_payload_key: str,
group_id: str,
group_payload_key: str
) -> List[dict]:
) -> list[dict]:
payloads = []
for i, text in enumerate(texts):
if text is None:
@ -1625,7 +1626,7 @@ class Qdrant(VectorStore):
metadata=payload.get(metadata_payload_key) or {},
)
def _build_condition(self, key: str, value: Any) -> List[rest.FieldCondition]:
def _build_condition(self, key: str, value: Any) -> list[rest.FieldCondition]:
from qdrant_client.http import models as rest
out = []
@ -1665,7 +1666,7 @@ class Qdrant(VectorStore):
]
)
def _embed_query(self, query: str) -> List[float]:
def _embed_query(self, query: str) -> list[float]:
"""Embed query text.
Used to provide backward compatibility with `embedding_function` argument.
@ -1685,7 +1686,7 @@ class Qdrant(VectorStore):
raise ValueError("Neither of embeddings or embedding_function is set")
return embedding.tolist() if hasattr(embedding, "tolist") else embedding
def _embed_texts(self, texts: Iterable[str]) -> List[List[float]]:
def _embed_texts(self, texts: Iterable[str]) -> list[list[float]]:
"""Embed search texts.
Used to provide backward compatibility with `embedding_function` argument.
@ -1715,11 +1716,11 @@ class Qdrant(VectorStore):
def _generate_rest_batches(
self,
texts: Iterable[str],
metadatas: Optional[List[dict]] = None,
metadatas: Optional[list[dict]] = None,
ids: Optional[Sequence[str]] = None,
batch_size: int = 64,
group_id: Optional[str] = None,
) -> Generator[Tuple[List[str], List[rest.PointStruct]], None, None]:
) -> Generator[tuple[list[str], list[rest.PointStruct]], None, None]:
from qdrant_client.http import models as rest
texts_iterator = iter(texts)

45
api/core/vector_store/vector/weaviate.py
View File

@ -2,7 +2,8 @@
from __future__ import annotations
import datetime
from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple, Type
from collections.abc import Callable, Iterable
from typing import Any, Optional
from uuid import uuid4
import numpy as np
@ -13,7 +14,7 @@ from langchain.vectorstores.base import VectorStore
from langchain.vectorstores.utils import maximal_marginal_relevance
def _default_schema(index_name: str) -> Dict:
def _default_schema(index_name: str) -> dict:
return {
"class": index_name,
"properties": [
@ -89,7 +90,7 @@ class Weaviate(VectorStore):
index_name: str,
text_key: str,
embedding: Optional[Embeddings] = None,
attributes: Optional[List[str]] = None,
attributes: Optional[list[str]] = None,
relevance_score_fn: Optional[
Callable[[float], float]
] = _default_score_normalizer,
@ -131,14 +132,14 @@ class Weaviate(VectorStore):
def add_texts(
self,
texts: Iterable[str],
metadatas: Optional[List[dict]] = None,
metadatas: Optional[list[dict]] = None,
**kwargs: Any,
) -> List[str]:
) -> list[str]:
"""Upload texts with metadata (properties) to Weaviate."""
from weaviate.util import get_valid_uuid
ids = []
embeddings: Optional[List[List[float]]] = None
embeddings: Optional[list[list[float]]] = None
if self._embedding:
if not isinstance(texts, list):
texts = list(texts)
@ -172,7 +173,7 @@ class Weaviate(VectorStore):
def similarity_search(
self, query: str, k: int = 4, **kwargs: Any
) -> List[Document]:
) -> list[Document]:
"""Return docs most similar to query.
Args:
@ -195,7 +196,7 @@ class Weaviate(VectorStore):
def similarity_search_by_text(
self, query: str, k: int = 4, **kwargs: Any
) -> List[Document]:
) -> list[Document]:
"""Return docs most similar to query.
Args:
@ -205,7 +206,7 @@ class Weaviate(VectorStore):
Returns:
List of Documents most similar to the query.
"""
content: Dict[str, Any] = {"concepts": [query]}
content: dict[str, Any] = {"concepts": [query]}
if kwargs.get("search_distance"):
content["certainty"] = kwargs.get("search_distance")
query_obj = self._client.query.get(self._index_name, self._query_attrs)
@ -224,7 +225,7 @@ class Weaviate(VectorStore):
def similarity_search_by_bm25(
self, query: str, k: int = 4, **kwargs: Any
) -> List[Document]:
) -> list[Document]:
"""Return docs using BM25F.
Args:
@ -234,7 +235,7 @@ class Weaviate(VectorStore):
Returns:
List of Documents most similar to the query.
"""
content: Dict[str, Any] = {"concepts": [query]}
content: dict[str, Any] = {"concepts": [query]}
if kwargs.get("search_distance"):
content["certainty"] = kwargs.get("search_distance")
query_obj = self._client.query.get(self._index_name, self._query_attrs)
@ -253,8 +254,8 @@ class Weaviate(VectorStore):
return docs
def similarity_search_by_vector(
self, embedding: List[float], k: int = 4, **kwargs: Any
) -> List[Document]:
self, embedding: list[float], k: int = 4, **kwargs: Any
) -> list[Document]:
"""Look up similar documents by embedding vector in Weaviate."""
vector = {"vector": embedding}
query_obj = self._client.query.get(self._index_name, self._query_attrs)
@ -278,7 +279,7 @@ class Weaviate(VectorStore):
fetch_k: int = 20,
lambda_mult: float = 0.5,
**kwargs: Any,
) -> List[Document]:
) -> list[Document]:
"""Return docs selected using the maximal marginal relevance.
Maximal marginal relevance optimizes for similarity to query AND diversity
@ -309,12 +310,12 @@ class Weaviate(VectorStore):
def max_marginal_relevance_search_by_vector(
self,
embedding: List[float],
embedding: list[float],
k: int = 4,
fetch_k: int = 20,
lambda_mult: float = 0.5,
**kwargs: Any,
) -> List[Document]:
) -> list[Document]:
"""Return docs selected using the maximal marginal relevance.
Maximal marginal relevance optimizes for similarity to query AND diversity
@ -359,7 +360,7 @@ class Weaviate(VectorStore):
def similarity_search_with_score(
self, query: str, k: int = 4, **kwargs: Any
) -> List[Tuple[Document, float]]:
) -> list[tuple[Document, float]]:
"""
Return list of documents most similar to the query
text and cosine distance in float for each.
@ -369,7 +370,7 @@ class Weaviate(VectorStore):
raise ValueError(
"_embedding cannot be None for similarity_search_with_score"
)
content: Dict[str, Any] = {"concepts": [query]}
content: dict[str, Any] = {"concepts": [query]}
if kwargs.get("search_distance"):
content["certainty"] = kwargs.get("search_distance")
query_obj = self._client.query.get(self._index_name, self._query_attrs)
@ -403,10 +404,10 @@ class Weaviate(VectorStore):
@classmethod
def from_texts(
cls: Type[Weaviate],
texts: List[str],
cls: type[Weaviate],
texts: list[str],
embedding: Embeddings,
metadatas: Optional[List[dict]] = None,
metadatas: Optional[list[dict]] = None,
**kwargs: Any,
) -> Weaviate:
"""Construct Weaviate wrapper from raw documents.
@ -490,7 +491,7 @@ class Weaviate(VectorStore):
by_text=by_text,
)
def delete(self, ids: Optional[List[str]] = None, **kwargs: Any) -> None:
def delete(self, ids: Optional[list[str]] = None, **kwargs: Any) -> None:
"""Delete by vector IDs.
Args: