JDSS/mem0
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(v3): migration crashes + entity linking on OSS (#4836)

Browse Source 
Co-authored-by: Soumil Rathi <soumilrathi@gmail.com>
Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
soumil-rathi
2026-04-15 14:46:28 +05:30
committed by GitHub
co-authored by Soumil Rathi Claude Opus 4.6
parent a5a688295e
commit d8d776636f
8 changed files with 90 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files
mem0/vector_stores/milvus.py
+40 -17
View File
@@ -46,6 +46,9 @@ class MilvusDB(VectorStoreBase):
self.embedding_model_dims = embedding_model_dims
self.metric_type = metric_type
self.client = MilvusClient(uri=url, token=token, db_name=db_name)
# Whether this collection has the `text` + `sparse` fields for v3 BM25.
# Pre-v3 collections lack them; writing a top-level `text` field is rejected.
self._has_bm25_schema = False
self.create_col(
collection_name=self.collection_name,
vector_size=self.embedding_model_dims,
@@ -68,6 +71,15 @@ class MilvusDB(VectorStoreBase):
if self.client.has_collection(collection_name):
logger.info(f"Collection {collection_name} already exists. Skipping creation.")
desc = self.client.describe_collection(collection_name=collection_name)
field_names = {f.get("name") for f in desc.get("fields", [])}
self._has_bm25_schema = "text" in field_names and "sparse" in field_names
if not self._has_bm25_schema:
logger.warning(
f"Collection '{collection_name}' predates v3 hybrid search (no 'text'/'sparse' fields). "
"BM25 keyword scoring will be disabled for this collection; semantic search works normally. "
"To enable hybrid search, use a fresh collection."
)
else:
fields = [
FieldSchema(name="id", dtype=DataType.VARCHAR, is_primary=True, max_length=512),
@@ -101,6 +113,7 @@ class MilvusDB(VectorStoreBase):
index_name="sparse_index",
)
self.client.create_collection(collection_name=collection_name, schema=schema, index_params=index_params)
self._has_bm25_schema = True
def insert(self, ids, vectors, payloads, **kwargs: Optional[dict[str, any]]):
"""Insert vectors into a collection.
@@ -110,17 +123,17 @@ class MilvusDB(VectorStoreBase):
payloads (List[Dict], optional): List of payloads corresponding to vectors.
ids (List[str], optional): List of IDs corresponding to vectors.
"""
# Batch insert all records at once for better performance and consistency
data = [
{
"id": idx,
"vectors": embedding,
"metadata": metadata,
# Batch insert all records at once for better performance and consistency.
# Only include the `text` field when the collection's schema has it — legacy
# collections created pre-v3 reject unknown top-level fields.
def _build_record(idx, embedding, metadata):
record = {"id": idx, "vectors": embedding, "metadata": metadata}
if self._has_bm25_schema:
# Populate the text field for BM25 sparse search; prefer lemmatized text, fall back to raw data
"text": (metadata.get("text_lemmatized") or metadata.get("data", ""))[:65535] if metadata else "",
}
for idx, embedding, metadata in zip(ids, vectors, payloads)
]
record["text"] = (metadata.get("text_lemmatized") or metadata.get("data", ""))[:65535] if metadata else ""
return record
data = [_build_record(idx, embedding, metadata) for idx, embedding, metadata in zip(ids, vectors, payloads)]
self.client.insert(collection_name=self.collection_name, data=data, **kwargs)
def _create_filter(self, filters: dict):
@@ -179,13 +192,21 @@ class MilvusDB(VectorStoreBase):
list: Search results.
"""
query_filter = self._create_filter(filters) if filters else None
hits = self.client.search(
collection_name=self.collection_name,
data=[vectors],
limit=top_k,
filter=query_filter,
output_fields=["*"],
)
# v3 collections carry both a dense `vectors` field and a sparse `sparse`
# field (for BM25), which makes anns_field ambiguous — Milvus rejects the
# query otherwise with "multiple anns_fields exist". Legacy single-vector
# collections don't need the hint, so only pass it when the hybrid schema
# is present.
search_kwargs = {
"collection_name": self.collection_name,
"data": [vectors],
"limit": top_k,
"filter": query_filter,
"output_fields": ["*"],
}
if self._has_bm25_schema:
search_kwargs["anns_field"] = "vectors"
hits = self.client.search(**search_kwargs)
result = self._parse_output(data=hits[0])
return result
@@ -206,6 +227,8 @@ class MilvusDB(VectorStoreBase):
list: Search results in the same format as search(), or None if sparse search
is not supported on this collection.
"""
if not self._has_bm25_schema:
return None
try:
query_filter = self._create_filter(filters) if filters else None
hits = self.client.search(
mem0/vector_stores/qdrant.py
+31 -13
View File
@@ -79,6 +79,10 @@ class Qdrant(VectorStoreBase):
self.embedding_model_dims = embedding_model_dims
self.on_disk = on_disk
self._bm25_encoder = None
# Whether this collection has the `bm25` named sparse vector slot.
# Pre-v3 collections lack it; writing a `bm25` sparse vector into such a
# collection is rejected by Qdrant ("Not existing vector name error: bm25").
self._has_bm25_slot = False
self.create_col(embedding_model_dims, on_disk)
def _get_bm25_encoder(self):
@@ -127,6 +131,15 @@ class Qdrant(VectorStoreBase):
for collection in response.collections:
if collection.name == self.collection_name:
logger.debug(f"Collection {self.collection_name} already exists. Skipping creation.")
info = self.client.get_collection(self.collection_name)
sparse_cfg = info.config.params.sparse_vectors
self._has_bm25_slot = bool(sparse_cfg and "bm25" in sparse_cfg)
if not self._has_bm25_slot:
logger.warning(
f"Collection '{self.collection_name}' predates v3 hybrid search (no 'bm25' sparse slot). "
"BM25 keyword scoring will be disabled for this collection; semantic search works normally. "
"To enable hybrid search, use a fresh collection."
)
self._create_filter_indexes()
return
@@ -139,6 +152,7 @@ class Qdrant(VectorStoreBase):
),
},
)
self._has_bm25_slot = True
self._create_filter_indexes()
def _create_filter_indexes(self):
@@ -177,13 +191,14 @@ class Qdrant(VectorStoreBase):
payload = payloads[idx] if payloads else {}
point_id = idx if ids is None else ids[idx]
# Build named vectors: dense + optional BM25 sparse
# Build named vectors: dense + optional BM25 sparse (only if collection has the slot).
named_vectors = {"": vector}
text_for_bm25 = payload.get("text_lemmatized") or payload.get("data", "")
if text_for_bm25:
sparse = self._encode_bm25(text_for_bm25)
if sparse is not None:
named_vectors["bm25"] = sparse
if self._has_bm25_slot:
text_for_bm25 = payload.get("text_lemmatized") or payload.get("data", "")
if text_for_bm25:
sparse = self._encode_bm25(text_for_bm25)
if sparse is not None:
named_vectors["bm25"] = sparse
points.append(PointStruct(id=point_id, vector=named_vectors, payload=payload))
@@ -382,7 +397,7 @@ class Qdrant(VectorStoreBase):
"""Batch search using Qdrant's query_batch_points for efficiency."""
query_filter = self._create_filter(filters) if filters else None
requests = [
models.QueryRequest(query=vec, filter=query_filter, limit=top_k)
models.QueryRequest(query=vec, filter=query_filter, limit=top_k, with_payload=True)
for vec in vectors_list
]
try:
@@ -407,6 +422,8 @@ class Qdrant(VectorStoreBase):
Returns:
list: Search results, or None if BM25 is not available.
"""
if not self._has_bm25_slot:
return None
sparse_query = self._encode_bm25(query)
if sparse_query is None:
return None
@@ -449,13 +466,14 @@ class Qdrant(VectorStoreBase):
payload (dict, optional): Updated payload. Defaults to None.
"""
if vector is not None and payload is not None:
# Full update: attach BM25 sparse vector alongside dense vector
# Full update: attach BM25 sparse vector alongside dense vector (only if slot exists).
named_vectors = {"": vector}
text_for_bm25 = payload.get("text_lemmatized") or payload.get("data", "")
if text_for_bm25:
sparse = self._encode_bm25(text_for_bm25)
if sparse is not None:
named_vectors["bm25"] = sparse
if self._has_bm25_slot:
text_for_bm25 = payload.get("text_lemmatized") or payload.get("data", "")
if text_for_bm25:
sparse = self._encode_bm25(text_for_bm25)
if sparse is not None:
named_vectors["bm25"] = sparse
point = PointStruct(id=vector_id, vector=named_vectors, payload=payload)
self.client.upsert(collection_name=self.collection_name, points=[point])
else: