From 398445c691e6a6b253e4643b16d7751221185e0c Mon Sep 17 00:00:00 2001 From: Saket Aryan Date: Fri, 10 Apr 2026 23:43:12 +0530 Subject: [PATCH] fix: filters did not support AND, OR and NOT in Qdant and other vector stores (#4780) Co-authored-by: Claude Opus 4.6 (1M context) --- mem0-ts/src/oss/src/utils/factory.ts | 3 ++ mem0/memory/main.py | 18 ++++++- mem0/vector_stores/milvus.py | 62 ++++++++++++++++++++++-- mem0/vector_stores/qdrant.py | 70 +++++++++++++++++++++++++--- mem0/vector_stores/redis.py | 2 +- 5 files changed, 142 insertions(+), 13 deletions(-) diff --git a/mem0-ts/src/oss/src/utils/factory.ts b/mem0-ts/src/oss/src/utils/factory.ts index 8c461d761..14a22ad46 100644 --- a/mem0-ts/src/oss/src/utils/factory.ts +++ b/mem0-ts/src/oss/src/utils/factory.ts @@ -32,6 +32,7 @@ import { LangchainLLM } from "../llms/langchain"; import { LangchainEmbedder } from "../embeddings/langchain"; import { LangchainVectorStore } from "../vector_stores/langchain"; import { AzureAISearch } from "../vector_stores/azure_ai_search"; +import { PGVector } from "../vector_stores/pgvector"; export class EmbedderFactory { static create(provider: string, config: EmbeddingConfig): Embedder { @@ -98,6 +99,8 @@ export class VectorStoreFactory { return new VectorizeDB(config as any); case "azure-ai-search": return new AzureAISearch(config as any); + case "pgvector": + return new PGVector(config as any); default: throw new Error(`Unsupported vector store provider: ${provider}`); } diff --git a/mem0/memory/main.py b/mem0/memory/main.py index ecd69616c..0e17b50e0 100644 --- a/mem0/memory/main.py +++ b/mem0/memory/main.py @@ -997,6 +997,13 @@ class Memory(MemoryBase): # Apply enhanced metadata filtering if advanced operators are detected if filters and self._has_advanced_operators(filters): processed_filters = self._process_metadata_filters(filters) + # Remove original logical/operator keys that _build_filters_and_metadata + # copied verbatim from input_filters — they have now been reprocessed. + for logical_key in ("AND", "OR", "NOT"): + effective_filters.pop(logical_key, None) + for fk in list(filters.keys()): + if fk not in ("AND", "OR", "NOT") and fk in effective_filters and isinstance(filters[fk], dict): + effective_filters.pop(fk, None) effective_filters.update(processed_filters) elif filters: # Simple filters, merge directly @@ -1237,7 +1244,7 @@ class Memory(MemoryBase): additional_metadata = {k: v for k, v in payload.items() if k not in core_and_promoted_keys} if additional_metadata: - if "metadata" not in memory_item_dict: + if not memory_item_dict.get("metadata"): memory_item_dict["metadata"] = {} memory_item_dict["metadata"].update(additional_metadata) @@ -2207,6 +2214,13 @@ class AsyncMemory(MemoryBase): # Apply enhanced metadata filtering if advanced operators are detected if filters and self._has_advanced_operators(filters): processed_filters = self._process_metadata_filters(filters) + # Remove original logical/operator keys that _build_filters_and_metadata + # copied verbatim from input_filters — they have now been reprocessed. + for logical_key in ("AND", "OR", "NOT"): + effective_filters.pop(logical_key, None) + for fk in list(filters.keys()): + if fk not in ("AND", "OR", "NOT") and fk in effective_filters and isinstance(filters[fk], dict): + effective_filters.pop(fk, None) effective_filters.update(processed_filters) elif filters: # Simple filters, merge directly @@ -2447,7 +2461,7 @@ class AsyncMemory(MemoryBase): additional_metadata = {k: v for k, v in payload.items() if k not in core_and_promoted_keys} if additional_metadata: - if "metadata" not in memory_item_dict: + if not memory_item_dict.get("metadata"): memory_item_dict["metadata"] = {} memory_item_dict["metadata"].update(additional_metadata) diff --git a/mem0/vector_stores/milvus.py b/mem0/vector_stores/milvus.py index 711634688..5b4b62d16 100644 --- a/mem0/vector_stores/milvus.py +++ b/mem0/vector_stores/milvus.py @@ -126,21 +126,74 @@ class MilvusDB(VectorStoreBase): def _create_filter(self, filters: dict): """Prepare filters for efficient query. + Supports simple equality, comparison operators (ne, in, nin, contains, + icontains, gt, gte, lt, lte), and logical combinators ($or, $not). + Args: - filters (dict): filters [user_id, agent_id, run_id] + filters (dict): filters dict produced by Memory._process_metadata_filters. Returns: - str: formated filter. + str: Milvus-compatible boolean expression. """ + return self._build_expr(filters) + + def _build_expr(self, filters: dict) -> str: + """Recursively build a Milvus filter expression string.""" operands = [] for key, value in filters.items(): - if isinstance(value, str): + if key == "$or": + or_parts = [self._build_expr(cond) for cond in value] + operands.append("(" + " or ".join(or_parts) + ")") + elif key == "$not": + not_parts = [self._build_expr(cond) for cond in value] + for part in not_parts: + operands.append(f"(not {part})") + elif isinstance(value, dict): + # Operator-based condition, e.g. {"ne": "work"} + for op, operand in value.items(): + operands.append(self._op_to_expr(key, op, operand)) + elif isinstance(value, str): operands.append(f'(metadata["{key}"] == "{value}")') else: operands.append(f'(metadata["{key}"] == {value})') - return " and ".join(operands) + @staticmethod + def _op_to_expr(key: str, op: str, value) -> str: + """Convert a single operator condition to a Milvus expression.""" + field = f'metadata["{key}"]' + if op == "eq": + if isinstance(value, str): + return f'({field} == "{value}")' + return f"({field} == {value})" + elif op == "ne": + if isinstance(value, str): + return f'({field} != "{value}")' + return f"({field} != {value})" + elif op == "gt": + return f"({field} > {value})" + elif op == "gte": + return f"({field} >= {value})" + elif op == "lt": + return f"({field} < {value})" + elif op == "lte": + return f"({field} <= {value})" + elif op == "in": + formatted = [f'"{v}"' if isinstance(v, str) else str(v) for v in value] + return f'({field} in [{", ".join(formatted)}])' + elif op == "nin": + formatted = [f'"{v}"' if isinstance(v, str) else str(v) for v in value] + return f'({field} not in [{", ".join(formatted)}])' + elif op == "contains": + # Milvus string LIKE is case-sensitive: field like "%value%" + return f'({field} like "%{value}%")' + elif op == "icontains": + # Milvus does not support lower() in filter expressions. + # Fall back to case-sensitive LIKE which matches substring presence. + return f'({field} like "%{value}%")' + else: + raise ValueError(f"Unsupported Milvus filter operator: {op}") + def _parse_output(self, data: list): """ Parse the output data. @@ -182,6 +235,7 @@ class MilvusDB(VectorStoreBase): hits = self.client.search( collection_name=self.collection_name, data=[vectors], + anns_field="vectors", limit=limit, filter=query_filter, output_fields=["*"], diff --git a/mem0/vector_stores/qdrant.py b/mem0/vector_stores/qdrant.py index fa4d56403..583bc76b7 100644 --- a/mem0/vector_stores/qdrant.py +++ b/mem0/vector_stores/qdrant.py @@ -7,6 +7,9 @@ from qdrant_client.models import ( Distance, FieldCondition, Filter, + MatchAny, + MatchExcept, + MatchText, MatchValue, PointIdsList, PointStruct, @@ -190,6 +193,14 @@ class Qdrant(VectorStoreBase): """ Create a Filter object from the provided filters. + Supports advanced operators produced by ``_process_metadata_filters``: + - Simple equality: ``{"key": "value"}`` + - Operator dicts: ``{"key": {"eq": …, "ne": …, "in": …, "nin": …, + "gt": …, "gte": …, "lt": …, "lte": …, + "contains": …, "icontains": …}}`` + - Range shorthand: ``{"key": {"gte": …, "lte": …}}`` + - Logical: ``{"$or": [...]}, {"$not": [...]}`` + Args: filters (dict): Filters to apply. @@ -199,13 +210,60 @@ class Qdrant(VectorStoreBase): if not filters: return None - conditions = [] + must = [] + should = [] + must_not = [] + for key, value in filters.items(): - if isinstance(value, dict) and "gte" in value and "lte" in value: - conditions.append(FieldCondition(key=key, range=Range(gte=value["gte"], lte=value["lte"]))) - else: - conditions.append(FieldCondition(key=key, match=MatchValue(value=value))) - return Filter(must=conditions) if conditions else None + # ── Logical combinators ────────────────────────────── + if key == "$or": + for sub_filter in value: + inner = self._create_filter(sub_filter) + if inner and inner.must: + should.extend(inner.must) + continue + if key == "$not": + for sub_filter in value: + inner = self._create_filter(sub_filter) + if inner and inner.must: + must_not.extend(inner.must) + continue + + # ── Operator dict ──────────────────────────────────── + if isinstance(value, dict): + # Range shorthand: {"gte": …, "lte": …} + if "gte" in value and "lte" in value and len(value) == 2: + must.append(FieldCondition(key=key, range=Range(gte=value["gte"], lte=value["lte"]))) + continue + + for op, operand in value.items(): + if op == "eq": + must.append(FieldCondition(key=key, match=MatchValue(value=operand))) + elif op == "ne": + must_not.append(FieldCondition(key=key, match=MatchValue(value=operand))) + elif op == "in": + must.append(FieldCondition(key=key, match=MatchAny(any=operand))) + elif op == "nin": + must.append(FieldCondition(key=key, match=MatchExcept(**{"except": operand}))) + elif op in ("gt", "gte", "lt", "lte"): + must.append(FieldCondition(key=key, range=Range(**{op: operand}))) + elif op in ("contains", "icontains"): + must.append(FieldCondition(key=key, match=MatchText(text=operand))) + else: + # Unknown operator — treat as simple equality on the + # whole dict value (backward-compatible fallback). + must.append(FieldCondition(key=key, match=MatchValue(value=value))) + break # only add once for the whole dict + continue + + # ── Simple equality ─────────────────────────────────── + must.append(FieldCondition(key=key, match=MatchValue(value=value))) + + return Filter( + must=must or None, + should=should or None, + must_not=must_not or None, + ) def search(self, query: str, vectors: list, limit: int = 5, filters: dict = None) -> list: """ diff --git a/mem0/vector_stores/redis.py b/mem0/vector_stores/redis.py index 770bf6a8e..20498f6de 100644 --- a/mem0/vector_stores/redis.py +++ b/mem0/vector_stores/redis.py @@ -159,7 +159,7 @@ class RedisDB(VectorStoreBase): return [ MemoryResult( id=result["memory_id"], - score=result["vector_distance"], + score=float(result["vector_distance"]), payload={ "hash": result["hash"], "data": result["memory"],