fix: filters did not support AND, OR and NOT in Qdant and other vector stores (#4780)

Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Saket Aryan
2026-04-10 23:43:12 +05:30
committed by GitHub
co-authored by Claude Opus 4.6
parent 90f9e6a550
commit 398445c691
5 changed files with 142 additions and 13 deletions
+3
View File
@@ -32,6 +32,7 @@ import { LangchainLLM } from "../llms/langchain";
import { LangchainEmbedder } from "../embeddings/langchain";
import { LangchainVectorStore } from "../vector_stores/langchain";
import { AzureAISearch } from "../vector_stores/azure_ai_search";
import { PGVector } from "../vector_stores/pgvector";
export class EmbedderFactory {
static create(provider: string, config: EmbeddingConfig): Embedder {
@@ -98,6 +99,8 @@ export class VectorStoreFactory {
return new VectorizeDB(config as any);
case "azure-ai-search":
return new AzureAISearch(config as any);
case "pgvector":
return new PGVector(config as any);
default:
throw new Error(`Unsupported vector store provider: ${provider}`);
}
+16 -2
View File
@@ -997,6 +997,13 @@ class Memory(MemoryBase):
# Apply enhanced metadata filtering if advanced operators are detected
if filters and self._has_advanced_operators(filters):
processed_filters = self._process_metadata_filters(filters)
# Remove original logical/operator keys that _build_filters_and_metadata
# copied verbatim from input_filters — they have now been reprocessed.
for logical_key in ("AND", "OR", "NOT"):
effective_filters.pop(logical_key, None)
for fk in list(filters.keys()):
if fk not in ("AND", "OR", "NOT") and fk in effective_filters and isinstance(filters[fk], dict):
effective_filters.pop(fk, None)
effective_filters.update(processed_filters)
elif filters:
# Simple filters, merge directly
@@ -1237,7 +1244,7 @@ class Memory(MemoryBase):
additional_metadata = {k: v for k, v in payload.items() if k not in core_and_promoted_keys}
if additional_metadata:
if "metadata" not in memory_item_dict:
if not memory_item_dict.get("metadata"):
memory_item_dict["metadata"] = {}
memory_item_dict["metadata"].update(additional_metadata)
@@ -2207,6 +2214,13 @@ class AsyncMemory(MemoryBase):
# Apply enhanced metadata filtering if advanced operators are detected
if filters and self._has_advanced_operators(filters):
processed_filters = self._process_metadata_filters(filters)
# Remove original logical/operator keys that _build_filters_and_metadata
# copied verbatim from input_filters — they have now been reprocessed.
for logical_key in ("AND", "OR", "NOT"):
effective_filters.pop(logical_key, None)
for fk in list(filters.keys()):
if fk not in ("AND", "OR", "NOT") and fk in effective_filters and isinstance(filters[fk], dict):
effective_filters.pop(fk, None)
effective_filters.update(processed_filters)
elif filters:
# Simple filters, merge directly
@@ -2447,7 +2461,7 @@ class AsyncMemory(MemoryBase):
additional_metadata = {k: v for k, v in payload.items() if k not in core_and_promoted_keys}
if additional_metadata:
if "metadata" not in memory_item_dict:
if not memory_item_dict.get("metadata"):
memory_item_dict["metadata"] = {}
memory_item_dict["metadata"].update(additional_metadata)
+58 -4
View File
@@ -126,21 +126,74 @@ class MilvusDB(VectorStoreBase):
def _create_filter(self, filters: dict):
"""Prepare filters for efficient query.
Supports simple equality, comparison operators (ne, in, nin, contains,
icontains, gt, gte, lt, lte), and logical combinators ($or, $not).
Args:
filters (dict): filters [user_id, agent_id, run_id]
filters (dict): filters dict produced by Memory._process_metadata_filters.
Returns:
str: formated filter.
str: Milvus-compatible boolean expression.
"""
return self._build_expr(filters)
def _build_expr(self, filters: dict) -> str:
"""Recursively build a Milvus filter expression string."""
operands = []
for key, value in filters.items():
if isinstance(value, str):
if key == "$or":
or_parts = [self._build_expr(cond) for cond in value]
operands.append("(" + " or ".join(or_parts) + ")")
elif key == "$not":
not_parts = [self._build_expr(cond) for cond in value]
for part in not_parts:
operands.append(f"(not {part})")
elif isinstance(value, dict):
# Operator-based condition, e.g. {"ne": "work"}
for op, operand in value.items():
operands.append(self._op_to_expr(key, op, operand))
elif isinstance(value, str):
operands.append(f'(metadata["{key}"] == "{value}")')
else:
operands.append(f'(metadata["{key}"] == {value})')
return " and ".join(operands)
@staticmethod
def _op_to_expr(key: str, op: str, value) -> str:
"""Convert a single operator condition to a Milvus expression."""
field = f'metadata["{key}"]'
if op == "eq":
if isinstance(value, str):
return f'({field} == "{value}")'
return f"({field} == {value})"
elif op == "ne":
if isinstance(value, str):
return f'({field} != "{value}")'
return f"({field} != {value})"
elif op == "gt":
return f"({field} > {value})"
elif op == "gte":
return f"({field} >= {value})"
elif op == "lt":
return f"({field} < {value})"
elif op == "lte":
return f"({field} <= {value})"
elif op == "in":
formatted = [f'"{v}"' if isinstance(v, str) else str(v) for v in value]
return f'({field} in [{", ".join(formatted)}])'
elif op == "nin":
formatted = [f'"{v}"' if isinstance(v, str) else str(v) for v in value]
return f'({field} not in [{", ".join(formatted)}])'
elif op == "contains":
# Milvus string LIKE is case-sensitive: field like "%value%"
return f'({field} like "%{value}%")'
elif op == "icontains":
# Milvus does not support lower() in filter expressions.
# Fall back to case-sensitive LIKE which matches substring presence.
return f'({field} like "%{value}%")'
else:
raise ValueError(f"Unsupported Milvus filter operator: {op}")
def _parse_output(self, data: list):
"""
Parse the output data.
@@ -182,6 +235,7 @@ class MilvusDB(VectorStoreBase):
hits = self.client.search(
collection_name=self.collection_name,
data=[vectors],
anns_field="vectors",
limit=limit,
filter=query_filter,
output_fields=["*"],
+64 -6
View File
@@ -7,6 +7,9 @@ from qdrant_client.models import (
Distance,
FieldCondition,
Filter,
MatchAny,
MatchExcept,
MatchText,
MatchValue,
PointIdsList,
PointStruct,
@@ -190,6 +193,14 @@ class Qdrant(VectorStoreBase):
"""
Create a Filter object from the provided filters.
Supports advanced operators produced by ``_process_metadata_filters``:
- Simple equality: ``{"key": "value"}``
- Operator dicts: ``{"key": {"eq": …, "ne": …, "in": …, "nin": …,
"gt": …, "gte": …, "lt": …, "lte": …,
"contains": …, "icontains": …}}``
- Range shorthand: ``{"key": {"gte": …, "lte": …}}``
- Logical: ``{"$or": [...]}, {"$not": [...]}``
Args:
filters (dict): Filters to apply.
@@ -199,13 +210,60 @@ class Qdrant(VectorStoreBase):
if not filters:
return None
conditions = []
must = []
should = []
must_not = []
for key, value in filters.items():
if isinstance(value, dict) and "gte" in value and "lte" in value:
conditions.append(FieldCondition(key=key, range=Range(gte=value["gte"], lte=value["lte"])))
else:
conditions.append(FieldCondition(key=key, match=MatchValue(value=value)))
return Filter(must=conditions) if conditions else None
# ── Logical combinators ──────────────────────────────
if key == "$or":
for sub_filter in value:
inner = self._create_filter(sub_filter)
if inner and inner.must:
should.extend(inner.must)
continue
if key == "$not":
for sub_filter in value:
inner = self._create_filter(sub_filter)
if inner and inner.must:
must_not.extend(inner.must)
continue
# ── Operator dict ────────────────────────────────────
if isinstance(value, dict):
# Range shorthand: {"gte": …, "lte": …}
if "gte" in value and "lte" in value and len(value) == 2:
must.append(FieldCondition(key=key, range=Range(gte=value["gte"], lte=value["lte"])))
continue
for op, operand in value.items():
if op == "eq":
must.append(FieldCondition(key=key, match=MatchValue(value=operand)))
elif op == "ne":
must_not.append(FieldCondition(key=key, match=MatchValue(value=operand)))
elif op == "in":
must.append(FieldCondition(key=key, match=MatchAny(any=operand)))
elif op == "nin":
must.append(FieldCondition(key=key, match=MatchExcept(**{"except": operand})))
elif op in ("gt", "gte", "lt", "lte"):
must.append(FieldCondition(key=key, range=Range(**{op: operand})))
elif op in ("contains", "icontains"):
must.append(FieldCondition(key=key, match=MatchText(text=operand)))
else:
# Unknown operator — treat as simple equality on the
# whole dict value (backward-compatible fallback).
must.append(FieldCondition(key=key, match=MatchValue(value=value)))
break # only add once for the whole dict
continue
# ── Simple equality ───────────────────────────────────
must.append(FieldCondition(key=key, match=MatchValue(value=value)))
return Filter(
must=must or None,
should=should or None,
must_not=must_not or None,
)
def search(self, query: str, vectors: list, limit: int = 5, filters: dict = None) -> list:
"""
+1 -1
View File
@@ -159,7 +159,7 @@ class RedisDB(VectorStoreBase):
return [
MemoryResult(
id=result["memory_id"],
score=result["vector_distance"],
score=float(result["vector_distance"]),
payload={
"hash": result["hash"],
"data": result["memory"],