fix: filters did not support AND, OR and NOT in Qdant and other vector stores (#4780)
Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.6
parent
90f9e6a550
commit
398445c691
@@ -32,6 +32,7 @@ import { LangchainLLM } from "../llms/langchain";
|
||||
import { LangchainEmbedder } from "../embeddings/langchain";
|
||||
import { LangchainVectorStore } from "../vector_stores/langchain";
|
||||
import { AzureAISearch } from "../vector_stores/azure_ai_search";
|
||||
import { PGVector } from "../vector_stores/pgvector";
|
||||
|
||||
export class EmbedderFactory {
|
||||
static create(provider: string, config: EmbeddingConfig): Embedder {
|
||||
@@ -98,6 +99,8 @@ export class VectorStoreFactory {
|
||||
return new VectorizeDB(config as any);
|
||||
case "azure-ai-search":
|
||||
return new AzureAISearch(config as any);
|
||||
case "pgvector":
|
||||
return new PGVector(config as any);
|
||||
default:
|
||||
throw new Error(`Unsupported vector store provider: ${provider}`);
|
||||
}
|
||||
|
||||
+16
-2
@@ -997,6 +997,13 @@ class Memory(MemoryBase):
|
||||
# Apply enhanced metadata filtering if advanced operators are detected
|
||||
if filters and self._has_advanced_operators(filters):
|
||||
processed_filters = self._process_metadata_filters(filters)
|
||||
# Remove original logical/operator keys that _build_filters_and_metadata
|
||||
# copied verbatim from input_filters — they have now been reprocessed.
|
||||
for logical_key in ("AND", "OR", "NOT"):
|
||||
effective_filters.pop(logical_key, None)
|
||||
for fk in list(filters.keys()):
|
||||
if fk not in ("AND", "OR", "NOT") and fk in effective_filters and isinstance(filters[fk], dict):
|
||||
effective_filters.pop(fk, None)
|
||||
effective_filters.update(processed_filters)
|
||||
elif filters:
|
||||
# Simple filters, merge directly
|
||||
@@ -1237,7 +1244,7 @@ class Memory(MemoryBase):
|
||||
|
||||
additional_metadata = {k: v for k, v in payload.items() if k not in core_and_promoted_keys}
|
||||
if additional_metadata:
|
||||
if "metadata" not in memory_item_dict:
|
||||
if not memory_item_dict.get("metadata"):
|
||||
memory_item_dict["metadata"] = {}
|
||||
memory_item_dict["metadata"].update(additional_metadata)
|
||||
|
||||
@@ -2207,6 +2214,13 @@ class AsyncMemory(MemoryBase):
|
||||
# Apply enhanced metadata filtering if advanced operators are detected
|
||||
if filters and self._has_advanced_operators(filters):
|
||||
processed_filters = self._process_metadata_filters(filters)
|
||||
# Remove original logical/operator keys that _build_filters_and_metadata
|
||||
# copied verbatim from input_filters — they have now been reprocessed.
|
||||
for logical_key in ("AND", "OR", "NOT"):
|
||||
effective_filters.pop(logical_key, None)
|
||||
for fk in list(filters.keys()):
|
||||
if fk not in ("AND", "OR", "NOT") and fk in effective_filters and isinstance(filters[fk], dict):
|
||||
effective_filters.pop(fk, None)
|
||||
effective_filters.update(processed_filters)
|
||||
elif filters:
|
||||
# Simple filters, merge directly
|
||||
@@ -2447,7 +2461,7 @@ class AsyncMemory(MemoryBase):
|
||||
|
||||
additional_metadata = {k: v for k, v in payload.items() if k not in core_and_promoted_keys}
|
||||
if additional_metadata:
|
||||
if "metadata" not in memory_item_dict:
|
||||
if not memory_item_dict.get("metadata"):
|
||||
memory_item_dict["metadata"] = {}
|
||||
memory_item_dict["metadata"].update(additional_metadata)
|
||||
|
||||
|
||||
@@ -126,21 +126,74 @@ class MilvusDB(VectorStoreBase):
|
||||
def _create_filter(self, filters: dict):
|
||||
"""Prepare filters for efficient query.
|
||||
|
||||
Supports simple equality, comparison operators (ne, in, nin, contains,
|
||||
icontains, gt, gte, lt, lte), and logical combinators ($or, $not).
|
||||
|
||||
Args:
|
||||
filters (dict): filters [user_id, agent_id, run_id]
|
||||
filters (dict): filters dict produced by Memory._process_metadata_filters.
|
||||
|
||||
Returns:
|
||||
str: formated filter.
|
||||
str: Milvus-compatible boolean expression.
|
||||
"""
|
||||
return self._build_expr(filters)
|
||||
|
||||
def _build_expr(self, filters: dict) -> str:
|
||||
"""Recursively build a Milvus filter expression string."""
|
||||
operands = []
|
||||
for key, value in filters.items():
|
||||
if isinstance(value, str):
|
||||
if key == "$or":
|
||||
or_parts = [self._build_expr(cond) for cond in value]
|
||||
operands.append("(" + " or ".join(or_parts) + ")")
|
||||
elif key == "$not":
|
||||
not_parts = [self._build_expr(cond) for cond in value]
|
||||
for part in not_parts:
|
||||
operands.append(f"(not {part})")
|
||||
elif isinstance(value, dict):
|
||||
# Operator-based condition, e.g. {"ne": "work"}
|
||||
for op, operand in value.items():
|
||||
operands.append(self._op_to_expr(key, op, operand))
|
||||
elif isinstance(value, str):
|
||||
operands.append(f'(metadata["{key}"] == "{value}")')
|
||||
else:
|
||||
operands.append(f'(metadata["{key}"] == {value})')
|
||||
|
||||
return " and ".join(operands)
|
||||
|
||||
@staticmethod
|
||||
def _op_to_expr(key: str, op: str, value) -> str:
|
||||
"""Convert a single operator condition to a Milvus expression."""
|
||||
field = f'metadata["{key}"]'
|
||||
if op == "eq":
|
||||
if isinstance(value, str):
|
||||
return f'({field} == "{value}")'
|
||||
return f"({field} == {value})"
|
||||
elif op == "ne":
|
||||
if isinstance(value, str):
|
||||
return f'({field} != "{value}")'
|
||||
return f"({field} != {value})"
|
||||
elif op == "gt":
|
||||
return f"({field} > {value})"
|
||||
elif op == "gte":
|
||||
return f"({field} >= {value})"
|
||||
elif op == "lt":
|
||||
return f"({field} < {value})"
|
||||
elif op == "lte":
|
||||
return f"({field} <= {value})"
|
||||
elif op == "in":
|
||||
formatted = [f'"{v}"' if isinstance(v, str) else str(v) for v in value]
|
||||
return f'({field} in [{", ".join(formatted)}])'
|
||||
elif op == "nin":
|
||||
formatted = [f'"{v}"' if isinstance(v, str) else str(v) for v in value]
|
||||
return f'({field} not in [{", ".join(formatted)}])'
|
||||
elif op == "contains":
|
||||
# Milvus string LIKE is case-sensitive: field like "%value%"
|
||||
return f'({field} like "%{value}%")'
|
||||
elif op == "icontains":
|
||||
# Milvus does not support lower() in filter expressions.
|
||||
# Fall back to case-sensitive LIKE which matches substring presence.
|
||||
return f'({field} like "%{value}%")'
|
||||
else:
|
||||
raise ValueError(f"Unsupported Milvus filter operator: {op}")
|
||||
|
||||
def _parse_output(self, data: list):
|
||||
"""
|
||||
Parse the output data.
|
||||
@@ -182,6 +235,7 @@ class MilvusDB(VectorStoreBase):
|
||||
hits = self.client.search(
|
||||
collection_name=self.collection_name,
|
||||
data=[vectors],
|
||||
anns_field="vectors",
|
||||
limit=limit,
|
||||
filter=query_filter,
|
||||
output_fields=["*"],
|
||||
|
||||
@@ -7,6 +7,9 @@ from qdrant_client.models import (
|
||||
Distance,
|
||||
FieldCondition,
|
||||
Filter,
|
||||
MatchAny,
|
||||
MatchExcept,
|
||||
MatchText,
|
||||
MatchValue,
|
||||
PointIdsList,
|
||||
PointStruct,
|
||||
@@ -190,6 +193,14 @@ class Qdrant(VectorStoreBase):
|
||||
"""
|
||||
Create a Filter object from the provided filters.
|
||||
|
||||
Supports advanced operators produced by ``_process_metadata_filters``:
|
||||
- Simple equality: ``{"key": "value"}``
|
||||
- Operator dicts: ``{"key": {"eq": …, "ne": …, "in": …, "nin": …,
|
||||
"gt": …, "gte": …, "lt": …, "lte": …,
|
||||
"contains": …, "icontains": …}}``
|
||||
- Range shorthand: ``{"key": {"gte": …, "lte": …}}``
|
||||
- Logical: ``{"$or": [...]}, {"$not": [...]}``
|
||||
|
||||
Args:
|
||||
filters (dict): Filters to apply.
|
||||
|
||||
@@ -199,13 +210,60 @@ class Qdrant(VectorStoreBase):
|
||||
if not filters:
|
||||
return None
|
||||
|
||||
conditions = []
|
||||
must = []
|
||||
should = []
|
||||
must_not = []
|
||||
|
||||
for key, value in filters.items():
|
||||
if isinstance(value, dict) and "gte" in value and "lte" in value:
|
||||
conditions.append(FieldCondition(key=key, range=Range(gte=value["gte"], lte=value["lte"])))
|
||||
else:
|
||||
conditions.append(FieldCondition(key=key, match=MatchValue(value=value)))
|
||||
return Filter(must=conditions) if conditions else None
|
||||
# ── Logical combinators ──────────────────────────────
|
||||
if key == "$or":
|
||||
for sub_filter in value:
|
||||
inner = self._create_filter(sub_filter)
|
||||
if inner and inner.must:
|
||||
should.extend(inner.must)
|
||||
continue
|
||||
if key == "$not":
|
||||
for sub_filter in value:
|
||||
inner = self._create_filter(sub_filter)
|
||||
if inner and inner.must:
|
||||
must_not.extend(inner.must)
|
||||
continue
|
||||
|
||||
# ── Operator dict ────────────────────────────────────
|
||||
if isinstance(value, dict):
|
||||
# Range shorthand: {"gte": …, "lte": …}
|
||||
if "gte" in value and "lte" in value and len(value) == 2:
|
||||
must.append(FieldCondition(key=key, range=Range(gte=value["gte"], lte=value["lte"])))
|
||||
continue
|
||||
|
||||
for op, operand in value.items():
|
||||
if op == "eq":
|
||||
must.append(FieldCondition(key=key, match=MatchValue(value=operand)))
|
||||
elif op == "ne":
|
||||
must_not.append(FieldCondition(key=key, match=MatchValue(value=operand)))
|
||||
elif op == "in":
|
||||
must.append(FieldCondition(key=key, match=MatchAny(any=operand)))
|
||||
elif op == "nin":
|
||||
must.append(FieldCondition(key=key, match=MatchExcept(**{"except": operand})))
|
||||
elif op in ("gt", "gte", "lt", "lte"):
|
||||
must.append(FieldCondition(key=key, range=Range(**{op: operand})))
|
||||
elif op in ("contains", "icontains"):
|
||||
must.append(FieldCondition(key=key, match=MatchText(text=operand)))
|
||||
else:
|
||||
# Unknown operator — treat as simple equality on the
|
||||
# whole dict value (backward-compatible fallback).
|
||||
must.append(FieldCondition(key=key, match=MatchValue(value=value)))
|
||||
break # only add once for the whole dict
|
||||
continue
|
||||
|
||||
# ── Simple equality ───────────────────────────────────
|
||||
must.append(FieldCondition(key=key, match=MatchValue(value=value)))
|
||||
|
||||
return Filter(
|
||||
must=must or None,
|
||||
should=should or None,
|
||||
must_not=must_not or None,
|
||||
)
|
||||
|
||||
def search(self, query: str, vectors: list, limit: int = 5, filters: dict = None) -> list:
|
||||
"""
|
||||
|
||||
@@ -159,7 +159,7 @@ class RedisDB(VectorStoreBase):
|
||||
return [
|
||||
MemoryResult(
|
||||
id=result["memory_id"],
|
||||
score=result["vector_distance"],
|
||||
score=float(result["vector_distance"]),
|
||||
payload={
|
||||
"hash": result["hash"],
|
||||
"data": result["memory"],
|
||||
|
||||
Reference in New Issue
Block a user