fix(valkey): escape special chars in FT.SEARCH tag filter values (#5750)
This commit is contained in:
@@ -41,7 +41,20 @@ class OutputData(BaseModel):
|
|||||||
payload: Dict
|
payload: Dict
|
||||||
|
|
||||||
|
|
||||||
|
_VALKEY_TAG_SPECIAL = set(r',.<>{}[]"\':;!@#$%^&*()-+=~| ')
|
||||||
|
|
||||||
|
|
||||||
class ValkeyDB(VectorStoreBase):
|
class ValkeyDB(VectorStoreBase):
|
||||||
|
@staticmethod
|
||||||
|
def _escape_tag_value(value):
|
||||||
|
"""Escape special characters in a Valkey FT.SEARCH tag filter value.
|
||||||
|
|
||||||
|
Without escaping, characters like * (wildcard) or | (OR) alter query
|
||||||
|
semantics and can bypass tenant-isolation filters.
|
||||||
|
"""
|
||||||
|
s = str(value)
|
||||||
|
return "".join(f"\\{c}" if c in _VALKEY_TAG_SPECIAL else c for c in s)
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
valkey_url: str,
|
valkey_url: str,
|
||||||
@@ -329,8 +342,8 @@ class ValkeyDB(VectorStoreBase):
|
|||||||
knn_part (str): The KNN part of the query.
|
knn_part (str): The KNN part of the query.
|
||||||
filters (dict, optional): Filters to apply to the search. Each key-value pair
|
filters (dict, optional): Filters to apply to the search. Each key-value pair
|
||||||
becomes a tag filter (@key:{value}). None values are ignored.
|
becomes a tag filter (@key:{value}). None values are ignored.
|
||||||
Values are used as-is (no validation) - wildcards, lists, etc. are
|
Values are escaped via _escape_tag_value() before interpolation
|
||||||
passed through literally to Valkey search. Multiple filters are
|
to prevent wildcard/operator injection. Multiple filters are
|
||||||
combined with AND logic (space-separated).
|
combined with AND logic (space-separated).
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
@@ -345,8 +358,8 @@ class ValkeyDB(VectorStoreBase):
|
|||||||
filter_parts = []
|
filter_parts = []
|
||||||
for key, value in filters.items():
|
for key, value in filters.items():
|
||||||
if value is not None:
|
if value is not None:
|
||||||
# Use the correct filter syntax for Valkey
|
escaped = self._escape_tag_value(value)
|
||||||
filter_parts.append(f"@{key}:{{{value}}}")
|
filter_parts.append(f"@{key}:{{{escaped}}}")
|
||||||
|
|
||||||
# No valid filter parts
|
# No valid filter parts
|
||||||
if not filter_parts:
|
if not filter_parts:
|
||||||
@@ -750,34 +763,6 @@ class ValkeyDB(VectorStoreBase):
|
|||||||
logger.exception(f"Error resetting index {self.collection_name}: {e}")
|
logger.exception(f"Error resetting index {self.collection_name}: {e}")
|
||||||
raise
|
raise
|
||||||
|
|
||||||
def _build_list_query(self, filters=None):
|
|
||||||
"""
|
|
||||||
Build a query for listing vectors.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
filters (dict, optional): Filters to apply to the list. Each key-value pair
|
|
||||||
becomes a tag filter (@key:{value}). None values are ignored.
|
|
||||||
Values are used as-is (no validation) - wildcards, lists, etc. are
|
|
||||||
passed through literally to Valkey search.
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
str: The query string. Returns "*" if no valid filters provided.
|
|
||||||
"""
|
|
||||||
# Default query
|
|
||||||
q = "*"
|
|
||||||
|
|
||||||
# Add filters if provided
|
|
||||||
if filters and any(value is not None for key, value in filters.items()):
|
|
||||||
filter_conditions = []
|
|
||||||
for key, value in filters.items():
|
|
||||||
if value is not None:
|
|
||||||
filter_conditions.append(f"@{key}:{{{value}}}")
|
|
||||||
|
|
||||||
if filter_conditions:
|
|
||||||
q = " ".join(filter_conditions)
|
|
||||||
|
|
||||||
return q
|
|
||||||
|
|
||||||
def list(self, filters: dict = None, top_k: int = None) -> list:
|
def list(self, filters: dict = None, top_k: int = None) -> list:
|
||||||
"""
|
"""
|
||||||
List all recent created memories from the vector store.
|
List all recent created memories from the vector store.
|
||||||
@@ -785,8 +770,8 @@ class ValkeyDB(VectorStoreBase):
|
|||||||
Args:
|
Args:
|
||||||
filters (dict, optional): Filters to apply to the list. Each key-value pair
|
filters (dict, optional): Filters to apply to the list. Each key-value pair
|
||||||
becomes a tag filter (@key:{value}). None values are ignored.
|
becomes a tag filter (@key:{value}). None values are ignored.
|
||||||
Values are used as-is without validation - wildcards, special characters,
|
Values are escaped via _escape_tag_value() before interpolation
|
||||||
lists, etc. are passed through literally to Valkey search.
|
to prevent wildcard/operator injection.
|
||||||
Multiple filters are combined with AND logic.
|
Multiple filters are combined with AND logic.
|
||||||
top_k (int, optional): Maximum number of results to return. Defaults to 1000
|
top_k (int, optional): Maximum number of results to return. Defaults to 1000
|
||||||
if not specified.
|
if not specified.
|
||||||
|
|||||||
@@ -490,26 +490,6 @@ def test_reset(valkey_db, mock_valkey_client):
|
|||||||
assert result is True
|
assert result is True
|
||||||
|
|
||||||
|
|
||||||
def test_build_list_query(valkey_db):
|
|
||||||
"""Test building a list query with and without filters."""
|
|
||||||
# Test without filters
|
|
||||||
query = valkey_db._build_list_query(None)
|
|
||||||
assert query == "*"
|
|
||||||
|
|
||||||
# Test with empty filters
|
|
||||||
query = valkey_db._build_list_query({})
|
|
||||||
assert query == "*"
|
|
||||||
|
|
||||||
# Test with filters
|
|
||||||
query = valkey_db._build_list_query({"user_id": "test_user"})
|
|
||||||
assert query == "@user_id:{test_user}"
|
|
||||||
|
|
||||||
# Test with multiple filters
|
|
||||||
query = valkey_db._build_list_query({"user_id": "test_user", "agent_id": "test_agent"})
|
|
||||||
assert "@user_id:{test_user}" in query
|
|
||||||
assert "@agent_id:{test_agent}" in query
|
|
||||||
|
|
||||||
|
|
||||||
def test_process_document_fields(valkey_db):
|
def test_process_document_fields(valkey_db):
|
||||||
"""Test processing document fields from hash results."""
|
"""Test processing document fields from hash results."""
|
||||||
# Create a mock result with all fields
|
# Create a mock result with all fields
|
||||||
@@ -1061,3 +1041,28 @@ def test_build_index_schema_indexes_memory_as_text(valkey_db):
|
|||||||
)
|
)
|
||||||
# And it must not be declared as TAG.
|
# And it must not be declared as TAG.
|
||||||
assert ["memory", "TAG"] != cmd[memory_idx : memory_idx + 2]
|
assert ["memory", "TAG"] != cmd[memory_idx : memory_idx + 2]
|
||||||
|
|
||||||
|
|
||||||
|
def test_escape_tag_value_wildcards(valkey_db):
|
||||||
|
"""Wildcard characters in filter values must be escaped to prevent query injection."""
|
||||||
|
assert "\\*" in valkey_db._escape_tag_value("*")
|
||||||
|
assert "\\|" in valkey_db._escape_tag_value("a|b")
|
||||||
|
|
||||||
|
|
||||||
|
def test_build_search_query_escapes_filter_values(valkey_db):
|
||||||
|
"""_build_search_query must escape special chars in filter values."""
|
||||||
|
knn_part = "[KNN 5 @embedding $vec_param AS vector_score]"
|
||||||
|
query = valkey_db._build_search_query(knn_part, {"user_id": "*"})
|
||||||
|
assert "\\*" in query
|
||||||
|
assert "@user_id:{\\*}" in query
|
||||||
|
|
||||||
|
|
||||||
|
def test_escape_tag_value_normal_strings(valkey_db):
|
||||||
|
"""Normal alphanumeric filter values must pass through unchanged."""
|
||||||
|
assert valkey_db._escape_tag_value("alice") == "alice"
|
||||||
|
assert valkey_db._escape_tag_value("user123") == "user123"
|
||||||
|
|
||||||
|
|
||||||
|
def test_escape_tag_value_hyphenated_user_id(valkey_db):
|
||||||
|
"""Hyphenated user IDs must have the hyphen escaped for exact-match."""
|
||||||
|
assert valkey_db._escape_tag_value("user-123") == r"user\-123"
|
||||||
|
|||||||
Reference in New Issue
Block a user