fix(valkey): escape special chars in FT.SEARCH tag filter values (#5750)
This commit is contained in:
@@ -41,7 +41,20 @@ class OutputData(BaseModel):
|
||||
payload: Dict
|
||||
|
||||
|
||||
_VALKEY_TAG_SPECIAL = set(r',.<>{}[]"\':;!@#$%^&*()-+=~| ')
|
||||
|
||||
|
||||
class ValkeyDB(VectorStoreBase):
|
||||
@staticmethod
|
||||
def _escape_tag_value(value):
|
||||
"""Escape special characters in a Valkey FT.SEARCH tag filter value.
|
||||
|
||||
Without escaping, characters like * (wildcard) or | (OR) alter query
|
||||
semantics and can bypass tenant-isolation filters.
|
||||
"""
|
||||
s = str(value)
|
||||
return "".join(f"\\{c}" if c in _VALKEY_TAG_SPECIAL else c for c in s)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
valkey_url: str,
|
||||
@@ -329,8 +342,8 @@ class ValkeyDB(VectorStoreBase):
|
||||
knn_part (str): The KNN part of the query.
|
||||
filters (dict, optional): Filters to apply to the search. Each key-value pair
|
||||
becomes a tag filter (@key:{value}). None values are ignored.
|
||||
Values are used as-is (no validation) - wildcards, lists, etc. are
|
||||
passed through literally to Valkey search. Multiple filters are
|
||||
Values are escaped via _escape_tag_value() before interpolation
|
||||
to prevent wildcard/operator injection. Multiple filters are
|
||||
combined with AND logic (space-separated).
|
||||
|
||||
Returns:
|
||||
@@ -345,8 +358,8 @@ class ValkeyDB(VectorStoreBase):
|
||||
filter_parts = []
|
||||
for key, value in filters.items():
|
||||
if value is not None:
|
||||
# Use the correct filter syntax for Valkey
|
||||
filter_parts.append(f"@{key}:{{{value}}}")
|
||||
escaped = self._escape_tag_value(value)
|
||||
filter_parts.append(f"@{key}:{{{escaped}}}")
|
||||
|
||||
# No valid filter parts
|
||||
if not filter_parts:
|
||||
@@ -750,34 +763,6 @@ class ValkeyDB(VectorStoreBase):
|
||||
logger.exception(f"Error resetting index {self.collection_name}: {e}")
|
||||
raise
|
||||
|
||||
def _build_list_query(self, filters=None):
|
||||
"""
|
||||
Build a query for listing vectors.
|
||||
|
||||
Args:
|
||||
filters (dict, optional): Filters to apply to the list. Each key-value pair
|
||||
becomes a tag filter (@key:{value}). None values are ignored.
|
||||
Values are used as-is (no validation) - wildcards, lists, etc. are
|
||||
passed through literally to Valkey search.
|
||||
|
||||
Returns:
|
||||
str: The query string. Returns "*" if no valid filters provided.
|
||||
"""
|
||||
# Default query
|
||||
q = "*"
|
||||
|
||||
# Add filters if provided
|
||||
if filters and any(value is not None for key, value in filters.items()):
|
||||
filter_conditions = []
|
||||
for key, value in filters.items():
|
||||
if value is not None:
|
||||
filter_conditions.append(f"@{key}:{{{value}}}")
|
||||
|
||||
if filter_conditions:
|
||||
q = " ".join(filter_conditions)
|
||||
|
||||
return q
|
||||
|
||||
def list(self, filters: dict = None, top_k: int = None) -> list:
|
||||
"""
|
||||
List all recent created memories from the vector store.
|
||||
@@ -785,8 +770,8 @@ class ValkeyDB(VectorStoreBase):
|
||||
Args:
|
||||
filters (dict, optional): Filters to apply to the list. Each key-value pair
|
||||
becomes a tag filter (@key:{value}). None values are ignored.
|
||||
Values are used as-is without validation - wildcards, special characters,
|
||||
lists, etc. are passed through literally to Valkey search.
|
||||
Values are escaped via _escape_tag_value() before interpolation
|
||||
to prevent wildcard/operator injection.
|
||||
Multiple filters are combined with AND logic.
|
||||
top_k (int, optional): Maximum number of results to return. Defaults to 1000
|
||||
if not specified.
|
||||
|
||||
@@ -490,26 +490,6 @@ def test_reset(valkey_db, mock_valkey_client):
|
||||
assert result is True
|
||||
|
||||
|
||||
def test_build_list_query(valkey_db):
|
||||
"""Test building a list query with and without filters."""
|
||||
# Test without filters
|
||||
query = valkey_db._build_list_query(None)
|
||||
assert query == "*"
|
||||
|
||||
# Test with empty filters
|
||||
query = valkey_db._build_list_query({})
|
||||
assert query == "*"
|
||||
|
||||
# Test with filters
|
||||
query = valkey_db._build_list_query({"user_id": "test_user"})
|
||||
assert query == "@user_id:{test_user}"
|
||||
|
||||
# Test with multiple filters
|
||||
query = valkey_db._build_list_query({"user_id": "test_user", "agent_id": "test_agent"})
|
||||
assert "@user_id:{test_user}" in query
|
||||
assert "@agent_id:{test_agent}" in query
|
||||
|
||||
|
||||
def test_process_document_fields(valkey_db):
|
||||
"""Test processing document fields from hash results."""
|
||||
# Create a mock result with all fields
|
||||
@@ -1061,3 +1041,28 @@ def test_build_index_schema_indexes_memory_as_text(valkey_db):
|
||||
)
|
||||
# And it must not be declared as TAG.
|
||||
assert ["memory", "TAG"] != cmd[memory_idx : memory_idx + 2]
|
||||
|
||||
|
||||
def test_escape_tag_value_wildcards(valkey_db):
|
||||
"""Wildcard characters in filter values must be escaped to prevent query injection."""
|
||||
assert "\\*" in valkey_db._escape_tag_value("*")
|
||||
assert "\\|" in valkey_db._escape_tag_value("a|b")
|
||||
|
||||
|
||||
def test_build_search_query_escapes_filter_values(valkey_db):
|
||||
"""_build_search_query must escape special chars in filter values."""
|
||||
knn_part = "[KNN 5 @embedding $vec_param AS vector_score]"
|
||||
query = valkey_db._build_search_query(knn_part, {"user_id": "*"})
|
||||
assert "\\*" in query
|
||||
assert "@user_id:{\\*}" in query
|
||||
|
||||
|
||||
def test_escape_tag_value_normal_strings(valkey_db):
|
||||
"""Normal alphanumeric filter values must pass through unchanged."""
|
||||
assert valkey_db._escape_tag_value("alice") == "alice"
|
||||
assert valkey_db._escape_tag_value("user123") == "user123"
|
||||
|
||||
|
||||
def test_escape_tag_value_hyphenated_user_id(valkey_db):
|
||||
"""Hyphenated user IDs must have the hyphen escaped for exact-match."""
|
||||
assert valkey_db._escape_tag_value("user-123") == r"user\-123"
|
||||
|
||||
Reference in New Issue
Block a user