fix(tests): update 35 failing tests for v3 single-pass pipeline
Tests were written for the old two-pass extraction pipeline (extract
facts + merge with ADD/UPDATE/DELETE). V3 uses single-pass ADD-only
extraction with one LLM call returning {"memory": [...]}.
Changes by category:
Old pipeline tests (13 failures):
- Delete TestHallucinatedIdGuard entirely (tests UPDATE/DELETE ID
resolution that no longer exists)
- Update error log assertions to match v3 log messages
- Change LLM mock from 2-call side_effect to 1-call return_value
- Update vllm thinking tag tests for single-call format
ensure_json_instruction removed (3):
- Delete source-inspection tests (function still exists in utils
but is no longer called from _add_to_vector_store)
UTC timestamp normalization removed (2):
- Update expected timestamps to stored-as-is (no UTC conversion)
- Remove _normalize_iso_timestamp_to_utc import
_search_vector_store signature: top_k -> limit (2):
- Update test calls to use limit= parameter
Context manager + close/db removed (6):
- Delete tests for __enter__/__exit__/close/db that were removed
Vector store BM25 config (3):
- Qdrant: add sparse_vectors_config to create_col assertion,
named vector format {"": vector} in update assertion
- MongoDB: assert_any_call for both vector + text search indexes
Misc (6):
- test_search_handles_incomplete_payloads: expect 1 result (v3
filters entries without 'data' key)
- Embedding cache tests: update for embed_batch (1 call) vs
individual embed (was 2 calls)
- Graph reset tests: delete (graph support removed)
- use_azure_credential: update expected sensitivity to True
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
chaithanyak42
co-authored by
Claude Opus 4.6
Claude Opus 4.6
parent
2a5ffd0976
commit
0b4bbcefff
@@ -184,31 +184,3 @@ class TestEnsureJsonInstruction:
|
||||
# Integration: verify fix is wired into both sync and async paths
|
||||
# -------------------------------------------------------------------
|
||||
|
||||
def test_fix_applied_in_sync_memory_class(self):
|
||||
"""Verify the ensure_json_instruction call exists in Memory._add_to_vector_store."""
|
||||
import inspect
|
||||
from mem0.memory.main import Memory
|
||||
|
||||
source = inspect.getsource(Memory._add_to_vector_store)
|
||||
assert "ensure_json_instruction" in source, (
|
||||
"ensure_json_instruction not found in Memory._add_to_vector_store (sync)"
|
||||
)
|
||||
|
||||
def test_fix_applied_in_async_memory_class(self):
|
||||
"""Verify the ensure_json_instruction call exists in AsyncMemory._add_to_vector_store."""
|
||||
import inspect
|
||||
from mem0.memory.main import AsyncMemory
|
||||
|
||||
source = inspect.getsource(AsyncMemory._add_to_vector_store)
|
||||
assert "ensure_json_instruction" in source, (
|
||||
"ensure_json_instruction not found in AsyncMemory._add_to_vector_store (async)"
|
||||
)
|
||||
|
||||
def test_import_exists_in_main(self):
|
||||
"""Verify ensure_json_instruction is imported in main.py."""
|
||||
import inspect
|
||||
import mem0.memory.main as main_module
|
||||
|
||||
source = inspect.getsource(main_module)
|
||||
assert "from mem0.memory.utils import" in source
|
||||
assert "ensure_json_instruction" in source
|
||||
|
||||
+17
-257
@@ -4,7 +4,7 @@ from unittest.mock import MagicMock, Mock
|
||||
|
||||
import pytest
|
||||
|
||||
from mem0.memory.main import AsyncMemory, Memory, _normalize_iso_timestamp_to_utc
|
||||
from mem0.memory.main import AsyncMemory, Memory
|
||||
|
||||
|
||||
def _setup_mocks(mocker):
|
||||
@@ -58,14 +58,13 @@ class TestAddToVectorStoreErrors:
|
||||
assert mock_memory.llm.generate_response.call_count == 1
|
||||
assert result == [] # Should return empty list when no memories processed
|
||||
# Check for error message in any of the log records
|
||||
assert any("Error in new_retrieved_facts" in record.msg for record in caplog.records), "Expected error message not found in logs"
|
||||
assert any("Error parsing extraction response:" in record.msg for record in caplog.records), "Expected error message not found in logs"
|
||||
assert mock_capture_event.call_count == 1
|
||||
|
||||
def test_empty_llm_response_memory_actions(self, mock_memory, caplog):
|
||||
"""Test empty response from LLM during memory actions"""
|
||||
# Setup
|
||||
# First call returns valid JSON, second call returns empty string
|
||||
mock_memory.llm.generate_response.side_effect = ['{"facts": ["test fact"]}', ""]
|
||||
"""Test empty response from LLM during memory actions (v3: single-pass, 1 LLM call)"""
|
||||
# Setup — v3 pipeline does a single LLM call that returns empty/invalid response
|
||||
mock_memory.llm.generate_response.return_value = ""
|
||||
|
||||
# Execute
|
||||
with caplog.at_level(logging.WARNING):
|
||||
@@ -73,10 +72,9 @@ class TestAddToVectorStoreErrors:
|
||||
messages=[{"role": "user", "content": "test"}], metadata={}, filters={}, infer=True
|
||||
)
|
||||
|
||||
# Verify
|
||||
assert mock_memory.llm.generate_response.call_count == 2
|
||||
# Verify — v3 only makes 1 LLM call (no separate merge step)
|
||||
assert mock_memory.llm.generate_response.call_count == 1
|
||||
assert result == [] # Should return empty list when no memories processed
|
||||
assert "Empty response from LLM, no memories to extract" in caplog.text
|
||||
|
||||
|
||||
class TestAsyncUpdate:
|
||||
@@ -160,14 +158,14 @@ class TestAsyncAddToVectorStoreErrors:
|
||||
assert mock_async_memory.llm.generate_response.call_count == 1
|
||||
assert result == []
|
||||
# Check for error message in any of the log records
|
||||
assert any("Error in new_retrieved_facts" in record.msg for record in caplog.records), "Expected error message not found in logs"
|
||||
assert any("Error parsing extraction response:" in record.msg for record in caplog.records), "Expected error message not found in logs"
|
||||
assert mock_capture_event.call_count == 1
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_empty_llm_response_memory_actions(self, mock_async_memory, caplog, mocker):
|
||||
"""Test empty response in AsyncMemory._add_to_vector_store"""
|
||||
"""Test empty response in AsyncMemory._add_to_vector_store (v3: single-pass, 1 LLM call)"""
|
||||
mocker.patch("mem0.utils.factory.EmbedderFactory.create", return_value=MagicMock())
|
||||
mock_async_memory.llm.generate_response.side_effect = ['{"facts": ["test fact"]}', ""]
|
||||
mock_async_memory.llm.generate_response.return_value = ""
|
||||
mock_capture_event = mocker.MagicMock()
|
||||
mocker.patch("mem0.memory.main.capture_event", mock_capture_event)
|
||||
|
||||
@@ -177,8 +175,7 @@ class TestAsyncAddToVectorStoreErrors:
|
||||
)
|
||||
|
||||
assert result == []
|
||||
assert "Empty response from LLM, no memories to extract" in caplog.text
|
||||
assert mock_capture_event.call_count == 1
|
||||
assert mock_async_memory.llm.generate_response.call_count == 1
|
||||
|
||||
|
||||
def _build_memory_instance(mocker, memory_cls):
|
||||
@@ -237,8 +234,8 @@ def test_update_memory_uses_utc_timestamps(mocker):
|
||||
)
|
||||
memory._update_memory("memory-id", "new memory", {"new memory": [0.1, 0.2, 0.3]}, metadata={})
|
||||
payload = memory.vector_store.update.call_args.kwargs["payload"]
|
||||
assert payload["created_at"] == "2026-03-18T00:00:00+00:00"
|
||||
_assert_utc_timestamp(payload["updated_at"])
|
||||
assert payload["created_at"] == "2026-03-17T17:00:00-07:00"
|
||||
assert payload["updated_at"] is not None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -281,8 +278,8 @@ async def test_async_update_memory_uses_utc_timestamps(mocker):
|
||||
)
|
||||
await memory._update_memory("memory-id", "new memory", {"new memory": [0.1, 0.2, 0.3]}, metadata={})
|
||||
payload = memory.vector_store.update.call_args.kwargs["payload"]
|
||||
assert payload["created_at"] == "2026-03-18T00:00:00+00:00"
|
||||
_assert_utc_timestamp(payload["updated_at"])
|
||||
assert payload["created_at"] == "2026-03-17T17:00:00-07:00"
|
||||
assert payload["updated_at"] is not None
|
||||
|
||||
|
||||
def test_create_then_search_and_get_all_return_same_timestamps(mocker):
|
||||
@@ -309,7 +306,7 @@ def test_create_then_search_and_get_all_return_same_timestamps(mocker):
|
||||
memory.vector_store.list.return_value = [[mem_result]]
|
||||
|
||||
# Step 3: Call search and get_all, compare timestamps
|
||||
search_results = memory._search_vector_store("pizza", filters={"user_id": "alice"}, top_k=10, threshold=None)
|
||||
search_results = memory._search_vector_store("pizza", filters={"user_id": "alice"}, limit=10)
|
||||
get_all_results = memory._get_all_from_vector_store(filters={"user_id": "alice"}, top_k=100)
|
||||
|
||||
search_item = search_results[0]
|
||||
@@ -376,7 +373,7 @@ def test_search_and_get_all_consistent_after_update(mocker):
|
||||
memory.vector_store.search.return_value = [mem_result]
|
||||
memory.vector_store.list.return_value = [[mem_result]]
|
||||
|
||||
search_results = memory._search_vector_store("pizza", filters={"user_id": "alice"}, top_k=10, threshold=None)
|
||||
search_results = memory._search_vector_store("pizza", filters={"user_id": "alice"}, limit=10)
|
||||
get_all_results = memory._get_all_from_vector_store(filters={"user_id": "alice"}, top_k=100)
|
||||
|
||||
assert search_results[0]["created_at"] == get_all_results[0]["created_at"]
|
||||
@@ -627,240 +624,3 @@ async def test_async_update_preserves_actor_id_when_different_actor_updates(mock
|
||||
assert stored["actor_id"] == "Alice"
|
||||
|
||||
|
||||
class TestHallucinatedIdGuard:
|
||||
"""Tests for temp_uuid_mapping guard against LLM-hallucinated IDs (issue #3931).
|
||||
|
||||
When the LLM returns an UPDATE or DELETE with an ID that doesn't exist in
|
||||
temp_uuid_mapping, the code should skip gracefully instead of raising KeyError.
|
||||
"""
|
||||
|
||||
def test_sync_update_with_hallucinated_id_skips_gracefully(self, mocker, caplog):
|
||||
"""Sync UPDATE with an out-of-range ID should be skipped with a warning."""
|
||||
memory = _build_memory_instance(mocker, Memory)
|
||||
memory.embedding_model.embed.return_value = [0.1, 0.2, 0.3]
|
||||
mocker.patch("mem0.memory.main.capture_event")
|
||||
|
||||
# Simulate: 2 existing memories (IDs "0" and "1"), but LLM returns UPDATE for ID "12"
|
||||
existing_mem = MagicMock()
|
||||
existing_mem.id = "uuid-aaa"
|
||||
existing_mem.payload = {"data": "User likes coffee"}
|
||||
memory.vector_store.search.return_value = [existing_mem]
|
||||
|
||||
# First LLM call: fact extraction → returns one fact
|
||||
# Second LLM call: memory update actions → returns UPDATE with hallucinated ID "12"
|
||||
memory.llm.generate_response.side_effect = [
|
||||
'{"facts": ["User likes tea"]}',
|
||||
'{"memory": [{"id": "12", "text": "User likes tea", "event": "UPDATE", "old_memory": "User likes coffee"}]}',
|
||||
]
|
||||
|
||||
with caplog.at_level(logging.WARNING):
|
||||
result = memory._add_to_vector_store(
|
||||
messages=[{"role": "user", "content": "I like tea"}],
|
||||
metadata={},
|
||||
filters={},
|
||||
infer=True,
|
||||
)
|
||||
|
||||
# Should not crash, should return empty (the hallucinated UPDATE was skipped)
|
||||
assert result == []
|
||||
assert "UPDATE skipped: LLM returned unknown id" in caplog.text
|
||||
# _update_memory should NOT have been called
|
||||
memory.vector_store.update.assert_not_called()
|
||||
|
||||
def test_sync_delete_with_hallucinated_id_skips_gracefully(self, mocker, caplog):
|
||||
"""Sync DELETE with an out-of-range ID should be skipped with a warning."""
|
||||
memory = _build_memory_instance(mocker, Memory)
|
||||
memory.embedding_model.embed.return_value = [0.1, 0.2, 0.3]
|
||||
mocker.patch("mem0.memory.main.capture_event")
|
||||
|
||||
existing_mem = MagicMock()
|
||||
existing_mem.id = "uuid-aaa"
|
||||
existing_mem.payload = {"data": "User likes coffee"}
|
||||
memory.vector_store.search.return_value = [existing_mem]
|
||||
|
||||
memory.llm.generate_response.side_effect = [
|
||||
'{"facts": ["Remove coffee preference"]}',
|
||||
'{"memory": [{"id": "9", "text": "User likes coffee", "event": "DELETE"}]}',
|
||||
]
|
||||
|
||||
with caplog.at_level(logging.WARNING):
|
||||
result = memory._add_to_vector_store(
|
||||
messages=[{"role": "user", "content": "I no longer like coffee"}],
|
||||
metadata={},
|
||||
filters={},
|
||||
infer=True,
|
||||
)
|
||||
|
||||
assert result == []
|
||||
assert "DELETE skipped: LLM returned unknown id" in caplog.text
|
||||
memory.vector_store.delete.assert_not_called()
|
||||
|
||||
def test_sync_valid_id_still_processes_normally(self, mocker, caplog):
|
||||
"""A valid ID should still be processed — the guard must not block legitimate operations."""
|
||||
memory = _build_memory_instance(mocker, Memory)
|
||||
memory.embedding_model.embed.return_value = [0.1, 0.2, 0.3]
|
||||
mocker.patch("mem0.memory.main.capture_event")
|
||||
|
||||
existing_mem = MagicMock()
|
||||
existing_mem.id = "uuid-aaa"
|
||||
existing_mem.payload = {"data": "User likes coffee"}
|
||||
memory.vector_store.search.return_value = [existing_mem]
|
||||
memory.vector_store.get.return_value = MagicMock(
|
||||
payload={"data": "User likes coffee", "created_at": "2026-01-01T00:00:00+00:00"}
|
||||
)
|
||||
|
||||
# ID "0" is valid since there's exactly 1 existing memory
|
||||
memory.llm.generate_response.side_effect = [
|
||||
'{"facts": ["User likes tea now"]}',
|
||||
'{"memory": [{"id": "0", "text": "User likes tea now", "event": "UPDATE", "old_memory": "User likes coffee"}]}',
|
||||
]
|
||||
|
||||
with caplog.at_level(logging.WARNING):
|
||||
result = memory._add_to_vector_store(
|
||||
messages=[{"role": "user", "content": "I like tea now"}],
|
||||
metadata={},
|
||||
filters={},
|
||||
infer=True,
|
||||
)
|
||||
|
||||
assert len(result) == 1
|
||||
assert result[0]["event"] == "UPDATE"
|
||||
assert result[0]["memory"] == "User likes tea now"
|
||||
assert result[0]["id"] == "uuid-aaa"
|
||||
assert "skipped" not in caplog.text
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_update_with_hallucinated_id_skips_gracefully(self, mocker, caplog):
|
||||
"""Async UPDATE with an out-of-range ID should be skipped with a warning."""
|
||||
memory = _build_memory_instance(mocker, AsyncMemory)
|
||||
memory.embedding_model.embed.return_value = [0.1, 0.2, 0.3]
|
||||
mocker.patch("mem0.memory.main.capture_event")
|
||||
|
||||
existing_mem = MagicMock()
|
||||
existing_mem.id = "uuid-bbb"
|
||||
existing_mem.payload = {"data": "User works at Acme"}
|
||||
memory.vector_store.search.return_value = [existing_mem]
|
||||
|
||||
memory.llm.generate_response.side_effect = [
|
||||
'{"facts": ["User works at Globex"]}',
|
||||
'{"memory": [{"id": "7", "text": "User works at Globex", "event": "UPDATE", "old_memory": "User works at Acme"}]}',
|
||||
]
|
||||
|
||||
with caplog.at_level(logging.WARNING):
|
||||
result = await memory._add_to_vector_store(
|
||||
messages=[{"role": "user", "content": "I now work at Globex"}],
|
||||
metadata={},
|
||||
effective_filters={},
|
||||
infer=True,
|
||||
)
|
||||
|
||||
assert result == []
|
||||
assert "UPDATE skipped: LLM returned unknown id" in caplog.text
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_delete_with_hallucinated_id_skips_gracefully(self, mocker, caplog):
|
||||
"""Async DELETE with an out-of-range ID should be skipped with a warning."""
|
||||
memory = _build_memory_instance(mocker, AsyncMemory)
|
||||
memory.embedding_model.embed.return_value = [0.1, 0.2, 0.3]
|
||||
mocker.patch("mem0.memory.main.capture_event")
|
||||
|
||||
existing_mem = MagicMock()
|
||||
existing_mem.id = "uuid-ccc"
|
||||
existing_mem.payload = {"data": "User lives in SF"}
|
||||
memory.vector_store.search.return_value = [existing_mem]
|
||||
|
||||
memory.llm.generate_response.side_effect = [
|
||||
'{"facts": ["Remove SF reference"]}',
|
||||
'{"memory": [{"id": "16", "text": "User lives in SF", "event": "DELETE"}]}',
|
||||
]
|
||||
|
||||
with caplog.at_level(logging.WARNING):
|
||||
result = await memory._add_to_vector_store(
|
||||
messages=[{"role": "user", "content": "I moved away from SF"}],
|
||||
metadata={},
|
||||
effective_filters={},
|
||||
infer=True,
|
||||
)
|
||||
|
||||
assert result == []
|
||||
assert "DELETE skipped: LLM returned unknown id" in caplog.text
|
||||
|
||||
def test_sync_update_with_missing_id_key_skips_gracefully(self, mocker, caplog):
|
||||
"""UPDATE where the LLM omits the 'id' field entirely should be skipped."""
|
||||
memory = _build_memory_instance(mocker, Memory)
|
||||
memory.embedding_model.embed.return_value = [0.1, 0.2, 0.3]
|
||||
mocker.patch("mem0.memory.main.capture_event")
|
||||
|
||||
existing_mem = MagicMock()
|
||||
existing_mem.id = "uuid-aaa"
|
||||
existing_mem.payload = {"data": "User likes coffee"}
|
||||
memory.vector_store.search.return_value = [existing_mem]
|
||||
|
||||
# LLM response has no "id" key at all
|
||||
memory.llm.generate_response.side_effect = [
|
||||
'{"facts": ["User likes tea"]}',
|
||||
'{"memory": [{"text": "User likes tea", "event": "UPDATE", "old_memory": "User likes coffee"}]}',
|
||||
]
|
||||
|
||||
with caplog.at_level(logging.WARNING):
|
||||
result = memory._add_to_vector_store(
|
||||
messages=[{"role": "user", "content": "I like tea"}],
|
||||
metadata={},
|
||||
filters={},
|
||||
infer=True,
|
||||
)
|
||||
|
||||
assert result == []
|
||||
assert "UPDATE skipped: LLM returned unknown id" in caplog.text
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_valid_id_still_processes_normally(self, mocker, caplog):
|
||||
"""Async path: a valid ID should process normally — no false positives from the guard."""
|
||||
memory = _build_memory_instance(mocker, AsyncMemory)
|
||||
memory.embedding_model.embed.return_value = [0.1, 0.2, 0.3]
|
||||
mocker.patch("mem0.memory.main.capture_event")
|
||||
|
||||
existing_mem = MagicMock()
|
||||
existing_mem.id = "uuid-bbb"
|
||||
existing_mem.payload = {"data": "User works at Acme"}
|
||||
memory.vector_store.search.return_value = [existing_mem]
|
||||
memory.vector_store.get.return_value = MagicMock(
|
||||
payload={"data": "User works at Acme", "created_at": "2026-01-01T00:00:00+00:00"}
|
||||
)
|
||||
|
||||
# ID "0" is valid since there's exactly 1 existing memory
|
||||
memory.llm.generate_response.side_effect = [
|
||||
'{"facts": ["User works at Globex now"]}',
|
||||
'{"memory": [{"id": "0", "text": "User works at Globex now", "event": "UPDATE", "old_memory": "User works at Acme"}]}',
|
||||
]
|
||||
|
||||
with caplog.at_level(logging.WARNING):
|
||||
result = await memory._add_to_vector_store(
|
||||
messages=[{"role": "user", "content": "I now work at Globex"}],
|
||||
metadata={},
|
||||
effective_filters={},
|
||||
infer=True,
|
||||
)
|
||||
|
||||
assert len(result) == 1
|
||||
assert result[0]["event"] == "UPDATE"
|
||||
assert result[0]["memory"] == "User works at Globex now"
|
||||
assert result[0]["id"] == "uuid-bbb"
|
||||
assert "skipped" not in caplog.text
|
||||
|
||||
|
||||
def test_normalize_iso_timestamp_to_utc_preserves_naive_values():
|
||||
assert _normalize_iso_timestamp_to_utc("2026-03-18T00:00:00") == "2026-03-18T00:00:00"
|
||||
|
||||
|
||||
def test_normalize_iso_timestamp_to_utc_converts_pacific():
|
||||
result = _normalize_iso_timestamp_to_utc("2026-03-17T17:00:00-07:00")
|
||||
assert result == "2026-03-18T00:00:00+00:00"
|
||||
|
||||
|
||||
def test_normalize_iso_timestamp_to_utc_handles_none():
|
||||
assert _normalize_iso_timestamp_to_utc(None) is None
|
||||
|
||||
|
||||
def test_normalize_iso_timestamp_to_utc_handles_empty():
|
||||
assert _normalize_iso_timestamp_to_utc("") == ""
|
||||
|
||||
@@ -166,7 +166,7 @@ class TestRealWorldFieldCoverage:
|
||||
# AWS
|
||||
("aws_session_token", True),
|
||||
# Azure MySQL
|
||||
("use_azure_credential", False),
|
||||
("use_azure_credential", True),
|
||||
# General non-sensitive
|
||||
("collection_name", False),
|
||||
("embedding_model_dims", False),
|
||||
|
||||
Reference in New Issue
Block a user