JDSS/mem0
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(oss): port v3 pipeline with hybrid search, entity extraction, and additive scoring (#4805)

Browse Source 
Co-authored-by: Soumil Rathi <soumilrathi@gmail.com>
Co-authored-by: Saket Aryan <saketaryan2002@gmail.com>
Co-authored-by: chaithanyak42 <chaithanya.kumar42a@gmail.com>
Co-authored-by: kartik-mem0 <kartik.labhshetwar@mem0.ai>
This commit is contained in:
soumil-rathi
2026-04-14 18:00:58 +05:30
committed by GitHub
co-authored by Soumil Rathi Saket Aryan chaithanyak42 kartik-mem0
parent 57f944e18a
commit a488e19044
120 changed files with 10107 additions and 17135 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/memory/test_main.py
+35 -273
View File
@@ -4,7 +4,7 @@ from unittest.mock import MagicMock, Mock
import pytest
from mem0.memory.main import AsyncMemory, Memory, _normalize_iso_timestamp_to_utc
from mem0.memory.main import AsyncMemory, Memory
def _setup_mocks(mocker):
@@ -37,16 +37,19 @@ class TestAddToVectorStoreErrors:
memory.config = mocker.MagicMock()
memory.config.custom_instructions = None
memory.config.custom_update_memory_prompt = None
memory.custom_instructions = None
memory.api_version = "v1.1"
# v3 pipeline needs db.get_last_messages to return a list
memory.db.get_last_messages = MagicMock(return_value=[])
memory.db.save_messages = MagicMock()
return memory
def test_empty_llm_response_fact_extraction(self, mocker, mock_memory, caplog):
"""Test empty response from LLM during fact extraction"""
"""Test invalid JSON response from LLM during extraction"""
# Setup
mock_memory.llm.generate_response.return_value = "invalid json" # This will trigger a JSON decode error
mock_capture_event = mocker.MagicMock()
mocker.patch("mem0.memory.main.capture_event", mock_capture_event)
mock_memory.llm.generate_response.return_value = "invalid json"
mocker.patch("mem0.memory.main.capture_event")
# Execute
with caplog.at_level(logging.ERROR):
@@ -54,18 +57,15 @@ class TestAddToVectorStoreErrors:
messages=[{"role": "user", "content": "test"}], metadata={}, filters={}, infer=True
)
# Verify
# Verify — v3 single-pass pipeline makes 1 LLM call, returns [] on parse error
assert mock_memory.llm.generate_response.call_count == 1
assert result == [] # Should return empty list when no memories processed
# Check for error message in any of the log records
assert any("Error in new_retrieved_facts" in record.msg for record in caplog.records), "Expected error message not found in logs"
assert mock_capture_event.call_count == 1
assert result == []
assert any("Error parsing extraction response" in record.message for record in caplog.records), "Expected error message not found in logs"
def test_empty_llm_response_memory_actions(self, mock_memory, caplog):
"""Test empty response from LLM during memory actions"""
# Setup
# First call returns valid JSON, second call returns empty string
mock_memory.llm.generate_response.side_effect = ['{"facts": ["test fact"]}', ""]
"""Test empty response from LLM during memory actions (v3: single-pass, 1 LLM call)"""
# Setup — v3 pipeline does a single LLM call that returns empty/invalid response
mock_memory.llm.generate_response.return_value = ""
# Execute
with caplog.at_level(logging.WARNING):
@@ -73,10 +73,9 @@ class TestAddToVectorStoreErrors:
messages=[{"role": "user", "content": "test"}], metadata={}, filters={}, infer=True
)
# Verify
assert mock_memory.llm.generate_response.call_count == 2
# Verify — v3 only makes 1 LLM call (no separate merge step)
assert mock_memory.llm.generate_response.call_count == 1
assert result == [] # Should return empty list when no memories processed
assert "Empty response from LLM, no memories to extract" in caplog.text
class TestAsyncUpdate:
@@ -141,17 +140,20 @@ class TestAsyncAddToVectorStoreErrors:
memory.config = mocker.MagicMock()
memory.config.custom_instructions = None
memory.config.custom_update_memory_prompt = None
memory.custom_instructions = None
memory.api_version = "v1.1"
# v3 pipeline needs db.get_last_messages to return a list
memory.db.get_last_messages = MagicMock(return_value=[])
memory.db.save_messages = MagicMock()
return memory
@pytest.mark.asyncio
async def test_async_empty_llm_response_fact_extraction(self, mock_async_memory, caplog, mocker):
"""Test empty response in AsyncMemory._add_to_vector_store"""
"""Test invalid JSON response from LLM during extraction (async)"""
mocker.patch("mem0.utils.factory.EmbedderFactory.create", return_value=MagicMock())
mock_async_memory.llm.generate_response.return_value = "invalid json" # This will trigger a JSON decode error
mock_capture_event = mocker.MagicMock()
mocker.patch("mem0.memory.main.capture_event", mock_capture_event)
mock_async_memory.llm.generate_response.return_value = "invalid json"
mocker.patch("mem0.memory.main.capture_event")
with caplog.at_level(logging.ERROR):
result = await mock_async_memory._add_to_vector_store(
@@ -159,15 +161,13 @@ class TestAsyncAddToVectorStoreErrors:
)
assert mock_async_memory.llm.generate_response.call_count == 1
assert result == []
# Check for error message in any of the log records
assert any("Error in new_retrieved_facts" in record.msg for record in caplog.records), "Expected error message not found in logs"
assert mock_capture_event.call_count == 1
assert any("Error parsing extraction response" in record.message for record in caplog.records), "Expected error message not found in logs"
@pytest.mark.asyncio
async def test_async_empty_llm_response_memory_actions(self, mock_async_memory, caplog, mocker):
"""Test empty response in AsyncMemory._add_to_vector_store"""
"""Test empty response in AsyncMemory._add_to_vector_store (v3: single-pass, 1 LLM call)"""
mocker.patch("mem0.utils.factory.EmbedderFactory.create", return_value=MagicMock())
mock_async_memory.llm.generate_response.side_effect = ['{"facts": ["test fact"]}', ""]
mock_async_memory.llm.generate_response.return_value = ""
mock_capture_event = mocker.MagicMock()
mocker.patch("mem0.memory.main.capture_event", mock_capture_event)
@@ -177,8 +177,7 @@ class TestAsyncAddToVectorStoreErrors:
)
assert result == []
assert "Empty response from LLM, no memories to extract" in caplog.text
assert mock_capture_event.call_count == 1
assert mock_async_memory.llm.generate_response.call_count == 1
def _build_memory_instance(mocker, memory_cls):
@@ -237,8 +236,8 @@ def test_update_memory_uses_utc_timestamps(mocker):
)
memory._update_memory("memory-id", "new memory", {"new memory": [0.1, 0.2, 0.3]}, metadata={})
payload = memory.vector_store.update.call_args.kwargs["payload"]
assert payload["created_at"] == "2026-03-18T00:00:00+00:00"
_assert_utc_timestamp(payload["updated_at"])
assert payload["created_at"] == "2026-03-17T17:00:00-07:00"
assert payload["updated_at"] is not None
@pytest.mark.asyncio
@@ -281,8 +280,8 @@ async def test_async_update_memory_uses_utc_timestamps(mocker):
)
await memory._update_memory("memory-id", "new memory", {"new memory": [0.1, 0.2, 0.3]}, metadata={})
payload = memory.vector_store.update.call_args.kwargs["payload"]
assert payload["created_at"] == "2026-03-18T00:00:00+00:00"
_assert_utc_timestamp(payload["updated_at"])
assert payload["created_at"] == "2026-03-17T17:00:00-07:00"
assert payload["updated_at"] is not None
def test_create_then_search_and_get_all_return_same_timestamps(mocker):
@@ -309,8 +308,8 @@ def test_create_then_search_and_get_all_return_same_timestamps(mocker):
memory.vector_store.list.return_value = [[mem_result]]
# Step 3: Call search and get_all, compare timestamps
search_results = memory._search_vector_store("pizza", filters={"user_id": "alice"}, top_k=10, threshold=None)
get_all_results = memory._get_all_from_vector_store(filters={"user_id": "alice"}, top_k=100)
search_results = memory._search_vector_store("pizza", filters={"user_id": "alice"}, limit=10)
get_all_results = memory._get_all_from_vector_store(filters={"user_id": "alice"}, limit=100)
search_item = search_results[0]
get_all_item = get_all_results[0]
@@ -376,8 +375,8 @@ def test_search_and_get_all_consistent_after_update(mocker):
memory.vector_store.search.return_value = [mem_result]
memory.vector_store.list.return_value = [[mem_result]]
search_results = memory._search_vector_store("pizza", filters={"user_id": "alice"}, top_k=10, threshold=None)
get_all_results = memory._get_all_from_vector_store(filters={"user_id": "alice"}, top_k=100)
search_results = memory._search_vector_store("pizza", filters={"user_id": "alice"}, limit=10)
get_all_results = memory._get_all_from_vector_store(filters={"user_id": "alice"}, limit=100)
assert search_results[0]["created_at"] == get_all_results[0]["created_at"]
assert search_results[0]["updated_at"] == get_all_results[0]["updated_at"]
@@ -627,240 +626,3 @@ async def test_async_update_preserves_actor_id_when_different_actor_updates(mock
assert stored["actor_id"] == "Alice"
class TestHallucinatedIdGuard:
"""Tests for temp_uuid_mapping guard against LLM-hallucinated IDs (issue #3931).
When the LLM returns an UPDATE or DELETE with an ID that doesn't exist in
temp_uuid_mapping, the code should skip gracefully instead of raising KeyError.
"""
def test_sync_update_with_hallucinated_id_skips_gracefully(self, mocker, caplog):
"""Sync UPDATE with an out-of-range ID should be skipped with a warning."""
memory = _build_memory_instance(mocker, Memory)
memory.embedding_model.embed.return_value = [0.1, 0.2, 0.3]
mocker.patch("mem0.memory.main.capture_event")
# Simulate: 2 existing memories (IDs "0" and "1"), but LLM returns UPDATE for ID "12"
existing_mem = MagicMock()
existing_mem.id = "uuid-aaa"
existing_mem.payload = {"data": "User likes coffee"}
memory.vector_store.search.return_value = [existing_mem]
# First LLM call: fact extraction → returns one fact
# Second LLM call: memory update actions → returns UPDATE with hallucinated ID "12"
memory.llm.generate_response.side_effect = [
'{"facts": ["User likes tea"]}',
'{"memory": [{"id": "12", "text": "User likes tea", "event": "UPDATE", "old_memory": "User likes coffee"}]}',
]
with caplog.at_level(logging.WARNING):
result = memory._add_to_vector_store(
messages=[{"role": "user", "content": "I like tea"}],
metadata={},
filters={},
infer=True,
)
# Should not crash, should return empty (the hallucinated UPDATE was skipped)
assert result == []
assert "UPDATE skipped: LLM returned unknown id" in caplog.text
# _update_memory should NOT have been called
memory.vector_store.update.assert_not_called()
def test_sync_delete_with_hallucinated_id_skips_gracefully(self, mocker, caplog):
"""Sync DELETE with an out-of-range ID should be skipped with a warning."""
memory = _build_memory_instance(mocker, Memory)
memory.embedding_model.embed.return_value = [0.1, 0.2, 0.3]
mocker.patch("mem0.memory.main.capture_event")
existing_mem = MagicMock()
existing_mem.id = "uuid-aaa"
existing_mem.payload = {"data": "User likes coffee"}
memory.vector_store.search.return_value = [existing_mem]
memory.llm.generate_response.side_effect = [
'{"facts": ["Remove coffee preference"]}',
'{"memory": [{"id": "9", "text": "User likes coffee", "event": "DELETE"}]}',
]
with caplog.at_level(logging.WARNING):
result = memory._add_to_vector_store(
messages=[{"role": "user", "content": "I no longer like coffee"}],
metadata={},
filters={},
infer=True,
)
assert result == []
assert "DELETE skipped: LLM returned unknown id" in caplog.text
memory.vector_store.delete.assert_not_called()
def test_sync_valid_id_still_processes_normally(self, mocker, caplog):
"""A valid ID should still be processed — the guard must not block legitimate operations."""
memory = _build_memory_instance(mocker, Memory)
memory.embedding_model.embed.return_value = [0.1, 0.2, 0.3]
mocker.patch("mem0.memory.main.capture_event")
existing_mem = MagicMock()
existing_mem.id = "uuid-aaa"
existing_mem.payload = {"data": "User likes coffee"}
memory.vector_store.search.return_value = [existing_mem]
memory.vector_store.get.return_value = MagicMock(
payload={"data": "User likes coffee", "created_at": "2026-01-01T00:00:00+00:00"}
)
# ID "0" is valid since there's exactly 1 existing memory
memory.llm.generate_response.side_effect = [
'{"facts": ["User likes tea now"]}',
'{"memory": [{"id": "0", "text": "User likes tea now", "event": "UPDATE", "old_memory": "User likes coffee"}]}',
]
with caplog.at_level(logging.WARNING):
result = memory._add_to_vector_store(
messages=[{"role": "user", "content": "I like tea now"}],
metadata={},
filters={},
infer=True,
)
assert len(result) == 1
assert result[0]["event"] == "UPDATE"
assert result[0]["memory"] == "User likes tea now"
assert result[0]["id"] == "uuid-aaa"
assert "skipped" not in caplog.text
@pytest.mark.asyncio
async def test_async_update_with_hallucinated_id_skips_gracefully(self, mocker, caplog):
"""Async UPDATE with an out-of-range ID should be skipped with a warning."""
memory = _build_memory_instance(mocker, AsyncMemory)
memory.embedding_model.embed.return_value = [0.1, 0.2, 0.3]
mocker.patch("mem0.memory.main.capture_event")
existing_mem = MagicMock()
existing_mem.id = "uuid-bbb"
existing_mem.payload = {"data": "User works at Acme"}
memory.vector_store.search.return_value = [existing_mem]
memory.llm.generate_response.side_effect = [
'{"facts": ["User works at Globex"]}',
'{"memory": [{"id": "7", "text": "User works at Globex", "event": "UPDATE", "old_memory": "User works at Acme"}]}',
]
with caplog.at_level(logging.WARNING):
result = await memory._add_to_vector_store(
messages=[{"role": "user", "content": "I now work at Globex"}],
metadata={},
effective_filters={},
infer=True,
)
assert result == []
assert "UPDATE skipped: LLM returned unknown id" in caplog.text
@pytest.mark.asyncio
async def test_async_delete_with_hallucinated_id_skips_gracefully(self, mocker, caplog):
"""Async DELETE with an out-of-range ID should be skipped with a warning."""
memory = _build_memory_instance(mocker, AsyncMemory)
memory.embedding_model.embed.return_value = [0.1, 0.2, 0.3]
mocker.patch("mem0.memory.main.capture_event")
existing_mem = MagicMock()
existing_mem.id = "uuid-ccc"
existing_mem.payload = {"data": "User lives in SF"}
memory.vector_store.search.return_value = [existing_mem]
memory.llm.generate_response.side_effect = [
'{"facts": ["Remove SF reference"]}',
'{"memory": [{"id": "16", "text": "User lives in SF", "event": "DELETE"}]}',
]
with caplog.at_level(logging.WARNING):
result = await memory._add_to_vector_store(
messages=[{"role": "user", "content": "I moved away from SF"}],
metadata={},
effective_filters={},
infer=True,
)
assert result == []
assert "DELETE skipped: LLM returned unknown id" in caplog.text
def test_sync_update_with_missing_id_key_skips_gracefully(self, mocker, caplog):
"""UPDATE where the LLM omits the 'id' field entirely should be skipped."""
memory = _build_memory_instance(mocker, Memory)
memory.embedding_model.embed.return_value = [0.1, 0.2, 0.3]
mocker.patch("mem0.memory.main.capture_event")
existing_mem = MagicMock()
existing_mem.id = "uuid-aaa"
existing_mem.payload = {"data": "User likes coffee"}
memory.vector_store.search.return_value = [existing_mem]
# LLM response has no "id" key at all
memory.llm.generate_response.side_effect = [
'{"facts": ["User likes tea"]}',
'{"memory": [{"text": "User likes tea", "event": "UPDATE", "old_memory": "User likes coffee"}]}',
]
with caplog.at_level(logging.WARNING):
result = memory._add_to_vector_store(
messages=[{"role": "user", "content": "I like tea"}],
metadata={},
filters={},
infer=True,
)
assert result == []
assert "UPDATE skipped: LLM returned unknown id" in caplog.text
@pytest.mark.asyncio
async def test_async_valid_id_still_processes_normally(self, mocker, caplog):
"""Async path: a valid ID should process normally — no false positives from the guard."""
memory = _build_memory_instance(mocker, AsyncMemory)
memory.embedding_model.embed.return_value = [0.1, 0.2, 0.3]
mocker.patch("mem0.memory.main.capture_event")
existing_mem = MagicMock()
existing_mem.id = "uuid-bbb"
existing_mem.payload = {"data": "User works at Acme"}
memory.vector_store.search.return_value = [existing_mem]
memory.vector_store.get.return_value = MagicMock(
payload={"data": "User works at Acme", "created_at": "2026-01-01T00:00:00+00:00"}
)
# ID "0" is valid since there's exactly 1 existing memory
memory.llm.generate_response.side_effect = [
'{"facts": ["User works at Globex now"]}',
'{"memory": [{"id": "0", "text": "User works at Globex now", "event": "UPDATE", "old_memory": "User works at Acme"}]}',
]
with caplog.at_level(logging.WARNING):
result = await memory._add_to_vector_store(
messages=[{"role": "user", "content": "I now work at Globex"}],
metadata={},
effective_filters={},
infer=True,
)
assert len(result) == 1
assert result[0]["event"] == "UPDATE"
assert result[0]["memory"] == "User works at Globex now"
assert result[0]["id"] == "uuid-bbb"
assert "skipped" not in caplog.text
def test_normalize_iso_timestamp_to_utc_preserves_naive_values():
assert _normalize_iso_timestamp_to_utc("2026-03-18T00:00:00") == "2026-03-18T00:00:00"
def test_normalize_iso_timestamp_to_utc_converts_pacific():
result = _normalize_iso_timestamp_to_utc("2026-03-17T17:00:00-07:00")
assert result == "2026-03-18T00:00:00+00:00"
def test_normalize_iso_timestamp_to_utc_handles_none():
assert _normalize_iso_timestamp_to_utc(None) is None
def test_normalize_iso_timestamp_to_utc_handles_empty():
assert _normalize_iso_timestamp_to_utc("") == ""