feat(oss): port v3 pipeline with hybrid search, entity extraction, and additive scoring (#4805)

Co-authored-by: Soumil Rathi <soumilrathi@gmail.com>
Co-authored-by: Saket Aryan <saketaryan2002@gmail.com>
Co-authored-by: chaithanyak42 <chaithanya.kumar42a@gmail.com>
Co-authored-by: kartik-mem0 <kartik.labhshetwar@mem0.ai>

tests/utils/test_lemmatization.py

@@ -0,0 +1,67 @@
+import pytest
+
+
+@pytest.fixture(autouse=True)
+def _ensure_spacy():
+    """Skip tests if spaCy model is not available."""
+    try:
+        import spacy
+        spacy.load("en_core_web_sm")
+    except Exception:
+        pytest.skip("spaCy en_core_web_sm model not available")
+
+
+class TestLemmatizeForBm25:
+    def test_basic_lemmatization(self):
+        from mem0.utils.lemmatization import lemmatize_for_bm25
+
+        result = lemmatize_for_bm25("The cats are running quickly")
+        assert "cat" in result
+        assert "run" in result or "running" in result
+        # Stop words and punctuation should be removed
+        assert "the" not in result.split()
+
+    def test_verb_forms_normalized(self):
+        from mem0.utils.lemmatization import lemmatize_for_bm25
+
+        result = lemmatize_for_bm25("she attended multiple meetings yesterday")
+        assert "attend" in result or "attended" in result
+        assert "meeting" in result  # -ing form preserved alongside lemma
+        # "multiple" is kept (not a spaCy stop word)
+
+    def test_ing_preservation(self):
+        from mem0.utils.lemmatization import lemmatize_for_bm25
+
+        result = lemmatize_for_bm25("attending the morning meeting")
+        tokens = result.split()
+        # Should have both the lemma and the -ing form
+        assert "attending" in tokens or "attend" in tokens
+
+    def test_empty_string(self):
+        from mem0.utils.lemmatization import lemmatize_for_bm25
+
+        result = lemmatize_for_bm25("")
+        assert result == ""
+
+    def test_punctuation_removed(self):
+        from mem0.utils.lemmatization import lemmatize_for_bm25
+
+        result = lemmatize_for_bm25("Hello, world! How are you?")
+        assert "," not in result
+        assert "!" not in result
+        assert "?" not in result
+
+    def test_lowercased(self):
+        from mem0.utils.lemmatization import lemmatize_for_bm25
+
+        result = lemmatize_for_bm25("PYTHON Programming LANGUAGE")
+        for token in result.split():
+            assert token == token.lower()
+
+    def test_stop_words_removed(self):
+        from mem0.utils.lemmatization import lemmatize_for_bm25
+
+        result = lemmatize_for_bm25("this is a very simple test of the system")
+        tokens = result.split()
+        for stop in ["this", "is", "a", "very", "of", "the"]:
+            assert stop not in tokens