fix(reranker): log reranking failures instead of swallowing them silently (#5717)
This commit is contained in:
@@ -1,3 +1,4 @@
|
|||||||
|
import logging
|
||||||
import os
|
import os
|
||||||
from typing import List, Dict, Any
|
from typing import List, Dict, Any
|
||||||
|
|
||||||
@@ -9,6 +10,8 @@ try:
|
|||||||
except ImportError:
|
except ImportError:
|
||||||
COHERE_AVAILABLE = False
|
COHERE_AVAILABLE = False
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
class CohereReranker(BaseReranker):
|
class CohereReranker(BaseReranker):
|
||||||
"""Cohere-based reranker implementation."""
|
"""Cohere-based reranker implementation."""
|
||||||
@@ -78,8 +81,9 @@ class CohereReranker(BaseReranker):
|
|||||||
|
|
||||||
return reranked_docs
|
return reranked_docs
|
||||||
|
|
||||||
except Exception:
|
except Exception as e:
|
||||||
# Fallback to original order if reranking fails
|
# Fallback to original order if reranking fails
|
||||||
|
logger.warning("Cohere reranking failed, falling back to original order: %s", e)
|
||||||
for doc in documents:
|
for doc in documents:
|
||||||
doc['rerank_score'] = 0.0
|
doc['rerank_score'] = 0.0
|
||||||
final_top_k = top_k or self.config.top_k
|
final_top_k = top_k or self.config.top_k
|
||||||
|
|||||||
@@ -1,3 +1,4 @@
|
|||||||
|
import logging
|
||||||
from typing import List, Dict, Any, Union
|
from typing import List, Dict, Any, Union
|
||||||
import numpy as np
|
import numpy as np
|
||||||
|
|
||||||
@@ -12,6 +13,8 @@ try:
|
|||||||
except ImportError:
|
except ImportError:
|
||||||
TRANSFORMERS_AVAILABLE = False
|
TRANSFORMERS_AVAILABLE = False
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
class HuggingFaceReranker(BaseReranker):
|
class HuggingFaceReranker(BaseReranker):
|
||||||
"""HuggingFace Transformers based reranker implementation."""
|
"""HuggingFace Transformers based reranker implementation."""
|
||||||
@@ -139,8 +142,9 @@ class HuggingFaceReranker(BaseReranker):
|
|||||||
|
|
||||||
return reranked_docs
|
return reranked_docs
|
||||||
|
|
||||||
except Exception:
|
except Exception as e:
|
||||||
# Fallback to original order if reranking fails
|
# Fallback to original order if reranking fails
|
||||||
|
logger.warning("HuggingFace reranking failed, falling back to original order: %s", e)
|
||||||
for doc in documents:
|
for doc in documents:
|
||||||
doc['rerank_score'] = 0.0
|
doc['rerank_score'] = 0.0
|
||||||
final_top_k = top_k or self.config.top_k
|
final_top_k = top_k or self.config.top_k
|
||||||
|
|||||||
@@ -1,3 +1,4 @@
|
|||||||
|
import logging
|
||||||
import re
|
import re
|
||||||
from typing import Any, Dict, List, Union
|
from typing import Any, Dict, List, Union
|
||||||
|
|
||||||
@@ -6,6 +7,8 @@ from mem0.configs.rerankers.llm import LLMRerankerConfig
|
|||||||
from mem0.reranker.base import BaseReranker
|
from mem0.reranker.base import BaseReranker
|
||||||
from mem0.utils.factory import LlmFactory
|
from mem0.utils.factory import LlmFactory
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
class LLMReranker(BaseReranker):
|
class LLMReranker(BaseReranker):
|
||||||
"""LLM-based reranker implementation."""
|
"""LLM-based reranker implementation."""
|
||||||
@@ -151,8 +154,9 @@ class LLMReranker(BaseReranker):
|
|||||||
scored_doc['rerank_score'] = score
|
scored_doc['rerank_score'] = score
|
||||||
scored_docs.append(scored_doc)
|
scored_docs.append(scored_doc)
|
||||||
|
|
||||||
except Exception:
|
except Exception as e:
|
||||||
# Fallback: assign neutral score if scoring fails
|
# Fallback: assign neutral score if scoring fails
|
||||||
|
logger.warning("LLM reranking failed for a document, assigning neutral score: %s", e)
|
||||||
scored_doc = doc.copy()
|
scored_doc = doc.copy()
|
||||||
scored_doc['rerank_score'] = 0.5
|
scored_doc['rerank_score'] = 0.5
|
||||||
scored_docs.append(scored_doc)
|
scored_docs.append(scored_doc)
|
||||||
|
|||||||
@@ -1,3 +1,4 @@
|
|||||||
|
import logging
|
||||||
from typing import List, Dict, Any, Union
|
from typing import List, Dict, Any, Union
|
||||||
import numpy as np
|
import numpy as np
|
||||||
|
|
||||||
@@ -11,6 +12,8 @@ try:
|
|||||||
except ImportError:
|
except ImportError:
|
||||||
SENTENCE_TRANSFORMERS_AVAILABLE = False
|
SENTENCE_TRANSFORMERS_AVAILABLE = False
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
class SentenceTransformerReranker(BaseReranker):
|
class SentenceTransformerReranker(BaseReranker):
|
||||||
"""Sentence Transformer based reranker implementation."""
|
"""Sentence Transformer based reranker implementation."""
|
||||||
@@ -102,8 +105,9 @@ class SentenceTransformerReranker(BaseReranker):
|
|||||||
|
|
||||||
return reranked_docs
|
return reranked_docs
|
||||||
|
|
||||||
except Exception:
|
except Exception as e:
|
||||||
# Fallback to original order if reranking fails
|
# Fallback to original order if reranking fails
|
||||||
|
logger.warning("SentenceTransformer reranking failed, falling back to original order: %s", e)
|
||||||
for doc in documents:
|
for doc in documents:
|
||||||
doc['rerank_score'] = 0.0
|
doc['rerank_score'] = 0.0
|
||||||
final_top_k = top_k or self.config.top_k
|
final_top_k = top_k or self.config.top_k
|
||||||
|
|||||||
@@ -1,3 +1,4 @@
|
|||||||
|
import logging
|
||||||
import os
|
import os
|
||||||
from typing import List, Dict, Any
|
from typing import List, Dict, Any
|
||||||
|
|
||||||
@@ -9,6 +10,8 @@ try:
|
|||||||
except ImportError:
|
except ImportError:
|
||||||
ZERO_ENTROPY_AVAILABLE = False
|
ZERO_ENTROPY_AVAILABLE = False
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
class ZeroEntropyReranker(BaseReranker):
|
class ZeroEntropyReranker(BaseReranker):
|
||||||
"""Zero Entropy-based reranker implementation."""
|
"""Zero Entropy-based reranker implementation."""
|
||||||
@@ -89,8 +92,9 @@ class ZeroEntropyReranker(BaseReranker):
|
|||||||
|
|
||||||
return reranked_docs
|
return reranked_docs
|
||||||
|
|
||||||
except Exception:
|
except Exception as e:
|
||||||
# Fallback to original order if reranking fails
|
# Fallback to original order if reranking fails
|
||||||
|
logger.warning("Zero Entropy reranking failed, falling back to original order: %s", e)
|
||||||
for doc in documents:
|
for doc in documents:
|
||||||
doc['rerank_score'] = 0.0
|
doc['rerank_score'] = 0.0
|
||||||
final_top_k = top_k or self.config.top_k
|
final_top_k = top_k or self.config.top_k
|
||||||
|
|||||||
@@ -0,0 +1,32 @@
|
|||||||
|
"""Reranker failures must be logged, not silently swallowed.
|
||||||
|
|
||||||
|
Uses the LLMReranker because it is constructible without heavy ML deps (the
|
||||||
|
``mock_llm`` fixture stubs the LLM factory). The fix under test is shared by all
|
||||||
|
reranker providers: the ``except`` fallback now emits a ``logger.warning`` before
|
||||||
|
degrading to the original order / a neutral score.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import logging
|
||||||
|
|
||||||
|
from mem0.reranker.llm_reranker import LLMReranker
|
||||||
|
|
||||||
|
|
||||||
|
class TestRerankerFailureLogging:
|
||||||
|
def test_llm_failure_is_logged_and_falls_back(self, mock_llm, caplog):
|
||||||
|
_factory, llm_instance = mock_llm
|
||||||
|
llm_instance.generate_response.side_effect = RuntimeError("upstream 500")
|
||||||
|
|
||||||
|
reranker = LLMReranker({"provider": "openai"})
|
||||||
|
docs = [{"memory": "alpha"}, {"memory": "beta"}]
|
||||||
|
|
||||||
|
with caplog.at_level(logging.WARNING, logger="mem0.reranker.llm_reranker"):
|
||||||
|
result = reranker.rerank("q", docs)
|
||||||
|
|
||||||
|
# Graceful degradation preserved: every doc still comes back, scored neutral.
|
||||||
|
assert len(result) == 2
|
||||||
|
assert all(d["rerank_score"] == 0.5 for d in result)
|
||||||
|
|
||||||
|
# The failure is no longer silent.
|
||||||
|
warnings = [r for r in caplog.records if r.levelno == logging.WARNING]
|
||||||
|
assert warnings, "expected a warning to be logged on reranking failure"
|
||||||
|
assert "upstream 500" in caplog.text
|
||||||
Reference in New Issue
Block a user