fix(oss): normalize malformed LLM fact output before embedding (#4224)
Co-authored-by: kartik-mem0 <kartik.labhshetwar@mem0.ai>
This commit is contained in:
co-authored by
kartik-mem0
parent
d7a34c24dd
commit
577a5a2feb
+28
-1
@@ -1,12 +1,15 @@
|
||||
import hashlib
|
||||
import logging
|
||||
import re
|
||||
|
||||
from mem0.configs.prompts import (
|
||||
AGENT_MEMORY_EXTRACTION_PROMPT,
|
||||
FACT_RETRIEVAL_PROMPT,
|
||||
USER_MEMORY_EXTRACTION_PROMPT,
|
||||
AGENT_MEMORY_EXTRACTION_PROMPT,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def get_fact_retrieval_messages(message, is_agent_memory=False):
|
||||
"""Get fact retrieval messages based on the memory type.
|
||||
@@ -77,6 +80,30 @@ def format_entities(entities):
|
||||
|
||||
return "\n".join(formatted_lines)
|
||||
|
||||
def normalize_facts(raw_facts):
|
||||
"""Normalize LLM-extracted facts to a list of strings.
|
||||
|
||||
Smaller LLMs (e.g. llama3.1:8b) sometimes return facts as objects
|
||||
like {"fact": "..."} or {"text": "..."} instead of plain strings.
|
||||
This mirrors the TypeScript FactRetrievalSchema validation.
|
||||
"""
|
||||
if not raw_facts:
|
||||
return []
|
||||
normalized = []
|
||||
for item in raw_facts:
|
||||
if isinstance(item, str):
|
||||
fact = item
|
||||
elif isinstance(item, dict):
|
||||
fact = item.get("fact") or item.get("text")
|
||||
if fact is None:
|
||||
logger.warning("Unexpected fact shape from LLM, skipping: %s", item)
|
||||
continue
|
||||
else:
|
||||
fact = str(item)
|
||||
if fact:
|
||||
normalized.append(fact)
|
||||
return normalized
|
||||
|
||||
|
||||
def remove_code_blocks(content: str) -> str:
|
||||
"""
|
||||
|
||||
Reference in New Issue
Block a user