JDSS/mem0
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(bedrock): omit topP for Anthropic Converse; use AWSBedrockConfig in LlmFactory (#4469)

Browse Source
This commit is contained in:
Himanshu
2026-03-25 11:22:52 +05:30
committed by GitHub
parent d1b4b304c7
commit 2e0f91e70d
4 changed files with 383 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files
mem0/configs/llms/aws_bedrock.py
+8 -4
View File
@@ -16,7 +16,7 @@ class AWSBedrockConfig(BaseLlmConfig):
model: Optional[str] = None,
temperature: float = 0.1,
max_tokens: int = 2000,
top_p: float = 0.9,
top_p: Optional[float] = None,
top_k: int = 1,
aws_access_key_id: Optional[str] = None,
aws_secret_access_key: Optional[str] = None,
@@ -33,7 +33,8 @@ class AWSBedrockConfig(BaseLlmConfig):
model: Bedrock model identifier (e.g., "amazon.nova-3-mini-20241119-v1:0")
temperature: Controls randomness (0.0 to 2.0)
max_tokens: Maximum tokens to generate
top_p: Nucleus sampling parameter (0.0 to 1.0)
top_p: Nucleus sampling (0.0–1.0). Default None omits topP on Converse
(required for Anthropic, which rejects temperature and topP together).
top_k: Top-k sampling parameter (1 to 40)
aws_access_key_id: AWS access key (optional, uses env vars if not provided)
aws_secret_access_key: AWS secret key (optional, uses env vars if not provided)
@@ -75,13 +76,16 @@ class AWSBedrockConfig(BaseLlmConfig):
def get_model_config(self) -> Dict[str, Any]:
"""Get model-specific configuration parameters."""
base_config = {
base_config: Dict[str, Any] = {
"temperature": self.temperature,
"max_tokens": self.max_tokens,
"top_p": self.top_p,
"top_k": self.top_k,
}
# Only include top_p when explicitly set by the user.
if self.top_p is not None:
base_config["top_p"] = self.top_p
# Add custom model kwargs
base_config.update(self.model_kwargs)
mem0/llms/aws_bedrock.py
+43 -41
View File
@@ -228,6 +228,12 @@ class AWSBedrockLLM(LLMBase):
return "\n\nHuman: " + "".join(formatted_messages) + "\n\nAssistant:"
def _merge_optional_top_p(self, target: Dict[str, Any], *, key: str = "top_p") -> None:
"""Add nucleus sampling to ``target`` only when ``model_config`` has ``top_p`` set."""
top_p = self.model_config.get("top_p")
if top_p is not None:
target[key] = top_p
def _prepare_input(self, prompt: str) -> Dict[str, Any]:
"""
Prepare input for the current provider's model.
@@ -268,44 +274,38 @@ class AWSBedrockLLM(LLMBase):
"messages": [{"role": "user", "content": prompt}],
"max_tokens": self.model_config.get("max_tokens", 5000),
"temperature": self.model_config.get("temperature", 0.1),
"top_p": self.model_config.get("top_p", 0.9),
}
self._merge_optional_top_p(input_body)
else:
# Legacy Amazon models
input_body = {
"inputText": prompt,
"textGenerationConfig": {
"maxTokenCount": self.model_config.get("max_tokens", 5000),
"topP": self.model_config.get("top_p", 0.9),
"temperature": self.model_config.get("temperature", 0.1),
},
}
# Remove None values
input_body["textGenerationConfig"] = {
k: v for k, v in input_body["textGenerationConfig"].items() if v is not None
text_gen_config: Dict[str, Any] = {
"maxTokenCount": self.model_config.get("max_tokens", 5000),
"temperature": self.model_config.get("temperature", 0.1),
}
self._merge_optional_top_p(text_gen_config, key="topP")
input_body = {"inputText": prompt, "textGenerationConfig": text_gen_config}
elif self.provider == "anthropic":
input_body = {
"messages": [{"role": "user", "content": [{"type": "text", "text": prompt}]}],
"max_tokens": self.model_config.get("max_tokens", 2000),
"temperature": self.model_config.get("temperature", 0.1),
"top_p": self.model_config.get("top_p", 0.9),
"anthropic_version": "bedrock-2023-05-31",
}
self._merge_optional_top_p(input_body)
elif self.provider == "meta":
input_body = {
"prompt": prompt,
"max_gen_len": self.model_config.get("max_tokens", 5000),
"temperature": self.model_config.get("temperature", 0.1),
"top_p": self.model_config.get("top_p", 0.9),
}
self._merge_optional_top_p(input_body)
elif self.provider == "mistral":
input_body = {
"prompt": prompt,
"max_tokens": self.model_config.get("max_tokens", 5000),
"temperature": self.model_config.get("temperature", 0.1),
"top_p": self.model_config.get("top_p", 0.9),
}
self._merge_optional_top_p(input_body)
else:
# Generic case - add all model config parameters
input_body.update(self.model_config)
@@ -479,6 +479,29 @@ class AWSBedrockLLM(LLMBase):
return converse_tools
def _default_max_tokens_for_converse(self) -> int:
"""Default maxTokens if ``max_tokens`` is missing (Nova: 5000, else 2000)."""
model_id = (self.config.model or "").lower()
if self.provider == "amazon" and "nova" in model_id:
return 5000
return 2000
def _build_inference_config(self) -> Dict[str, Any]:
"""Build Converse ``inferenceConfig``. Anthropic allows only one of temperature or topP; we keep temperature and omit topP."""
inference_config: Dict[str, Any] = {
"maxTokens": self.model_config.get("max_tokens", self._default_max_tokens_for_converse()),
"temperature": self.model_config.get("temperature", 0.1),
}
top_p = self.model_config.get("top_p")
if top_p is not None:
if self.provider == "anthropic":
logger.debug("Omitting topP for Anthropic Converse (using temperature); top_p=%s", top_p)
else:
inference_config["topP"] = top_p
return inference_config
def _generate_with_tools(self, messages: List[Dict[str, str]], tools: List[Dict], stream: bool = False) -> Dict[str, Any]:
"""Generate response with tool calling support using correct message format."""
# Format messages for tool-enabled models
@@ -501,11 +524,7 @@ class AWSBedrockLLM(LLMBase):
converse_params = {
"modelId": self.config.model,
"messages": formatted_messages,
"inferenceConfig": {
"maxTokens": self.model_config.get("max_tokens", 2000),
"temperature": self.model_config.get("temperature", 0.1),
"topP": self.model_config.get("top_p", 0.9),
}
"inferenceConfig": self._build_inference_config(),
}
# Add system message if present (for Anthropic)
@@ -531,11 +550,7 @@ class AWSBedrockLLM(LLMBase):
converse_params = {
"modelId": self.config.model,
"messages": formatted_messages,
"inferenceConfig": {
"maxTokens": self.model_config.get("max_tokens", 2000),
"temperature": self.model_config.get("temperature", 0.1),
"topP": self.model_config.get("top_p", 0.9),
}
"inferenceConfig": self._build_inference_config(),
}
# Add system message if present
@@ -554,26 +569,13 @@ class AWSBedrockLLM(LLMBase):
return str(response)
elif self.provider == "amazon" and "nova" in self.config.model.lower():
# Nova models use converse API even without tools
# Nova models use the Converse API even without tools
formatted_messages = self._format_messages_amazon(messages)
input_body = {
"messages": formatted_messages,
"max_tokens": self.model_config.get("max_tokens", 5000),
"temperature": self.model_config.get("temperature", 0.1),
"top_p": self.model_config.get("top_p", 0.9),
}
# Use converse API for Nova models
response = self.client.converse(
modelId=self.config.model,
messages=input_body["messages"],
inferenceConfig={
"maxTokens": input_body["max_tokens"],
"temperature": input_body["temperature"],
"topP": input_body["top_p"],
}
messages=formatted_messages,
inferenceConfig=self._build_inference_config(),
)
return self._parse_response(response)
else:
# For other providers and legacy Amazon models (like Titan)
mem0/utils/factory.py
+2 -1
View File
@@ -3,6 +3,7 @@ from typing import Dict, Optional, Union
from mem0.configs.embeddings.base import BaseEmbedderConfig
from mem0.configs.llms.anthropic import AnthropicConfig
from mem0.configs.llms.aws_bedrock import AWSBedrockConfig
from mem0.configs.llms.azure import AzureOpenAIConfig
from mem0.configs.llms.base import BaseLlmConfig
from mem0.configs.llms.deepseek import DeepSeekConfig
@@ -38,7 +39,7 @@ class LlmFactory:
"openai": ("mem0.llms.openai.OpenAILLM", OpenAIConfig),
"groq": ("mem0.llms.groq.GroqLLM", BaseLlmConfig),
"together": ("mem0.llms.together.TogetherLLM", BaseLlmConfig),
"aws_bedrock": ("mem0.llms.aws_bedrock.AWSBedrockLLM", BaseLlmConfig),
"aws_bedrock": ("mem0.llms.aws_bedrock.AWSBedrockLLM", AWSBedrockConfig),
"litellm": ("mem0.llms.litellm.LiteLLM", BaseLlmConfig),
"azure_openai": ("mem0.llms.azure_openai.AzureOpenAILLM", AzureOpenAIConfig),
"openai_structured": ("mem0.llms.openai_structured.OpenAIStructuredLLM", OpenAIConfig),