JDSS/mem0
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(ai-sdk): added file support for multimodal capabilities with memory context (#3500)

Browse Source
This commit is contained in:
Saket Aryan
2025-09-25 10:32:06 +05:30
committed by GitHub
parent a199ee4ff8
commit 6e1d02c137
5 changed files with 262 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
vercel-ai-sdk/src/mem0-utils.ts
+185 -6
View File
@@ -1,19 +1,62 @@
import { LanguageModelV2Prompt } from '@ai-sdk/provider';
import { Mem0ConfigSettings } from './mem0-types';
import { loadApiKey } from '@ai-sdk/provider-utils';
interface MultimodalContent {
type: 'text' | 'image_url' | 'mdx_url' | 'pdf_url';
text?: string;
image_url?: {
url: string;
};
mdx_url?: {
url: string;
};
pdf_url?: {
url: string;
};
}
interface FileContent {
type: 'file';
data: string; // fileDataUrl
mediaType: string; // e.g., 'application/pdf', 'text/markdown', 'image/jpeg'
}
interface Message {
role: string;
content: string | Array<{type: string, text: string}>;
content: string | MultimodalContent | Array<MultimodalContent>;
}
const flattenPrompt = (prompt: LanguageModelV2Prompt) => {
try {
return prompt.map((part) => {
if (part.role === "user") {
return part.content
.filter((obj) => obj.type === 'text')
.map((obj) => obj.text)
.join(" ");
if (typeof part.content === 'string') {
return part.content;
} else if (Array.isArray(part.content)) {
return part.content
.filter((obj) => obj.type === 'text')
.map((obj) => obj.text)
.join(" ");
} else if (part.content && typeof part.content === 'object' && 'type' in part.content) {
const content = part.content as any;
if (content.type === 'text' && content.text) {
return content.text;
} else if (content.type === 'file') {
// For file content, we'll include a descriptive placeholder
if (content.mediaType === 'application/pdf') {
return '[PDF document]';
} else if (content.mediaType === 'text/markdown' || content.mediaType === 'application/mdx') {
return '[Markdown document]';
} else if (content.mediaType && content.mediaType.startsWith('image/')) {
return '[Image]';
} else {
return '[File attachment]';
}
}
}
// For non-text content (images, pdfs, mdx), we'll include a placeholder
// This helps maintain context for memory search while not breaking the text flow
return "[multimodal content]";
}
return "";
}).join(" ");
@@ -33,7 +76,7 @@ const convertToMem0Format = (messages: LanguageModelV2Prompt) => {
content: message.content,
};
}
else {
else if (Array.isArray(message.content)) {
return message.content.map((obj: any) => {
try {
if (obj.type === "text") {
@@ -41,6 +84,69 @@ const convertToMem0Format = (messages: LanguageModelV2Prompt) => {
role: message.role,
content: obj.text,
};
} else if (obj.type === "file") {
// Handle LanguageModelV2Prompt file format
if (obj.mediaType === "application/pdf") {
return {
role: message.role,
content: {
type: "pdf_url",
pdf_url: {
url: obj.data
}
}
};
} else if (obj.mediaType === "text/markdown" || obj.mediaType === "application/mdx") {
return {
role: message.role,
content: {
type: "mdx_url",
mdx_url: {
url: obj.data
}
}
};
} else if (obj.mediaType && obj.mediaType.startsWith("image/")) {
return {
role: message.role,
content: {
type: "image_url",
image_url: {
url: obj.data
}
}
};
}
} else if (obj.type === "image_url" || obj.type === "image") {
return {
role: message.role,
content: {
type: "image_url",
image_url: {
url: obj.image_url?.url || obj.image?.url || obj.url
}
}
};
} else if (obj.type === "mdx_url" || obj.type === "mdx") {
return {
role: message.role,
content: {
type: "mdx_url",
mdx_url: {
url: obj.mdx_url?.url || obj.mdx?.url || obj.url
}
}
};
} else if (obj.type === "pdf_url" || obj.type === "pdf") {
return {
role: message.role,
content: {
type: "pdf_url",
pdf_url: {
url: obj.pdf_url?.url || obj.pdf?.url || obj.url
}
}
};
}
return null;
} catch (error) {
@@ -48,6 +154,79 @@ const convertToMem0Format = (messages: LanguageModelV2Prompt) => {
return null;
}
}).filter((item: null) => item !== null);
} else {
// Handle single multimodal content object
const obj = message.content;
if (obj.type === "text") {
return {
role: message.role,
content: obj.text,
};
} else if (obj.type === "file") {
// Handle LanguageModelV2Prompt file format
if (obj.mediaType === "application/pdf") {
return {
role: message.role,
content: {
type: "pdf_url",
pdf_url: {
url: obj.data
}
}
};
} else if (obj.mediaType === "text/markdown" || obj.mediaType === "application/mdx") {
return {
role: message.role,
content: {
type: "mdx_url",
mdx_url: {
url: obj.data
}
}
};
} else if (obj.mediaType && obj.mediaType.startsWith("image/")) {
return {
role: message.role,
content: {
type: "image_url",
image_url: {
url: obj.data
}
}
};
}
} else if (obj.type === "image_url" || obj.type === "image") {
return {
role: message.role,
content: {
type: "image_url",
image_url: {
url: obj.image_url?.url || obj.image?.url || obj.url
}
}
};
} else if (obj.type === "mdx_url" || obj.type === "mdx") {
return {
role: message.role,
content: {
type: "mdx_url",
mdx_url: {
url: obj.mdx_url?.url || obj.mdx?.url || obj.url
}
}
};
} else if (obj.type === "pdf_url" || obj.type === "pdf") {
return {
role: message.role,
content: {
type: "pdf_url",
pdf_url: {
url: obj.pdf_url?.url || obj.pdf?.url || obj.url
}
}
};
}
return null;
}
} catch (error) {
console.error("Error processing message:", error);
vercel-ai-sdk/src/provider-response-provider.ts
+5
View File
@@ -59,6 +59,11 @@ class Mem0AITextGenerator implements LanguageModelV2 {
})(modelId);
break;
case "google":
this.languageModel = createGoogleGenerativeAI({
apiKey: config?.apiKey,
...provider_config as GoogleGenerativeAIProviderSettings,
})(modelId);
break;
case "gemini":
this.languageModel = createGoogleGenerativeAI({
apiKey: config?.apiKey,