diff --git a/docs/changelog.mdx b/docs/changelog.mdx index 0a03a8a52..986ccfec5 100644 --- a/docs/changelog.mdx +++ b/docs/changelog.mdx @@ -1087,6 +1087,11 @@ mode: "wide" + +**New Features:** +- **Vercel AI SDK:** Added file support for multimodal capabilities with memory context + + **Bug Fix:** - **Vercel AI SDK:** Fixed streaming response in the AI SDK. diff --git a/docs/integrations/vercel-ai-sdk.mdx b/docs/integrations/vercel-ai-sdk.mdx index 7983ce0a6..29d18c392 100644 --- a/docs/integrations/vercel-ai-sdk.mdx +++ b/docs/integrations/vercel-ai-sdk.mdx @@ -203,6 +203,72 @@ console.log(sources); The same can be done for `streamText` as well. +### 6. File Support with Memory Context + +Mem0 AI SDK supports file processing with memory context. Here's an example of analyzing a PDF file: + +```typescript +import { streamText } from "ai"; +import { createMem0 } from "@mem0/vercel-ai-provider"; +import { readFileSync } from 'fs'; +import { join } from 'path'; + +const mem0 = createMem0({ + provider: "google", + mem0ApiKey: "m0-xxx", + config: { + apiKey: "google-api-key" + }, + mem0Config: { + user_id: "alice", + }, +}); + +async function main() { + // Read the PDF file + const filePath = join(process.cwd(), 'my_pdf.pdf'); + const fileBuffer = readFileSync(filePath); + + // Convert the file's arrayBuffer to a Base64 data URL + const arrayBuffer = fileBuffer.buffer.slice(fileBuffer.byteOffset, fileBuffer.byteOffset + fileBuffer.byteLength); + const uint8Array = new Uint8Array(arrayBuffer); + + // Convert Uint8Array to an array of characters + const charArray = Array.from(uint8Array, byte => String.fromCharCode(byte)); + const binaryString = charArray.join(''); + const base64Data = Buffer.from(binaryString, 'binary').toString('base64'); + const fileDataUrl = `data:application/pdf;base64,${base64Data}`; + + const { textStream } = streamText({ + model: mem0("gemini-2.5-flash"), + messages: [ + { + role: 'user', + content: [ + { + type: 'text', + text: 'Analyze the following PDF and generate a summary.', + }, + { + type: 'file', + data: fileDataUrl, + mediaType: 'application/pdf', + }, + ], + }, + ], + }); + + for await (const textPart of textStream) { + process.stdout.write(textPart); + } +} + +main(); +``` + +> **Note**: File support is available with providers that support multimodal capabilities like Google's Gemini models. The example shows how to process PDF files, but you can also work with images, text files, and other supported formats. + ## Graph Memory Mem0 AI SDK now supports Graph Memory. You can enable it by setting `enable_graph` to `true` in the `mem0Config` object. diff --git a/vercel-ai-sdk/package.json b/vercel-ai-sdk/package.json index 34afea7a1..1ed69210a 100644 --- a/vercel-ai-sdk/package.json +++ b/vercel-ai-sdk/package.json @@ -1,6 +1,6 @@ { "name": "@mem0/vercel-ai-provider", - "version": "2.0.2", + "version": "2.0.3", "description": "Vercel AI Provider for providing memory to LLMs", "main": "./dist/index.js", "module": "./dist/index.mjs", diff --git a/vercel-ai-sdk/src/mem0-utils.ts b/vercel-ai-sdk/src/mem0-utils.ts index 2a3906720..9059a97a4 100644 --- a/vercel-ai-sdk/src/mem0-utils.ts +++ b/vercel-ai-sdk/src/mem0-utils.ts @@ -1,19 +1,62 @@ import { LanguageModelV2Prompt } from '@ai-sdk/provider'; import { Mem0ConfigSettings } from './mem0-types'; import { loadApiKey } from '@ai-sdk/provider-utils'; +interface MultimodalContent { + type: 'text' | 'image_url' | 'mdx_url' | 'pdf_url'; + text?: string; + image_url?: { + url: string; + }; + mdx_url?: { + url: string; + }; + pdf_url?: { + url: string; + }; +} + +interface FileContent { + type: 'file'; + data: string; // fileDataUrl + mediaType: string; // e.g., 'application/pdf', 'text/markdown', 'image/jpeg' +} + interface Message { role: string; - content: string | Array<{type: string, text: string}>; + content: string | MultimodalContent | Array; } const flattenPrompt = (prompt: LanguageModelV2Prompt) => { try { return prompt.map((part) => { if (part.role === "user") { - return part.content - .filter((obj) => obj.type === 'text') - .map((obj) => obj.text) - .join(" "); + if (typeof part.content === 'string') { + return part.content; + } else if (Array.isArray(part.content)) { + return part.content + .filter((obj) => obj.type === 'text') + .map((obj) => obj.text) + .join(" "); + } else if (part.content && typeof part.content === 'object' && 'type' in part.content) { + const content = part.content as any; + if (content.type === 'text' && content.text) { + return content.text; + } else if (content.type === 'file') { + // For file content, we'll include a descriptive placeholder + if (content.mediaType === 'application/pdf') { + return '[PDF document]'; + } else if (content.mediaType === 'text/markdown' || content.mediaType === 'application/mdx') { + return '[Markdown document]'; + } else if (content.mediaType && content.mediaType.startsWith('image/')) { + return '[Image]'; + } else { + return '[File attachment]'; + } + } + } + // For non-text content (images, pdfs, mdx), we'll include a placeholder + // This helps maintain context for memory search while not breaking the text flow + return "[multimodal content]"; } return ""; }).join(" "); @@ -33,7 +76,7 @@ const convertToMem0Format = (messages: LanguageModelV2Prompt) => { content: message.content, }; } - else { + else if (Array.isArray(message.content)) { return message.content.map((obj: any) => { try { if (obj.type === "text") { @@ -41,6 +84,69 @@ const convertToMem0Format = (messages: LanguageModelV2Prompt) => { role: message.role, content: obj.text, }; + } else if (obj.type === "file") { + // Handle LanguageModelV2Prompt file format + if (obj.mediaType === "application/pdf") { + return { + role: message.role, + content: { + type: "pdf_url", + pdf_url: { + url: obj.data + } + } + }; + } else if (obj.mediaType === "text/markdown" || obj.mediaType === "application/mdx") { + return { + role: message.role, + content: { + type: "mdx_url", + mdx_url: { + url: obj.data + } + } + }; + } else if (obj.mediaType && obj.mediaType.startsWith("image/")) { + return { + role: message.role, + content: { + type: "image_url", + image_url: { + url: obj.data + } + } + }; + } + } else if (obj.type === "image_url" || obj.type === "image") { + return { + role: message.role, + content: { + type: "image_url", + image_url: { + url: obj.image_url?.url || obj.image?.url || obj.url + } + } + }; + } else if (obj.type === "mdx_url" || obj.type === "mdx") { + return { + role: message.role, + content: { + type: "mdx_url", + mdx_url: { + url: obj.mdx_url?.url || obj.mdx?.url || obj.url + } + } + }; + } else if (obj.type === "pdf_url" || obj.type === "pdf") { + return { + role: message.role, + content: { + type: "pdf_url", + pdf_url: { + url: obj.pdf_url?.url || obj.pdf?.url || obj.url + } + } + }; } return null; } catch (error) { @@ -48,6 +154,79 @@ const convertToMem0Format = (messages: LanguageModelV2Prompt) => { return null; } }).filter((item: null) => item !== null); + } else { + // Handle single multimodal content object + const obj = message.content; + if (obj.type === "text") { + return { + role: message.role, + content: obj.text, + }; + } else if (obj.type === "file") { + // Handle LanguageModelV2Prompt file format + if (obj.mediaType === "application/pdf") { + return { + role: message.role, + content: { + type: "pdf_url", + pdf_url: { + url: obj.data + } + } + }; + } else if (obj.mediaType === "text/markdown" || obj.mediaType === "application/mdx") { + return { + role: message.role, + content: { + type: "mdx_url", + mdx_url: { + url: obj.data + } + } + }; + } else if (obj.mediaType && obj.mediaType.startsWith("image/")) { + return { + role: message.role, + content: { + type: "image_url", + image_url: { + url: obj.data + } + } + }; + } + } else if (obj.type === "image_url" || obj.type === "image") { + return { + role: message.role, + content: { + type: "image_url", + image_url: { + url: obj.image_url?.url || obj.image?.url || obj.url + } + } + }; + } else if (obj.type === "mdx_url" || obj.type === "mdx") { + return { + role: message.role, + content: { + type: "mdx_url", + mdx_url: { + url: obj.mdx_url?.url || obj.mdx?.url || obj.url + } + } + }; + } else if (obj.type === "pdf_url" || obj.type === "pdf") { + return { + role: message.role, + content: { + type: "pdf_url", + pdf_url: { + url: obj.pdf_url?.url || obj.pdf?.url || obj.url + } + } + }; + } + return null; } } catch (error) { console.error("Error processing message:", error); diff --git a/vercel-ai-sdk/src/provider-response-provider.ts b/vercel-ai-sdk/src/provider-response-provider.ts index d7132b1e4..c849bd289 100644 --- a/vercel-ai-sdk/src/provider-response-provider.ts +++ b/vercel-ai-sdk/src/provider-response-provider.ts @@ -59,6 +59,11 @@ class Mem0AITextGenerator implements LanguageModelV2 { })(modelId); break; case "google": + this.languageModel = createGoogleGenerativeAI({ + apiKey: config?.apiKey, + ...provider_config as GoogleGenerativeAIProviderSettings, + })(modelId); + break; case "gemini": this.languageModel = createGoogleGenerativeAI({ apiKey: config?.apiKey,