feat(mem0-ts): add MongoDB vector store provider to OSS TypeScript SDK (#5793)
Co-authored-by: kartik-mem0 <kartik.labhshetwar@mem0.ai>
This commit is contained in:
@@ -2,13 +2,15 @@
|
||||
title: "MongoDB"
|
||||
description: "Use MongoDB as a vector database in Mem0 with built-in vector search for high-dimensional similarity queries."
|
||||
---
|
||||
|
||||
# MongoDB
|
||||
|
||||
[MongoDB](https://www.mongodb.com/) is a versatile document database that supports vector search capabilities, allowing for efficient high-dimensional similarity searches over large datasets with robust scalability and performance.
|
||||
|
||||
## Usage
|
||||
|
||||
```python
|
||||
<CodeGroup>
|
||||
```python Python
|
||||
import os
|
||||
from mem0 import Memory
|
||||
|
||||
@@ -20,30 +22,90 @@ config = {
|
||||
"config": {
|
||||
"db_name": "mem0-db",
|
||||
"collection_name": "mem0-collection",
|
||||
"mongo_uri":"mongodb://username:password@localhost:27017"
|
||||
"mongo_uri": "mongodb://username:password@localhost:27017"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
m = Memory.from_config(config)
|
||||
messages = [
|
||||
{"role": "user", "content": "I'm planning to watch a movie tonight. Any recommendations?"},
|
||||
{"role": "assistant", "content": "How about thriller movies? They can be quite engaging."},
|
||||
{"role": "user", "content": "I’m not a big fan of thriller movies but I love sci-fi movies."},
|
||||
{"role": "assistant", "content": "Got it! I'll avoid thriller recommendations and suggest sci-fi movies in the future."}
|
||||
{
|
||||
"role": "user",
|
||||
"content": "I'm planning to watch a movie tonight. Any recommendations?",
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "How about thriller movies? They can be quite engaging.",
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "I’m not a big fan of thriller movies but I love sci-fi movies.",
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "Got it! I'll avoid thriller recommendations and suggest sci-fi movies in the future.",
|
||||
},
|
||||
]
|
||||
|
||||
m.add(messages, user_id="alice", metadata={"category": "movies"})
|
||||
```
|
||||
|
||||
```typescript TypeScript
|
||||
import { Memory } from "mem0ai/oss";
|
||||
|
||||
const config = {
|
||||
vectorStore: {
|
||||
provider: "mongodb",
|
||||
config: {
|
||||
dbName: "mem0-db",
|
||||
collectionName: "mem0-collection",
|
||||
url: "mongodb://username:password@localhost:27017",
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const memory = new Memory(config);
|
||||
|
||||
const messages = [
|
||||
{
|
||||
role: "user",
|
||||
content: "I'm planning to watch a movie tonight. Any recommendations?",
|
||||
},
|
||||
{
|
||||
role: "assistant",
|
||||
content: "How about thriller movies? They can be quite engaging.",
|
||||
},
|
||||
{
|
||||
role: "user",
|
||||
content: "I’m not a big fan of thriller movies but I love sci-fi movies.",
|
||||
},
|
||||
{
|
||||
role: "assistant",
|
||||
content:
|
||||
"Got it! I'll avoid thriller recommendations and suggest sci-fi movies in the future.",
|
||||
},
|
||||
];
|
||||
|
||||
await memory.add(messages, {
|
||||
userId: "alice",
|
||||
metadata: {
|
||||
category: "movies",
|
||||
},
|
||||
});
|
||||
```
|
||||
</CodeGroup>
|
||||
|
||||
## Config
|
||||
|
||||
Here are the parameters available for configuring MongoDB:
|
||||
|
||||
| Parameter | Description | Default Value |
|
||||
| --- | --- | --- |
|
||||
| db_name | Name of the MongoDB database | `"mem0_db"` |
|
||||
| collection_name | Name of the MongoDB collection | `"mem0"` |
|
||||
| embedding_model_dims | Dimensions of the embedding vectors | `1536` |
|
||||
| mongo_uri | The MongoDB URI connection string | `mongodb://localhost:27017` |
|
||||
| Python | TypeScript | Description | Default Value |
|
||||
| --- | --- | --- | --- |
|
||||
| db_name | dbName | Name of the MongoDB database | "mem0_db" |
|
||||
| collection_name | collectionName | Name of the MongoDB collection | "mem0" |
|
||||
| embedding_model_dims | embeddingModelDims | Dimensions of the embedding vectors | 1536 |
|
||||
| mongo_uri | url | The MongoDB URI connection string | mongodb://localhost:27017 |
|
||||
|
||||
> **Note**: If `mongo_uri` is not provided, it will default to `mongodb://localhost:27017`.
|
||||
> **Note**: If `mongo_uri` (Python) or `url` (TypeScript) is not provided, it defaults to `mongodb://localhost:27017`. A local instance must be running MongoDB v8.2+ for vector search to work.
|
||||
|
||||
> **Note**: The vector search index builds asynchronously after the first write. A search issued right after the first `add()` may return no results (and log an "index not initialized" message) until the index finishes building. This takes a few seconds on a local deployment and up to about a minute on Atlas. This is expected; the search returns results once the index is ready.
|
||||
|
||||
@@ -129,6 +129,7 @@
|
||||
"cloudflare": "^4.2.0",
|
||||
"fastembed": "^2.1.0",
|
||||
"groq-sdk": "0.3.0",
|
||||
"mongodb": "^7.0.0",
|
||||
"ollama": "^0.5.14",
|
||||
"pg": "8.11.3",
|
||||
"redis": "^4.6.13",
|
||||
|
||||
Generated
+3
@@ -104,6 +104,9 @@ importers:
|
||||
groq-sdk:
|
||||
specifier: 0.3.0
|
||||
version: 0.3.0
|
||||
mongodb:
|
||||
specifier: ^7.0.0
|
||||
version: 7.2.0
|
||||
mysql2:
|
||||
specifier: ^3.0.0
|
||||
version: 3.22.5(@types/node@22.19.21)
|
||||
|
||||
@@ -40,5 +40,6 @@ export * from "./vector_stores/s3_vectors";
|
||||
export * from "./vector_stores/vertex_ai_vector_search";
|
||||
export * from "./vector_stores/pinecone";
|
||||
export * from "./vector_stores/turbopuffer";
|
||||
export * from "./vector_stores/mongodb";
|
||||
export * from "./vector_stores/opensearch";
|
||||
export * from "./utils/factory";
|
||||
|
||||
@@ -0,0 +1,292 @@
|
||||
const mockInsertOne = jest.fn();
|
||||
const mockDeleteOne = jest.fn();
|
||||
const mockInsertMany = jest.fn();
|
||||
const mockFindOne = jest.fn();
|
||||
const mockUpdateOne = jest.fn();
|
||||
const mockListSearchIndexes = jest.fn();
|
||||
const mockCreateSearchIndex = jest.fn();
|
||||
const mockDrop = jest.fn();
|
||||
const mockToArray = jest.fn();
|
||||
const mockLimit = jest.fn().mockReturnThis();
|
||||
const mockFind = jest.fn().mockReturnValue({
|
||||
limit: mockLimit,
|
||||
toArray: mockToArray,
|
||||
});
|
||||
const mockAggregate = jest.fn().mockReturnValue({
|
||||
toArray: mockToArray,
|
||||
});
|
||||
const mockListCollections = jest.fn();
|
||||
const mockClose = jest.fn();
|
||||
|
||||
const mockCollection = {
|
||||
insertOne: mockInsertOne,
|
||||
deleteOne: mockDeleteOne,
|
||||
insertMany: mockInsertMany,
|
||||
findOne: mockFindOne,
|
||||
updateOne: mockUpdateOne,
|
||||
listSearchIndexes: mockListSearchIndexes,
|
||||
createSearchIndex: mockCreateSearchIndex,
|
||||
drop: mockDrop,
|
||||
find: mockFind,
|
||||
aggregate: mockAggregate,
|
||||
};
|
||||
|
||||
const mockDb = {
|
||||
collection: jest.fn().mockReturnValue(mockCollection),
|
||||
listCollections: mockListCollections,
|
||||
};
|
||||
|
||||
const mockMongoClient = jest.fn().mockImplementation(() => {
|
||||
return {
|
||||
db: jest.fn().mockReturnValue(mockDb),
|
||||
close: mockClose,
|
||||
};
|
||||
});
|
||||
|
||||
jest.mock("mongodb", () => {
|
||||
return {
|
||||
MongoClient: mockMongoClient,
|
||||
};
|
||||
});
|
||||
|
||||
import { MongoDB } from "../vector_stores/mongodb";
|
||||
|
||||
describe("MongoDB Vector Store", () => {
|
||||
let store: MongoDB;
|
||||
|
||||
beforeEach(() => {
|
||||
jest.clearAllMocks();
|
||||
mockListCollections.mockReturnValue({
|
||||
toArray: jest.fn().mockResolvedValue([]),
|
||||
});
|
||||
mockListSearchIndexes.mockReturnValue({
|
||||
toArray: jest.fn().mockResolvedValue([]),
|
||||
});
|
||||
store = new MongoDB({
|
||||
url: "mongodb://localhost:27017",
|
||||
dbName: "test_db",
|
||||
collectionName: "test_col",
|
||||
embeddingModelDims: 4,
|
||||
});
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await store.close();
|
||||
});
|
||||
|
||||
it("should initialize client and check/create collection and indexes", async () => {
|
||||
await store.initialize();
|
||||
|
||||
expect(mockListCollections).toHaveBeenCalledWith({ name: "test_col" });
|
||||
expect(mockCollection.insertOne).toHaveBeenCalledWith({
|
||||
_id: 0,
|
||||
placeholder: true,
|
||||
});
|
||||
expect(mockCollection.deleteOne).toHaveBeenCalledWith({ _id: 0 });
|
||||
expect(mockCreateSearchIndex).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("should insert documents correctly", async () => {
|
||||
mockInsertMany.mockResolvedValue({ insertedCount: 2 });
|
||||
|
||||
await store.insert(
|
||||
[
|
||||
[0.1, 0.2, 0.3, 0.4],
|
||||
[0.5, 0.6, 0.7, 0.8],
|
||||
],
|
||||
["id1", "id2"],
|
||||
[{ user: "alice" }, { user: "bob" }],
|
||||
);
|
||||
|
||||
expect(mockInsertMany).toHaveBeenCalledWith([
|
||||
{
|
||||
_id: "id1",
|
||||
embedding: [0.1, 0.2, 0.3, 0.4],
|
||||
payload: { user: "alice" },
|
||||
},
|
||||
{ _id: "id2", embedding: [0.5, 0.6, 0.7, 0.8], payload: { user: "bob" } },
|
||||
]);
|
||||
});
|
||||
|
||||
it("should perform vector search correctly without filters", async () => {
|
||||
mockListSearchIndexes.mockReturnValue({
|
||||
toArray: jest.fn().mockResolvedValue([{ name: "test_col_vector_index" }]),
|
||||
});
|
||||
mockToArray.mockResolvedValue([
|
||||
{ _id: "id1", score: 0.95, payload: { text: "hello" } },
|
||||
{ _id: "id2", score: 0.85, payload: { text: "world" } },
|
||||
]);
|
||||
|
||||
const results = await store.search([0.1, 0.2, 0.3, 0.4], 2);
|
||||
|
||||
expect(results).toEqual([
|
||||
{ id: "id1", score: 0.95, payload: { text: "hello" } },
|
||||
{ id: "id2", score: 0.85, payload: { text: "world" } },
|
||||
]);
|
||||
|
||||
expect(mockAggregate).toHaveBeenCalledWith([
|
||||
{
|
||||
$vectorSearch: {
|
||||
index: "test_col_vector_index",
|
||||
limit: 2,
|
||||
numCandidates: 40,
|
||||
queryVector: [0.1, 0.2, 0.3, 0.4],
|
||||
path: "embedding",
|
||||
},
|
||||
},
|
||||
{ $set: { score: { $meta: "vectorSearchScore" } } },
|
||||
{ $project: { embedding: 0 } },
|
||||
]);
|
||||
});
|
||||
|
||||
it("should perform vector search correctly with filters", async () => {
|
||||
mockListSearchIndexes.mockReturnValue({
|
||||
toArray: jest.fn().mockResolvedValue([{ name: "test_col_vector_index" }]),
|
||||
});
|
||||
mockToArray.mockResolvedValue([]);
|
||||
|
||||
await store.search([0.1, 0.2, 0.3, 0.4], 2, {
|
||||
user: "alice",
|
||||
role: "admin",
|
||||
});
|
||||
|
||||
expect(mockAggregate).toHaveBeenCalledWith([
|
||||
{
|
||||
$vectorSearch: {
|
||||
index: "test_col_vector_index",
|
||||
limit: 2,
|
||||
numCandidates: 40,
|
||||
queryVector: [0.1, 0.2, 0.3, 0.4],
|
||||
path: "embedding",
|
||||
},
|
||||
},
|
||||
{
|
||||
$match: {
|
||||
$and: [{ "payload.user": "alice" }, { "payload.role": "admin" }],
|
||||
},
|
||||
},
|
||||
{ $set: { score: { $meta: "vectorSearchScore" } } },
|
||||
{ $project: { embedding: 0 } },
|
||||
]);
|
||||
});
|
||||
|
||||
it("should reject invalid object/dict filter values", async () => {
|
||||
await expect(
|
||||
store.search([0.1, 0.2, 0.3, 0.4], 5, { user: { name: "alice" } }),
|
||||
).rejects.toThrow("Filter value for 'user' must be a scalar");
|
||||
|
||||
await expect(
|
||||
store.search([0.1, 0.2, 0.3, 0.4], 5, { user: [{ name: "alice" }] }),
|
||||
).rejects.toThrow("Filter list for 'user' contains an object");
|
||||
});
|
||||
|
||||
it("should perform keyword search correctly", async () => {
|
||||
mockToArray.mockResolvedValue([
|
||||
{ _id: "id1", score: 1.5, payload: { data: "test search" } },
|
||||
]);
|
||||
|
||||
const results = await store.keywordSearch("test", 1);
|
||||
|
||||
expect(results).toEqual([
|
||||
{ id: "id1", score: 1.5, payload: { data: "test search" } },
|
||||
]);
|
||||
|
||||
expect(mockAggregate).toHaveBeenCalledWith([
|
||||
{
|
||||
$search: {
|
||||
index: "test_col_text_search_index",
|
||||
text: {
|
||||
query: "test",
|
||||
path: ["payload.data", "payload.text_lemmatized"],
|
||||
},
|
||||
},
|
||||
},
|
||||
{ $set: { score: { $meta: "searchScore" } } },
|
||||
{ $project: { embedding: 0 } },
|
||||
{ $limit: 1 },
|
||||
]);
|
||||
});
|
||||
|
||||
it("should perform get correctly", async () => {
|
||||
mockFindOne.mockResolvedValue({
|
||||
_id: "id1",
|
||||
payload: { data: "get-test" },
|
||||
});
|
||||
|
||||
const result = await store.get("id1");
|
||||
expect(result).toEqual({ id: "id1", payload: { data: "get-test" } });
|
||||
expect(mockFindOne).toHaveBeenCalledWith({ _id: "id1" });
|
||||
});
|
||||
|
||||
it("should return null when get document does not exist", async () => {
|
||||
mockFindOne.mockResolvedValue(null);
|
||||
|
||||
const result = await store.get("id-non-existent");
|
||||
expect(result).toBeNull();
|
||||
});
|
||||
|
||||
it("should update document correctly", async () => {
|
||||
mockUpdateOne.mockResolvedValue({ matchedCount: 1 });
|
||||
|
||||
await store.update("id1", [0.1, 0.2, 0.3, 0.4], { name: "new-alice" });
|
||||
|
||||
expect(mockUpdateOne).toHaveBeenCalledWith(
|
||||
{ _id: "id1" },
|
||||
{
|
||||
$set: {
|
||||
embedding: [0.1, 0.2, 0.3, 0.4],
|
||||
"payload.name": "new-alice",
|
||||
},
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
it("should delete document correctly", async () => {
|
||||
mockDeleteOne.mockResolvedValue({ deletedCount: 1 });
|
||||
|
||||
await store.delete("id1");
|
||||
|
||||
expect(mockDeleteOne).toHaveBeenCalledWith({ _id: "id1" });
|
||||
});
|
||||
|
||||
it("should delete collection correctly", async () => {
|
||||
mockDrop.mockResolvedValue(true);
|
||||
|
||||
await store.deleteCol();
|
||||
|
||||
expect(mockDrop).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("should list documents correctly with filters", async () => {
|
||||
mockToArray.mockResolvedValue([{ _id: "id1", payload: { user: "alice" } }]);
|
||||
|
||||
const [results, count] = await store.list({ user: "alice" }, 10);
|
||||
|
||||
expect(results).toEqual([{ id: "id1", payload: { user: "alice" } }]);
|
||||
expect(count).toBe(1);
|
||||
expect(mockFind).toHaveBeenCalledWith({
|
||||
$and: [{ "payload.user": "alice" }],
|
||||
});
|
||||
expect(mockLimit).toHaveBeenCalledWith(10);
|
||||
});
|
||||
|
||||
it("should manage user ID correctly", async () => {
|
||||
mockFindOne.mockResolvedValue(null);
|
||||
mockUpdateOne.mockResolvedValue({});
|
||||
|
||||
const userId1 = await store.getUserId();
|
||||
expect(userId1).toBeDefined();
|
||||
expect(typeof userId1).toBe("string");
|
||||
|
||||
mockFindOne.mockResolvedValue({ user_id: "custom-user-123" });
|
||||
const userId2 = await store.getUserId();
|
||||
expect(userId2).toBe("custom-user-123");
|
||||
|
||||
await store.setUserId("new-custom-user");
|
||||
expect(mockUpdateOne).toHaveBeenCalledWith(
|
||||
{},
|
||||
{ $set: { user_id: "new-custom-user" } },
|
||||
{ upsert: true },
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -52,6 +52,7 @@ import { CassandraDB } from "../vector_stores/cassandra";
|
||||
import { PineconeDB } from "../vector_stores/pinecone";
|
||||
import { S3Vectors } from "../vector_stores/s3_vectors";
|
||||
import { TurbopufferDB } from "../vector_stores/turbopuffer";
|
||||
import { MongoDB } from "../vector_stores/mongodb";
|
||||
|
||||
export class EmbedderFactory {
|
||||
static create(provider: string, config: EmbeddingConfig): Embedder {
|
||||
@@ -159,6 +160,8 @@ export class VectorStoreFactory {
|
||||
return new S3Vectors(config as any);
|
||||
case "turbopuffer":
|
||||
return new TurbopufferDB(config as any);
|
||||
case "mongodb":
|
||||
return new MongoDB(config as any);
|
||||
default:
|
||||
throw new Error(`Unsupported vector store provider: ${provider}`);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,462 @@
|
||||
import { MongoClient, Collection, Db } from "mongodb";
|
||||
import { VectorStore } from "./base";
|
||||
import { SearchFilters, VectorStoreConfig, VectorStoreResult } from "../types";
|
||||
|
||||
export interface MongoDBConfig extends VectorStoreConfig {
|
||||
url?: string;
|
||||
dbName?: string;
|
||||
collectionName?: string;
|
||||
embeddingModelDims?: number;
|
||||
dimension?: number;
|
||||
client?: MongoClient;
|
||||
}
|
||||
|
||||
export class MongoDB implements VectorStore {
|
||||
private client: MongoClient;
|
||||
private db: Db;
|
||||
private collection!: Collection;
|
||||
private readonly collectionName: string;
|
||||
private readonly dbName: string;
|
||||
private readonly embeddingModelDims: number;
|
||||
private readonly indexName: string;
|
||||
private _initPromise?: Promise<void>;
|
||||
|
||||
constructor(config: MongoDBConfig) {
|
||||
this.collectionName = config.collectionName || "mem0";
|
||||
this.dbName = config.dbName || "mem0_db";
|
||||
this.embeddingModelDims =
|
||||
config.embeddingModelDims || config.dimension || 1536;
|
||||
this.indexName = `${this.collectionName}_vector_index`;
|
||||
|
||||
if (config.client) {
|
||||
this.client = config.client;
|
||||
} else {
|
||||
const url = config.url || "mongodb://localhost:27017";
|
||||
this.client = new MongoClient(url, { appName: "Mem0" });
|
||||
}
|
||||
|
||||
this.db = this.client.db(this.dbName);
|
||||
}
|
||||
|
||||
async initialize(): Promise<void> {
|
||||
if (!this._initPromise) {
|
||||
this._initPromise = this._doInitialize();
|
||||
}
|
||||
return this._initPromise;
|
||||
}
|
||||
|
||||
private async _doInitialize(): Promise<void> {
|
||||
try {
|
||||
const collections = await this.db
|
||||
.listCollections({ name: this.collectionName })
|
||||
.toArray();
|
||||
if (collections.length === 0) {
|
||||
this.collection = this.db.collection(this.collectionName);
|
||||
await this.collection.insertOne({ _id: 0 as any, placeholder: true });
|
||||
await this.collection.deleteOne({ _id: 0 as any });
|
||||
} else {
|
||||
this.collection = this.db.collection(this.collectionName);
|
||||
}
|
||||
|
||||
// Create Vector Search Index
|
||||
try {
|
||||
let foundIndex = false;
|
||||
try {
|
||||
const indexes = await this.collection.listSearchIndexes().toArray();
|
||||
foundIndex = indexes.some((idx) => idx.name === this.indexName);
|
||||
} catch (e) {
|
||||
// listSearchIndexes might not be supported/available on non-Atlas or legacy clusters
|
||||
}
|
||||
|
||||
if (!foundIndex) {
|
||||
await this.collection.createSearchIndex({
|
||||
name: this.indexName,
|
||||
type: "vectorSearch",
|
||||
definition: {
|
||||
fields: [
|
||||
{
|
||||
type: "vector",
|
||||
path: "embedding",
|
||||
numDimensions: this.embeddingModelDims,
|
||||
similarity: "cosine",
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
}
|
||||
} catch (e: any) {
|
||||
console.warn(
|
||||
`Could not verify or create vector search index: ${e.message}`,
|
||||
);
|
||||
}
|
||||
|
||||
// Create Text Search Index for keywordSearch
|
||||
const textIndexName = `${this.collectionName}_text_search_index`;
|
||||
try {
|
||||
let foundTextIndex = false;
|
||||
try {
|
||||
const indexes = await this.collection.listSearchIndexes().toArray();
|
||||
foundTextIndex = indexes.some((idx) => idx.name === textIndexName);
|
||||
} catch (e) {
|
||||
// ignore
|
||||
}
|
||||
|
||||
if (!foundTextIndex) {
|
||||
await this.collection.createSearchIndex({
|
||||
name: textIndexName,
|
||||
definition: {
|
||||
mappings: {
|
||||
dynamic: false,
|
||||
fields: {
|
||||
payload: {
|
||||
type: "document",
|
||||
fields: {
|
||||
data: { type: "string" },
|
||||
text_lemmatized: { type: "string" },
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
}
|
||||
} catch (e: any) {
|
||||
console.warn(
|
||||
`Could not create text search index '${textIndexName}': ${e.message}. ` +
|
||||
`Atlas Search may not be available. keywordSearch() will not work.`,
|
||||
);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error("Error initializing MongoDB:", error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
private validateFilterValue(key: string, value: any): void {
|
||||
if (typeof value === "object" && value !== null) {
|
||||
if (Array.isArray(value)) {
|
||||
for (const item of value) {
|
||||
if (
|
||||
typeof item === "object" &&
|
||||
item !== null &&
|
||||
!Array.isArray(item)
|
||||
) {
|
||||
throw new Error(
|
||||
`Filter list for '${key}' contains an object, which may contain MongoDB query operators.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
throw new Error(
|
||||
`Filter value for '${key}' must be a scalar (string, number, boolean), not an object. Objects may contain MongoDB query operators.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async insert(
|
||||
vectors: number[][],
|
||||
ids: string[],
|
||||
payloads: Record<string, any>[],
|
||||
): Promise<void> {
|
||||
await this.initialize();
|
||||
|
||||
const documents = vectors.map((vector, idx) => ({
|
||||
_id: ids[idx] as any,
|
||||
embedding: vector,
|
||||
payload: payloads[idx] || {},
|
||||
}));
|
||||
|
||||
try {
|
||||
await this.collection.insertMany(documents);
|
||||
} catch (error) {
|
||||
console.error("Error inserting data:", error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async search(
|
||||
query: number[],
|
||||
topK: number = 5,
|
||||
filters?: SearchFilters,
|
||||
): Promise<VectorStoreResult[]> {
|
||||
await this.initialize();
|
||||
|
||||
if (filters) {
|
||||
for (const [key, value] of Object.entries(filters)) {
|
||||
this.validateFilterValue(key, value);
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
let foundIndex = false;
|
||||
try {
|
||||
const indexes = await this.collection.listSearchIndexes().toArray();
|
||||
foundIndex = indexes.some((idx) => idx.name === this.indexName);
|
||||
} catch (e) {
|
||||
// listSearchIndexes might not be supported/available on non-Atlas or legacy clusters
|
||||
foundIndex = true;
|
||||
}
|
||||
|
||||
if (!foundIndex) {
|
||||
console.error(`Index '${this.indexName}' does not exist.`);
|
||||
return [];
|
||||
}
|
||||
|
||||
const pipeline: any[] = [
|
||||
{
|
||||
$vectorSearch: {
|
||||
index: this.indexName,
|
||||
limit: topK,
|
||||
numCandidates: Math.min(topK * 20, 10000),
|
||||
queryVector: query,
|
||||
path: "embedding",
|
||||
},
|
||||
},
|
||||
{ $set: { score: { $meta: "vectorSearchScore" } } },
|
||||
{ $project: { embedding: 0 } },
|
||||
];
|
||||
|
||||
if (filters && Object.keys(filters).length > 0) {
|
||||
const filterConditions: any[] = [];
|
||||
for (const [key, value] of Object.entries(filters)) {
|
||||
filterConditions.push({ [`payload.${key}`]: value });
|
||||
}
|
||||
if (filterConditions.length > 0) {
|
||||
pipeline.splice(1, 0, { $match: { $and: filterConditions } });
|
||||
}
|
||||
}
|
||||
|
||||
const results = await this.collection.aggregate(pipeline).toArray();
|
||||
|
||||
return results.map((doc) => ({
|
||||
id: String(doc._id),
|
||||
score: doc.score,
|
||||
payload: doc.payload || {},
|
||||
}));
|
||||
} catch (error) {
|
||||
// The vector index builds asynchronously after creation; a search issued
|
||||
// before it is queryable throws "Index not initialized". Log the message
|
||||
// (matching the Python provider) rather than the full error object, and
|
||||
// return no results until the index finishes building.
|
||||
console.error(
|
||||
"Error during vector search:",
|
||||
error instanceof Error ? error.message : error,
|
||||
);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
async keywordSearch(
|
||||
query: string,
|
||||
topK: number = 5,
|
||||
filters?: SearchFilters,
|
||||
): Promise<VectorStoreResult[] | null> {
|
||||
await this.initialize();
|
||||
|
||||
if (filters) {
|
||||
for (const [key, value] of Object.entries(filters)) {
|
||||
this.validateFilterValue(key, value);
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
const textIndexName = `${this.collectionName}_text_search_index`;
|
||||
const pipeline: any[] = [
|
||||
{
|
||||
$search: {
|
||||
index: textIndexName,
|
||||
text: {
|
||||
query: query,
|
||||
path: ["payload.data", "payload.text_lemmatized"],
|
||||
},
|
||||
},
|
||||
},
|
||||
{ $set: { score: { $meta: "searchScore" } } },
|
||||
{ $project: { embedding: 0 } },
|
||||
];
|
||||
|
||||
if (filters && Object.keys(filters).length > 0) {
|
||||
const filterConditions: any[] = [];
|
||||
for (const [key, value] of Object.entries(filters)) {
|
||||
filterConditions.push({ [`payload.${key}`]: value });
|
||||
}
|
||||
if (filterConditions.length > 0) {
|
||||
pipeline.splice(1, 0, { $match: { $and: filterConditions } });
|
||||
}
|
||||
}
|
||||
|
||||
pipeline.push({ $limit: topK });
|
||||
|
||||
const results = await this.collection.aggregate(pipeline).toArray();
|
||||
|
||||
return results.map((doc) => ({
|
||||
id: String(doc._id),
|
||||
score: doc.score,
|
||||
payload: doc.payload || {},
|
||||
}));
|
||||
} catch (error) {
|
||||
console.error(
|
||||
"Error during keyword search:",
|
||||
error instanceof Error ? error.message : error,
|
||||
);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async get(vectorId: string): Promise<VectorStoreResult | null> {
|
||||
await this.initialize();
|
||||
try {
|
||||
const doc = await this.collection.findOne({ _id: vectorId as any });
|
||||
if (doc) {
|
||||
return {
|
||||
id: String(doc._id),
|
||||
payload: doc.payload || {},
|
||||
};
|
||||
}
|
||||
return null;
|
||||
} catch (error) {
|
||||
console.error("Error retrieving document:", error);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async update(
|
||||
vectorId: string,
|
||||
vector: number[],
|
||||
payload: Record<string, any>,
|
||||
): Promise<void> {
|
||||
await this.initialize();
|
||||
const updateFields: any = {};
|
||||
if (vector) {
|
||||
updateFields.embedding = vector;
|
||||
}
|
||||
if (payload) {
|
||||
for (const [key, value] of Object.entries(payload)) {
|
||||
updateFields[`payload.${key}`] = value;
|
||||
}
|
||||
}
|
||||
|
||||
if (Object.keys(updateFields).length > 0) {
|
||||
try {
|
||||
const result = await this.collection.updateOne(
|
||||
{ _id: vectorId as any },
|
||||
{ $set: updateFields },
|
||||
);
|
||||
if (result.matchedCount === 0) {
|
||||
console.warn(`No document found with ID '${vectorId}' to update.`);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error("Error updating document:", error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async delete(vectorId: string): Promise<void> {
|
||||
await this.initialize();
|
||||
try {
|
||||
const result = await this.collection.deleteOne({ _id: vectorId as any });
|
||||
if (result.deletedCount === 0) {
|
||||
console.warn(`No document found with ID '${vectorId}' to delete.`);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error("Error deleting document:", error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async deleteCol(): Promise<void> {
|
||||
await this.initialize();
|
||||
try {
|
||||
await this.collection.drop();
|
||||
} catch (error) {
|
||||
console.error("Error deleting collection:", error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async list(
|
||||
filters?: SearchFilters,
|
||||
topK: number = 100,
|
||||
): Promise<[VectorStoreResult[], number]> {
|
||||
await this.initialize();
|
||||
|
||||
if (filters) {
|
||||
for (const [key, value] of Object.entries(filters)) {
|
||||
this.validateFilterValue(key, value);
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
let query: any = {};
|
||||
if (filters && Object.keys(filters).length > 0) {
|
||||
const filterConditions: any[] = [];
|
||||
for (const [key, value] of Object.entries(filters)) {
|
||||
filterConditions.push({ [`payload.${key}`]: value });
|
||||
}
|
||||
if (filterConditions.length > 0) {
|
||||
query = { $and: filterConditions };
|
||||
}
|
||||
}
|
||||
|
||||
const results = await this.collection.find(query).limit(topK).toArray();
|
||||
|
||||
const output = results.map((doc) => ({
|
||||
id: String(doc._id),
|
||||
payload: doc.payload || {},
|
||||
}));
|
||||
|
||||
return [output, results.length];
|
||||
} catch (error) {
|
||||
console.error("Error listing documents:", error);
|
||||
return [[], 0];
|
||||
}
|
||||
}
|
||||
|
||||
async getUserId(): Promise<string> {
|
||||
await this.initialize();
|
||||
try {
|
||||
const migrationsCol = this.db.collection("memory_migrations");
|
||||
const doc = await migrationsCol.findOne({});
|
||||
if (doc && doc.user_id) {
|
||||
return doc.user_id;
|
||||
}
|
||||
|
||||
const randomUserId =
|
||||
Math.random().toString(36).substring(2, 15) +
|
||||
Math.random().toString(36).substring(2, 15);
|
||||
|
||||
await migrationsCol.updateOne(
|
||||
{},
|
||||
{ $set: { user_id: randomUserId } },
|
||||
{ upsert: true },
|
||||
);
|
||||
return randomUserId;
|
||||
} catch (error) {
|
||||
console.error("Error getting user ID:", error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async setUserId(userId: string): Promise<void> {
|
||||
await this.initialize();
|
||||
try {
|
||||
const migrationsCol = this.db.collection("memory_migrations");
|
||||
await migrationsCol.updateOne(
|
||||
{},
|
||||
{ $set: { user_id: userId } },
|
||||
{ upsert: true },
|
||||
);
|
||||
} catch (error) {
|
||||
console.error("Error setting user ID:", error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async close(): Promise<void> {
|
||||
if (this.client) {
|
||||
await this.client.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -31,6 +31,7 @@ const external = [
|
||||
"natural",
|
||||
"mysql2",
|
||||
"@turbopuffer/turbopuffer",
|
||||
"mongodb",
|
||||
"@opensearch-project/opensearch",
|
||||
"@elastic/elasticsearch",
|
||||
];
|
||||
|
||||
Reference in New Issue
Block a user