fix(ts-sdk): replace sqlite3 with better-sqlite3 to fix native binding resolution (#4270)
This commit is contained in:
@@ -729,6 +729,23 @@ mode: "wide"
|
||||
|
||||
<Tab title="TypeScript">
|
||||
|
||||
<Update label="2026-03-09" description="v2.3.0">
|
||||
|
||||
**Breaking Changes:**
|
||||
- **Dependencies:** Minimum Node.js version for OSS sqlite features is now Node 20+ (due to `better-sqlite3` v12)
|
||||
|
||||
**Bug Fixes:**
|
||||
- **OSS Storage:** Replaced `sqlite3` with `better-sqlite3` to fix native binding resolution failures under jiti-based loaders (e.g. OpenClaw plugin system). Fixes issues where the `bindings` module walked V8 stack frames with synthetic filenames, failing to locate the native `.node` addon.
|
||||
- **OSS Storage:** Fixed async init race condition in `SQLiteManager` — `init()` is now synchronous
|
||||
- **OSS Vector Store:** Migrated `MemoryVectorStore` from `sqlite3` to `better-sqlite3` with transactional batch inserts
|
||||
|
||||
**Improvements:**
|
||||
- **Performance:** Cached prepared statements in `SQLiteManager` for faster history operations
|
||||
- **Performance:** Batch `insert()` in `MemoryVectorStore` wrapped in a transaction for atomicity
|
||||
- **Build:** Updated `tsup.config.ts` externals from `sqlite3` to `better-sqlite3`
|
||||
|
||||
</Update>
|
||||
|
||||
<Update label="2026-02-17" description="v2.2.3">
|
||||
|
||||
**New Features & Updates:**
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "mem0ai",
|
||||
"version": "2.2.4",
|
||||
"version": "2.3.0",
|
||||
"description": "The Memory Layer For Your AI Apps",
|
||||
"main": "./dist/index.js",
|
||||
"module": "./dist/index.mjs",
|
||||
@@ -77,6 +77,7 @@
|
||||
"author": "Deshraj Yadav",
|
||||
"license": "Apache-2.0",
|
||||
"devDependencies": {
|
||||
"@types/better-sqlite3": "^7.6.13",
|
||||
"@types/node": "^22.7.6",
|
||||
"@types/uuid": "^9.0.8",
|
||||
"dotenv": "^16.4.5",
|
||||
@@ -108,14 +109,13 @@
|
||||
"@supabase/supabase-js": "^2.49.1",
|
||||
"@types/jest": "29.5.14",
|
||||
"@types/pg": "8.11.0",
|
||||
"@types/sqlite3": "3.1.11",
|
||||
"better-sqlite3": "^12.6.2",
|
||||
"cloudflare": "^4.2.0",
|
||||
"groq-sdk": "0.3.0",
|
||||
"neo4j-driver": "^5.28.1",
|
||||
"ollama": "^0.5.14",
|
||||
"pg": "8.11.3",
|
||||
"redis": "^4.6.13",
|
||||
"sqlite3": "5.1.7"
|
||||
"redis": "^4.6.13"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
@@ -127,7 +127,7 @@
|
||||
"pnpm": {
|
||||
"onlyBuiltDependencies": [
|
||||
"esbuild",
|
||||
"sqlite3"
|
||||
"better-sqlite3"
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
Generated
+29
-765
File diff suppressed because it is too large
Load Diff
@@ -19,7 +19,6 @@
|
||||
"@types/node": "^20.11.19",
|
||||
"@types/pg": "^8.11.0",
|
||||
"@types/redis": "^4.0.10",
|
||||
"@types/sqlite3": "^3.1.11",
|
||||
"@types/uuid": "^9.0.8",
|
||||
"cloudflare": "^4.2.0",
|
||||
"dotenv": "^16.4.4",
|
||||
@@ -27,7 +26,7 @@
|
||||
"openai": "^4.28.0",
|
||||
"pg": "^8.11.3",
|
||||
"redis": "^4.7.0",
|
||||
"sqlite3": "^5.1.7",
|
||||
"better-sqlite3": "^12.6.2",
|
||||
"uuid": "^9.0.1",
|
||||
"zod": "^3.22.4"
|
||||
},
|
||||
|
||||
@@ -1,16 +1,18 @@
|
||||
import sqlite3 from "sqlite3";
|
||||
import Database from "better-sqlite3";
|
||||
import { HistoryManager } from "./base";
|
||||
|
||||
export class SQLiteManager implements HistoryManager {
|
||||
private db: sqlite3.Database;
|
||||
private db: Database.Database;
|
||||
private stmtInsert!: Database.Statement;
|
||||
private stmtSelect!: Database.Statement;
|
||||
|
||||
constructor(dbPath: string) {
|
||||
this.db = new sqlite3.Database(dbPath);
|
||||
this.init().catch(console.error);
|
||||
this.db = new Database(dbPath);
|
||||
this.init();
|
||||
}
|
||||
|
||||
private async init() {
|
||||
await this.run(`
|
||||
private init(): void {
|
||||
this.db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS memory_history (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
memory_id TEXT NOT NULL,
|
||||
@@ -22,24 +24,14 @@ export class SQLiteManager implements HistoryManager {
|
||||
is_deleted INTEGER DEFAULT 0
|
||||
)
|
||||
`);
|
||||
}
|
||||
|
||||
private async run(sql: string, params: any[] = []): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
this.db.run(sql, params, (err) => {
|
||||
if (err) reject(err);
|
||||
else resolve();
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
private async all(sql: string, params: any[] = []): Promise<any[]> {
|
||||
return new Promise((resolve, reject) => {
|
||||
this.db.all(sql, params, (err, rows) => {
|
||||
if (err) reject(err);
|
||||
else resolve(rows);
|
||||
});
|
||||
});
|
||||
this.stmtInsert = this.db.prepare(
|
||||
`INSERT INTO memory_history
|
||||
(memory_id, previous_value, new_value, action, created_at, updated_at, is_deleted)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)`,
|
||||
);
|
||||
this.stmtSelect = this.db.prepare(
|
||||
"SELECT * FROM memory_history WHERE memory_id = ? ORDER BY id DESC",
|
||||
);
|
||||
}
|
||||
|
||||
async addHistory(
|
||||
@@ -51,32 +43,24 @@ export class SQLiteManager implements HistoryManager {
|
||||
updatedAt?: string,
|
||||
isDeleted: number = 0,
|
||||
): Promise<void> {
|
||||
await this.run(
|
||||
`INSERT INTO memory_history
|
||||
(memory_id, previous_value, new_value, action, created_at, updated_at, is_deleted)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)`,
|
||||
[
|
||||
this.stmtInsert.run(
|
||||
memoryId,
|
||||
previousValue,
|
||||
newValue,
|
||||
action,
|
||||
createdAt,
|
||||
updatedAt,
|
||||
createdAt ?? null,
|
||||
updatedAt ?? null,
|
||||
isDeleted,
|
||||
],
|
||||
);
|
||||
}
|
||||
|
||||
async getHistory(memoryId: string): Promise<any[]> {
|
||||
return this.all(
|
||||
"SELECT * FROM memory_history WHERE memory_id = ? ORDER BY id DESC",
|
||||
[memoryId],
|
||||
);
|
||||
return this.stmtSelect.all(memoryId) as any[];
|
||||
}
|
||||
|
||||
async reset(): Promise<void> {
|
||||
await this.run("DROP TABLE IF EXISTS memory_history");
|
||||
await this.init();
|
||||
this.db.exec("DROP TABLE IF EXISTS memory_history");
|
||||
this.init();
|
||||
}
|
||||
|
||||
close(): void {
|
||||
|
||||
@@ -0,0 +1,377 @@
|
||||
/**
|
||||
* Tests for the sqlite3 → better-sqlite3 migration.
|
||||
*
|
||||
* Covers:
|
||||
* - SQLiteManager: all HistoryManager interface methods
|
||||
* - MemoryVectorStore: insert, search, get, update, delete, list, userId mgmt
|
||||
* - File-based persistence and in-memory mode
|
||||
* - Backward compatibility: same schema, same data shapes
|
||||
*/
|
||||
|
||||
import { SQLiteManager } from "../storage/SQLiteManager";
|
||||
import { MemoryVectorStore } from "../vector_stores/memory";
|
||||
import fs from "fs";
|
||||
import path from "path";
|
||||
import os from "os";
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// SQLiteManager tests
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe("SQLiteManager (better-sqlite3)", () => {
|
||||
let mgr: SQLiteManager;
|
||||
|
||||
beforeEach(() => {
|
||||
mgr = new SQLiteManager(":memory:");
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
mgr.close();
|
||||
});
|
||||
|
||||
it("creates an in-memory database without errors", () => {
|
||||
expect(mgr).toBeDefined();
|
||||
});
|
||||
|
||||
it("addHistory inserts a row that getHistory returns", async () => {
|
||||
await mgr.addHistory(
|
||||
"mem-001",
|
||||
null,
|
||||
"User likes TypeScript",
|
||||
"ADD",
|
||||
"2026-01-01T00:00:00Z",
|
||||
);
|
||||
|
||||
const history = await mgr.getHistory("mem-001");
|
||||
expect(history).toHaveLength(1);
|
||||
expect(history[0].memory_id).toBe("mem-001");
|
||||
expect(history[0].new_value).toBe("User likes TypeScript");
|
||||
expect(history[0].action).toBe("ADD");
|
||||
expect(history[0].previous_value).toBeNull();
|
||||
expect(history[0].is_deleted).toBe(0);
|
||||
});
|
||||
|
||||
it("returns history in DESC order (most recent first)", async () => {
|
||||
await mgr.addHistory("mem-002", null, "First", "ADD", "2026-01-01");
|
||||
await mgr.addHistory(
|
||||
"mem-002",
|
||||
"First",
|
||||
"Updated",
|
||||
"UPDATE",
|
||||
"2026-01-01",
|
||||
"2026-01-02",
|
||||
);
|
||||
await mgr.addHistory(
|
||||
"mem-002",
|
||||
"Updated",
|
||||
null,
|
||||
"DELETE",
|
||||
undefined,
|
||||
undefined,
|
||||
1,
|
||||
);
|
||||
|
||||
const history = await mgr.getHistory("mem-002");
|
||||
expect(history).toHaveLength(3);
|
||||
expect(history[0].action).toBe("DELETE");
|
||||
expect(history[1].action).toBe("UPDATE");
|
||||
expect(history[2].action).toBe("ADD");
|
||||
expect(history[0].is_deleted).toBe(1);
|
||||
});
|
||||
|
||||
it("isolates history by memory_id", async () => {
|
||||
await mgr.addHistory("mem-A", null, "Fact A", "ADD", "2026-01-01");
|
||||
await mgr.addHistory("mem-B", null, "Fact B", "ADD", "2026-01-01");
|
||||
|
||||
expect(await mgr.getHistory("mem-A")).toHaveLength(1);
|
||||
expect(await mgr.getHistory("mem-B")).toHaveLength(1);
|
||||
expect((await mgr.getHistory("mem-A"))[0].new_value).toBe("Fact A");
|
||||
expect((await mgr.getHistory("mem-B"))[0].new_value).toBe("Fact B");
|
||||
});
|
||||
|
||||
it("handles NULL/undefined optional fields correctly", async () => {
|
||||
await mgr.addHistory(
|
||||
"mem-null",
|
||||
null,
|
||||
null,
|
||||
"DELETE",
|
||||
undefined,
|
||||
undefined,
|
||||
1,
|
||||
);
|
||||
|
||||
const history = await mgr.getHistory("mem-null");
|
||||
expect(history).toHaveLength(1);
|
||||
expect(history[0].previous_value).toBeNull();
|
||||
expect(history[0].new_value).toBeNull();
|
||||
expect(history[0].created_at).toBeNull();
|
||||
expect(history[0].updated_at).toBeNull();
|
||||
});
|
||||
|
||||
it("reset() clears all history and allows re-insertion", async () => {
|
||||
await mgr.addHistory("mem-003", null, "Fact", "ADD", "2026-01-01");
|
||||
expect(await mgr.getHistory("mem-003")).toHaveLength(1);
|
||||
|
||||
await mgr.reset();
|
||||
expect(await mgr.getHistory("mem-003")).toHaveLength(0);
|
||||
|
||||
await mgr.addHistory("mem-004", null, "New fact", "ADD", "2026-02-01");
|
||||
expect(await mgr.getHistory("mem-004")).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("works with file-based database and persists data", async () => {
|
||||
const dbPath = path.join(os.tmpdir(), `mem0-test-history-${Date.now()}.db`);
|
||||
|
||||
try {
|
||||
const fmgr = new SQLiteManager(dbPath);
|
||||
await fmgr.addHistory(
|
||||
"mem-file",
|
||||
null,
|
||||
"Persistent",
|
||||
"ADD",
|
||||
"2026-01-01",
|
||||
);
|
||||
fmgr.close();
|
||||
|
||||
expect(fs.existsSync(dbPath)).toBe(true);
|
||||
expect(fs.statSync(dbPath).size).toBeGreaterThan(0);
|
||||
|
||||
// Reopen and verify
|
||||
const fmgr2 = new SQLiteManager(dbPath);
|
||||
const history = await fmgr2.getHistory("mem-file");
|
||||
expect(history).toHaveLength(1);
|
||||
expect(history[0].new_value).toBe("Persistent");
|
||||
fmgr2.close();
|
||||
} finally {
|
||||
if (fs.existsSync(dbPath)) fs.unlinkSync(dbPath);
|
||||
}
|
||||
});
|
||||
|
||||
it("handles many rapid insertions", async () => {
|
||||
for (let i = 0; i < 100; i++) {
|
||||
await mgr.addHistory(
|
||||
`mem-rapid-${i}`,
|
||||
null,
|
||||
`Fact ${i}`,
|
||||
"ADD",
|
||||
new Date().toISOString(),
|
||||
);
|
||||
}
|
||||
|
||||
for (let i = 0; i < 100; i++) {
|
||||
const h = await mgr.getHistory(`mem-rapid-${i}`);
|
||||
expect(h).toHaveLength(1);
|
||||
expect(h[0].new_value).toBe(`Fact ${i}`);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// MemoryVectorStore tests
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe("MemoryVectorStore (better-sqlite3)", () => {
|
||||
const DIM = 4;
|
||||
let store: MemoryVectorStore;
|
||||
let dbPath: string;
|
||||
|
||||
function normalize(v: number[]): number[] {
|
||||
const norm = Math.sqrt(v.reduce((s, x) => s + x * x, 0));
|
||||
return v.map((x) => x / norm);
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
dbPath = path.join(os.tmpdir(), `mem0-test-vectors-${Date.now()}.db`);
|
||||
store = new MemoryVectorStore({ dimension: DIM, dbPath } as any);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
if (fs.existsSync(dbPath)) fs.unlinkSync(dbPath);
|
||||
});
|
||||
|
||||
it("insert + get returns the stored payload", async () => {
|
||||
const v = normalize([1, 0, 0, 0]);
|
||||
await store.insert([v], ["id-1"], [{ data: "hello", userId: "u1" }]);
|
||||
|
||||
const result = await store.get("id-1");
|
||||
expect(result).not.toBeNull();
|
||||
expect(result!.id).toBe("id-1");
|
||||
expect(result!.payload.data).toBe("hello");
|
||||
expect(result!.payload.userId).toBe("u1");
|
||||
});
|
||||
|
||||
it("get returns null for non-existent id", async () => {
|
||||
const result = await store.get("nope");
|
||||
expect(result).toBeNull();
|
||||
});
|
||||
|
||||
it("search returns results sorted by cosine similarity", async () => {
|
||||
const v1 = normalize([1, 0, 0, 0]);
|
||||
const v2 = normalize([0, 1, 0, 0]);
|
||||
const v3 = normalize([1, 1, 0, 0]);
|
||||
|
||||
await store.insert(
|
||||
[v1, v2, v3],
|
||||
["id-1", "id-2", "id-3"],
|
||||
[{ data: "exact" }, { data: "orthogonal" }, { data: "close" }],
|
||||
);
|
||||
|
||||
const results = await store.search(v1, 3);
|
||||
expect(results).toHaveLength(3);
|
||||
expect(results[0].id).toBe("id-1");
|
||||
expect(results[0].score).toBeCloseTo(1.0, 5);
|
||||
expect(results[1].id).toBe("id-3");
|
||||
expect(results[2].id).toBe("id-2");
|
||||
expect(results[2].score).toBeCloseTo(0, 5);
|
||||
});
|
||||
|
||||
it("search respects limit", async () => {
|
||||
const vectors = [];
|
||||
const ids = [];
|
||||
const payloads = [];
|
||||
for (let i = 0; i < 10; i++) {
|
||||
const v = [0, 0, 0, 0];
|
||||
v[i % DIM] = 1;
|
||||
vectors.push(normalize(v));
|
||||
ids.push(`id-${i}`);
|
||||
payloads.push({ data: `item-${i}` });
|
||||
}
|
||||
await store.insert(vectors, ids, payloads);
|
||||
|
||||
const results = await store.search(normalize([1, 0, 0, 0]), 3);
|
||||
expect(results).toHaveLength(3);
|
||||
});
|
||||
|
||||
it("search respects filters", async () => {
|
||||
const v = normalize([1, 0, 0, 0]);
|
||||
await store.insert(
|
||||
[v, v],
|
||||
["id-1", "id-2"],
|
||||
[
|
||||
{ data: "a", userId: "alice" },
|
||||
{ data: "b", userId: "bob" },
|
||||
],
|
||||
);
|
||||
|
||||
const results = await store.search(v, 10, { userId: "alice" });
|
||||
expect(results).toHaveLength(1);
|
||||
expect(results[0].id).toBe("id-1");
|
||||
});
|
||||
|
||||
it("search throws on dimension mismatch", async () => {
|
||||
await expect(store.search([1, 0, 0], 10)).rejects.toThrow(
|
||||
"dimension mismatch",
|
||||
);
|
||||
});
|
||||
|
||||
it("insert throws on dimension mismatch", async () => {
|
||||
await expect(
|
||||
store.insert([[1, 0, 0]], ["id-1"], [{ data: "x" }]),
|
||||
).rejects.toThrow("dimension mismatch");
|
||||
});
|
||||
|
||||
it("update modifies the stored vector and payload", async () => {
|
||||
const v1 = normalize([1, 0, 0, 0]);
|
||||
const v2 = normalize([0, 1, 0, 0]);
|
||||
await store.insert([v1], ["id-1"], [{ data: "original" }]);
|
||||
|
||||
await store.update("id-1", v2, { data: "updated" });
|
||||
|
||||
const result = await store.get("id-1");
|
||||
expect(result!.payload.data).toBe("updated");
|
||||
|
||||
const results = await store.search(v2, 1);
|
||||
expect(results[0].id).toBe("id-1");
|
||||
expect(results[0].score).toBeCloseTo(1.0, 5);
|
||||
});
|
||||
|
||||
it("delete removes the vector", async () => {
|
||||
const v = normalize([1, 0, 0, 0]);
|
||||
await store.insert([v], ["id-1"], [{ data: "doomed" }]);
|
||||
|
||||
await store.delete("id-1");
|
||||
expect(await store.get("id-1")).toBeNull();
|
||||
});
|
||||
|
||||
it("deleteCol drops and recreates table", async () => {
|
||||
const v = normalize([1, 0, 0, 0]);
|
||||
await store.insert([v], ["id-1"], [{ data: "will be gone" }]);
|
||||
|
||||
await store.deleteCol();
|
||||
expect(await store.get("id-1")).toBeNull();
|
||||
|
||||
await store.insert([v], ["id-2"], [{ data: "fresh" }]);
|
||||
expect(await store.get("id-2")).not.toBeNull();
|
||||
});
|
||||
|
||||
it("list returns all vectors with optional filters", async () => {
|
||||
const v = normalize([1, 0, 0, 0]);
|
||||
await store.insert(
|
||||
[v, v, v],
|
||||
["id-1", "id-2", "id-3"],
|
||||
[
|
||||
{ data: "a", userId: "alice" },
|
||||
{ data: "b", userId: "bob" },
|
||||
{ data: "c", userId: "alice" },
|
||||
],
|
||||
);
|
||||
|
||||
const [all, totalAll] = await store.list();
|
||||
expect(all).toHaveLength(3);
|
||||
expect(totalAll).toBe(3);
|
||||
|
||||
const [filtered, totalFiltered] = await store.list({ userId: "alice" });
|
||||
expect(filtered).toHaveLength(2);
|
||||
expect(totalFiltered).toBe(2);
|
||||
});
|
||||
|
||||
it("list respects limit", async () => {
|
||||
const v = normalize([1, 0, 0, 0]);
|
||||
await store.insert(
|
||||
[v, v, v],
|
||||
["id-1", "id-2", "id-3"],
|
||||
[{ data: "a" }, { data: "b" }, { data: "c" }],
|
||||
);
|
||||
|
||||
const [results] = await store.list(undefined, 2);
|
||||
expect(results).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("getUserId generates and persists a userId", async () => {
|
||||
const id1 = await store.getUserId();
|
||||
expect(typeof id1).toBe("string");
|
||||
expect(id1.length).toBeGreaterThan(0);
|
||||
|
||||
const id2 = await store.getUserId();
|
||||
expect(id2).toBe(id1);
|
||||
});
|
||||
|
||||
it("setUserId overrides the stored userId", async () => {
|
||||
await store.setUserId("custom-user-123");
|
||||
const id = await store.getUserId();
|
||||
expect(id).toBe("custom-user-123");
|
||||
});
|
||||
|
||||
it("INSERT OR REPLACE upserts on id conflict", async () => {
|
||||
const v1 = normalize([1, 0, 0, 0]);
|
||||
const v2 = normalize([0, 1, 0, 0]);
|
||||
await store.insert([v1], ["id-1"], [{ data: "original" }]);
|
||||
await store.insert([v2], ["id-1"], [{ data: "replaced" }]);
|
||||
|
||||
const result = await store.get("id-1");
|
||||
expect(result!.payload.data).toBe("replaced");
|
||||
|
||||
const [all] = await store.list();
|
||||
expect(all).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("file-based database persists across reopens", async () => {
|
||||
const v = normalize([1, 0, 0, 0]);
|
||||
await store.insert([v], ["id-persist"], [{ data: "persistent" }]);
|
||||
|
||||
const store2 = new MemoryVectorStore({ dimension: DIM, dbPath } as any);
|
||||
const result = await store2.get("id-persist");
|
||||
expect(result).not.toBeNull();
|
||||
expect(result!.payload.data).toBe("persistent");
|
||||
});
|
||||
});
|
||||
@@ -1,6 +1,6 @@
|
||||
import { VectorStore } from "./base";
|
||||
import { SearchFilters, VectorStoreConfig, VectorStoreResult } from "../types";
|
||||
import sqlite3 from "sqlite3";
|
||||
import Database from "better-sqlite3";
|
||||
import path from "path";
|
||||
|
||||
interface MemoryVector {
|
||||
@@ -10,7 +10,7 @@ interface MemoryVector {
|
||||
}
|
||||
|
||||
export class MemoryVectorStore implements VectorStore {
|
||||
private db: sqlite3.Database;
|
||||
private db: Database.Database;
|
||||
private dimension: number;
|
||||
private dbPath: string;
|
||||
|
||||
@@ -20,12 +20,12 @@ export class MemoryVectorStore implements VectorStore {
|
||||
if (config.dbPath) {
|
||||
this.dbPath = config.dbPath;
|
||||
}
|
||||
this.db = new sqlite3.Database(this.dbPath);
|
||||
this.init().catch(console.error);
|
||||
this.db = new Database(this.dbPath);
|
||||
this.init();
|
||||
}
|
||||
|
||||
private async init() {
|
||||
await this.run(`
|
||||
private init(): void {
|
||||
this.db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS vectors (
|
||||
id TEXT PRIMARY KEY,
|
||||
vector BLOB NOT NULL,
|
||||
@@ -33,7 +33,7 @@ export class MemoryVectorStore implements VectorStore {
|
||||
)
|
||||
`);
|
||||
|
||||
await this.run(`
|
||||
this.db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS memory_migrations (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
user_id TEXT NOT NULL UNIQUE
|
||||
@@ -41,33 +41,6 @@ export class MemoryVectorStore implements VectorStore {
|
||||
`);
|
||||
}
|
||||
|
||||
private async run(sql: string, params: any[] = []): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
this.db.run(sql, params, (err) => {
|
||||
if (err) reject(err);
|
||||
else resolve();
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
private async all(sql: string, params: any[] = []): Promise<any[]> {
|
||||
return new Promise((resolve, reject) => {
|
||||
this.db.all(sql, params, (err, rows) => {
|
||||
if (err) reject(err);
|
||||
else resolve(rows);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
private async getOne(sql: string, params: any[] = []): Promise<any> {
|
||||
return new Promise((resolve, reject) => {
|
||||
this.db.get(sql, params, (err, row) => {
|
||||
if (err) reject(err);
|
||||
else resolve(row);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
private cosineSimilarity(a: number[], b: number[]): number {
|
||||
let dotProduct = 0;
|
||||
let normA = 0;
|
||||
@@ -92,18 +65,23 @@ export class MemoryVectorStore implements VectorStore {
|
||||
ids: string[],
|
||||
payloads: Record<string, any>[],
|
||||
): Promise<void> {
|
||||
for (let i = 0; i < vectors.length; i++) {
|
||||
if (vectors[i].length !== this.dimension) {
|
||||
throw new Error(
|
||||
`Vector dimension mismatch. Expected ${this.dimension}, got ${vectors[i].length}`,
|
||||
);
|
||||
}
|
||||
const vectorBuffer = Buffer.from(new Float32Array(vectors[i]).buffer);
|
||||
await this.run(
|
||||
const stmt = this.db.prepare(
|
||||
`INSERT OR REPLACE INTO vectors (id, vector, payload) VALUES (?, ?, ?)`,
|
||||
[ids[i], vectorBuffer, JSON.stringify(payloads[i])],
|
||||
);
|
||||
const insertMany = this.db.transaction(
|
||||
(vecs: number[][], vIds: string[], vPayloads: Record<string, any>[]) => {
|
||||
for (let i = 0; i < vecs.length; i++) {
|
||||
if (vecs[i].length !== this.dimension) {
|
||||
throw new Error(
|
||||
`Vector dimension mismatch. Expected ${this.dimension}, got ${vecs[i].length}`,
|
||||
);
|
||||
}
|
||||
const vectorBuffer = Buffer.from(new Float32Array(vecs[i]).buffer);
|
||||
stmt.run(vIds[i], vectorBuffer, JSON.stringify(vPayloads[i]));
|
||||
}
|
||||
},
|
||||
);
|
||||
insertMany(vectors, ids, payloads);
|
||||
}
|
||||
|
||||
async search(
|
||||
@@ -117,11 +95,15 @@ export class MemoryVectorStore implements VectorStore {
|
||||
);
|
||||
}
|
||||
|
||||
const rows = await this.all(`SELECT * FROM vectors`);
|
||||
const rows = this.db.prepare(`SELECT * FROM vectors`).all() as any[];
|
||||
const results: VectorStoreResult[] = [];
|
||||
|
||||
for (const row of rows) {
|
||||
const vector = new Float32Array(row.vector.buffer);
|
||||
const vector = new Float32Array(
|
||||
row.vector.buffer,
|
||||
row.vector.byteOffset,
|
||||
row.vector.byteLength / 4,
|
||||
);
|
||||
const payload = JSON.parse(row.payload);
|
||||
const memoryVector: MemoryVector = {
|
||||
id: row.id,
|
||||
@@ -144,9 +126,9 @@ export class MemoryVectorStore implements VectorStore {
|
||||
}
|
||||
|
||||
async get(vectorId: string): Promise<VectorStoreResult | null> {
|
||||
const row = await this.getOne(`SELECT * FROM vectors WHERE id = ?`, [
|
||||
vectorId,
|
||||
]);
|
||||
const row = this.db
|
||||
.prepare(`SELECT * FROM vectors WHERE id = ?`)
|
||||
.get(vectorId) as any;
|
||||
if (!row) return null;
|
||||
|
||||
const payload = JSON.parse(row.payload);
|
||||
@@ -167,34 +149,38 @@ export class MemoryVectorStore implements VectorStore {
|
||||
);
|
||||
}
|
||||
const vectorBuffer = Buffer.from(new Float32Array(vector).buffer);
|
||||
await this.run(`UPDATE vectors SET vector = ?, payload = ? WHERE id = ?`, [
|
||||
vectorBuffer,
|
||||
JSON.stringify(payload),
|
||||
vectorId,
|
||||
]);
|
||||
this.db
|
||||
.prepare(`UPDATE vectors SET vector = ?, payload = ? WHERE id = ?`)
|
||||
.run(vectorBuffer, JSON.stringify(payload), vectorId);
|
||||
}
|
||||
|
||||
async delete(vectorId: string): Promise<void> {
|
||||
await this.run(`DELETE FROM vectors WHERE id = ?`, [vectorId]);
|
||||
this.db.prepare(`DELETE FROM vectors WHERE id = ?`).run(vectorId);
|
||||
}
|
||||
|
||||
async deleteCol(): Promise<void> {
|
||||
await this.run(`DROP TABLE IF EXISTS vectors`);
|
||||
await this.init();
|
||||
this.db.exec(`DROP TABLE IF EXISTS vectors`);
|
||||
this.init();
|
||||
}
|
||||
|
||||
async list(
|
||||
filters?: SearchFilters,
|
||||
limit: number = 100,
|
||||
): Promise<[VectorStoreResult[], number]> {
|
||||
const rows = await this.all(`SELECT * FROM vectors`);
|
||||
const rows = this.db.prepare(`SELECT * FROM vectors`).all() as any[];
|
||||
const results: VectorStoreResult[] = [];
|
||||
|
||||
for (const row of rows) {
|
||||
const payload = JSON.parse(row.payload);
|
||||
const memoryVector: MemoryVector = {
|
||||
id: row.id,
|
||||
vector: Array.from(new Float32Array(row.vector.buffer)),
|
||||
vector: Array.from(
|
||||
new Float32Array(
|
||||
row.vector.buffer,
|
||||
row.vector.byteOffset,
|
||||
row.vector.byteLength / 4,
|
||||
),
|
||||
),
|
||||
payload,
|
||||
};
|
||||
|
||||
@@ -210,9 +196,9 @@ export class MemoryVectorStore implements VectorStore {
|
||||
}
|
||||
|
||||
async getUserId(): Promise<string> {
|
||||
const row = await this.getOne(
|
||||
`SELECT user_id FROM memory_migrations LIMIT 1`,
|
||||
);
|
||||
const row = this.db
|
||||
.prepare(`SELECT user_id FROM memory_migrations LIMIT 1`)
|
||||
.get() as any;
|
||||
if (row) {
|
||||
return row.user_id;
|
||||
}
|
||||
@@ -221,20 +207,20 @@ export class MemoryVectorStore implements VectorStore {
|
||||
const randomUserId =
|
||||
Math.random().toString(36).substring(2, 15) +
|
||||
Math.random().toString(36).substring(2, 15);
|
||||
await this.run(`INSERT INTO memory_migrations (user_id) VALUES (?)`, [
|
||||
randomUserId,
|
||||
]);
|
||||
this.db
|
||||
.prepare(`INSERT INTO memory_migrations (user_id) VALUES (?)`)
|
||||
.run(randomUserId);
|
||||
return randomUserId;
|
||||
}
|
||||
|
||||
async setUserId(userId: string): Promise<void> {
|
||||
await this.run(`DELETE FROM memory_migrations`);
|
||||
await this.run(`INSERT INTO memory_migrations (user_id) VALUES (?)`, [
|
||||
userId,
|
||||
]);
|
||||
this.db.prepare(`DELETE FROM memory_migrations`).run();
|
||||
this.db
|
||||
.prepare(`INSERT INTO memory_migrations (user_id) VALUES (?)`)
|
||||
.run(userId);
|
||||
}
|
||||
|
||||
async initialize(): Promise<void> {
|
||||
await this.init();
|
||||
this.init();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,7 +7,7 @@ const external = [
|
||||
"uuid",
|
||||
"pg",
|
||||
"zod",
|
||||
"sqlite3",
|
||||
"better-sqlite3",
|
||||
"@qdrant/js-client-rest",
|
||||
"redis",
|
||||
];
|
||||
|
||||
Reference in New Issue
Block a user