fix(ts-sdk): replace sqlite3 with better-sqlite3 to fix native binding resolution (#4270)

This commit is contained in:
Utkarsh
2026-03-09 23:30:36 +05:30
committed by GitHub
parent 7482c48692
commit 36537c8326
8 changed files with 515 additions and 888 deletions
+17
View File
@@ -729,6 +729,23 @@ mode: "wide"
<Tab title="TypeScript">
<Update label="2026-03-09" description="v2.3.0">
**Breaking Changes:**
- **Dependencies:** Minimum Node.js version for OSS sqlite features is now Node 20+ (due to `better-sqlite3` v12)
**Bug Fixes:**
- **OSS Storage:** Replaced `sqlite3` with `better-sqlite3` to fix native binding resolution failures under jiti-based loaders (e.g. OpenClaw plugin system). Fixes issues where the `bindings` module walked V8 stack frames with synthetic filenames, failing to locate the native `.node` addon.
- **OSS Storage:** Fixed async init race condition in `SQLiteManager` — `init()` is now synchronous
- **OSS Vector Store:** Migrated `MemoryVectorStore` from `sqlite3` to `better-sqlite3` with transactional batch inserts
**Improvements:**
- **Performance:** Cached prepared statements in `SQLiteManager` for faster history operations
- **Performance:** Batch `insert()` in `MemoryVectorStore` wrapped in a transaction for atomicity
- **Build:** Updated `tsup.config.ts` externals from `sqlite3` to `better-sqlite3`
</Update>
<Update label="2026-02-17" description="v2.2.3">
**New Features & Updates:**
+5 -5
View File
@@ -1,6 +1,6 @@
{
"name": "mem0ai",
"version": "2.2.4",
"version": "2.3.0",
"description": "The Memory Layer For Your AI Apps",
"main": "./dist/index.js",
"module": "./dist/index.mjs",
@@ -77,6 +77,7 @@
"author": "Deshraj Yadav",
"license": "Apache-2.0",
"devDependencies": {
"@types/better-sqlite3": "^7.6.13",
"@types/node": "^22.7.6",
"@types/uuid": "^9.0.8",
"dotenv": "^16.4.5",
@@ -108,14 +109,13 @@
"@supabase/supabase-js": "^2.49.1",
"@types/jest": "29.5.14",
"@types/pg": "8.11.0",
"@types/sqlite3": "3.1.11",
"better-sqlite3": "^12.6.2",
"cloudflare": "^4.2.0",
"groq-sdk": "0.3.0",
"neo4j-driver": "^5.28.1",
"ollama": "^0.5.14",
"pg": "8.11.3",
"redis": "^4.6.13",
"sqlite3": "5.1.7"
"redis": "^4.6.13"
},
"engines": {
"node": ">=18"
@@ -127,7 +127,7 @@
"pnpm": {
"onlyBuiltDependencies": [
"esbuild",
"sqlite3"
"better-sqlite3"
]
}
}
+29 -765
View File
File diff suppressed because it is too large Load Diff
+1 -2
View File
@@ -19,7 +19,6 @@
"@types/node": "^20.11.19",
"@types/pg": "^8.11.0",
"@types/redis": "^4.0.10",
"@types/sqlite3": "^3.1.11",
"@types/uuid": "^9.0.8",
"cloudflare": "^4.2.0",
"dotenv": "^16.4.4",
@@ -27,7 +26,7 @@
"openai": "^4.28.0",
"pg": "^8.11.3",
"redis": "^4.7.0",
"sqlite3": "^5.1.7",
"better-sqlite3": "^12.6.2",
"uuid": "^9.0.1",
"zod": "^3.22.4"
},
+22 -38
View File
@@ -1,16 +1,18 @@
import sqlite3 from "sqlite3";
import Database from "better-sqlite3";
import { HistoryManager } from "./base";
export class SQLiteManager implements HistoryManager {
private db: sqlite3.Database;
private db: Database.Database;
private stmtInsert!: Database.Statement;
private stmtSelect!: Database.Statement;
constructor(dbPath: string) {
this.db = new sqlite3.Database(dbPath);
this.init().catch(console.error);
this.db = new Database(dbPath);
this.init();
}
private async init() {
await this.run(`
private init(): void {
this.db.exec(`
CREATE TABLE IF NOT EXISTS memory_history (
id INTEGER PRIMARY KEY AUTOINCREMENT,
memory_id TEXT NOT NULL,
@@ -22,24 +24,14 @@ export class SQLiteManager implements HistoryManager {
is_deleted INTEGER DEFAULT 0
)
`);
}
private async run(sql: string, params: any[] = []): Promise<void> {
return new Promise((resolve, reject) => {
this.db.run(sql, params, (err) => {
if (err) reject(err);
else resolve();
});
});
}
private async all(sql: string, params: any[] = []): Promise<any[]> {
return new Promise((resolve, reject) => {
this.db.all(sql, params, (err, rows) => {
if (err) reject(err);
else resolve(rows);
});
});
this.stmtInsert = this.db.prepare(
`INSERT INTO memory_history
(memory_id, previous_value, new_value, action, created_at, updated_at, is_deleted)
VALUES (?, ?, ?, ?, ?, ?, ?)`,
);
this.stmtSelect = this.db.prepare(
"SELECT * FROM memory_history WHERE memory_id = ? ORDER BY id DESC",
);
}
async addHistory(
@@ -51,32 +43,24 @@ export class SQLiteManager implements HistoryManager {
updatedAt?: string,
isDeleted: number = 0,
): Promise<void> {
await this.run(
`INSERT INTO memory_history
(memory_id, previous_value, new_value, action, created_at, updated_at, is_deleted)
VALUES (?, ?, ?, ?, ?, ?, ?)`,
[
this.stmtInsert.run(
memoryId,
previousValue,
newValue,
action,
createdAt,
updatedAt,
createdAt ?? null,
updatedAt ?? null,
isDeleted,
],
);
}
async getHistory(memoryId: string): Promise<any[]> {
return this.all(
"SELECT * FROM memory_history WHERE memory_id = ? ORDER BY id DESC",
[memoryId],
);
return this.stmtSelect.all(memoryId) as any[];
}
async reset(): Promise<void> {
await this.run("DROP TABLE IF EXISTS memory_history");
await this.init();
this.db.exec("DROP TABLE IF EXISTS memory_history");
this.init();
}
close(): void {
@@ -0,0 +1,377 @@
/**
* Tests for the sqlite3 → better-sqlite3 migration.
*
* Covers:
* - SQLiteManager: all HistoryManager interface methods
* - MemoryVectorStore: insert, search, get, update, delete, list, userId mgmt
* - File-based persistence and in-memory mode
* - Backward compatibility: same schema, same data shapes
*/
import { SQLiteManager } from "../storage/SQLiteManager";
import { MemoryVectorStore } from "../vector_stores/memory";
import fs from "fs";
import path from "path";
import os from "os";
// ---------------------------------------------------------------------------
// SQLiteManager tests
// ---------------------------------------------------------------------------
describe("SQLiteManager (better-sqlite3)", () => {
let mgr: SQLiteManager;
beforeEach(() => {
mgr = new SQLiteManager(":memory:");
});
afterEach(() => {
mgr.close();
});
it("creates an in-memory database without errors", () => {
expect(mgr).toBeDefined();
});
it("addHistory inserts a row that getHistory returns", async () => {
await mgr.addHistory(
"mem-001",
null,
"User likes TypeScript",
"ADD",
"2026-01-01T00:00:00Z",
);
const history = await mgr.getHistory("mem-001");
expect(history).toHaveLength(1);
expect(history[0].memory_id).toBe("mem-001");
expect(history[0].new_value).toBe("User likes TypeScript");
expect(history[0].action).toBe("ADD");
expect(history[0].previous_value).toBeNull();
expect(history[0].is_deleted).toBe(0);
});
it("returns history in DESC order (most recent first)", async () => {
await mgr.addHistory("mem-002", null, "First", "ADD", "2026-01-01");
await mgr.addHistory(
"mem-002",
"First",
"Updated",
"UPDATE",
"2026-01-01",
"2026-01-02",
);
await mgr.addHistory(
"mem-002",
"Updated",
null,
"DELETE",
undefined,
undefined,
1,
);
const history = await mgr.getHistory("mem-002");
expect(history).toHaveLength(3);
expect(history[0].action).toBe("DELETE");
expect(history[1].action).toBe("UPDATE");
expect(history[2].action).toBe("ADD");
expect(history[0].is_deleted).toBe(1);
});
it("isolates history by memory_id", async () => {
await mgr.addHistory("mem-A", null, "Fact A", "ADD", "2026-01-01");
await mgr.addHistory("mem-B", null, "Fact B", "ADD", "2026-01-01");
expect(await mgr.getHistory("mem-A")).toHaveLength(1);
expect(await mgr.getHistory("mem-B")).toHaveLength(1);
expect((await mgr.getHistory("mem-A"))[0].new_value).toBe("Fact A");
expect((await mgr.getHistory("mem-B"))[0].new_value).toBe("Fact B");
});
it("handles NULL/undefined optional fields correctly", async () => {
await mgr.addHistory(
"mem-null",
null,
null,
"DELETE",
undefined,
undefined,
1,
);
const history = await mgr.getHistory("mem-null");
expect(history).toHaveLength(1);
expect(history[0].previous_value).toBeNull();
expect(history[0].new_value).toBeNull();
expect(history[0].created_at).toBeNull();
expect(history[0].updated_at).toBeNull();
});
it("reset() clears all history and allows re-insertion", async () => {
await mgr.addHistory("mem-003", null, "Fact", "ADD", "2026-01-01");
expect(await mgr.getHistory("mem-003")).toHaveLength(1);
await mgr.reset();
expect(await mgr.getHistory("mem-003")).toHaveLength(0);
await mgr.addHistory("mem-004", null, "New fact", "ADD", "2026-02-01");
expect(await mgr.getHistory("mem-004")).toHaveLength(1);
});
it("works with file-based database and persists data", async () => {
const dbPath = path.join(os.tmpdir(), `mem0-test-history-${Date.now()}.db`);
try {
const fmgr = new SQLiteManager(dbPath);
await fmgr.addHistory(
"mem-file",
null,
"Persistent",
"ADD",
"2026-01-01",
);
fmgr.close();
expect(fs.existsSync(dbPath)).toBe(true);
expect(fs.statSync(dbPath).size).toBeGreaterThan(0);
// Reopen and verify
const fmgr2 = new SQLiteManager(dbPath);
const history = await fmgr2.getHistory("mem-file");
expect(history).toHaveLength(1);
expect(history[0].new_value).toBe("Persistent");
fmgr2.close();
} finally {
if (fs.existsSync(dbPath)) fs.unlinkSync(dbPath);
}
});
it("handles many rapid insertions", async () => {
for (let i = 0; i < 100; i++) {
await mgr.addHistory(
`mem-rapid-${i}`,
null,
`Fact ${i}`,
"ADD",
new Date().toISOString(),
);
}
for (let i = 0; i < 100; i++) {
const h = await mgr.getHistory(`mem-rapid-${i}`);
expect(h).toHaveLength(1);
expect(h[0].new_value).toBe(`Fact ${i}`);
}
});
});
// ---------------------------------------------------------------------------
// MemoryVectorStore tests
// ---------------------------------------------------------------------------
describe("MemoryVectorStore (better-sqlite3)", () => {
const DIM = 4;
let store: MemoryVectorStore;
let dbPath: string;
function normalize(v: number[]): number[] {
const norm = Math.sqrt(v.reduce((s, x) => s + x * x, 0));
return v.map((x) => x / norm);
}
beforeEach(() => {
dbPath = path.join(os.tmpdir(), `mem0-test-vectors-${Date.now()}.db`);
store = new MemoryVectorStore({ dimension: DIM, dbPath } as any);
});
afterEach(() => {
if (fs.existsSync(dbPath)) fs.unlinkSync(dbPath);
});
it("insert + get returns the stored payload", async () => {
const v = normalize([1, 0, 0, 0]);
await store.insert([v], ["id-1"], [{ data: "hello", userId: "u1" }]);
const result = await store.get("id-1");
expect(result).not.toBeNull();
expect(result!.id).toBe("id-1");
expect(result!.payload.data).toBe("hello");
expect(result!.payload.userId).toBe("u1");
});
it("get returns null for non-existent id", async () => {
const result = await store.get("nope");
expect(result).toBeNull();
});
it("search returns results sorted by cosine similarity", async () => {
const v1 = normalize([1, 0, 0, 0]);
const v2 = normalize([0, 1, 0, 0]);
const v3 = normalize([1, 1, 0, 0]);
await store.insert(
[v1, v2, v3],
["id-1", "id-2", "id-3"],
[{ data: "exact" }, { data: "orthogonal" }, { data: "close" }],
);
const results = await store.search(v1, 3);
expect(results).toHaveLength(3);
expect(results[0].id).toBe("id-1");
expect(results[0].score).toBeCloseTo(1.0, 5);
expect(results[1].id).toBe("id-3");
expect(results[2].id).toBe("id-2");
expect(results[2].score).toBeCloseTo(0, 5);
});
it("search respects limit", async () => {
const vectors = [];
const ids = [];
const payloads = [];
for (let i = 0; i < 10; i++) {
const v = [0, 0, 0, 0];
v[i % DIM] = 1;
vectors.push(normalize(v));
ids.push(`id-${i}`);
payloads.push({ data: `item-${i}` });
}
await store.insert(vectors, ids, payloads);
const results = await store.search(normalize([1, 0, 0, 0]), 3);
expect(results).toHaveLength(3);
});
it("search respects filters", async () => {
const v = normalize([1, 0, 0, 0]);
await store.insert(
[v, v],
["id-1", "id-2"],
[
{ data: "a", userId: "alice" },
{ data: "b", userId: "bob" },
],
);
const results = await store.search(v, 10, { userId: "alice" });
expect(results).toHaveLength(1);
expect(results[0].id).toBe("id-1");
});
it("search throws on dimension mismatch", async () => {
await expect(store.search([1, 0, 0], 10)).rejects.toThrow(
"dimension mismatch",
);
});
it("insert throws on dimension mismatch", async () => {
await expect(
store.insert([[1, 0, 0]], ["id-1"], [{ data: "x" }]),
).rejects.toThrow("dimension mismatch");
});
it("update modifies the stored vector and payload", async () => {
const v1 = normalize([1, 0, 0, 0]);
const v2 = normalize([0, 1, 0, 0]);
await store.insert([v1], ["id-1"], [{ data: "original" }]);
await store.update("id-1", v2, { data: "updated" });
const result = await store.get("id-1");
expect(result!.payload.data).toBe("updated");
const results = await store.search(v2, 1);
expect(results[0].id).toBe("id-1");
expect(results[0].score).toBeCloseTo(1.0, 5);
});
it("delete removes the vector", async () => {
const v = normalize([1, 0, 0, 0]);
await store.insert([v], ["id-1"], [{ data: "doomed" }]);
await store.delete("id-1");
expect(await store.get("id-1")).toBeNull();
});
it("deleteCol drops and recreates table", async () => {
const v = normalize([1, 0, 0, 0]);
await store.insert([v], ["id-1"], [{ data: "will be gone" }]);
await store.deleteCol();
expect(await store.get("id-1")).toBeNull();
await store.insert([v], ["id-2"], [{ data: "fresh" }]);
expect(await store.get("id-2")).not.toBeNull();
});
it("list returns all vectors with optional filters", async () => {
const v = normalize([1, 0, 0, 0]);
await store.insert(
[v, v, v],
["id-1", "id-2", "id-3"],
[
{ data: "a", userId: "alice" },
{ data: "b", userId: "bob" },
{ data: "c", userId: "alice" },
],
);
const [all, totalAll] = await store.list();
expect(all).toHaveLength(3);
expect(totalAll).toBe(3);
const [filtered, totalFiltered] = await store.list({ userId: "alice" });
expect(filtered).toHaveLength(2);
expect(totalFiltered).toBe(2);
});
it("list respects limit", async () => {
const v = normalize([1, 0, 0, 0]);
await store.insert(
[v, v, v],
["id-1", "id-2", "id-3"],
[{ data: "a" }, { data: "b" }, { data: "c" }],
);
const [results] = await store.list(undefined, 2);
expect(results).toHaveLength(2);
});
it("getUserId generates and persists a userId", async () => {
const id1 = await store.getUserId();
expect(typeof id1).toBe("string");
expect(id1.length).toBeGreaterThan(0);
const id2 = await store.getUserId();
expect(id2).toBe(id1);
});
it("setUserId overrides the stored userId", async () => {
await store.setUserId("custom-user-123");
const id = await store.getUserId();
expect(id).toBe("custom-user-123");
});
it("INSERT OR REPLACE upserts on id conflict", async () => {
const v1 = normalize([1, 0, 0, 0]);
const v2 = normalize([0, 1, 0, 0]);
await store.insert([v1], ["id-1"], [{ data: "original" }]);
await store.insert([v2], ["id-1"], [{ data: "replaced" }]);
const result = await store.get("id-1");
expect(result!.payload.data).toBe("replaced");
const [all] = await store.list();
expect(all).toHaveLength(1);
});
it("file-based database persists across reopens", async () => {
const v = normalize([1, 0, 0, 0]);
await store.insert([v], ["id-persist"], [{ data: "persistent" }]);
const store2 = new MemoryVectorStore({ dimension: DIM, dbPath } as any);
const result = await store2.get("id-persist");
expect(result).not.toBeNull();
expect(result!.payload.data).toBe("persistent");
});
});
+55 -69
View File
@@ -1,6 +1,6 @@
import { VectorStore } from "./base";
import { SearchFilters, VectorStoreConfig, VectorStoreResult } from "../types";
import sqlite3 from "sqlite3";
import Database from "better-sqlite3";
import path from "path";
interface MemoryVector {
@@ -10,7 +10,7 @@ interface MemoryVector {
}
export class MemoryVectorStore implements VectorStore {
private db: sqlite3.Database;
private db: Database.Database;
private dimension: number;
private dbPath: string;
@@ -20,12 +20,12 @@ export class MemoryVectorStore implements VectorStore {
if (config.dbPath) {
this.dbPath = config.dbPath;
}
this.db = new sqlite3.Database(this.dbPath);
this.init().catch(console.error);
this.db = new Database(this.dbPath);
this.init();
}
private async init() {
await this.run(`
private init(): void {
this.db.exec(`
CREATE TABLE IF NOT EXISTS vectors (
id TEXT PRIMARY KEY,
vector BLOB NOT NULL,
@@ -33,7 +33,7 @@ export class MemoryVectorStore implements VectorStore {
)
`);
await this.run(`
this.db.exec(`
CREATE TABLE IF NOT EXISTS memory_migrations (
id INTEGER PRIMARY KEY AUTOINCREMENT,
user_id TEXT NOT NULL UNIQUE
@@ -41,33 +41,6 @@ export class MemoryVectorStore implements VectorStore {
`);
}
private async run(sql: string, params: any[] = []): Promise<void> {
return new Promise((resolve, reject) => {
this.db.run(sql, params, (err) => {
if (err) reject(err);
else resolve();
});
});
}
private async all(sql: string, params: any[] = []): Promise<any[]> {
return new Promise((resolve, reject) => {
this.db.all(sql, params, (err, rows) => {
if (err) reject(err);
else resolve(rows);
});
});
}
private async getOne(sql: string, params: any[] = []): Promise<any> {
return new Promise((resolve, reject) => {
this.db.get(sql, params, (err, row) => {
if (err) reject(err);
else resolve(row);
});
});
}
private cosineSimilarity(a: number[], b: number[]): number {
let dotProduct = 0;
let normA = 0;
@@ -92,18 +65,23 @@ export class MemoryVectorStore implements VectorStore {
ids: string[],
payloads: Record<string, any>[],
): Promise<void> {
for (let i = 0; i < vectors.length; i++) {
if (vectors[i].length !== this.dimension) {
throw new Error(
`Vector dimension mismatch. Expected ${this.dimension}, got ${vectors[i].length}`,
);
}
const vectorBuffer = Buffer.from(new Float32Array(vectors[i]).buffer);
await this.run(
const stmt = this.db.prepare(
`INSERT OR REPLACE INTO vectors (id, vector, payload) VALUES (?, ?, ?)`,
[ids[i], vectorBuffer, JSON.stringify(payloads[i])],
);
const insertMany = this.db.transaction(
(vecs: number[][], vIds: string[], vPayloads: Record<string, any>[]) => {
for (let i = 0; i < vecs.length; i++) {
if (vecs[i].length !== this.dimension) {
throw new Error(
`Vector dimension mismatch. Expected ${this.dimension}, got ${vecs[i].length}`,
);
}
const vectorBuffer = Buffer.from(new Float32Array(vecs[i]).buffer);
stmt.run(vIds[i], vectorBuffer, JSON.stringify(vPayloads[i]));
}
},
);
insertMany(vectors, ids, payloads);
}
async search(
@@ -117,11 +95,15 @@ export class MemoryVectorStore implements VectorStore {
);
}
const rows = await this.all(`SELECT * FROM vectors`);
const rows = this.db.prepare(`SELECT * FROM vectors`).all() as any[];
const results: VectorStoreResult[] = [];
for (const row of rows) {
const vector = new Float32Array(row.vector.buffer);
const vector = new Float32Array(
row.vector.buffer,
row.vector.byteOffset,
row.vector.byteLength / 4,
);
const payload = JSON.parse(row.payload);
const memoryVector: MemoryVector = {
id: row.id,
@@ -144,9 +126,9 @@ export class MemoryVectorStore implements VectorStore {
}
async get(vectorId: string): Promise<VectorStoreResult | null> {
const row = await this.getOne(`SELECT * FROM vectors WHERE id = ?`, [
vectorId,
]);
const row = this.db
.prepare(`SELECT * FROM vectors WHERE id = ?`)
.get(vectorId) as any;
if (!row) return null;
const payload = JSON.parse(row.payload);
@@ -167,34 +149,38 @@ export class MemoryVectorStore implements VectorStore {
);
}
const vectorBuffer = Buffer.from(new Float32Array(vector).buffer);
await this.run(`UPDATE vectors SET vector = ?, payload = ? WHERE id = ?`, [
vectorBuffer,
JSON.stringify(payload),
vectorId,
]);
this.db
.prepare(`UPDATE vectors SET vector = ?, payload = ? WHERE id = ?`)
.run(vectorBuffer, JSON.stringify(payload), vectorId);
}
async delete(vectorId: string): Promise<void> {
await this.run(`DELETE FROM vectors WHERE id = ?`, [vectorId]);
this.db.prepare(`DELETE FROM vectors WHERE id = ?`).run(vectorId);
}
async deleteCol(): Promise<void> {
await this.run(`DROP TABLE IF EXISTS vectors`);
await this.init();
this.db.exec(`DROP TABLE IF EXISTS vectors`);
this.init();
}
async list(
filters?: SearchFilters,
limit: number = 100,
): Promise<[VectorStoreResult[], number]> {
const rows = await this.all(`SELECT * FROM vectors`);
const rows = this.db.prepare(`SELECT * FROM vectors`).all() as any[];
const results: VectorStoreResult[] = [];
for (const row of rows) {
const payload = JSON.parse(row.payload);
const memoryVector: MemoryVector = {
id: row.id,
vector: Array.from(new Float32Array(row.vector.buffer)),
vector: Array.from(
new Float32Array(
row.vector.buffer,
row.vector.byteOffset,
row.vector.byteLength / 4,
),
),
payload,
};
@@ -210,9 +196,9 @@ export class MemoryVectorStore implements VectorStore {
}
async getUserId(): Promise<string> {
const row = await this.getOne(
`SELECT user_id FROM memory_migrations LIMIT 1`,
);
const row = this.db
.prepare(`SELECT user_id FROM memory_migrations LIMIT 1`)
.get() as any;
if (row) {
return row.user_id;
}
@@ -221,20 +207,20 @@ export class MemoryVectorStore implements VectorStore {
const randomUserId =
Math.random().toString(36).substring(2, 15) +
Math.random().toString(36).substring(2, 15);
await this.run(`INSERT INTO memory_migrations (user_id) VALUES (?)`, [
randomUserId,
]);
this.db
.prepare(`INSERT INTO memory_migrations (user_id) VALUES (?)`)
.run(randomUserId);
return randomUserId;
}
async setUserId(userId: string): Promise<void> {
await this.run(`DELETE FROM memory_migrations`);
await this.run(`INSERT INTO memory_migrations (user_id) VALUES (?)`, [
userId,
]);
this.db.prepare(`DELETE FROM memory_migrations`).run();
this.db
.prepare(`INSERT INTO memory_migrations (user_id) VALUES (?)`)
.run(userId);
}
async initialize(): Promise<void> {
await this.init();
this.init();
}
}
+1 -1
View File
@@ -7,7 +7,7 @@ const external = [
"uuid",
"pg",
"zod",
"sqlite3",
"better-sqlite3",
"@qdrant/js-client-rest",
"redis",
];