fix: delete_all now drains all pages, not just the first batch (close… (#4872)
Co-authored-by: kartik-mem0 <kartik.labhshetwar@mem0.ai>
This commit is contained in:
@@ -105,6 +105,10 @@ const ENTITY_PARAMS = [
|
|||||||
// in both casings (the default store promotes camelCase on read); actor_id has no camelCase alias.
|
// in both casings (the default store promotes camelCase on read); actor_id has no camelCase alias.
|
||||||
const IDENTITY_KEYS = [...ENTITY_PARAMS, "actor_id"];
|
const IDENTITY_KEYS = [...ENTITY_PARAMS, "actor_id"];
|
||||||
|
|
||||||
|
// Batch size for deleteAll pagination. Larger than most vector store default
|
||||||
|
// page limits (~100) to minimize roundtrips while bounded to avoid memory pressure.
|
||||||
|
const DELETE_ALL_BATCH_SIZE = 1000;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Validates that no top-level entity parameters are passed in config.
|
* Validates that no top-level entity parameters are passed in config.
|
||||||
* @throws Error if entity params are found at top level
|
* @throws Error if entity params are found at top level
|
||||||
@@ -1727,18 +1731,39 @@ export class Memory {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
const [memories] = await this.vectorStore.list(filters);
|
let deletedCount = 0;
|
||||||
for (const memory of memories) {
|
const seenBatches = new Set<string>();
|
||||||
await this.deleteMemory(memory.id);
|
|
||||||
|
while (true) {
|
||||||
|
const [batch] = await this.vectorStore.list(
|
||||||
|
filters,
|
||||||
|
DELETE_ALL_BATCH_SIZE,
|
||||||
|
);
|
||||||
|
if (!batch.length) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
const batchKey = batch
|
||||||
|
.map((memory) => String(memory.id))
|
||||||
|
.sort()
|
||||||
|
.join(",");
|
||||||
|
if (seenBatches.has(batchKey)) {
|
||||||
|
logger.warn("Stopping deleteAll after a repeated memory batch");
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
seenBatches.add(batchKey);
|
||||||
|
for (const memory of batch) {
|
||||||
|
await this.deleteMemory(memory.id);
|
||||||
|
}
|
||||||
|
deletedCount += batch.length;
|
||||||
}
|
}
|
||||||
|
|
||||||
const result = { message: "Memories deleted successfully!" };
|
const result = { message: "Memories deleted successfully!" };
|
||||||
if (memories.length > 0) {
|
if (deletedCount > 0) {
|
||||||
await this._displayDecayUsageNotice({
|
await this._displayDecayUsageNotice({
|
||||||
triggerFunction: "delete_all",
|
triggerFunction: "delete_all",
|
||||||
triggerSource: "delete_all",
|
triggerSource: "delete_all",
|
||||||
triggerReason: "bulk_delete",
|
triggerReason: "bulk_delete",
|
||||||
deletedCount: memories.length,
|
deletedCount,
|
||||||
});
|
});
|
||||||
} else {
|
} else {
|
||||||
await this._displayFirstRunNotice("delete_all");
|
await this._displayFirstRunNotice("delete_all");
|
||||||
|
|||||||
@@ -649,6 +649,66 @@ describe("Memory - deleteAll()", () => {
|
|||||||
"At least one filter is required to delete all memories",
|
"At least one filter is required to delete all memories",
|
||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test("deletes past the vector store default page limit", async () => {
|
||||||
|
const mem = createMemory();
|
||||||
|
const internals = mem as any;
|
||||||
|
await internals._ensureInitialized();
|
||||||
|
const pagedUserId = `deleteall_paged_${Date.now()}`;
|
||||||
|
const total = 150;
|
||||||
|
|
||||||
|
await internals.vectorStore.insert(
|
||||||
|
Array.from({ length: total }, () => new Array(1536).fill(0.1)),
|
||||||
|
Array.from({ length: total }, (_, index) => `paged-${index}`),
|
||||||
|
Array.from({ length: total }, (_, index) => ({
|
||||||
|
data: `Fact ${index}`,
|
||||||
|
hash: `hash-${index}`,
|
||||||
|
user_id: pagedUserId,
|
||||||
|
createdAt: new Date().toISOString(),
|
||||||
|
})),
|
||||||
|
);
|
||||||
|
|
||||||
|
const [before] = await internals.vectorStore.list(
|
||||||
|
{ user_id: pagedUserId },
|
||||||
|
total * 2,
|
||||||
|
);
|
||||||
|
expect(before).toHaveLength(total);
|
||||||
|
|
||||||
|
const result = await mem.deleteAll({ userId: pagedUserId });
|
||||||
|
expect(result.message).toBe("Memories deleted successfully!");
|
||||||
|
|
||||||
|
const [after] = await internals.vectorStore.list(
|
||||||
|
{ user_id: pagedUserId },
|
||||||
|
total * 2,
|
||||||
|
);
|
||||||
|
expect(after).toHaveLength(0);
|
||||||
|
}, 60000);
|
||||||
|
|
||||||
|
test("stops when the vector store keeps returning the same batch", async () => {
|
||||||
|
const mem = createMemory();
|
||||||
|
const internals = mem as any;
|
||||||
|
await internals._ensureInitialized();
|
||||||
|
const stale = [
|
||||||
|
{ id: "stale-1", payload: { data: "x", user_id: "u1" } },
|
||||||
|
{ id: "stale-2", payload: { data: "x", user_id: "u1" } },
|
||||||
|
];
|
||||||
|
const listSpy = jest
|
||||||
|
.spyOn(internals.vectorStore, "list")
|
||||||
|
.mockResolvedValueOnce([stale, stale.length])
|
||||||
|
.mockResolvedValueOnce([stale, stale.length])
|
||||||
|
.mockImplementation(async () => {
|
||||||
|
throw new Error("deleteAll should have stopped after a repeated batch");
|
||||||
|
});
|
||||||
|
const deleteSpy = jest
|
||||||
|
.spyOn(internals, "deleteMemory")
|
||||||
|
.mockResolvedValue("stale-1");
|
||||||
|
|
||||||
|
const result = await mem.deleteAll({ userId: "u1" });
|
||||||
|
|
||||||
|
expect(result.message).toBe("Memories deleted successfully!");
|
||||||
|
expect(listSpy).toHaveBeenCalledTimes(2);
|
||||||
|
expect(deleteSpy).toHaveBeenCalledTimes(2);
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
// ─── getAll() ────────────────────────────────────────────
|
// ─── getAll() ────────────────────────────────────────────
|
||||||
|
|||||||
@@ -362,7 +362,10 @@ describe("Memory Input Validation", () => {
|
|||||||
|
|
||||||
await memory.deleteAll({ userId: " alice " });
|
await memory.deleteAll({ userId: " alice " });
|
||||||
|
|
||||||
expect(listSpy).toHaveBeenCalledWith({ user_id: "alice" });
|
expect(listSpy).toHaveBeenCalledWith(
|
||||||
|
{ user_id: "alice" },
|
||||||
|
expect.any(Number),
|
||||||
|
);
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -5177,20 +5177,22 @@ describe("Memory class – backward compat with all providers", () => {
|
|||||||
updated_at: new Date().toISOString(),
|
updated_at: new Date().toISOString(),
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
mockVStore.list.mockResolvedValue([
|
mockVStore.list
|
||||||
[
|
.mockResolvedValueOnce([
|
||||||
{
|
[
|
||||||
id: memoryId,
|
{
|
||||||
payload: {
|
id: memoryId,
|
||||||
memory: "test",
|
payload: {
|
||||||
hash: "h",
|
memory: "test",
|
||||||
created_at: new Date().toISOString(),
|
hash: "h",
|
||||||
updated_at: new Date().toISOString(),
|
created_at: new Date().toISOString(),
|
||||||
|
updated_at: new Date().toISOString(),
|
||||||
|
},
|
||||||
},
|
},
|
||||||
},
|
],
|
||||||
],
|
1,
|
||||||
1,
|
])
|
||||||
]);
|
.mockResolvedValue([[], 0]);
|
||||||
mockVectorStoreFactory.create.mockReturnValue(mockVStore);
|
mockVectorStoreFactory.create.mockReturnValue(mockVStore);
|
||||||
|
|
||||||
const mem = new MemoryClass({
|
const mem = new MemoryClass({
|
||||||
|
|||||||
Reference in New Issue
Block a user