fix(oss): auto-detect embedding dimension to fix Qdrant mismatch with non-OpenAI embedders (#4297)
Co-authored-by: utkarsh240799 <utkarsh240799@users.noreply.github.com> Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Utkarsh
@@ -43,13 +43,23 @@ export class ConfigManager {
|
||||
const defaultConf = DEFAULT_MEMORY_CONFIG.vectorStore.config;
|
||||
const userConf = userConfig.vectorStore?.config;
|
||||
|
||||
// Resolve the vector store dimension. If the user explicitly
|
||||
// provided one, use it. Otherwise leave it undefined so that
|
||||
// Memory._autoInitialize() can auto-detect it by running a
|
||||
// probe embedding at startup — this makes *any* embedder work
|
||||
// out of the box without the user needing to know or set the
|
||||
// dimension manually.
|
||||
const explicitDimension =
|
||||
userConf?.dimension ||
|
||||
userConfig.embedder?.config?.embeddingDims ||
|
||||
undefined;
|
||||
|
||||
// Prioritize user-provided client instance
|
||||
if (userConf?.client && typeof userConf.client === "object") {
|
||||
return {
|
||||
client: userConf.client,
|
||||
// Include other fields from userConf if necessary, or omit defaults
|
||||
collectionName: userConf.collectionName, // Can be undefined
|
||||
dimension: userConf.dimension || defaultConf.dimension, // Merge dimension
|
||||
collectionName: userConf.collectionName,
|
||||
dimension: explicitDimension,
|
||||
...userConf, // Include any other passthrough fields from user
|
||||
};
|
||||
} else {
|
||||
@@ -57,7 +67,7 @@ export class ConfigManager {
|
||||
return {
|
||||
collectionName:
|
||||
userConf?.collectionName || defaultConf.collectionName,
|
||||
dimension: userConf?.dimension || defaultConf.dimension,
|
||||
dimension: explicitDimension,
|
||||
// Ensure client is not carried over from defaults if not provided by user
|
||||
client: undefined,
|
||||
// Include other passthrough fields from userConf even if no client
|
||||
|
||||
@@ -28,7 +28,7 @@ export class GoogleEmbedder implements Embedder {
|
||||
const response = await this.google.models.embedContent({
|
||||
model: this.model,
|
||||
contents: texts,
|
||||
config: { outputDimensionality: 768 },
|
||||
config: { outputDimensionality: this.embeddingDims },
|
||||
});
|
||||
return response.embeddings!.map((item) => item.values!);
|
||||
}
|
||||
|
||||
@@ -41,7 +41,7 @@ export class Memory {
|
||||
private config: MemoryConfig;
|
||||
private customPrompt: string | undefined;
|
||||
private embedder: Embedder;
|
||||
private vectorStore: VectorStore;
|
||||
private vectorStore!: VectorStore;
|
||||
private llm: LLM;
|
||||
private db: HistoryManager;
|
||||
private collectionName: string | undefined;
|
||||
@@ -49,6 +49,8 @@ export class Memory {
|
||||
private graphMemory?: MemoryGraph;
|
||||
private enableGraph: boolean;
|
||||
telemetryId: string;
|
||||
private _initPromise: Promise<void>;
|
||||
private _initError?: Error;
|
||||
|
||||
constructor(config: Partial<MemoryConfig> = {}) {
|
||||
// Merge and validate config
|
||||
@@ -59,10 +61,9 @@ export class Memory {
|
||||
this.config.embedder.provider,
|
||||
this.config.embedder.config,
|
||||
);
|
||||
this.vectorStore = VectorStoreFactory.create(
|
||||
this.config.vectorStore.provider,
|
||||
this.config.vectorStore.config,
|
||||
);
|
||||
// Vector store creation is deferred to _autoInitialize() so that
|
||||
// the embedding dimension can be auto-detected first when not
|
||||
// explicitly configured.
|
||||
this.llm = LLMFactory.create(
|
||||
this.config.llm.provider,
|
||||
this.config.llm.config,
|
||||
@@ -86,8 +87,67 @@ export class Memory {
|
||||
this.graphMemory = new MemoryGraph(this.config);
|
||||
}
|
||||
|
||||
// Initialize telemetry if vector store is initialized
|
||||
this._initializeTelemetry();
|
||||
// Auto-detect embedding dimension (if needed), create vector store,
|
||||
// and initialize it. All public methods await this before proceeding.
|
||||
this._initPromise = this._autoInitialize().catch((error) => {
|
||||
this._initError =
|
||||
error instanceof Error ? error : new Error(String(error));
|
||||
console.error(this._initError);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* If no explicit dimension was provided, runs a probe embedding to
|
||||
* detect it. Then creates and initializes the vector store.
|
||||
*/
|
||||
private async _autoInitialize(): Promise<void> {
|
||||
if (!this.config.vectorStore.config.dimension) {
|
||||
try {
|
||||
const probe = await this.embedder.embed("dimension probe");
|
||||
this.config.vectorStore.config.dimension = probe.length;
|
||||
} catch (error: any) {
|
||||
throw new Error(
|
||||
`Failed to auto-detect embedding dimension from provider '${this.config.embedder.provider}': ${error.message}. ` +
|
||||
`Please set 'dimension' in vectorStore.config or 'embeddingDims' in embedder.config explicitly.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
this.vectorStore = VectorStoreFactory.create(
|
||||
this.config.vectorStore.provider,
|
||||
this.config.vectorStore.config,
|
||||
);
|
||||
|
||||
// The vector store constructor may fire initialize() asynchronously
|
||||
// (e.g. Qdrant). Explicitly await it here to guarantee the backing
|
||||
// store (collections, tables, etc.) is ready before any public method
|
||||
// attempts to read or write.
|
||||
await this.vectorStore.initialize();
|
||||
|
||||
await this._initializeTelemetry();
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensures that auto-initialization (dimension detection + vector store
|
||||
* creation) has completed before any public method proceeds.
|
||||
* If a previous init attempt failed, retries automatically.
|
||||
*/
|
||||
private async _ensureInitialized(): Promise<void> {
|
||||
await this._initPromise;
|
||||
if (this._initError) {
|
||||
// Clear failed state and retry — the embedder or vector store
|
||||
// may have been transiently unavailable at startup.
|
||||
this._initError = undefined;
|
||||
this._initPromise = this._autoInitialize().catch((error) => {
|
||||
this._initError =
|
||||
error instanceof Error ? error : new Error(String(error));
|
||||
console.error(this._initError);
|
||||
});
|
||||
await this._initPromise;
|
||||
if (this._initError) {
|
||||
throw this._initError;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async _initializeTelemetry() {
|
||||
@@ -147,6 +207,7 @@ export class Memory {
|
||||
messages: string | Message[],
|
||||
config: AddMemoryOptions,
|
||||
): Promise<SearchResult> {
|
||||
await this._ensureInitialized();
|
||||
await this._captureEvent("add", {
|
||||
message_count: Array.isArray(messages) ? messages.length : 1,
|
||||
has_metadata: !!config.metadata,
|
||||
@@ -372,6 +433,7 @@ export class Memory {
|
||||
}
|
||||
|
||||
async get(memoryId: string): Promise<MemoryItem | null> {
|
||||
await this._ensureInitialized();
|
||||
const memory = await this.vectorStore.get(memoryId);
|
||||
if (!memory) return null;
|
||||
|
||||
@@ -413,6 +475,7 @@ export class Memory {
|
||||
query: string,
|
||||
config: SearchMemoryOptions,
|
||||
): Promise<SearchResult> {
|
||||
await this._ensureInitialized();
|
||||
await this._captureEvent("search", {
|
||||
query_length: query.length,
|
||||
limit: config.limit,
|
||||
@@ -479,6 +542,7 @@ export class Memory {
|
||||
}
|
||||
|
||||
async update(memoryId: string, data: string): Promise<{ message: string }> {
|
||||
await this._ensureInitialized();
|
||||
await this._captureEvent("update", { memory_id: memoryId });
|
||||
const embedding = await this.embedder.embed(data);
|
||||
await this.updateMemory(memoryId, data, { [data]: embedding });
|
||||
@@ -486,6 +550,7 @@ export class Memory {
|
||||
}
|
||||
|
||||
async delete(memoryId: string): Promise<{ message: string }> {
|
||||
await this._ensureInitialized();
|
||||
await this._captureEvent("delete", { memory_id: memoryId });
|
||||
await this.deleteMemory(memoryId);
|
||||
return { message: "Memory deleted successfully!" };
|
||||
@@ -494,6 +559,7 @@ export class Memory {
|
||||
async deleteAll(
|
||||
config: DeleteAllMemoryOptions,
|
||||
): Promise<{ message: string }> {
|
||||
await this._ensureInitialized();
|
||||
await this._captureEvent("delete_all", {
|
||||
has_user_id: !!config.userId,
|
||||
has_agent_id: !!config.agentId,
|
||||
@@ -521,10 +587,12 @@ export class Memory {
|
||||
}
|
||||
|
||||
async history(memoryId: string): Promise<any[]> {
|
||||
await this._ensureInitialized();
|
||||
return this.db.getHistory(memoryId);
|
||||
}
|
||||
|
||||
async reset(): Promise<void> {
|
||||
await this._ensureInitialized();
|
||||
await this._captureEvent("reset");
|
||||
await this.db.reset();
|
||||
|
||||
@@ -549,28 +617,30 @@ export class Memory {
|
||||
await this.graphMemory.deleteAll({ userId: "default" }); // Assuming this is okay, or needs similar check?
|
||||
}
|
||||
|
||||
// Re-initialize factories/clients based on the original config
|
||||
// Re-initialize factories/clients based on the original config.
|
||||
// Dimension is already set in this.config from the initial probe,
|
||||
// so _autoInitialize will skip the probe and just re-create the store.
|
||||
this.embedder = EmbedderFactory.create(
|
||||
this.config.embedder.provider,
|
||||
this.config.embedder.config,
|
||||
);
|
||||
// Re-create vector store instance - crucial for Langchain to reset wrapper state if needed
|
||||
this.vectorStore = VectorStoreFactory.create(
|
||||
this.config.vectorStore.provider,
|
||||
this.config.vectorStore.config, // This will pass the original client instance back
|
||||
);
|
||||
this.llm = LLMFactory.create(
|
||||
this.config.llm.provider,
|
||||
this.config.llm.config,
|
||||
);
|
||||
// Re-init DB if needed (though db.reset() likely handles its state)
|
||||
// Re-init Graph if needed
|
||||
|
||||
// Re-initialize telemetry
|
||||
this._initializeTelemetry();
|
||||
// Re-create vector store via _autoInitialize (which handles dimension + creation)
|
||||
this._initError = undefined;
|
||||
this._initPromise = this._autoInitialize().catch((error) => {
|
||||
this._initError =
|
||||
error instanceof Error ? error : new Error(String(error));
|
||||
console.error(this._initError);
|
||||
});
|
||||
await this._initPromise;
|
||||
}
|
||||
|
||||
async getAll(config: GetAllMemoryOptions): Promise<SearchResult> {
|
||||
await this._ensureInitialized();
|
||||
await this._captureEvent("get_all", {
|
||||
limit: config.limit,
|
||||
has_user_id: !!config.userId,
|
||||
|
||||
@@ -81,6 +81,7 @@ export class AzureAISearch implements VectorStore {
|
||||
private readonly hybridSearch: boolean;
|
||||
private readonly vectorFilterMode: string;
|
||||
private readonly apiKey: string | undefined;
|
||||
private _initPromise?: Promise<void>;
|
||||
|
||||
constructor(config: AzureAISearchConfig) {
|
||||
this.serviceName = config.serviceName;
|
||||
@@ -117,6 +118,13 @@ export class AzureAISearch implements VectorStore {
|
||||
* Initialize the Azure AI Search index if it doesn't exist
|
||||
*/
|
||||
async initialize(): Promise<void> {
|
||||
if (!this._initPromise) {
|
||||
this._initPromise = this._doInitialize();
|
||||
}
|
||||
return this._initPromise;
|
||||
}
|
||||
|
||||
private async _doInitialize(): Promise<void> {
|
||||
try {
|
||||
const collections = await this.listCols();
|
||||
if (!collections.includes(this.indexName)) {
|
||||
|
||||
@@ -32,6 +32,7 @@ export class Qdrant implements VectorStore {
|
||||
private client: QdrantClient;
|
||||
private readonly collectionName: string;
|
||||
private dimension: number;
|
||||
private _initPromise?: Promise<void>;
|
||||
|
||||
constructor(config: QdrantConfig) {
|
||||
if (config.client) {
|
||||
@@ -211,22 +212,8 @@ export class Qdrant implements VectorStore {
|
||||
|
||||
async getUserId(): Promise<string> {
|
||||
try {
|
||||
// First check if the collection exists
|
||||
const collections = await this.client.getCollections();
|
||||
const userCollectionExists = collections.collections.some(
|
||||
(col: { name: string }) => col.name === "memory_migrations",
|
||||
);
|
||||
|
||||
if (!userCollectionExists) {
|
||||
// Create the collection if it doesn't exist
|
||||
await this.client.createCollection("memory_migrations", {
|
||||
vectors: {
|
||||
size: 1,
|
||||
distance: "Cosine",
|
||||
on_disk: false,
|
||||
},
|
||||
});
|
||||
}
|
||||
// Ensure collection exists (idempotent — handles race conditions)
|
||||
await this.ensureCollection("memory_migrations", 1);
|
||||
|
||||
// Now try to get the user ID
|
||||
const result = await this.client.scroll("memory_migrations", {
|
||||
@@ -286,66 +273,61 @@ export class Qdrant implements VectorStore {
|
||||
}
|
||||
}
|
||||
|
||||
async initialize(): Promise<void> {
|
||||
private async ensureCollection(
|
||||
name: string,
|
||||
size: number,
|
||||
): Promise<void> {
|
||||
try {
|
||||
// Create collection if it doesn't exist
|
||||
const collections = await this.client.getCollections();
|
||||
const exists = collections.collections.some(
|
||||
(c) => c.name === this.collectionName,
|
||||
);
|
||||
|
||||
if (!exists) {
|
||||
try {
|
||||
await this.client.createCollection(this.collectionName, {
|
||||
vectors: {
|
||||
size: this.dimension,
|
||||
distance: "Cosine",
|
||||
},
|
||||
});
|
||||
} catch (error: any) {
|
||||
// Handle case where collection was created between our check and create
|
||||
if (error?.status === 409) {
|
||||
// Collection already exists - verify it has the correct configuration
|
||||
const collectionInfo = await this.client.getCollection(
|
||||
this.collectionName,
|
||||
);
|
||||
await this.client.createCollection(name, {
|
||||
vectors: {
|
||||
size,
|
||||
distance: "Cosine",
|
||||
},
|
||||
});
|
||||
} catch (error: any) {
|
||||
if (error?.status === 409) {
|
||||
// Collection already exists — verify configuration for the main collection
|
||||
if (name === this.collectionName) {
|
||||
try {
|
||||
const collectionInfo = await this.client.getCollection(name);
|
||||
const vectorConfig = collectionInfo.config?.params?.vectors;
|
||||
|
||||
if (!vectorConfig || vectorConfig.size !== this.dimension) {
|
||||
if (vectorConfig && vectorConfig.size !== size) {
|
||||
throw new Error(
|
||||
`Collection ${this.collectionName} exists but has wrong configuration. ` +
|
||||
`Expected vector size: ${this.dimension}, got: ${vectorConfig?.size}`,
|
||||
`Collection ${name} exists but has wrong vector size. ` +
|
||||
`Expected: ${size}, got: ${vectorConfig.size}`,
|
||||
);
|
||||
}
|
||||
// Collection exists with correct configuration - we can proceed
|
||||
} else {
|
||||
throw error;
|
||||
} catch (verifyError: any) {
|
||||
// Re-throw dimension mismatch errors
|
||||
if (verifyError?.message?.includes("wrong vector size")) {
|
||||
throw verifyError;
|
||||
}
|
||||
// Transient errors (e.g. 500 while collection is being committed)
|
||||
// are non-fatal — the collection exists per the 409.
|
||||
console.warn(
|
||||
`Collection '${name}' exists (409) but dimension verification failed: ${verifyError?.message || verifyError}. Proceeding anyway.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
// Otherwise collection exists and is fine — proceed
|
||||
} else {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Create memory_migrations collection if it doesn't exist
|
||||
const userExists = collections.collections.some(
|
||||
(c) => c.name === "memory_migrations",
|
||||
);
|
||||
async initialize(): Promise<void> {
|
||||
if (!this._initPromise) {
|
||||
this._initPromise = this._doInitialize();
|
||||
}
|
||||
return this._initPromise;
|
||||
}
|
||||
|
||||
if (!userExists) {
|
||||
try {
|
||||
await this.client.createCollection("memory_migrations", {
|
||||
vectors: {
|
||||
size: 1, // Minimal size since we only store user_id
|
||||
distance: "Cosine",
|
||||
},
|
||||
});
|
||||
} catch (error: any) {
|
||||
// Handle case where collection was created between our check and create
|
||||
if (error?.status === 409) {
|
||||
// Collection already exists - we can proceed
|
||||
} else {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
private async _doInitialize(): Promise<void> {
|
||||
try {
|
||||
await this.ensureCollection(this.collectionName, this.dimension);
|
||||
await this.ensureCollection("memory_migrations", 1);
|
||||
} catch (error) {
|
||||
console.error("Error initializing Qdrant:", error);
|
||||
throw error;
|
||||
|
||||
@@ -139,6 +139,7 @@ export class RedisDB implements VectorStore {
|
||||
private readonly indexName: string;
|
||||
private readonly indexPrefix: string;
|
||||
private readonly schema: RedisSchema;
|
||||
private _initPromise?: Promise<void>;
|
||||
|
||||
constructor(config: RedisConfig) {
|
||||
this.indexName = config.collectionName;
|
||||
@@ -240,6 +241,13 @@ export class RedisDB implements VectorStore {
|
||||
}
|
||||
|
||||
async initialize(): Promise<void> {
|
||||
if (!this._initPromise) {
|
||||
this._initPromise = this._doInitialize();
|
||||
}
|
||||
return this._initPromise;
|
||||
}
|
||||
|
||||
private async _doInitialize(): Promise<void> {
|
||||
try {
|
||||
await this.client.connect();
|
||||
console.log("Connected to Redis");
|
||||
|
||||
@@ -86,6 +86,7 @@ export class SupabaseDB implements VectorStore {
|
||||
private readonly tableName: string;
|
||||
private readonly embeddingColumnName: string;
|
||||
private readonly metadataColumnName: string;
|
||||
private _initPromise?: Promise<void>;
|
||||
|
||||
constructor(config: SupabaseConfig) {
|
||||
this.client = createClient(config.supabaseUrl, config.supabaseKey);
|
||||
@@ -100,6 +101,13 @@ export class SupabaseDB implements VectorStore {
|
||||
}
|
||||
|
||||
async initialize(): Promise<void> {
|
||||
if (!this._initPromise) {
|
||||
this._initPromise = this._doInitialize();
|
||||
}
|
||||
return this._initPromise;
|
||||
}
|
||||
|
||||
private async _doInitialize(): Promise<void> {
|
||||
try {
|
||||
// Verify table exists and vector operations work by attempting a test insert
|
||||
const testVector = Array(1536).fill(0);
|
||||
|
||||
@@ -20,6 +20,7 @@ export class VectorizeDB implements VectorStore {
|
||||
private dimensions: number;
|
||||
private indexName: string;
|
||||
private accountId: string;
|
||||
private _initPromise?: Promise<void>;
|
||||
|
||||
constructor(config: VectorizeConfig) {
|
||||
this.client = new Cloudflare({ apiToken: config.apiKey });
|
||||
@@ -343,6 +344,13 @@ export class VectorizeDB implements VectorStore {
|
||||
}
|
||||
|
||||
async initialize(): Promise<void> {
|
||||
if (!this._initPromise) {
|
||||
this._initPromise = this._doInitialize();
|
||||
}
|
||||
return this._initPromise;
|
||||
}
|
||||
|
||||
private async _doInitialize(): Promise<void> {
|
||||
try {
|
||||
// Check if the index already exists
|
||||
let indexFound = false;
|
||||
|
||||
@@ -0,0 +1,88 @@
|
||||
/// <reference types="jest" />
|
||||
import { ConfigManager } from "../src/config/manager";
|
||||
|
||||
describe("ConfigManager", () => {
|
||||
describe("mergeConfig - dimension handling", () => {
|
||||
const baseLlm = {
|
||||
provider: "openai",
|
||||
config: { apiKey: "test-key" },
|
||||
};
|
||||
|
||||
it("should leave dimension undefined when no explicit dimension or embeddingDims provided", () => {
|
||||
const config = ConfigManager.mergeConfig({
|
||||
embedder: { provider: "openai", config: { apiKey: "test-key" } },
|
||||
vectorStore: { provider: "memory", config: { collectionName: "test" } },
|
||||
llm: baseLlm,
|
||||
});
|
||||
|
||||
// Dimension should be undefined so Memory._autoInitialize() will
|
||||
// auto-detect it via a probe embedding at runtime.
|
||||
expect(config.vectorStore.config.dimension).toBeUndefined();
|
||||
});
|
||||
|
||||
it("should use embeddingDims from embedder config when provided", () => {
|
||||
const config = ConfigManager.mergeConfig({
|
||||
embedder: {
|
||||
provider: "ollama",
|
||||
config: { model: "nomic-embed-text", embeddingDims: 768 },
|
||||
},
|
||||
vectorStore: { provider: "qdrant", config: { collectionName: "test" } },
|
||||
llm: baseLlm,
|
||||
});
|
||||
|
||||
expect(config.vectorStore.config.dimension).toBe(768);
|
||||
});
|
||||
|
||||
it("should prefer explicit vector store dimension over embedder dims", () => {
|
||||
const config = ConfigManager.mergeConfig({
|
||||
embedder: {
|
||||
provider: "ollama",
|
||||
config: { model: "nomic-embed-text", embeddingDims: 768 },
|
||||
},
|
||||
vectorStore: {
|
||||
provider: "qdrant",
|
||||
config: { collectionName: "test", dimension: 1024 },
|
||||
},
|
||||
llm: baseLlm,
|
||||
});
|
||||
|
||||
expect(config.vectorStore.config.dimension).toBe(1024);
|
||||
});
|
||||
|
||||
it("should leave dimension undefined when using a custom client without explicit dims", () => {
|
||||
const mockClient = { someMethod: () => {} };
|
||||
const config = ConfigManager.mergeConfig({
|
||||
embedder: {
|
||||
provider: "ollama",
|
||||
config: { model: "nomic-embed-text" },
|
||||
},
|
||||
vectorStore: {
|
||||
provider: "qdrant",
|
||||
config: { collectionName: "test", client: mockClient },
|
||||
},
|
||||
llm: baseLlm,
|
||||
});
|
||||
|
||||
// No embeddingDims and no explicit dimension → should be undefined
|
||||
// for auto-detection at runtime.
|
||||
expect(config.vectorStore.config.dimension).toBeUndefined();
|
||||
});
|
||||
|
||||
it("should use embeddingDims when using a custom client", () => {
|
||||
const mockClient = { someMethod: () => {} };
|
||||
const config = ConfigManager.mergeConfig({
|
||||
embedder: {
|
||||
provider: "ollama",
|
||||
config: { model: "nomic-embed-text", embeddingDims: 768 },
|
||||
},
|
||||
vectorStore: {
|
||||
provider: "qdrant",
|
||||
config: { collectionName: "test", client: mockClient },
|
||||
},
|
||||
llm: baseLlm,
|
||||
});
|
||||
|
||||
expect(config.vectorStore.config.dimension).toBe(768);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,519 @@
|
||||
/// <reference types="jest" />
|
||||
/**
|
||||
* Tests for embedding dimension auto-detection.
|
||||
*
|
||||
* Covers:
|
||||
* - ConfigManager: dimension resolution logic
|
||||
* - Memory class: probe-based auto-detection, lazy init gate, backward compat
|
||||
* - MemoryVectorStore: backward compat with explicit dimensions
|
||||
* - Explicit error messages on probe failure
|
||||
*/
|
||||
|
||||
import { ConfigManager } from "../src/config/manager";
|
||||
import { MemoryVectorStore } from "../src/vector_stores/memory";
|
||||
import * as fs from "fs";
|
||||
import * as path from "path";
|
||||
import * as os from "os";
|
||||
|
||||
jest.setTimeout(15000);
|
||||
|
||||
// ───────────────────────────────────────────────────────────────────────────
|
||||
// 1. ConfigManager – dimension resolution
|
||||
// ───────────────────────────────────────────────────────────────────────────
|
||||
describe("ConfigManager – dimension resolution", () => {
|
||||
const baseLlm = { provider: "openai", config: { apiKey: "k" } };
|
||||
|
||||
it("leaves dimension undefined when nothing explicit is set", () => {
|
||||
const cfg = ConfigManager.mergeConfig({
|
||||
embedder: { provider: "openai", config: { apiKey: "k" } },
|
||||
vectorStore: { provider: "memory", config: { collectionName: "t" } },
|
||||
llm: baseLlm,
|
||||
});
|
||||
expect(cfg.vectorStore.config.dimension).toBeUndefined();
|
||||
});
|
||||
|
||||
it("uses embeddingDims from embedder config", () => {
|
||||
const cfg = ConfigManager.mergeConfig({
|
||||
embedder: {
|
||||
provider: "ollama",
|
||||
config: { model: "nomic-embed-text", embeddingDims: 768 },
|
||||
},
|
||||
vectorStore: { provider: "qdrant", config: { collectionName: "t" } },
|
||||
llm: baseLlm,
|
||||
});
|
||||
expect(cfg.vectorStore.config.dimension).toBe(768);
|
||||
});
|
||||
|
||||
it("prefers explicit vectorStore.dimension over embeddingDims", () => {
|
||||
const cfg = ConfigManager.mergeConfig({
|
||||
embedder: {
|
||||
provider: "ollama",
|
||||
config: { model: "nomic-embed-text", embeddingDims: 768 },
|
||||
},
|
||||
vectorStore: {
|
||||
provider: "qdrant",
|
||||
config: { collectionName: "t", dimension: 1024 },
|
||||
},
|
||||
llm: baseLlm,
|
||||
});
|
||||
expect(cfg.vectorStore.config.dimension).toBe(1024);
|
||||
});
|
||||
|
||||
it("leaves dimension undefined for custom client without explicit dims", () => {
|
||||
const cfg = ConfigManager.mergeConfig({
|
||||
embedder: { provider: "ollama", config: { model: "nomic-embed-text" } },
|
||||
vectorStore: {
|
||||
provider: "qdrant",
|
||||
config: { collectionName: "t", client: {} },
|
||||
},
|
||||
llm: baseLlm,
|
||||
});
|
||||
expect(cfg.vectorStore.config.dimension).toBeUndefined();
|
||||
});
|
||||
|
||||
it("uses embeddingDims with a custom client", () => {
|
||||
const cfg = ConfigManager.mergeConfig({
|
||||
embedder: {
|
||||
provider: "ollama",
|
||||
config: { model: "nomic-embed-text", embeddingDims: 768 },
|
||||
},
|
||||
vectorStore: {
|
||||
provider: "qdrant",
|
||||
config: { collectionName: "t", client: {} },
|
||||
},
|
||||
llm: baseLlm,
|
||||
});
|
||||
expect(cfg.vectorStore.config.dimension).toBe(768);
|
||||
});
|
||||
|
||||
it("preserves all other vectorStore config fields", () => {
|
||||
const cfg = ConfigManager.mergeConfig({
|
||||
embedder: { provider: "openai", config: { apiKey: "k" } },
|
||||
vectorStore: {
|
||||
provider: "qdrant",
|
||||
config: {
|
||||
collectionName: "my-coll",
|
||||
host: "my-host",
|
||||
port: 6333,
|
||||
apiKey: "qdrant-key",
|
||||
},
|
||||
},
|
||||
llm: baseLlm,
|
||||
});
|
||||
expect(cfg.vectorStore.config.collectionName).toBe("my-coll");
|
||||
expect(cfg.vectorStore.config.host).toBe("my-host");
|
||||
expect(cfg.vectorStore.config.port).toBe(6333);
|
||||
expect(cfg.vectorStore.config.apiKey).toBe("qdrant-key");
|
||||
});
|
||||
|
||||
it("leaves dimension undefined with empty config", () => {
|
||||
const cfg = ConfigManager.mergeConfig({
|
||||
embedder: { provider: "openai", config: {} },
|
||||
vectorStore: { provider: "memory", config: {} },
|
||||
llm: baseLlm,
|
||||
});
|
||||
expect(cfg.vectorStore.config.dimension).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
// ───────────────────────────────────────────────────────────────────────────
|
||||
// 2. MemoryVectorStore – backward compat with explicit dimensions
|
||||
// ───────────────────────────────────────────────────────────────────────────
|
||||
describe("MemoryVectorStore – backward compat", () => {
|
||||
let tmpDir: string;
|
||||
|
||||
beforeEach(() => {
|
||||
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "mem0-test-"));
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
fs.rmSync(tmpDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it("defaults to dimension 1536 when not specified", async () => {
|
||||
const store = new MemoryVectorStore({
|
||||
collectionName: "test",
|
||||
dbPath: path.join(tmpDir, "vs.db"),
|
||||
});
|
||||
|
||||
const vector = new Array(1536).fill(0.1);
|
||||
await store.insert([vector], ["id-1"], [{ data: "hello" }]);
|
||||
const result = await store.get("id-1");
|
||||
expect(result).not.toBeNull();
|
||||
});
|
||||
|
||||
it("explicit dimension=1536 still works", async () => {
|
||||
const store = new MemoryVectorStore({
|
||||
collectionName: "test",
|
||||
dimension: 1536,
|
||||
dbPath: path.join(tmpDir, "vs.db"),
|
||||
});
|
||||
|
||||
const vector = new Array(1536).fill(0.1);
|
||||
await store.insert([vector], ["id-1"], [{ data: "hello" }]);
|
||||
const result = await store.get("id-1");
|
||||
expect(result).not.toBeNull();
|
||||
});
|
||||
|
||||
it("explicit dimension rejects mismatched vectors", async () => {
|
||||
const store = new MemoryVectorStore({
|
||||
collectionName: "test",
|
||||
dimension: 1536,
|
||||
dbPath: path.join(tmpDir, "vs.db"),
|
||||
});
|
||||
|
||||
const wrongVector = new Array(768).fill(0.1);
|
||||
await expect(
|
||||
store.insert([wrongVector], ["id-1"], [{ data: "hello" }]),
|
||||
).rejects.toThrow("Vector dimension mismatch");
|
||||
});
|
||||
|
||||
it("search validates dimension", async () => {
|
||||
const store = new MemoryVectorStore({
|
||||
collectionName: "test",
|
||||
dimension: 4,
|
||||
dbPath: path.join(tmpDir, "vs.db"),
|
||||
});
|
||||
|
||||
await expect(store.search([1, 2, 3], 1)).rejects.toThrow(
|
||||
"Query dimension mismatch",
|
||||
);
|
||||
});
|
||||
|
||||
it("custom dimension=768 works end-to-end", async () => {
|
||||
const store = new MemoryVectorStore({
|
||||
collectionName: "test",
|
||||
dimension: 768,
|
||||
dbPath: path.join(tmpDir, "vs.db"),
|
||||
});
|
||||
|
||||
await store.insert(
|
||||
[
|
||||
[1, ...new Array(767).fill(0)],
|
||||
[0, 1, ...new Array(766).fill(0)],
|
||||
],
|
||||
["a", "b"],
|
||||
[{ data: "alpha" }, { data: "beta" }],
|
||||
);
|
||||
|
||||
const results = await store.search([1, ...new Array(767).fill(0)], 2);
|
||||
expect(results.length).toBe(2);
|
||||
expect(results[0].id).toBe("a");
|
||||
});
|
||||
|
||||
it("getUserId and setUserId still work", async () => {
|
||||
const store = new MemoryVectorStore({
|
||||
collectionName: "test",
|
||||
dbPath: path.join(tmpDir, "vs.db"),
|
||||
});
|
||||
|
||||
const userId = await store.getUserId();
|
||||
expect(typeof userId).toBe("string");
|
||||
expect(userId.length).toBeGreaterThan(0);
|
||||
|
||||
await store.setUserId("custom-user");
|
||||
const newUserId = await store.getUserId();
|
||||
expect(newUserId).toBe("custom-user");
|
||||
});
|
||||
|
||||
it("initialize() is idempotent", async () => {
|
||||
const store = new MemoryVectorStore({
|
||||
collectionName: "test",
|
||||
dbPath: path.join(tmpDir, "vs.db"),
|
||||
});
|
||||
|
||||
await store.initialize();
|
||||
await store.initialize();
|
||||
await store.initialize();
|
||||
});
|
||||
});
|
||||
|
||||
// ───────────────────────────────────────────────────────────────────────────
|
||||
// 3. Memory class – auto-init with probe, lazy gate, backward compat
|
||||
// ───────────────────────────────────────────────────────────────────────────
|
||||
describe("Memory – auto-initialization", () => {
|
||||
let mockEmbedderFactory: any;
|
||||
let mockVectorStoreFactory: any;
|
||||
let mockLlmFactory: any;
|
||||
let mockHistoryFactory: any;
|
||||
let MemoryClass: any;
|
||||
|
||||
function createMockEmbedder(dims: number) {
|
||||
return {
|
||||
embed: jest.fn().mockResolvedValue(new Array(dims).fill(0)),
|
||||
embedBatch: jest.fn().mockResolvedValue([new Array(dims).fill(0)]),
|
||||
};
|
||||
}
|
||||
|
||||
function createMockVectorStore() {
|
||||
return {
|
||||
insert: jest.fn().mockResolvedValue(undefined),
|
||||
search: jest.fn().mockResolvedValue([]),
|
||||
get: jest.fn().mockResolvedValue(null),
|
||||
update: jest.fn().mockResolvedValue(undefined),
|
||||
delete: jest.fn().mockResolvedValue(undefined),
|
||||
deleteCol: jest.fn().mockResolvedValue(undefined),
|
||||
list: jest.fn().mockResolvedValue([[], 0]),
|
||||
getUserId: jest.fn().mockResolvedValue("test-user-id"),
|
||||
setUserId: jest.fn().mockResolvedValue(undefined),
|
||||
initialize: jest.fn().mockResolvedValue(undefined),
|
||||
};
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
jest.resetModules();
|
||||
|
||||
const mockEmbedder = createMockEmbedder(768);
|
||||
const mockVStore = createMockVectorStore();
|
||||
|
||||
mockEmbedderFactory = { create: jest.fn().mockReturnValue(mockEmbedder) };
|
||||
mockVectorStoreFactory = { create: jest.fn().mockReturnValue(mockVStore) };
|
||||
mockLlmFactory = {
|
||||
create: jest.fn().mockReturnValue({
|
||||
generateResponse: jest.fn().mockResolvedValue('{"facts":[]}'),
|
||||
}),
|
||||
};
|
||||
mockHistoryFactory = {
|
||||
create: jest.fn().mockReturnValue({
|
||||
addHistory: jest.fn().mockResolvedValue(undefined),
|
||||
getHistory: jest.fn().mockResolvedValue([]),
|
||||
reset: jest.fn().mockResolvedValue(undefined),
|
||||
}),
|
||||
};
|
||||
|
||||
jest.doMock("../src/utils/factory", () => ({
|
||||
EmbedderFactory: mockEmbedderFactory,
|
||||
VectorStoreFactory: mockVectorStoreFactory,
|
||||
LLMFactory: mockLlmFactory,
|
||||
HistoryManagerFactory: mockHistoryFactory,
|
||||
}));
|
||||
|
||||
jest.doMock("../src/utils/telemetry", () => ({
|
||||
captureClientEvent: jest.fn().mockResolvedValue(undefined),
|
||||
}));
|
||||
|
||||
MemoryClass = require("../src/memory").Memory;
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
jest.restoreAllMocks();
|
||||
jest.resetModules();
|
||||
});
|
||||
|
||||
it("probes embedder to detect dimension when none set", async () => {
|
||||
const mockEmbedder = createMockEmbedder(768);
|
||||
const mockVStore = createMockVectorStore();
|
||||
mockEmbedderFactory.create.mockReturnValue(mockEmbedder);
|
||||
mockVectorStoreFactory.create.mockReturnValue(mockVStore);
|
||||
|
||||
const mem = new MemoryClass({
|
||||
embedder: { provider: "ollama", config: { model: "nomic-embed-text" } },
|
||||
vectorStore: { provider: "qdrant", config: { collectionName: "test" } },
|
||||
llm: { provider: "openai", config: { apiKey: "k" } },
|
||||
disableHistory: true,
|
||||
});
|
||||
|
||||
await mem.getAll({ userId: "u1" });
|
||||
|
||||
// Should have called embed("dimension probe") to detect dimension
|
||||
expect(mockEmbedder.embed).toHaveBeenCalledWith("dimension probe");
|
||||
|
||||
// VectorStoreFactory should have been called with detected dimension
|
||||
const vsCreateCall = mockVectorStoreFactory.create.mock.calls[0];
|
||||
expect(vsCreateCall[1].dimension).toBe(768);
|
||||
});
|
||||
|
||||
it("skips probe when explicit dimension provided", async () => {
|
||||
const mockEmbedder = createMockEmbedder(1536);
|
||||
const mockVStore = createMockVectorStore();
|
||||
mockEmbedderFactory.create.mockReturnValue(mockEmbedder);
|
||||
mockVectorStoreFactory.create.mockReturnValue(mockVStore);
|
||||
|
||||
const mem = new MemoryClass({
|
||||
embedder: { provider: "openai", config: { apiKey: "k" } },
|
||||
vectorStore: {
|
||||
provider: "memory",
|
||||
config: { collectionName: "test", dimension: 1536 },
|
||||
},
|
||||
llm: { provider: "openai", config: { apiKey: "k" } },
|
||||
disableHistory: true,
|
||||
});
|
||||
|
||||
await mem.getAll({ userId: "u1" });
|
||||
|
||||
// embed should NOT have been called for probing
|
||||
expect(mockEmbedder.embed).not.toHaveBeenCalledWith("dimension probe");
|
||||
|
||||
// VectorStoreFactory gets the explicit dimension
|
||||
const vsCreateCall = mockVectorStoreFactory.create.mock.calls[0];
|
||||
expect(vsCreateCall[1].dimension).toBe(1536);
|
||||
});
|
||||
|
||||
it("skips probe when embeddingDims provided", async () => {
|
||||
const mockEmbedder = createMockEmbedder(768);
|
||||
const mockVStore = createMockVectorStore();
|
||||
mockEmbedderFactory.create.mockReturnValue(mockEmbedder);
|
||||
mockVectorStoreFactory.create.mockReturnValue(mockVStore);
|
||||
|
||||
const mem = new MemoryClass({
|
||||
embedder: {
|
||||
provider: "ollama",
|
||||
config: { model: "nomic-embed-text", embeddingDims: 768 },
|
||||
},
|
||||
vectorStore: { provider: "qdrant", config: { collectionName: "test" } },
|
||||
llm: { provider: "openai", config: { apiKey: "k" } },
|
||||
disableHistory: true,
|
||||
});
|
||||
|
||||
await mem.getAll({ userId: "u1" });
|
||||
|
||||
// ConfigManager resolves dimension from embeddingDims → no probe needed
|
||||
expect(mockEmbedder.embed).not.toHaveBeenCalledWith("dimension probe");
|
||||
});
|
||||
|
||||
it("all public methods wait for initialization", async () => {
|
||||
let resolveProbe: () => void;
|
||||
let probeCallCount = 0;
|
||||
const mockEmbedder = {
|
||||
embed: jest.fn().mockImplementation(() => {
|
||||
probeCallCount++;
|
||||
if (probeCallCount === 1) {
|
||||
// First call is the dimension probe — hang until manually resolved
|
||||
return new Promise<number[]>((resolve) => {
|
||||
resolveProbe = () => resolve(new Array(768).fill(0));
|
||||
});
|
||||
}
|
||||
// Subsequent calls (from search, etc.) resolve immediately
|
||||
return Promise.resolve(new Array(768).fill(0));
|
||||
}),
|
||||
embedBatch: jest.fn(),
|
||||
};
|
||||
const mockVStore = createMockVectorStore();
|
||||
mockEmbedderFactory.create.mockReturnValue(mockEmbedder);
|
||||
mockVectorStoreFactory.create.mockReturnValue(mockVStore);
|
||||
|
||||
const mem = new MemoryClass({
|
||||
embedder: { provider: "ollama", config: { model: "test" } },
|
||||
vectorStore: { provider: "qdrant", config: { collectionName: "t" } },
|
||||
llm: { provider: "openai", config: { apiKey: "k" } },
|
||||
disableHistory: true,
|
||||
});
|
||||
|
||||
let getAllDone = false;
|
||||
let searchDone = false;
|
||||
let getDone = false;
|
||||
|
||||
const getAllP = mem.getAll({ userId: "u" }).then(() => (getAllDone = true));
|
||||
const searchP = mem
|
||||
.search("q", { userId: "u" })
|
||||
.then(() => (searchDone = true));
|
||||
const getP = mem.get("id").then(() => (getDone = true));
|
||||
|
||||
await new Promise((r) => setTimeout(r, 50));
|
||||
expect(getAllDone).toBe(false);
|
||||
expect(searchDone).toBe(false);
|
||||
expect(getDone).toBe(false);
|
||||
|
||||
// Resolve the probe — init completes — methods unblock
|
||||
resolveProbe!();
|
||||
await Promise.all([getAllP, searchP, getP]);
|
||||
expect(getAllDone).toBe(true);
|
||||
expect(searchDone).toBe(true);
|
||||
expect(getDone).toBe(true);
|
||||
});
|
||||
|
||||
it("reset re-creates vector store with correct dimension", async () => {
|
||||
const mockEmbedder = createMockEmbedder(768);
|
||||
const mockVStore = createMockVectorStore();
|
||||
mockEmbedderFactory.create.mockReturnValue(mockEmbedder);
|
||||
mockVectorStoreFactory.create.mockReturnValue(mockVStore);
|
||||
|
||||
const mem = new MemoryClass({
|
||||
embedder: { provider: "ollama", config: { model: "nomic-embed-text" } },
|
||||
vectorStore: { provider: "qdrant", config: { collectionName: "test" } },
|
||||
llm: { provider: "openai", config: { apiKey: "k" } },
|
||||
disableHistory: true,
|
||||
});
|
||||
|
||||
await mem.getAll({ userId: "u1" });
|
||||
expect(mockVectorStoreFactory.create).toHaveBeenCalledTimes(1);
|
||||
|
||||
// Reset should re-create vector store
|
||||
const mockVStore2 = createMockVectorStore();
|
||||
mockVectorStoreFactory.create.mockReturnValue(mockVStore2);
|
||||
await mem.reset();
|
||||
expect(mockVectorStoreFactory.create).toHaveBeenCalledTimes(2);
|
||||
|
||||
// Second creation should still have dimension=768 (cached from first probe)
|
||||
const secondCall = mockVectorStoreFactory.create.mock.calls[1];
|
||||
expect(secondCall[1].dimension).toBe(768);
|
||||
});
|
||||
|
||||
it("backward compat: full explicit config works without probe", async () => {
|
||||
const mockEmbedder = createMockEmbedder(1536);
|
||||
const mockVStore = createMockVectorStore();
|
||||
mockEmbedderFactory.create.mockReturnValue(mockEmbedder);
|
||||
mockVectorStoreFactory.create.mockReturnValue(mockVStore);
|
||||
|
||||
const mem = new MemoryClass({
|
||||
version: "v1.1",
|
||||
embedder: {
|
||||
provider: "openai",
|
||||
config: { apiKey: "sk-fake", model: "text-embedding-3-small" },
|
||||
},
|
||||
vectorStore: {
|
||||
provider: "memory",
|
||||
config: { collectionName: "test-memories", dimension: 1536 },
|
||||
},
|
||||
llm: {
|
||||
provider: "openai",
|
||||
config: { apiKey: "sk-fake", model: "gpt-4-turbo-preview" },
|
||||
},
|
||||
historyDbPath: ":memory:",
|
||||
disableHistory: true,
|
||||
});
|
||||
|
||||
await mem.getAll({ userId: "u1" });
|
||||
expect(mockEmbedder.embed).not.toHaveBeenCalledWith("dimension probe");
|
||||
});
|
||||
|
||||
it("throws explicit error when probe fails", async () => {
|
||||
const mockEmbedder = {
|
||||
embed: jest.fn().mockRejectedValue(new Error("Connection refused")),
|
||||
embedBatch: jest.fn(),
|
||||
};
|
||||
mockEmbedderFactory.create.mockReturnValue(mockEmbedder);
|
||||
|
||||
// Suppress console.error for this test
|
||||
const consoleSpy = jest
|
||||
.spyOn(console, "error")
|
||||
.mockImplementation(() => {});
|
||||
|
||||
const mem = new MemoryClass({
|
||||
embedder: { provider: "ollama", config: { model: "nomic-embed-text" } },
|
||||
vectorStore: { provider: "qdrant", config: { collectionName: "test" } },
|
||||
llm: { provider: "openai", config: { apiKey: "k" } },
|
||||
disableHistory: true,
|
||||
});
|
||||
|
||||
// getAll should reject with the init error
|
||||
await expect(mem.getAll({ userId: "u1" })).rejects.toThrow(
|
||||
"auto-detect embedding dimension",
|
||||
);
|
||||
|
||||
// Verify the error was logged and contains helpful information
|
||||
const errorCall = consoleSpy.mock.calls.find(
|
||||
(call) =>
|
||||
call[0] instanceof Error &&
|
||||
call[0].message.includes("auto-detect embedding dimension"),
|
||||
);
|
||||
expect(errorCall).toBeDefined();
|
||||
const errorMsg = (errorCall![0] as Error).message;
|
||||
expect(errorMsg).toContain("ollama");
|
||||
expect(errorMsg).toContain("Connection refused");
|
||||
expect(errorMsg).toContain("dimension");
|
||||
expect(errorMsg).toContain("embeddingDims");
|
||||
|
||||
consoleSpy.mockRestore();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,521 @@
|
||||
/// <reference types="jest" />
|
||||
/**
|
||||
* End-to-end tests for Qdrant dimension mismatch fix.
|
||||
*
|
||||
* Requires a running Qdrant instance at localhost:6333 (v1.13.x).
|
||||
* These tests replicate the exact scenarios from issues #4212, #4173, #4056.
|
||||
*
|
||||
* Skipped automatically when Qdrant is not available.
|
||||
*
|
||||
* Run: npx jest --config jest.config.js src/oss/tests/qdrant-e2e.test.ts --forceExit
|
||||
*/
|
||||
|
||||
import { QdrantClient } from "@qdrant/js-client-rest";
|
||||
import { Qdrant } from "../src/vector_stores/qdrant";
|
||||
import { v4 as uuidv4 } from "uuid";
|
||||
jest.setTimeout(30000);
|
||||
|
||||
const QDRANT_HOST = "localhost";
|
||||
const QDRANT_PORT = 6333;
|
||||
|
||||
// Check if Qdrant is reachable synchronously at load time using
|
||||
// a sync check via child_process so describe.skip works correctly.
|
||||
function isQdrantAvailable(): boolean {
|
||||
try {
|
||||
const { execSync } = require("child_process");
|
||||
execSync(
|
||||
`node -e "const s=require('net').createConnection({host:'${QDRANT_HOST}',port:${QDRANT_PORT}});s.on('connect',()=>{s.destroy();process.exit(0)});s.on('error',()=>process.exit(1));s.setTimeout(2000,()=>process.exit(1))"`,
|
||||
{ timeout: 3000, stdio: "ignore" },
|
||||
);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
const qdrantAvailable = isQdrantAvailable();
|
||||
if (!qdrantAvailable) {
|
||||
console.warn("Qdrant not available at localhost:6333 — skipping e2e tests");
|
||||
}
|
||||
|
||||
let qdrantClient: QdrantClient;
|
||||
|
||||
beforeAll(async () => {
|
||||
if (!qdrantAvailable) return;
|
||||
qdrantClient = new QdrantClient({ host: QDRANT_HOST, port: QDRANT_PORT });
|
||||
const collections = await qdrantClient.getCollections();
|
||||
expect(collections).toBeDefined();
|
||||
});
|
||||
|
||||
// Helper: delete a collection if it exists
|
||||
async function deleteCollectionIfExists(name: string) {
|
||||
try {
|
||||
await qdrantClient.deleteCollection(name);
|
||||
} catch {
|
||||
// Collection doesn't exist — fine
|
||||
}
|
||||
}
|
||||
|
||||
// Helper: create a fake embedder that produces vectors of a given dimension
|
||||
function createFakeEmbedder(dims: number) {
|
||||
return {
|
||||
embed: jest.fn().mockImplementation(async (_text: string) => {
|
||||
const vec = new Array(dims).fill(0);
|
||||
for (let i = 0; i < _text.length && i < dims; i++) {
|
||||
vec[i] = _text.charCodeAt(i) / 255;
|
||||
}
|
||||
return vec;
|
||||
}),
|
||||
embedBatch: jest.fn().mockImplementation(async (texts: string[]) => {
|
||||
return Promise.all(
|
||||
texts.map(async (t) => {
|
||||
const vec = new Array(dims).fill(0);
|
||||
for (let i = 0; i < t.length && i < dims; i++) {
|
||||
vec[i] = t.charCodeAt(i) / 255;
|
||||
}
|
||||
return vec;
|
||||
}),
|
||||
);
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
// Conditionally skip tests when Qdrant is unavailable
|
||||
const describeIfQdrant = qdrantAvailable ? describe : describe.skip;
|
||||
|
||||
afterAll(async () => {
|
||||
await deleteCollectionIfExists("e2e_test_768");
|
||||
await deleteCollectionIfExists("e2e_test_1536");
|
||||
await deleteCollectionIfExists("e2e_test_race");
|
||||
await deleteCollectionIfExists("e2e_test_race2");
|
||||
await deleteCollectionIfExists("e2e_test_noexplicit");
|
||||
await deleteCollectionIfExists("e2e_test_explicit");
|
||||
await deleteCollectionIfExists("e2e_test_embdims");
|
||||
await deleteCollectionIfExists("e2e_test_autodetect");
|
||||
await deleteCollectionIfExists("memory_migrations");
|
||||
});
|
||||
|
||||
// ───────────────────────────────────────────────────────────────────────────
|
||||
// 1. Reproduce #4212 / #4173: dimension mismatch with 768-dim embedder
|
||||
// ───────────────────────────────────────────────────────────────────────────
|
||||
describeIfQdrant("Issue #4212/#4173: Qdrant dimension mismatch", () => {
|
||||
it("BEFORE FIX scenario: 768-dim vector into 1536-dim collection → Bad Request", async () => {
|
||||
const collectionName = "e2e_test_1536";
|
||||
await deleteCollectionIfExists(collectionName);
|
||||
await qdrantClient.createCollection(collectionName, {
|
||||
vectors: { size: 1536, distance: "Cosine" },
|
||||
});
|
||||
|
||||
// Insert a 768-dim vector — this is what nomic-embed-text produces
|
||||
const vector768 = new Array(768).fill(0.1);
|
||||
try {
|
||||
await qdrantClient.upsert(collectionName, {
|
||||
points: [
|
||||
{ id: "test-1", vector: vector768, payload: { data: "hello" } },
|
||||
],
|
||||
});
|
||||
fail("Expected Qdrant to reject 768-dim vector into 1536-dim collection");
|
||||
} catch (error: any) {
|
||||
// This is the exact "Bad Request" error users were hitting
|
||||
expect(error.status).toBe(400);
|
||||
}
|
||||
|
||||
await deleteCollectionIfExists(collectionName);
|
||||
});
|
||||
|
||||
it("AFTER FIX: Qdrant store with dimension=768 works end-to-end", async () => {
|
||||
const collectionName = "e2e_test_768";
|
||||
await deleteCollectionIfExists(collectionName);
|
||||
await deleteCollectionIfExists("memory_migrations");
|
||||
|
||||
// Create Qdrant store with correct dimension (what our auto-detect provides)
|
||||
const store = new Qdrant({
|
||||
host: QDRANT_HOST,
|
||||
port: QDRANT_PORT,
|
||||
collectionName,
|
||||
embeddingModelDims: 768,
|
||||
dimension: 768,
|
||||
});
|
||||
await store.initialize();
|
||||
|
||||
// Verify collection was created with 768 dims
|
||||
const info = await qdrantClient.getCollection(collectionName);
|
||||
expect(info.config?.params?.vectors?.size).toBe(768);
|
||||
|
||||
// Insert 768-dim vectors (what nomic-embed-text produces)
|
||||
const vec1 = new Array(768).fill(0);
|
||||
vec1[0] = 1.0;
|
||||
const vec2 = new Array(768).fill(0);
|
||||
vec2[1] = 1.0;
|
||||
|
||||
const id1 = uuidv4();
|
||||
const id2 = uuidv4();
|
||||
|
||||
await store.insert([vec1, vec2], [id1, id2], [
|
||||
{ data: "hello", userId: "u1" },
|
||||
{ data: "world", userId: "u1" },
|
||||
]);
|
||||
|
||||
// Search with 768-dim query — this USED TO fail with Bad Request
|
||||
const results = await store.search(vec1, 2, { userId: "u1" });
|
||||
expect(results.length).toBe(2);
|
||||
expect(results[0].id).toBe(id1); // Most similar to itself
|
||||
expect(results[0].score).toBeGreaterThan(0.9);
|
||||
|
||||
// Get by ID
|
||||
const item = await store.get(id1);
|
||||
expect(item).not.toBeNull();
|
||||
expect(item!.payload.data).toBe("hello");
|
||||
|
||||
// Update with 768-dim vector
|
||||
const vec3 = new Array(768).fill(0);
|
||||
vec3[2] = 1.0;
|
||||
await store.update(id1, vec3, { data: "updated", userId: "u1" });
|
||||
const updated = await store.get(id1);
|
||||
expect(updated!.payload.data).toBe("updated");
|
||||
|
||||
// Delete
|
||||
await store.delete(id2);
|
||||
const deleted = await store.get(id2);
|
||||
expect(deleted).toBeNull();
|
||||
|
||||
// List
|
||||
const [listed, count] = await store.list({ userId: "u1" });
|
||||
expect(count).toBe(1);
|
||||
expect(listed[0].payload.data).toBe("updated");
|
||||
|
||||
await deleteCollectionIfExists(collectionName);
|
||||
await deleteCollectionIfExists("memory_migrations");
|
||||
});
|
||||
|
||||
it("AFTER FIX: Memory auto-detects 768 dims via probe (full integration)", async () => {
|
||||
const collectionName = "e2e_test_autodetect";
|
||||
await deleteCollectionIfExists(collectionName);
|
||||
await deleteCollectionIfExists("memory_migrations");
|
||||
|
||||
const fakeEmbedder = createFakeEmbedder(768);
|
||||
|
||||
// Mock only the non-Qdrant factories to avoid Google SDK import crash
|
||||
jest.resetModules();
|
||||
jest.doMock("../src/utils/factory", () => {
|
||||
// Import Qdrant directly (avoids loading Google embedder via factory)
|
||||
const { Qdrant: QdrantStore } = require("../src/vector_stores/qdrant");
|
||||
return {
|
||||
EmbedderFactory: { create: jest.fn().mockReturnValue(fakeEmbedder) },
|
||||
VectorStoreFactory: {
|
||||
create: jest.fn().mockImplementation((_provider: string, config: any) => {
|
||||
return new QdrantStore(config);
|
||||
}),
|
||||
},
|
||||
LLMFactory: {
|
||||
create: jest.fn().mockReturnValue({
|
||||
generateResponse: jest.fn().mockResolvedValue('{"facts":[]}'),
|
||||
}),
|
||||
},
|
||||
HistoryManagerFactory: {
|
||||
create: jest.fn().mockReturnValue({
|
||||
addHistory: jest.fn().mockResolvedValue(undefined),
|
||||
getHistory: jest.fn().mockResolvedValue([]),
|
||||
reset: jest.fn().mockResolvedValue(undefined),
|
||||
}),
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
jest.doMock("../src/utils/telemetry", () => ({
|
||||
captureClientEvent: jest.fn().mockResolvedValue(undefined),
|
||||
}));
|
||||
|
||||
const { Memory } = require("../src/memory");
|
||||
|
||||
// This is the EXACT config from issue #4212 — NO dimension specified
|
||||
const mem = new Memory({
|
||||
embedder: {
|
||||
provider: "ollama",
|
||||
config: { model: "nomic-embed-text" },
|
||||
},
|
||||
vectorStore: {
|
||||
provider: "qdrant",
|
||||
config: {
|
||||
host: QDRANT_HOST,
|
||||
port: QDRANT_PORT,
|
||||
collectionName,
|
||||
},
|
||||
},
|
||||
llm: { provider: "openai", config: { apiKey: "fake" } },
|
||||
disableHistory: true,
|
||||
});
|
||||
|
||||
// This triggers init — probe should detect 768 dims
|
||||
await mem.getAll({ userId: "test-user" });
|
||||
|
||||
// Verify the probe was called
|
||||
expect(fakeEmbedder.embed).toHaveBeenCalledWith("dimension probe");
|
||||
|
||||
// Verify Qdrant collection was created with auto-detected 768 dims
|
||||
const collectionInfo = await qdrantClient.getCollection(collectionName);
|
||||
expect(collectionInfo.config?.params?.vectors?.size).toBe(768);
|
||||
|
||||
// Search should work (this used to throw Bad Request)
|
||||
const searchResult = await mem.search("hello world", {
|
||||
userId: "test-user",
|
||||
});
|
||||
expect(searchResult).toBeDefined();
|
||||
|
||||
await deleteCollectionIfExists(collectionName);
|
||||
await deleteCollectionIfExists("memory_migrations");
|
||||
jest.resetModules();
|
||||
});
|
||||
|
||||
it("AFTER FIX: explicit dimension=768 skips probe (backward compat)", async () => {
|
||||
const collectionName = "e2e_test_explicit";
|
||||
await deleteCollectionIfExists(collectionName);
|
||||
await deleteCollectionIfExists("memory_migrations");
|
||||
|
||||
const fakeEmbedder = createFakeEmbedder(768);
|
||||
|
||||
jest.resetModules();
|
||||
jest.doMock("../src/utils/factory", () => {
|
||||
const { Qdrant: QdrantStore } = require("../src/vector_stores/qdrant");
|
||||
return {
|
||||
EmbedderFactory: { create: jest.fn().mockReturnValue(fakeEmbedder) },
|
||||
VectorStoreFactory: {
|
||||
create: jest.fn().mockImplementation((_provider: string, config: any) => {
|
||||
return new QdrantStore(config);
|
||||
}),
|
||||
},
|
||||
LLMFactory: {
|
||||
create: jest.fn().mockReturnValue({
|
||||
generateResponse: jest.fn().mockResolvedValue('{"facts":[]}'),
|
||||
}),
|
||||
},
|
||||
HistoryManagerFactory: {
|
||||
create: jest.fn().mockReturnValue({
|
||||
addHistory: jest.fn().mockResolvedValue(undefined),
|
||||
getHistory: jest.fn().mockResolvedValue([]),
|
||||
reset: jest.fn().mockResolvedValue(undefined),
|
||||
}),
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
jest.doMock("../src/utils/telemetry", () => ({
|
||||
captureClientEvent: jest.fn().mockResolvedValue(undefined),
|
||||
}));
|
||||
|
||||
const { Memory } = require("../src/memory");
|
||||
|
||||
// Workaround config from #4212 — explicit dimension
|
||||
const mem = new Memory({
|
||||
embedder: {
|
||||
provider: "ollama",
|
||||
config: { model: "nomic-embed-text" },
|
||||
},
|
||||
vectorStore: {
|
||||
provider: "qdrant",
|
||||
config: {
|
||||
host: QDRANT_HOST,
|
||||
port: QDRANT_PORT,
|
||||
collectionName,
|
||||
dimension: 768,
|
||||
},
|
||||
},
|
||||
llm: { provider: "openai", config: { apiKey: "fake" } },
|
||||
disableHistory: true,
|
||||
});
|
||||
|
||||
await mem.getAll({ userId: "test-user" });
|
||||
|
||||
// Probe should NOT have been called
|
||||
expect(fakeEmbedder.embed).not.toHaveBeenCalledWith("dimension probe");
|
||||
|
||||
const collectionInfo = await qdrantClient.getCollection(collectionName);
|
||||
expect(collectionInfo.config?.params?.vectors?.size).toBe(768);
|
||||
|
||||
await deleteCollectionIfExists(collectionName);
|
||||
await deleteCollectionIfExists("memory_migrations");
|
||||
jest.resetModules();
|
||||
});
|
||||
|
||||
it("AFTER FIX: embeddingDims in embedder config skips probe", async () => {
|
||||
const collectionName = "e2e_test_embdims";
|
||||
await deleteCollectionIfExists(collectionName);
|
||||
await deleteCollectionIfExists("memory_migrations");
|
||||
|
||||
const fakeEmbedder = createFakeEmbedder(768);
|
||||
|
||||
jest.resetModules();
|
||||
jest.doMock("../src/utils/factory", () => {
|
||||
const { Qdrant: QdrantStore } = require("../src/vector_stores/qdrant");
|
||||
return {
|
||||
EmbedderFactory: { create: jest.fn().mockReturnValue(fakeEmbedder) },
|
||||
VectorStoreFactory: {
|
||||
create: jest.fn().mockImplementation((_provider: string, config: any) => {
|
||||
return new QdrantStore(config);
|
||||
}),
|
||||
},
|
||||
LLMFactory: {
|
||||
create: jest.fn().mockReturnValue({
|
||||
generateResponse: jest.fn().mockResolvedValue('{"facts":[]}'),
|
||||
}),
|
||||
},
|
||||
HistoryManagerFactory: {
|
||||
create: jest.fn().mockReturnValue({
|
||||
addHistory: jest.fn().mockResolvedValue(undefined),
|
||||
getHistory: jest.fn().mockResolvedValue([]),
|
||||
reset: jest.fn().mockResolvedValue(undefined),
|
||||
}),
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
jest.doMock("../src/utils/telemetry", () => ({
|
||||
captureClientEvent: jest.fn().mockResolvedValue(undefined),
|
||||
}));
|
||||
|
||||
const { Memory } = require("../src/memory");
|
||||
|
||||
const mem = new Memory({
|
||||
embedder: {
|
||||
provider: "ollama",
|
||||
config: { model: "nomic-embed-text", embeddingDims: 768 },
|
||||
},
|
||||
vectorStore: {
|
||||
provider: "qdrant",
|
||||
config: {
|
||||
host: QDRANT_HOST,
|
||||
port: QDRANT_PORT,
|
||||
collectionName,
|
||||
},
|
||||
},
|
||||
llm: { provider: "openai", config: { apiKey: "fake" } },
|
||||
disableHistory: true,
|
||||
});
|
||||
|
||||
await mem.getAll({ userId: "test-user" });
|
||||
|
||||
// Probe should NOT have been called — dimension inferred from embeddingDims
|
||||
expect(fakeEmbedder.embed).not.toHaveBeenCalledWith("dimension probe");
|
||||
|
||||
const collectionInfo = await qdrantClient.getCollection(collectionName);
|
||||
expect(collectionInfo.config?.params?.vectors?.size).toBe(768);
|
||||
|
||||
await deleteCollectionIfExists(collectionName);
|
||||
await deleteCollectionIfExists("memory_migrations");
|
||||
jest.resetModules();
|
||||
});
|
||||
});
|
||||
|
||||
// ───────────────────────────────────────────────────────────────────────────
|
||||
// 2. Reproduce #4056 issue 1: Collection creation race condition
|
||||
// ───────────────────────────────────────────────────────────────────────────
|
||||
describeIfQdrant("Issue #4056: Qdrant race condition", () => {
|
||||
it("concurrent ensureCollection calls don't crash (no 409 error leak)", async () => {
|
||||
const collectionName = "e2e_test_race";
|
||||
await deleteCollectionIfExists(collectionName);
|
||||
await deleteCollectionIfExists("memory_migrations");
|
||||
|
||||
// Create 5 Qdrant instances concurrently — this simulates the race
|
||||
// that caused "Collection memory_migrations already exists!" in #4056
|
||||
const instances = Array.from(
|
||||
{ length: 5 },
|
||||
() =>
|
||||
new Qdrant({
|
||||
host: QDRANT_HOST,
|
||||
port: QDRANT_PORT,
|
||||
collectionName,
|
||||
embeddingModelDims: 768,
|
||||
dimension: 768,
|
||||
}),
|
||||
);
|
||||
|
||||
// All should initialize without throwing 409 Conflict
|
||||
await Promise.all(instances.map((inst) => inst.initialize()));
|
||||
|
||||
// Verify collection exists with correct dimension
|
||||
const info = await qdrantClient.getCollection(collectionName);
|
||||
expect(info.config?.params?.vectors?.size).toBe(768);
|
||||
|
||||
// memory_migrations should also exist (created by initialize)
|
||||
const migrInfo = await qdrantClient.getCollection("memory_migrations");
|
||||
expect(migrInfo.config?.params?.vectors?.size).toBe(1);
|
||||
|
||||
await deleteCollectionIfExists(collectionName);
|
||||
await deleteCollectionIfExists("memory_migrations");
|
||||
});
|
||||
|
||||
it("getUserId works after concurrent initialization", async () => {
|
||||
const collectionName = "e2e_test_race2";
|
||||
await deleteCollectionIfExists(collectionName);
|
||||
await deleteCollectionIfExists("memory_migrations");
|
||||
|
||||
const instance = new Qdrant({
|
||||
host: QDRANT_HOST,
|
||||
port: QDRANT_PORT,
|
||||
collectionName,
|
||||
embeddingModelDims: 768,
|
||||
dimension: 768,
|
||||
});
|
||||
|
||||
await instance.initialize();
|
||||
|
||||
// getUserId should work without 409 crash
|
||||
const userId = await instance.getUserId();
|
||||
expect(typeof userId).toBe("string");
|
||||
expect(userId.length).toBeGreaterThan(0);
|
||||
|
||||
// setUserId + getUserId roundtrip
|
||||
await instance.setUserId("custom-e2e-user");
|
||||
const updated = await instance.getUserId();
|
||||
expect(updated).toBe("custom-e2e-user");
|
||||
|
||||
await deleteCollectionIfExists(collectionName);
|
||||
await deleteCollectionIfExists("memory_migrations");
|
||||
});
|
||||
});
|
||||
|
||||
// ───────────────────────────────────────────────────────────────────────────
|
||||
// 3. Reproduce #4056 issue 2: memory_migrations dimension isolation
|
||||
// ───────────────────────────────────────────────────────────────────────────
|
||||
describeIfQdrant("Issue #4056: memory_migrations dimension isolation", () => {
|
||||
it("memory_migrations uses dim=1 independently of main collection dim=768", async () => {
|
||||
const collectionName = "e2e_test_noexplicit";
|
||||
await deleteCollectionIfExists(collectionName);
|
||||
await deleteCollectionIfExists("memory_migrations");
|
||||
|
||||
const instance = new Qdrant({
|
||||
host: QDRANT_HOST,
|
||||
port: QDRANT_PORT,
|
||||
collectionName,
|
||||
embeddingModelDims: 768,
|
||||
dimension: 768,
|
||||
});
|
||||
|
||||
await instance.initialize();
|
||||
|
||||
// Allow Qdrant a moment to fully commit collections
|
||||
await new Promise((r) => setTimeout(r, 500));
|
||||
|
||||
// Main collection should be 768
|
||||
const mainInfo = await qdrantClient.getCollection(collectionName);
|
||||
expect(mainInfo.config?.params?.vectors?.size).toBe(768);
|
||||
|
||||
// memory_migrations should be 1 (NOT 768!)
|
||||
// This was the bug in #4056 issue 2 — telemetry used wrong dimension
|
||||
const migrationsInfo =
|
||||
await qdrantClient.getCollection("memory_migrations");
|
||||
expect(migrationsInfo.config?.params?.vectors?.size).toBe(1);
|
||||
|
||||
// getUserId should work — vector dim=1 in memory_migrations
|
||||
const userId = await instance.getUserId();
|
||||
expect(typeof userId).toBe("string");
|
||||
|
||||
// setUserId should also work
|
||||
await instance.setUserId("custom-test-user");
|
||||
const newUserId = await instance.getUserId();
|
||||
expect(newUserId).toBe("custom-test-user");
|
||||
|
||||
await deleteCollectionIfExists(collectionName);
|
||||
await deleteCollectionIfExists("memory_migrations");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,427 @@
|
||||
/// <reference types="jest" />
|
||||
/**
|
||||
* End-to-end tests for Redis vector store with init guard fix.
|
||||
*
|
||||
* Requires a running Redis Stack instance at localhost:6379.
|
||||
* Skipped automatically when Redis is not available.
|
||||
*
|
||||
* Run: npx jest --config jest.config.js src/oss/tests/redis-e2e.test.ts --forceExit
|
||||
*/
|
||||
|
||||
import { createClient } from "redis";
|
||||
import { RedisDB } from "../src/vector_stores/redis";
|
||||
import { v4 as uuidv4 } from "uuid";
|
||||
jest.setTimeout(30000);
|
||||
|
||||
const REDIS_HOST = "localhost";
|
||||
const REDIS_PORT = 6379;
|
||||
const REDIS_URL = `redis://${REDIS_HOST}:${REDIS_PORT}`;
|
||||
const COLLECTION_NAME = "e2e_redis_test";
|
||||
|
||||
// Check if Redis is reachable synchronously at load time
|
||||
function isRedisAvailable(): boolean {
|
||||
try {
|
||||
const { execSync } = require("child_process");
|
||||
execSync(
|
||||
`node -e "const s=require('net').createConnection({host:'${REDIS_HOST}',port:${REDIS_PORT}});s.on('connect',()=>{s.destroy();process.exit(0)});s.on('error',()=>process.exit(1));s.setTimeout(2000,()=>process.exit(1))"`,
|
||||
{ timeout: 3000, stdio: "ignore" },
|
||||
);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
const redisAvailable = isRedisAvailable();
|
||||
if (!redisAvailable) {
|
||||
console.warn("Redis not available at localhost:6379 — skipping e2e tests");
|
||||
}
|
||||
|
||||
// Standalone client for cleanup
|
||||
let cleanupClient: ReturnType<typeof createClient>;
|
||||
|
||||
async function cleanupRedis() {
|
||||
if (!redisAvailable) return;
|
||||
try {
|
||||
// Drop the index if it exists
|
||||
await cleanupClient.ft.dropIndex(COLLECTION_NAME);
|
||||
} catch {
|
||||
// Index doesn't exist — fine
|
||||
}
|
||||
|
||||
// Delete all keys with our prefix
|
||||
const keys = await cleanupClient.keys(`mem0:${COLLECTION_NAME}:*`);
|
||||
if (keys.length > 0) {
|
||||
await cleanupClient.del(keys);
|
||||
}
|
||||
|
||||
// Clean up memory_migrations key
|
||||
await cleanupClient.del("memory_migrations:1");
|
||||
}
|
||||
|
||||
beforeAll(async () => {
|
||||
if (!redisAvailable) return;
|
||||
|
||||
cleanupClient = createClient({ url: REDIS_URL });
|
||||
await cleanupClient.connect();
|
||||
|
||||
// Verify Redis Stack is running with search module
|
||||
const modules = (await cleanupClient.moduleList()) as unknown as any[];
|
||||
const hasSearch = modules.some((mod: any[]) => {
|
||||
const moduleMap = new Map();
|
||||
for (let i = 0; i < mod.length; i += 2) {
|
||||
moduleMap.set(mod[i], mod[i + 1]);
|
||||
}
|
||||
return moduleMap.get("name")?.toLowerCase() === "search";
|
||||
});
|
||||
expect(hasSearch).toBe(true);
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
if (!redisAvailable) return;
|
||||
await cleanupRedis();
|
||||
await cleanupClient.quit();
|
||||
});
|
||||
|
||||
// Conditionally skip tests when Redis is unavailable
|
||||
const describeIfRedis = redisAvailable ? describe : describe.skip;
|
||||
|
||||
// ───────────────────────────────────────────────────────────────────────────
|
||||
// 1. Basic initialization and idempotent init guard
|
||||
// ───────────────────────────────────────────────────────────────────────────
|
||||
describeIfRedis("Redis: initialization", () => {
|
||||
afterEach(async () => {
|
||||
await cleanupRedis();
|
||||
});
|
||||
|
||||
it("initializes successfully and creates index", async () => {
|
||||
const store = new RedisDB({
|
||||
redisUrl: REDIS_URL,
|
||||
collectionName: COLLECTION_NAME,
|
||||
embeddingModelDims: 128,
|
||||
});
|
||||
await store.initialize();
|
||||
|
||||
// Verify the index was created by querying index info
|
||||
const info = await cleanupClient.ft.info(COLLECTION_NAME);
|
||||
expect(info).toBeDefined();
|
||||
expect(info.indexName).toBe(COLLECTION_NAME);
|
||||
|
||||
await store.close();
|
||||
});
|
||||
|
||||
it("idempotent initialize() — multiple calls don't crash", async () => {
|
||||
const store = new RedisDB({
|
||||
redisUrl: REDIS_URL,
|
||||
collectionName: COLLECTION_NAME,
|
||||
embeddingModelDims: 128,
|
||||
});
|
||||
|
||||
// Call initialize multiple times concurrently
|
||||
await Promise.all([
|
||||
store.initialize(),
|
||||
store.initialize(),
|
||||
store.initialize(),
|
||||
]);
|
||||
|
||||
// Should still work fine
|
||||
const info = await cleanupClient.ft.info(COLLECTION_NAME);
|
||||
expect(info).toBeDefined();
|
||||
|
||||
await store.close();
|
||||
});
|
||||
});
|
||||
|
||||
// ───────────────────────────────────────────────────────────────────────────
|
||||
// 2. Full CRUD operations
|
||||
// ───────────────────────────────────────────────────────────────────────────
|
||||
describeIfRedis("Redis: CRUD operations", () => {
|
||||
let store: RedisDB;
|
||||
|
||||
beforeEach(async () => {
|
||||
await cleanupRedis();
|
||||
store = new RedisDB({
|
||||
redisUrl: REDIS_URL,
|
||||
collectionName: COLLECTION_NAME,
|
||||
embeddingModelDims: 4, // Small dims for testing
|
||||
});
|
||||
await store.initialize();
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await store.close();
|
||||
await cleanupRedis();
|
||||
});
|
||||
|
||||
it("insert and search vectors", async () => {
|
||||
const id1 = uuidv4();
|
||||
const id2 = uuidv4();
|
||||
const vec1 = [1.0, 0.0, 0.0, 0.0];
|
||||
const vec2 = [0.0, 1.0, 0.0, 0.0];
|
||||
|
||||
await store.insert(
|
||||
[vec1, vec2],
|
||||
[id1, id2],
|
||||
[
|
||||
{
|
||||
data: "hello world",
|
||||
hash: "h1",
|
||||
userId: "user1",
|
||||
createdAt: new Date().toISOString(),
|
||||
},
|
||||
{
|
||||
data: "goodbye world",
|
||||
hash: "h2",
|
||||
userId: "user1",
|
||||
createdAt: new Date().toISOString(),
|
||||
},
|
||||
],
|
||||
);
|
||||
|
||||
// Search — vec1 should be most similar to itself
|
||||
const results = await store.search(vec1, 2, { userId: "user1" });
|
||||
expect(results.length).toBe(2);
|
||||
// The first result should be closest to the query
|
||||
expect(results[0].id).toBe(id1);
|
||||
expect(results[0].score).toBeDefined();
|
||||
expect(results[0].payload).toBeDefined();
|
||||
});
|
||||
|
||||
it("get vector by ID", async () => {
|
||||
const id = uuidv4();
|
||||
const vec = [0.5, 0.5, 0.0, 0.0];
|
||||
|
||||
await store.insert(
|
||||
[vec],
|
||||
[id],
|
||||
[
|
||||
{
|
||||
data: "test memory",
|
||||
hash: "h-test",
|
||||
userId: "user1",
|
||||
createdAt: new Date().toISOString(),
|
||||
},
|
||||
],
|
||||
);
|
||||
|
||||
const result = await store.get(id);
|
||||
expect(result).not.toBeNull();
|
||||
expect(result!.id).toBe(id);
|
||||
expect(result!.payload.data).toBe("test memory");
|
||||
expect(result!.payload.hash).toBe("h-test");
|
||||
});
|
||||
|
||||
it("get non-existent vector returns null", async () => {
|
||||
const result = await store.get("non-existent-id");
|
||||
expect(result).toBeNull();
|
||||
});
|
||||
|
||||
it("update vector and payload", async () => {
|
||||
const id = uuidv4();
|
||||
const vec = [1.0, 0.0, 0.0, 0.0];
|
||||
|
||||
await store.insert(
|
||||
[vec],
|
||||
[id],
|
||||
[
|
||||
{
|
||||
data: "original",
|
||||
hash: "h-orig",
|
||||
userId: "user1",
|
||||
createdAt: new Date().toISOString(),
|
||||
},
|
||||
],
|
||||
);
|
||||
|
||||
// Update with new vector and payload
|
||||
const newVec = [0.0, 0.0, 1.0, 0.0];
|
||||
await store.update(id, newVec, {
|
||||
data: "updated memory",
|
||||
hash: "h-updated",
|
||||
userId: "user1",
|
||||
createdAt: new Date().toISOString(),
|
||||
updatedAt: new Date().toISOString(),
|
||||
});
|
||||
|
||||
const result = await store.get(id);
|
||||
expect(result).not.toBeNull();
|
||||
expect(result!.payload.data).toBe("updated memory");
|
||||
expect(result!.payload.hash).toBe("h-updated");
|
||||
});
|
||||
|
||||
it("delete vector", async () => {
|
||||
const id = uuidv4();
|
||||
const vec = [0.0, 0.0, 0.0, 1.0];
|
||||
|
||||
await store.insert(
|
||||
[vec],
|
||||
[id],
|
||||
[
|
||||
{
|
||||
data: "to be deleted",
|
||||
hash: "h-del",
|
||||
userId: "user1",
|
||||
createdAt: new Date().toISOString(),
|
||||
},
|
||||
],
|
||||
);
|
||||
|
||||
// Verify it exists
|
||||
const before = await store.get(id);
|
||||
expect(before).not.toBeNull();
|
||||
|
||||
// Delete
|
||||
await store.delete(id);
|
||||
|
||||
// Verify it's gone
|
||||
const after = await store.get(id);
|
||||
expect(after).toBeNull();
|
||||
});
|
||||
|
||||
it("list vectors with filters", async () => {
|
||||
const id1 = uuidv4();
|
||||
const id2 = uuidv4();
|
||||
const id3 = uuidv4();
|
||||
|
||||
await store.insert(
|
||||
[
|
||||
[1, 0, 0, 0],
|
||||
[0, 1, 0, 0],
|
||||
[0, 0, 1, 0],
|
||||
],
|
||||
[id1, id2, id3],
|
||||
[
|
||||
{
|
||||
data: "mem1",
|
||||
hash: "h1",
|
||||
userId: "usera",
|
||||
createdAt: new Date().toISOString(),
|
||||
},
|
||||
{
|
||||
data: "mem2",
|
||||
hash: "h2",
|
||||
userId: "usera",
|
||||
createdAt: new Date().toISOString(),
|
||||
},
|
||||
{
|
||||
data: "mem3",
|
||||
hash: "h3",
|
||||
userId: "userb",
|
||||
createdAt: new Date().toISOString(),
|
||||
},
|
||||
],
|
||||
);
|
||||
|
||||
// List all
|
||||
const [all, allCount] = await store.list();
|
||||
expect(allCount).toBe(3);
|
||||
expect(all.length).toBe(3);
|
||||
|
||||
// List with filter
|
||||
const [filtered, filteredCount] = await store.list({
|
||||
userId: "usera",
|
||||
});
|
||||
expect(filteredCount).toBe(2);
|
||||
expect(filtered.length).toBe(2);
|
||||
});
|
||||
});
|
||||
|
||||
// ───────────────────────────────────────────────────────────────────────────
|
||||
// 3. getUserId / setUserId
|
||||
// ───────────────────────────────────────────────────────────────────────────
|
||||
describeIfRedis("Redis: getUserId / setUserId", () => {
|
||||
let store: RedisDB;
|
||||
|
||||
beforeEach(async () => {
|
||||
await cleanupRedis();
|
||||
store = new RedisDB({
|
||||
redisUrl: REDIS_URL,
|
||||
collectionName: COLLECTION_NAME,
|
||||
embeddingModelDims: 4,
|
||||
});
|
||||
await store.initialize();
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await store.close();
|
||||
await cleanupRedis();
|
||||
});
|
||||
|
||||
it("getUserId generates random ID if none exists", async () => {
|
||||
const userId = await store.getUserId();
|
||||
expect(typeof userId).toBe("string");
|
||||
expect(userId.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("setUserId + getUserId roundtrip", async () => {
|
||||
await store.setUserId("custom-redis-user");
|
||||
const retrieved = await store.getUserId();
|
||||
expect(retrieved).toBe("custom-redis-user");
|
||||
});
|
||||
|
||||
it("getUserId returns same value on subsequent calls", async () => {
|
||||
const first = await store.getUserId();
|
||||
const second = await store.getUserId();
|
||||
expect(first).toBe(second);
|
||||
});
|
||||
});
|
||||
|
||||
// ───────────────────────────────────────────────────────────────────────────
|
||||
// 4. Dimension handling (our fix ensures correct dims from Memory)
|
||||
// ───────────────────────────────────────────────────────────────────────────
|
||||
describeIfRedis("Redis: dimension handling", () => {
|
||||
afterEach(async () => {
|
||||
await cleanupRedis();
|
||||
});
|
||||
|
||||
it("creates index with correct dimensions from config", async () => {
|
||||
const store = new RedisDB({
|
||||
redisUrl: REDIS_URL,
|
||||
collectionName: COLLECTION_NAME,
|
||||
embeddingModelDims: 768,
|
||||
});
|
||||
await store.initialize();
|
||||
|
||||
// Verify the index has the right dimension in its schema
|
||||
const info = await cleanupClient.ft.info(COLLECTION_NAME);
|
||||
// Check that the vector field has DIM=768
|
||||
const attributes = info.attributes as any[];
|
||||
const vectorAttr = attributes.find(
|
||||
(a: any) => a.identifier === "embedding" || a.attribute === "embedding",
|
||||
);
|
||||
expect(vectorAttr).toBeDefined();
|
||||
|
||||
await store.close();
|
||||
});
|
||||
|
||||
it("insert with matching dimension succeeds", async () => {
|
||||
const dims = 128;
|
||||
const store = new RedisDB({
|
||||
redisUrl: REDIS_URL,
|
||||
collectionName: COLLECTION_NAME,
|
||||
embeddingModelDims: dims,
|
||||
});
|
||||
await store.initialize();
|
||||
|
||||
const id = uuidv4();
|
||||
const vec = new Array(dims).fill(0.1);
|
||||
|
||||
await store.insert(
|
||||
[vec],
|
||||
[id],
|
||||
[
|
||||
{
|
||||
data: "test",
|
||||
hash: "h1",
|
||||
createdAt: new Date().toISOString(),
|
||||
},
|
||||
],
|
||||
);
|
||||
|
||||
const result = await store.get(id);
|
||||
expect(result).not.toBeNull();
|
||||
expect(result!.id).toBe(id);
|
||||
|
||||
await store.close();
|
||||
});
|
||||
});
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user