fix(oss): auto-detect embedding dimension to fix Qdrant mismatch with non-OpenAI embedders (#4297)
Co-authored-by: utkarsh240799 <utkarsh240799@users.noreply.github.com> Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
co-authored by
utkarsh240799
Claude Opus 4.6
parent
5b3acf416b
commit
59c3b050bd
@@ -43,13 +43,23 @@ export class ConfigManager {
|
|||||||
const defaultConf = DEFAULT_MEMORY_CONFIG.vectorStore.config;
|
const defaultConf = DEFAULT_MEMORY_CONFIG.vectorStore.config;
|
||||||
const userConf = userConfig.vectorStore?.config;
|
const userConf = userConfig.vectorStore?.config;
|
||||||
|
|
||||||
|
// Resolve the vector store dimension. If the user explicitly
|
||||||
|
// provided one, use it. Otherwise leave it undefined so that
|
||||||
|
// Memory._autoInitialize() can auto-detect it by running a
|
||||||
|
// probe embedding at startup — this makes *any* embedder work
|
||||||
|
// out of the box without the user needing to know or set the
|
||||||
|
// dimension manually.
|
||||||
|
const explicitDimension =
|
||||||
|
userConf?.dimension ||
|
||||||
|
userConfig.embedder?.config?.embeddingDims ||
|
||||||
|
undefined;
|
||||||
|
|
||||||
// Prioritize user-provided client instance
|
// Prioritize user-provided client instance
|
||||||
if (userConf?.client && typeof userConf.client === "object") {
|
if (userConf?.client && typeof userConf.client === "object") {
|
||||||
return {
|
return {
|
||||||
client: userConf.client,
|
client: userConf.client,
|
||||||
// Include other fields from userConf if necessary, or omit defaults
|
collectionName: userConf.collectionName,
|
||||||
collectionName: userConf.collectionName, // Can be undefined
|
dimension: explicitDimension,
|
||||||
dimension: userConf.dimension || defaultConf.dimension, // Merge dimension
|
|
||||||
...userConf, // Include any other passthrough fields from user
|
...userConf, // Include any other passthrough fields from user
|
||||||
};
|
};
|
||||||
} else {
|
} else {
|
||||||
@@ -57,7 +67,7 @@ export class ConfigManager {
|
|||||||
return {
|
return {
|
||||||
collectionName:
|
collectionName:
|
||||||
userConf?.collectionName || defaultConf.collectionName,
|
userConf?.collectionName || defaultConf.collectionName,
|
||||||
dimension: userConf?.dimension || defaultConf.dimension,
|
dimension: explicitDimension,
|
||||||
// Ensure client is not carried over from defaults if not provided by user
|
// Ensure client is not carried over from defaults if not provided by user
|
||||||
client: undefined,
|
client: undefined,
|
||||||
// Include other passthrough fields from userConf even if no client
|
// Include other passthrough fields from userConf even if no client
|
||||||
|
|||||||
@@ -28,7 +28,7 @@ export class GoogleEmbedder implements Embedder {
|
|||||||
const response = await this.google.models.embedContent({
|
const response = await this.google.models.embedContent({
|
||||||
model: this.model,
|
model: this.model,
|
||||||
contents: texts,
|
contents: texts,
|
||||||
config: { outputDimensionality: 768 },
|
config: { outputDimensionality: this.embeddingDims },
|
||||||
});
|
});
|
||||||
return response.embeddings!.map((item) => item.values!);
|
return response.embeddings!.map((item) => item.values!);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -41,7 +41,7 @@ export class Memory {
|
|||||||
private config: MemoryConfig;
|
private config: MemoryConfig;
|
||||||
private customPrompt: string | undefined;
|
private customPrompt: string | undefined;
|
||||||
private embedder: Embedder;
|
private embedder: Embedder;
|
||||||
private vectorStore: VectorStore;
|
private vectorStore!: VectorStore;
|
||||||
private llm: LLM;
|
private llm: LLM;
|
||||||
private db: HistoryManager;
|
private db: HistoryManager;
|
||||||
private collectionName: string | undefined;
|
private collectionName: string | undefined;
|
||||||
@@ -49,6 +49,8 @@ export class Memory {
|
|||||||
private graphMemory?: MemoryGraph;
|
private graphMemory?: MemoryGraph;
|
||||||
private enableGraph: boolean;
|
private enableGraph: boolean;
|
||||||
telemetryId: string;
|
telemetryId: string;
|
||||||
|
private _initPromise: Promise<void>;
|
||||||
|
private _initError?: Error;
|
||||||
|
|
||||||
constructor(config: Partial<MemoryConfig> = {}) {
|
constructor(config: Partial<MemoryConfig> = {}) {
|
||||||
// Merge and validate config
|
// Merge and validate config
|
||||||
@@ -59,10 +61,9 @@ export class Memory {
|
|||||||
this.config.embedder.provider,
|
this.config.embedder.provider,
|
||||||
this.config.embedder.config,
|
this.config.embedder.config,
|
||||||
);
|
);
|
||||||
this.vectorStore = VectorStoreFactory.create(
|
// Vector store creation is deferred to _autoInitialize() so that
|
||||||
this.config.vectorStore.provider,
|
// the embedding dimension can be auto-detected first when not
|
||||||
this.config.vectorStore.config,
|
// explicitly configured.
|
||||||
);
|
|
||||||
this.llm = LLMFactory.create(
|
this.llm = LLMFactory.create(
|
||||||
this.config.llm.provider,
|
this.config.llm.provider,
|
||||||
this.config.llm.config,
|
this.config.llm.config,
|
||||||
@@ -86,8 +87,67 @@ export class Memory {
|
|||||||
this.graphMemory = new MemoryGraph(this.config);
|
this.graphMemory = new MemoryGraph(this.config);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Initialize telemetry if vector store is initialized
|
// Auto-detect embedding dimension (if needed), create vector store,
|
||||||
this._initializeTelemetry();
|
// and initialize it. All public methods await this before proceeding.
|
||||||
|
this._initPromise = this._autoInitialize().catch((error) => {
|
||||||
|
this._initError =
|
||||||
|
error instanceof Error ? error : new Error(String(error));
|
||||||
|
console.error(this._initError);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* If no explicit dimension was provided, runs a probe embedding to
|
||||||
|
* detect it. Then creates and initializes the vector store.
|
||||||
|
*/
|
||||||
|
private async _autoInitialize(): Promise<void> {
|
||||||
|
if (!this.config.vectorStore.config.dimension) {
|
||||||
|
try {
|
||||||
|
const probe = await this.embedder.embed("dimension probe");
|
||||||
|
this.config.vectorStore.config.dimension = probe.length;
|
||||||
|
} catch (error: any) {
|
||||||
|
throw new Error(
|
||||||
|
`Failed to auto-detect embedding dimension from provider '${this.config.embedder.provider}': ${error.message}. ` +
|
||||||
|
`Please set 'dimension' in vectorStore.config or 'embeddingDims' in embedder.config explicitly.`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
this.vectorStore = VectorStoreFactory.create(
|
||||||
|
this.config.vectorStore.provider,
|
||||||
|
this.config.vectorStore.config,
|
||||||
|
);
|
||||||
|
|
||||||
|
// The vector store constructor may fire initialize() asynchronously
|
||||||
|
// (e.g. Qdrant). Explicitly await it here to guarantee the backing
|
||||||
|
// store (collections, tables, etc.) is ready before any public method
|
||||||
|
// attempts to read or write.
|
||||||
|
await this.vectorStore.initialize();
|
||||||
|
|
||||||
|
await this._initializeTelemetry();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Ensures that auto-initialization (dimension detection + vector store
|
||||||
|
* creation) has completed before any public method proceeds.
|
||||||
|
* If a previous init attempt failed, retries automatically.
|
||||||
|
*/
|
||||||
|
private async _ensureInitialized(): Promise<void> {
|
||||||
|
await this._initPromise;
|
||||||
|
if (this._initError) {
|
||||||
|
// Clear failed state and retry — the embedder or vector store
|
||||||
|
// may have been transiently unavailable at startup.
|
||||||
|
this._initError = undefined;
|
||||||
|
this._initPromise = this._autoInitialize().catch((error) => {
|
||||||
|
this._initError =
|
||||||
|
error instanceof Error ? error : new Error(String(error));
|
||||||
|
console.error(this._initError);
|
||||||
|
});
|
||||||
|
await this._initPromise;
|
||||||
|
if (this._initError) {
|
||||||
|
throw this._initError;
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private async _initializeTelemetry() {
|
private async _initializeTelemetry() {
|
||||||
@@ -147,6 +207,7 @@ export class Memory {
|
|||||||
messages: string | Message[],
|
messages: string | Message[],
|
||||||
config: AddMemoryOptions,
|
config: AddMemoryOptions,
|
||||||
): Promise<SearchResult> {
|
): Promise<SearchResult> {
|
||||||
|
await this._ensureInitialized();
|
||||||
await this._captureEvent("add", {
|
await this._captureEvent("add", {
|
||||||
message_count: Array.isArray(messages) ? messages.length : 1,
|
message_count: Array.isArray(messages) ? messages.length : 1,
|
||||||
has_metadata: !!config.metadata,
|
has_metadata: !!config.metadata,
|
||||||
@@ -372,6 +433,7 @@ export class Memory {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async get(memoryId: string): Promise<MemoryItem | null> {
|
async get(memoryId: string): Promise<MemoryItem | null> {
|
||||||
|
await this._ensureInitialized();
|
||||||
const memory = await this.vectorStore.get(memoryId);
|
const memory = await this.vectorStore.get(memoryId);
|
||||||
if (!memory) return null;
|
if (!memory) return null;
|
||||||
|
|
||||||
@@ -413,6 +475,7 @@ export class Memory {
|
|||||||
query: string,
|
query: string,
|
||||||
config: SearchMemoryOptions,
|
config: SearchMemoryOptions,
|
||||||
): Promise<SearchResult> {
|
): Promise<SearchResult> {
|
||||||
|
await this._ensureInitialized();
|
||||||
await this._captureEvent("search", {
|
await this._captureEvent("search", {
|
||||||
query_length: query.length,
|
query_length: query.length,
|
||||||
limit: config.limit,
|
limit: config.limit,
|
||||||
@@ -479,6 +542,7 @@ export class Memory {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async update(memoryId: string, data: string): Promise<{ message: string }> {
|
async update(memoryId: string, data: string): Promise<{ message: string }> {
|
||||||
|
await this._ensureInitialized();
|
||||||
await this._captureEvent("update", { memory_id: memoryId });
|
await this._captureEvent("update", { memory_id: memoryId });
|
||||||
const embedding = await this.embedder.embed(data);
|
const embedding = await this.embedder.embed(data);
|
||||||
await this.updateMemory(memoryId, data, { [data]: embedding });
|
await this.updateMemory(memoryId, data, { [data]: embedding });
|
||||||
@@ -486,6 +550,7 @@ export class Memory {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async delete(memoryId: string): Promise<{ message: string }> {
|
async delete(memoryId: string): Promise<{ message: string }> {
|
||||||
|
await this._ensureInitialized();
|
||||||
await this._captureEvent("delete", { memory_id: memoryId });
|
await this._captureEvent("delete", { memory_id: memoryId });
|
||||||
await this.deleteMemory(memoryId);
|
await this.deleteMemory(memoryId);
|
||||||
return { message: "Memory deleted successfully!" };
|
return { message: "Memory deleted successfully!" };
|
||||||
@@ -494,6 +559,7 @@ export class Memory {
|
|||||||
async deleteAll(
|
async deleteAll(
|
||||||
config: DeleteAllMemoryOptions,
|
config: DeleteAllMemoryOptions,
|
||||||
): Promise<{ message: string }> {
|
): Promise<{ message: string }> {
|
||||||
|
await this._ensureInitialized();
|
||||||
await this._captureEvent("delete_all", {
|
await this._captureEvent("delete_all", {
|
||||||
has_user_id: !!config.userId,
|
has_user_id: !!config.userId,
|
||||||
has_agent_id: !!config.agentId,
|
has_agent_id: !!config.agentId,
|
||||||
@@ -521,10 +587,12 @@ export class Memory {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async history(memoryId: string): Promise<any[]> {
|
async history(memoryId: string): Promise<any[]> {
|
||||||
|
await this._ensureInitialized();
|
||||||
return this.db.getHistory(memoryId);
|
return this.db.getHistory(memoryId);
|
||||||
}
|
}
|
||||||
|
|
||||||
async reset(): Promise<void> {
|
async reset(): Promise<void> {
|
||||||
|
await this._ensureInitialized();
|
||||||
await this._captureEvent("reset");
|
await this._captureEvent("reset");
|
||||||
await this.db.reset();
|
await this.db.reset();
|
||||||
|
|
||||||
@@ -549,28 +617,30 @@ export class Memory {
|
|||||||
await this.graphMemory.deleteAll({ userId: "default" }); // Assuming this is okay, or needs similar check?
|
await this.graphMemory.deleteAll({ userId: "default" }); // Assuming this is okay, or needs similar check?
|
||||||
}
|
}
|
||||||
|
|
||||||
// Re-initialize factories/clients based on the original config
|
// Re-initialize factories/clients based on the original config.
|
||||||
|
// Dimension is already set in this.config from the initial probe,
|
||||||
|
// so _autoInitialize will skip the probe and just re-create the store.
|
||||||
this.embedder = EmbedderFactory.create(
|
this.embedder = EmbedderFactory.create(
|
||||||
this.config.embedder.provider,
|
this.config.embedder.provider,
|
||||||
this.config.embedder.config,
|
this.config.embedder.config,
|
||||||
);
|
);
|
||||||
// Re-create vector store instance - crucial for Langchain to reset wrapper state if needed
|
|
||||||
this.vectorStore = VectorStoreFactory.create(
|
|
||||||
this.config.vectorStore.provider,
|
|
||||||
this.config.vectorStore.config, // This will pass the original client instance back
|
|
||||||
);
|
|
||||||
this.llm = LLMFactory.create(
|
this.llm = LLMFactory.create(
|
||||||
this.config.llm.provider,
|
this.config.llm.provider,
|
||||||
this.config.llm.config,
|
this.config.llm.config,
|
||||||
);
|
);
|
||||||
// Re-init DB if needed (though db.reset() likely handles its state)
|
|
||||||
// Re-init Graph if needed
|
|
||||||
|
|
||||||
// Re-initialize telemetry
|
// Re-create vector store via _autoInitialize (which handles dimension + creation)
|
||||||
this._initializeTelemetry();
|
this._initError = undefined;
|
||||||
|
this._initPromise = this._autoInitialize().catch((error) => {
|
||||||
|
this._initError =
|
||||||
|
error instanceof Error ? error : new Error(String(error));
|
||||||
|
console.error(this._initError);
|
||||||
|
});
|
||||||
|
await this._initPromise;
|
||||||
}
|
}
|
||||||
|
|
||||||
async getAll(config: GetAllMemoryOptions): Promise<SearchResult> {
|
async getAll(config: GetAllMemoryOptions): Promise<SearchResult> {
|
||||||
|
await this._ensureInitialized();
|
||||||
await this._captureEvent("get_all", {
|
await this._captureEvent("get_all", {
|
||||||
limit: config.limit,
|
limit: config.limit,
|
||||||
has_user_id: !!config.userId,
|
has_user_id: !!config.userId,
|
||||||
|
|||||||
@@ -81,6 +81,7 @@ export class AzureAISearch implements VectorStore {
|
|||||||
private readonly hybridSearch: boolean;
|
private readonly hybridSearch: boolean;
|
||||||
private readonly vectorFilterMode: string;
|
private readonly vectorFilterMode: string;
|
||||||
private readonly apiKey: string | undefined;
|
private readonly apiKey: string | undefined;
|
||||||
|
private _initPromise?: Promise<void>;
|
||||||
|
|
||||||
constructor(config: AzureAISearchConfig) {
|
constructor(config: AzureAISearchConfig) {
|
||||||
this.serviceName = config.serviceName;
|
this.serviceName = config.serviceName;
|
||||||
@@ -117,6 +118,13 @@ export class AzureAISearch implements VectorStore {
|
|||||||
* Initialize the Azure AI Search index if it doesn't exist
|
* Initialize the Azure AI Search index if it doesn't exist
|
||||||
*/
|
*/
|
||||||
async initialize(): Promise<void> {
|
async initialize(): Promise<void> {
|
||||||
|
if (!this._initPromise) {
|
||||||
|
this._initPromise = this._doInitialize();
|
||||||
|
}
|
||||||
|
return this._initPromise;
|
||||||
|
}
|
||||||
|
|
||||||
|
private async _doInitialize(): Promise<void> {
|
||||||
try {
|
try {
|
||||||
const collections = await this.listCols();
|
const collections = await this.listCols();
|
||||||
if (!collections.includes(this.indexName)) {
|
if (!collections.includes(this.indexName)) {
|
||||||
|
|||||||
@@ -32,6 +32,7 @@ export class Qdrant implements VectorStore {
|
|||||||
private client: QdrantClient;
|
private client: QdrantClient;
|
||||||
private readonly collectionName: string;
|
private readonly collectionName: string;
|
||||||
private dimension: number;
|
private dimension: number;
|
||||||
|
private _initPromise?: Promise<void>;
|
||||||
|
|
||||||
constructor(config: QdrantConfig) {
|
constructor(config: QdrantConfig) {
|
||||||
if (config.client) {
|
if (config.client) {
|
||||||
@@ -211,22 +212,8 @@ export class Qdrant implements VectorStore {
|
|||||||
|
|
||||||
async getUserId(): Promise<string> {
|
async getUserId(): Promise<string> {
|
||||||
try {
|
try {
|
||||||
// First check if the collection exists
|
// Ensure collection exists (idempotent — handles race conditions)
|
||||||
const collections = await this.client.getCollections();
|
await this.ensureCollection("memory_migrations", 1);
|
||||||
const userCollectionExists = collections.collections.some(
|
|
||||||
(col: { name: string }) => col.name === "memory_migrations",
|
|
||||||
);
|
|
||||||
|
|
||||||
if (!userCollectionExists) {
|
|
||||||
// Create the collection if it doesn't exist
|
|
||||||
await this.client.createCollection("memory_migrations", {
|
|
||||||
vectors: {
|
|
||||||
size: 1,
|
|
||||||
distance: "Cosine",
|
|
||||||
on_disk: false,
|
|
||||||
},
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
// Now try to get the user ID
|
// Now try to get the user ID
|
||||||
const result = await this.client.scroll("memory_migrations", {
|
const result = await this.client.scroll("memory_migrations", {
|
||||||
@@ -286,66 +273,61 @@ export class Qdrant implements VectorStore {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async initialize(): Promise<void> {
|
private async ensureCollection(
|
||||||
|
name: string,
|
||||||
|
size: number,
|
||||||
|
): Promise<void> {
|
||||||
try {
|
try {
|
||||||
// Create collection if it doesn't exist
|
await this.client.createCollection(name, {
|
||||||
const collections = await this.client.getCollections();
|
vectors: {
|
||||||
const exists = collections.collections.some(
|
size,
|
||||||
(c) => c.name === this.collectionName,
|
distance: "Cosine",
|
||||||
);
|
},
|
||||||
|
});
|
||||||
if (!exists) {
|
} catch (error: any) {
|
||||||
try {
|
if (error?.status === 409) {
|
||||||
await this.client.createCollection(this.collectionName, {
|
// Collection already exists — verify configuration for the main collection
|
||||||
vectors: {
|
if (name === this.collectionName) {
|
||||||
size: this.dimension,
|
try {
|
||||||
distance: "Cosine",
|
const collectionInfo = await this.client.getCollection(name);
|
||||||
},
|
|
||||||
});
|
|
||||||
} catch (error: any) {
|
|
||||||
// Handle case where collection was created between our check and create
|
|
||||||
if (error?.status === 409) {
|
|
||||||
// Collection already exists - verify it has the correct configuration
|
|
||||||
const collectionInfo = await this.client.getCollection(
|
|
||||||
this.collectionName,
|
|
||||||
);
|
|
||||||
const vectorConfig = collectionInfo.config?.params?.vectors;
|
const vectorConfig = collectionInfo.config?.params?.vectors;
|
||||||
|
|
||||||
if (!vectorConfig || vectorConfig.size !== this.dimension) {
|
if (vectorConfig && vectorConfig.size !== size) {
|
||||||
throw new Error(
|
throw new Error(
|
||||||
`Collection ${this.collectionName} exists but has wrong configuration. ` +
|
`Collection ${name} exists but has wrong vector size. ` +
|
||||||
`Expected vector size: ${this.dimension}, got: ${vectorConfig?.size}`,
|
`Expected: ${size}, got: ${vectorConfig.size}`,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
// Collection exists with correct configuration - we can proceed
|
} catch (verifyError: any) {
|
||||||
} else {
|
// Re-throw dimension mismatch errors
|
||||||
throw error;
|
if (verifyError?.message?.includes("wrong vector size")) {
|
||||||
|
throw verifyError;
|
||||||
|
}
|
||||||
|
// Transient errors (e.g. 500 while collection is being committed)
|
||||||
|
// are non-fatal — the collection exists per the 409.
|
||||||
|
console.warn(
|
||||||
|
`Collection '${name}' exists (409) but dimension verification failed: ${verifyError?.message || verifyError}. Proceeding anyway.`,
|
||||||
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
// Otherwise collection exists and is fine — proceed
|
||||||
|
} else {
|
||||||
|
throw error;
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Create memory_migrations collection if it doesn't exist
|
async initialize(): Promise<void> {
|
||||||
const userExists = collections.collections.some(
|
if (!this._initPromise) {
|
||||||
(c) => c.name === "memory_migrations",
|
this._initPromise = this._doInitialize();
|
||||||
);
|
}
|
||||||
|
return this._initPromise;
|
||||||
|
}
|
||||||
|
|
||||||
if (!userExists) {
|
private async _doInitialize(): Promise<void> {
|
||||||
try {
|
try {
|
||||||
await this.client.createCollection("memory_migrations", {
|
await this.ensureCollection(this.collectionName, this.dimension);
|
||||||
vectors: {
|
await this.ensureCollection("memory_migrations", 1);
|
||||||
size: 1, // Minimal size since we only store user_id
|
|
||||||
distance: "Cosine",
|
|
||||||
},
|
|
||||||
});
|
|
||||||
} catch (error: any) {
|
|
||||||
// Handle case where collection was created between our check and create
|
|
||||||
if (error?.status === 409) {
|
|
||||||
// Collection already exists - we can proceed
|
|
||||||
} else {
|
|
||||||
throw error;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error("Error initializing Qdrant:", error);
|
console.error("Error initializing Qdrant:", error);
|
||||||
throw error;
|
throw error;
|
||||||
|
|||||||
@@ -139,6 +139,7 @@ export class RedisDB implements VectorStore {
|
|||||||
private readonly indexName: string;
|
private readonly indexName: string;
|
||||||
private readonly indexPrefix: string;
|
private readonly indexPrefix: string;
|
||||||
private readonly schema: RedisSchema;
|
private readonly schema: RedisSchema;
|
||||||
|
private _initPromise?: Promise<void>;
|
||||||
|
|
||||||
constructor(config: RedisConfig) {
|
constructor(config: RedisConfig) {
|
||||||
this.indexName = config.collectionName;
|
this.indexName = config.collectionName;
|
||||||
@@ -240,6 +241,13 @@ export class RedisDB implements VectorStore {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async initialize(): Promise<void> {
|
async initialize(): Promise<void> {
|
||||||
|
if (!this._initPromise) {
|
||||||
|
this._initPromise = this._doInitialize();
|
||||||
|
}
|
||||||
|
return this._initPromise;
|
||||||
|
}
|
||||||
|
|
||||||
|
private async _doInitialize(): Promise<void> {
|
||||||
try {
|
try {
|
||||||
await this.client.connect();
|
await this.client.connect();
|
||||||
console.log("Connected to Redis");
|
console.log("Connected to Redis");
|
||||||
|
|||||||
@@ -86,6 +86,7 @@ export class SupabaseDB implements VectorStore {
|
|||||||
private readonly tableName: string;
|
private readonly tableName: string;
|
||||||
private readonly embeddingColumnName: string;
|
private readonly embeddingColumnName: string;
|
||||||
private readonly metadataColumnName: string;
|
private readonly metadataColumnName: string;
|
||||||
|
private _initPromise?: Promise<void>;
|
||||||
|
|
||||||
constructor(config: SupabaseConfig) {
|
constructor(config: SupabaseConfig) {
|
||||||
this.client = createClient(config.supabaseUrl, config.supabaseKey);
|
this.client = createClient(config.supabaseUrl, config.supabaseKey);
|
||||||
@@ -100,6 +101,13 @@ export class SupabaseDB implements VectorStore {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async initialize(): Promise<void> {
|
async initialize(): Promise<void> {
|
||||||
|
if (!this._initPromise) {
|
||||||
|
this._initPromise = this._doInitialize();
|
||||||
|
}
|
||||||
|
return this._initPromise;
|
||||||
|
}
|
||||||
|
|
||||||
|
private async _doInitialize(): Promise<void> {
|
||||||
try {
|
try {
|
||||||
// Verify table exists and vector operations work by attempting a test insert
|
// Verify table exists and vector operations work by attempting a test insert
|
||||||
const testVector = Array(1536).fill(0);
|
const testVector = Array(1536).fill(0);
|
||||||
|
|||||||
@@ -20,6 +20,7 @@ export class VectorizeDB implements VectorStore {
|
|||||||
private dimensions: number;
|
private dimensions: number;
|
||||||
private indexName: string;
|
private indexName: string;
|
||||||
private accountId: string;
|
private accountId: string;
|
||||||
|
private _initPromise?: Promise<void>;
|
||||||
|
|
||||||
constructor(config: VectorizeConfig) {
|
constructor(config: VectorizeConfig) {
|
||||||
this.client = new Cloudflare({ apiToken: config.apiKey });
|
this.client = new Cloudflare({ apiToken: config.apiKey });
|
||||||
@@ -343,6 +344,13 @@ export class VectorizeDB implements VectorStore {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async initialize(): Promise<void> {
|
async initialize(): Promise<void> {
|
||||||
|
if (!this._initPromise) {
|
||||||
|
this._initPromise = this._doInitialize();
|
||||||
|
}
|
||||||
|
return this._initPromise;
|
||||||
|
}
|
||||||
|
|
||||||
|
private async _doInitialize(): Promise<void> {
|
||||||
try {
|
try {
|
||||||
// Check if the index already exists
|
// Check if the index already exists
|
||||||
let indexFound = false;
|
let indexFound = false;
|
||||||
|
|||||||
@@ -0,0 +1,88 @@
|
|||||||
|
/// <reference types="jest" />
|
||||||
|
import { ConfigManager } from "../src/config/manager";
|
||||||
|
|
||||||
|
describe("ConfigManager", () => {
|
||||||
|
describe("mergeConfig - dimension handling", () => {
|
||||||
|
const baseLlm = {
|
||||||
|
provider: "openai",
|
||||||
|
config: { apiKey: "test-key" },
|
||||||
|
};
|
||||||
|
|
||||||
|
it("should leave dimension undefined when no explicit dimension or embeddingDims provided", () => {
|
||||||
|
const config = ConfigManager.mergeConfig({
|
||||||
|
embedder: { provider: "openai", config: { apiKey: "test-key" } },
|
||||||
|
vectorStore: { provider: "memory", config: { collectionName: "test" } },
|
||||||
|
llm: baseLlm,
|
||||||
|
});
|
||||||
|
|
||||||
|
// Dimension should be undefined so Memory._autoInitialize() will
|
||||||
|
// auto-detect it via a probe embedding at runtime.
|
||||||
|
expect(config.vectorStore.config.dimension).toBeUndefined();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("should use embeddingDims from embedder config when provided", () => {
|
||||||
|
const config = ConfigManager.mergeConfig({
|
||||||
|
embedder: {
|
||||||
|
provider: "ollama",
|
||||||
|
config: { model: "nomic-embed-text", embeddingDims: 768 },
|
||||||
|
},
|
||||||
|
vectorStore: { provider: "qdrant", config: { collectionName: "test" } },
|
||||||
|
llm: baseLlm,
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(config.vectorStore.config.dimension).toBe(768);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("should prefer explicit vector store dimension over embedder dims", () => {
|
||||||
|
const config = ConfigManager.mergeConfig({
|
||||||
|
embedder: {
|
||||||
|
provider: "ollama",
|
||||||
|
config: { model: "nomic-embed-text", embeddingDims: 768 },
|
||||||
|
},
|
||||||
|
vectorStore: {
|
||||||
|
provider: "qdrant",
|
||||||
|
config: { collectionName: "test", dimension: 1024 },
|
||||||
|
},
|
||||||
|
llm: baseLlm,
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(config.vectorStore.config.dimension).toBe(1024);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("should leave dimension undefined when using a custom client without explicit dims", () => {
|
||||||
|
const mockClient = { someMethod: () => {} };
|
||||||
|
const config = ConfigManager.mergeConfig({
|
||||||
|
embedder: {
|
||||||
|
provider: "ollama",
|
||||||
|
config: { model: "nomic-embed-text" },
|
||||||
|
},
|
||||||
|
vectorStore: {
|
||||||
|
provider: "qdrant",
|
||||||
|
config: { collectionName: "test", client: mockClient },
|
||||||
|
},
|
||||||
|
llm: baseLlm,
|
||||||
|
});
|
||||||
|
|
||||||
|
// No embeddingDims and no explicit dimension → should be undefined
|
||||||
|
// for auto-detection at runtime.
|
||||||
|
expect(config.vectorStore.config.dimension).toBeUndefined();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("should use embeddingDims when using a custom client", () => {
|
||||||
|
const mockClient = { someMethod: () => {} };
|
||||||
|
const config = ConfigManager.mergeConfig({
|
||||||
|
embedder: {
|
||||||
|
provider: "ollama",
|
||||||
|
config: { model: "nomic-embed-text", embeddingDims: 768 },
|
||||||
|
},
|
||||||
|
vectorStore: {
|
||||||
|
provider: "qdrant",
|
||||||
|
config: { collectionName: "test", client: mockClient },
|
||||||
|
},
|
||||||
|
llm: baseLlm,
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(config.vectorStore.config.dimension).toBe(768);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,519 @@
|
|||||||
|
/// <reference types="jest" />
|
||||||
|
/**
|
||||||
|
* Tests for embedding dimension auto-detection.
|
||||||
|
*
|
||||||
|
* Covers:
|
||||||
|
* - ConfigManager: dimension resolution logic
|
||||||
|
* - Memory class: probe-based auto-detection, lazy init gate, backward compat
|
||||||
|
* - MemoryVectorStore: backward compat with explicit dimensions
|
||||||
|
* - Explicit error messages on probe failure
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { ConfigManager } from "../src/config/manager";
|
||||||
|
import { MemoryVectorStore } from "../src/vector_stores/memory";
|
||||||
|
import * as fs from "fs";
|
||||||
|
import * as path from "path";
|
||||||
|
import * as os from "os";
|
||||||
|
|
||||||
|
jest.setTimeout(15000);
|
||||||
|
|
||||||
|
// ───────────────────────────────────────────────────────────────────────────
|
||||||
|
// 1. ConfigManager – dimension resolution
|
||||||
|
// ───────────────────────────────────────────────────────────────────────────
|
||||||
|
describe("ConfigManager – dimension resolution", () => {
|
||||||
|
const baseLlm = { provider: "openai", config: { apiKey: "k" } };
|
||||||
|
|
||||||
|
it("leaves dimension undefined when nothing explicit is set", () => {
|
||||||
|
const cfg = ConfigManager.mergeConfig({
|
||||||
|
embedder: { provider: "openai", config: { apiKey: "k" } },
|
||||||
|
vectorStore: { provider: "memory", config: { collectionName: "t" } },
|
||||||
|
llm: baseLlm,
|
||||||
|
});
|
||||||
|
expect(cfg.vectorStore.config.dimension).toBeUndefined();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("uses embeddingDims from embedder config", () => {
|
||||||
|
const cfg = ConfigManager.mergeConfig({
|
||||||
|
embedder: {
|
||||||
|
provider: "ollama",
|
||||||
|
config: { model: "nomic-embed-text", embeddingDims: 768 },
|
||||||
|
},
|
||||||
|
vectorStore: { provider: "qdrant", config: { collectionName: "t" } },
|
||||||
|
llm: baseLlm,
|
||||||
|
});
|
||||||
|
expect(cfg.vectorStore.config.dimension).toBe(768);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("prefers explicit vectorStore.dimension over embeddingDims", () => {
|
||||||
|
const cfg = ConfigManager.mergeConfig({
|
||||||
|
embedder: {
|
||||||
|
provider: "ollama",
|
||||||
|
config: { model: "nomic-embed-text", embeddingDims: 768 },
|
||||||
|
},
|
||||||
|
vectorStore: {
|
||||||
|
provider: "qdrant",
|
||||||
|
config: { collectionName: "t", dimension: 1024 },
|
||||||
|
},
|
||||||
|
llm: baseLlm,
|
||||||
|
});
|
||||||
|
expect(cfg.vectorStore.config.dimension).toBe(1024);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("leaves dimension undefined for custom client without explicit dims", () => {
|
||||||
|
const cfg = ConfigManager.mergeConfig({
|
||||||
|
embedder: { provider: "ollama", config: { model: "nomic-embed-text" } },
|
||||||
|
vectorStore: {
|
||||||
|
provider: "qdrant",
|
||||||
|
config: { collectionName: "t", client: {} },
|
||||||
|
},
|
||||||
|
llm: baseLlm,
|
||||||
|
});
|
||||||
|
expect(cfg.vectorStore.config.dimension).toBeUndefined();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("uses embeddingDims with a custom client", () => {
|
||||||
|
const cfg = ConfigManager.mergeConfig({
|
||||||
|
embedder: {
|
||||||
|
provider: "ollama",
|
||||||
|
config: { model: "nomic-embed-text", embeddingDims: 768 },
|
||||||
|
},
|
||||||
|
vectorStore: {
|
||||||
|
provider: "qdrant",
|
||||||
|
config: { collectionName: "t", client: {} },
|
||||||
|
},
|
||||||
|
llm: baseLlm,
|
||||||
|
});
|
||||||
|
expect(cfg.vectorStore.config.dimension).toBe(768);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("preserves all other vectorStore config fields", () => {
|
||||||
|
const cfg = ConfigManager.mergeConfig({
|
||||||
|
embedder: { provider: "openai", config: { apiKey: "k" } },
|
||||||
|
vectorStore: {
|
||||||
|
provider: "qdrant",
|
||||||
|
config: {
|
||||||
|
collectionName: "my-coll",
|
||||||
|
host: "my-host",
|
||||||
|
port: 6333,
|
||||||
|
apiKey: "qdrant-key",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
llm: baseLlm,
|
||||||
|
});
|
||||||
|
expect(cfg.vectorStore.config.collectionName).toBe("my-coll");
|
||||||
|
expect(cfg.vectorStore.config.host).toBe("my-host");
|
||||||
|
expect(cfg.vectorStore.config.port).toBe(6333);
|
||||||
|
expect(cfg.vectorStore.config.apiKey).toBe("qdrant-key");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("leaves dimension undefined with empty config", () => {
|
||||||
|
const cfg = ConfigManager.mergeConfig({
|
||||||
|
embedder: { provider: "openai", config: {} },
|
||||||
|
vectorStore: { provider: "memory", config: {} },
|
||||||
|
llm: baseLlm,
|
||||||
|
});
|
||||||
|
expect(cfg.vectorStore.config.dimension).toBeUndefined();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
// ───────────────────────────────────────────────────────────────────────────
|
||||||
|
// 2. MemoryVectorStore – backward compat with explicit dimensions
|
||||||
|
// ───────────────────────────────────────────────────────────────────────────
|
||||||
|
describe("MemoryVectorStore – backward compat", () => {
|
||||||
|
let tmpDir: string;
|
||||||
|
|
||||||
|
beforeEach(() => {
|
||||||
|
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "mem0-test-"));
|
||||||
|
});
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
fs.rmSync(tmpDir, { recursive: true, force: true });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("defaults to dimension 1536 when not specified", async () => {
|
||||||
|
const store = new MemoryVectorStore({
|
||||||
|
collectionName: "test",
|
||||||
|
dbPath: path.join(tmpDir, "vs.db"),
|
||||||
|
});
|
||||||
|
|
||||||
|
const vector = new Array(1536).fill(0.1);
|
||||||
|
await store.insert([vector], ["id-1"], [{ data: "hello" }]);
|
||||||
|
const result = await store.get("id-1");
|
||||||
|
expect(result).not.toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("explicit dimension=1536 still works", async () => {
|
||||||
|
const store = new MemoryVectorStore({
|
||||||
|
collectionName: "test",
|
||||||
|
dimension: 1536,
|
||||||
|
dbPath: path.join(tmpDir, "vs.db"),
|
||||||
|
});
|
||||||
|
|
||||||
|
const vector = new Array(1536).fill(0.1);
|
||||||
|
await store.insert([vector], ["id-1"], [{ data: "hello" }]);
|
||||||
|
const result = await store.get("id-1");
|
||||||
|
expect(result).not.toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("explicit dimension rejects mismatched vectors", async () => {
|
||||||
|
const store = new MemoryVectorStore({
|
||||||
|
collectionName: "test",
|
||||||
|
dimension: 1536,
|
||||||
|
dbPath: path.join(tmpDir, "vs.db"),
|
||||||
|
});
|
||||||
|
|
||||||
|
const wrongVector = new Array(768).fill(0.1);
|
||||||
|
await expect(
|
||||||
|
store.insert([wrongVector], ["id-1"], [{ data: "hello" }]),
|
||||||
|
).rejects.toThrow("Vector dimension mismatch");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("search validates dimension", async () => {
|
||||||
|
const store = new MemoryVectorStore({
|
||||||
|
collectionName: "test",
|
||||||
|
dimension: 4,
|
||||||
|
dbPath: path.join(tmpDir, "vs.db"),
|
||||||
|
});
|
||||||
|
|
||||||
|
await expect(store.search([1, 2, 3], 1)).rejects.toThrow(
|
||||||
|
"Query dimension mismatch",
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("custom dimension=768 works end-to-end", async () => {
|
||||||
|
const store = new MemoryVectorStore({
|
||||||
|
collectionName: "test",
|
||||||
|
dimension: 768,
|
||||||
|
dbPath: path.join(tmpDir, "vs.db"),
|
||||||
|
});
|
||||||
|
|
||||||
|
await store.insert(
|
||||||
|
[
|
||||||
|
[1, ...new Array(767).fill(0)],
|
||||||
|
[0, 1, ...new Array(766).fill(0)],
|
||||||
|
],
|
||||||
|
["a", "b"],
|
||||||
|
[{ data: "alpha" }, { data: "beta" }],
|
||||||
|
);
|
||||||
|
|
||||||
|
const results = await store.search([1, ...new Array(767).fill(0)], 2);
|
||||||
|
expect(results.length).toBe(2);
|
||||||
|
expect(results[0].id).toBe("a");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("getUserId and setUserId still work", async () => {
|
||||||
|
const store = new MemoryVectorStore({
|
||||||
|
collectionName: "test",
|
||||||
|
dbPath: path.join(tmpDir, "vs.db"),
|
||||||
|
});
|
||||||
|
|
||||||
|
const userId = await store.getUserId();
|
||||||
|
expect(typeof userId).toBe("string");
|
||||||
|
expect(userId.length).toBeGreaterThan(0);
|
||||||
|
|
||||||
|
await store.setUserId("custom-user");
|
||||||
|
const newUserId = await store.getUserId();
|
||||||
|
expect(newUserId).toBe("custom-user");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("initialize() is idempotent", async () => {
|
||||||
|
const store = new MemoryVectorStore({
|
||||||
|
collectionName: "test",
|
||||||
|
dbPath: path.join(tmpDir, "vs.db"),
|
||||||
|
});
|
||||||
|
|
||||||
|
await store.initialize();
|
||||||
|
await store.initialize();
|
||||||
|
await store.initialize();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
// ───────────────────────────────────────────────────────────────────────────
|
||||||
|
// 3. Memory class – auto-init with probe, lazy gate, backward compat
|
||||||
|
// ───────────────────────────────────────────────────────────────────────────
|
||||||
|
describe("Memory – auto-initialization", () => {
|
||||||
|
let mockEmbedderFactory: any;
|
||||||
|
let mockVectorStoreFactory: any;
|
||||||
|
let mockLlmFactory: any;
|
||||||
|
let mockHistoryFactory: any;
|
||||||
|
let MemoryClass: any;
|
||||||
|
|
||||||
|
function createMockEmbedder(dims: number) {
|
||||||
|
return {
|
||||||
|
embed: jest.fn().mockResolvedValue(new Array(dims).fill(0)),
|
||||||
|
embedBatch: jest.fn().mockResolvedValue([new Array(dims).fill(0)]),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function createMockVectorStore() {
|
||||||
|
return {
|
||||||
|
insert: jest.fn().mockResolvedValue(undefined),
|
||||||
|
search: jest.fn().mockResolvedValue([]),
|
||||||
|
get: jest.fn().mockResolvedValue(null),
|
||||||
|
update: jest.fn().mockResolvedValue(undefined),
|
||||||
|
delete: jest.fn().mockResolvedValue(undefined),
|
||||||
|
deleteCol: jest.fn().mockResolvedValue(undefined),
|
||||||
|
list: jest.fn().mockResolvedValue([[], 0]),
|
||||||
|
getUserId: jest.fn().mockResolvedValue("test-user-id"),
|
||||||
|
setUserId: jest.fn().mockResolvedValue(undefined),
|
||||||
|
initialize: jest.fn().mockResolvedValue(undefined),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
beforeEach(() => {
|
||||||
|
jest.resetModules();
|
||||||
|
|
||||||
|
const mockEmbedder = createMockEmbedder(768);
|
||||||
|
const mockVStore = createMockVectorStore();
|
||||||
|
|
||||||
|
mockEmbedderFactory = { create: jest.fn().mockReturnValue(mockEmbedder) };
|
||||||
|
mockVectorStoreFactory = { create: jest.fn().mockReturnValue(mockVStore) };
|
||||||
|
mockLlmFactory = {
|
||||||
|
create: jest.fn().mockReturnValue({
|
||||||
|
generateResponse: jest.fn().mockResolvedValue('{"facts":[]}'),
|
||||||
|
}),
|
||||||
|
};
|
||||||
|
mockHistoryFactory = {
|
||||||
|
create: jest.fn().mockReturnValue({
|
||||||
|
addHistory: jest.fn().mockResolvedValue(undefined),
|
||||||
|
getHistory: jest.fn().mockResolvedValue([]),
|
||||||
|
reset: jest.fn().mockResolvedValue(undefined),
|
||||||
|
}),
|
||||||
|
};
|
||||||
|
|
||||||
|
jest.doMock("../src/utils/factory", () => ({
|
||||||
|
EmbedderFactory: mockEmbedderFactory,
|
||||||
|
VectorStoreFactory: mockVectorStoreFactory,
|
||||||
|
LLMFactory: mockLlmFactory,
|
||||||
|
HistoryManagerFactory: mockHistoryFactory,
|
||||||
|
}));
|
||||||
|
|
||||||
|
jest.doMock("../src/utils/telemetry", () => ({
|
||||||
|
captureClientEvent: jest.fn().mockResolvedValue(undefined),
|
||||||
|
}));
|
||||||
|
|
||||||
|
MemoryClass = require("../src/memory").Memory;
|
||||||
|
});
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
jest.restoreAllMocks();
|
||||||
|
jest.resetModules();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("probes embedder to detect dimension when none set", async () => {
|
||||||
|
const mockEmbedder = createMockEmbedder(768);
|
||||||
|
const mockVStore = createMockVectorStore();
|
||||||
|
mockEmbedderFactory.create.mockReturnValue(mockEmbedder);
|
||||||
|
mockVectorStoreFactory.create.mockReturnValue(mockVStore);
|
||||||
|
|
||||||
|
const mem = new MemoryClass({
|
||||||
|
embedder: { provider: "ollama", config: { model: "nomic-embed-text" } },
|
||||||
|
vectorStore: { provider: "qdrant", config: { collectionName: "test" } },
|
||||||
|
llm: { provider: "openai", config: { apiKey: "k" } },
|
||||||
|
disableHistory: true,
|
||||||
|
});
|
||||||
|
|
||||||
|
await mem.getAll({ userId: "u1" });
|
||||||
|
|
||||||
|
// Should have called embed("dimension probe") to detect dimension
|
||||||
|
expect(mockEmbedder.embed).toHaveBeenCalledWith("dimension probe");
|
||||||
|
|
||||||
|
// VectorStoreFactory should have been called with detected dimension
|
||||||
|
const vsCreateCall = mockVectorStoreFactory.create.mock.calls[0];
|
||||||
|
expect(vsCreateCall[1].dimension).toBe(768);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("skips probe when explicit dimension provided", async () => {
|
||||||
|
const mockEmbedder = createMockEmbedder(1536);
|
||||||
|
const mockVStore = createMockVectorStore();
|
||||||
|
mockEmbedderFactory.create.mockReturnValue(mockEmbedder);
|
||||||
|
mockVectorStoreFactory.create.mockReturnValue(mockVStore);
|
||||||
|
|
||||||
|
const mem = new MemoryClass({
|
||||||
|
embedder: { provider: "openai", config: { apiKey: "k" } },
|
||||||
|
vectorStore: {
|
||||||
|
provider: "memory",
|
||||||
|
config: { collectionName: "test", dimension: 1536 },
|
||||||
|
},
|
||||||
|
llm: { provider: "openai", config: { apiKey: "k" } },
|
||||||
|
disableHistory: true,
|
||||||
|
});
|
||||||
|
|
||||||
|
await mem.getAll({ userId: "u1" });
|
||||||
|
|
||||||
|
// embed should NOT have been called for probing
|
||||||
|
expect(mockEmbedder.embed).not.toHaveBeenCalledWith("dimension probe");
|
||||||
|
|
||||||
|
// VectorStoreFactory gets the explicit dimension
|
||||||
|
const vsCreateCall = mockVectorStoreFactory.create.mock.calls[0];
|
||||||
|
expect(vsCreateCall[1].dimension).toBe(1536);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("skips probe when embeddingDims provided", async () => {
|
||||||
|
const mockEmbedder = createMockEmbedder(768);
|
||||||
|
const mockVStore = createMockVectorStore();
|
||||||
|
mockEmbedderFactory.create.mockReturnValue(mockEmbedder);
|
||||||
|
mockVectorStoreFactory.create.mockReturnValue(mockVStore);
|
||||||
|
|
||||||
|
const mem = new MemoryClass({
|
||||||
|
embedder: {
|
||||||
|
provider: "ollama",
|
||||||
|
config: { model: "nomic-embed-text", embeddingDims: 768 },
|
||||||
|
},
|
||||||
|
vectorStore: { provider: "qdrant", config: { collectionName: "test" } },
|
||||||
|
llm: { provider: "openai", config: { apiKey: "k" } },
|
||||||
|
disableHistory: true,
|
||||||
|
});
|
||||||
|
|
||||||
|
await mem.getAll({ userId: "u1" });
|
||||||
|
|
||||||
|
// ConfigManager resolves dimension from embeddingDims → no probe needed
|
||||||
|
expect(mockEmbedder.embed).not.toHaveBeenCalledWith("dimension probe");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("all public methods wait for initialization", async () => {
|
||||||
|
let resolveProbe: () => void;
|
||||||
|
let probeCallCount = 0;
|
||||||
|
const mockEmbedder = {
|
||||||
|
embed: jest.fn().mockImplementation(() => {
|
||||||
|
probeCallCount++;
|
||||||
|
if (probeCallCount === 1) {
|
||||||
|
// First call is the dimension probe — hang until manually resolved
|
||||||
|
return new Promise<number[]>((resolve) => {
|
||||||
|
resolveProbe = () => resolve(new Array(768).fill(0));
|
||||||
|
});
|
||||||
|
}
|
||||||
|
// Subsequent calls (from search, etc.) resolve immediately
|
||||||
|
return Promise.resolve(new Array(768).fill(0));
|
||||||
|
}),
|
||||||
|
embedBatch: jest.fn(),
|
||||||
|
};
|
||||||
|
const mockVStore = createMockVectorStore();
|
||||||
|
mockEmbedderFactory.create.mockReturnValue(mockEmbedder);
|
||||||
|
mockVectorStoreFactory.create.mockReturnValue(mockVStore);
|
||||||
|
|
||||||
|
const mem = new MemoryClass({
|
||||||
|
embedder: { provider: "ollama", config: { model: "test" } },
|
||||||
|
vectorStore: { provider: "qdrant", config: { collectionName: "t" } },
|
||||||
|
llm: { provider: "openai", config: { apiKey: "k" } },
|
||||||
|
disableHistory: true,
|
||||||
|
});
|
||||||
|
|
||||||
|
let getAllDone = false;
|
||||||
|
let searchDone = false;
|
||||||
|
let getDone = false;
|
||||||
|
|
||||||
|
const getAllP = mem.getAll({ userId: "u" }).then(() => (getAllDone = true));
|
||||||
|
const searchP = mem
|
||||||
|
.search("q", { userId: "u" })
|
||||||
|
.then(() => (searchDone = true));
|
||||||
|
const getP = mem.get("id").then(() => (getDone = true));
|
||||||
|
|
||||||
|
await new Promise((r) => setTimeout(r, 50));
|
||||||
|
expect(getAllDone).toBe(false);
|
||||||
|
expect(searchDone).toBe(false);
|
||||||
|
expect(getDone).toBe(false);
|
||||||
|
|
||||||
|
// Resolve the probe — init completes — methods unblock
|
||||||
|
resolveProbe!();
|
||||||
|
await Promise.all([getAllP, searchP, getP]);
|
||||||
|
expect(getAllDone).toBe(true);
|
||||||
|
expect(searchDone).toBe(true);
|
||||||
|
expect(getDone).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("reset re-creates vector store with correct dimension", async () => {
|
||||||
|
const mockEmbedder = createMockEmbedder(768);
|
||||||
|
const mockVStore = createMockVectorStore();
|
||||||
|
mockEmbedderFactory.create.mockReturnValue(mockEmbedder);
|
||||||
|
mockVectorStoreFactory.create.mockReturnValue(mockVStore);
|
||||||
|
|
||||||
|
const mem = new MemoryClass({
|
||||||
|
embedder: { provider: "ollama", config: { model: "nomic-embed-text" } },
|
||||||
|
vectorStore: { provider: "qdrant", config: { collectionName: "test" } },
|
||||||
|
llm: { provider: "openai", config: { apiKey: "k" } },
|
||||||
|
disableHistory: true,
|
||||||
|
});
|
||||||
|
|
||||||
|
await mem.getAll({ userId: "u1" });
|
||||||
|
expect(mockVectorStoreFactory.create).toHaveBeenCalledTimes(1);
|
||||||
|
|
||||||
|
// Reset should re-create vector store
|
||||||
|
const mockVStore2 = createMockVectorStore();
|
||||||
|
mockVectorStoreFactory.create.mockReturnValue(mockVStore2);
|
||||||
|
await mem.reset();
|
||||||
|
expect(mockVectorStoreFactory.create).toHaveBeenCalledTimes(2);
|
||||||
|
|
||||||
|
// Second creation should still have dimension=768 (cached from first probe)
|
||||||
|
const secondCall = mockVectorStoreFactory.create.mock.calls[1];
|
||||||
|
expect(secondCall[1].dimension).toBe(768);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("backward compat: full explicit config works without probe", async () => {
|
||||||
|
const mockEmbedder = createMockEmbedder(1536);
|
||||||
|
const mockVStore = createMockVectorStore();
|
||||||
|
mockEmbedderFactory.create.mockReturnValue(mockEmbedder);
|
||||||
|
mockVectorStoreFactory.create.mockReturnValue(mockVStore);
|
||||||
|
|
||||||
|
const mem = new MemoryClass({
|
||||||
|
version: "v1.1",
|
||||||
|
embedder: {
|
||||||
|
provider: "openai",
|
||||||
|
config: { apiKey: "sk-fake", model: "text-embedding-3-small" },
|
||||||
|
},
|
||||||
|
vectorStore: {
|
||||||
|
provider: "memory",
|
||||||
|
config: { collectionName: "test-memories", dimension: 1536 },
|
||||||
|
},
|
||||||
|
llm: {
|
||||||
|
provider: "openai",
|
||||||
|
config: { apiKey: "sk-fake", model: "gpt-4-turbo-preview" },
|
||||||
|
},
|
||||||
|
historyDbPath: ":memory:",
|
||||||
|
disableHistory: true,
|
||||||
|
});
|
||||||
|
|
||||||
|
await mem.getAll({ userId: "u1" });
|
||||||
|
expect(mockEmbedder.embed).not.toHaveBeenCalledWith("dimension probe");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("throws explicit error when probe fails", async () => {
|
||||||
|
const mockEmbedder = {
|
||||||
|
embed: jest.fn().mockRejectedValue(new Error("Connection refused")),
|
||||||
|
embedBatch: jest.fn(),
|
||||||
|
};
|
||||||
|
mockEmbedderFactory.create.mockReturnValue(mockEmbedder);
|
||||||
|
|
||||||
|
// Suppress console.error for this test
|
||||||
|
const consoleSpy = jest
|
||||||
|
.spyOn(console, "error")
|
||||||
|
.mockImplementation(() => {});
|
||||||
|
|
||||||
|
const mem = new MemoryClass({
|
||||||
|
embedder: { provider: "ollama", config: { model: "nomic-embed-text" } },
|
||||||
|
vectorStore: { provider: "qdrant", config: { collectionName: "test" } },
|
||||||
|
llm: { provider: "openai", config: { apiKey: "k" } },
|
||||||
|
disableHistory: true,
|
||||||
|
});
|
||||||
|
|
||||||
|
// getAll should reject with the init error
|
||||||
|
await expect(mem.getAll({ userId: "u1" })).rejects.toThrow(
|
||||||
|
"auto-detect embedding dimension",
|
||||||
|
);
|
||||||
|
|
||||||
|
// Verify the error was logged and contains helpful information
|
||||||
|
const errorCall = consoleSpy.mock.calls.find(
|
||||||
|
(call) =>
|
||||||
|
call[0] instanceof Error &&
|
||||||
|
call[0].message.includes("auto-detect embedding dimension"),
|
||||||
|
);
|
||||||
|
expect(errorCall).toBeDefined();
|
||||||
|
const errorMsg = (errorCall![0] as Error).message;
|
||||||
|
expect(errorMsg).toContain("ollama");
|
||||||
|
expect(errorMsg).toContain("Connection refused");
|
||||||
|
expect(errorMsg).toContain("dimension");
|
||||||
|
expect(errorMsg).toContain("embeddingDims");
|
||||||
|
|
||||||
|
consoleSpy.mockRestore();
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,521 @@
|
|||||||
|
/// <reference types="jest" />
|
||||||
|
/**
|
||||||
|
* End-to-end tests for Qdrant dimension mismatch fix.
|
||||||
|
*
|
||||||
|
* Requires a running Qdrant instance at localhost:6333 (v1.13.x).
|
||||||
|
* These tests replicate the exact scenarios from issues #4212, #4173, #4056.
|
||||||
|
*
|
||||||
|
* Skipped automatically when Qdrant is not available.
|
||||||
|
*
|
||||||
|
* Run: npx jest --config jest.config.js src/oss/tests/qdrant-e2e.test.ts --forceExit
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { QdrantClient } from "@qdrant/js-client-rest";
|
||||||
|
import { Qdrant } from "../src/vector_stores/qdrant";
|
||||||
|
import { v4 as uuidv4 } from "uuid";
|
||||||
|
jest.setTimeout(30000);
|
||||||
|
|
||||||
|
const QDRANT_HOST = "localhost";
|
||||||
|
const QDRANT_PORT = 6333;
|
||||||
|
|
||||||
|
// Check if Qdrant is reachable synchronously at load time using
|
||||||
|
// a sync check via child_process so describe.skip works correctly.
|
||||||
|
function isQdrantAvailable(): boolean {
|
||||||
|
try {
|
||||||
|
const { execSync } = require("child_process");
|
||||||
|
execSync(
|
||||||
|
`node -e "const s=require('net').createConnection({host:'${QDRANT_HOST}',port:${QDRANT_PORT}});s.on('connect',()=>{s.destroy();process.exit(0)});s.on('error',()=>process.exit(1));s.setTimeout(2000,()=>process.exit(1))"`,
|
||||||
|
{ timeout: 3000, stdio: "ignore" },
|
||||||
|
);
|
||||||
|
return true;
|
||||||
|
} catch {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const qdrantAvailable = isQdrantAvailable();
|
||||||
|
if (!qdrantAvailable) {
|
||||||
|
console.warn("Qdrant not available at localhost:6333 — skipping e2e tests");
|
||||||
|
}
|
||||||
|
|
||||||
|
let qdrantClient: QdrantClient;
|
||||||
|
|
||||||
|
beforeAll(async () => {
|
||||||
|
if (!qdrantAvailable) return;
|
||||||
|
qdrantClient = new QdrantClient({ host: QDRANT_HOST, port: QDRANT_PORT });
|
||||||
|
const collections = await qdrantClient.getCollections();
|
||||||
|
expect(collections).toBeDefined();
|
||||||
|
});
|
||||||
|
|
||||||
|
// Helper: delete a collection if it exists
|
||||||
|
async function deleteCollectionIfExists(name: string) {
|
||||||
|
try {
|
||||||
|
await qdrantClient.deleteCollection(name);
|
||||||
|
} catch {
|
||||||
|
// Collection doesn't exist — fine
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Helper: create a fake embedder that produces vectors of a given dimension
|
||||||
|
function createFakeEmbedder(dims: number) {
|
||||||
|
return {
|
||||||
|
embed: jest.fn().mockImplementation(async (_text: string) => {
|
||||||
|
const vec = new Array(dims).fill(0);
|
||||||
|
for (let i = 0; i < _text.length && i < dims; i++) {
|
||||||
|
vec[i] = _text.charCodeAt(i) / 255;
|
||||||
|
}
|
||||||
|
return vec;
|
||||||
|
}),
|
||||||
|
embedBatch: jest.fn().mockImplementation(async (texts: string[]) => {
|
||||||
|
return Promise.all(
|
||||||
|
texts.map(async (t) => {
|
||||||
|
const vec = new Array(dims).fill(0);
|
||||||
|
for (let i = 0; i < t.length && i < dims; i++) {
|
||||||
|
vec[i] = t.charCodeAt(i) / 255;
|
||||||
|
}
|
||||||
|
return vec;
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
}),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
// Conditionally skip tests when Qdrant is unavailable
|
||||||
|
const describeIfQdrant = qdrantAvailable ? describe : describe.skip;
|
||||||
|
|
||||||
|
afterAll(async () => {
|
||||||
|
await deleteCollectionIfExists("e2e_test_768");
|
||||||
|
await deleteCollectionIfExists("e2e_test_1536");
|
||||||
|
await deleteCollectionIfExists("e2e_test_race");
|
||||||
|
await deleteCollectionIfExists("e2e_test_race2");
|
||||||
|
await deleteCollectionIfExists("e2e_test_noexplicit");
|
||||||
|
await deleteCollectionIfExists("e2e_test_explicit");
|
||||||
|
await deleteCollectionIfExists("e2e_test_embdims");
|
||||||
|
await deleteCollectionIfExists("e2e_test_autodetect");
|
||||||
|
await deleteCollectionIfExists("memory_migrations");
|
||||||
|
});
|
||||||
|
|
||||||
|
// ───────────────────────────────────────────────────────────────────────────
|
||||||
|
// 1. Reproduce #4212 / #4173: dimension mismatch with 768-dim embedder
|
||||||
|
// ───────────────────────────────────────────────────────────────────────────
|
||||||
|
describeIfQdrant("Issue #4212/#4173: Qdrant dimension mismatch", () => {
|
||||||
|
it("BEFORE FIX scenario: 768-dim vector into 1536-dim collection → Bad Request", async () => {
|
||||||
|
const collectionName = "e2e_test_1536";
|
||||||
|
await deleteCollectionIfExists(collectionName);
|
||||||
|
await qdrantClient.createCollection(collectionName, {
|
||||||
|
vectors: { size: 1536, distance: "Cosine" },
|
||||||
|
});
|
||||||
|
|
||||||
|
// Insert a 768-dim vector — this is what nomic-embed-text produces
|
||||||
|
const vector768 = new Array(768).fill(0.1);
|
||||||
|
try {
|
||||||
|
await qdrantClient.upsert(collectionName, {
|
||||||
|
points: [
|
||||||
|
{ id: "test-1", vector: vector768, payload: { data: "hello" } },
|
||||||
|
],
|
||||||
|
});
|
||||||
|
fail("Expected Qdrant to reject 768-dim vector into 1536-dim collection");
|
||||||
|
} catch (error: any) {
|
||||||
|
// This is the exact "Bad Request" error users were hitting
|
||||||
|
expect(error.status).toBe(400);
|
||||||
|
}
|
||||||
|
|
||||||
|
await deleteCollectionIfExists(collectionName);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("AFTER FIX: Qdrant store with dimension=768 works end-to-end", async () => {
|
||||||
|
const collectionName = "e2e_test_768";
|
||||||
|
await deleteCollectionIfExists(collectionName);
|
||||||
|
await deleteCollectionIfExists("memory_migrations");
|
||||||
|
|
||||||
|
// Create Qdrant store with correct dimension (what our auto-detect provides)
|
||||||
|
const store = new Qdrant({
|
||||||
|
host: QDRANT_HOST,
|
||||||
|
port: QDRANT_PORT,
|
||||||
|
collectionName,
|
||||||
|
embeddingModelDims: 768,
|
||||||
|
dimension: 768,
|
||||||
|
});
|
||||||
|
await store.initialize();
|
||||||
|
|
||||||
|
// Verify collection was created with 768 dims
|
||||||
|
const info = await qdrantClient.getCollection(collectionName);
|
||||||
|
expect(info.config?.params?.vectors?.size).toBe(768);
|
||||||
|
|
||||||
|
// Insert 768-dim vectors (what nomic-embed-text produces)
|
||||||
|
const vec1 = new Array(768).fill(0);
|
||||||
|
vec1[0] = 1.0;
|
||||||
|
const vec2 = new Array(768).fill(0);
|
||||||
|
vec2[1] = 1.0;
|
||||||
|
|
||||||
|
const id1 = uuidv4();
|
||||||
|
const id2 = uuidv4();
|
||||||
|
|
||||||
|
await store.insert([vec1, vec2], [id1, id2], [
|
||||||
|
{ data: "hello", userId: "u1" },
|
||||||
|
{ data: "world", userId: "u1" },
|
||||||
|
]);
|
||||||
|
|
||||||
|
// Search with 768-dim query — this USED TO fail with Bad Request
|
||||||
|
const results = await store.search(vec1, 2, { userId: "u1" });
|
||||||
|
expect(results.length).toBe(2);
|
||||||
|
expect(results[0].id).toBe(id1); // Most similar to itself
|
||||||
|
expect(results[0].score).toBeGreaterThan(0.9);
|
||||||
|
|
||||||
|
// Get by ID
|
||||||
|
const item = await store.get(id1);
|
||||||
|
expect(item).not.toBeNull();
|
||||||
|
expect(item!.payload.data).toBe("hello");
|
||||||
|
|
||||||
|
// Update with 768-dim vector
|
||||||
|
const vec3 = new Array(768).fill(0);
|
||||||
|
vec3[2] = 1.0;
|
||||||
|
await store.update(id1, vec3, { data: "updated", userId: "u1" });
|
||||||
|
const updated = await store.get(id1);
|
||||||
|
expect(updated!.payload.data).toBe("updated");
|
||||||
|
|
||||||
|
// Delete
|
||||||
|
await store.delete(id2);
|
||||||
|
const deleted = await store.get(id2);
|
||||||
|
expect(deleted).toBeNull();
|
||||||
|
|
||||||
|
// List
|
||||||
|
const [listed, count] = await store.list({ userId: "u1" });
|
||||||
|
expect(count).toBe(1);
|
||||||
|
expect(listed[0].payload.data).toBe("updated");
|
||||||
|
|
||||||
|
await deleteCollectionIfExists(collectionName);
|
||||||
|
await deleteCollectionIfExists("memory_migrations");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("AFTER FIX: Memory auto-detects 768 dims via probe (full integration)", async () => {
|
||||||
|
const collectionName = "e2e_test_autodetect";
|
||||||
|
await deleteCollectionIfExists(collectionName);
|
||||||
|
await deleteCollectionIfExists("memory_migrations");
|
||||||
|
|
||||||
|
const fakeEmbedder = createFakeEmbedder(768);
|
||||||
|
|
||||||
|
// Mock only the non-Qdrant factories to avoid Google SDK import crash
|
||||||
|
jest.resetModules();
|
||||||
|
jest.doMock("../src/utils/factory", () => {
|
||||||
|
// Import Qdrant directly (avoids loading Google embedder via factory)
|
||||||
|
const { Qdrant: QdrantStore } = require("../src/vector_stores/qdrant");
|
||||||
|
return {
|
||||||
|
EmbedderFactory: { create: jest.fn().mockReturnValue(fakeEmbedder) },
|
||||||
|
VectorStoreFactory: {
|
||||||
|
create: jest.fn().mockImplementation((_provider: string, config: any) => {
|
||||||
|
return new QdrantStore(config);
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
LLMFactory: {
|
||||||
|
create: jest.fn().mockReturnValue({
|
||||||
|
generateResponse: jest.fn().mockResolvedValue('{"facts":[]}'),
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
HistoryManagerFactory: {
|
||||||
|
create: jest.fn().mockReturnValue({
|
||||||
|
addHistory: jest.fn().mockResolvedValue(undefined),
|
||||||
|
getHistory: jest.fn().mockResolvedValue([]),
|
||||||
|
reset: jest.fn().mockResolvedValue(undefined),
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
|
jest.doMock("../src/utils/telemetry", () => ({
|
||||||
|
captureClientEvent: jest.fn().mockResolvedValue(undefined),
|
||||||
|
}));
|
||||||
|
|
||||||
|
const { Memory } = require("../src/memory");
|
||||||
|
|
||||||
|
// This is the EXACT config from issue #4212 — NO dimension specified
|
||||||
|
const mem = new Memory({
|
||||||
|
embedder: {
|
||||||
|
provider: "ollama",
|
||||||
|
config: { model: "nomic-embed-text" },
|
||||||
|
},
|
||||||
|
vectorStore: {
|
||||||
|
provider: "qdrant",
|
||||||
|
config: {
|
||||||
|
host: QDRANT_HOST,
|
||||||
|
port: QDRANT_PORT,
|
||||||
|
collectionName,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
llm: { provider: "openai", config: { apiKey: "fake" } },
|
||||||
|
disableHistory: true,
|
||||||
|
});
|
||||||
|
|
||||||
|
// This triggers init — probe should detect 768 dims
|
||||||
|
await mem.getAll({ userId: "test-user" });
|
||||||
|
|
||||||
|
// Verify the probe was called
|
||||||
|
expect(fakeEmbedder.embed).toHaveBeenCalledWith("dimension probe");
|
||||||
|
|
||||||
|
// Verify Qdrant collection was created with auto-detected 768 dims
|
||||||
|
const collectionInfo = await qdrantClient.getCollection(collectionName);
|
||||||
|
expect(collectionInfo.config?.params?.vectors?.size).toBe(768);
|
||||||
|
|
||||||
|
// Search should work (this used to throw Bad Request)
|
||||||
|
const searchResult = await mem.search("hello world", {
|
||||||
|
userId: "test-user",
|
||||||
|
});
|
||||||
|
expect(searchResult).toBeDefined();
|
||||||
|
|
||||||
|
await deleteCollectionIfExists(collectionName);
|
||||||
|
await deleteCollectionIfExists("memory_migrations");
|
||||||
|
jest.resetModules();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("AFTER FIX: explicit dimension=768 skips probe (backward compat)", async () => {
|
||||||
|
const collectionName = "e2e_test_explicit";
|
||||||
|
await deleteCollectionIfExists(collectionName);
|
||||||
|
await deleteCollectionIfExists("memory_migrations");
|
||||||
|
|
||||||
|
const fakeEmbedder = createFakeEmbedder(768);
|
||||||
|
|
||||||
|
jest.resetModules();
|
||||||
|
jest.doMock("../src/utils/factory", () => {
|
||||||
|
const { Qdrant: QdrantStore } = require("../src/vector_stores/qdrant");
|
||||||
|
return {
|
||||||
|
EmbedderFactory: { create: jest.fn().mockReturnValue(fakeEmbedder) },
|
||||||
|
VectorStoreFactory: {
|
||||||
|
create: jest.fn().mockImplementation((_provider: string, config: any) => {
|
||||||
|
return new QdrantStore(config);
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
LLMFactory: {
|
||||||
|
create: jest.fn().mockReturnValue({
|
||||||
|
generateResponse: jest.fn().mockResolvedValue('{"facts":[]}'),
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
HistoryManagerFactory: {
|
||||||
|
create: jest.fn().mockReturnValue({
|
||||||
|
addHistory: jest.fn().mockResolvedValue(undefined),
|
||||||
|
getHistory: jest.fn().mockResolvedValue([]),
|
||||||
|
reset: jest.fn().mockResolvedValue(undefined),
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
|
jest.doMock("../src/utils/telemetry", () => ({
|
||||||
|
captureClientEvent: jest.fn().mockResolvedValue(undefined),
|
||||||
|
}));
|
||||||
|
|
||||||
|
const { Memory } = require("../src/memory");
|
||||||
|
|
||||||
|
// Workaround config from #4212 — explicit dimension
|
||||||
|
const mem = new Memory({
|
||||||
|
embedder: {
|
||||||
|
provider: "ollama",
|
||||||
|
config: { model: "nomic-embed-text" },
|
||||||
|
},
|
||||||
|
vectorStore: {
|
||||||
|
provider: "qdrant",
|
||||||
|
config: {
|
||||||
|
host: QDRANT_HOST,
|
||||||
|
port: QDRANT_PORT,
|
||||||
|
collectionName,
|
||||||
|
dimension: 768,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
llm: { provider: "openai", config: { apiKey: "fake" } },
|
||||||
|
disableHistory: true,
|
||||||
|
});
|
||||||
|
|
||||||
|
await mem.getAll({ userId: "test-user" });
|
||||||
|
|
||||||
|
// Probe should NOT have been called
|
||||||
|
expect(fakeEmbedder.embed).not.toHaveBeenCalledWith("dimension probe");
|
||||||
|
|
||||||
|
const collectionInfo = await qdrantClient.getCollection(collectionName);
|
||||||
|
expect(collectionInfo.config?.params?.vectors?.size).toBe(768);
|
||||||
|
|
||||||
|
await deleteCollectionIfExists(collectionName);
|
||||||
|
await deleteCollectionIfExists("memory_migrations");
|
||||||
|
jest.resetModules();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("AFTER FIX: embeddingDims in embedder config skips probe", async () => {
|
||||||
|
const collectionName = "e2e_test_embdims";
|
||||||
|
await deleteCollectionIfExists(collectionName);
|
||||||
|
await deleteCollectionIfExists("memory_migrations");
|
||||||
|
|
||||||
|
const fakeEmbedder = createFakeEmbedder(768);
|
||||||
|
|
||||||
|
jest.resetModules();
|
||||||
|
jest.doMock("../src/utils/factory", () => {
|
||||||
|
const { Qdrant: QdrantStore } = require("../src/vector_stores/qdrant");
|
||||||
|
return {
|
||||||
|
EmbedderFactory: { create: jest.fn().mockReturnValue(fakeEmbedder) },
|
||||||
|
VectorStoreFactory: {
|
||||||
|
create: jest.fn().mockImplementation((_provider: string, config: any) => {
|
||||||
|
return new QdrantStore(config);
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
LLMFactory: {
|
||||||
|
create: jest.fn().mockReturnValue({
|
||||||
|
generateResponse: jest.fn().mockResolvedValue('{"facts":[]}'),
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
HistoryManagerFactory: {
|
||||||
|
create: jest.fn().mockReturnValue({
|
||||||
|
addHistory: jest.fn().mockResolvedValue(undefined),
|
||||||
|
getHistory: jest.fn().mockResolvedValue([]),
|
||||||
|
reset: jest.fn().mockResolvedValue(undefined),
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
|
jest.doMock("../src/utils/telemetry", () => ({
|
||||||
|
captureClientEvent: jest.fn().mockResolvedValue(undefined),
|
||||||
|
}));
|
||||||
|
|
||||||
|
const { Memory } = require("../src/memory");
|
||||||
|
|
||||||
|
const mem = new Memory({
|
||||||
|
embedder: {
|
||||||
|
provider: "ollama",
|
||||||
|
config: { model: "nomic-embed-text", embeddingDims: 768 },
|
||||||
|
},
|
||||||
|
vectorStore: {
|
||||||
|
provider: "qdrant",
|
||||||
|
config: {
|
||||||
|
host: QDRANT_HOST,
|
||||||
|
port: QDRANT_PORT,
|
||||||
|
collectionName,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
llm: { provider: "openai", config: { apiKey: "fake" } },
|
||||||
|
disableHistory: true,
|
||||||
|
});
|
||||||
|
|
||||||
|
await mem.getAll({ userId: "test-user" });
|
||||||
|
|
||||||
|
// Probe should NOT have been called — dimension inferred from embeddingDims
|
||||||
|
expect(fakeEmbedder.embed).not.toHaveBeenCalledWith("dimension probe");
|
||||||
|
|
||||||
|
const collectionInfo = await qdrantClient.getCollection(collectionName);
|
||||||
|
expect(collectionInfo.config?.params?.vectors?.size).toBe(768);
|
||||||
|
|
||||||
|
await deleteCollectionIfExists(collectionName);
|
||||||
|
await deleteCollectionIfExists("memory_migrations");
|
||||||
|
jest.resetModules();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
// ───────────────────────────────────────────────────────────────────────────
|
||||||
|
// 2. Reproduce #4056 issue 1: Collection creation race condition
|
||||||
|
// ───────────────────────────────────────────────────────────────────────────
|
||||||
|
describeIfQdrant("Issue #4056: Qdrant race condition", () => {
|
||||||
|
it("concurrent ensureCollection calls don't crash (no 409 error leak)", async () => {
|
||||||
|
const collectionName = "e2e_test_race";
|
||||||
|
await deleteCollectionIfExists(collectionName);
|
||||||
|
await deleteCollectionIfExists("memory_migrations");
|
||||||
|
|
||||||
|
// Create 5 Qdrant instances concurrently — this simulates the race
|
||||||
|
// that caused "Collection memory_migrations already exists!" in #4056
|
||||||
|
const instances = Array.from(
|
||||||
|
{ length: 5 },
|
||||||
|
() =>
|
||||||
|
new Qdrant({
|
||||||
|
host: QDRANT_HOST,
|
||||||
|
port: QDRANT_PORT,
|
||||||
|
collectionName,
|
||||||
|
embeddingModelDims: 768,
|
||||||
|
dimension: 768,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
// All should initialize without throwing 409 Conflict
|
||||||
|
await Promise.all(instances.map((inst) => inst.initialize()));
|
||||||
|
|
||||||
|
// Verify collection exists with correct dimension
|
||||||
|
const info = await qdrantClient.getCollection(collectionName);
|
||||||
|
expect(info.config?.params?.vectors?.size).toBe(768);
|
||||||
|
|
||||||
|
// memory_migrations should also exist (created by initialize)
|
||||||
|
const migrInfo = await qdrantClient.getCollection("memory_migrations");
|
||||||
|
expect(migrInfo.config?.params?.vectors?.size).toBe(1);
|
||||||
|
|
||||||
|
await deleteCollectionIfExists(collectionName);
|
||||||
|
await deleteCollectionIfExists("memory_migrations");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("getUserId works after concurrent initialization", async () => {
|
||||||
|
const collectionName = "e2e_test_race2";
|
||||||
|
await deleteCollectionIfExists(collectionName);
|
||||||
|
await deleteCollectionIfExists("memory_migrations");
|
||||||
|
|
||||||
|
const instance = new Qdrant({
|
||||||
|
host: QDRANT_HOST,
|
||||||
|
port: QDRANT_PORT,
|
||||||
|
collectionName,
|
||||||
|
embeddingModelDims: 768,
|
||||||
|
dimension: 768,
|
||||||
|
});
|
||||||
|
|
||||||
|
await instance.initialize();
|
||||||
|
|
||||||
|
// getUserId should work without 409 crash
|
||||||
|
const userId = await instance.getUserId();
|
||||||
|
expect(typeof userId).toBe("string");
|
||||||
|
expect(userId.length).toBeGreaterThan(0);
|
||||||
|
|
||||||
|
// setUserId + getUserId roundtrip
|
||||||
|
await instance.setUserId("custom-e2e-user");
|
||||||
|
const updated = await instance.getUserId();
|
||||||
|
expect(updated).toBe("custom-e2e-user");
|
||||||
|
|
||||||
|
await deleteCollectionIfExists(collectionName);
|
||||||
|
await deleteCollectionIfExists("memory_migrations");
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
// ───────────────────────────────────────────────────────────────────────────
|
||||||
|
// 3. Reproduce #4056 issue 2: memory_migrations dimension isolation
|
||||||
|
// ───────────────────────────────────────────────────────────────────────────
|
||||||
|
describeIfQdrant("Issue #4056: memory_migrations dimension isolation", () => {
|
||||||
|
it("memory_migrations uses dim=1 independently of main collection dim=768", async () => {
|
||||||
|
const collectionName = "e2e_test_noexplicit";
|
||||||
|
await deleteCollectionIfExists(collectionName);
|
||||||
|
await deleteCollectionIfExists("memory_migrations");
|
||||||
|
|
||||||
|
const instance = new Qdrant({
|
||||||
|
host: QDRANT_HOST,
|
||||||
|
port: QDRANT_PORT,
|
||||||
|
collectionName,
|
||||||
|
embeddingModelDims: 768,
|
||||||
|
dimension: 768,
|
||||||
|
});
|
||||||
|
|
||||||
|
await instance.initialize();
|
||||||
|
|
||||||
|
// Allow Qdrant a moment to fully commit collections
|
||||||
|
await new Promise((r) => setTimeout(r, 500));
|
||||||
|
|
||||||
|
// Main collection should be 768
|
||||||
|
const mainInfo = await qdrantClient.getCollection(collectionName);
|
||||||
|
expect(mainInfo.config?.params?.vectors?.size).toBe(768);
|
||||||
|
|
||||||
|
// memory_migrations should be 1 (NOT 768!)
|
||||||
|
// This was the bug in #4056 issue 2 — telemetry used wrong dimension
|
||||||
|
const migrationsInfo =
|
||||||
|
await qdrantClient.getCollection("memory_migrations");
|
||||||
|
expect(migrationsInfo.config?.params?.vectors?.size).toBe(1);
|
||||||
|
|
||||||
|
// getUserId should work — vector dim=1 in memory_migrations
|
||||||
|
const userId = await instance.getUserId();
|
||||||
|
expect(typeof userId).toBe("string");
|
||||||
|
|
||||||
|
// setUserId should also work
|
||||||
|
await instance.setUserId("custom-test-user");
|
||||||
|
const newUserId = await instance.getUserId();
|
||||||
|
expect(newUserId).toBe("custom-test-user");
|
||||||
|
|
||||||
|
await deleteCollectionIfExists(collectionName);
|
||||||
|
await deleteCollectionIfExists("memory_migrations");
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,427 @@
|
|||||||
|
/// <reference types="jest" />
|
||||||
|
/**
|
||||||
|
* End-to-end tests for Redis vector store with init guard fix.
|
||||||
|
*
|
||||||
|
* Requires a running Redis Stack instance at localhost:6379.
|
||||||
|
* Skipped automatically when Redis is not available.
|
||||||
|
*
|
||||||
|
* Run: npx jest --config jest.config.js src/oss/tests/redis-e2e.test.ts --forceExit
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { createClient } from "redis";
|
||||||
|
import { RedisDB } from "../src/vector_stores/redis";
|
||||||
|
import { v4 as uuidv4 } from "uuid";
|
||||||
|
jest.setTimeout(30000);
|
||||||
|
|
||||||
|
const REDIS_HOST = "localhost";
|
||||||
|
const REDIS_PORT = 6379;
|
||||||
|
const REDIS_URL = `redis://${REDIS_HOST}:${REDIS_PORT}`;
|
||||||
|
const COLLECTION_NAME = "e2e_redis_test";
|
||||||
|
|
||||||
|
// Check if Redis is reachable synchronously at load time
|
||||||
|
function isRedisAvailable(): boolean {
|
||||||
|
try {
|
||||||
|
const { execSync } = require("child_process");
|
||||||
|
execSync(
|
||||||
|
`node -e "const s=require('net').createConnection({host:'${REDIS_HOST}',port:${REDIS_PORT}});s.on('connect',()=>{s.destroy();process.exit(0)});s.on('error',()=>process.exit(1));s.setTimeout(2000,()=>process.exit(1))"`,
|
||||||
|
{ timeout: 3000, stdio: "ignore" },
|
||||||
|
);
|
||||||
|
return true;
|
||||||
|
} catch {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const redisAvailable = isRedisAvailable();
|
||||||
|
if (!redisAvailable) {
|
||||||
|
console.warn("Redis not available at localhost:6379 — skipping e2e tests");
|
||||||
|
}
|
||||||
|
|
||||||
|
// Standalone client for cleanup
|
||||||
|
let cleanupClient: ReturnType<typeof createClient>;
|
||||||
|
|
||||||
|
async function cleanupRedis() {
|
||||||
|
if (!redisAvailable) return;
|
||||||
|
try {
|
||||||
|
// Drop the index if it exists
|
||||||
|
await cleanupClient.ft.dropIndex(COLLECTION_NAME);
|
||||||
|
} catch {
|
||||||
|
// Index doesn't exist — fine
|
||||||
|
}
|
||||||
|
|
||||||
|
// Delete all keys with our prefix
|
||||||
|
const keys = await cleanupClient.keys(`mem0:${COLLECTION_NAME}:*`);
|
||||||
|
if (keys.length > 0) {
|
||||||
|
await cleanupClient.del(keys);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Clean up memory_migrations key
|
||||||
|
await cleanupClient.del("memory_migrations:1");
|
||||||
|
}
|
||||||
|
|
||||||
|
beforeAll(async () => {
|
||||||
|
if (!redisAvailable) return;
|
||||||
|
|
||||||
|
cleanupClient = createClient({ url: REDIS_URL });
|
||||||
|
await cleanupClient.connect();
|
||||||
|
|
||||||
|
// Verify Redis Stack is running with search module
|
||||||
|
const modules = (await cleanupClient.moduleList()) as unknown as any[];
|
||||||
|
const hasSearch = modules.some((mod: any[]) => {
|
||||||
|
const moduleMap = new Map();
|
||||||
|
for (let i = 0; i < mod.length; i += 2) {
|
||||||
|
moduleMap.set(mod[i], mod[i + 1]);
|
||||||
|
}
|
||||||
|
return moduleMap.get("name")?.toLowerCase() === "search";
|
||||||
|
});
|
||||||
|
expect(hasSearch).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
afterAll(async () => {
|
||||||
|
if (!redisAvailable) return;
|
||||||
|
await cleanupRedis();
|
||||||
|
await cleanupClient.quit();
|
||||||
|
});
|
||||||
|
|
||||||
|
// Conditionally skip tests when Redis is unavailable
|
||||||
|
const describeIfRedis = redisAvailable ? describe : describe.skip;
|
||||||
|
|
||||||
|
// ───────────────────────────────────────────────────────────────────────────
|
||||||
|
// 1. Basic initialization and idempotent init guard
|
||||||
|
// ───────────────────────────────────────────────────────────────────────────
|
||||||
|
describeIfRedis("Redis: initialization", () => {
|
||||||
|
afterEach(async () => {
|
||||||
|
await cleanupRedis();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("initializes successfully and creates index", async () => {
|
||||||
|
const store = new RedisDB({
|
||||||
|
redisUrl: REDIS_URL,
|
||||||
|
collectionName: COLLECTION_NAME,
|
||||||
|
embeddingModelDims: 128,
|
||||||
|
});
|
||||||
|
await store.initialize();
|
||||||
|
|
||||||
|
// Verify the index was created by querying index info
|
||||||
|
const info = await cleanupClient.ft.info(COLLECTION_NAME);
|
||||||
|
expect(info).toBeDefined();
|
||||||
|
expect(info.indexName).toBe(COLLECTION_NAME);
|
||||||
|
|
||||||
|
await store.close();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("idempotent initialize() — multiple calls don't crash", async () => {
|
||||||
|
const store = new RedisDB({
|
||||||
|
redisUrl: REDIS_URL,
|
||||||
|
collectionName: COLLECTION_NAME,
|
||||||
|
embeddingModelDims: 128,
|
||||||
|
});
|
||||||
|
|
||||||
|
// Call initialize multiple times concurrently
|
||||||
|
await Promise.all([
|
||||||
|
store.initialize(),
|
||||||
|
store.initialize(),
|
||||||
|
store.initialize(),
|
||||||
|
]);
|
||||||
|
|
||||||
|
// Should still work fine
|
||||||
|
const info = await cleanupClient.ft.info(COLLECTION_NAME);
|
||||||
|
expect(info).toBeDefined();
|
||||||
|
|
||||||
|
await store.close();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
// ───────────────────────────────────────────────────────────────────────────
|
||||||
|
// 2. Full CRUD operations
|
||||||
|
// ───────────────────────────────────────────────────────────────────────────
|
||||||
|
describeIfRedis("Redis: CRUD operations", () => {
|
||||||
|
let store: RedisDB;
|
||||||
|
|
||||||
|
beforeEach(async () => {
|
||||||
|
await cleanupRedis();
|
||||||
|
store = new RedisDB({
|
||||||
|
redisUrl: REDIS_URL,
|
||||||
|
collectionName: COLLECTION_NAME,
|
||||||
|
embeddingModelDims: 4, // Small dims for testing
|
||||||
|
});
|
||||||
|
await store.initialize();
|
||||||
|
});
|
||||||
|
|
||||||
|
afterEach(async () => {
|
||||||
|
await store.close();
|
||||||
|
await cleanupRedis();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("insert and search vectors", async () => {
|
||||||
|
const id1 = uuidv4();
|
||||||
|
const id2 = uuidv4();
|
||||||
|
const vec1 = [1.0, 0.0, 0.0, 0.0];
|
||||||
|
const vec2 = [0.0, 1.0, 0.0, 0.0];
|
||||||
|
|
||||||
|
await store.insert(
|
||||||
|
[vec1, vec2],
|
||||||
|
[id1, id2],
|
||||||
|
[
|
||||||
|
{
|
||||||
|
data: "hello world",
|
||||||
|
hash: "h1",
|
||||||
|
userId: "user1",
|
||||||
|
createdAt: new Date().toISOString(),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
data: "goodbye world",
|
||||||
|
hash: "h2",
|
||||||
|
userId: "user1",
|
||||||
|
createdAt: new Date().toISOString(),
|
||||||
|
},
|
||||||
|
],
|
||||||
|
);
|
||||||
|
|
||||||
|
// Search — vec1 should be most similar to itself
|
||||||
|
const results = await store.search(vec1, 2, { userId: "user1" });
|
||||||
|
expect(results.length).toBe(2);
|
||||||
|
// The first result should be closest to the query
|
||||||
|
expect(results[0].id).toBe(id1);
|
||||||
|
expect(results[0].score).toBeDefined();
|
||||||
|
expect(results[0].payload).toBeDefined();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("get vector by ID", async () => {
|
||||||
|
const id = uuidv4();
|
||||||
|
const vec = [0.5, 0.5, 0.0, 0.0];
|
||||||
|
|
||||||
|
await store.insert(
|
||||||
|
[vec],
|
||||||
|
[id],
|
||||||
|
[
|
||||||
|
{
|
||||||
|
data: "test memory",
|
||||||
|
hash: "h-test",
|
||||||
|
userId: "user1",
|
||||||
|
createdAt: new Date().toISOString(),
|
||||||
|
},
|
||||||
|
],
|
||||||
|
);
|
||||||
|
|
||||||
|
const result = await store.get(id);
|
||||||
|
expect(result).not.toBeNull();
|
||||||
|
expect(result!.id).toBe(id);
|
||||||
|
expect(result!.payload.data).toBe("test memory");
|
||||||
|
expect(result!.payload.hash).toBe("h-test");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("get non-existent vector returns null", async () => {
|
||||||
|
const result = await store.get("non-existent-id");
|
||||||
|
expect(result).toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("update vector and payload", async () => {
|
||||||
|
const id = uuidv4();
|
||||||
|
const vec = [1.0, 0.0, 0.0, 0.0];
|
||||||
|
|
||||||
|
await store.insert(
|
||||||
|
[vec],
|
||||||
|
[id],
|
||||||
|
[
|
||||||
|
{
|
||||||
|
data: "original",
|
||||||
|
hash: "h-orig",
|
||||||
|
userId: "user1",
|
||||||
|
createdAt: new Date().toISOString(),
|
||||||
|
},
|
||||||
|
],
|
||||||
|
);
|
||||||
|
|
||||||
|
// Update with new vector and payload
|
||||||
|
const newVec = [0.0, 0.0, 1.0, 0.0];
|
||||||
|
await store.update(id, newVec, {
|
||||||
|
data: "updated memory",
|
||||||
|
hash: "h-updated",
|
||||||
|
userId: "user1",
|
||||||
|
createdAt: new Date().toISOString(),
|
||||||
|
updatedAt: new Date().toISOString(),
|
||||||
|
});
|
||||||
|
|
||||||
|
const result = await store.get(id);
|
||||||
|
expect(result).not.toBeNull();
|
||||||
|
expect(result!.payload.data).toBe("updated memory");
|
||||||
|
expect(result!.payload.hash).toBe("h-updated");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("delete vector", async () => {
|
||||||
|
const id = uuidv4();
|
||||||
|
const vec = [0.0, 0.0, 0.0, 1.0];
|
||||||
|
|
||||||
|
await store.insert(
|
||||||
|
[vec],
|
||||||
|
[id],
|
||||||
|
[
|
||||||
|
{
|
||||||
|
data: "to be deleted",
|
||||||
|
hash: "h-del",
|
||||||
|
userId: "user1",
|
||||||
|
createdAt: new Date().toISOString(),
|
||||||
|
},
|
||||||
|
],
|
||||||
|
);
|
||||||
|
|
||||||
|
// Verify it exists
|
||||||
|
const before = await store.get(id);
|
||||||
|
expect(before).not.toBeNull();
|
||||||
|
|
||||||
|
// Delete
|
||||||
|
await store.delete(id);
|
||||||
|
|
||||||
|
// Verify it's gone
|
||||||
|
const after = await store.get(id);
|
||||||
|
expect(after).toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("list vectors with filters", async () => {
|
||||||
|
const id1 = uuidv4();
|
||||||
|
const id2 = uuidv4();
|
||||||
|
const id3 = uuidv4();
|
||||||
|
|
||||||
|
await store.insert(
|
||||||
|
[
|
||||||
|
[1, 0, 0, 0],
|
||||||
|
[0, 1, 0, 0],
|
||||||
|
[0, 0, 1, 0],
|
||||||
|
],
|
||||||
|
[id1, id2, id3],
|
||||||
|
[
|
||||||
|
{
|
||||||
|
data: "mem1",
|
||||||
|
hash: "h1",
|
||||||
|
userId: "usera",
|
||||||
|
createdAt: new Date().toISOString(),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
data: "mem2",
|
||||||
|
hash: "h2",
|
||||||
|
userId: "usera",
|
||||||
|
createdAt: new Date().toISOString(),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
data: "mem3",
|
||||||
|
hash: "h3",
|
||||||
|
userId: "userb",
|
||||||
|
createdAt: new Date().toISOString(),
|
||||||
|
},
|
||||||
|
],
|
||||||
|
);
|
||||||
|
|
||||||
|
// List all
|
||||||
|
const [all, allCount] = await store.list();
|
||||||
|
expect(allCount).toBe(3);
|
||||||
|
expect(all.length).toBe(3);
|
||||||
|
|
||||||
|
// List with filter
|
||||||
|
const [filtered, filteredCount] = await store.list({
|
||||||
|
userId: "usera",
|
||||||
|
});
|
||||||
|
expect(filteredCount).toBe(2);
|
||||||
|
expect(filtered.length).toBe(2);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
// ───────────────────────────────────────────────────────────────────────────
|
||||||
|
// 3. getUserId / setUserId
|
||||||
|
// ───────────────────────────────────────────────────────────────────────────
|
||||||
|
describeIfRedis("Redis: getUserId / setUserId", () => {
|
||||||
|
let store: RedisDB;
|
||||||
|
|
||||||
|
beforeEach(async () => {
|
||||||
|
await cleanupRedis();
|
||||||
|
store = new RedisDB({
|
||||||
|
redisUrl: REDIS_URL,
|
||||||
|
collectionName: COLLECTION_NAME,
|
||||||
|
embeddingModelDims: 4,
|
||||||
|
});
|
||||||
|
await store.initialize();
|
||||||
|
});
|
||||||
|
|
||||||
|
afterEach(async () => {
|
||||||
|
await store.close();
|
||||||
|
await cleanupRedis();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("getUserId generates random ID if none exists", async () => {
|
||||||
|
const userId = await store.getUserId();
|
||||||
|
expect(typeof userId).toBe("string");
|
||||||
|
expect(userId.length).toBeGreaterThan(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("setUserId + getUserId roundtrip", async () => {
|
||||||
|
await store.setUserId("custom-redis-user");
|
||||||
|
const retrieved = await store.getUserId();
|
||||||
|
expect(retrieved).toBe("custom-redis-user");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("getUserId returns same value on subsequent calls", async () => {
|
||||||
|
const first = await store.getUserId();
|
||||||
|
const second = await store.getUserId();
|
||||||
|
expect(first).toBe(second);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
// ───────────────────────────────────────────────────────────────────────────
|
||||||
|
// 4. Dimension handling (our fix ensures correct dims from Memory)
|
||||||
|
// ───────────────────────────────────────────────────────────────────────────
|
||||||
|
describeIfRedis("Redis: dimension handling", () => {
|
||||||
|
afterEach(async () => {
|
||||||
|
await cleanupRedis();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("creates index with correct dimensions from config", async () => {
|
||||||
|
const store = new RedisDB({
|
||||||
|
redisUrl: REDIS_URL,
|
||||||
|
collectionName: COLLECTION_NAME,
|
||||||
|
embeddingModelDims: 768,
|
||||||
|
});
|
||||||
|
await store.initialize();
|
||||||
|
|
||||||
|
// Verify the index has the right dimension in its schema
|
||||||
|
const info = await cleanupClient.ft.info(COLLECTION_NAME);
|
||||||
|
// Check that the vector field has DIM=768
|
||||||
|
const attributes = info.attributes as any[];
|
||||||
|
const vectorAttr = attributes.find(
|
||||||
|
(a: any) => a.identifier === "embedding" || a.attribute === "embedding",
|
||||||
|
);
|
||||||
|
expect(vectorAttr).toBeDefined();
|
||||||
|
|
||||||
|
await store.close();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("insert with matching dimension succeeds", async () => {
|
||||||
|
const dims = 128;
|
||||||
|
const store = new RedisDB({
|
||||||
|
redisUrl: REDIS_URL,
|
||||||
|
collectionName: COLLECTION_NAME,
|
||||||
|
embeddingModelDims: dims,
|
||||||
|
});
|
||||||
|
await store.initialize();
|
||||||
|
|
||||||
|
const id = uuidv4();
|
||||||
|
const vec = new Array(dims).fill(0.1);
|
||||||
|
|
||||||
|
await store.insert(
|
||||||
|
[vec],
|
||||||
|
[id],
|
||||||
|
[
|
||||||
|
{
|
||||||
|
data: "test",
|
||||||
|
hash: "h1",
|
||||||
|
createdAt: new Date().toISOString(),
|
||||||
|
},
|
||||||
|
],
|
||||||
|
);
|
||||||
|
|
||||||
|
const result = await store.get(id);
|
||||||
|
expect(result).not.toBeNull();
|
||||||
|
expect(result!.id).toBe(id);
|
||||||
|
|
||||||
|
await store.close();
|
||||||
|
});
|
||||||
|
});
|
||||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user