feat(ts-sdk): add OpenSearch vector store (#5810)

Co-authored-by: kartik-mem0 <kartik.labhshetwar@mem0.ai>
This commit is contained in:
AxelRay
2026-07-07 15:40:20 +05:30
committed by GitHub
co-authored by kartik-mem0
parent 002fe46ab3
commit 87276ef968
9 changed files with 886 additions and 3 deletions
+62 -3
View File
@@ -6,12 +6,18 @@ description: "Use OpenSearch as a vector database in Mem0 with k-NN search suppo
### Installation
OpenSearch support requires additional dependencies. Install them with:
OpenSearch support requires an additional client library. Install the one for your SDK:
```bash
<CodeGroup>
```bash Python
pip install opensearch-py
```
```bash TypeScript
npm install @opensearch-project/opensearch
```
</CodeGroup>
### Prerequisites
Before using OpenSearch with Mem0, you need to set up a collection in AWS OpenSearch Service.
@@ -26,7 +32,8 @@ You can create a collection through the AWS Console:
### Usage
```python
<CodeGroup>
```python Python
import os
from mem0 import Memory
import boto3
@@ -56,8 +63,43 @@ config = {
}
```
```typescript TypeScript
import { Memory } from 'mem0ai/oss';
// Basic self-hosted OpenSearch. For AWS OpenSearch Serverless, build an
// @opensearch-project/opensearch Client with AwsSigv4Signer and pass it as
// `client` instead of host/port/user/password.
const config = {
vectorStore: {
provider: 'opensearch',
config: {
collectionName: 'mem0',
embeddingModelDims: 1024,
host: 'localhost',
port: 9200,
user: 'admin',
password: 'admin',
useSSL: false,
verifyCerts: false,
},
},
};
const memory = new Memory(config);
const messages = [
{ role: "user", content: "I'm planning to watch a movie tonight. Any recommendations?" },
{ role: "assistant", content: "How about thriller movies? They can be quite engaging." },
{ role: "user", content: "I'm not a big fan of thriller movies but I love sci-fi movies." },
{ role: "assistant", content: "Got it! I'll avoid thriller recommendations and suggest sci-fi movies in the future." },
];
await memory.add(messages, { userId: "alice", metadata: { category: "movies" } });
```
</CodeGroup>
### Configuration Options
<Tabs>
<Tab title="Python">
| Parameter | Type | Default | Description |
|-----------|------|---------|-------------|
| `collection_name` | string | required | Name of the OpenSearch index |
@@ -68,6 +110,23 @@ config = {
| `use_ssl` | bool | False | Enable SSL/TLS connection |
| `verify_certs` | bool | False | Verify SSL certificates |
| `auto_refresh` | bool | False | Automatically refresh index after insert. OpenSearch refreshes every ~1 second by default, so this is rarely needed. |
</Tab>
<Tab title="TypeScript">
| Parameter | Type | Default | Description |
|-----------|------|---------|-------------|
| `collectionName` | string | required | Name of the OpenSearch index |
| `embeddingModelDims` | number | 1536 | Dimension of embedding vectors |
| `host` | string | `localhost` | OpenSearch endpoint host |
| `port` | number | 9200 | Port number |
| `httpAuth` | object | None | Authentication credentials, an object or `[user, password]` tuple |
| `user` | string | None | Username for basic auth (used together with `password`) |
| `password` | string | None | Password for basic auth (used together with `user`) |
| `useSSL` | boolean | false | Enable SSL/TLS connection |
| `verifyCerts` | boolean | false | Verify SSL certificates |
| `autoRefresh` | boolean | false | Refresh the index after each write so new memories are searchable immediately. Not supported on AWS Serverless. |
| `client` | object | None | Preconfigured OpenSearch client, e.g. one built with AwsSigv4Signer for AWS auth |
</Tab>
</Tabs>
<Note>
The defaults above match a local OpenSearch instance. The AWS OpenSearch Serverless
+1
View File
@@ -116,6 +116,7 @@
"@google/genai": "^1.40.0",
"@langchain/core": "^1.1.47",
"@mistralai/mistralai": "^1.5.2",
"@opensearch-project/opensearch": "^3.5.1",
"@pinecone-database/pinecone": "^8.0.0",
"@qdrant/js-client-rest": "^1.18.0",
"@supabase/supabase-js": "^2.49.1",
+40
View File
@@ -56,6 +56,9 @@ importers:
'@mistralai/mistralai':
specifier: ^1.5.2
version: 1.15.1
'@opensearch-project/opensearch':
specifier: ^3.5.1
version: 3.6.0
'@pinecone-database/pinecone':
specifier: ^8.0.0
version: 8.0.0
@@ -872,6 +875,10 @@ packages:
resolution: {integrity: sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==}
engines: {node: '>= 8'}
'@opensearch-project/opensearch@3.6.0':
resolution: {integrity: sha512-ow1A/Z7MBlNL/JZzTY4+M+8psiVh66g0Z7EQSl/qbT0U6zLBsVDYh6nvm8D4e4tIzQtn6ODbA2OsqYGvqZj76g==}
engines: {node: '>=14', yarn: ^1.22.10}
'@pinecone-database/pinecone@8.0.0':
resolution: {integrity: sha512-dItFqLdis2Pd5lC67aKn8HhvXajzlOz4+0RyK1CRcZMdSwG8YxUCBtH4yXPC8/6uLxfr2dvMWnRFuKgtPwwajQ==}
engines: {node: '>=20.0.0'}
@@ -1458,6 +1465,9 @@ packages:
resolution: {integrity: sha512-NZKeq9AfyQvEeNlN0zSYAaWrmBffJh3IELMZfRpJVWgrpEbtEpnjvzqBPf+mxoI287JohRDoa+/nsfqqiZmF6g==}
engines: {node: '>= 6.0.0'}
aws4@1.13.2:
resolution: {integrity: sha512-lHe62zvbTB5eEABUVi/AwVh0ZKY9rMMDhmm+eeyuuUQbQ3+J+fONVQOZyj+DdrvD4BY33uYniyRJ4UJIaSKAfw==}
axios@1.17.0:
resolution: {integrity: sha512-J8SwNxprqqpbfenehxWYXE7CW+wM1BB4w3+N+g+/Wx40xM4rsLrfPmHHxSWIxJLYDgSY/HqlFPIYb2/S3rxafw==}
@@ -2136,6 +2146,10 @@ packages:
resolution: {integrity: sha512-puUZAUKT5m8Zzvs72XWy3HtvVbTWljRE66cP60bxJzAqf2DgICo7lYTY2IHUmLnNpjYvw5bvmoHvPc0QO2a62w==}
engines: {node: ^16.14.0 || >=18.0.0}
hpagent@1.2.0:
resolution: {integrity: sha512-A91dYTeIB6NoXG+PxTQpCCDDnfHsW9kc06Lvpu1TEe9gnd6ZFeiBoRO9JvzEv6xK7EX97/dUE8g/vBMTqTS3CA==}
engines: {node: '>=14'}
html-escaper@2.0.2:
resolution: {integrity: sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg==}
@@ -2441,6 +2455,10 @@ packages:
json-stringify-safe@5.0.1:
resolution: {integrity: sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==}
json11@2.0.2:
resolution: {integrity: sha512-HIrd50UPYmP6sqLuLbFVm75g16o0oZrVfxrsY0EEys22klz8mRoWlX9KAEDOSOR9Q34rcxsyC8oDveGrCz5uLQ==}
hasBin: true
json5@2.2.3:
resolution: {integrity: sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==}
engines: {node: '>=6'}
@@ -3181,6 +3199,9 @@ packages:
safer-buffer@2.1.2:
resolution: {integrity: sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==}
secure-json-parse@2.7.0:
resolution: {integrity: sha512-6aU+Rwsezw7VR8/nyvKTx8QpWH9FrcYiXXlqC4z5d5XQBDRqtbfsRjnwGyqbi3gddNtWHuEk9OANUotL26qKUw==}
semver-compare@1.0.0:
resolution: {integrity: sha512-YM3/ITh2MJ5MtzaM429anh+x2jiLVjqILF4m4oyQB18W7Ggea7BfqdH/wGMK7dDiMghv/6WG7znWMwUDzJiXow==}
@@ -4765,6 +4786,17 @@ snapshots:
'@nodelib/fs.scandir': 2.1.5
fastq: 1.20.1
'@opensearch-project/opensearch@3.6.0':
dependencies:
aws4: 1.13.2
debug: 4.4.3(supports-color@5.5.0)
hpagent: 1.2.0
json11: 2.0.2
ms: 2.1.3
secure-json-parse: 2.7.0
transitivePeerDependencies:
- supports-color
'@pinecone-database/pinecone@8.0.0': {}
'@pkgjs/parseargs@0.11.0':
@@ -5307,6 +5339,8 @@ snapshots:
aws-ssl-profiles@1.1.2: {}
aws4@1.13.2: {}
axios@1.17.0:
dependencies:
follow-redirects: 1.16.0
@@ -6068,6 +6102,8 @@ snapshots:
dependencies:
lru-cache: 10.4.3
hpagent@1.2.0: {}
html-escaper@2.0.2: {}
http-proxy-agent@7.0.2:
@@ -6557,6 +6593,8 @@ snapshots:
json-stringify-safe@5.0.1: {}
json11@2.0.2: {}
json5@2.2.3: {}
jsonwebtoken@9.0.3:
@@ -7297,6 +7335,8 @@ snapshots:
safer-buffer@2.1.2: {}
secure-json-parse@2.7.0: {}
semver-compare@1.0.0: {}
semver@6.3.1: {}
+1
View File
@@ -15,6 +15,7 @@
"dependencies": {
"@anthropic-ai/sdk": "^0.18.0",
"@google/genai": "^0.7.0",
"@opensearch-project/opensearch": "^3.5.1",
"@qdrant/js-client-rest": "^1.13.0",
"@upstash/vector": "^1.2.3",
"@types/node": "^20.11.19",
+1
View File
@@ -39,4 +39,5 @@ export * from "./vector_stores/s3_vectors";
export * from "./vector_stores/vertex_ai_vector_search";
export * from "./vector_stores/pinecone";
export * from "./vector_stores/turbopuffer";
export * from "./vector_stores/opensearch";
export * from "./utils/factory";
+3
View File
@@ -43,6 +43,7 @@ import { LangchainEmbedder } from "../embeddings/langchain";
import { LangchainVectorStore } from "../vector_stores/langchain";
import { AzureAISearch } from "../vector_stores/azure_ai_search";
import { PGVector } from "../vector_stores/pgvector";
import { OpenSearchDB } from "../vector_stores/opensearch";
import { UpstashVector } from "../vector_stores/upstash_vector";
import { AzureMySQLDB } from "../vector_stores/azure_mysql";
import { VertexAIVectorSearch } from "../vector_stores/vertex_ai_vector_search";
@@ -140,6 +141,8 @@ export class VectorStoreFactory {
return new VertexAIVectorSearch(config as any);
case "pgvector":
return new PGVector(config as any);
case "opensearch":
return new OpenSearchDB(config as any);
case "upstash_vector":
return new UpstashVector(config as any);
case "azure_mysql":
@@ -0,0 +1,621 @@
import { Client } from "@opensearch-project/opensearch";
import { VectorStore } from "./base";
import { SearchFilters, VectorStoreConfig, VectorStoreResult } from "../types";
type OpenSearchAuth =
| {
username?: string;
password?: string;
}
| {
user?: string;
password?: string;
}
| Record<string, any>;
interface OpenSearchConfig extends VectorStoreConfig {
client?: Client;
host?: string;
port?: number;
httpAuth?: OpenSearchAuth | [string, string];
user?: string;
password?: string;
useSSL?: boolean;
verifyCerts?: boolean;
collectionName: string;
embeddingModelDims: number;
autoRefresh?: boolean;
}
interface OpenSearchHit {
_id?: string;
_score?: number;
_source?: {
id?: string;
vector_field?: number[];
payload?: Record<string, any>;
};
}
const KEY_MAP: Record<string, string> = {
$and: "AND",
$or: "OR",
$not: "NOT",
};
function responseBody<T>(response: T | { body: T }): T {
if (
response &&
typeof response === "object" &&
"body" in response &&
(response as { body: T }).body !== undefined
) {
return (response as { body: T }).body;
}
return response as T;
}
function escapeWildcard(value: string): string {
return value.replace(/([\\*?])/g, "\\$1");
}
export class OpenSearchDB implements VectorStore {
private client: Client;
private readonly collectionName: string;
private readonly embeddingModelDims: number;
private readonly autoRefresh: boolean;
private _initPromise?: Promise<void>;
constructor(config: OpenSearchConfig) {
this.collectionName = config.collectionName;
this.embeddingModelDims = config.embeddingModelDims;
this.autoRefresh = config.autoRefresh ?? false;
if (config.client) {
this.client = config.client;
} else {
const useSSL = config.useSSL ?? false;
const host = config.host || "localhost";
const port = config.port || 9200;
const auth =
config.httpAuth ||
(config.user && config.password
? { username: config.user, password: config.password }
: undefined);
this.client = new Client({
node: `${useSSL ? "https" : "http"}://${host}:${port}`,
auth: this.normalizeAuth(auth),
ssl: {
// Default false to match the Python SDK: self-hosted OpenSearch
// commonly uses self-signed certs, so verification is opt-in.
rejectUnauthorized: config.verifyCerts ?? false,
},
});
}
this.initialize().catch(console.error);
}
async initialize(): Promise<void> {
if (!this._initPromise) {
this._initPromise = this._doInitialize();
}
return this._initPromise;
}
private async _doInitialize(): Promise<void> {
await this.createCol(this.collectionName, this.embeddingModelDims);
await this.ensureMigrationIndex();
}
private normalizeAuth(auth?: OpenSearchAuth | [string, string]): any {
if (!auth) return undefined;
if (Array.isArray(auth)) {
return {
username: auth[0],
password: auth[1],
};
}
if ("user" in auth && auth.user) {
const { user, ...rest } = auth;
return {
...rest,
username: user,
};
}
return auth;
}
private async indexExists(index: string): Promise<boolean> {
const response = await this.client.indices.exists({ index });
return Boolean(responseBody<boolean>(response));
}
private async createCol(name: string, vectorSize: number): Promise<void> {
if (await this.indexExists(name)) return;
await this.client.indices.create({
index: name,
body: {
settings: {
index: {
knn: true,
},
},
mappings: {
properties: {
vector_field: {
type: "knn_vector",
dimension: vectorSize,
method: {
engine: "nmslib",
name: "hnsw",
space_type: "cosinesimil",
},
},
// payload is a dynamic object so that string sub-fields (user_id,
// agent_id, run_id, and any user metadata) each get an automatic
// `.keyword` sub-field, which the filter builder queries for exact
// match term/terms clauses. Mapping these as explicit `keyword`
// fields would remove the `.keyword` sub-field and make every
// scoped filter silently match nothing.
payload: { type: "object" },
id: { type: "keyword" },
},
},
},
});
}
private async ensureMigrationIndex(): Promise<void> {
if (await this.indexExists("memory_migrations")) return;
await this.client.indices.create({
index: "memory_migrations",
body: {
mappings: {
properties: {
user_id: { type: "keyword" },
},
},
},
});
}
private validateVector(vector: number[], index: number): void {
if (!vector) {
throw new Error(`Vector at index ${index} is null or undefined.`);
}
if (vector.length === 0) {
throw new Error(
`Vector at index ${index} is empty. Expected dimension ${this.embeddingModelDims}.`,
);
}
if (vector.length !== this.embeddingModelDims) {
throw new Error(
`Vector at index ${index} has dimension ${vector.length}, but index ` +
`'${this.collectionName}' expects dimension ${this.embeddingModelDims}.`,
);
}
}
async insert(
vectors: number[][],
ids: string[],
payloads: Record<string, any>[],
): Promise<void> {
vectors.forEach((vector, index) => this.validateVector(vector, index));
const operations = vectors.flatMap((vector, index) => {
const id = ids[index] || String(index);
return [
{ index: { _index: this.collectionName, _id: id } },
{
vector_field: vector,
payload: payloads[index] || {},
id,
},
];
});
if (operations.length === 0) return;
const response = responseBody<{ errors?: boolean; items?: any[] }>(
await this.client.bulk({
refresh: this.autoRefresh,
body: operations,
}),
);
if (response.errors) {
const failedItem = response.items?.find((item) => {
const action = item.index || item.create || item.update || item.delete;
return action?.error;
});
throw new Error(
`OpenSearch bulk insert failed: ${JSON.stringify(failedItem)}`,
);
}
}
async keywordSearch(
query: string,
topK: number = 5,
filters?: SearchFilters,
): Promise<VectorStoreResult[] | null> {
const boolQuery: Record<string, any> = {
should: [
{ match: { "payload.data": query } },
{ match: { "payload.text_lemmatized": query } },
],
minimum_should_match: 1,
};
const filter = this.buildFilterClauses(filters);
if (filter.length) {
boolQuery.filter = filter;
}
const response = responseBody<{ hits: { hits: OpenSearchHit[] } }>(
(await this.client.search({
index: this.collectionName,
body: {
size: topK,
query: { bool: boolQuery },
},
})) as any,
);
return response.hits.hits.map((hit) => this.hitToResult(hit));
}
async search(
query: number[],
topK: number = 5,
filters?: SearchFilters,
): Promise<VectorStoreResult[]> {
const knnQuery = {
knn: {
vector_field: {
vector: query,
k: topK * 2,
},
},
};
const filter = this.buildFilterClauses(filters);
const searchQuery = filter.length
? {
bool: {
must: knnQuery,
filter,
},
}
: knnQuery;
const response = responseBody<{ hits: { hits: OpenSearchHit[] } }>(
(await this.client.search({
index: this.collectionName,
body: {
size: topK * 2,
query: searchQuery,
},
})) as any,
);
return response.hits.hits
.slice(0, topK)
.map((hit) => this.hitToResult(hit));
}
async get(vectorId: string): Promise<VectorStoreResult | null> {
try {
const response = responseBody<{ _source?: OpenSearchHit["_source"] }>(
await this.client.get({
index: this.collectionName,
id: vectorId,
}),
);
if (!response._source) return null;
return {
id: response._source.id || vectorId,
payload: response._source.payload || {},
};
} catch (error: any) {
if (error?.statusCode === 404 || error?.meta?.statusCode === 404) {
return null;
}
throw error;
}
}
async update(
vectorId: string,
vector: number[],
payload: Record<string, any>,
): Promise<void> {
if (vector) {
this.validateVector(vector, 0);
}
await this.client.update({
index: this.collectionName,
id: vectorId,
body: {
doc: {
...(vector && { vector_field: vector }),
...(payload && { payload }),
id: vectorId,
},
},
refresh: this.autoRefresh,
});
}
async delete(vectorId: string): Promise<void> {
try {
await this.client.delete({
index: this.collectionName,
id: vectorId,
refresh: this.autoRefresh,
});
} catch (error: any) {
if (error?.statusCode === 404 || error?.meta?.statusCode === 404) {
return;
}
throw error;
}
}
async deleteCol(): Promise<void> {
if (!(await this.indexExists(this.collectionName))) return;
await this.client.indices.delete({ index: this.collectionName });
}
async list(
filters?: SearchFilters,
topK: number = 100,
): Promise<[VectorStoreResult[], number]> {
const filter = this.buildFilterClauses(filters);
const query = filter.length ? { bool: { filter } } : { match_all: {} };
const response = responseBody<{
hits: {
total?: number | { value: number };
hits: OpenSearchHit[];
};
}>(
(await this.client.search({
index: this.collectionName,
body: {
size: topK,
query,
},
})) as any,
);
const results = response.hits.hits.map((hit) => this.hitToResult(hit));
const total =
typeof response.hits.total === "number"
? response.hits.total
: response.hits.total?.value || results.length;
return [results, total];
}
async reset(): Promise<void> {
await this.deleteCol();
await this.createCol(this.collectionName, this.embeddingModelDims);
}
async getUserId(): Promise<string> {
await this.ensureMigrationIndex();
const response = responseBody<{ hits: { hits: OpenSearchHit[] } }>(
(await this.client.search({
index: "memory_migrations",
body: {
size: 1,
query: { match_all: {} },
},
})) as any,
);
const existing = response.hits.hits[0]?._source?.payload?.user_id;
if (existing) return String(existing);
const userId =
Math.random().toString(36).substring(2, 15) +
Math.random().toString(36).substring(2, 15);
await this.setUserId(userId);
return userId;
}
async setUserId(userId: string): Promise<void> {
await this.ensureMigrationIndex();
await this.client.index({
index: "memory_migrations",
id: "user_id",
body: {
user_id: userId,
payload: { user_id: userId },
},
refresh: this.autoRefresh,
});
}
private hitToResult(hit: OpenSearchHit): VectorStoreResult {
return {
id: hit._source?.id || hit._id || "",
payload: hit._source?.payload || {},
score: hit._score,
};
}
private buildFilterClauses(filters?: SearchFilters): any[] {
if (!filters || Object.keys(filters).length === 0) return [];
const normalized: Record<string, any> = {};
for (const [key, value] of Object.entries(filters)) {
normalized[KEY_MAP[key] || key] = value;
}
return Object.entries(normalized)
.map(([key, value]) => this.buildFilterClause(key, value))
.filter((clause): clause is Record<string, any> => Boolean(clause));
}
private buildFilterClause(
key: string,
value: any,
): Record<string, any> | null {
if (value === null || value === undefined) return null;
if (key === "AND" || key === "OR" || key === "NOT") {
if (!Array.isArray(value)) {
throw new Error(`${key} filter value must be an array.`);
}
const clauses = value
.flatMap((filter) => this.buildFilterClauses(filter))
.filter(Boolean);
if (clauses.length === 0) return null;
if (key === "AND") {
return { bool: { filter: clauses } };
}
if (key === "OR") {
return { bool: { should: clauses, minimum_should_match: 1 } };
}
return { bool: { must_not: clauses } };
}
if (value === "*") {
return { exists: { field: this.payloadField(key, false) } };
}
if (Array.isArray(value)) {
value.forEach((item) => this.assertScalarValue(key, item));
return { terms: { [this.payloadField(key, true)]: value } };
}
if (typeof value === "object") {
return this.buildOperatorClause(key, value);
}
return {
term: { [this.payloadField(key, typeof value === "string")]: value },
};
}
private buildOperatorClause(
key: string,
value: Record<string, any>,
): Record<string, any> {
const clauses = Object.entries(value).map(([operator, operatorValue]) => {
switch (operator) {
case "eq":
this.assertScalarValue(key, operatorValue);
return {
term: {
[this.payloadField(key, typeof operatorValue === "string")]:
operatorValue,
},
};
case "ne":
this.assertScalarValue(key, operatorValue);
return {
bool: {
must_not: [
{
term: {
[this.payloadField(key, typeof operatorValue === "string")]:
operatorValue,
},
},
],
},
};
case "in":
this.assertScalarArray(key, operatorValue);
return { terms: { [this.payloadField(key, true)]: operatorValue } };
case "nin":
this.assertScalarArray(key, operatorValue);
return {
bool: {
must_not: [
{ terms: { [this.payloadField(key, true)]: operatorValue } },
],
},
};
case "gt":
case "gte":
case "lt":
case "lte":
this.assertScalarValue(key, operatorValue);
return {
range: {
[this.payloadField(key, false)]: {
[operator]: operatorValue,
},
},
};
case "contains":
case "icontains":
this.assertScalarValue(key, operatorValue);
return {
wildcard: {
[this.payloadField(key, true)]: {
value: `*${escapeWildcard(String(operatorValue))}*`,
case_insensitive: operator === "icontains",
},
},
};
default:
throw new Error(`Unsupported filter operator: ${operator}`);
}
});
return clauses.length === 1 ? clauses[0] : { bool: { filter: clauses } };
}
private payloadField(key: string, keyword: boolean): string {
if (key.startsWith("payload.")) {
return keyword && !key.endsWith(".keyword") ? `${key}.keyword` : key;
}
const field = `payload.${key}`;
return keyword ? `${field}.keyword` : field;
}
// Filter values become OpenSearch term/terms/range/wildcard leaves. Allowing
// an object here lets a caller inject raw query parameters (e.g. a `term`
// object form with `boost`/`case_insensitive`), so reject non-scalar leaves.
// Mirrors the Python SDK's `_validate_filter` scalar allow-list (PR #5986).
private assertScalarValue(key: string, value: any): void {
if (value !== null && typeof value === "object") {
throw new Error(
`Filter value for '${key}' must be a string, number, or boolean, got ` +
`${Array.isArray(value) ? "an array" : "an object"}.`,
);
}
}
private assertScalarArray(key: string, value: any): void {
if (!Array.isArray(value)) {
throw new Error(`Filter value for '${key}' must be an array.`);
}
value.forEach((item) => this.assertScalarValue(key, item));
}
}
@@ -0,0 +1,156 @@
import { OpenSearchDB } from "../src/vector_stores/opensearch";
// These tests run entirely against a mocked OpenSearch client (no live cluster).
// They lock the index mapping and the filter query paths together: `payload` must
// stay a dynamic object so that a `.keyword` sub-field exists for the exact-match
// term clauses the filter builder emits. Mapping payload sub-keys as explicit
// `keyword` fields removes that sub-field and makes every scoped filter match
// nothing, so the createCol + search assertions here guard that regression.
describe("OpenSearchDB", () => {
const collectionName = "memories";
const dims = 3;
function createClient(overrides: Record<string, any> = {}) {
const { indices: indicesOverride, ...rest } = overrides;
return {
indices: {
exists: jest.fn().mockResolvedValue(false),
create: jest.fn().mockResolvedValue({ body: { acknowledged: true } }),
delete: jest.fn().mockResolvedValue({ body: { acknowledged: true } }),
...(indicesOverride || {}),
},
bulk: jest.fn().mockResolvedValue({ body: { errors: false, items: [] } }),
search: jest.fn().mockResolvedValue({
body: { hits: { hits: [], total: { value: 0 } } },
}),
get: jest.fn().mockResolvedValue({ body: { _source: null } }),
update: jest.fn().mockResolvedValue({ body: {} }),
delete: jest.fn().mockResolvedValue({ body: {} }),
index: jest.fn().mockResolvedValue({ body: {} }),
...rest,
};
}
async function createStore(client: any) {
const store = new OpenSearchDB({
collectionName,
embeddingModelDims: dims,
client: client as any,
});
await store.initialize();
return store;
}
it("maps payload as a dynamic object and enables knn without a slow refresh override", async () => {
const client = createClient();
await createStore(client);
const createCall = client.indices.create.mock.calls.find(
([arg]: any[]) => arg.index === collectionName,
);
expect(createCall).toBeDefined();
const body = createCall[0].body;
// Dynamic payload object is what gives user_id/agent_id/run_id their
// `.keyword` sub-field. Enumerated keyword sub-properties would break filters.
expect(body.mappings.properties.payload).toEqual({ type: "object" });
expect(body.mappings.properties.payload.properties).toBeUndefined();
// No dead metadata mirror of payload.
expect(body.mappings.properties.metadata).toBeUndefined();
// knn on, and no 10s refresh_interval that would hide freshly added memories.
expect(body.settings.index.knn).toBe(true);
expect(body.settings.index.refresh_interval).toBeUndefined();
});
it("inserts payload and id without a duplicate metadata field", async () => {
const client = createClient();
const store = await createStore(client);
await store.insert(
[[0.1, 0.2, 0.3]],
["mem-1"],
[{ user_id: "alice", data: "Alice likes pizza" }],
);
expect(client.bulk).toHaveBeenCalledTimes(1);
const operations = client.bulk.mock.calls[0][0].body;
const [action, doc] = operations;
expect(action).toEqual({ index: { _index: collectionName, _id: "mem-1" } });
expect(doc.payload).toEqual({
user_id: "alice",
data: "Alice likes pizza",
});
expect(doc.id).toBe("mem-1");
// No dead metadata/text mirror fields (Python live insert writes neither).
expect(doc).not.toHaveProperty("metadata");
expect(doc).not.toHaveProperty("text");
});
it("scopes search by user_id via the payload.<key>.keyword term path", async () => {
const client = createClient();
const store = await createStore(client);
await store.search([0.1, 0.2, 0.3], 5, { user_id: "alice" });
const searchCall = client.search.mock.calls.find(
([arg]: any[]) => arg.index === collectionName,
);
expect(searchCall).toBeDefined();
const filter = searchCall[0].body.query.bool.filter;
expect(filter).toContainEqual({
term: { "payload.user_id.keyword": "alice" },
});
});
it("lists filtered results with a total count", async () => {
const client = createClient({
search: jest.fn().mockResolvedValue({
body: {
hits: {
total: { value: 1 },
hits: [
{
_id: "mem-1",
_score: 1,
_source: { id: "mem-1", payload: { user_id: "alice" } },
},
],
},
},
}),
});
const store = await createStore(client);
const [results, total] = await store.list({ user_id: "alice" });
expect(total).toBe(1);
expect(results).toEqual([
{ id: "mem-1", payload: { user_id: "alice" }, score: 1 },
]);
});
it("rejects object filter values that could inject query parameters", async () => {
const client = createClient();
const store = await createStore(client);
// A term query accepts an object form ({value, boost, case_insensitive}),
// so an object leaf would let a caller inject raw query params. Guarded.
await expect(
store.search([0.1, 0.2, 0.3], 5, {
user_id: { eq: { boost: 999, value: "x" } },
} as any),
).rejects.toThrow(/must be a string, number, or boolean/);
expect(client.search).not.toHaveBeenCalled();
});
it("returns null from get when the document is missing (404)", async () => {
const client = createClient({
get: jest.fn().mockRejectedValue({ statusCode: 404 }),
});
const store = await createStore(client);
await expect(store.get("missing")).resolves.toBeNull();
});
});
+1
View File
@@ -31,6 +31,7 @@ const external = [
"natural",
"mysql2",
"@turbopuffer/turbopuffer",
"@opensearch-project/opensearch",
];
const define = {