diff --git a/docs/components/vectordbs/dbs/oracledb.mdx b/docs/components/vectordbs/dbs/oracledb.mdx index 0f8367bc6..178414555 100644 --- a/docs/components/vectordbs/dbs/oracledb.mdx +++ b/docs/components/vectordbs/dbs/oracledb.mdx @@ -8,12 +8,18 @@ description: "Use Oracle Database AI Vector Search as a vector store in Mem0 for ### Requirements - Oracle Database 23.4 or later, with a user that can create tables and vector indexes -- The `python-oracledb` driver. In thick mode, Oracle Client 23.4 or later is also required. +- The `python-oracledb` or `node-oracledb` driver. In thick mode, Oracle Client 23.4 or later is also required. -```bash + +```bash Python pip install oracledb ``` +```bash TypeScript +npm install oracledb +``` + + ### Usage @@ -47,11 +53,58 @@ messages = [ ] m.add(messages, user_id="alice", metadata={"category": "movies"}) ``` + +```typescript TypeScript +import { Memory } from "mem0ai/oss"; + +const config = { + vectorStore: { + provider: "oracledb", + config: { + collectionName: "mem0", + embeddingModelDims: 1536, + connectionParams: { + user: "mem0_user", + password: "your-password", + connectString: "localhost:1521/FREEPDB1", + }, + }, + }, +}; + +const memory = new Memory(config); + +const messages = [ + { + role: "user", + content: "I'm planning to watch a movie tonight. Any recommendations?", + }, + { + role: "assistant", + content: "How about thriller movies? They can be quite engaging.", + }, + { + role: "user", + content: "I'm not a big fan of thriller movies but I love sci-fi movies.", + }, + { + role: "assistant", + content: + "Got it! I'll avoid thriller recommendations and suggest sci-fi movies in the future.", + }, +]; + +await memory.add(messages, { + userId: "alice", + metadata: { category: "movies" }, +}); +``` -To reuse a connection or pool you already manage, pass it as `client` instead of `connection_params`: +To reuse a connection or pool you already manage, pass it as `client` instead of the connection parameters: -```python + +```python Python import oracledb pool = oracledb.create_pool(user="mem0_user", password="your-password", dsn="localhost:1521/FREEPDB1") @@ -64,33 +117,52 @@ config = { } ``` +```typescript TypeScript +import oracledb from "oracledb"; + +const pool = await oracledb.createPool({ + user: "mem0_user", + password: "your-password", + connectString: "localhost:1521/FREEPDB1", +}); + +const config = { + vectorStore: { + provider: "oracledb", + config: { client: pool }, + }, +}; +``` + + ### Config Here are the parameters available for configuring Oracle AI Vector Search: -| Parameter | Description | Default Value | -| --- | --- | --- | -| `connection_params` | Connection settings passed to `python-oracledb`, such as `user`, `password` and `dsn`. See the [connection handling guide](https://python-oracledb.readthedocs.io/en/latest/user_guide/connection_handling.html). | `None` | -| `use_connection_pool` | Create a connection pool from `connection_params` instead of a single connection | `True` | -| `client` | An existing `oracledb.Connection` or `oracledb.ConnectionPool` to use instead of building one from `connection_params` | `None` | -| `collection_name` | Name of the Oracle table that stores vectors and payloads | `mem0` | -| `embedding_model_dims` | Dimension of your embedding vectors, must be greater than 0 | `1536` | -| `distance_metric` | Distance function used for indexing and search: `COSINE`, `EUCLIDEAN`, `EUCLIDEAN_SQUARED`, `DOT`, `HAMMING` or `MANHATTAN` | `COSINE` | -| `do_create_index` | Whether to create a vector index on the collection | `True` | -| `index_type` | Vector index type: `HNSW` or `IVF` | `HNSW` | -| `index_name` | Name of the vector index | `_VEC_IDX` | -| `index_parameters` | Index tuning parameters. For `HNSW`: `neighbors`, `efconstruction`. For `IVF`: `neighbor partitions`, `samples_per_partition`, `min_vectors_per_partition`. | `None` | -| `index_accuracy` | Target index accuracy from 1 to 100, applied as `WITH TARGET ACCURACY ` | `None` | +| Python | TypeScript | Description | Default Value | +| --- | --- | --- | --- | +| `connection_params` | `connectionParams` | Connection settings passed to the Oracle driver, such as `user`, `password` and `dsn` (`connectString` in TypeScript). See the [Python](https://python-oracledb.readthedocs.io/en/latest/user_guide/connection_handling.html) or [Node.js](https://node-oracledb.readthedocs.io/en/latest/user_guide/connection_handling.html) connection handling guide. | `None` | +| `use_connection_pool` | `useConnectionPool` | Create a connection pool from the connection parameters instead of a single connection | `True` | +| `client` | `client` | An existing Oracle connection or pool to use instead of building one from the connection parameters | `None` | +| `collection_name` | `collectionName` | Name of the Oracle table that stores vectors and payloads | `mem0` | +| `embedding_model_dims` | `embeddingModelDims` | Dimension of your embedding vectors, must be greater than 0 | `1536` | +| `distance_metric` | `distanceMetric` | Distance function used for indexing and search: `COSINE`, `EUCLIDEAN`, `EUCLIDEAN_SQUARED`, `DOT`, `HAMMING` or `MANHATTAN` | `COSINE` | +| `do_create_index` | `doCreateIndex` | Whether to create a vector index on the collection | `True` | +| `index_type` | `indexType` | Vector index type: `HNSW` or `IVF` | `HNSW` | +| `index_name` | `indexName` | Name of the vector index | `_VEC_IDX` | +| `index_parameters` | `indexParameters` | Index tuning parameters. For `HNSW`: `neighbors`, `efconstruction`. For `IVF`: `neighbor partitions`, `samples_per_partition`, `min_vectors_per_partition`. | `None` | +| `index_accuracy` | `indexAccuracy` | Target index accuracy from 1 to 100, applied as `WITH TARGET ACCURACY ` | `None` | - When you pass a pre-built `client`, Mem0 uses it as-is and ignores `connection_params` and `use_connection_pool`. Mem0 does not close a client it did not create. + When you pass a pre-built `client`, Mem0 uses it as-is and ignores the connection parameters and pooling options. Mem0 does not close a client it did not create. ### Vector indexes Set the index type with `index_type` and tune it with `index_parameters`: -```python + +```python Python config = { "vector_store": { "provider": "oracledb", @@ -104,6 +176,25 @@ config = { } ``` +```typescript TypeScript +const config = { + vectorStore: { + provider: "oracledb", + config: { + connectionParams: { + user: "mem0_user", + password: "your-password", + connectString: "localhost:1521/FREEPDB1", + }, + indexType: "HNSW", + indexParameters: { neighbors: 32, efconstruction: 200 }, + indexAccuracy: 95, + }, + }, +}; +``` + + For the full list of supported options, see the Oracle [`CREATE VECTOR INDEX`](https://docs.oracle.com/en/database/oracle/oracle-database/26/sqlrf/create-vector-index.html) reference. ### Search scores @@ -121,14 +212,23 @@ Filters run against the JSON `payload` column and support: | Comparison | `{"score": {"gte": 0.5}}`, also `eq`, `ne`, `gt`, `lt`, `lte` | | Membership | `{"category": {"in": ["movies", "books"]}}`, also `nin` | | String matching | `{"title": {"contains": "sci-fi"}}`, also `icontains` for case-insensitive | -| Logical groups | `{"AND": [...]}`, `{"OR": [...]}`, `{"NOT": [...]}` | +| Logical groups | `{"AND": [...]}`, `{"OR": [...]}`, `{"NOT": [...]}`, also `$and`, `$or`, `$not` | Multiple fields at the top level are combined with `AND`: -```python + +```python Python m.search( "movie recommendations", user_id="alice", filters={"category": {"in": ["movies", "books"]}, "rating": {"gte": 4}}, ) ``` + +```typescript TypeScript +await memory.search("movie recommendations", { + userId: "alice", + filters: { category: { in: ["movies", "books"] }, rating: { gte: 4 } }, +}); +``` + diff --git a/mem0-ts/package.json b/mem0-ts/package.json index 31e14d743..75bdb3a93 100644 --- a/mem0-ts/package.json +++ b/mem0-ts/package.json @@ -86,6 +86,7 @@ "license": "Apache-2.0", "devDependencies": { "@types/better-sqlite3": "^7.6.13", + "@types/oracledb": "^7.0.1", "@types/node": "^22.7.6", "@types/uuid": "^9.0.8", "dotenv": "^16.4.5", @@ -139,6 +140,7 @@ "mongodb": "^7.0.0", "weaviate-client": "^3.0.0", "ollama": "^0.5.14", + "oracledb": "^6.5.0 || ^7.0.0", "pg": "8.11.3", "redis": "^4.6.13", "@elastic/elasticsearch": "^9.0.0", @@ -252,6 +254,9 @@ }, "iovalkey": { "optional": true + }, + "oracledb": { + "optional": true } }, "engines": { diff --git a/mem0-ts/pnpm-lock.yaml b/mem0-ts/pnpm-lock.yaml index 4b997496f..bafe9be1a 100644 --- a/mem0-ts/pnpm-lock.yaml +++ b/mem0-ts/pnpm-lock.yaml @@ -153,6 +153,9 @@ importers: openai: specifier: ^4.93.0 version: 4.104.0(ws@5.2.5)(zod@3.25.76) + oracledb: + specifier: ^6.5.0 || ^7.0.0 + version: 7.0.1 pg: specifier: 8.11.3 version: 8.11.3 @@ -181,6 +184,9 @@ importers: '@types/node': specifier: ^22.7.6 version: 22.19.21 + '@types/oracledb': + specifier: ^7.0.1 + version: 7.0.1 '@types/uuid': specifier: ^9.0.8 version: 9.0.8 @@ -1911,6 +1917,9 @@ packages: '@types/normalize-package-data@2.4.4': resolution: {integrity: sha512-37i+OaWTh9qeK4LSHPsyRC7NahnGotNuZvjLSgcPzblpHB3rrCJxAOgI5gCdKm7coonsaX1Of0ILiTcnZjbfxA==} + '@types/oracledb@7.0.1': + resolution: {integrity: sha512-0A6m9YE4yu73KXehr5D6cbyALUzZoANGY4bG5cAPQIpJoJG4eMVPbR1Vau8ycnC25V/+F4Au1pdFSy8ODOAD0w==} + '@types/pad-left@2.1.1': resolution: {integrity: sha512-Xd22WCRBydkGSApl5Bw0PhAOHKSVjNL3E3AwzKaps96IMraPqy5BvZIsBVK6JLwdybUzjHnuWVwpDd0JjTfHXA==} @@ -3796,6 +3805,10 @@ packages: openid-client@5.7.1: resolution: {integrity: sha512-jDBPgSVfTnkIh71Hg9pRvtJc6wTwqjRkN88+gCFtYWrlP4Yx2Dsrow8uPi3qLr/aeymPF3o2+dS+wOpglK04ew==} + oracledb@7.0.1: + resolution: {integrity: sha512-xlM0Ceh6A5stQLAdEfKf3pgCSkbOjQLo2ZPEi3+kXklz+KbZD3fLi/nsTSbQeZZNFBSFDNxdn1Ek3+bxG40M8w==} + engines: {node: '>=14.17'} + p-finally@1.0.0: resolution: {integrity: sha512-LICb2p9CB7FS+0eR1oqWnHhp0FljGLZCWBE9aix0Uye9W8LTQPwMTYVGWQWIw9RdQiDg4+epXQODwIYJtSJaow==} engines: {node: '>=4'} @@ -7247,6 +7260,10 @@ snapshots: '@types/normalize-package-data@2.4.4': {} + '@types/oracledb@7.0.1': + dependencies: + '@types/node': 22.19.21 + '@types/pad-left@2.1.1': {} '@types/pg@8.11.0': @@ -9349,6 +9366,8 @@ snapshots: object-hash: 2.2.0 oidc-token-hash: 5.2.0 + oracledb@7.0.1: {} + p-finally@1.0.0: {} p-limit@2.3.0: diff --git a/mem0-ts/src/oss/src/index.ts b/mem0-ts/src/oss/src/index.ts index 848f9d431..c780a2edf 100644 --- a/mem0-ts/src/oss/src/index.ts +++ b/mem0-ts/src/oss/src/index.ts @@ -51,6 +51,7 @@ export * from "./vector_stores/milvus"; export * from "./vector_stores/mongodb"; export * from "./vector_stores/opensearch"; export * from "./vector_stores/weaviate"; +export * from "./vector_stores/oracledb"; export * from "./rerankers/base"; export * from "./rerankers/cohere"; export * from "./rerankers/llm"; diff --git a/mem0-ts/src/oss/src/utils/factory.ts b/mem0-ts/src/oss/src/utils/factory.ts index 5100867dc..16de0301f 100644 --- a/mem0-ts/src/oss/src/utils/factory.ts +++ b/mem0-ts/src/oss/src/utils/factory.ts @@ -71,6 +71,7 @@ import { TurbopufferDB } from "../vector_stores/turbopuffer"; import { Milvus } from "../vector_stores/milvus"; import { MongoDB } from "../vector_stores/mongodb"; import { WeaviateDB } from "../vector_stores/weaviate"; +import { OracleAIVectorSearch } from "../vector_stores/oracledb"; export class EmbedderFactory { static create(provider: string, config: EmbeddingConfig): Embedder { @@ -205,6 +206,8 @@ export class VectorStoreFactory { return new MongoDB(config as any); case "weaviate": return new WeaviateDB(config as any); + case "oracledb": + return new OracleAIVectorSearch(config as any); default: throw new Error(`Unsupported vector store provider: ${provider}`); } diff --git a/mem0-ts/src/oss/src/vector_stores/oracledb.ts b/mem0-ts/src/oss/src/vector_stores/oracledb.ts new file mode 100644 index 000000000..51c4c0568 --- /dev/null +++ b/mem0-ts/src/oss/src/vector_stores/oracledb.ts @@ -0,0 +1,741 @@ +import type { Connection, Pool } from "oracledb"; +import { v4 as uuidv4 } from "uuid"; +import { VectorStore } from "./base"; +import { SearchFilters, VectorStoreConfig, VectorStoreResult } from "../types"; +import { loadPeer } from "../utils/load_peer"; + +const DISTANCE_METRICS = [ + "COSINE", + "EUCLIDEAN", + "EUCLIDEAN_SQUARED", + "DOT", + "HAMMING", + "MANHATTAN", +] as const; + +type DistanceMetric = (typeof DISTANCE_METRICS)[number]; +type IndexType = "HNSW" | "IVF"; + +const SCORE_FROM_DISTANCE: Record number> = { + COSINE: (d) => Math.max(0, Math.min(1, 1 - d)), + EUCLIDEAN: (d) => 1 / (1 + Math.max(0, d)), + EUCLIDEAN_SQUARED: (d) => 1 / (1 + Math.sqrt(Math.max(0, d))), + HAMMING: (d) => 1 / (1 + Math.max(0, d)), + MANHATTAN: (d) => 1 / (1 + Math.max(0, d)), + DOT: (d) => -d, +}; + +const INDEX_PARAMETER_RANGES: Record< + IndexType, + Record +> = { + HNSW: { + neighbors: [2, 2048], + efconstruction: [1, 65535], + }, + IVF: { + "neighbor partitions": [1, 10_000_000], + samples_per_partition: [1, Number.MAX_SAFE_INTEGER], + min_vectors_per_partition: [0, Number.MAX_SAFE_INTEGER], + }, +}; + +const IDENTIFIER_RE = /^(?:"[^"]+"|[^".]+)(?:\.(?:"[^"]+"|[^".]+))*$/; +const METADATA_KEY_RE = /^[a-zA-Z0-9_.[\],\s*]+$/; + +export function quoteIdentifier(name: string): string { + const trimmed = name.trim(); + if (!IDENTIFIER_RE.test(trimmed)) { + throw new Error(`Identifier name ${name} is not valid.`); + } + return [...trimmed.matchAll(/"([^"]+)"|([^".]+)/g)] + .map((m) => `"${m[1] ?? m[2]}"`) + .join("."); +} + +function jsonPath(metadataKey: string): string { + if (!METADATA_KEY_RE.test(metadataKey)) { + throw new Error( + `Invalid metadata key '${metadataKey}'. Only letters, numbers, underscores, ` + + `nesting via '.', and array wildcards '[*]' are allowed.`, + ); + } + return metadataKey + .split(".") + .map((part) => + part.endsWith("[*]") ? `."${part.slice(0, -3)}"[*]` : `."${part}"`, + ) + .join(""); +} + +const COMPARISON_OPERATORS: Record = { + eq: "==", + ne: "!=", + gt: ">", + gte: ">=", + lt: "<", + lte: "<=", +}; + +const FIELD_OPERATORS = new Set([ + ...Object.keys(COMPARISON_OPERATORS), + "in", + "nin", + "contains", + "icontains", +]); + +const LOGICAL_OPERATORS: Record = { + $and: "and", + $or: "or", + $not: "not", + AND: "and", + OR: "or", + NOT: "not", +}; + +function isScalar(value: any): boolean { + return value === null || (typeof value !== "object" && !Array.isArray(value)); +} + +function bindFilterValue( + value: any, + binds: Record, +): [string, string] { + const name = `f_${Object.keys(binds).length}`; + binds[name] = value; + return [`$${name}`, `:${name} AS "${name}"`]; +} + +function jsonExists( + path: string, + predicate: string, + passings: string[], +): string { + const passingClause = + passings.length > 0 ? ` PASSING ${passings.join(", ")}` : ""; + return `JSON_EXISTS(payload, '$${path}?(${predicate})'${passingClause})`; +} + +function buildFieldCondition( + metadataKey: string, + value: any, + binds: Record, +): string { + const path = jsonPath(metadataKey); + + if (value === "*") { + return `JSON_EXISTS(payload, '$${path}')`; + } + + if (isScalar(value)) { + if (value === null) { + return jsonExists(path, "@ == null", []); + } + const [variable, passing] = bindFilterValue(value, binds); + return jsonExists(path, `@ == ${variable}`, [passing]); + } + + if (Array.isArray(value)) { + throw new Error( + `Oracle filter for field '${metadataKey}' must be a scalar or an operator object`, + ); + } + + const operators = Object.entries(value); + if (operators.length === 0) { + throw new Error( + `Operator filter for field '${metadataKey}' must not be empty`, + ); + } + + const unsupported = operators + .map(([op]) => op) + .filter((op) => !FIELD_OPERATORS.has(op)); + if (unsupported.length > 0) { + throw new Error( + `Unsupported Oracle filter operator(s) for field '${metadataKey}': ${unsupported.sort().join(", ")}`, + ); + } + + const predicates: string[] = []; + const passings: string[] = []; + const additionalClauses: string[] = []; + + for (const [operator, operand] of operators) { + if (operator in COMPARISON_OPERATORS) { + if (!isScalar(operand)) { + throw new Error( + `Oracle filter operator '${operator}' requires a scalar value`, + ); + } + if (operand === null) { + if (operator !== "eq" && operator !== "ne") { + throw new Error( + `Oracle filter operator '${operator}' does not support null`, + ); + } + predicates.push(`@ ${COMPARISON_OPERATORS[operator]} null`); + continue; + } + const [variable, passing] = bindFilterValue(operand, binds); + predicates.push(`@ ${COMPARISON_OPERATORS[operator]} ${variable}`); + passings.push(passing); + continue; + } + + if (operator === "in" || operator === "nin") { + if (!Array.isArray(operand) || operand.length === 0) { + throw new Error( + `Oracle filter operator '${operator}' requires a non-empty array`, + ); + } + + const variables: string[] = []; + const listPassings: string[] = []; + for (const item of operand) { + if (!isScalar(item)) { + throw new Error( + `Oracle filter operator '${operator}' requires scalar values`, + ); + } + if (item === null) { + variables.push("null"); + continue; + } + const [variable, passing] = bindFilterValue(item, binds); + variables.push(variable); + listPassings.push(passing); + } + + const membership = jsonExists( + path, + `@ in (${variables.join(", ")})`, + listPassings, + ); + additionalClauses.push( + operator === "in" ? membership : `NOT (${membership})`, + ); + continue; + } + + if (typeof operand !== "string") { + throw new Error( + `Oracle filter operator '${operator}' requires a string value`, + ); + } + + if (operator === "contains") { + const [variable, passing] = bindFilterValue(operand, binds); + predicates.push(`@ has substring ${variable}`); + passings.push(passing); + } else { + const [variable, passing] = bindFilterValue(operand.toLowerCase(), binds); + predicates.push(`@.lower() has substring ${variable}`); + passings.push(passing); + } + } + + const clauses = [...additionalClauses]; + if (predicates.length > 0) { + clauses.unshift(jsonExists(path, predicates.join(" && "), passings)); + } + + return clauses.length === 1 ? clauses[0] : `(${clauses.join(" AND ")})`; +} + +export function buildFilterGroup( + filters: Record, + binds: Record, +): string { + const entries = Object.entries(filters ?? {}); + if (entries.length === 0) { + throw new Error("Oracle filter groups must be non-empty objects"); + } + + const clauses: string[] = []; + for (const [key, value] of entries) { + const logicalOperator = LOGICAL_OPERATORS[key]; + if (logicalOperator) { + if (!Array.isArray(value) || value.length === 0) { + throw new Error( + `Logical filter operator '${key}' requires a non-empty array`, + ); + } + const nested = value.map((condition) => + buildFilterGroup(condition, binds), + ); + if (logicalOperator === "not") { + clauses.push(`NOT (${nested.join(" OR ")})`); + } else { + clauses.push( + `(${nested.join(logicalOperator === "and" ? " AND " : " OR ")})`, + ); + } + continue; + } + + if (key.startsWith("$")) { + throw new Error(`Unsupported Oracle logical filter operator: ${key}`); + } + + clauses.push(buildFieldCondition(key, value, binds)); + } + + return clauses.length === 1 ? clauses[0] : `(${clauses.join(" AND ")})`; +} + +export function buildWhereClause( + filters?: SearchFilters, +): [string, Record] { + if (!filters || Object.keys(filters).length === 0) { + return ["", {}]; + } + const binds: Record = {}; + return [`WHERE ${buildFilterGroup(filters, binds)}`, binds]; +} + +interface OracleDBConfig extends VectorStoreConfig { + connectionParams?: Record; + useConnectionPool?: boolean; + client?: Connection | Pool; + collectionName?: string; + embeddingModelDims?: number; + distanceMetric?: DistanceMetric; + doCreateIndex?: boolean; + indexType?: IndexType; + indexName?: string; + indexParameters?: Record; + indexAccuracy?: number; +} + +export class OracleAIVectorSearch implements VectorStore { + private readonly collectionName: string; + private readonly indexName: string; + private readonly embeddingModelDims: number; + private readonly distanceMetric: DistanceMetric; + private readonly indexType: IndexType; + private readonly indexParameters: Record; + private readonly indexAccuracy?: number; + private readonly doCreateIndex: boolean; + private readonly config: OracleDBConfig; + private oracledb: any; + private client?: Connection | Pool; + private ownsClient = false; + private _initPromise?: Promise; + + constructor(config: OracleDBConfig) { + if (!config.connectionParams && !config.client) { + throw new Error( + "Must provide at least one of `connectionParams` and `client`", + ); + } + + this.collectionName = quoteIdentifier(config.collectionName || "mem0"); + this.indexName = quoteIdentifier( + config.indexName || `${config.collectionName || "mem0"}_VEC_IDX`, + ); + + this.embeddingModelDims = config.embeddingModelDims ?? 1536; + if ( + !Number.isInteger(this.embeddingModelDims) || + this.embeddingModelDims <= 0 + ) { + throw new Error("`embeddingModelDims` must be a positive integer"); + } + + const distanceMetric = (config.distanceMetric ?? + "COSINE") as string as DistanceMetric; + this.distanceMetric = distanceMetric.toUpperCase() as DistanceMetric; + if (!DISTANCE_METRICS.includes(this.distanceMetric)) { + throw new Error(`Unsupported distance metric: ${config.distanceMetric}`); + } + + const indexType = (config.indexType ?? "HNSW") as string; + this.indexType = indexType.toUpperCase() as IndexType; + if (this.indexType !== "HNSW" && this.indexType !== "IVF") { + throw new Error(`Unsupported index type: ${config.indexType}`); + } + + this.indexAccuracy = config.indexAccuracy; + if ( + this.indexAccuracy !== undefined && + (!Number.isInteger(this.indexAccuracy) || + this.indexAccuracy <= 0 || + this.indexAccuracy > 100) + ) { + throw new Error("`indexAccuracy` must be an integer between 1 and 100"); + } + + this.indexParameters = this.validateIndexParameters(config.indexParameters); + this.doCreateIndex = config.doCreateIndex ?? true; + this.config = config; + } + + private validateIndexParameters( + parameters?: Record, + ): Record { + if (!parameters) return {}; + + const allowed = INDEX_PARAMETER_RANGES[this.indexType]; + const validated: Record = {}; + + for (const [key, value] of Object.entries(parameters)) { + const range = allowed[key]; + if (!range) { + throw new Error( + `Unsupported ${this.indexType} index parameter '${key}'. ` + + `Allowed: ${Object.keys(allowed).join(", ")}`, + ); + } + if (!Number.isInteger(value) || value < range[0] || value > range[1]) { + throw new Error( + `Index parameter '${key}' must be an integer between ${range[0]} and ${range[1]}`, + ); + } + validated[key] = value; + } + + return validated; + } + + async initialize(): Promise { + if (!this._initPromise) { + this._initPromise = this._doInitialize(); + } + return this._initPromise; + } + + private async _doInitialize(): Promise { + const sdk = await loadPeer( + "oracledb", + "Oracle AI Vector Search", + () => import("oracledb"), + ); + this.oracledb = sdk.default ?? sdk; + + if (this.config.client) { + this.client = this.config.client; + } else if (this.config.useConnectionPool ?? true) { + this.client = await this.oracledb.createPool({ + poolMin: 1, + poolMax: 4, + ...this.config.connectionParams, + }); + this.ownsClient = true; + } else { + this.client = await this.oracledb.getConnection( + this.config.connectionParams, + ); + this.ownsClient = true; + } + + await this.assertVectorSupport(); + await this.createCol(); + } + + private isPool(client: Connection | Pool): client is Pool { + return typeof (client as Pool).getConnection === "function"; + } + + private async withConnection( + fn: (connection: Connection) => Promise, + commit = false, + ): Promise { + const client = this.client!; + + if (!this.isPool(client)) { + const connection = client as Connection; + try { + const result = await fn(connection); + if (commit) await connection.commit(); + return result; + } catch (err) { + await connection.rollback(); + throw err; + } + } + + const connection = await client.getConnection(); + try { + const result = await fn(connection); + if (commit) await connection.commit(); + return result; + } catch (err) { + await connection.rollback(); + throw err; + } finally { + await connection.close(); + } + } + + private async assertVectorSupport(): Promise { + if (!this.oracledb.thin) { + const [major, minor] = [ + Math.floor(this.oracledb.oracleClientVersion / 100000000), + Math.floor(this.oracledb.oracleClientVersion / 100000) % 100, + ]; + if (major < 23 || (major === 23 && minor < 4)) { + throw new Error( + `Oracle DB client driver version ${this.oracledb.oracleClientVersionString} ` + + "not supported, must be >=23.4 for vector support", + ); + } + } + + const version = await this.withConnection( + async (connection) => connection.oracleServerVersionString, + ); + const [major, minor] = version.split(".").map(Number); + if (major < 23 || (major === 23 && minor < 4)) { + throw new Error( + `Oracle DB version ${version} not supported, must be >=23.4 for vector support`, + ); + } + } + + private createIndexDdl(): string { + const accuracy = this.indexAccuracy + ? `WITH TARGET ACCURACY ${this.indexAccuracy}` + : ""; + + const parameterEntries = Object.entries(this.indexParameters); + const parameters = + parameterEntries.length > 0 + ? `PARAMETERS (${[ + `type ${this.indexType}`, + ...parameterEntries.map(([key, value]) => `${key} ${value}`), + ].join(", ")})` + : ""; + + const organization = + this.indexType === "HNSW" + ? "INMEMORY NEIGHBOR GRAPH" + : "NEIGHBOR PARTITIONS"; + + return ( + `CREATE VECTOR INDEX IF NOT EXISTS ${this.indexName} ON ${this.collectionName} (vector) ` + + `ORGANIZATION ${organization} DISTANCE ${this.distanceMetric} ${accuracy} ${parameters}` + ); + } + + private async createCol(): Promise { + await this.withConnection(async (connection) => { + await connection.execute(` + CREATE TABLE IF NOT EXISTS ${this.collectionName} ( + id VARCHAR2(36) PRIMARY KEY, + vector VECTOR(${this.embeddingModelDims}), + payload JSON + ) + `); + + await connection.execute(` + CREATE TABLE IF NOT EXISTS memory_migrations ( + id NUMBER PRIMARY KEY, + user_id VARCHAR2(255) NOT NULL + ) + `); + + if (this.doCreateIndex) { + await connection.execute(this.createIndexDdl()); + } + }, true); + } + + private loadPayload(value: any): Record { + if (value === null || value === undefined) return {}; + if (typeof value === "string") return JSON.parse(value); + if (Buffer.isBuffer(value)) return JSON.parse(value.toString("utf-8")); + return value; + } + + private vectorBind(vector: number[]) { + return { + type: this.oracledb.DB_TYPE_VECTOR, + val: new Float32Array(vector), + }; + } + + private payloadBind(payload: Record) { + return { type: this.oracledb.DB_TYPE_JSON, val: payload }; + } + + async insert( + vectors: number[][], + ids: string[], + payloads: Record[], + ): Promise { + await this.initialize(); + + await this.withConnection(async (connection) => { + for (let i = 0; i < vectors.length; i++) { + await connection.execute( + `INSERT INTO ${this.collectionName} (id, vector, payload) VALUES (:id, :vector, :payload)`, + { + id: ids[i], + vector: this.vectorBind(vectors[i]), + payload: this.payloadBind(payloads[i] ?? {}), + }, + ); + } + }, true); + } + + async search( + query: number[], + topK: number = 5, + filters?: SearchFilters, + ): Promise { + await this.initialize(); + + const [whereClause, filterBinds] = buildWhereClause(filters); + const sql = + `SELECT id, payload, VECTOR_DISTANCE(vector, :query_vec, ${this.distanceMetric}) distance ` + + `FROM ${this.collectionName} ${whereClause} ORDER BY distance FETCH APPROX FIRST :max_rows ROWS ONLY`; + + const rows = await this.withConnection(async (connection) => { + const result = await connection.execute(sql, { + query_vec: this.vectorBind(query), + max_rows: topK, + ...filterBinds, + }); + return result.rows ?? []; + }); + + return rows.map((row) => ({ + id: row[0], + payload: this.loadPayload(row[1]), + score: SCORE_FROM_DISTANCE[this.distanceMetric](Number(row[2])), + })); + } + + async get(vectorId: string): Promise { + await this.initialize(); + + const rows = await this.withConnection(async (connection) => { + const result = await connection.execute( + `SELECT id, payload FROM ${this.collectionName} WHERE id = :vector_id`, + { vector_id: vectorId }, + ); + return result.rows ?? []; + }); + + if (rows.length === 0) return null; + return { id: rows[0][0], payload: this.loadPayload(rows[0][1]) }; + } + + async update( + vectorId: string, + vector: number[], + payload: Record, + ): Promise { + await this.initialize(); + + const assignments: string[] = []; + const binds: Record = { vector_id: vectorId }; + + if (vector) { + assignments.push("vector = :vector"); + binds.vector = this.vectorBind(vector); + } + if (payload) { + assignments.push("payload = :payload"); + binds.payload = this.payloadBind(payload); + } + if (assignments.length === 0) return; + + await this.withConnection( + (connection) => + connection.execute( + `UPDATE ${this.collectionName} SET ${assignments.join(", ")} WHERE id = :vector_id`, + binds, + ), + true, + ); + } + + async delete(vectorId: string): Promise { + await this.initialize(); + + await this.withConnection( + (connection) => + connection.execute( + `DELETE FROM ${this.collectionName} WHERE id = :vector_id`, + { vector_id: vectorId }, + ), + true, + ); + } + + async deleteCol(): Promise { + await this.initialize(); + + await this.withConnection( + (connection) => + connection.execute(`DROP TABLE ${this.collectionName} PURGE`), + true, + ); + } + + async list( + filters?: SearchFilters, + topK: number = 100, + ): Promise<[VectorStoreResult[], number]> { + await this.initialize(); + + const [whereClause, filterBinds] = buildWhereClause(filters); + + return this.withConnection(async (connection) => { + const listResult = await connection.execute( + `SELECT id, payload FROM ${this.collectionName} ${whereClause} FETCH FIRST :max_rows ROWS ONLY`, + { ...filterBinds, max_rows: topK }, + ); + const countResult = await connection.execute( + `SELECT COUNT(*) FROM ${this.collectionName} ${whereClause}`, + filterBinds, + ); + + const results = (listResult.rows ?? []).map((row) => ({ + id: row[0], + payload: this.loadPayload(row[1]), + })); + + return [results, Number(countResult.rows?.[0]?.[0] ?? 0)]; + }); + } + + async getUserId(): Promise { + await this.initialize(); + + const rows = await this.withConnection(async (connection) => { + const result = await connection.execute( + "SELECT user_id FROM memory_migrations WHERE id = 1", + ); + return result.rows ?? []; + }); + + if (rows.length > 0) return rows[0][0]; + + const generatedUserId = uuidv4(); + await this.setUserId(generatedUserId); + return generatedUserId; + } + + async setUserId(userId: string): Promise { + await this.initialize(); + + await this.withConnection(async (connection) => { + await connection.execute("DELETE FROM memory_migrations WHERE id = 1"); + await connection.execute( + "INSERT INTO memory_migrations (id, user_id) VALUES (1, :user_id)", + { user_id: userId }, + ); + }, true); + } + + async close(): Promise { + if (this.client && this.ownsClient) { + await this.client.close(); + } + } +} diff --git a/mem0-ts/src/oss/tests/oracledb.unit.test.ts b/mem0-ts/src/oss/tests/oracledb.unit.test.ts new file mode 100644 index 000000000..ea50045bc --- /dev/null +++ b/mem0-ts/src/oss/tests/oracledb.unit.test.ts @@ -0,0 +1,328 @@ +/// +/** Oracle AI Vector Search filter, config and SQL tests. The driver is mocked, so no database is needed. */ +const DB_TYPE_VECTOR = { name: "DB_TYPE_VECTOR" }; +const DB_TYPE_JSON = { name: "DB_TYPE_JSON" }; + +jest.mock("oracledb", () => ({ thin: true, DB_TYPE_VECTOR, DB_TYPE_JSON }), { + virtual: true, +}); + +import { + OracleAIVectorSearch, + buildWhereClause, + quoteIdentifier, +} from "../src/vector_stores/oracledb"; + +type Call = { sql: string; binds: any }; + +function fakeConnection(calls: Call[], resultsBySql: Array) { + let selectIndex = 0; + return { + oracleServerVersionString: "23.4.0.24.05", + async execute(sql: string, binds: any = {}) { + calls.push({ sql: sql.replace(/\s+/g, " ").trim(), binds }); + if (/^\s*SELECT/i.test(sql)) { + return { rows: resultsBySql[selectIndex++] ?? [] }; + } + return { rows: [] }; + }, + async commit() {}, + async rollback() {}, + async close() {}, + }; +} + +function makeStore( + calls: Call[], + results: Array = [], + overrides = {}, +) { + return new OracleAIVectorSearch({ + client: fakeConnection(calls, results) as any, + collectionName: "mem0", + embeddingModelDims: 3, + ...overrides, + } as any); +} + +describe("quoteIdentifier", () => { + it("quotes a bare name", () => { + expect(quoteIdentifier("mem0")).toBe('"mem0"'); + }); + + it("quotes each segment of a schema-qualified name", () => { + expect(quoteIdentifier("app.mem0")).toBe('"app"."mem0"'); + }); + + it("preserves already-quoted segments", () => { + expect(quoteIdentifier('"App"."Mem0"')).toBe('"App"."Mem0"'); + }); + + it("rejects a name that would break out of the quoting", () => { + expect(() => quoteIdentifier('mem0" (x); DROP TABLE t--')).toThrow( + /is not valid/, + ); + }); +}); + +describe("buildWhereClause", () => { + it("returns no clause for empty filters", () => { + expect(buildWhereClause(undefined)).toEqual(["", {}]); + expect(buildWhereClause({})).toEqual(["", {}]); + }); + + it("binds a scalar equality instead of inlining it", () => { + const [clause, binds] = buildWhereClause({ user_id: "alice" }); + expect(clause).toBe( + `WHERE JSON_EXISTS(payload, '$."user_id"?(@ == $f_0)' PASSING :f_0 AS "f_0")`, + ); + expect(binds).toEqual({ f_0: "alice" }); + }); + + it("ANDs multiple fields", () => { + const [clause, binds] = buildWhereClause({ + user_id: "alice", + agent_id: "bot", + }); + expect(clause.startsWith("WHERE (")).toBe(true); + expect(clause).toContain(" AND "); + expect(binds).toEqual({ f_0: "alice", f_1: "bot" }); + }); + + it("applies every operator in a compound range filter", () => { + const [clause, binds] = buildWhereClause({ age: { gte: 10, lte: 20 } }); + expect(clause).toContain("@ >= $f_0 && @ <= $f_1"); + expect(binds).toEqual({ f_0: 10, f_1: 20 }); + }); + + it("builds an existence check for the wildcard filter", () => { + const [clause, binds] = buildWhereClause({ user_id: "*" }); + expect(clause).toBe(`WHERE JSON_EXISTS(payload, '$."user_id"')`); + expect(binds).toEqual({}); + }); + + it("builds membership for in and negates it for nin", () => { + const [inClause] = buildWhereClause({ user_id: { in: ["a", "b"] } }); + expect(inClause).toContain("@ in ($f_0, $f_1)"); + expect(inClause).not.toContain("NOT ("); + + const [ninClause] = buildWhereClause({ user_id: { nin: ["a"] } }); + expect(ninClause).toContain("NOT ("); + }); + + it("lowercases the operand for icontains", () => { + const [clause, binds] = buildWhereClause({ data: { icontains: "SciFi" } }); + expect(clause).toContain("@.lower() has substring $f_0"); + expect(binds).toEqual({ f_0: "scifi" }); + }); + + it("ORs the branches of a $or group", () => { + const [clause, binds] = buildWhereClause({ + $or: [{ user_id: "alice" }, { agent_id: "bot" }], + }); + expect(clause).toContain(" OR "); + expect(binds).toEqual({ f_0: "alice", f_1: "bot" }); + }); + + it("negates a $not group", () => { + const [clause] = buildWhereClause({ $not: [{ user_id: "alice" }] }); + expect(clause.startsWith("WHERE NOT (")).toBe(true); + }); + + it("nests logical groups", () => { + const [clause, binds] = buildWhereClause({ + user_id: "alice", + $or: [{ agent_id: "bot" }, { run_id: "r1" }], + }); + expect(clause).toContain(" AND "); + expect(clause).toContain(" OR "); + expect(Object.keys(binds)).toEqual(["f_0", "f_1", "f_2"]); + }); + + it("compares against JSON null without a bind", () => { + const [clause, binds] = buildWhereClause({ agent_id: null }); + expect(clause).toBe( + `WHERE JSON_EXISTS(payload, '$."agent_id"?(@ == null)')`, + ); + expect(binds).toEqual({}); + }); + + it("rejects a metadata key that could escape the JSON path", () => { + expect(() => buildWhereClause({ 'a"?(1==1))--': "x" })).toThrow( + /Invalid metadata key/, + ); + }); + + it("rejects an unsupported field operator", () => { + expect(() => buildWhereClause({ age: { regex: "^a" } })).toThrow( + /Unsupported Oracle filter operator/, + ); + }); + + it("rejects an unsupported logical operator", () => { + expect(() => buildWhereClause({ $nor: [{ a: 1 }] })).toThrow( + /Unsupported Oracle logical filter operator/, + ); + }); + + it("rejects an empty in list", () => { + expect(() => buildWhereClause({ user_id: { in: [] } })).toThrow( + /non-empty array/, + ); + }); + + it("rejects a non-scalar comparison operand", () => { + expect(() => buildWhereClause({ age: { gt: [1] } })).toThrow( + /requires a scalar value/, + ); + }); +}); + +describe("OracleAIVectorSearch config validation", () => { + it("requires connectionParams or client", () => { + expect(() => new OracleAIVectorSearch({} as any)).toThrow( + /connectionParams.*client/, + ); + }); + + it("rejects an unsupported distance metric", () => { + expect(() => makeStore([], [], { distanceMetric: "JACCARD" })).toThrow( + /Unsupported distance metric/, + ); + }); + + it("rejects a non-positive embedding dimension", () => { + expect(() => makeStore([], [], { embeddingModelDims: 0 })).toThrow( + /positive integer/, + ); + }); + + it("rejects an out-of-range index accuracy", () => { + expect(() => makeStore([], [], { indexAccuracy: 101 })).toThrow( + /between 1 and 100/, + ); + }); + + it("rejects an index parameter that does not belong to the index type", () => { + expect(() => + makeStore([], [], { + indexType: "HNSW", + indexParameters: { samples_per_partition: 10 }, + }), + ).toThrow(/Unsupported HNSW index parameter/); + }); + + it("rejects an index parameter outside its allowed range", () => { + expect(() => + makeStore([], [], { indexParameters: { neighbors: 1 } }), + ).toThrow(/between 2 and 2048/); + }); +}); + +describe("OracleAIVectorSearch SQL", () => { + it("creates the table and a vector index on initialize", async () => { + const calls: Call[] = []; + await makeStore(calls, [], { + indexParameters: { neighbors: 32, efconstruction: 200 }, + indexAccuracy: 95, + }).initialize(); + + const ddl = calls.map((c) => c.sql).join("\n"); + expect(ddl).toContain( + 'CREATE TABLE IF NOT EXISTS "mem0" ( id VARCHAR2(36) PRIMARY KEY, vector VECTOR(3), payload JSON )', + ); + expect(ddl).toContain( + 'CREATE VECTOR INDEX IF NOT EXISTS "mem0_VEC_IDX" ON "mem0" (vector) ORGANIZATION INMEMORY NEIGHBOR GRAPH DISTANCE COSINE WITH TARGET ACCURACY 95 PARAMETERS (type HNSW, neighbors 32, efconstruction 200)', + ); + }); + + it("skips index creation when doCreateIndex is false", async () => { + const calls: Call[] = []; + await makeStore(calls, [], { doCreateIndex: false }).initialize(); + expect(calls.map((c) => c.sql).join("\n")).not.toContain( + "CREATE VECTOR INDEX", + ); + }); + + it("binds vectors as DB_TYPE_VECTOR and payloads as DB_TYPE_JSON on insert", async () => { + const calls: Call[] = []; + await makeStore(calls).insert([[1, 2, 3]], ["id-1"], [{ data: "hello" }]); + + const insert = calls.find((c) => c.sql.startsWith("INSERT INTO"))!; + expect(insert.binds.id).toBe("id-1"); + expect(insert.binds.vector.type).toBe(DB_TYPE_VECTOR); + expect(insert.binds.vector.val).toEqual(new Float32Array([1, 2, 3])); + expect(insert.binds.payload).toEqual({ + type: DB_TYPE_JSON, + val: { data: "hello" }, + }); + }); + + it("converts cosine distance to a similarity score", async () => { + const calls: Call[] = []; + const store = makeStore(calls, [[["id-1", { data: "hello" }, 0.25]]]); + const results = await store.search([1, 2, 3], 5); + + expect(results).toEqual([ + { id: "id-1", payload: { data: "hello" }, score: 0.75 }, + ]); + const select = calls.find((c) => c.sql.startsWith("SELECT id, payload,"))!; + expect(select.sql).toContain( + "VECTOR_DISTANCE(vector, :query_vec, COSINE) distance", + ); + expect(select.sql).toContain("FETCH APPROX FIRST :max_rows ROWS ONLY"); + expect(select.binds.max_rows).toBe(5); + }); + + it("inverts the sign of a DOT distance", async () => { + const store = makeStore([], [[["id-1", {}, -0.4]]], { + distanceMetric: "DOT", + }); + const [result] = await store.search([1, 2, 3]); + expect(result.score).toBeCloseTo(0.4); + }); + + it("parses a payload returned as a JSON string", async () => { + const store = makeStore([], [[["id-1", '{"data":"hello"}']]]); + expect(await store.get("id-1")).toEqual({ + id: "id-1", + payload: { data: "hello" }, + }); + }); + + it("returns null when get finds no row", async () => { + expect(await makeStore([], [[]]).get("missing")).toBeNull(); + }); + + it("generates and persists a UUID user id when none is stored", async () => { + const calls: Call[] = []; + const userId = await makeStore(calls, [[]]).getUserId(); + + expect(userId).toMatch( + /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/, + ); + const insert = calls.find((c) => + c.sql.startsWith("INSERT INTO memory_migrations"), + )!; + expect(insert.binds).toEqual({ user_id: userId }); + }); + + it("returns the stored user id when one exists", async () => { + expect(await makeStore([], [[["alice"]]]).getUserId()).toBe("alice"); + }); + + it("returns rows and the total count from list", async () => { + const calls: Call[] = []; + const store = makeStore(calls, [[["id-1", { data: "hello" }]], [[7]]]); + const [results, count] = await store.list({ user_id: "alice" }, 10); + + expect(results).toEqual([{ id: "id-1", payload: { data: "hello" } }]); + expect(count).toBe(7); + + const list = calls.find((c) => + c.sql.startsWith("SELECT id, payload FROM"), + )!; + expect(list.sql).toContain("WHERE JSON_EXISTS(payload,"); + expect(list.binds).toEqual({ f_0: "alice", max_rows: 10 }); + }); +}); diff --git a/mem0-ts/tsup.config.ts b/mem0-ts/tsup.config.ts index a98af7eb4..ea0ccc7ab 100644 --- a/mem0-ts/tsup.config.ts +++ b/mem0-ts/tsup.config.ts @@ -44,6 +44,7 @@ const external = [ "@elastic/elasticsearch", "chromadb", "weaviate-client", + "oracledb", ]; const define = {