feat(ts-oss): add Oracle AI Vector Search vector store (#6690)
This commit is contained in:
Kartik
@@ -51,6 +51,7 @@ export * from "./vector_stores/milvus";
|
||||
export * from "./vector_stores/mongodb";
|
||||
export * from "./vector_stores/opensearch";
|
||||
export * from "./vector_stores/weaviate";
|
||||
export * from "./vector_stores/oracledb";
|
||||
export * from "./rerankers/base";
|
||||
export * from "./rerankers/cohere";
|
||||
export * from "./rerankers/llm";
|
||||
|
||||
@@ -71,6 +71,7 @@ import { TurbopufferDB } from "../vector_stores/turbopuffer";
|
||||
import { Milvus } from "../vector_stores/milvus";
|
||||
import { MongoDB } from "../vector_stores/mongodb";
|
||||
import { WeaviateDB } from "../vector_stores/weaviate";
|
||||
import { OracleAIVectorSearch } from "../vector_stores/oracledb";
|
||||
|
||||
export class EmbedderFactory {
|
||||
static create(provider: string, config: EmbeddingConfig): Embedder {
|
||||
@@ -205,6 +206,8 @@ export class VectorStoreFactory {
|
||||
return new MongoDB(config as any);
|
||||
case "weaviate":
|
||||
return new WeaviateDB(config as any);
|
||||
case "oracledb":
|
||||
return new OracleAIVectorSearch(config as any);
|
||||
default:
|
||||
throw new Error(`Unsupported vector store provider: ${provider}`);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,741 @@
|
||||
import type { Connection, Pool } from "oracledb";
|
||||
import { v4 as uuidv4 } from "uuid";
|
||||
import { VectorStore } from "./base";
|
||||
import { SearchFilters, VectorStoreConfig, VectorStoreResult } from "../types";
|
||||
import { loadPeer } from "../utils/load_peer";
|
||||
|
||||
const DISTANCE_METRICS = [
|
||||
"COSINE",
|
||||
"EUCLIDEAN",
|
||||
"EUCLIDEAN_SQUARED",
|
||||
"DOT",
|
||||
"HAMMING",
|
||||
"MANHATTAN",
|
||||
] as const;
|
||||
|
||||
type DistanceMetric = (typeof DISTANCE_METRICS)[number];
|
||||
type IndexType = "HNSW" | "IVF";
|
||||
|
||||
const SCORE_FROM_DISTANCE: Record<DistanceMetric, (d: number) => number> = {
|
||||
COSINE: (d) => Math.max(0, Math.min(1, 1 - d)),
|
||||
EUCLIDEAN: (d) => 1 / (1 + Math.max(0, d)),
|
||||
EUCLIDEAN_SQUARED: (d) => 1 / (1 + Math.sqrt(Math.max(0, d))),
|
||||
HAMMING: (d) => 1 / (1 + Math.max(0, d)),
|
||||
MANHATTAN: (d) => 1 / (1 + Math.max(0, d)),
|
||||
DOT: (d) => -d,
|
||||
};
|
||||
|
||||
const INDEX_PARAMETER_RANGES: Record<
|
||||
IndexType,
|
||||
Record<string, [number, number]>
|
||||
> = {
|
||||
HNSW: {
|
||||
neighbors: [2, 2048],
|
||||
efconstruction: [1, 65535],
|
||||
},
|
||||
IVF: {
|
||||
"neighbor partitions": [1, 10_000_000],
|
||||
samples_per_partition: [1, Number.MAX_SAFE_INTEGER],
|
||||
min_vectors_per_partition: [0, Number.MAX_SAFE_INTEGER],
|
||||
},
|
||||
};
|
||||
|
||||
const IDENTIFIER_RE = /^(?:"[^"]+"|[^".]+)(?:\.(?:"[^"]+"|[^".]+))*$/;
|
||||
const METADATA_KEY_RE = /^[a-zA-Z0-9_.[\],\s*]+$/;
|
||||
|
||||
export function quoteIdentifier(name: string): string {
|
||||
const trimmed = name.trim();
|
||||
if (!IDENTIFIER_RE.test(trimmed)) {
|
||||
throw new Error(`Identifier name ${name} is not valid.`);
|
||||
}
|
||||
return [...trimmed.matchAll(/"([^"]+)"|([^".]+)/g)]
|
||||
.map((m) => `"${m[1] ?? m[2]}"`)
|
||||
.join(".");
|
||||
}
|
||||
|
||||
function jsonPath(metadataKey: string): string {
|
||||
if (!METADATA_KEY_RE.test(metadataKey)) {
|
||||
throw new Error(
|
||||
`Invalid metadata key '${metadataKey}'. Only letters, numbers, underscores, ` +
|
||||
`nesting via '.', and array wildcards '[*]' are allowed.`,
|
||||
);
|
||||
}
|
||||
return metadataKey
|
||||
.split(".")
|
||||
.map((part) =>
|
||||
part.endsWith("[*]") ? `."${part.slice(0, -3)}"[*]` : `."${part}"`,
|
||||
)
|
||||
.join("");
|
||||
}
|
||||
|
||||
const COMPARISON_OPERATORS: Record<string, string> = {
|
||||
eq: "==",
|
||||
ne: "!=",
|
||||
gt: ">",
|
||||
gte: ">=",
|
||||
lt: "<",
|
||||
lte: "<=",
|
||||
};
|
||||
|
||||
const FIELD_OPERATORS = new Set([
|
||||
...Object.keys(COMPARISON_OPERATORS),
|
||||
"in",
|
||||
"nin",
|
||||
"contains",
|
||||
"icontains",
|
||||
]);
|
||||
|
||||
const LOGICAL_OPERATORS: Record<string, "and" | "or" | "not"> = {
|
||||
$and: "and",
|
||||
$or: "or",
|
||||
$not: "not",
|
||||
AND: "and",
|
||||
OR: "or",
|
||||
NOT: "not",
|
||||
};
|
||||
|
||||
function isScalar(value: any): boolean {
|
||||
return value === null || (typeof value !== "object" && !Array.isArray(value));
|
||||
}
|
||||
|
||||
function bindFilterValue(
|
||||
value: any,
|
||||
binds: Record<string, any>,
|
||||
): [string, string] {
|
||||
const name = `f_${Object.keys(binds).length}`;
|
||||
binds[name] = value;
|
||||
return [`$${name}`, `:${name} AS "${name}"`];
|
||||
}
|
||||
|
||||
function jsonExists(
|
||||
path: string,
|
||||
predicate: string,
|
||||
passings: string[],
|
||||
): string {
|
||||
const passingClause =
|
||||
passings.length > 0 ? ` PASSING ${passings.join(", ")}` : "";
|
||||
return `JSON_EXISTS(payload, '$${path}?(${predicate})'${passingClause})`;
|
||||
}
|
||||
|
||||
function buildFieldCondition(
|
||||
metadataKey: string,
|
||||
value: any,
|
||||
binds: Record<string, any>,
|
||||
): string {
|
||||
const path = jsonPath(metadataKey);
|
||||
|
||||
if (value === "*") {
|
||||
return `JSON_EXISTS(payload, '$${path}')`;
|
||||
}
|
||||
|
||||
if (isScalar(value)) {
|
||||
if (value === null) {
|
||||
return jsonExists(path, "@ == null", []);
|
||||
}
|
||||
const [variable, passing] = bindFilterValue(value, binds);
|
||||
return jsonExists(path, `@ == ${variable}`, [passing]);
|
||||
}
|
||||
|
||||
if (Array.isArray(value)) {
|
||||
throw new Error(
|
||||
`Oracle filter for field '${metadataKey}' must be a scalar or an operator object`,
|
||||
);
|
||||
}
|
||||
|
||||
const operators = Object.entries(value);
|
||||
if (operators.length === 0) {
|
||||
throw new Error(
|
||||
`Operator filter for field '${metadataKey}' must not be empty`,
|
||||
);
|
||||
}
|
||||
|
||||
const unsupported = operators
|
||||
.map(([op]) => op)
|
||||
.filter((op) => !FIELD_OPERATORS.has(op));
|
||||
if (unsupported.length > 0) {
|
||||
throw new Error(
|
||||
`Unsupported Oracle filter operator(s) for field '${metadataKey}': ${unsupported.sort().join(", ")}`,
|
||||
);
|
||||
}
|
||||
|
||||
const predicates: string[] = [];
|
||||
const passings: string[] = [];
|
||||
const additionalClauses: string[] = [];
|
||||
|
||||
for (const [operator, operand] of operators) {
|
||||
if (operator in COMPARISON_OPERATORS) {
|
||||
if (!isScalar(operand)) {
|
||||
throw new Error(
|
||||
`Oracle filter operator '${operator}' requires a scalar value`,
|
||||
);
|
||||
}
|
||||
if (operand === null) {
|
||||
if (operator !== "eq" && operator !== "ne") {
|
||||
throw new Error(
|
||||
`Oracle filter operator '${operator}' does not support null`,
|
||||
);
|
||||
}
|
||||
predicates.push(`@ ${COMPARISON_OPERATORS[operator]} null`);
|
||||
continue;
|
||||
}
|
||||
const [variable, passing] = bindFilterValue(operand, binds);
|
||||
predicates.push(`@ ${COMPARISON_OPERATORS[operator]} ${variable}`);
|
||||
passings.push(passing);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (operator === "in" || operator === "nin") {
|
||||
if (!Array.isArray(operand) || operand.length === 0) {
|
||||
throw new Error(
|
||||
`Oracle filter operator '${operator}' requires a non-empty array`,
|
||||
);
|
||||
}
|
||||
|
||||
const variables: string[] = [];
|
||||
const listPassings: string[] = [];
|
||||
for (const item of operand) {
|
||||
if (!isScalar(item)) {
|
||||
throw new Error(
|
||||
`Oracle filter operator '${operator}' requires scalar values`,
|
||||
);
|
||||
}
|
||||
if (item === null) {
|
||||
variables.push("null");
|
||||
continue;
|
||||
}
|
||||
const [variable, passing] = bindFilterValue(item, binds);
|
||||
variables.push(variable);
|
||||
listPassings.push(passing);
|
||||
}
|
||||
|
||||
const membership = jsonExists(
|
||||
path,
|
||||
`@ in (${variables.join(", ")})`,
|
||||
listPassings,
|
||||
);
|
||||
additionalClauses.push(
|
||||
operator === "in" ? membership : `NOT (${membership})`,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (typeof operand !== "string") {
|
||||
throw new Error(
|
||||
`Oracle filter operator '${operator}' requires a string value`,
|
||||
);
|
||||
}
|
||||
|
||||
if (operator === "contains") {
|
||||
const [variable, passing] = bindFilterValue(operand, binds);
|
||||
predicates.push(`@ has substring ${variable}`);
|
||||
passings.push(passing);
|
||||
} else {
|
||||
const [variable, passing] = bindFilterValue(operand.toLowerCase(), binds);
|
||||
predicates.push(`@.lower() has substring ${variable}`);
|
||||
passings.push(passing);
|
||||
}
|
||||
}
|
||||
|
||||
const clauses = [...additionalClauses];
|
||||
if (predicates.length > 0) {
|
||||
clauses.unshift(jsonExists(path, predicates.join(" && "), passings));
|
||||
}
|
||||
|
||||
return clauses.length === 1 ? clauses[0] : `(${clauses.join(" AND ")})`;
|
||||
}
|
||||
|
||||
export function buildFilterGroup(
|
||||
filters: Record<string, any>,
|
||||
binds: Record<string, any>,
|
||||
): string {
|
||||
const entries = Object.entries(filters ?? {});
|
||||
if (entries.length === 0) {
|
||||
throw new Error("Oracle filter groups must be non-empty objects");
|
||||
}
|
||||
|
||||
const clauses: string[] = [];
|
||||
for (const [key, value] of entries) {
|
||||
const logicalOperator = LOGICAL_OPERATORS[key];
|
||||
if (logicalOperator) {
|
||||
if (!Array.isArray(value) || value.length === 0) {
|
||||
throw new Error(
|
||||
`Logical filter operator '${key}' requires a non-empty array`,
|
||||
);
|
||||
}
|
||||
const nested = value.map((condition) =>
|
||||
buildFilterGroup(condition, binds),
|
||||
);
|
||||
if (logicalOperator === "not") {
|
||||
clauses.push(`NOT (${nested.join(" OR ")})`);
|
||||
} else {
|
||||
clauses.push(
|
||||
`(${nested.join(logicalOperator === "and" ? " AND " : " OR ")})`,
|
||||
);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if (key.startsWith("$")) {
|
||||
throw new Error(`Unsupported Oracle logical filter operator: ${key}`);
|
||||
}
|
||||
|
||||
clauses.push(buildFieldCondition(key, value, binds));
|
||||
}
|
||||
|
||||
return clauses.length === 1 ? clauses[0] : `(${clauses.join(" AND ")})`;
|
||||
}
|
||||
|
||||
export function buildWhereClause(
|
||||
filters?: SearchFilters,
|
||||
): [string, Record<string, any>] {
|
||||
if (!filters || Object.keys(filters).length === 0) {
|
||||
return ["", {}];
|
||||
}
|
||||
const binds: Record<string, any> = {};
|
||||
return [`WHERE ${buildFilterGroup(filters, binds)}`, binds];
|
||||
}
|
||||
|
||||
interface OracleDBConfig extends VectorStoreConfig {
|
||||
connectionParams?: Record<string, any>;
|
||||
useConnectionPool?: boolean;
|
||||
client?: Connection | Pool;
|
||||
collectionName?: string;
|
||||
embeddingModelDims?: number;
|
||||
distanceMetric?: DistanceMetric;
|
||||
doCreateIndex?: boolean;
|
||||
indexType?: IndexType;
|
||||
indexName?: string;
|
||||
indexParameters?: Record<string, number>;
|
||||
indexAccuracy?: number;
|
||||
}
|
||||
|
||||
export class OracleAIVectorSearch implements VectorStore {
|
||||
private readonly collectionName: string;
|
||||
private readonly indexName: string;
|
||||
private readonly embeddingModelDims: number;
|
||||
private readonly distanceMetric: DistanceMetric;
|
||||
private readonly indexType: IndexType;
|
||||
private readonly indexParameters: Record<string, number>;
|
||||
private readonly indexAccuracy?: number;
|
||||
private readonly doCreateIndex: boolean;
|
||||
private readonly config: OracleDBConfig;
|
||||
private oracledb: any;
|
||||
private client?: Connection | Pool;
|
||||
private ownsClient = false;
|
||||
private _initPromise?: Promise<void>;
|
||||
|
||||
constructor(config: OracleDBConfig) {
|
||||
if (!config.connectionParams && !config.client) {
|
||||
throw new Error(
|
||||
"Must provide at least one of `connectionParams` and `client`",
|
||||
);
|
||||
}
|
||||
|
||||
this.collectionName = quoteIdentifier(config.collectionName || "mem0");
|
||||
this.indexName = quoteIdentifier(
|
||||
config.indexName || `${config.collectionName || "mem0"}_VEC_IDX`,
|
||||
);
|
||||
|
||||
this.embeddingModelDims = config.embeddingModelDims ?? 1536;
|
||||
if (
|
||||
!Number.isInteger(this.embeddingModelDims) ||
|
||||
this.embeddingModelDims <= 0
|
||||
) {
|
||||
throw new Error("`embeddingModelDims` must be a positive integer");
|
||||
}
|
||||
|
||||
const distanceMetric = (config.distanceMetric ??
|
||||
"COSINE") as string as DistanceMetric;
|
||||
this.distanceMetric = distanceMetric.toUpperCase() as DistanceMetric;
|
||||
if (!DISTANCE_METRICS.includes(this.distanceMetric)) {
|
||||
throw new Error(`Unsupported distance metric: ${config.distanceMetric}`);
|
||||
}
|
||||
|
||||
const indexType = (config.indexType ?? "HNSW") as string;
|
||||
this.indexType = indexType.toUpperCase() as IndexType;
|
||||
if (this.indexType !== "HNSW" && this.indexType !== "IVF") {
|
||||
throw new Error(`Unsupported index type: ${config.indexType}`);
|
||||
}
|
||||
|
||||
this.indexAccuracy = config.indexAccuracy;
|
||||
if (
|
||||
this.indexAccuracy !== undefined &&
|
||||
(!Number.isInteger(this.indexAccuracy) ||
|
||||
this.indexAccuracy <= 0 ||
|
||||
this.indexAccuracy > 100)
|
||||
) {
|
||||
throw new Error("`indexAccuracy` must be an integer between 1 and 100");
|
||||
}
|
||||
|
||||
this.indexParameters = this.validateIndexParameters(config.indexParameters);
|
||||
this.doCreateIndex = config.doCreateIndex ?? true;
|
||||
this.config = config;
|
||||
}
|
||||
|
||||
private validateIndexParameters(
|
||||
parameters?: Record<string, number>,
|
||||
): Record<string, number> {
|
||||
if (!parameters) return {};
|
||||
|
||||
const allowed = INDEX_PARAMETER_RANGES[this.indexType];
|
||||
const validated: Record<string, number> = {};
|
||||
|
||||
for (const [key, value] of Object.entries(parameters)) {
|
||||
const range = allowed[key];
|
||||
if (!range) {
|
||||
throw new Error(
|
||||
`Unsupported ${this.indexType} index parameter '${key}'. ` +
|
||||
`Allowed: ${Object.keys(allowed).join(", ")}`,
|
||||
);
|
||||
}
|
||||
if (!Number.isInteger(value) || value < range[0] || value > range[1]) {
|
||||
throw new Error(
|
||||
`Index parameter '${key}' must be an integer between ${range[0]} and ${range[1]}`,
|
||||
);
|
||||
}
|
||||
validated[key] = value;
|
||||
}
|
||||
|
||||
return validated;
|
||||
}
|
||||
|
||||
async initialize(): Promise<void> {
|
||||
if (!this._initPromise) {
|
||||
this._initPromise = this._doInitialize();
|
||||
}
|
||||
return this._initPromise;
|
||||
}
|
||||
|
||||
private async _doInitialize(): Promise<void> {
|
||||
const sdk = await loadPeer(
|
||||
"oracledb",
|
||||
"Oracle AI Vector Search",
|
||||
() => import("oracledb"),
|
||||
);
|
||||
this.oracledb = sdk.default ?? sdk;
|
||||
|
||||
if (this.config.client) {
|
||||
this.client = this.config.client;
|
||||
} else if (this.config.useConnectionPool ?? true) {
|
||||
this.client = await this.oracledb.createPool({
|
||||
poolMin: 1,
|
||||
poolMax: 4,
|
||||
...this.config.connectionParams,
|
||||
});
|
||||
this.ownsClient = true;
|
||||
} else {
|
||||
this.client = await this.oracledb.getConnection(
|
||||
this.config.connectionParams,
|
||||
);
|
||||
this.ownsClient = true;
|
||||
}
|
||||
|
||||
await this.assertVectorSupport();
|
||||
await this.createCol();
|
||||
}
|
||||
|
||||
private isPool(client: Connection | Pool): client is Pool {
|
||||
return typeof (client as Pool).getConnection === "function";
|
||||
}
|
||||
|
||||
private async withConnection<T>(
|
||||
fn: (connection: Connection) => Promise<T>,
|
||||
commit = false,
|
||||
): Promise<T> {
|
||||
const client = this.client!;
|
||||
|
||||
if (!this.isPool(client)) {
|
||||
const connection = client as Connection;
|
||||
try {
|
||||
const result = await fn(connection);
|
||||
if (commit) await connection.commit();
|
||||
return result;
|
||||
} catch (err) {
|
||||
await connection.rollback();
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
const connection = await client.getConnection();
|
||||
try {
|
||||
const result = await fn(connection);
|
||||
if (commit) await connection.commit();
|
||||
return result;
|
||||
} catch (err) {
|
||||
await connection.rollback();
|
||||
throw err;
|
||||
} finally {
|
||||
await connection.close();
|
||||
}
|
||||
}
|
||||
|
||||
private async assertVectorSupport(): Promise<void> {
|
||||
if (!this.oracledb.thin) {
|
||||
const [major, minor] = [
|
||||
Math.floor(this.oracledb.oracleClientVersion / 100000000),
|
||||
Math.floor(this.oracledb.oracleClientVersion / 100000) % 100,
|
||||
];
|
||||
if (major < 23 || (major === 23 && minor < 4)) {
|
||||
throw new Error(
|
||||
`Oracle DB client driver version ${this.oracledb.oracleClientVersionString} ` +
|
||||
"not supported, must be >=23.4 for vector support",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const version = await this.withConnection(
|
||||
async (connection) => connection.oracleServerVersionString,
|
||||
);
|
||||
const [major, minor] = version.split(".").map(Number);
|
||||
if (major < 23 || (major === 23 && minor < 4)) {
|
||||
throw new Error(
|
||||
`Oracle DB version ${version} not supported, must be >=23.4 for vector support`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private createIndexDdl(): string {
|
||||
const accuracy = this.indexAccuracy
|
||||
? `WITH TARGET ACCURACY ${this.indexAccuracy}`
|
||||
: "";
|
||||
|
||||
const parameterEntries = Object.entries(this.indexParameters);
|
||||
const parameters =
|
||||
parameterEntries.length > 0
|
||||
? `PARAMETERS (${[
|
||||
`type ${this.indexType}`,
|
||||
...parameterEntries.map(([key, value]) => `${key} ${value}`),
|
||||
].join(", ")})`
|
||||
: "";
|
||||
|
||||
const organization =
|
||||
this.indexType === "HNSW"
|
||||
? "INMEMORY NEIGHBOR GRAPH"
|
||||
: "NEIGHBOR PARTITIONS";
|
||||
|
||||
return (
|
||||
`CREATE VECTOR INDEX IF NOT EXISTS ${this.indexName} ON ${this.collectionName} (vector) ` +
|
||||
`ORGANIZATION ${organization} DISTANCE ${this.distanceMetric} ${accuracy} ${parameters}`
|
||||
);
|
||||
}
|
||||
|
||||
private async createCol(): Promise<void> {
|
||||
await this.withConnection(async (connection) => {
|
||||
await connection.execute(`
|
||||
CREATE TABLE IF NOT EXISTS ${this.collectionName} (
|
||||
id VARCHAR2(36) PRIMARY KEY,
|
||||
vector VECTOR(${this.embeddingModelDims}),
|
||||
payload JSON
|
||||
)
|
||||
`);
|
||||
|
||||
await connection.execute(`
|
||||
CREATE TABLE IF NOT EXISTS memory_migrations (
|
||||
id NUMBER PRIMARY KEY,
|
||||
user_id VARCHAR2(255) NOT NULL
|
||||
)
|
||||
`);
|
||||
|
||||
if (this.doCreateIndex) {
|
||||
await connection.execute(this.createIndexDdl());
|
||||
}
|
||||
}, true);
|
||||
}
|
||||
|
||||
private loadPayload(value: any): Record<string, any> {
|
||||
if (value === null || value === undefined) return {};
|
||||
if (typeof value === "string") return JSON.parse(value);
|
||||
if (Buffer.isBuffer(value)) return JSON.parse(value.toString("utf-8"));
|
||||
return value;
|
||||
}
|
||||
|
||||
private vectorBind(vector: number[]) {
|
||||
return {
|
||||
type: this.oracledb.DB_TYPE_VECTOR,
|
||||
val: new Float32Array(vector),
|
||||
};
|
||||
}
|
||||
|
||||
private payloadBind(payload: Record<string, any>) {
|
||||
return { type: this.oracledb.DB_TYPE_JSON, val: payload };
|
||||
}
|
||||
|
||||
async insert(
|
||||
vectors: number[][],
|
||||
ids: string[],
|
||||
payloads: Record<string, any>[],
|
||||
): Promise<void> {
|
||||
await this.initialize();
|
||||
|
||||
await this.withConnection(async (connection) => {
|
||||
for (let i = 0; i < vectors.length; i++) {
|
||||
await connection.execute(
|
||||
`INSERT INTO ${this.collectionName} (id, vector, payload) VALUES (:id, :vector, :payload)`,
|
||||
{
|
||||
id: ids[i],
|
||||
vector: this.vectorBind(vectors[i]),
|
||||
payload: this.payloadBind(payloads[i] ?? {}),
|
||||
},
|
||||
);
|
||||
}
|
||||
}, true);
|
||||
}
|
||||
|
||||
async search(
|
||||
query: number[],
|
||||
topK: number = 5,
|
||||
filters?: SearchFilters,
|
||||
): Promise<VectorStoreResult[]> {
|
||||
await this.initialize();
|
||||
|
||||
const [whereClause, filterBinds] = buildWhereClause(filters);
|
||||
const sql =
|
||||
`SELECT id, payload, VECTOR_DISTANCE(vector, :query_vec, ${this.distanceMetric}) distance ` +
|
||||
`FROM ${this.collectionName} ${whereClause} ORDER BY distance FETCH APPROX FIRST :max_rows ROWS ONLY`;
|
||||
|
||||
const rows = await this.withConnection(async (connection) => {
|
||||
const result = await connection.execute<any[]>(sql, {
|
||||
query_vec: this.vectorBind(query),
|
||||
max_rows: topK,
|
||||
...filterBinds,
|
||||
});
|
||||
return result.rows ?? [];
|
||||
});
|
||||
|
||||
return rows.map((row) => ({
|
||||
id: row[0],
|
||||
payload: this.loadPayload(row[1]),
|
||||
score: SCORE_FROM_DISTANCE[this.distanceMetric](Number(row[2])),
|
||||
}));
|
||||
}
|
||||
|
||||
async get(vectorId: string): Promise<VectorStoreResult | null> {
|
||||
await this.initialize();
|
||||
|
||||
const rows = await this.withConnection(async (connection) => {
|
||||
const result = await connection.execute<any[]>(
|
||||
`SELECT id, payload FROM ${this.collectionName} WHERE id = :vector_id`,
|
||||
{ vector_id: vectorId },
|
||||
);
|
||||
return result.rows ?? [];
|
||||
});
|
||||
|
||||
if (rows.length === 0) return null;
|
||||
return { id: rows[0][0], payload: this.loadPayload(rows[0][1]) };
|
||||
}
|
||||
|
||||
async update(
|
||||
vectorId: string,
|
||||
vector: number[],
|
||||
payload: Record<string, any>,
|
||||
): Promise<void> {
|
||||
await this.initialize();
|
||||
|
||||
const assignments: string[] = [];
|
||||
const binds: Record<string, any> = { vector_id: vectorId };
|
||||
|
||||
if (vector) {
|
||||
assignments.push("vector = :vector");
|
||||
binds.vector = this.vectorBind(vector);
|
||||
}
|
||||
if (payload) {
|
||||
assignments.push("payload = :payload");
|
||||
binds.payload = this.payloadBind(payload);
|
||||
}
|
||||
if (assignments.length === 0) return;
|
||||
|
||||
await this.withConnection(
|
||||
(connection) =>
|
||||
connection.execute(
|
||||
`UPDATE ${this.collectionName} SET ${assignments.join(", ")} WHERE id = :vector_id`,
|
||||
binds,
|
||||
),
|
||||
true,
|
||||
);
|
||||
}
|
||||
|
||||
async delete(vectorId: string): Promise<void> {
|
||||
await this.initialize();
|
||||
|
||||
await this.withConnection(
|
||||
(connection) =>
|
||||
connection.execute(
|
||||
`DELETE FROM ${this.collectionName} WHERE id = :vector_id`,
|
||||
{ vector_id: vectorId },
|
||||
),
|
||||
true,
|
||||
);
|
||||
}
|
||||
|
||||
async deleteCol(): Promise<void> {
|
||||
await this.initialize();
|
||||
|
||||
await this.withConnection(
|
||||
(connection) =>
|
||||
connection.execute(`DROP TABLE ${this.collectionName} PURGE`),
|
||||
true,
|
||||
);
|
||||
}
|
||||
|
||||
async list(
|
||||
filters?: SearchFilters,
|
||||
topK: number = 100,
|
||||
): Promise<[VectorStoreResult[], number]> {
|
||||
await this.initialize();
|
||||
|
||||
const [whereClause, filterBinds] = buildWhereClause(filters);
|
||||
|
||||
return this.withConnection(async (connection) => {
|
||||
const listResult = await connection.execute<any[]>(
|
||||
`SELECT id, payload FROM ${this.collectionName} ${whereClause} FETCH FIRST :max_rows ROWS ONLY`,
|
||||
{ ...filterBinds, max_rows: topK },
|
||||
);
|
||||
const countResult = await connection.execute<any[]>(
|
||||
`SELECT COUNT(*) FROM ${this.collectionName} ${whereClause}`,
|
||||
filterBinds,
|
||||
);
|
||||
|
||||
const results = (listResult.rows ?? []).map((row) => ({
|
||||
id: row[0],
|
||||
payload: this.loadPayload(row[1]),
|
||||
}));
|
||||
|
||||
return [results, Number(countResult.rows?.[0]?.[0] ?? 0)];
|
||||
});
|
||||
}
|
||||
|
||||
async getUserId(): Promise<string> {
|
||||
await this.initialize();
|
||||
|
||||
const rows = await this.withConnection(async (connection) => {
|
||||
const result = await connection.execute<any[]>(
|
||||
"SELECT user_id FROM memory_migrations WHERE id = 1",
|
||||
);
|
||||
return result.rows ?? [];
|
||||
});
|
||||
|
||||
if (rows.length > 0) return rows[0][0];
|
||||
|
||||
const generatedUserId = uuidv4();
|
||||
await this.setUserId(generatedUserId);
|
||||
return generatedUserId;
|
||||
}
|
||||
|
||||
async setUserId(userId: string): Promise<void> {
|
||||
await this.initialize();
|
||||
|
||||
await this.withConnection(async (connection) => {
|
||||
await connection.execute("DELETE FROM memory_migrations WHERE id = 1");
|
||||
await connection.execute(
|
||||
"INSERT INTO memory_migrations (id, user_id) VALUES (1, :user_id)",
|
||||
{ user_id: userId },
|
||||
);
|
||||
}, true);
|
||||
}
|
||||
|
||||
async close(): Promise<void> {
|
||||
if (this.client && this.ownsClient) {
|
||||
await this.client.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,328 @@
|
||||
/// <reference types="jest" />
|
||||
/** Oracle AI Vector Search filter, config and SQL tests. The driver is mocked, so no database is needed. */
|
||||
const DB_TYPE_VECTOR = { name: "DB_TYPE_VECTOR" };
|
||||
const DB_TYPE_JSON = { name: "DB_TYPE_JSON" };
|
||||
|
||||
jest.mock("oracledb", () => ({ thin: true, DB_TYPE_VECTOR, DB_TYPE_JSON }), {
|
||||
virtual: true,
|
||||
});
|
||||
|
||||
import {
|
||||
OracleAIVectorSearch,
|
||||
buildWhereClause,
|
||||
quoteIdentifier,
|
||||
} from "../src/vector_stores/oracledb";
|
||||
|
||||
type Call = { sql: string; binds: any };
|
||||
|
||||
function fakeConnection(calls: Call[], resultsBySql: Array<any[][]>) {
|
||||
let selectIndex = 0;
|
||||
return {
|
||||
oracleServerVersionString: "23.4.0.24.05",
|
||||
async execute(sql: string, binds: any = {}) {
|
||||
calls.push({ sql: sql.replace(/\s+/g, " ").trim(), binds });
|
||||
if (/^\s*SELECT/i.test(sql)) {
|
||||
return { rows: resultsBySql[selectIndex++] ?? [] };
|
||||
}
|
||||
return { rows: [] };
|
||||
},
|
||||
async commit() {},
|
||||
async rollback() {},
|
||||
async close() {},
|
||||
};
|
||||
}
|
||||
|
||||
function makeStore(
|
||||
calls: Call[],
|
||||
results: Array<any[][]> = [],
|
||||
overrides = {},
|
||||
) {
|
||||
return new OracleAIVectorSearch({
|
||||
client: fakeConnection(calls, results) as any,
|
||||
collectionName: "mem0",
|
||||
embeddingModelDims: 3,
|
||||
...overrides,
|
||||
} as any);
|
||||
}
|
||||
|
||||
describe("quoteIdentifier", () => {
|
||||
it("quotes a bare name", () => {
|
||||
expect(quoteIdentifier("mem0")).toBe('"mem0"');
|
||||
});
|
||||
|
||||
it("quotes each segment of a schema-qualified name", () => {
|
||||
expect(quoteIdentifier("app.mem0")).toBe('"app"."mem0"');
|
||||
});
|
||||
|
||||
it("preserves already-quoted segments", () => {
|
||||
expect(quoteIdentifier('"App"."Mem0"')).toBe('"App"."Mem0"');
|
||||
});
|
||||
|
||||
it("rejects a name that would break out of the quoting", () => {
|
||||
expect(() => quoteIdentifier('mem0" (x); DROP TABLE t--')).toThrow(
|
||||
/is not valid/,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("buildWhereClause", () => {
|
||||
it("returns no clause for empty filters", () => {
|
||||
expect(buildWhereClause(undefined)).toEqual(["", {}]);
|
||||
expect(buildWhereClause({})).toEqual(["", {}]);
|
||||
});
|
||||
|
||||
it("binds a scalar equality instead of inlining it", () => {
|
||||
const [clause, binds] = buildWhereClause({ user_id: "alice" });
|
||||
expect(clause).toBe(
|
||||
`WHERE JSON_EXISTS(payload, '$."user_id"?(@ == $f_0)' PASSING :f_0 AS "f_0")`,
|
||||
);
|
||||
expect(binds).toEqual({ f_0: "alice" });
|
||||
});
|
||||
|
||||
it("ANDs multiple fields", () => {
|
||||
const [clause, binds] = buildWhereClause({
|
||||
user_id: "alice",
|
||||
agent_id: "bot",
|
||||
});
|
||||
expect(clause.startsWith("WHERE (")).toBe(true);
|
||||
expect(clause).toContain(" AND ");
|
||||
expect(binds).toEqual({ f_0: "alice", f_1: "bot" });
|
||||
});
|
||||
|
||||
it("applies every operator in a compound range filter", () => {
|
||||
const [clause, binds] = buildWhereClause({ age: { gte: 10, lte: 20 } });
|
||||
expect(clause).toContain("@ >= $f_0 && @ <= $f_1");
|
||||
expect(binds).toEqual({ f_0: 10, f_1: 20 });
|
||||
});
|
||||
|
||||
it("builds an existence check for the wildcard filter", () => {
|
||||
const [clause, binds] = buildWhereClause({ user_id: "*" });
|
||||
expect(clause).toBe(`WHERE JSON_EXISTS(payload, '$."user_id"')`);
|
||||
expect(binds).toEqual({});
|
||||
});
|
||||
|
||||
it("builds membership for in and negates it for nin", () => {
|
||||
const [inClause] = buildWhereClause({ user_id: { in: ["a", "b"] } });
|
||||
expect(inClause).toContain("@ in ($f_0, $f_1)");
|
||||
expect(inClause).not.toContain("NOT (");
|
||||
|
||||
const [ninClause] = buildWhereClause({ user_id: { nin: ["a"] } });
|
||||
expect(ninClause).toContain("NOT (");
|
||||
});
|
||||
|
||||
it("lowercases the operand for icontains", () => {
|
||||
const [clause, binds] = buildWhereClause({ data: { icontains: "SciFi" } });
|
||||
expect(clause).toContain("@.lower() has substring $f_0");
|
||||
expect(binds).toEqual({ f_0: "scifi" });
|
||||
});
|
||||
|
||||
it("ORs the branches of a $or group", () => {
|
||||
const [clause, binds] = buildWhereClause({
|
||||
$or: [{ user_id: "alice" }, { agent_id: "bot" }],
|
||||
});
|
||||
expect(clause).toContain(" OR ");
|
||||
expect(binds).toEqual({ f_0: "alice", f_1: "bot" });
|
||||
});
|
||||
|
||||
it("negates a $not group", () => {
|
||||
const [clause] = buildWhereClause({ $not: [{ user_id: "alice" }] });
|
||||
expect(clause.startsWith("WHERE NOT (")).toBe(true);
|
||||
});
|
||||
|
||||
it("nests logical groups", () => {
|
||||
const [clause, binds] = buildWhereClause({
|
||||
user_id: "alice",
|
||||
$or: [{ agent_id: "bot" }, { run_id: "r1" }],
|
||||
});
|
||||
expect(clause).toContain(" AND ");
|
||||
expect(clause).toContain(" OR ");
|
||||
expect(Object.keys(binds)).toEqual(["f_0", "f_1", "f_2"]);
|
||||
});
|
||||
|
||||
it("compares against JSON null without a bind", () => {
|
||||
const [clause, binds] = buildWhereClause({ agent_id: null });
|
||||
expect(clause).toBe(
|
||||
`WHERE JSON_EXISTS(payload, '$."agent_id"?(@ == null)')`,
|
||||
);
|
||||
expect(binds).toEqual({});
|
||||
});
|
||||
|
||||
it("rejects a metadata key that could escape the JSON path", () => {
|
||||
expect(() => buildWhereClause({ 'a"?(1==1))--': "x" })).toThrow(
|
||||
/Invalid metadata key/,
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects an unsupported field operator", () => {
|
||||
expect(() => buildWhereClause({ age: { regex: "^a" } })).toThrow(
|
||||
/Unsupported Oracle filter operator/,
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects an unsupported logical operator", () => {
|
||||
expect(() => buildWhereClause({ $nor: [{ a: 1 }] })).toThrow(
|
||||
/Unsupported Oracle logical filter operator/,
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects an empty in list", () => {
|
||||
expect(() => buildWhereClause({ user_id: { in: [] } })).toThrow(
|
||||
/non-empty array/,
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects a non-scalar comparison operand", () => {
|
||||
expect(() => buildWhereClause({ age: { gt: [1] } })).toThrow(
|
||||
/requires a scalar value/,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("OracleAIVectorSearch config validation", () => {
|
||||
it("requires connectionParams or client", () => {
|
||||
expect(() => new OracleAIVectorSearch({} as any)).toThrow(
|
||||
/connectionParams.*client/,
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects an unsupported distance metric", () => {
|
||||
expect(() => makeStore([], [], { distanceMetric: "JACCARD" })).toThrow(
|
||||
/Unsupported distance metric/,
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects a non-positive embedding dimension", () => {
|
||||
expect(() => makeStore([], [], { embeddingModelDims: 0 })).toThrow(
|
||||
/positive integer/,
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects an out-of-range index accuracy", () => {
|
||||
expect(() => makeStore([], [], { indexAccuracy: 101 })).toThrow(
|
||||
/between 1 and 100/,
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects an index parameter that does not belong to the index type", () => {
|
||||
expect(() =>
|
||||
makeStore([], [], {
|
||||
indexType: "HNSW",
|
||||
indexParameters: { samples_per_partition: 10 },
|
||||
}),
|
||||
).toThrow(/Unsupported HNSW index parameter/);
|
||||
});
|
||||
|
||||
it("rejects an index parameter outside its allowed range", () => {
|
||||
expect(() =>
|
||||
makeStore([], [], { indexParameters: { neighbors: 1 } }),
|
||||
).toThrow(/between 2 and 2048/);
|
||||
});
|
||||
});
|
||||
|
||||
describe("OracleAIVectorSearch SQL", () => {
|
||||
it("creates the table and a vector index on initialize", async () => {
|
||||
const calls: Call[] = [];
|
||||
await makeStore(calls, [], {
|
||||
indexParameters: { neighbors: 32, efconstruction: 200 },
|
||||
indexAccuracy: 95,
|
||||
}).initialize();
|
||||
|
||||
const ddl = calls.map((c) => c.sql).join("\n");
|
||||
expect(ddl).toContain(
|
||||
'CREATE TABLE IF NOT EXISTS "mem0" ( id VARCHAR2(36) PRIMARY KEY, vector VECTOR(3), payload JSON )',
|
||||
);
|
||||
expect(ddl).toContain(
|
||||
'CREATE VECTOR INDEX IF NOT EXISTS "mem0_VEC_IDX" ON "mem0" (vector) ORGANIZATION INMEMORY NEIGHBOR GRAPH DISTANCE COSINE WITH TARGET ACCURACY 95 PARAMETERS (type HNSW, neighbors 32, efconstruction 200)',
|
||||
);
|
||||
});
|
||||
|
||||
it("skips index creation when doCreateIndex is false", async () => {
|
||||
const calls: Call[] = [];
|
||||
await makeStore(calls, [], { doCreateIndex: false }).initialize();
|
||||
expect(calls.map((c) => c.sql).join("\n")).not.toContain(
|
||||
"CREATE VECTOR INDEX",
|
||||
);
|
||||
});
|
||||
|
||||
it("binds vectors as DB_TYPE_VECTOR and payloads as DB_TYPE_JSON on insert", async () => {
|
||||
const calls: Call[] = [];
|
||||
await makeStore(calls).insert([[1, 2, 3]], ["id-1"], [{ data: "hello" }]);
|
||||
|
||||
const insert = calls.find((c) => c.sql.startsWith("INSERT INTO"))!;
|
||||
expect(insert.binds.id).toBe("id-1");
|
||||
expect(insert.binds.vector.type).toBe(DB_TYPE_VECTOR);
|
||||
expect(insert.binds.vector.val).toEqual(new Float32Array([1, 2, 3]));
|
||||
expect(insert.binds.payload).toEqual({
|
||||
type: DB_TYPE_JSON,
|
||||
val: { data: "hello" },
|
||||
});
|
||||
});
|
||||
|
||||
it("converts cosine distance to a similarity score", async () => {
|
||||
const calls: Call[] = [];
|
||||
const store = makeStore(calls, [[["id-1", { data: "hello" }, 0.25]]]);
|
||||
const results = await store.search([1, 2, 3], 5);
|
||||
|
||||
expect(results).toEqual([
|
||||
{ id: "id-1", payload: { data: "hello" }, score: 0.75 },
|
||||
]);
|
||||
const select = calls.find((c) => c.sql.startsWith("SELECT id, payload,"))!;
|
||||
expect(select.sql).toContain(
|
||||
"VECTOR_DISTANCE(vector, :query_vec, COSINE) distance",
|
||||
);
|
||||
expect(select.sql).toContain("FETCH APPROX FIRST :max_rows ROWS ONLY");
|
||||
expect(select.binds.max_rows).toBe(5);
|
||||
});
|
||||
|
||||
it("inverts the sign of a DOT distance", async () => {
|
||||
const store = makeStore([], [[["id-1", {}, -0.4]]], {
|
||||
distanceMetric: "DOT",
|
||||
});
|
||||
const [result] = await store.search([1, 2, 3]);
|
||||
expect(result.score).toBeCloseTo(0.4);
|
||||
});
|
||||
|
||||
it("parses a payload returned as a JSON string", async () => {
|
||||
const store = makeStore([], [[["id-1", '{"data":"hello"}']]]);
|
||||
expect(await store.get("id-1")).toEqual({
|
||||
id: "id-1",
|
||||
payload: { data: "hello" },
|
||||
});
|
||||
});
|
||||
|
||||
it("returns null when get finds no row", async () => {
|
||||
expect(await makeStore([], [[]]).get("missing")).toBeNull();
|
||||
});
|
||||
|
||||
it("generates and persists a UUID user id when none is stored", async () => {
|
||||
const calls: Call[] = [];
|
||||
const userId = await makeStore(calls, [[]]).getUserId();
|
||||
|
||||
expect(userId).toMatch(
|
||||
/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/,
|
||||
);
|
||||
const insert = calls.find((c) =>
|
||||
c.sql.startsWith("INSERT INTO memory_migrations"),
|
||||
)!;
|
||||
expect(insert.binds).toEqual({ user_id: userId });
|
||||
});
|
||||
|
||||
it("returns the stored user id when one exists", async () => {
|
||||
expect(await makeStore([], [[["alice"]]]).getUserId()).toBe("alice");
|
||||
});
|
||||
|
||||
it("returns rows and the total count from list", async () => {
|
||||
const calls: Call[] = [];
|
||||
const store = makeStore(calls, [[["id-1", { data: "hello" }]], [[7]]]);
|
||||
const [results, count] = await store.list({ user_id: "alice" }, 10);
|
||||
|
||||
expect(results).toEqual([{ id: "id-1", payload: { data: "hello" } }]);
|
||||
expect(count).toBe(7);
|
||||
|
||||
const list = calls.find((c) =>
|
||||
c.sql.startsWith("SELECT id, payload FROM"),
|
||||
)!;
|
||||
expect(list.sql).toContain("WHERE JSON_EXISTS(payload,");
|
||||
expect(list.binds).toEqual({ f_0: "alice", max_rows: 10 });
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user