feat(vector-stores): add S3 Vectors provider to TypeScript OSS SDK (#5822)
Co-authored-by: kartik-mem0 <kartik.labhshetwar@mem0.ai>
This commit is contained in:
Rod Boev
@@ -28,4 +28,5 @@ export * from "./vector_stores/langchain";
|
||||
export * from "./vector_stores/vectorize";
|
||||
export * from "./vector_stores/azure_ai_search";
|
||||
export * from "./vector_stores/pgvector";
|
||||
export * from "./vector_stores/s3_vectors";
|
||||
export * from "./utils/factory";
|
||||
|
||||
@@ -38,6 +38,7 @@ import { LangchainEmbedder } from "../embeddings/langchain";
|
||||
import { LangchainVectorStore } from "../vector_stores/langchain";
|
||||
import { AzureAISearch } from "../vector_stores/azure_ai_search";
|
||||
import { PGVector } from "../vector_stores/pgvector";
|
||||
import { S3Vectors } from "../vector_stores/s3_vectors";
|
||||
|
||||
export class EmbedderFactory {
|
||||
static create(provider: string, config: EmbeddingConfig): Embedder {
|
||||
@@ -116,6 +117,9 @@ export class VectorStoreFactory {
|
||||
return new AzureAISearch(config as any);
|
||||
case "pgvector":
|
||||
return new PGVector(config as any);
|
||||
case "s3-vectors":
|
||||
case "s3_vectors":
|
||||
return new S3Vectors(config as any);
|
||||
default:
|
||||
throw new Error(`Unsupported vector store provider: ${provider}`);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,841 @@
|
||||
import {
|
||||
CreateIndexCommand,
|
||||
CreateVectorBucketCommand,
|
||||
DeleteIndexCommand,
|
||||
DeleteVectorsCommand,
|
||||
GetIndexCommand,
|
||||
GetVectorBucketCommand,
|
||||
GetVectorsCommand,
|
||||
ListVectorsCommand,
|
||||
PutVectorsCommand,
|
||||
QueryVectorsCommand,
|
||||
S3VectorsClient,
|
||||
type DistanceMetric,
|
||||
type GetOutputVector,
|
||||
type ListOutputVector,
|
||||
type QueryOutputVector,
|
||||
type S3VectorsClientConfig,
|
||||
type VectorData,
|
||||
} from "@aws-sdk/client-s3vectors";
|
||||
import { VectorStore } from "./base";
|
||||
import { SearchFilters, VectorStoreConfig, VectorStoreResult } from "../types";
|
||||
|
||||
const MIGRATION_INDEX_NAME = "memory_migrations";
|
||||
const MIGRATION_VECTOR_KEY = "mem0-user";
|
||||
const DEFAULT_PAGE_SIZE = 500;
|
||||
const ALWAYS_FALSE_FILTER_KEY = "$alwaysFalse";
|
||||
|
||||
type S3Filter = Record<string, any>;
|
||||
|
||||
interface S3VectorsConfig extends VectorStoreConfig {
|
||||
vectorBucketName: string;
|
||||
collectionName: string;
|
||||
embeddingModelDims?: number;
|
||||
dimension?: number;
|
||||
distanceMetric?: DistanceMetric | "cosine" | "euclidean";
|
||||
region?: string;
|
||||
regionName?: string;
|
||||
client?: S3VectorsClientLike;
|
||||
clientConfig?: S3VectorsClientConfig;
|
||||
}
|
||||
|
||||
interface S3VectorsClientLike {
|
||||
send(command: any): Promise<any>;
|
||||
}
|
||||
|
||||
export class S3Vectors implements VectorStore {
|
||||
private readonly client: S3VectorsClientLike;
|
||||
private readonly vectorBucketName: string;
|
||||
private readonly collectionName: string;
|
||||
private readonly dimension: number;
|
||||
private readonly distanceMetric: DistanceMetric | "cosine" | "euclidean";
|
||||
private _initPromise?: Promise<void>;
|
||||
private cachedUserId?: string;
|
||||
|
||||
constructor(config: S3VectorsConfig) {
|
||||
if (!config.vectorBucketName) {
|
||||
throw new Error("vectorBucketName is required");
|
||||
}
|
||||
if (!config.collectionName) {
|
||||
throw new Error("collectionName is required");
|
||||
}
|
||||
|
||||
const dimension = config.embeddingModelDims ?? config.dimension;
|
||||
if (!dimension || dimension < 1) {
|
||||
throw new Error("embeddingModelDims or dimension is required");
|
||||
}
|
||||
|
||||
this.vectorBucketName = config.vectorBucketName;
|
||||
this.collectionName = config.collectionName;
|
||||
this.dimension = dimension;
|
||||
this.distanceMetric = config.distanceMetric || "cosine";
|
||||
this.client =
|
||||
config.client ||
|
||||
new S3VectorsClient({
|
||||
...(config.clientConfig || {}),
|
||||
...(config.region || config.regionName
|
||||
? { region: config.region || config.regionName }
|
||||
: {}),
|
||||
});
|
||||
|
||||
void this.initialize().catch(console.error);
|
||||
}
|
||||
|
||||
async initialize(): Promise<void> {
|
||||
if (!this._initPromise) {
|
||||
this._initPromise = this._doInitialize();
|
||||
}
|
||||
return this._initPromise;
|
||||
}
|
||||
|
||||
private async _doInitialize(): Promise<void> {
|
||||
await this.ensureBucketExists();
|
||||
await this.ensureIndexExists(
|
||||
this.collectionName,
|
||||
this.dimension,
|
||||
this.distanceMetric,
|
||||
);
|
||||
}
|
||||
|
||||
async insert(
|
||||
vectors: number[][],
|
||||
ids: string[],
|
||||
payloads: Record<string, any>[],
|
||||
): Promise<void> {
|
||||
await this.initialize();
|
||||
this.assertBatchDimensions(vectors, "Insert");
|
||||
|
||||
await this.client.send(
|
||||
new PutVectorsCommand({
|
||||
vectorBucketName: this.vectorBucketName,
|
||||
indexName: this.collectionName,
|
||||
vectors: vectors.map((vector, index) => ({
|
||||
key: ids[index],
|
||||
data: this.toVectorData(vector),
|
||||
metadata: payloads[index] || {},
|
||||
})),
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
async keywordSearch(): Promise<null> {
|
||||
return null;
|
||||
}
|
||||
|
||||
async search(
|
||||
query: number[],
|
||||
topK: number = 5,
|
||||
filters?: SearchFilters,
|
||||
): Promise<VectorStoreResult[]> {
|
||||
await this.initialize();
|
||||
this.assertVectorDimension(query, "Query");
|
||||
|
||||
const filter = this.convertFilters(filters);
|
||||
if (filter && this.isAlwaysFalseFilter(filter)) {
|
||||
return [];
|
||||
}
|
||||
const response = await this.client.send(
|
||||
new QueryVectorsCommand({
|
||||
vectorBucketName: this.vectorBucketName,
|
||||
indexName: this.collectionName,
|
||||
queryVector: this.toVectorData(query),
|
||||
topK,
|
||||
filter,
|
||||
returnMetadata: true,
|
||||
returnDistance: true,
|
||||
}),
|
||||
);
|
||||
|
||||
const responseDistanceMetric =
|
||||
response.distanceMetric || this.distanceMetric;
|
||||
return (response.vectors || [])
|
||||
.map((vector: QueryOutputVector) =>
|
||||
this.normalizeQueryVector(vector, responseDistanceMetric),
|
||||
)
|
||||
.slice(0, topK);
|
||||
}
|
||||
|
||||
async get(vectorId: string): Promise<VectorStoreResult | null> {
|
||||
await this.initialize();
|
||||
|
||||
try {
|
||||
const response = await this.client.send(
|
||||
new GetVectorsCommand({
|
||||
vectorBucketName: this.vectorBucketName,
|
||||
indexName: this.collectionName,
|
||||
keys: [vectorId],
|
||||
returnMetadata: true,
|
||||
}),
|
||||
);
|
||||
|
||||
const vector = response.vectors?.[0];
|
||||
if (!vector) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return this.normalizeStoredVector(vector);
|
||||
} catch (error) {
|
||||
if (this.isNotFound(error)) {
|
||||
return null;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async update(
|
||||
vectorId: string,
|
||||
vector: number[],
|
||||
payload: Record<string, any>,
|
||||
): Promise<void> {
|
||||
await this.initialize();
|
||||
|
||||
let nextVector = vector;
|
||||
let nextPayload = payload || {};
|
||||
|
||||
if (vector.length === 0 || !payload) {
|
||||
const existing = await this.fetchStoredVector(vectorId);
|
||||
if (!existing) {
|
||||
throw new Error(`Vector with ID ${vectorId} not found`);
|
||||
}
|
||||
nextVector = vector.length > 0 ? vector : existing.vector;
|
||||
nextPayload = payload || existing.payload;
|
||||
}
|
||||
|
||||
this.assertVectorDimension(nextVector, "Vector");
|
||||
|
||||
await this.client.send(
|
||||
new PutVectorsCommand({
|
||||
vectorBucketName: this.vectorBucketName,
|
||||
indexName: this.collectionName,
|
||||
vectors: [
|
||||
{
|
||||
key: vectorId,
|
||||
data: this.toVectorData(nextVector),
|
||||
metadata: nextPayload,
|
||||
},
|
||||
],
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
async delete(vectorId: string): Promise<void> {
|
||||
await this.initialize();
|
||||
|
||||
await this.client.send(
|
||||
new DeleteVectorsCommand({
|
||||
vectorBucketName: this.vectorBucketName,
|
||||
indexName: this.collectionName,
|
||||
keys: [vectorId],
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
async deleteCol(): Promise<void> {
|
||||
await this.initialize();
|
||||
|
||||
try {
|
||||
await this.client.send(
|
||||
new DeleteIndexCommand({
|
||||
vectorBucketName: this.vectorBucketName,
|
||||
indexName: this.collectionName,
|
||||
}),
|
||||
);
|
||||
} catch (error) {
|
||||
if (this.isNotFound(error)) {
|
||||
return;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async list(
|
||||
filters?: SearchFilters,
|
||||
topK: number = 100,
|
||||
): Promise<[VectorStoreResult[], number]> {
|
||||
await this.initialize();
|
||||
|
||||
const filter = this.convertFilters(filters);
|
||||
const results: VectorStoreResult[] = [];
|
||||
let nextToken: string | undefined;
|
||||
|
||||
// Stop paginating once we have topK matches. Both callers of list()
|
||||
// discard the count, so scanning the whole index just to total every
|
||||
// match is wasted round-trips (O(index size) on getAll/deleteAll).
|
||||
// Return the page length like qdrant does.
|
||||
do {
|
||||
const response = await this.client.send(
|
||||
new ListVectorsCommand({
|
||||
vectorBucketName: this.vectorBucketName,
|
||||
indexName: this.collectionName,
|
||||
maxResults: DEFAULT_PAGE_SIZE,
|
||||
nextToken,
|
||||
returnMetadata: true,
|
||||
}),
|
||||
);
|
||||
|
||||
for (const vector of response.vectors || []) {
|
||||
if (results.length >= topK) {
|
||||
break;
|
||||
}
|
||||
const normalized = this.normalizeListedVector(vector);
|
||||
if (filter && !this.matchesFilter(normalized.payload, filter)) {
|
||||
continue;
|
||||
}
|
||||
results.push(normalized);
|
||||
}
|
||||
|
||||
nextToken = response.nextToken;
|
||||
} while (nextToken && results.length < topK);
|
||||
|
||||
return [results, results.length];
|
||||
}
|
||||
|
||||
async getUserId(): Promise<string> {
|
||||
await this.initialize();
|
||||
if (this.cachedUserId) {
|
||||
return this.cachedUserId;
|
||||
}
|
||||
|
||||
await this.ensureMigrationIndex();
|
||||
|
||||
const response = await this.client.send(
|
||||
new GetVectorsCommand({
|
||||
vectorBucketName: this.vectorBucketName,
|
||||
indexName: MIGRATION_INDEX_NAME,
|
||||
keys: [MIGRATION_VECTOR_KEY],
|
||||
returnMetadata: true,
|
||||
}),
|
||||
);
|
||||
|
||||
const metadata = this.normalizeMetadata(response.vectors?.[0]?.metadata);
|
||||
const userId = metadata.user_id;
|
||||
if (typeof userId === "string" && userId.length > 0) {
|
||||
this.cachedUserId = userId;
|
||||
return userId;
|
||||
}
|
||||
|
||||
const randomUserId =
|
||||
Math.random().toString(36).substring(2, 15) +
|
||||
Math.random().toString(36).substring(2, 15);
|
||||
await this.setUserId(randomUserId);
|
||||
return randomUserId;
|
||||
}
|
||||
|
||||
async setUserId(userId: string): Promise<void> {
|
||||
await this.initialize();
|
||||
await this.ensureMigrationIndex();
|
||||
|
||||
await this.client.send(
|
||||
new PutVectorsCommand({
|
||||
vectorBucketName: this.vectorBucketName,
|
||||
indexName: MIGRATION_INDEX_NAME,
|
||||
vectors: [
|
||||
{
|
||||
key: MIGRATION_VECTOR_KEY,
|
||||
data: this.toVectorData([0]),
|
||||
metadata: { user_id: userId },
|
||||
},
|
||||
],
|
||||
}),
|
||||
);
|
||||
|
||||
this.cachedUserId = userId;
|
||||
}
|
||||
|
||||
private async ensureBucketExists(): Promise<void> {
|
||||
try {
|
||||
await this.client.send(
|
||||
new GetVectorBucketCommand({
|
||||
vectorBucketName: this.vectorBucketName,
|
||||
}),
|
||||
);
|
||||
} catch (error) {
|
||||
if (!this.isNotFound(error)) {
|
||||
throw error;
|
||||
}
|
||||
|
||||
try {
|
||||
await this.client.send(
|
||||
new CreateVectorBucketCommand({
|
||||
vectorBucketName: this.vectorBucketName,
|
||||
}),
|
||||
);
|
||||
} catch (createError) {
|
||||
if (!this.isConflict(createError)) {
|
||||
throw createError;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async ensureIndexExists(
|
||||
indexName: string,
|
||||
dimension: number,
|
||||
distanceMetric: DistanceMetric | "cosine" | "euclidean",
|
||||
): Promise<void> {
|
||||
try {
|
||||
await this.client.send(
|
||||
new GetIndexCommand({
|
||||
vectorBucketName: this.vectorBucketName,
|
||||
indexName,
|
||||
}),
|
||||
);
|
||||
} catch (error) {
|
||||
if (!this.isNotFound(error)) {
|
||||
throw error;
|
||||
}
|
||||
|
||||
try {
|
||||
await this.client.send(
|
||||
new CreateIndexCommand({
|
||||
vectorBucketName: this.vectorBucketName,
|
||||
indexName,
|
||||
dataType: "float32",
|
||||
dimension,
|
||||
distanceMetric,
|
||||
}),
|
||||
);
|
||||
} catch (createError) {
|
||||
if (!this.isConflict(createError)) {
|
||||
throw createError;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async ensureMigrationIndex(): Promise<void> {
|
||||
await this.ensureIndexExists(MIGRATION_INDEX_NAME, 1, "euclidean");
|
||||
}
|
||||
|
||||
private async fetchStoredVector(
|
||||
vectorId: string,
|
||||
): Promise<{ vector: number[]; payload: Record<string, any> } | null> {
|
||||
const response = await this.client.send(
|
||||
new GetVectorsCommand({
|
||||
vectorBucketName: this.vectorBucketName,
|
||||
indexName: this.collectionName,
|
||||
keys: [vectorId],
|
||||
returnData: true,
|
||||
returnMetadata: true,
|
||||
}),
|
||||
);
|
||||
|
||||
const vector = response.vectors?.[0];
|
||||
const data = this.fromVectorData(vector?.data);
|
||||
if (!vector || !data) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
vector: data,
|
||||
payload: this.normalizeMetadata(vector.metadata),
|
||||
};
|
||||
}
|
||||
|
||||
private convertFilters(filters?: SearchFilters): S3Filter | undefined {
|
||||
if (!filters || Object.keys(filters).length === 0) {
|
||||
return undefined;
|
||||
}
|
||||
const normalized = this.convertFilterNode(filters);
|
||||
return this.isNoOpFilter(normalized) ? undefined : normalized;
|
||||
}
|
||||
|
||||
private convertFilterNode(node: Record<string, any>): S3Filter {
|
||||
const clauses: S3Filter[] = [];
|
||||
|
||||
for (const [key, value] of Object.entries(node)) {
|
||||
if (key === "$and" || key === "$or") {
|
||||
if (!Array.isArray(value) || value.length === 0) {
|
||||
throw new Error(`${key} filter requires a non-empty list`);
|
||||
}
|
||||
const normalizedEntries = value.map((entry) =>
|
||||
this.convertFilterNode(entry),
|
||||
);
|
||||
const entries = normalizedEntries.filter(
|
||||
(entry) => !this.isNoOpFilter(entry),
|
||||
);
|
||||
if (key === "$and") {
|
||||
if (entries.some((entry) => this.isAlwaysFalseFilter(entry))) {
|
||||
return { [ALWAYS_FALSE_FILTER_KEY]: true };
|
||||
}
|
||||
if (entries.length === 0) {
|
||||
continue;
|
||||
}
|
||||
} else {
|
||||
if (normalizedEntries.some((entry) => this.isNoOpFilter(entry))) {
|
||||
continue;
|
||||
}
|
||||
const viableEntries = entries.filter(
|
||||
(entry) => !this.isAlwaysFalseFilter(entry),
|
||||
);
|
||||
if (viableEntries.length === 0) {
|
||||
return { [ALWAYS_FALSE_FILTER_KEY]: true };
|
||||
}
|
||||
clauses.push(
|
||||
viableEntries.length === 1
|
||||
? viableEntries[0]
|
||||
: { [key]: viableEntries },
|
||||
);
|
||||
continue;
|
||||
}
|
||||
clauses.push(entries.length === 1 ? entries[0] : { [key]: entries });
|
||||
continue;
|
||||
}
|
||||
|
||||
if (key === "$not") {
|
||||
if (!Array.isArray(value) || value.length === 0) {
|
||||
throw new Error("$not filter requires a non-empty list");
|
||||
}
|
||||
|
||||
const normalizedEntries = value.map((entry) =>
|
||||
this.convertFilterNode(entry),
|
||||
);
|
||||
if (normalizedEntries.some((entry) => this.isNoOpFilter(entry))) {
|
||||
return { [ALWAYS_FALSE_FILTER_KEY]: true };
|
||||
}
|
||||
const negated = normalizedEntries.filter(
|
||||
(entry) => !this.isAlwaysFalseFilter(entry),
|
||||
);
|
||||
if (negated.length === 0) {
|
||||
continue;
|
||||
}
|
||||
const negatedFilters = negated.map((entry) => this.negateFilter(entry));
|
||||
clauses.push(
|
||||
negatedFilters.length === 1
|
||||
? negatedFilters[0]
|
||||
: { $and: negatedFilters },
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
const fieldFilter = this.convertFieldFilter(key, value);
|
||||
if (this.isAlwaysFalseFilter(fieldFilter)) {
|
||||
return fieldFilter;
|
||||
}
|
||||
if (this.isNoOpFilter(fieldFilter)) {
|
||||
continue;
|
||||
}
|
||||
clauses.push(fieldFilter);
|
||||
}
|
||||
|
||||
if (clauses.some((entry) => this.isAlwaysFalseFilter(entry))) {
|
||||
return { [ALWAYS_FALSE_FILTER_KEY]: true };
|
||||
}
|
||||
if (clauses.length === 1) {
|
||||
return clauses[0];
|
||||
}
|
||||
if (clauses.length === 0) {
|
||||
return {};
|
||||
}
|
||||
|
||||
return { $and: clauses };
|
||||
}
|
||||
|
||||
private convertFieldFilter(key: string, value: any): S3Filter {
|
||||
if (value === "*") {
|
||||
return { [key]: { $exists: true } };
|
||||
}
|
||||
|
||||
if (Array.isArray(value)) {
|
||||
if (value.length === 0) {
|
||||
return { [ALWAYS_FALSE_FILTER_KEY]: true };
|
||||
}
|
||||
return { [key]: { $in: value } };
|
||||
}
|
||||
|
||||
if (typeof value !== "object" || value === null) {
|
||||
return { [key]: { $eq: value } };
|
||||
}
|
||||
|
||||
const operators: Record<string, any> = {};
|
||||
|
||||
for (const [operator, operand] of Object.entries(value)) {
|
||||
switch (operator) {
|
||||
case "eq":
|
||||
operators.$eq = operand;
|
||||
break;
|
||||
case "ne":
|
||||
operators.$ne = operand;
|
||||
break;
|
||||
case "gt":
|
||||
operators.$gt = operand;
|
||||
break;
|
||||
case "gte":
|
||||
operators.$gte = operand;
|
||||
break;
|
||||
case "lt":
|
||||
operators.$lt = operand;
|
||||
break;
|
||||
case "lte":
|
||||
operators.$lte = operand;
|
||||
break;
|
||||
case "in":
|
||||
if (!Array.isArray(operand) || operand.length === 0) {
|
||||
return { [ALWAYS_FALSE_FILTER_KEY]: true };
|
||||
}
|
||||
operators.$in = operand;
|
||||
break;
|
||||
case "nin":
|
||||
if (!Array.isArray(operand) || operand.length === 0) {
|
||||
break;
|
||||
}
|
||||
operators.$nin = operand;
|
||||
break;
|
||||
case "contains":
|
||||
case "icontains":
|
||||
case "startsWith":
|
||||
throw new Error(
|
||||
`S3 Vectors does not support '${operator}' metadata filters.`,
|
||||
);
|
||||
default:
|
||||
throw new Error(
|
||||
`Unsupported S3 Vectors filter operator: ${operator}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if (Object.keys(operators).length === 0) {
|
||||
return {};
|
||||
}
|
||||
return { [key]: operators };
|
||||
}
|
||||
|
||||
private negateFilter(filter: S3Filter): S3Filter {
|
||||
if (Array.isArray(filter.$and)) {
|
||||
return {
|
||||
$or: filter.$and.map((entry: S3Filter) => this.negateFilter(entry)),
|
||||
};
|
||||
}
|
||||
|
||||
if (Array.isArray(filter.$or)) {
|
||||
return {
|
||||
$and: filter.$or.map((entry: S3Filter) => this.negateFilter(entry)),
|
||||
};
|
||||
}
|
||||
|
||||
const entries = Object.entries(filter);
|
||||
if (entries.length !== 1) {
|
||||
throw new Error("S3 Vectors cannot negate this filter shape.");
|
||||
}
|
||||
|
||||
const [key, rawValue] = entries[0];
|
||||
if (key.startsWith("$")) {
|
||||
throw new Error("S3 Vectors cannot negate this filter shape.");
|
||||
}
|
||||
|
||||
const value =
|
||||
typeof rawValue === "object" && rawValue !== null
|
||||
? rawValue
|
||||
: { $eq: rawValue };
|
||||
const negated: Record<string, any> = {};
|
||||
|
||||
for (const [operator, operand] of Object.entries(value)) {
|
||||
switch (operator) {
|
||||
case "$eq":
|
||||
negated.$ne = operand;
|
||||
break;
|
||||
case "$ne":
|
||||
negated.$eq = operand;
|
||||
break;
|
||||
case "$gt":
|
||||
negated.$lte = operand;
|
||||
break;
|
||||
case "$gte":
|
||||
negated.$lt = operand;
|
||||
break;
|
||||
case "$lt":
|
||||
negated.$gte = operand;
|
||||
break;
|
||||
case "$lte":
|
||||
negated.$gt = operand;
|
||||
break;
|
||||
case "$in":
|
||||
negated.$nin = operand;
|
||||
break;
|
||||
case "$nin":
|
||||
negated.$in = operand;
|
||||
break;
|
||||
case "$exists":
|
||||
negated.$exists = !operand;
|
||||
break;
|
||||
default:
|
||||
throw new Error("S3 Vectors cannot negate this filter shape.");
|
||||
}
|
||||
}
|
||||
|
||||
return { [key]: negated };
|
||||
}
|
||||
|
||||
private matchesFilter(
|
||||
metadata: Record<string, any>,
|
||||
filter: S3Filter,
|
||||
): boolean {
|
||||
if (this.isAlwaysFalseFilter(filter)) {
|
||||
return false;
|
||||
}
|
||||
if (Array.isArray(filter.$and)) {
|
||||
return filter.$and.every((entry: S3Filter) =>
|
||||
this.matchesFilter(metadata, entry),
|
||||
);
|
||||
}
|
||||
|
||||
if (Array.isArray(filter.$or)) {
|
||||
return filter.$or.some((entry: S3Filter) =>
|
||||
this.matchesFilter(metadata, entry),
|
||||
);
|
||||
}
|
||||
|
||||
return Object.entries(filter).every(([key, rawValue]) =>
|
||||
this.matchesFieldFilter(metadata[key], rawValue),
|
||||
);
|
||||
}
|
||||
|
||||
private matchesFieldFilter(metadataValue: any, rawValue: any): boolean {
|
||||
const value =
|
||||
typeof rawValue === "object" && rawValue !== null
|
||||
? rawValue
|
||||
: { $eq: rawValue };
|
||||
|
||||
return Object.entries(value).every(([operator, operand]) =>
|
||||
this.evaluateOperator(metadataValue, operator, operand),
|
||||
);
|
||||
}
|
||||
|
||||
private evaluateOperator(
|
||||
metadataValue: any,
|
||||
operator: string,
|
||||
operand: any,
|
||||
): boolean {
|
||||
switch (operator) {
|
||||
case "$eq":
|
||||
if (Array.isArray(metadataValue)) {
|
||||
return metadataValue.some((entry) => entry === operand);
|
||||
}
|
||||
return metadataValue === operand;
|
||||
case "$ne":
|
||||
if (Array.isArray(metadataValue)) {
|
||||
return !metadataValue.some((entry) => entry === operand);
|
||||
}
|
||||
return metadataValue !== operand;
|
||||
case "$gt":
|
||||
return typeof metadataValue === "number" && metadataValue > operand;
|
||||
case "$gte":
|
||||
return typeof metadataValue === "number" && metadataValue >= operand;
|
||||
case "$lt":
|
||||
return typeof metadataValue === "number" && metadataValue < operand;
|
||||
case "$lte":
|
||||
return typeof metadataValue === "number" && metadataValue <= operand;
|
||||
case "$in":
|
||||
if (!Array.isArray(operand) || operand.length === 0) {
|
||||
return false;
|
||||
}
|
||||
if (Array.isArray(metadataValue)) {
|
||||
return metadataValue.some((entry) => operand.includes(entry));
|
||||
}
|
||||
return operand.includes(metadataValue);
|
||||
case "$nin":
|
||||
if (!Array.isArray(operand) || operand.length === 0) {
|
||||
return true;
|
||||
}
|
||||
if (Array.isArray(metadataValue)) {
|
||||
return metadataValue.every((entry) => !operand.includes(entry));
|
||||
}
|
||||
return !operand.includes(metadataValue);
|
||||
case "$exists":
|
||||
return operand
|
||||
? metadataValue !== undefined
|
||||
: metadataValue === undefined;
|
||||
default:
|
||||
throw new Error(`Unsupported S3 Vectors filter operator: ${operator}`);
|
||||
}
|
||||
}
|
||||
|
||||
private toVectorData(vector: number[]): VectorData {
|
||||
return { float32: vector };
|
||||
}
|
||||
|
||||
private fromVectorData(data?: VectorData): number[] | undefined {
|
||||
if (!data || !("float32" in data) || !Array.isArray(data.float32)) {
|
||||
return undefined;
|
||||
}
|
||||
return data.float32;
|
||||
}
|
||||
|
||||
private normalizeMetadata(metadata: any): Record<string, any> {
|
||||
if (metadata && typeof metadata === "object" && !Array.isArray(metadata)) {
|
||||
return { ...(metadata as Record<string, any>) };
|
||||
}
|
||||
return {};
|
||||
}
|
||||
|
||||
private normalizeQueryVector(
|
||||
vector: QueryOutputVector,
|
||||
distanceMetric: DistanceMetric | "cosine" | "euclidean",
|
||||
): VectorStoreResult {
|
||||
return {
|
||||
id: String(vector.key),
|
||||
payload: this.normalizeMetadata(vector.metadata),
|
||||
score: this.normalizeScore(vector.distance, distanceMetric),
|
||||
};
|
||||
}
|
||||
|
||||
private normalizeStoredVector(
|
||||
vector: GetOutputVector | ListOutputVector,
|
||||
): VectorStoreResult {
|
||||
return {
|
||||
id: String(vector.key),
|
||||
payload: this.normalizeMetadata(vector.metadata),
|
||||
};
|
||||
}
|
||||
|
||||
private normalizeListedVector(vector: ListOutputVector): VectorStoreResult {
|
||||
return this.normalizeStoredVector(vector);
|
||||
}
|
||||
|
||||
private normalizeScore(
|
||||
distance?: number,
|
||||
distanceMetric: DistanceMetric | "cosine" | "euclidean" = this
|
||||
.distanceMetric,
|
||||
): number | undefined {
|
||||
if (distance === undefined || distance === null) {
|
||||
return undefined;
|
||||
}
|
||||
if (!Number.isFinite(distance)) {
|
||||
return undefined;
|
||||
}
|
||||
if (distanceMetric === "euclidean") {
|
||||
return 1 / (1 + distance);
|
||||
}
|
||||
return Math.max(0, Math.min(1, 1 - distance));
|
||||
}
|
||||
|
||||
private assertVectorDimension(vector: number[], context: string): void {
|
||||
if (vector.length !== this.dimension) {
|
||||
throw new Error(
|
||||
`${context} dimension mismatch. Expected ${this.dimension}, got ${vector.length}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private assertBatchDimensions(vectors: number[][], context: string): void {
|
||||
for (const vector of vectors) {
|
||||
this.assertVectorDimension(vector, context);
|
||||
}
|
||||
}
|
||||
|
||||
private isNotFound(error: any): boolean {
|
||||
return error?.name === "NotFoundException";
|
||||
}
|
||||
|
||||
private isConflict(error: any): boolean {
|
||||
return error?.name === "ConflictException";
|
||||
}
|
||||
|
||||
private isAlwaysFalseFilter(filter: S3Filter): boolean {
|
||||
return filter[ALWAYS_FALSE_FILTER_KEY] === true;
|
||||
}
|
||||
|
||||
private isNoOpFilter(filter: S3Filter): boolean {
|
||||
return Object.keys(filter).length === 0;
|
||||
}
|
||||
}
|
||||
@@ -138,6 +138,11 @@ jest.mock("../src/vector_stores/pgvector", () => ({
|
||||
.fn()
|
||||
.mockImplementation((config) => ({ type: "pgvector", config })),
|
||||
}));
|
||||
jest.mock("../src/vector_stores/s3_vectors", () => ({
|
||||
S3Vectors: jest
|
||||
.fn()
|
||||
.mockImplementation((config) => ({ type: "s3-vectors", config })),
|
||||
}));
|
||||
jest.mock("../src/storage/SupabaseHistoryManager", () => ({
|
||||
SupabaseHistoryManager: jest
|
||||
.fn()
|
||||
@@ -260,6 +265,8 @@ describe("VectorStoreFactory", () => {
|
||||
["vectorize"],
|
||||
["azure-ai-search"],
|
||||
["pgvector"],
|
||||
["s3-vectors"],
|
||||
["s3_vectors"],
|
||||
])("creates vector store for provider '%s'", (provider) => {
|
||||
expect(() =>
|
||||
VectorStoreFactory.create(provider, dummyVSConfig),
|
||||
|
||||
@@ -14,7 +14,6 @@ describe("tsup.config.ts externals", () => {
|
||||
beforeAll(() => {
|
||||
const pkgPath = path.resolve(__dirname, "../../../package.json");
|
||||
const pkg = JSON.parse(fs.readFileSync(pkgPath, "utf-8"));
|
||||
// Filter out @types/* packages — they are type-only and not bundled at runtime
|
||||
peerDeps = Object.keys(pkg.peerDependencies || {}).filter(
|
||||
(dep) => !dep.startsWith("@types/"),
|
||||
);
|
||||
@@ -22,16 +21,16 @@ describe("tsup.config.ts externals", () => {
|
||||
|
||||
const tsupConfigPath = path.resolve(__dirname, "../../../tsup.config.ts");
|
||||
const tsupContent = fs.readFileSync(tsupConfigPath, "utf-8");
|
||||
|
||||
// Extract strings from the external array (supports double, single, and backtick quotes)
|
||||
const externalMatch = tsupContent.match(
|
||||
/const external\s*=\s*\[([\s\S]*?)\];/,
|
||||
);
|
||||
|
||||
if (!externalMatch) {
|
||||
throw new Error("Could not find external array in tsup.config.ts");
|
||||
}
|
||||
|
||||
const matches = externalMatch[1].match(/["'`]([^"'`]+)["'`]/g);
|
||||
externalDeps = (matches || []).map((m) => m.replace(/["'`]/g, ""));
|
||||
externalDeps = (matches || []).map((value) => value.replace(/["'`]/g, ""));
|
||||
});
|
||||
|
||||
it("should have every peerDependency in the external array", () => {
|
||||
|
||||
@@ -720,7 +720,358 @@ describe("AzureAISearch – backward compat with mocked client", () => {
|
||||
});
|
||||
|
||||
// ───────────────────────────────────────────────────────────────────────────
|
||||
// 6. Vectorize — mock Cloudflare client, test idempotent init
|
||||
// 6. S3 Vectors — mock AWS client, test interface + init
|
||||
// ───────────────────────────────────────────────────────────────────────────
|
||||
describe("S3 Vectors – backward compat with mocked client", () => {
|
||||
function createMockS3VectorsClient(options?: {
|
||||
queryDistance?: number;
|
||||
queryDistanceMetric?: "cosine" | "euclidean";
|
||||
}) {
|
||||
const buckets = new Set<string>();
|
||||
const indexes = new Map<
|
||||
string,
|
||||
{
|
||||
dimension: number;
|
||||
distanceMetric: string;
|
||||
}
|
||||
>();
|
||||
const vectors = new Map<
|
||||
string,
|
||||
{
|
||||
key: string;
|
||||
data: { float32: number[] };
|
||||
metadata: Record<string, any>;
|
||||
}
|
||||
>();
|
||||
|
||||
const vectorMapKey = (indexName: string, key: string) =>
|
||||
`${indexName}:${key}`;
|
||||
|
||||
return {
|
||||
send: jest.fn().mockImplementation(async (command: any) => {
|
||||
const name = command.constructor.name;
|
||||
const input = command.input;
|
||||
|
||||
switch (name) {
|
||||
case "GetVectorBucketCommand":
|
||||
if (!buckets.has(input.vectorBucketName)) {
|
||||
const error: any = new Error("Bucket not found");
|
||||
error.name = "NotFoundException";
|
||||
throw error;
|
||||
}
|
||||
return {};
|
||||
case "CreateVectorBucketCommand":
|
||||
if (buckets.has(input.vectorBucketName)) {
|
||||
const error: any = new Error("Bucket exists");
|
||||
error.name = "ConflictException";
|
||||
throw error;
|
||||
}
|
||||
buckets.add(input.vectorBucketName);
|
||||
return {};
|
||||
case "GetIndexCommand":
|
||||
if (!indexes.has(input.indexName)) {
|
||||
const error: any = new Error("Index not found");
|
||||
error.name = "NotFoundException";
|
||||
throw error;
|
||||
}
|
||||
return {};
|
||||
case "CreateIndexCommand":
|
||||
if (indexes.has(input.indexName)) {
|
||||
const error: any = new Error("Index exists");
|
||||
error.name = "ConflictException";
|
||||
throw error;
|
||||
}
|
||||
indexes.set(input.indexName, {
|
||||
dimension: input.dimension,
|
||||
distanceMetric: input.distanceMetric,
|
||||
});
|
||||
return {};
|
||||
case "PutVectorsCommand":
|
||||
for (const vector of input.vectors || []) {
|
||||
vectors.set(vectorMapKey(input.indexName, vector.key), {
|
||||
key: vector.key,
|
||||
data: vector.data,
|
||||
metadata: vector.metadata || {},
|
||||
});
|
||||
}
|
||||
return {};
|
||||
case "QueryVectorsCommand":
|
||||
return {
|
||||
distanceMetric: options?.queryDistanceMetric ?? "cosine",
|
||||
vectors: [
|
||||
{
|
||||
key: "doc-1",
|
||||
metadata: { user_id: "u1", topic: "alpha" },
|
||||
distance: options?.queryDistance ?? 0.25,
|
||||
},
|
||||
],
|
||||
};
|
||||
case "GetVectorsCommand":
|
||||
return {
|
||||
vectors: (input.keys || [])
|
||||
.map((key: string) =>
|
||||
vectors.get(vectorMapKey(input.indexName, key)),
|
||||
)
|
||||
.filter(Boolean),
|
||||
};
|
||||
case "DeleteVectorsCommand":
|
||||
for (const key of input.keys || []) {
|
||||
vectors.delete(vectorMapKey(input.indexName, key));
|
||||
}
|
||||
return {};
|
||||
case "DeleteIndexCommand":
|
||||
indexes.delete(input.indexName);
|
||||
for (const key of Array.from(vectors.keys())) {
|
||||
if (key.startsWith(`${input.indexName}:`)) {
|
||||
vectors.delete(key);
|
||||
}
|
||||
}
|
||||
return {};
|
||||
case "ListVectorsCommand":
|
||||
return {
|
||||
vectors: Array.from(vectors.values())
|
||||
.filter((entry) =>
|
||||
vectorMapKey(input.indexName, entry.key).startsWith(
|
||||
`${input.indexName}:`,
|
||||
),
|
||||
)
|
||||
.map((entry) => ({
|
||||
key: entry.key,
|
||||
metadata: entry.metadata,
|
||||
})),
|
||||
};
|
||||
default:
|
||||
throw new Error(`Unexpected S3Vectors command: ${name}`);
|
||||
}
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
function findCommandInput(
|
||||
client: { send: jest.Mock },
|
||||
commandName: string,
|
||||
): Record<string, any> | undefined {
|
||||
const match = client.send.mock.calls.find(
|
||||
([command]) => command.constructor.name === commandName,
|
||||
);
|
||||
return match?.[0]?.input;
|
||||
}
|
||||
|
||||
it("implements full VectorStore interface", () => {
|
||||
const { S3Vectors } = require("../src/vector_stores/s3_vectors");
|
||||
const store = new S3Vectors({
|
||||
client: createMockS3VectorsClient(),
|
||||
vectorBucketName: "test-bucket",
|
||||
collectionName: "test-index",
|
||||
embeddingModelDims: 3,
|
||||
});
|
||||
expect(typeof store.insert).toBe("function");
|
||||
expect(typeof store.search).toBe("function");
|
||||
expect(typeof store.get).toBe("function");
|
||||
expect(typeof store.update).toBe("function");
|
||||
expect(typeof store.delete).toBe("function");
|
||||
expect(typeof store.deleteCol).toBe("function");
|
||||
expect(typeof store.list).toBe("function");
|
||||
expect(typeof store.getUserId).toBe("function");
|
||||
expect(typeof store.setUserId).toBe("function");
|
||||
expect(typeof store.initialize).toBe("function");
|
||||
});
|
||||
|
||||
it("initialize() is idempotent (same promise returned)", async () => {
|
||||
const { S3Vectors } = require("../src/vector_stores/s3_vectors");
|
||||
const mockClient = createMockS3VectorsClient();
|
||||
const store = new S3Vectors({
|
||||
client: mockClient,
|
||||
vectorBucketName: "test-bucket",
|
||||
collectionName: "test-index",
|
||||
embeddingModelDims: 3,
|
||||
});
|
||||
|
||||
const p1 = store.initialize();
|
||||
const p2 = store.initialize();
|
||||
const p3 = store.initialize();
|
||||
|
||||
await Promise.all([p1, p2, p3]);
|
||||
|
||||
expect(
|
||||
mockClient.send.mock.calls.filter(
|
||||
([command]) => command.constructor.name === "CreateVectorBucketCommand",
|
||||
),
|
||||
).toHaveLength(1);
|
||||
expect(
|
||||
mockClient.send.mock.calls.filter(
|
||||
([command]) => command.constructor.name === "CreateIndexCommand",
|
||||
),
|
||||
).toHaveLength(1);
|
||||
expect(
|
||||
mockClient.send.mock.calls.filter(
|
||||
([command]) => command.constructor.name === "GetVectorBucketCommand",
|
||||
),
|
||||
).toHaveLength(1);
|
||||
expect(
|
||||
mockClient.send.mock.calls.filter(
|
||||
([command]) => command.constructor.name === "GetIndexCommand",
|
||||
),
|
||||
).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("shapes S3 write requests and normalizes search results", async () => {
|
||||
const { S3Vectors } = require("../src/vector_stores/s3_vectors");
|
||||
const mockClient = createMockS3VectorsClient();
|
||||
const store = new S3Vectors({
|
||||
client: mockClient,
|
||||
vectorBucketName: "test-bucket",
|
||||
collectionName: "test-index",
|
||||
embeddingModelDims: 3,
|
||||
});
|
||||
|
||||
await store.initialize();
|
||||
await store.insert(
|
||||
[[0.1, 0.2, 0.3]],
|
||||
["doc-1"],
|
||||
[{ user_id: "u1", topic: "alpha" }],
|
||||
);
|
||||
|
||||
const putInput = findCommandInput(mockClient, "PutVectorsCommand");
|
||||
expect(putInput).toMatchObject({
|
||||
vectorBucketName: "test-bucket",
|
||||
indexName: "test-index",
|
||||
vectors: [
|
||||
{
|
||||
key: "doc-1",
|
||||
data: { float32: [0.1, 0.2, 0.3] },
|
||||
metadata: { user_id: "u1", topic: "alpha" },
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
const results = await store.search([0.1, 0.2, 0.3], 5, { user_id: "u1" });
|
||||
|
||||
const queryInput = findCommandInput(mockClient, "QueryVectorsCommand");
|
||||
expect(queryInput).toMatchObject({
|
||||
vectorBucketName: "test-bucket",
|
||||
indexName: "test-index",
|
||||
queryVector: { float32: [0.1, 0.2, 0.3] },
|
||||
topK: 5,
|
||||
filter: { user_id: { $eq: "u1" } },
|
||||
returnMetadata: true,
|
||||
returnDistance: true,
|
||||
});
|
||||
expect(results).toEqual([
|
||||
{
|
||||
id: "doc-1",
|
||||
payload: { user_id: "u1", topic: "alpha" },
|
||||
score: 0.75,
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("normalizes euclidean distances without collapsing scores to zero", async () => {
|
||||
const { S3Vectors } = require("../src/vector_stores/s3_vectors");
|
||||
const store = new S3Vectors({
|
||||
client: createMockS3VectorsClient({
|
||||
queryDistance: 1.5,
|
||||
queryDistanceMetric: "euclidean",
|
||||
}),
|
||||
vectorBucketName: "test-bucket",
|
||||
collectionName: "test-index",
|
||||
embeddingModelDims: 3,
|
||||
distanceMetric: "cosine",
|
||||
});
|
||||
|
||||
const [result] = await store.search([0.1, 0.2, 0.3], 1);
|
||||
|
||||
expect(result).toEqual({
|
||||
id: "doc-1",
|
||||
payload: { user_id: "u1", topic: "alpha" },
|
||||
score: 0.4,
|
||||
});
|
||||
});
|
||||
|
||||
it("normalizes empty in and nin operands before search hits AWS", async () => {
|
||||
const { S3Vectors } = require("../src/vector_stores/s3_vectors");
|
||||
const mockClient = createMockS3VectorsClient();
|
||||
const store = new S3Vectors({
|
||||
client: mockClient,
|
||||
vectorBucketName: "test-bucket",
|
||||
collectionName: "test-index",
|
||||
embeddingModelDims: 3,
|
||||
});
|
||||
|
||||
const baselineQueryCalls = mockClient.send.mock.calls.filter(
|
||||
([command]) => command.constructor.name === "QueryVectorsCommand",
|
||||
).length;
|
||||
const emptyInResults = await store.search([0.1, 0.2, 0.3], 5, {
|
||||
topic: { in: [] },
|
||||
});
|
||||
expect(emptyInResults).toEqual([]);
|
||||
expect(
|
||||
mockClient.send.mock.calls.filter(
|
||||
([command]) => command.constructor.name === "QueryVectorsCommand",
|
||||
),
|
||||
).toHaveLength(baselineQueryCalls);
|
||||
|
||||
await store.search([0.1, 0.2, 0.3], 5, {
|
||||
topic: { nin: [] },
|
||||
});
|
||||
|
||||
const queryInput = findCommandInput(mockClient, "QueryVectorsCommand");
|
||||
expect(queryInput).toMatchObject({
|
||||
vectorBucketName: "test-bucket",
|
||||
indexName: "test-index",
|
||||
queryVector: { float32: [0.1, 0.2, 0.3] },
|
||||
topK: 5,
|
||||
returnMetadata: true,
|
||||
returnDistance: true,
|
||||
});
|
||||
expect(queryInput.filter).toBeUndefined();
|
||||
});
|
||||
it("applies client-side list filters, including empty $nin operands", async () => {
|
||||
const { S3Vectors } = require("../src/vector_stores/s3_vectors");
|
||||
const mockClient = createMockS3VectorsClient();
|
||||
const store = new S3Vectors({
|
||||
client: mockClient,
|
||||
vectorBucketName: "test-bucket",
|
||||
collectionName: "test-index",
|
||||
embeddingModelDims: 3,
|
||||
});
|
||||
|
||||
await store.initialize();
|
||||
await store.insert(
|
||||
[
|
||||
[0.1, 0.2, 0.3],
|
||||
[0.3, 0.2, 0.1],
|
||||
],
|
||||
["doc-1", "doc-2"],
|
||||
[
|
||||
{ user_id: "u1", topic: "alpha", tags: ["keep"] },
|
||||
{ user_id: "u2", topic: "beta", tags: ["skip"] },
|
||||
],
|
||||
);
|
||||
|
||||
const [allRows, allCount] = await store.list({ topic: { nin: [] } }, 10);
|
||||
expect(allCount).toBe(2);
|
||||
expect(allRows.map((row) => row.id)).toEqual(["doc-1", "doc-2"]);
|
||||
|
||||
const [filteredRows, filteredCount] = await store.list(
|
||||
{
|
||||
$and: [{ topic: { nin: ["beta"] } }, { tags: { in: ["keep"] } }],
|
||||
},
|
||||
10,
|
||||
);
|
||||
|
||||
expect(filteredCount).toBe(1);
|
||||
expect(filteredRows).toEqual([
|
||||
{
|
||||
id: "doc-1",
|
||||
payload: { user_id: "u1", topic: "alpha", tags: ["keep"] },
|
||||
},
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
// ───────────────────────────────────────────────────────────────────────────
|
||||
// 7. Vectorize — mock Cloudflare client, test idempotent init
|
||||
// ───────────────────────────────────────────────────────────────────────────
|
||||
describe("Vectorize – backward compat with mocked client", () => {
|
||||
let VectorizeDB: any;
|
||||
|
||||
Reference in New Issue
Block a user