feat(vector-stores): add Weaviate adapter to TypeScript OSS SDK (#5800)
Co-authored-by: kartik-mem0 <kartik.labhshetwar@mem0.ai>
This commit is contained in:
@@ -44,4 +44,5 @@ export * from "./vector_stores/turbopuffer";
|
||||
export * from "./vector_stores/milvus";
|
||||
export * from "./vector_stores/mongodb";
|
||||
export * from "./vector_stores/opensearch";
|
||||
export * from "./vector_stores/weaviate";
|
||||
export * from "./utils/factory";
|
||||
|
||||
@@ -58,6 +58,7 @@ import { S3Vectors } from "../vector_stores/s3_vectors";
|
||||
import { TurbopufferDB } from "../vector_stores/turbopuffer";
|
||||
import { Milvus } from "../vector_stores/milvus";
|
||||
import { MongoDB } from "../vector_stores/mongodb";
|
||||
import { WeaviateDB } from "../vector_stores/weaviate";
|
||||
|
||||
export class EmbedderFactory {
|
||||
static create(provider: string, config: EmbeddingConfig): Embedder {
|
||||
@@ -177,6 +178,8 @@ export class VectorStoreFactory {
|
||||
return new Milvus(config as any);
|
||||
case "mongodb":
|
||||
return new MongoDB(config as any);
|
||||
case "weaviate":
|
||||
return new WeaviateDB(config as any);
|
||||
default:
|
||||
throw new Error(`Unsupported vector store provider: ${provider}`);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,217 @@
|
||||
import weaviate, { Filters, type WeaviateClient } from "weaviate-client";
|
||||
import { v4 as uuidv4 } from "uuid";
|
||||
import { VectorStore } from "./base";
|
||||
import { SearchFilters, VectorStoreConfig, VectorStoreResult } from "../types";
|
||||
|
||||
interface WeaviateConfig extends VectorStoreConfig {
|
||||
client?: WeaviateClient;
|
||||
clusterUrl?: string;
|
||||
apiKey?: string;
|
||||
additionalHeaders?: Record<string, string>;
|
||||
collectionName: string;
|
||||
embeddingModelDims: number;
|
||||
}
|
||||
|
||||
const RETURN_PROPERTIES = [
|
||||
"ids",
|
||||
"hash",
|
||||
"metadata",
|
||||
"data",
|
||||
"created_at",
|
||||
"category",
|
||||
"updated_at",
|
||||
"user_id",
|
||||
"agent_id",
|
||||
"run_id",
|
||||
];
|
||||
|
||||
export class WeaviateDB implements VectorStore {
|
||||
private _config: WeaviateConfig;
|
||||
private _client!: WeaviateClient;
|
||||
private _col!: any;
|
||||
private _userId: string;
|
||||
private _initPromise?: Promise<void>;
|
||||
|
||||
constructor(config: WeaviateConfig) {
|
||||
this._config = config;
|
||||
this._userId = "";
|
||||
this.initialize().catch(console.error);
|
||||
}
|
||||
|
||||
initialize(): Promise<void> {
|
||||
return (this._initPromise ??= this._doInitialize());
|
||||
}
|
||||
|
||||
private async _doInitialize(): Promise<void> {
|
||||
const { client, clusterUrl, apiKey, additionalHeaders, collectionName } =
|
||||
this._config;
|
||||
|
||||
if (client) {
|
||||
this._client = client;
|
||||
} else if (clusterUrl?.includes("localhost")) {
|
||||
this._client = await weaviate.connectToLocal({
|
||||
headers: additionalHeaders,
|
||||
});
|
||||
} else if (apiKey) {
|
||||
this._client = await weaviate.connectToWeaviateCloud(clusterUrl!, {
|
||||
authCredentials: new weaviate.ApiKey(apiKey),
|
||||
headers: additionalHeaders,
|
||||
});
|
||||
} else {
|
||||
if (!clusterUrl) {
|
||||
throw new Error(
|
||||
"WeaviateDB: clusterUrl is required when client and apiKey are not provided",
|
||||
);
|
||||
}
|
||||
const parsed = new URL(clusterUrl);
|
||||
const httpSecure = parsed.protocol === "https:";
|
||||
this._client = await weaviate.connectToCustom({
|
||||
httpHost: parsed.hostname,
|
||||
httpPort: parsed.port
|
||||
? parseInt(parsed.port, 10)
|
||||
: httpSecure
|
||||
? 443
|
||||
: 8080,
|
||||
httpSecure,
|
||||
grpcHost: parsed.hostname,
|
||||
grpcPort: 50051,
|
||||
grpcSecure: false,
|
||||
headers: additionalHeaders,
|
||||
});
|
||||
}
|
||||
|
||||
const exists = await this._client.collections.exists(collectionName);
|
||||
if (!exists) {
|
||||
await this._client.collections.create({
|
||||
name: collectionName,
|
||||
properties: RETURN_PROPERTIES.map((name) => ({
|
||||
name,
|
||||
dataType: "text" as const,
|
||||
})),
|
||||
vectorizers: weaviate.configure.vectorizer.none(),
|
||||
vectorIndex: weaviate.configure.vectorIndex.hnsw(),
|
||||
} as any);
|
||||
}
|
||||
|
||||
this._col = this._client.collections.get(collectionName);
|
||||
}
|
||||
|
||||
private _buildFilters(filters?: SearchFilters) {
|
||||
if (!filters) return undefined;
|
||||
const conditions = (["user_id", "agent_id", "run_id"] as const)
|
||||
.filter((key) => filters[key] != null)
|
||||
.map((key) => this._col.filter.byProperty(key).equal(filters[key]));
|
||||
return conditions.length ? Filters.and(...conditions) : undefined;
|
||||
}
|
||||
|
||||
async insert(
|
||||
vectors: number[][],
|
||||
ids: string[],
|
||||
payloads: Record<string, any>[],
|
||||
): Promise<void> {
|
||||
await this.initialize();
|
||||
const objects = vectors.map((vector, i) => ({
|
||||
id: ids[i],
|
||||
properties: payloads[i],
|
||||
vectors: vector,
|
||||
}));
|
||||
await this._col.data.insertMany(objects);
|
||||
}
|
||||
|
||||
async search(
|
||||
query: number[],
|
||||
topK?: number,
|
||||
filters?: SearchFilters,
|
||||
): Promise<VectorStoreResult[]> {
|
||||
await this.initialize();
|
||||
const result = await this._col.query.nearVector(query, {
|
||||
limit: topK ?? 10,
|
||||
filters: this._buildFilters(filters),
|
||||
returnMetadata: ["distance"],
|
||||
});
|
||||
return result.objects.map((obj: any) => ({
|
||||
id: obj.uuid,
|
||||
payload: obj.properties,
|
||||
score: 1 - obj.metadata.distance,
|
||||
}));
|
||||
}
|
||||
|
||||
async keywordSearch(
|
||||
query: string,
|
||||
topK?: number,
|
||||
filters?: SearchFilters,
|
||||
): Promise<VectorStoreResult[] | null> {
|
||||
await this.initialize();
|
||||
const result = await this._col.query.bm25(query, {
|
||||
queryProperties: ["data"],
|
||||
limit: topK ?? 10,
|
||||
filters: this._buildFilters(filters),
|
||||
returnMetadata: ["score"],
|
||||
});
|
||||
return result.objects.map((obj: any) => ({
|
||||
id: obj.uuid,
|
||||
payload: obj.properties,
|
||||
score: obj.metadata.score,
|
||||
}));
|
||||
}
|
||||
|
||||
async get(vectorId: string): Promise<VectorStoreResult | null> {
|
||||
await this.initialize();
|
||||
const obj = await this._col.query.fetchObjectById(vectorId, {
|
||||
returnProperties: RETURN_PROPERTIES,
|
||||
});
|
||||
if (!obj) return null;
|
||||
return { id: obj.uuid, payload: obj.properties };
|
||||
}
|
||||
|
||||
async update(
|
||||
vectorId: string,
|
||||
vector: number[],
|
||||
payload: Record<string, any>,
|
||||
): Promise<void> {
|
||||
await this.initialize();
|
||||
await this._col.data.update({
|
||||
id: vectorId,
|
||||
properties: payload,
|
||||
vectors: vector,
|
||||
});
|
||||
}
|
||||
|
||||
async delete(vectorId: string): Promise<void> {
|
||||
await this.initialize();
|
||||
await this._col.data.deleteById(vectorId);
|
||||
}
|
||||
|
||||
async deleteCol(): Promise<void> {
|
||||
await this.initialize();
|
||||
await this._client.collections.delete(this._config.collectionName);
|
||||
}
|
||||
|
||||
async list(
|
||||
filters?: SearchFilters,
|
||||
topK?: number,
|
||||
): Promise<[VectorStoreResult[], number]> {
|
||||
await this.initialize();
|
||||
const result = await this._col.query.fetchObjects({
|
||||
limit: topK ?? 100,
|
||||
filters: this._buildFilters(filters),
|
||||
returnProperties: RETURN_PROPERTIES,
|
||||
});
|
||||
const results = result.objects.map((obj: any) => ({
|
||||
id: obj.uuid,
|
||||
payload: obj.properties,
|
||||
}));
|
||||
return [results, results.length];
|
||||
}
|
||||
|
||||
async getUserId(): Promise<string> {
|
||||
if (!this._userId) {
|
||||
this._userId = uuidv4();
|
||||
}
|
||||
return this._userId;
|
||||
}
|
||||
|
||||
async setUserId(userId: string): Promise<void> {
|
||||
this._userId = userId;
|
||||
}
|
||||
}
|
||||
@@ -193,6 +193,11 @@ jest.mock("../src/vector_stores/s3_vectors", () => ({
|
||||
.fn()
|
||||
.mockImplementation((config) => ({ type: "s3-vectors", config })),
|
||||
}));
|
||||
jest.mock("../src/vector_stores/weaviate", () => ({
|
||||
WeaviateDB: jest
|
||||
.fn()
|
||||
.mockImplementation((config) => ({ type: "weaviate", config })),
|
||||
}));
|
||||
jest.mock("../src/storage/SupabaseHistoryManager", () => ({
|
||||
SupabaseHistoryManager: jest
|
||||
.fn()
|
||||
@@ -328,6 +333,7 @@ describe("VectorStoreFactory", () => {
|
||||
["cassandra"],
|
||||
["s3-vectors"],
|
||||
["s3_vectors"],
|
||||
["weaviate"],
|
||||
])("creates vector store for provider '%s'", (provider) => {
|
||||
expect(() =>
|
||||
VectorStoreFactory.create(provider, dummyVSConfig),
|
||||
|
||||
@@ -2281,3 +2281,152 @@ describe("Memory class – backward compat with all providers", () => {
|
||||
consoleSpy.mockRestore();
|
||||
});
|
||||
});
|
||||
|
||||
// ───────────────────────────────────────────────────────────────────────────
|
||||
// WeaviateDB — mock client, behavioral surface checks
|
||||
// ───────────────────────────────────────────────────────────────────────────
|
||||
describe("WeaviateDB – backward compat with mocked client", () => {
|
||||
let WeaviateDB: any;
|
||||
let mockClient: any;
|
||||
let mockCol: any;
|
||||
|
||||
beforeEach(() => {
|
||||
jest.resetModules();
|
||||
|
||||
mockCol = {
|
||||
data: {
|
||||
insertMany: jest.fn().mockResolvedValue({}),
|
||||
deleteById: jest.fn().mockResolvedValue({}),
|
||||
update: jest.fn().mockResolvedValue({}),
|
||||
},
|
||||
query: {
|
||||
nearVector: jest.fn().mockResolvedValue({ objects: [] }),
|
||||
bm25: jest.fn().mockResolvedValue({ objects: [] }),
|
||||
fetchObjectById: jest.fn().mockResolvedValue(null),
|
||||
fetchObjects: jest.fn().mockResolvedValue({ objects: [] }),
|
||||
},
|
||||
filter: {
|
||||
byProperty: jest
|
||||
.fn()
|
||||
.mockReturnValue({ equal: jest.fn().mockReturnValue({}) }),
|
||||
},
|
||||
};
|
||||
|
||||
mockClient = {
|
||||
collections: {
|
||||
exists: jest.fn().mockResolvedValue(false),
|
||||
create: jest.fn().mockResolvedValue({}),
|
||||
get: jest.fn().mockReturnValue(mockCol),
|
||||
delete: jest.fn().mockResolvedValue({}),
|
||||
},
|
||||
};
|
||||
|
||||
jest.doMock("weaviate-client", () => ({
|
||||
default: {
|
||||
connectToLocal: jest.fn().mockResolvedValue(mockClient),
|
||||
connectToWeaviateCloud: jest.fn().mockResolvedValue(mockClient),
|
||||
connectToCustom: jest.fn().mockResolvedValue(mockClient),
|
||||
ApiKey: jest.fn().mockReturnValue({}),
|
||||
configure: {
|
||||
vectorizer: { none: jest.fn().mockReturnValue({}) },
|
||||
vectorIndex: { hnsw: jest.fn().mockReturnValue({}) },
|
||||
},
|
||||
},
|
||||
Filters: { and: jest.fn().mockReturnValue({ __mock: "filter" }) },
|
||||
__esModule: true,
|
||||
}));
|
||||
|
||||
WeaviateDB = require("../src/vector_stores/weaviate").WeaviateDB;
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
jest.restoreAllMocks();
|
||||
jest.resetModules();
|
||||
});
|
||||
|
||||
it("implements full VectorStore interface", () => {
|
||||
const store = new WeaviateDB({
|
||||
client: mockClient,
|
||||
collectionName: "test",
|
||||
embeddingModelDims: 768,
|
||||
});
|
||||
expect(typeof store.insert).toBe("function");
|
||||
expect(typeof store.search).toBe("function");
|
||||
expect(typeof store.keywordSearch).toBe("function");
|
||||
expect(typeof store.get).toBe("function");
|
||||
expect(typeof store.update).toBe("function");
|
||||
expect(typeof store.delete).toBe("function");
|
||||
expect(typeof store.deleteCol).toBe("function");
|
||||
expect(typeof store.list).toBe("function");
|
||||
expect(typeof store.getUserId).toBe("function");
|
||||
expect(typeof store.setUserId).toBe("function");
|
||||
expect(typeof store.initialize).toBe("function");
|
||||
});
|
||||
|
||||
it("initialize() is idempotent (same promise returned)", async () => {
|
||||
const store = new WeaviateDB({
|
||||
client: mockClient,
|
||||
collectionName: "test",
|
||||
embeddingModelDims: 768,
|
||||
});
|
||||
const p1 = store.initialize();
|
||||
const p2 = store.initialize();
|
||||
const p3 = store.initialize();
|
||||
await Promise.all([p1, p2, p3]);
|
||||
expect(mockClient.collections.create).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("insert shapes insertMany request correctly", async () => {
|
||||
const store = new WeaviateDB({
|
||||
client: mockClient,
|
||||
collectionName: "test",
|
||||
embeddingModelDims: 3,
|
||||
});
|
||||
await store.initialize();
|
||||
await store.insert([[0.1, 0.2, 0.3]], ["id-1"], [{ data: "hello" }]);
|
||||
expect(mockCol.data.insertMany).toHaveBeenCalledWith(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
id: "id-1",
|
||||
properties: { data: "hello" },
|
||||
vectors: [0.1, 0.2, 0.3],
|
||||
}),
|
||||
]),
|
||||
);
|
||||
});
|
||||
|
||||
it("search normalizes nearVector result to id/payload/score", async () => {
|
||||
mockCol.query.nearVector.mockResolvedValue({
|
||||
objects: [
|
||||
{
|
||||
uuid: "id-1",
|
||||
properties: { data: "x" },
|
||||
metadata: { distance: 0.2 },
|
||||
},
|
||||
],
|
||||
});
|
||||
const store = new WeaviateDB({
|
||||
client: mockClient,
|
||||
collectionName: "test",
|
||||
embeddingModelDims: 3,
|
||||
});
|
||||
await store.initialize();
|
||||
const results = await store.search([0.1, 0.2, 0.3], 1);
|
||||
expect(results).toHaveLength(1);
|
||||
expect(results[0]).toMatchObject({
|
||||
id: "id-1",
|
||||
payload: { data: "x" },
|
||||
score: 0.8,
|
||||
});
|
||||
});
|
||||
|
||||
it("getUserId / setUserId roundtrip", async () => {
|
||||
const store = new WeaviateDB({
|
||||
client: mockClient,
|
||||
collectionName: "test",
|
||||
embeddingModelDims: 768,
|
||||
});
|
||||
await store.setUserId("custom-user");
|
||||
expect(await store.getUserId()).toBe("custom-user");
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user