feat(oss): add reranker + per-search rerank to the TypeScript OSS SDK (#6055)
This commit is contained in:
@@ -177,6 +177,7 @@ export class ConfigManager {
|
||||
})(),
|
||||
disableHistory:
|
||||
userConfig.disableHistory || DEFAULT_MEMORY_CONFIG.disableHistory,
|
||||
reranker: userConfig.reranker,
|
||||
};
|
||||
|
||||
// Validate the merged config
|
||||
|
||||
@@ -45,4 +45,9 @@ export * from "./vector_stores/milvus";
|
||||
export * from "./vector_stores/mongodb";
|
||||
export * from "./vector_stores/opensearch";
|
||||
export * from "./vector_stores/weaviate";
|
||||
export * from "./rerankers/base";
|
||||
export * from "./rerankers/cohere";
|
||||
export * from "./rerankers/llm";
|
||||
export * from "./rerankers/zeroentropy";
|
||||
export * from "./rerankers/cross_encoder";
|
||||
export * from "./utils/factory";
|
||||
|
||||
@@ -13,6 +13,7 @@ import {
|
||||
LLMFactory,
|
||||
VectorStoreFactory,
|
||||
HistoryManagerFactory,
|
||||
RerankerFactory,
|
||||
} from "../utils/factory";
|
||||
import {
|
||||
FactRetrievalSchema,
|
||||
@@ -28,6 +29,7 @@ import {
|
||||
import { DummyHistoryManager } from "../storage/DummyHistoryManager";
|
||||
import { Embedder } from "../embeddings/base";
|
||||
import { LLM } from "../llms/base";
|
||||
import { Reranker } from "../rerankers/base";
|
||||
import { VectorStore } from "../vector_stores/base";
|
||||
import { ConfigManager } from "../config/manager";
|
||||
|
||||
@@ -164,6 +166,7 @@ export class Memory {
|
||||
private embedder: Embedder;
|
||||
private vectorStore!: VectorStore;
|
||||
private llm: LLM;
|
||||
private reranker: Reranker | null = null;
|
||||
private db: HistoryManager;
|
||||
private collectionName: string | undefined;
|
||||
private apiVersion: string;
|
||||
@@ -188,6 +191,12 @@ export class Memory {
|
||||
this.config.llm.provider,
|
||||
this.config.llm.config,
|
||||
);
|
||||
if (this.config.reranker) {
|
||||
this.reranker = RerankerFactory.create(
|
||||
this.config.reranker.provider,
|
||||
this.config.reranker.config,
|
||||
);
|
||||
}
|
||||
if (this.config.disableHistory) {
|
||||
this.db = new DummyHistoryManager();
|
||||
} else {
|
||||
@@ -1541,8 +1550,30 @@ export class Memory {
|
||||
};
|
||||
});
|
||||
|
||||
// Step 10: Optionally re-rank with the configured reranker. Opt-in per
|
||||
// search via `rerank: true`; a no-op when no reranker is configured.
|
||||
const invokeReranker = Boolean(
|
||||
config.rerank && this.reranker && results.length > 0,
|
||||
);
|
||||
let finalResults = results;
|
||||
if (invokeReranker) {
|
||||
try {
|
||||
const ranked = await this.reranker!.rerank(
|
||||
query,
|
||||
results.map((r) => r.memory),
|
||||
topK,
|
||||
);
|
||||
finalResults = ranked.map((r) => ({
|
||||
...results[r.index],
|
||||
rerankScore: r.rerankScore,
|
||||
}));
|
||||
} catch (e) {
|
||||
console.warn(`Reranking failed, using original results: ${e}`);
|
||||
}
|
||||
}
|
||||
|
||||
const result = {
|
||||
results,
|
||||
results: finalResults,
|
||||
};
|
||||
const searchElapsedMs = Date.now() - searchStartMs;
|
||||
if (temporalUsageNotice) {
|
||||
|
||||
@@ -36,6 +36,11 @@ export interface SearchMemoryOptions {
|
||||
threshold?: number;
|
||||
explain?: boolean;
|
||||
referenceDate?: number | string | Date | null;
|
||||
/**
|
||||
* Re-rank the results with the configured reranker before returning. No-op
|
||||
* when no `reranker` is configured on the Memory.
|
||||
*/
|
||||
rerank?: boolean;
|
||||
/** Include expired memories in the results. Defaults to false. */
|
||||
showExpired?: boolean;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
export interface RerankResult {
|
||||
/** Index into the input `documents` array. */
|
||||
index: number;
|
||||
/** Relevance of the document to the query, 0..1, higher = more relevant. */
|
||||
rerankScore: number;
|
||||
}
|
||||
|
||||
export interface Reranker {
|
||||
/**
|
||||
* Rank `documents` by relevance to `query`.
|
||||
*
|
||||
* Returns results sorted by descending relevance. When `topK` is given, at
|
||||
* most that many results are returned. Each result's `index` points back into
|
||||
* the input `documents` array so callers can recover the original item.
|
||||
*/
|
||||
rerank(
|
||||
query: string,
|
||||
documents: string[],
|
||||
topK?: number,
|
||||
): Promise<RerankResult[]>;
|
||||
}
|
||||
@@ -0,0 +1,140 @@
|
||||
const mockRerank = jest.fn();
|
||||
|
||||
jest.mock("cohere-ai", () => ({
|
||||
CohereClient: jest.fn().mockImplementation(() => ({
|
||||
rerank: mockRerank,
|
||||
})),
|
||||
}));
|
||||
|
||||
import { CohereClient } from "cohere-ai";
|
||||
import { CohereReranker } from "./cohere";
|
||||
|
||||
describe("CohereReranker", () => {
|
||||
beforeEach(() => {
|
||||
mockRerank.mockReset();
|
||||
(CohereClient as unknown as jest.Mock).mockClear();
|
||||
});
|
||||
|
||||
it("throws when no API key is provided or configured", () => {
|
||||
const originalEnv = process.env.COHERE_API_KEY;
|
||||
delete process.env.COHERE_API_KEY;
|
||||
|
||||
expect(() => new CohereReranker({})).toThrow(/Cohere API key is required/);
|
||||
|
||||
if (originalEnv !== undefined) process.env.COHERE_API_KEY = originalEnv;
|
||||
});
|
||||
|
||||
it("sends the query, documents, topN, and default model to Cohere", async () => {
|
||||
mockRerank.mockResolvedValue({ results: [] });
|
||||
const reranker = new CohereReranker({ apiKey: "key" });
|
||||
|
||||
await reranker.rerank("capital of US?", ["a", "b", "c"], 2);
|
||||
|
||||
expect(mockRerank).toHaveBeenCalledWith({
|
||||
model: "rerank-v3.5",
|
||||
query: "capital of US?",
|
||||
documents: ["a", "b", "c"],
|
||||
topN: 2,
|
||||
returnDocuments: false,
|
||||
maxChunksPerDoc: undefined,
|
||||
});
|
||||
});
|
||||
|
||||
it("defaults topN to documents.length when neither the call nor config sets a top_k", async () => {
|
||||
mockRerank.mockResolvedValue({ results: [] });
|
||||
const reranker = new CohereReranker({ apiKey: "key" });
|
||||
|
||||
await reranker.rerank("q", ["a", "b", "c"]);
|
||||
|
||||
expect(mockRerank).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ topN: 3 }),
|
||||
);
|
||||
});
|
||||
|
||||
it("forwards returnDocuments and maxChunksPerDoc from config", async () => {
|
||||
mockRerank.mockResolvedValue({ results: [] });
|
||||
const reranker = new CohereReranker({
|
||||
apiKey: "key",
|
||||
returnDocuments: true,
|
||||
maxChunksPerDoc: 5,
|
||||
});
|
||||
|
||||
await reranker.rerank("q", ["a"]);
|
||||
|
||||
expect(mockRerank).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ returnDocuments: true, maxChunksPerDoc: 5 }),
|
||||
);
|
||||
});
|
||||
|
||||
it("returns Cohere's ranked results as {index, rerankScore}", async () => {
|
||||
mockRerank.mockResolvedValue({
|
||||
results: [
|
||||
{ index: 2, relevanceScore: 0.9 },
|
||||
{ index: 0, relevanceScore: 0.31 },
|
||||
],
|
||||
});
|
||||
const reranker = new CohereReranker({ apiKey: "key" });
|
||||
|
||||
const results = await reranker.rerank("q", ["x", "y", "z"]);
|
||||
|
||||
expect(results).toEqual([
|
||||
{ index: 2, rerankScore: 0.9 },
|
||||
{ index: 0, rerankScore: 0.31 },
|
||||
]);
|
||||
});
|
||||
|
||||
it("uses a custom model when provided", async () => {
|
||||
mockRerank.mockResolvedValue({ results: [] });
|
||||
const reranker = new CohereReranker({
|
||||
apiKey: "key",
|
||||
model: "rerank-v4.0-pro",
|
||||
});
|
||||
|
||||
await reranker.rerank("q", ["a"]);
|
||||
|
||||
expect(mockRerank).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ model: "rerank-v4.0-pro" }),
|
||||
);
|
||||
});
|
||||
|
||||
it("returns an empty array without calling Cohere when there are no documents", async () => {
|
||||
const reranker = new CohereReranker({ apiKey: "key" });
|
||||
|
||||
const results = await reranker.rerank("q", []);
|
||||
|
||||
expect(results).toEqual([]);
|
||||
expect(mockRerank).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("falls back to the original order with rerankScore 0.0 when the Cohere API call fails", async () => {
|
||||
mockRerank.mockRejectedValue(new Error("cohere is down"));
|
||||
const warnSpy = jest.spyOn(console, "warn").mockImplementation(() => {});
|
||||
const reranker = new CohereReranker({ apiKey: "key" });
|
||||
|
||||
const results = await reranker.rerank("q", ["a", "b", "c"]);
|
||||
|
||||
expect(results).toEqual([
|
||||
{ index: 0, rerankScore: 0.0 },
|
||||
{ index: 1, rerankScore: 0.0 },
|
||||
{ index: 2, rerankScore: 0.0 },
|
||||
]);
|
||||
expect(warnSpy).toHaveBeenCalled();
|
||||
|
||||
warnSpy.mockRestore();
|
||||
});
|
||||
|
||||
it("slices the fallback results by topK when the Cohere API call fails", async () => {
|
||||
mockRerank.mockRejectedValue(new Error("cohere is down"));
|
||||
jest.spyOn(console, "warn").mockImplementation(() => {});
|
||||
const reranker = new CohereReranker({ apiKey: "key", topK: 2 });
|
||||
|
||||
const results = await reranker.rerank("q", ["a", "b", "c"]);
|
||||
|
||||
expect(results).toEqual([
|
||||
{ index: 0, rerankScore: 0.0 },
|
||||
{ index: 1, rerankScore: 0.0 },
|
||||
]);
|
||||
|
||||
(console.warn as jest.Mock).mockRestore();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,61 @@
|
||||
import { CohereClient } from "cohere-ai";
|
||||
import { RerankerConfig } from "../types";
|
||||
import { Reranker, RerankResult } from "./base";
|
||||
|
||||
const DEFAULT_MODEL = "rerank-v3.5";
|
||||
|
||||
export class CohereReranker implements Reranker {
|
||||
private client: CohereClient;
|
||||
private model: string;
|
||||
private topK?: number;
|
||||
private returnDocuments: boolean;
|
||||
private maxChunksPerDoc?: number;
|
||||
|
||||
constructor(config: RerankerConfig) {
|
||||
const apiKey = config.apiKey || process.env.COHERE_API_KEY;
|
||||
if (!apiKey) {
|
||||
throw new Error(
|
||||
"Cohere API key is required. Set COHERE_API_KEY environment variable or pass apiKey in config.",
|
||||
);
|
||||
}
|
||||
this.client = new CohereClient({ token: apiKey });
|
||||
this.model = config.model || DEFAULT_MODEL;
|
||||
this.topK = config.topK;
|
||||
this.returnDocuments = config.returnDocuments ?? false;
|
||||
this.maxChunksPerDoc = config.maxChunksPerDoc;
|
||||
}
|
||||
|
||||
async rerank(
|
||||
query: string,
|
||||
documents: string[],
|
||||
topK?: number,
|
||||
): Promise<RerankResult[]> {
|
||||
if (documents.length === 0) return [];
|
||||
|
||||
try {
|
||||
const response = await this.client.rerank({
|
||||
model: this.model,
|
||||
query,
|
||||
documents,
|
||||
topN: topK || this.topK || documents.length,
|
||||
returnDocuments: this.returnDocuments,
|
||||
maxChunksPerDoc: this.maxChunksPerDoc,
|
||||
});
|
||||
|
||||
return response.results.map((result) => ({
|
||||
index: result.index,
|
||||
rerankScore: result.relevanceScore,
|
||||
}));
|
||||
} catch (e) {
|
||||
console.warn(
|
||||
`Cohere reranking failed, falling back to original order: ${e}`,
|
||||
);
|
||||
const scored = documents.map((_, index) => ({
|
||||
index,
|
||||
rerankScore: 0.0,
|
||||
}));
|
||||
const finalTopK = topK || this.topK;
|
||||
return finalTopK ? scored.slice(0, finalTopK) : scored;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,189 @@
|
||||
const mockModelFromPretrained = jest.fn();
|
||||
const mockTokenizerFromPretrained = jest.fn();
|
||||
|
||||
jest.mock("@huggingface/transformers", () => ({
|
||||
AutoModelForSequenceClassification: {
|
||||
from_pretrained: mockModelFromPretrained,
|
||||
},
|
||||
AutoTokenizer: { from_pretrained: mockTokenizerFromPretrained },
|
||||
}));
|
||||
|
||||
import { CrossEncoderReranker } from "./cross_encoder";
|
||||
|
||||
const sigmoid = (x: number) => 1 / (1 + Math.exp(-x));
|
||||
|
||||
/** Wire the mocked tokenizer + model so the model returns `logits` for a call. */
|
||||
function setupModel(logits: number[][]) {
|
||||
const tokenizer = jest.fn().mockReturnValue({ input_ids: [] });
|
||||
mockTokenizerFromPretrained.mockResolvedValue(tokenizer);
|
||||
const model = jest
|
||||
.fn()
|
||||
.mockResolvedValue({ logits: { tolist: () => logits } });
|
||||
mockModelFromPretrained.mockResolvedValue(model);
|
||||
return { tokenizer, model };
|
||||
}
|
||||
|
||||
describe("CrossEncoderReranker", () => {
|
||||
beforeEach(() => {
|
||||
mockModelFromPretrained.mockReset();
|
||||
mockTokenizerFromPretrained.mockReset();
|
||||
});
|
||||
|
||||
it("scores each document and returns them sorted by relevance, sigmoid-normalized to [0,1]", async () => {
|
||||
setupModel([[0.0], [2.0], [-1.0]]);
|
||||
const reranker = new CrossEncoderReranker({}, "default-model");
|
||||
|
||||
const results = await reranker.rerank("q", ["a", "b", "c"]);
|
||||
|
||||
// sigmoid: b(2.0)=0.88 > a(0.0)=0.5 > c(-1.0)=0.27
|
||||
expect(results.map((r) => r.index)).toEqual([1, 0, 2]);
|
||||
expect(results[0].rerankScore).toBeCloseTo(sigmoid(2.0), 5);
|
||||
expect(results[1].rerankScore).toBeCloseTo(sigmoid(0.0), 5);
|
||||
expect(results[2].rerankScore).toBeCloseTo(sigmoid(-1.0), 5);
|
||||
});
|
||||
|
||||
it("pairs the query with each document via text_pair when tokenizing", async () => {
|
||||
const { tokenizer } = setupModel([[0.1], [0.2]]);
|
||||
const reranker = new CrossEncoderReranker(
|
||||
{ maxLength: 128 },
|
||||
"default-model",
|
||||
);
|
||||
|
||||
await reranker.rerank("what is x", ["doc one", "doc two"]);
|
||||
|
||||
expect(tokenizer).toHaveBeenCalledWith(
|
||||
["what is x", "what is x"],
|
||||
expect.objectContaining({
|
||||
text_pair: ["doc one", "doc two"],
|
||||
padding: true,
|
||||
truncation: true,
|
||||
max_length: 128,
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("applies the topK limit", async () => {
|
||||
setupModel([[0.0], [2.0], [-1.0]]);
|
||||
const reranker = new CrossEncoderReranker({}, "default-model");
|
||||
|
||||
const results = await reranker.rerank("q", ["a", "b", "c"], 2);
|
||||
|
||||
expect(results).toHaveLength(2);
|
||||
expect(results.map((r) => r.index)).toEqual([1, 0]);
|
||||
});
|
||||
|
||||
it("falls back to config.topK when the rerank() call omits one", async () => {
|
||||
setupModel([[0.0], [2.0], [-1.0]]);
|
||||
const reranker = new CrossEncoderReranker({ topK: 1 }, "default-model");
|
||||
|
||||
const results = await reranker.rerank("q", ["a", "b", "c"]);
|
||||
|
||||
expect(results).toHaveLength(1);
|
||||
expect(results.map((r) => r.index)).toEqual([1]);
|
||||
});
|
||||
|
||||
it("returns [] without loading the model when there are no documents", async () => {
|
||||
const reranker = new CrossEncoderReranker({}, "default-model");
|
||||
|
||||
const results = await reranker.rerank("q", []);
|
||||
|
||||
expect(results).toEqual([]);
|
||||
expect(mockModelFromPretrained).not.toHaveBeenCalled();
|
||||
expect(mockTokenizerFromPretrained).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("returns raw logits as scores when normalize is false", async () => {
|
||||
setupModel([[2.0], [0.0]]);
|
||||
const reranker = new CrossEncoderReranker(
|
||||
{ normalize: false },
|
||||
"default-model",
|
||||
);
|
||||
|
||||
const results = await reranker.rerank("q", ["a", "b"]);
|
||||
|
||||
expect(results.map((r) => r.index)).toEqual([0, 1]);
|
||||
expect(results[0].rerankScore).toBe(2.0);
|
||||
expect(results[1].rerankScore).toBe(0.0);
|
||||
});
|
||||
|
||||
it("loads the model and tokenizer only once across multiple rerank calls", async () => {
|
||||
setupModel([[0.5]]);
|
||||
const reranker = new CrossEncoderReranker({}, "default-model");
|
||||
|
||||
await reranker.rerank("q", ["a"]);
|
||||
await reranker.rerank("q2", ["b"]);
|
||||
|
||||
expect(mockModelFromPretrained).toHaveBeenCalledTimes(1);
|
||||
expect(mockTokenizerFromPretrained).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("loads the default model, or the configured model when provided", async () => {
|
||||
setupModel([[0.5]]);
|
||||
|
||||
await new CrossEncoderReranker({}, "the-default").rerank("q", ["a"]);
|
||||
expect(mockModelFromPretrained).toHaveBeenCalledWith(
|
||||
"the-default",
|
||||
expect.any(Object),
|
||||
);
|
||||
|
||||
mockModelFromPretrained.mockClear();
|
||||
setupModel([[0.5]]);
|
||||
await new CrossEncoderReranker(
|
||||
{ model: "custom/model" },
|
||||
"the-default",
|
||||
).rerank("q", ["a"]);
|
||||
expect(mockModelFromPretrained).toHaveBeenCalledWith(
|
||||
"custom/model",
|
||||
expect.any(Object),
|
||||
);
|
||||
});
|
||||
|
||||
it("applies a default maxLength (as the huggingface provider passes 512) when config omits one", async () => {
|
||||
const { tokenizer } = setupModel([[0.5]]);
|
||||
const reranker = new CrossEncoderReranker({}, "default-model", 512);
|
||||
|
||||
await reranker.rerank("q", ["a"]);
|
||||
|
||||
expect(tokenizer).toHaveBeenCalledWith(
|
||||
["q"],
|
||||
expect.objectContaining({ max_length: 512 }),
|
||||
);
|
||||
});
|
||||
|
||||
it("falls back to the original order with rerankScore 0.0 when the model fails to load", async () => {
|
||||
mockModelFromPretrained.mockResolvedValue(jest.fn());
|
||||
mockTokenizerFromPretrained.mockRejectedValue(
|
||||
new Error("model download failed"),
|
||||
);
|
||||
const warnSpy = jest.spyOn(console, "warn").mockImplementation(() => {});
|
||||
const reranker = new CrossEncoderReranker({}, "default-model");
|
||||
|
||||
const results = await reranker.rerank("q", ["a", "b", "c"]);
|
||||
|
||||
expect(results).toEqual([
|
||||
{ index: 0, rerankScore: 0.0 },
|
||||
{ index: 1, rerankScore: 0.0 },
|
||||
{ index: 2, rerankScore: 0.0 },
|
||||
]);
|
||||
expect(warnSpy).toHaveBeenCalled();
|
||||
|
||||
warnSpy.mockRestore();
|
||||
});
|
||||
|
||||
it("falls back to the original order with rerankScore 0.0, sliced by topK, when scoring fails", async () => {
|
||||
const tokenizer = jest.fn().mockReturnValue({ input_ids: [] });
|
||||
mockTokenizerFromPretrained.mockResolvedValue(tokenizer);
|
||||
mockModelFromPretrained.mockResolvedValue(
|
||||
jest.fn().mockRejectedValue(new Error("forward pass failed")),
|
||||
);
|
||||
const warnSpy = jest.spyOn(console, "warn").mockImplementation(() => {});
|
||||
const reranker = new CrossEncoderReranker({}, "default-model");
|
||||
|
||||
const results = await reranker.rerank("q", ["a", "b"], 1);
|
||||
|
||||
expect(results).toEqual([{ index: 0, rerankScore: 0.0 }]);
|
||||
expect(warnSpy).toHaveBeenCalled();
|
||||
|
||||
warnSpy.mockRestore();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,98 @@
|
||||
import { RerankerConfig } from "../types";
|
||||
import { Reranker, RerankResult } from "./base";
|
||||
|
||||
const sigmoid = (x: number) => 1 / (1 + Math.exp(-x));
|
||||
|
||||
export class CrossEncoderReranker implements Reranker {
|
||||
private modelId: string;
|
||||
private device?: string;
|
||||
private maxLength?: number;
|
||||
private normalize: boolean;
|
||||
private topK?: number;
|
||||
// ponytail: batchSize/showProgressBar are accepted for config parity with the
|
||||
// Python SDK but are no-ops here — a memory search reranks a small candidate
|
||||
// set in a single forward pass. Chunk by batchSize if that ever grows.
|
||||
private loaded?: Promise<{ model: any; tokenizer: any }>;
|
||||
|
||||
constructor(
|
||||
config: RerankerConfig,
|
||||
defaultModel: string,
|
||||
defaultMaxLength?: number,
|
||||
) {
|
||||
this.modelId = config.model || defaultModel;
|
||||
this.device = config.device;
|
||||
this.maxLength = config.maxLength ?? defaultMaxLength;
|
||||
this.normalize = config.normalize ?? true;
|
||||
this.topK = config.topK;
|
||||
}
|
||||
|
||||
private load() {
|
||||
if (!this.loaded) {
|
||||
this.loaded = (async () => {
|
||||
// Lazy-load Transformers.js (and its onnxruntime native binding) only
|
||||
// when a rerank actually runs. A static import would pull onnxruntime
|
||||
// into every `new Memory()`, colliding on Linux with fastembed's
|
||||
// separate onnxruntime version — see the merge with the FastEmbed
|
||||
// embedder. Deferring it keeps memory construction free of ONNX.
|
||||
const { AutoModelForSequenceClassification, AutoTokenizer } =
|
||||
await import("@huggingface/transformers");
|
||||
const options: any = {};
|
||||
if (this.device) options.device = this.device;
|
||||
const model = await AutoModelForSequenceClassification.from_pretrained(
|
||||
this.modelId,
|
||||
options,
|
||||
);
|
||||
const tokenizer = await AutoTokenizer.from_pretrained(this.modelId);
|
||||
return { model, tokenizer };
|
||||
})();
|
||||
}
|
||||
return this.loaded;
|
||||
}
|
||||
|
||||
async rerank(
|
||||
query: string,
|
||||
documents: string[],
|
||||
topK?: number,
|
||||
): Promise<RerankResult[]> {
|
||||
if (documents.length === 0) return [];
|
||||
|
||||
try {
|
||||
const { model, tokenizer } = await this.load();
|
||||
|
||||
const inputs = tokenizer(
|
||||
documents.map(() => query),
|
||||
{
|
||||
text_pair: documents,
|
||||
padding: true,
|
||||
truncation: true,
|
||||
...(this.maxLength ? { max_length: this.maxLength } : {}),
|
||||
},
|
||||
);
|
||||
|
||||
const { logits } = await model(inputs);
|
||||
const rows: unknown[] = logits.tolist();
|
||||
|
||||
const scored = rows.map((row, index) => {
|
||||
const logit = Array.isArray(row) ? (row[0] as number) : (row as number);
|
||||
return {
|
||||
index,
|
||||
rerankScore: this.normalize ? sigmoid(logit) : logit,
|
||||
};
|
||||
});
|
||||
|
||||
scored.sort((a, b) => b.rerankScore - a.rerankScore);
|
||||
const finalTopK = topK || this.topK;
|
||||
return finalTopK ? scored.slice(0, finalTopK) : scored;
|
||||
} catch (e) {
|
||||
console.warn(
|
||||
`Cross-encoder reranking failed, falling back to original order: ${e}`,
|
||||
);
|
||||
const scored = documents.map((_, index) => ({
|
||||
index,
|
||||
rerankScore: 0.0,
|
||||
}));
|
||||
const finalTopK = topK || this.topK;
|
||||
return finalTopK ? scored.slice(0, finalTopK) : scored;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,174 @@
|
||||
import { LLM } from "../llms/base";
|
||||
import { LLMReranker } from "./llm";
|
||||
|
||||
// Duplicated rather than imported from ./llm.ts so this test catches drift.
|
||||
const EXPECTED_SYSTEM_PROMPT = `You are a relevance scoring assistant. Given a query and a document, score how relevant the document is to the query.
|
||||
|
||||
Score the relevance on a scale from 0.0 to 1.0, where:
|
||||
- 1.0 = Perfectly relevant and directly answers the query
|
||||
- 0.8-0.9 = Highly relevant with good information
|
||||
- 0.6-0.7 = Moderately relevant with some useful information
|
||||
- 0.4-0.5 = Slightly relevant with limited useful information
|
||||
- 0.0-0.3 = Not relevant or no useful information
|
||||
|
||||
Respond with only a single numerical score between 0.0 and 1.0. Do not include any explanation or additional text.`;
|
||||
|
||||
/**
|
||||
* Fake LLM that scores a document by looking up the document text inside the
|
||||
* prompt. Test document tokens must be distinct and must not be substrings of
|
||||
* the prompt boilerplate (e.g. avoid "a"/"b"), or the lookup resolves the
|
||||
* wrong doc.
|
||||
*/
|
||||
function makeLLM(scoreByDoc: Record<string, string>): LLM {
|
||||
return {
|
||||
generateResponse: async (
|
||||
messages: Array<{ role: string; content: string }>,
|
||||
) => {
|
||||
const prompt = messages.map((m) => m.content).join("\n");
|
||||
const doc = Object.keys(scoreByDoc).find((d) => prompt.includes(d));
|
||||
return doc ? scoreByDoc[doc] : "no number here";
|
||||
},
|
||||
generateChat: async () => ({ content: "", role: "assistant" }),
|
||||
};
|
||||
}
|
||||
|
||||
describe("LLMReranker", () => {
|
||||
it("sorts documents by descending relevance score", async () => {
|
||||
const llm = makeLLM({ cats: "0.2", dogs: "0.9", fish: "0.5" });
|
||||
const reranker = new LLMReranker({}, llm);
|
||||
|
||||
const results = await reranker.rerank("pets", ["cats", "dogs", "fish"]);
|
||||
|
||||
expect(results.map((r) => r.index)).toEqual([1, 2, 0]);
|
||||
expect(results.map((r) => r.rerankScore)).toEqual([0.9, 0.5, 0.2]);
|
||||
});
|
||||
|
||||
it("clamps scores to the [0, 1] range", async () => {
|
||||
const llm = makeLLM({ zebra: "1.5", walrus: "-0.3" });
|
||||
const reranker = new LLMReranker({}, llm);
|
||||
|
||||
const results = await reranker.rerank("q", ["zebra", "walrus"]);
|
||||
|
||||
const byIndex = new Map(results.map((r) => [r.index, r.rerankScore]));
|
||||
expect(byIndex.get(0)).toBe(1); // "zebra" 1.5 -> clamped to 1
|
||||
expect(byIndex.get(1)).toBe(0); // "walrus" -0.3 -> clamped to 0
|
||||
});
|
||||
|
||||
it("truncates results to topK", async () => {
|
||||
const llm = makeLLM({ alpha: "0.1", bravo: "0.8", charlie: "0.5" });
|
||||
const reranker = new LLMReranker({}, llm);
|
||||
|
||||
const results = await reranker.rerank(
|
||||
"q",
|
||||
["alpha", "bravo", "charlie"],
|
||||
2,
|
||||
);
|
||||
|
||||
expect(results).toHaveLength(2);
|
||||
expect(results.map((r) => r.index)).toEqual([1, 2]); // bravo(0.8), charlie(0.5)
|
||||
});
|
||||
|
||||
it("falls back to config.topK when the rerank() call omits one", async () => {
|
||||
const llm = makeLLM({ alpha: "0.1", bravo: "0.8", charlie: "0.5" });
|
||||
const reranker = new LLMReranker({ topK: 1 }, llm);
|
||||
|
||||
const results = await reranker.rerank("q", ["alpha", "bravo", "charlie"]);
|
||||
|
||||
expect(results).toHaveLength(1);
|
||||
expect(results[0].index).toBe(1); // bravo(0.8)
|
||||
});
|
||||
|
||||
it("falls back to a neutral score of 0.5 (not 0) when the LLM output has no number", async () => {
|
||||
const llm = makeLLM({ junk: "I cannot rate this" });
|
||||
const reranker = new LLMReranker({}, llm);
|
||||
|
||||
const results = await reranker.rerank("q", ["junk"]);
|
||||
|
||||
expect(results[0].rerankScore).toBe(0.5);
|
||||
});
|
||||
|
||||
it("prefers a decimal match over an integer match when extracting the score", async () => {
|
||||
const llm = makeLLM({ item: "The score is 0.73 out of 1" });
|
||||
const reranker = new LLMReranker({}, llm);
|
||||
|
||||
const results = await reranker.rerank("q", ["item"]);
|
||||
|
||||
expect(results[0].rerankScore).toBe(0.73);
|
||||
});
|
||||
|
||||
it("falls back to an integer match when no decimal is present", async () => {
|
||||
const llm = makeLLM({ item: "I'd say this is a solid 1" });
|
||||
const reranker = new LLMReranker({}, llm);
|
||||
|
||||
const results = await reranker.rerank("q", ["item"]);
|
||||
|
||||
expect(results[0].rerankScore).toBe(1);
|
||||
});
|
||||
|
||||
it("assigns a neutral 0.5 score (not 0.0) when a per-document LLM call fails, and still returns that document", async () => {
|
||||
const llm: LLM = {
|
||||
generateResponse: jest
|
||||
.fn()
|
||||
.mockResolvedValueOnce("0.9") // scores "good"
|
||||
.mockRejectedValueOnce(new Error("rate limited")), // scores "bad"
|
||||
generateChat: async () => ({ content: "", role: "assistant" }),
|
||||
};
|
||||
const warnSpy = jest.spyOn(console, "warn").mockImplementation(() => {});
|
||||
const reranker = new LLMReranker({}, llm);
|
||||
|
||||
const results = await reranker.rerank("q", ["good", "bad"]);
|
||||
|
||||
expect(results).toHaveLength(2);
|
||||
const byIndex = new Map(results.map((r) => [r.index, r.rerankScore]));
|
||||
expect(byIndex.get(0)).toBe(0.9);
|
||||
expect(byIndex.get(1)).toBe(0.5);
|
||||
expect(warnSpy).toHaveBeenCalled();
|
||||
|
||||
warnSpy.mockRestore();
|
||||
});
|
||||
|
||||
it("sends the exact system prompt and a separate user message with the query and document", async () => {
|
||||
const generateResponse = jest.fn().mockResolvedValue("0.5");
|
||||
const llm: LLM = {
|
||||
generateResponse,
|
||||
generateChat: async () => ({ content: "", role: "assistant" }),
|
||||
};
|
||||
const reranker = new LLMReranker({}, llm);
|
||||
|
||||
await reranker.rerank("what is the capital?", [
|
||||
"Paris is the capital of France.",
|
||||
]);
|
||||
|
||||
expect(generateResponse).toHaveBeenCalledWith([
|
||||
{ role: "system", content: EXPECTED_SYSTEM_PROMPT },
|
||||
{
|
||||
role: "user",
|
||||
content:
|
||||
"Query: what is the capital?\n\nDocument: Paris is the capital of France.",
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("truncates the query and document to 4000 characters before sending", async () => {
|
||||
const generateResponse = jest.fn().mockResolvedValue("0.5");
|
||||
const llm: LLM = {
|
||||
generateResponse,
|
||||
generateChat: async () => ({ content: "", role: "assistant" }),
|
||||
};
|
||||
const reranker = new LLMReranker({}, llm);
|
||||
const longQuery = "q".repeat(5000);
|
||||
const longDoc = "d".repeat(5000);
|
||||
|
||||
await reranker.rerank(longQuery, [longDoc]);
|
||||
|
||||
const userMessage = generateResponse.mock.calls[0][0][1];
|
||||
const sentQuery = userMessage.content.match(/^Query: (q+)/)[1];
|
||||
const sentDoc = userMessage.content.match(/Document: (d+)/)[1];
|
||||
expect(sentQuery).toHaveLength(4000);
|
||||
expect(sentDoc).toHaveLength(4000);
|
||||
});
|
||||
|
||||
it("throws when no LLM is provided", () => {
|
||||
expect(() => new LLMReranker({}, undefined as unknown as LLM)).toThrow();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,87 @@
|
||||
import { RerankerConfig } from "../types";
|
||||
import { LLM, LLMResponse } from "../llms/base";
|
||||
import { Reranker, RerankResult } from "./base";
|
||||
|
||||
const SYSTEM_PROMPT = `You are a relevance scoring assistant. Given a query and a document, score how relevant the document is to the query.
|
||||
|
||||
Score the relevance on a scale from 0.0 to 1.0, where:
|
||||
- 1.0 = Perfectly relevant and directly answers the query
|
||||
- 0.8-0.9 = Highly relevant with good information
|
||||
- 0.6-0.7 = Moderately relevant with some useful information
|
||||
- 0.4-0.5 = Slightly relevant with limited useful information
|
||||
- 0.0-0.3 = Not relevant or no useful information
|
||||
|
||||
Respond with only a single numerical score between 0.0 and 1.0. Do not include any explanation or additional text.`;
|
||||
|
||||
const MAX_INPUT_LEN = 4000;
|
||||
|
||||
export class LLMReranker implements Reranker {
|
||||
private llm: LLM;
|
||||
private topK?: number;
|
||||
|
||||
constructor(config: RerankerConfig, llm: LLM) {
|
||||
if (!llm) {
|
||||
throw new Error(
|
||||
"LLMReranker requires an LLM instance; RerankerFactory should always provide one for the llm_reranker provider.",
|
||||
);
|
||||
}
|
||||
this.llm = llm;
|
||||
this.topK = config.topK;
|
||||
}
|
||||
|
||||
async rerank(
|
||||
query: string,
|
||||
documents: string[],
|
||||
topK?: number,
|
||||
): Promise<RerankResult[]> {
|
||||
if (documents.length === 0) return [];
|
||||
|
||||
const scored = await Promise.all(
|
||||
documents.map(async (document, index) => {
|
||||
try {
|
||||
const rerankScore = await this.score(query, document);
|
||||
return { index, rerankScore };
|
||||
} catch (e) {
|
||||
console.warn(
|
||||
`LLM reranking failed for a document, assigning neutral score: ${e}`,
|
||||
);
|
||||
return { index, rerankScore: 0.5 };
|
||||
}
|
||||
}),
|
||||
);
|
||||
|
||||
scored.sort((a, b) => b.rerankScore - a.rerankScore);
|
||||
const finalTopK = topK || this.topK;
|
||||
return finalTopK ? scored.slice(0, finalTopK) : scored;
|
||||
}
|
||||
|
||||
private async score(query: string, document: string): Promise<number> {
|
||||
const safeQuery = query.slice(0, MAX_INPUT_LEN);
|
||||
const safeDoc = document.slice(0, MAX_INPUT_LEN);
|
||||
const userMessage = `Query: ${safeQuery}\n\nDocument: ${safeDoc}`;
|
||||
|
||||
const response = await this.llm.generateResponse([
|
||||
{ role: "system", content: SYSTEM_PROMPT },
|
||||
{ role: "user", content: userMessage },
|
||||
]);
|
||||
|
||||
const text =
|
||||
typeof response === "string"
|
||||
? response
|
||||
: ((response as LLMResponse)?.content ?? "");
|
||||
|
||||
return this.extractScore(text);
|
||||
}
|
||||
|
||||
private extractScore(responseText: string): number {
|
||||
const matches =
|
||||
responseText.match(/-?\d+\.\d+/g) || responseText.match(/-?\d+/g);
|
||||
|
||||
if (matches && matches.length > 0) {
|
||||
const score = parseFloat(matches[0]);
|
||||
return Math.min(Math.max(score, 0.0), 1.0);
|
||||
}
|
||||
|
||||
return 0.5;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,133 @@
|
||||
const mockRerank = jest.fn();
|
||||
|
||||
jest.mock("zeroentropy", () => ({
|
||||
ZeroEntropy: jest.fn().mockImplementation(() => ({
|
||||
models: { rerank: mockRerank },
|
||||
})),
|
||||
}));
|
||||
|
||||
import { ZeroEntropy } from "zeroentropy";
|
||||
import { ZeroEntropyReranker } from "./zeroentropy";
|
||||
|
||||
describe("ZeroEntropyReranker", () => {
|
||||
beforeEach(() => {
|
||||
mockRerank.mockReset();
|
||||
(ZeroEntropy as unknown as jest.Mock).mockClear();
|
||||
});
|
||||
|
||||
it("throws when no API key is provided or configured", () => {
|
||||
const originalEnv = process.env.ZERO_ENTROPY_API_KEY;
|
||||
delete process.env.ZERO_ENTROPY_API_KEY;
|
||||
|
||||
expect(() => new ZeroEntropyReranker({})).toThrow(
|
||||
/Zero Entropy API key is required/,
|
||||
);
|
||||
|
||||
if (originalEnv !== undefined)
|
||||
process.env.ZERO_ENTROPY_API_KEY = originalEnv;
|
||||
});
|
||||
|
||||
it("sends the query, documents, and default model to ZeroEntropy without a top_n parameter", async () => {
|
||||
mockRerank.mockResolvedValue({ results: [] });
|
||||
const reranker = new ZeroEntropyReranker({ apiKey: "key" });
|
||||
|
||||
await reranker.rerank("capital of US?", ["a", "b", "c"], 2);
|
||||
|
||||
expect(mockRerank).toHaveBeenCalledWith({
|
||||
model: "zerank-1",
|
||||
query: "capital of US?",
|
||||
documents: ["a", "b", "c"],
|
||||
});
|
||||
});
|
||||
|
||||
it("maps ZeroEntropy's results (relevance_score) to {index, rerankScore}", async () => {
|
||||
mockRerank.mockResolvedValue({
|
||||
results: [
|
||||
{ index: 2, relevance_score: 0.9 },
|
||||
{ index: 0, relevance_score: 0.31 },
|
||||
],
|
||||
});
|
||||
const reranker = new ZeroEntropyReranker({ apiKey: "key" });
|
||||
|
||||
const results = await reranker.rerank("q", ["x", "y", "z"]);
|
||||
|
||||
expect(results).toEqual([
|
||||
{ index: 2, rerankScore: 0.9 },
|
||||
{ index: 0, rerankScore: 0.31 },
|
||||
]);
|
||||
});
|
||||
|
||||
it("sorts unsorted API results by descending relevance score client-side", async () => {
|
||||
mockRerank.mockResolvedValue({
|
||||
results: [
|
||||
{ index: 0, relevance_score: 0.2 },
|
||||
{ index: 1, relevance_score: 0.9 },
|
||||
{ index: 2, relevance_score: 0.5 },
|
||||
],
|
||||
});
|
||||
const reranker = new ZeroEntropyReranker({ apiKey: "key" });
|
||||
|
||||
const results = await reranker.rerank("q", ["a", "b", "c"]);
|
||||
|
||||
expect(results.map((r) => r.index)).toEqual([1, 2, 0]);
|
||||
expect(results.map((r) => r.rerankScore)).toEqual([0.9, 0.5, 0.2]);
|
||||
});
|
||||
|
||||
it("slices to topK client-side after sorting", async () => {
|
||||
mockRerank.mockResolvedValue({
|
||||
results: [
|
||||
{ index: 0, relevance_score: 0.2 },
|
||||
{ index: 1, relevance_score: 0.9 },
|
||||
{ index: 2, relevance_score: 0.5 },
|
||||
],
|
||||
});
|
||||
const reranker = new ZeroEntropyReranker({ apiKey: "key" });
|
||||
|
||||
const results = await reranker.rerank("q", ["a", "b", "c"], 2);
|
||||
|
||||
expect(results).toEqual([
|
||||
{ index: 1, rerankScore: 0.9 },
|
||||
{ index: 2, rerankScore: 0.5 },
|
||||
]);
|
||||
});
|
||||
|
||||
it("uses a custom model when provided", async () => {
|
||||
mockRerank.mockResolvedValue({ results: [] });
|
||||
const reranker = new ZeroEntropyReranker({
|
||||
apiKey: "key",
|
||||
model: "zerank-1-small",
|
||||
});
|
||||
|
||||
await reranker.rerank("q", ["a"]);
|
||||
|
||||
expect(mockRerank).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ model: "zerank-1-small" }),
|
||||
);
|
||||
});
|
||||
|
||||
it("returns an empty array without calling ZeroEntropy when there are no documents", async () => {
|
||||
const reranker = new ZeroEntropyReranker({ apiKey: "key" });
|
||||
|
||||
const results = await reranker.rerank("q", []);
|
||||
|
||||
expect(results).toEqual([]);
|
||||
expect(mockRerank).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("falls back to the original order with rerankScore 0.0 when the API call fails", async () => {
|
||||
mockRerank.mockRejectedValue(new Error("zero entropy is down"));
|
||||
const warnSpy = jest.spyOn(console, "warn").mockImplementation(() => {});
|
||||
const reranker = new ZeroEntropyReranker({ apiKey: "key" });
|
||||
|
||||
const results = await reranker.rerank("q", ["a", "b", "c"]);
|
||||
|
||||
expect(results).toEqual([
|
||||
{ index: 0, rerankScore: 0.0 },
|
||||
{ index: 1, rerankScore: 0.0 },
|
||||
{ index: 2, rerankScore: 0.0 },
|
||||
]);
|
||||
expect(warnSpy).toHaveBeenCalled();
|
||||
|
||||
warnSpy.mockRestore();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,58 @@
|
||||
import { ZeroEntropy } from "zeroentropy";
|
||||
import { RerankerConfig } from "../types";
|
||||
import { Reranker, RerankResult } from "./base";
|
||||
|
||||
const DEFAULT_MODEL = "zerank-1";
|
||||
|
||||
export class ZeroEntropyReranker implements Reranker {
|
||||
private client: ZeroEntropy;
|
||||
private model: string;
|
||||
private topK?: number;
|
||||
|
||||
constructor(config: RerankerConfig) {
|
||||
const apiKey = config.apiKey || process.env.ZERO_ENTROPY_API_KEY;
|
||||
if (!apiKey) {
|
||||
throw new Error(
|
||||
"Zero Entropy API key is required. Set ZERO_ENTROPY_API_KEY environment variable or pass apiKey in config.",
|
||||
);
|
||||
}
|
||||
this.client = new ZeroEntropy({ apiKey });
|
||||
this.model = config.model || DEFAULT_MODEL;
|
||||
this.topK = config.topK;
|
||||
}
|
||||
|
||||
async rerank(
|
||||
query: string,
|
||||
documents: string[],
|
||||
topK?: number,
|
||||
): Promise<RerankResult[]> {
|
||||
if (documents.length === 0) return [];
|
||||
|
||||
try {
|
||||
const response = await this.client.models.rerank({
|
||||
model: this.model,
|
||||
query,
|
||||
documents,
|
||||
});
|
||||
|
||||
const scored = response.results.map((result) => ({
|
||||
index: result.index,
|
||||
rerankScore: result.relevance_score,
|
||||
}));
|
||||
scored.sort((a, b) => b.rerankScore - a.rerankScore);
|
||||
|
||||
const finalTopK = topK || this.topK;
|
||||
return finalTopK ? scored.slice(0, finalTopK) : scored;
|
||||
} catch (e) {
|
||||
console.warn(
|
||||
`Zero Entropy reranking failed, falling back to original order: ${e}`,
|
||||
);
|
||||
const scored = documents.map((_, index) => ({
|
||||
index,
|
||||
rerankScore: 0.0,
|
||||
}));
|
||||
const finalTopK = topK || this.topK;
|
||||
return finalTopK ? scored.slice(0, finalTopK) : scored;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -60,6 +60,53 @@ export interface LLMConfig {
|
||||
maxTokens?: number;
|
||||
}
|
||||
|
||||
export interface RerankerConfig {
|
||||
apiKey?: string;
|
||||
/** The reranker model to use. Default varies by provider. */
|
||||
model?: string;
|
||||
/** Maximum number of documents to return after reranking. Default: unset (return all). */
|
||||
topK?: number;
|
||||
/** `cohere` only. Return document texts in the response. Default: `false`. */
|
||||
returnDocuments?: boolean;
|
||||
/** `cohere` only. Maximum number of chunks per document. Default: unset. */
|
||||
maxChunksPerDoc?: number;
|
||||
/**
|
||||
* `sentence_transformer` / `huggingface` only. Transformers.js device, e.g.
|
||||
* `"cpu"`, `"wasm"`, `"webgpu"`. Default: unset (auto-detect).
|
||||
*/
|
||||
device?: string;
|
||||
/** `huggingface` only. Max token length per query-document pair. Default: `512`. */
|
||||
maxLength?: number;
|
||||
/**
|
||||
* `sentence_transformer` / `huggingface` only. Sigmoid-normalize raw logits
|
||||
* to `[0, 1]`. Default: `true`; set `false` to surface raw logits.
|
||||
*/
|
||||
normalize?: boolean;
|
||||
/** No-op: a search reranks a small candidate set in one forward pass. */
|
||||
batchSize?: number;
|
||||
/** No-op in this runtime. */
|
||||
showProgressBar?: boolean;
|
||||
/**
|
||||
* `llm_reranker` only. LLM provider used to build the scoring LLM when
|
||||
* `llm` is not set. Default: `"openai"`.
|
||||
*/
|
||||
provider?: string;
|
||||
/** `llm_reranker` only. Temperature for LLM generation. Default: `0.0`. */
|
||||
temperature?: number;
|
||||
/** `llm_reranker` only. Maximum tokens for the LLM response. Default: `100`. */
|
||||
maxTokens?: number;
|
||||
/**
|
||||
* `llm_reranker` only. Nested LLM configuration. When set, it overrides the
|
||||
* top-level `provider`/`model`/`temperature`/`maxTokens`/`apiKey`, which
|
||||
* then only act as defaults for fields missing from `llm.config`.
|
||||
*/
|
||||
llm?: {
|
||||
provider: string;
|
||||
config: LLMConfig;
|
||||
};
|
||||
[key: string]: any;
|
||||
}
|
||||
|
||||
export interface MemoryConfig {
|
||||
version?: string;
|
||||
embedder: {
|
||||
@@ -74,6 +121,10 @@ export interface MemoryConfig {
|
||||
provider: string;
|
||||
config: LLMConfig;
|
||||
};
|
||||
reranker?: {
|
||||
provider: string;
|
||||
config: RerankerConfig;
|
||||
};
|
||||
historyStore?: HistoryStoreConfig;
|
||||
disableHistory?: boolean;
|
||||
historyDbPath?: string;
|
||||
@@ -87,6 +138,8 @@ export interface MemoryItem {
|
||||
createdAt?: string;
|
||||
updatedAt?: string;
|
||||
score?: number;
|
||||
/** Relevance score added by the reranker, alongside (not replacing) `score`. */
|
||||
rerankScore?: number;
|
||||
metadata?: Record<string, any>;
|
||||
attributedTo?: string;
|
||||
}
|
||||
@@ -156,5 +209,11 @@ export const MemoryConfigSchema = z.object({
|
||||
config: z.record(z.string(), z.any()),
|
||||
})
|
||||
.optional(),
|
||||
reranker: z
|
||||
.object({
|
||||
provider: z.string(),
|
||||
config: z.record(z.string(), z.any()),
|
||||
})
|
||||
.optional(),
|
||||
disableHistory: z.boolean().optional(),
|
||||
});
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
jest.mock("zeroentropy", () => ({ ZeroEntropy: jest.fn() }));
|
||||
jest.mock("@huggingface/transformers", () => ({
|
||||
AutoModelForSequenceClassification: { from_pretrained: jest.fn() },
|
||||
AutoTokenizer: { from_pretrained: jest.fn() },
|
||||
}));
|
||||
|
||||
import { RerankerFactory } from "./factory";
|
||||
import { CohereReranker } from "../rerankers/cohere";
|
||||
import { LLMReranker } from "../rerankers/llm";
|
||||
import { ZeroEntropyReranker } from "../rerankers/zeroentropy";
|
||||
import { CrossEncoderReranker } from "../rerankers/cross_encoder";
|
||||
|
||||
describe("RerankerFactory", () => {
|
||||
const originalOpenAiKey = process.env.OPENAI_API_KEY;
|
||||
|
||||
afterEach(() => {
|
||||
if (originalOpenAiKey === undefined) {
|
||||
delete process.env.OPENAI_API_KEY;
|
||||
} else {
|
||||
process.env.OPENAI_API_KEY = originalOpenAiKey;
|
||||
}
|
||||
});
|
||||
|
||||
it("creates a CohereReranker for provider 'cohere'", () => {
|
||||
const reranker = RerankerFactory.create("cohere", { apiKey: "key" });
|
||||
expect(reranker).toBeInstanceOf(CohereReranker);
|
||||
});
|
||||
|
||||
it("matches the provider name case-insensitively", () => {
|
||||
const reranker = RerankerFactory.create("Cohere", { apiKey: "key" });
|
||||
expect(reranker).toBeInstanceOf(CohereReranker);
|
||||
});
|
||||
|
||||
it("creates a ZeroEntropyReranker for provider 'zero_entropy'", () => {
|
||||
const reranker = RerankerFactory.create("zero_entropy", {
|
||||
apiKey: "key",
|
||||
});
|
||||
expect(reranker).toBeInstanceOf(ZeroEntropyReranker);
|
||||
});
|
||||
|
||||
it("creates a CrossEncoderReranker for provider 'sentence_transformer'", () => {
|
||||
const reranker = RerankerFactory.create("sentence_transformer", {});
|
||||
expect(reranker).toBeInstanceOf(CrossEncoderReranker);
|
||||
});
|
||||
|
||||
it("creates a CrossEncoderReranker for provider 'huggingface'", () => {
|
||||
const reranker = RerankerFactory.create("huggingface", {});
|
||||
expect(reranker).toBeInstanceOf(CrossEncoderReranker);
|
||||
});
|
||||
|
||||
it("creates an LLMReranker for provider 'llm_reranker', building a default openai LLM from top-level config", () => {
|
||||
const reranker = RerankerFactory.create("llm_reranker", {
|
||||
apiKey: "key",
|
||||
});
|
||||
expect(reranker).toBeInstanceOf(LLMReranker);
|
||||
});
|
||||
|
||||
it("creates an LLMReranker that builds its own LLM from a nested config.llm", () => {
|
||||
const reranker = RerankerFactory.create("llm_reranker", {
|
||||
llm: { provider: "openai", config: { apiKey: "x" } },
|
||||
});
|
||||
expect(reranker).toBeInstanceOf(LLMReranker);
|
||||
});
|
||||
|
||||
it("prefers the nested llm.provider over the top-level provider when building the llm_reranker's LLM", () => {
|
||||
// If the top-level `provider` were used instead of the nested one, this
|
||||
// would throw ("Unsupported LLM provider: not-a-real-provider").
|
||||
const reranker = RerankerFactory.create("llm_reranker", {
|
||||
provider: "not-a-real-provider",
|
||||
llm: { provider: "openai", config: { apiKey: "key" } },
|
||||
});
|
||||
expect(reranker).toBeInstanceOf(LLMReranker);
|
||||
});
|
||||
|
||||
it("throws for the 'llm_reranker' provider when the default LLM has no API key available", () => {
|
||||
delete process.env.OPENAI_API_KEY;
|
||||
|
||||
expect(() => RerankerFactory.create("llm_reranker", {})).toThrow();
|
||||
});
|
||||
|
||||
it("throws for an unsupported provider", () => {
|
||||
expect(() => RerankerFactory.create("banana", {})).toThrow(
|
||||
/unsupported reranker provider/i,
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -12,8 +12,14 @@ import {
|
||||
EmbeddingConfig,
|
||||
HistoryStoreConfig,
|
||||
LLMConfig,
|
||||
RerankerConfig,
|
||||
VectorStoreConfig,
|
||||
} from "../types";
|
||||
import { Reranker } from "../rerankers/base";
|
||||
import { CohereReranker } from "../rerankers/cohere";
|
||||
import { LLMReranker } from "../rerankers/llm";
|
||||
import { ZeroEntropyReranker } from "../rerankers/zeroentropy";
|
||||
import { CrossEncoderReranker } from "../rerankers/cross_encoder";
|
||||
import { Embedder } from "../embeddings/base";
|
||||
import { LLM } from "../llms/base";
|
||||
import { VectorStore } from "../vector_stores/base";
|
||||
@@ -186,6 +192,69 @@ export class VectorStoreFactory {
|
||||
}
|
||||
}
|
||||
|
||||
export class RerankerFactory {
|
||||
static create(provider: string, config: RerankerConfig): Reranker {
|
||||
switch (provider.toLowerCase()) {
|
||||
case "cohere":
|
||||
return new CohereReranker(config);
|
||||
case "zero_entropy":
|
||||
return new ZeroEntropyReranker(config);
|
||||
case "sentence_transformer":
|
||||
return new CrossEncoderReranker(
|
||||
config,
|
||||
"Xenova/ms-marco-MiniLM-L-6-v2",
|
||||
);
|
||||
case "huggingface":
|
||||
return new CrossEncoderReranker(
|
||||
config,
|
||||
"Xenova/bge-reranker-base",
|
||||
512,
|
||||
);
|
||||
case "llm_reranker": {
|
||||
const llm = RerankerFactory.buildLLMRerankerLLM(config);
|
||||
return new LLMReranker(config, llm);
|
||||
}
|
||||
default:
|
||||
throw new Error(`Unsupported reranker provider: ${provider}`);
|
||||
}
|
||||
}
|
||||
|
||||
private static buildLLMRerankerLLM(config: RerankerConfig): LLM {
|
||||
const nested = config.llm;
|
||||
let llmProvider: string;
|
||||
let llmConfig: LLMConfig;
|
||||
|
||||
if (nested) {
|
||||
llmProvider = nested.provider || config.provider || "openai";
|
||||
llmConfig = { ...(nested.config || {}) };
|
||||
if (llmConfig.model === undefined) {
|
||||
llmConfig.model = config.model ?? "gpt-4o-mini";
|
||||
}
|
||||
if (llmConfig.temperature === undefined) {
|
||||
llmConfig.temperature = config.temperature ?? 0.0;
|
||||
}
|
||||
if (llmConfig.maxTokens === undefined) {
|
||||
llmConfig.maxTokens = config.maxTokens ?? 100;
|
||||
}
|
||||
if (config.apiKey && llmConfig.apiKey === undefined) {
|
||||
llmConfig.apiKey = config.apiKey;
|
||||
}
|
||||
} else {
|
||||
llmProvider = config.provider || "openai";
|
||||
llmConfig = {
|
||||
model: config.model ?? "gpt-4o-mini",
|
||||
temperature: config.temperature ?? 0.0,
|
||||
maxTokens: config.maxTokens ?? 100,
|
||||
};
|
||||
if (config.apiKey) {
|
||||
llmConfig.apiKey = config.apiKey;
|
||||
}
|
||||
}
|
||||
|
||||
return LLMFactory.create(llmProvider, llmConfig);
|
||||
}
|
||||
}
|
||||
|
||||
export class HistoryManagerFactory {
|
||||
static create(provider: string, config: HistoryStoreConfig): HistoryManager {
|
||||
switch (provider.toLowerCase()) {
|
||||
|
||||
@@ -0,0 +1,184 @@
|
||||
/**
|
||||
* Reranker integration tests for Memory.search().
|
||||
*
|
||||
* Verifies the per-search `rerank` flag: when a reranker is configured and
|
||||
* `rerank: true` is passed, search reorders results by the reranker's output;
|
||||
* otherwise results pass through unchanged. Failures degrade gracefully.
|
||||
*/
|
||||
/// <reference types="jest" />
|
||||
import { Memory } from "../src/memory";
|
||||
import { CohereReranker } from "../src/rerankers/cohere";
|
||||
import type { RerankResult } from "../src/rerankers/base";
|
||||
|
||||
jest.setTimeout(15000);
|
||||
|
||||
jest.mock("../src/embeddings/google", () => ({
|
||||
GoogleEmbedder: jest.fn(),
|
||||
}));
|
||||
jest.mock("../src/llms/google", () => ({
|
||||
GoogleLLM: jest.fn(),
|
||||
}));
|
||||
jest.mock("../src/llms/openai", () => ({
|
||||
OpenAILLM: jest.fn().mockImplementation(() => ({
|
||||
generateResponse: jest
|
||||
.fn()
|
||||
.mockResolvedValue(JSON.stringify({ memory: [] })),
|
||||
})),
|
||||
}));
|
||||
|
||||
const mockEmbedding = new Array(1536).fill(0.1);
|
||||
jest.mock("../src/embeddings/openai", () => ({
|
||||
OpenAIEmbedder: jest.fn().mockImplementation(() => ({
|
||||
embed: jest.fn().mockResolvedValue(mockEmbedding),
|
||||
embedBatch: jest
|
||||
.fn()
|
||||
.mockImplementation((texts: string[]) =>
|
||||
Promise.resolve(texts.map(() => mockEmbedding)),
|
||||
),
|
||||
embeddingDims: 1536,
|
||||
})),
|
||||
}));
|
||||
|
||||
function createMemory(config: Record<string, any> = {}): Memory {
|
||||
return new Memory({
|
||||
version: "v1.1",
|
||||
embedder: {
|
||||
provider: "openai",
|
||||
config: { apiKey: "test-key", model: "text-embedding-3-small" },
|
||||
},
|
||||
vectorStore: {
|
||||
provider: "memory",
|
||||
config: {
|
||||
collectionName: `test-rerank-${Date.now()}-${Math.random()}`,
|
||||
dimension: 1536,
|
||||
dbPath: ":memory:",
|
||||
},
|
||||
},
|
||||
llm: {
|
||||
provider: "openai",
|
||||
config: { apiKey: "test-key", model: "gpt-5-mini" },
|
||||
},
|
||||
historyDbPath: ":memory:",
|
||||
...config,
|
||||
});
|
||||
}
|
||||
|
||||
// Two semantic results whose natural (score-sorted) order is [alpha, bravo].
|
||||
async function primeSearch(m: any) {
|
||||
await m._ensureInitialized();
|
||||
m.embedder = { embed: jest.fn().mockResolvedValue(mockEmbedding) };
|
||||
m.vectorStore.search = jest.fn().mockResolvedValue([
|
||||
{ id: "a", score: 0.9, payload: { data: "alpha" } },
|
||||
{ id: "b", score: 0.8, payload: { data: "bravo" } },
|
||||
]);
|
||||
m.vectorStore.keywordSearch = jest.fn().mockResolvedValue(null);
|
||||
}
|
||||
|
||||
describe("Memory.search reranking", () => {
|
||||
it("reorders results by the reranker when rerank:true, adding rerankScore while preserving the original vector score", async () => {
|
||||
const memory = createMemory();
|
||||
const m = memory as any;
|
||||
await primeSearch(m);
|
||||
|
||||
const rerank = jest
|
||||
.fn<Promise<RerankResult[]>, [string, string[], number?]>()
|
||||
.mockResolvedValue([
|
||||
{ index: 1, rerankScore: 0.99 }, // bravo
|
||||
{ index: 0, rerankScore: 0.4 }, // alpha
|
||||
]);
|
||||
m.reranker = { rerank };
|
||||
|
||||
const result = await m.search("what did i eat", {
|
||||
filters: { user_id: "u1" },
|
||||
rerank: true,
|
||||
});
|
||||
|
||||
expect(rerank).toHaveBeenCalledWith(
|
||||
"what did i eat",
|
||||
["alpha", "bravo"],
|
||||
expect.any(Number),
|
||||
);
|
||||
expect(result.results.map((r: any) => r.memory)).toEqual([
|
||||
"bravo",
|
||||
"alpha",
|
||||
]);
|
||||
expect(result.results[0].rerankScore).toBe(0.99);
|
||||
expect(result.results[1].rerankScore).toBe(0.4);
|
||||
// The original vector similarity `score` must survive reranking.
|
||||
expect(result.results[0].score).toBe(0.8); // bravo's original vector score
|
||||
expect(result.results[1].score).toBe(0.9); // alpha's original vector score
|
||||
|
||||
await memory.reset();
|
||||
});
|
||||
|
||||
it("leaves results untouched and does not call the reranker when rerank is omitted", async () => {
|
||||
const memory = createMemory();
|
||||
const m = memory as any;
|
||||
await primeSearch(m);
|
||||
const rerank = jest.fn();
|
||||
m.reranker = { rerank };
|
||||
|
||||
const result = await m.search("what did i eat", {
|
||||
filters: { user_id: "u1" },
|
||||
});
|
||||
|
||||
expect(rerank).not.toHaveBeenCalled();
|
||||
expect(result.results.map((r: any) => r.memory)).toEqual([
|
||||
"alpha",
|
||||
"bravo",
|
||||
]);
|
||||
expect(result.results[0].rerankScore).toBeUndefined();
|
||||
|
||||
await memory.reset();
|
||||
});
|
||||
|
||||
it("is a no-op (no throw) when rerank:true but no reranker is configured", async () => {
|
||||
const memory = createMemory();
|
||||
const m = memory as any;
|
||||
await primeSearch(m);
|
||||
|
||||
const result = await m.search("what did i eat", {
|
||||
filters: { user_id: "u1" },
|
||||
rerank: true,
|
||||
});
|
||||
|
||||
expect(result.results.map((r: any) => r.memory)).toEqual([
|
||||
"alpha",
|
||||
"bravo",
|
||||
]);
|
||||
|
||||
await memory.reset();
|
||||
});
|
||||
|
||||
it("falls back to the original results when the reranker throws", async () => {
|
||||
const memory = createMemory();
|
||||
const m = memory as any;
|
||||
await primeSearch(m);
|
||||
m.reranker = {
|
||||
rerank: jest.fn().mockRejectedValue(new Error("provider down")),
|
||||
};
|
||||
const warnSpy = jest.spyOn(console, "warn").mockImplementation(() => {});
|
||||
|
||||
const result = await m.search("what did i eat", {
|
||||
filters: { user_id: "u1" },
|
||||
rerank: true,
|
||||
});
|
||||
|
||||
expect(result.results.map((r: any) => r.memory)).toEqual([
|
||||
"alpha",
|
||||
"bravo",
|
||||
]);
|
||||
expect(warnSpy).toHaveBeenCalled();
|
||||
|
||||
warnSpy.mockRestore();
|
||||
await memory.reset();
|
||||
});
|
||||
|
||||
it("wires a reranker from config in the constructor", () => {
|
||||
const memory = createMemory({
|
||||
reranker: { provider: "cohere", config: { apiKey: "test-key" } },
|
||||
});
|
||||
|
||||
expect((memory as any).reranker).toBeInstanceOf(CohereReranker);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user