feat(oss): add reranker + per-search rerank to the TypeScript OSS SDK (#6055)

This commit is contained in:
Kartik
2026-07-09 22:18:38 +05:30
committed by GitHub
parent 2a4aa232b2
commit 6a801bfe2f
32 changed files with 2173 additions and 19 deletions
+1
View File
@@ -177,6 +177,7 @@ export class ConfigManager {
})(),
disableHistory:
userConfig.disableHistory || DEFAULT_MEMORY_CONFIG.disableHistory,
reranker: userConfig.reranker,
};
// Validate the merged config
+5
View File
@@ -45,4 +45,9 @@ export * from "./vector_stores/milvus";
export * from "./vector_stores/mongodb";
export * from "./vector_stores/opensearch";
export * from "./vector_stores/weaviate";
export * from "./rerankers/base";
export * from "./rerankers/cohere";
export * from "./rerankers/llm";
export * from "./rerankers/zeroentropy";
export * from "./rerankers/cross_encoder";
export * from "./utils/factory";
+32 -1
View File
@@ -13,6 +13,7 @@ import {
LLMFactory,
VectorStoreFactory,
HistoryManagerFactory,
RerankerFactory,
} from "../utils/factory";
import {
FactRetrievalSchema,
@@ -28,6 +29,7 @@ import {
import { DummyHistoryManager } from "../storage/DummyHistoryManager";
import { Embedder } from "../embeddings/base";
import { LLM } from "../llms/base";
import { Reranker } from "../rerankers/base";
import { VectorStore } from "../vector_stores/base";
import { ConfigManager } from "../config/manager";
@@ -164,6 +166,7 @@ export class Memory {
private embedder: Embedder;
private vectorStore!: VectorStore;
private llm: LLM;
private reranker: Reranker | null = null;
private db: HistoryManager;
private collectionName: string | undefined;
private apiVersion: string;
@@ -188,6 +191,12 @@ export class Memory {
this.config.llm.provider,
this.config.llm.config,
);
if (this.config.reranker) {
this.reranker = RerankerFactory.create(
this.config.reranker.provider,
this.config.reranker.config,
);
}
if (this.config.disableHistory) {
this.db = new DummyHistoryManager();
} else {
@@ -1541,8 +1550,30 @@ export class Memory {
};
});
// Step 10: Optionally re-rank with the configured reranker. Opt-in per
// search via `rerank: true`; a no-op when no reranker is configured.
const invokeReranker = Boolean(
config.rerank && this.reranker && results.length > 0,
);
let finalResults = results;
if (invokeReranker) {
try {
const ranked = await this.reranker!.rerank(
query,
results.map((r) => r.memory),
topK,
);
finalResults = ranked.map((r) => ({
...results[r.index],
rerankScore: r.rerankScore,
}));
} catch (e) {
console.warn(`Reranking failed, using original results: ${e}`);
}
}
const result = {
results,
results: finalResults,
};
const searchElapsedMs = Date.now() - searchStartMs;
if (temporalUsageNotice) {
@@ -36,6 +36,11 @@ export interface SearchMemoryOptions {
threshold?: number;
explain?: boolean;
referenceDate?: number | string | Date | null;
/**
* Re-rank the results with the configured reranker before returning. No-op
* when no `reranker` is configured on the Memory.
*/
rerank?: boolean;
/** Include expired memories in the results. Defaults to false. */
showExpired?: boolean;
}
+21
View File
@@ -0,0 +1,21 @@
export interface RerankResult {
/** Index into the input `documents` array. */
index: number;
/** Relevance of the document to the query, 0..1, higher = more relevant. */
rerankScore: number;
}
export interface Reranker {
/**
* Rank `documents` by relevance to `query`.
*
* Returns results sorted by descending relevance. When `topK` is given, at
* most that many results are returned. Each result's `index` points back into
* the input `documents` array so callers can recover the original item.
*/
rerank(
query: string,
documents: string[],
topK?: number,
): Promise<RerankResult[]>;
}
@@ -0,0 +1,140 @@
const mockRerank = jest.fn();
jest.mock("cohere-ai", () => ({
CohereClient: jest.fn().mockImplementation(() => ({
rerank: mockRerank,
})),
}));
import { CohereClient } from "cohere-ai";
import { CohereReranker } from "./cohere";
describe("CohereReranker", () => {
beforeEach(() => {
mockRerank.mockReset();
(CohereClient as unknown as jest.Mock).mockClear();
});
it("throws when no API key is provided or configured", () => {
const originalEnv = process.env.COHERE_API_KEY;
delete process.env.COHERE_API_KEY;
expect(() => new CohereReranker({})).toThrow(/Cohere API key is required/);
if (originalEnv !== undefined) process.env.COHERE_API_KEY = originalEnv;
});
it("sends the query, documents, topN, and default model to Cohere", async () => {
mockRerank.mockResolvedValue({ results: [] });
const reranker = new CohereReranker({ apiKey: "key" });
await reranker.rerank("capital of US?", ["a", "b", "c"], 2);
expect(mockRerank).toHaveBeenCalledWith({
model: "rerank-v3.5",
query: "capital of US?",
documents: ["a", "b", "c"],
topN: 2,
returnDocuments: false,
maxChunksPerDoc: undefined,
});
});
it("defaults topN to documents.length when neither the call nor config sets a top_k", async () => {
mockRerank.mockResolvedValue({ results: [] });
const reranker = new CohereReranker({ apiKey: "key" });
await reranker.rerank("q", ["a", "b", "c"]);
expect(mockRerank).toHaveBeenCalledWith(
expect.objectContaining({ topN: 3 }),
);
});
it("forwards returnDocuments and maxChunksPerDoc from config", async () => {
mockRerank.mockResolvedValue({ results: [] });
const reranker = new CohereReranker({
apiKey: "key",
returnDocuments: true,
maxChunksPerDoc: 5,
});
await reranker.rerank("q", ["a"]);
expect(mockRerank).toHaveBeenCalledWith(
expect.objectContaining({ returnDocuments: true, maxChunksPerDoc: 5 }),
);
});
it("returns Cohere's ranked results as {index, rerankScore}", async () => {
mockRerank.mockResolvedValue({
results: [
{ index: 2, relevanceScore: 0.9 },
{ index: 0, relevanceScore: 0.31 },
],
});
const reranker = new CohereReranker({ apiKey: "key" });
const results = await reranker.rerank("q", ["x", "y", "z"]);
expect(results).toEqual([
{ index: 2, rerankScore: 0.9 },
{ index: 0, rerankScore: 0.31 },
]);
});
it("uses a custom model when provided", async () => {
mockRerank.mockResolvedValue({ results: [] });
const reranker = new CohereReranker({
apiKey: "key",
model: "rerank-v4.0-pro",
});
await reranker.rerank("q", ["a"]);
expect(mockRerank).toHaveBeenCalledWith(
expect.objectContaining({ model: "rerank-v4.0-pro" }),
);
});
it("returns an empty array without calling Cohere when there are no documents", async () => {
const reranker = new CohereReranker({ apiKey: "key" });
const results = await reranker.rerank("q", []);
expect(results).toEqual([]);
expect(mockRerank).not.toHaveBeenCalled();
});
it("falls back to the original order with rerankScore 0.0 when the Cohere API call fails", async () => {
mockRerank.mockRejectedValue(new Error("cohere is down"));
const warnSpy = jest.spyOn(console, "warn").mockImplementation(() => {});
const reranker = new CohereReranker({ apiKey: "key" });
const results = await reranker.rerank("q", ["a", "b", "c"]);
expect(results).toEqual([
{ index: 0, rerankScore: 0.0 },
{ index: 1, rerankScore: 0.0 },
{ index: 2, rerankScore: 0.0 },
]);
expect(warnSpy).toHaveBeenCalled();
warnSpy.mockRestore();
});
it("slices the fallback results by topK when the Cohere API call fails", async () => {
mockRerank.mockRejectedValue(new Error("cohere is down"));
jest.spyOn(console, "warn").mockImplementation(() => {});
const reranker = new CohereReranker({ apiKey: "key", topK: 2 });
const results = await reranker.rerank("q", ["a", "b", "c"]);
expect(results).toEqual([
{ index: 0, rerankScore: 0.0 },
{ index: 1, rerankScore: 0.0 },
]);
(console.warn as jest.Mock).mockRestore();
});
});
+61
View File
@@ -0,0 +1,61 @@
import { CohereClient } from "cohere-ai";
import { RerankerConfig } from "../types";
import { Reranker, RerankResult } from "./base";
const DEFAULT_MODEL = "rerank-v3.5";
export class CohereReranker implements Reranker {
private client: CohereClient;
private model: string;
private topK?: number;
private returnDocuments: boolean;
private maxChunksPerDoc?: number;
constructor(config: RerankerConfig) {
const apiKey = config.apiKey || process.env.COHERE_API_KEY;
if (!apiKey) {
throw new Error(
"Cohere API key is required. Set COHERE_API_KEY environment variable or pass apiKey in config.",
);
}
this.client = new CohereClient({ token: apiKey });
this.model = config.model || DEFAULT_MODEL;
this.topK = config.topK;
this.returnDocuments = config.returnDocuments ?? false;
this.maxChunksPerDoc = config.maxChunksPerDoc;
}
async rerank(
query: string,
documents: string[],
topK?: number,
): Promise<RerankResult[]> {
if (documents.length === 0) return [];
try {
const response = await this.client.rerank({
model: this.model,
query,
documents,
topN: topK || this.topK || documents.length,
returnDocuments: this.returnDocuments,
maxChunksPerDoc: this.maxChunksPerDoc,
});
return response.results.map((result) => ({
index: result.index,
rerankScore: result.relevanceScore,
}));
} catch (e) {
console.warn(
`Cohere reranking failed, falling back to original order: ${e}`,
);
const scored = documents.map((_, index) => ({
index,
rerankScore: 0.0,
}));
const finalTopK = topK || this.topK;
return finalTopK ? scored.slice(0, finalTopK) : scored;
}
}
}
@@ -0,0 +1,189 @@
const mockModelFromPretrained = jest.fn();
const mockTokenizerFromPretrained = jest.fn();
jest.mock("@huggingface/transformers", () => ({
AutoModelForSequenceClassification: {
from_pretrained: mockModelFromPretrained,
},
AutoTokenizer: { from_pretrained: mockTokenizerFromPretrained },
}));
import { CrossEncoderReranker } from "./cross_encoder";
const sigmoid = (x: number) => 1 / (1 + Math.exp(-x));
/** Wire the mocked tokenizer + model so the model returns `logits` for a call. */
function setupModel(logits: number[][]) {
const tokenizer = jest.fn().mockReturnValue({ input_ids: [] });
mockTokenizerFromPretrained.mockResolvedValue(tokenizer);
const model = jest
.fn()
.mockResolvedValue({ logits: { tolist: () => logits } });
mockModelFromPretrained.mockResolvedValue(model);
return { tokenizer, model };
}
describe("CrossEncoderReranker", () => {
beforeEach(() => {
mockModelFromPretrained.mockReset();
mockTokenizerFromPretrained.mockReset();
});
it("scores each document and returns them sorted by relevance, sigmoid-normalized to [0,1]", async () => {
setupModel([[0.0], [2.0], [-1.0]]);
const reranker = new CrossEncoderReranker({}, "default-model");
const results = await reranker.rerank("q", ["a", "b", "c"]);
// sigmoid: b(2.0)=0.88 > a(0.0)=0.5 > c(-1.0)=0.27
expect(results.map((r) => r.index)).toEqual([1, 0, 2]);
expect(results[0].rerankScore).toBeCloseTo(sigmoid(2.0), 5);
expect(results[1].rerankScore).toBeCloseTo(sigmoid(0.0), 5);
expect(results[2].rerankScore).toBeCloseTo(sigmoid(-1.0), 5);
});
it("pairs the query with each document via text_pair when tokenizing", async () => {
const { tokenizer } = setupModel([[0.1], [0.2]]);
const reranker = new CrossEncoderReranker(
{ maxLength: 128 },
"default-model",
);
await reranker.rerank("what is x", ["doc one", "doc two"]);
expect(tokenizer).toHaveBeenCalledWith(
["what is x", "what is x"],
expect.objectContaining({
text_pair: ["doc one", "doc two"],
padding: true,
truncation: true,
max_length: 128,
}),
);
});
it("applies the topK limit", async () => {
setupModel([[0.0], [2.0], [-1.0]]);
const reranker = new CrossEncoderReranker({}, "default-model");
const results = await reranker.rerank("q", ["a", "b", "c"], 2);
expect(results).toHaveLength(2);
expect(results.map((r) => r.index)).toEqual([1, 0]);
});
it("falls back to config.topK when the rerank() call omits one", async () => {
setupModel([[0.0], [2.0], [-1.0]]);
const reranker = new CrossEncoderReranker({ topK: 1 }, "default-model");
const results = await reranker.rerank("q", ["a", "b", "c"]);
expect(results).toHaveLength(1);
expect(results.map((r) => r.index)).toEqual([1]);
});
it("returns [] without loading the model when there are no documents", async () => {
const reranker = new CrossEncoderReranker({}, "default-model");
const results = await reranker.rerank("q", []);
expect(results).toEqual([]);
expect(mockModelFromPretrained).not.toHaveBeenCalled();
expect(mockTokenizerFromPretrained).not.toHaveBeenCalled();
});
it("returns raw logits as scores when normalize is false", async () => {
setupModel([[2.0], [0.0]]);
const reranker = new CrossEncoderReranker(
{ normalize: false },
"default-model",
);
const results = await reranker.rerank("q", ["a", "b"]);
expect(results.map((r) => r.index)).toEqual([0, 1]);
expect(results[0].rerankScore).toBe(2.0);
expect(results[1].rerankScore).toBe(0.0);
});
it("loads the model and tokenizer only once across multiple rerank calls", async () => {
setupModel([[0.5]]);
const reranker = new CrossEncoderReranker({}, "default-model");
await reranker.rerank("q", ["a"]);
await reranker.rerank("q2", ["b"]);
expect(mockModelFromPretrained).toHaveBeenCalledTimes(1);
expect(mockTokenizerFromPretrained).toHaveBeenCalledTimes(1);
});
it("loads the default model, or the configured model when provided", async () => {
setupModel([[0.5]]);
await new CrossEncoderReranker({}, "the-default").rerank("q", ["a"]);
expect(mockModelFromPretrained).toHaveBeenCalledWith(
"the-default",
expect.any(Object),
);
mockModelFromPretrained.mockClear();
setupModel([[0.5]]);
await new CrossEncoderReranker(
{ model: "custom/model" },
"the-default",
).rerank("q", ["a"]);
expect(mockModelFromPretrained).toHaveBeenCalledWith(
"custom/model",
expect.any(Object),
);
});
it("applies a default maxLength (as the huggingface provider passes 512) when config omits one", async () => {
const { tokenizer } = setupModel([[0.5]]);
const reranker = new CrossEncoderReranker({}, "default-model", 512);
await reranker.rerank("q", ["a"]);
expect(tokenizer).toHaveBeenCalledWith(
["q"],
expect.objectContaining({ max_length: 512 }),
);
});
it("falls back to the original order with rerankScore 0.0 when the model fails to load", async () => {
mockModelFromPretrained.mockResolvedValue(jest.fn());
mockTokenizerFromPretrained.mockRejectedValue(
new Error("model download failed"),
);
const warnSpy = jest.spyOn(console, "warn").mockImplementation(() => {});
const reranker = new CrossEncoderReranker({}, "default-model");
const results = await reranker.rerank("q", ["a", "b", "c"]);
expect(results).toEqual([
{ index: 0, rerankScore: 0.0 },
{ index: 1, rerankScore: 0.0 },
{ index: 2, rerankScore: 0.0 },
]);
expect(warnSpy).toHaveBeenCalled();
warnSpy.mockRestore();
});
it("falls back to the original order with rerankScore 0.0, sliced by topK, when scoring fails", async () => {
const tokenizer = jest.fn().mockReturnValue({ input_ids: [] });
mockTokenizerFromPretrained.mockResolvedValue(tokenizer);
mockModelFromPretrained.mockResolvedValue(
jest.fn().mockRejectedValue(new Error("forward pass failed")),
);
const warnSpy = jest.spyOn(console, "warn").mockImplementation(() => {});
const reranker = new CrossEncoderReranker({}, "default-model");
const results = await reranker.rerank("q", ["a", "b"], 1);
expect(results).toEqual([{ index: 0, rerankScore: 0.0 }]);
expect(warnSpy).toHaveBeenCalled();
warnSpy.mockRestore();
});
});
@@ -0,0 +1,98 @@
import { RerankerConfig } from "../types";
import { Reranker, RerankResult } from "./base";
const sigmoid = (x: number) => 1 / (1 + Math.exp(-x));
export class CrossEncoderReranker implements Reranker {
private modelId: string;
private device?: string;
private maxLength?: number;
private normalize: boolean;
private topK?: number;
// ponytail: batchSize/showProgressBar are accepted for config parity with the
// Python SDK but are no-ops here — a memory search reranks a small candidate
// set in a single forward pass. Chunk by batchSize if that ever grows.
private loaded?: Promise<{ model: any; tokenizer: any }>;
constructor(
config: RerankerConfig,
defaultModel: string,
defaultMaxLength?: number,
) {
this.modelId = config.model || defaultModel;
this.device = config.device;
this.maxLength = config.maxLength ?? defaultMaxLength;
this.normalize = config.normalize ?? true;
this.topK = config.topK;
}
private load() {
if (!this.loaded) {
this.loaded = (async () => {
// Lazy-load Transformers.js (and its onnxruntime native binding) only
// when a rerank actually runs. A static import would pull onnxruntime
// into every `new Memory()`, colliding on Linux with fastembed's
// separate onnxruntime version — see the merge with the FastEmbed
// embedder. Deferring it keeps memory construction free of ONNX.
const { AutoModelForSequenceClassification, AutoTokenizer } =
await import("@huggingface/transformers");
const options: any = {};
if (this.device) options.device = this.device;
const model = await AutoModelForSequenceClassification.from_pretrained(
this.modelId,
options,
);
const tokenizer = await AutoTokenizer.from_pretrained(this.modelId);
return { model, tokenizer };
})();
}
return this.loaded;
}
async rerank(
query: string,
documents: string[],
topK?: number,
): Promise<RerankResult[]> {
if (documents.length === 0) return [];
try {
const { model, tokenizer } = await this.load();
const inputs = tokenizer(
documents.map(() => query),
{
text_pair: documents,
padding: true,
truncation: true,
...(this.maxLength ? { max_length: this.maxLength } : {}),
},
);
const { logits } = await model(inputs);
const rows: unknown[] = logits.tolist();
const scored = rows.map((row, index) => {
const logit = Array.isArray(row) ? (row[0] as number) : (row as number);
return {
index,
rerankScore: this.normalize ? sigmoid(logit) : logit,
};
});
scored.sort((a, b) => b.rerankScore - a.rerankScore);
const finalTopK = topK || this.topK;
return finalTopK ? scored.slice(0, finalTopK) : scored;
} catch (e) {
console.warn(
`Cross-encoder reranking failed, falling back to original order: ${e}`,
);
const scored = documents.map((_, index) => ({
index,
rerankScore: 0.0,
}));
const finalTopK = topK || this.topK;
return finalTopK ? scored.slice(0, finalTopK) : scored;
}
}
}
+174
View File
@@ -0,0 +1,174 @@
import { LLM } from "../llms/base";
import { LLMReranker } from "./llm";
// Duplicated rather than imported from ./llm.ts so this test catches drift.
const EXPECTED_SYSTEM_PROMPT = `You are a relevance scoring assistant. Given a query and a document, score how relevant the document is to the query.
Score the relevance on a scale from 0.0 to 1.0, where:
- 1.0 = Perfectly relevant and directly answers the query
- 0.8-0.9 = Highly relevant with good information
- 0.6-0.7 = Moderately relevant with some useful information
- 0.4-0.5 = Slightly relevant with limited useful information
- 0.0-0.3 = Not relevant or no useful information
Respond with only a single numerical score between 0.0 and 1.0. Do not include any explanation or additional text.`;
/**
* Fake LLM that scores a document by looking up the document text inside the
* prompt. Test document tokens must be distinct and must not be substrings of
* the prompt boilerplate (e.g. avoid "a"/"b"), or the lookup resolves the
* wrong doc.
*/
function makeLLM(scoreByDoc: Record<string, string>): LLM {
return {
generateResponse: async (
messages: Array<{ role: string; content: string }>,
) => {
const prompt = messages.map((m) => m.content).join("\n");
const doc = Object.keys(scoreByDoc).find((d) => prompt.includes(d));
return doc ? scoreByDoc[doc] : "no number here";
},
generateChat: async () => ({ content: "", role: "assistant" }),
};
}
describe("LLMReranker", () => {
it("sorts documents by descending relevance score", async () => {
const llm = makeLLM({ cats: "0.2", dogs: "0.9", fish: "0.5" });
const reranker = new LLMReranker({}, llm);
const results = await reranker.rerank("pets", ["cats", "dogs", "fish"]);
expect(results.map((r) => r.index)).toEqual([1, 2, 0]);
expect(results.map((r) => r.rerankScore)).toEqual([0.9, 0.5, 0.2]);
});
it("clamps scores to the [0, 1] range", async () => {
const llm = makeLLM({ zebra: "1.5", walrus: "-0.3" });
const reranker = new LLMReranker({}, llm);
const results = await reranker.rerank("q", ["zebra", "walrus"]);
const byIndex = new Map(results.map((r) => [r.index, r.rerankScore]));
expect(byIndex.get(0)).toBe(1); // "zebra" 1.5 -> clamped to 1
expect(byIndex.get(1)).toBe(0); // "walrus" -0.3 -> clamped to 0
});
it("truncates results to topK", async () => {
const llm = makeLLM({ alpha: "0.1", bravo: "0.8", charlie: "0.5" });
const reranker = new LLMReranker({}, llm);
const results = await reranker.rerank(
"q",
["alpha", "bravo", "charlie"],
2,
);
expect(results).toHaveLength(2);
expect(results.map((r) => r.index)).toEqual([1, 2]); // bravo(0.8), charlie(0.5)
});
it("falls back to config.topK when the rerank() call omits one", async () => {
const llm = makeLLM({ alpha: "0.1", bravo: "0.8", charlie: "0.5" });
const reranker = new LLMReranker({ topK: 1 }, llm);
const results = await reranker.rerank("q", ["alpha", "bravo", "charlie"]);
expect(results).toHaveLength(1);
expect(results[0].index).toBe(1); // bravo(0.8)
});
it("falls back to a neutral score of 0.5 (not 0) when the LLM output has no number", async () => {
const llm = makeLLM({ junk: "I cannot rate this" });
const reranker = new LLMReranker({}, llm);
const results = await reranker.rerank("q", ["junk"]);
expect(results[0].rerankScore).toBe(0.5);
});
it("prefers a decimal match over an integer match when extracting the score", async () => {
const llm = makeLLM({ item: "The score is 0.73 out of 1" });
const reranker = new LLMReranker({}, llm);
const results = await reranker.rerank("q", ["item"]);
expect(results[0].rerankScore).toBe(0.73);
});
it("falls back to an integer match when no decimal is present", async () => {
const llm = makeLLM({ item: "I'd say this is a solid 1" });
const reranker = new LLMReranker({}, llm);
const results = await reranker.rerank("q", ["item"]);
expect(results[0].rerankScore).toBe(1);
});
it("assigns a neutral 0.5 score (not 0.0) when a per-document LLM call fails, and still returns that document", async () => {
const llm: LLM = {
generateResponse: jest
.fn()
.mockResolvedValueOnce("0.9") // scores "good"
.mockRejectedValueOnce(new Error("rate limited")), // scores "bad"
generateChat: async () => ({ content: "", role: "assistant" }),
};
const warnSpy = jest.spyOn(console, "warn").mockImplementation(() => {});
const reranker = new LLMReranker({}, llm);
const results = await reranker.rerank("q", ["good", "bad"]);
expect(results).toHaveLength(2);
const byIndex = new Map(results.map((r) => [r.index, r.rerankScore]));
expect(byIndex.get(0)).toBe(0.9);
expect(byIndex.get(1)).toBe(0.5);
expect(warnSpy).toHaveBeenCalled();
warnSpy.mockRestore();
});
it("sends the exact system prompt and a separate user message with the query and document", async () => {
const generateResponse = jest.fn().mockResolvedValue("0.5");
const llm: LLM = {
generateResponse,
generateChat: async () => ({ content: "", role: "assistant" }),
};
const reranker = new LLMReranker({}, llm);
await reranker.rerank("what is the capital?", [
"Paris is the capital of France.",
]);
expect(generateResponse).toHaveBeenCalledWith([
{ role: "system", content: EXPECTED_SYSTEM_PROMPT },
{
role: "user",
content:
"Query: what is the capital?\n\nDocument: Paris is the capital of France.",
},
]);
});
it("truncates the query and document to 4000 characters before sending", async () => {
const generateResponse = jest.fn().mockResolvedValue("0.5");
const llm: LLM = {
generateResponse,
generateChat: async () => ({ content: "", role: "assistant" }),
};
const reranker = new LLMReranker({}, llm);
const longQuery = "q".repeat(5000);
const longDoc = "d".repeat(5000);
await reranker.rerank(longQuery, [longDoc]);
const userMessage = generateResponse.mock.calls[0][0][1];
const sentQuery = userMessage.content.match(/^Query: (q+)/)[1];
const sentDoc = userMessage.content.match(/Document: (d+)/)[1];
expect(sentQuery).toHaveLength(4000);
expect(sentDoc).toHaveLength(4000);
});
it("throws when no LLM is provided", () => {
expect(() => new LLMReranker({}, undefined as unknown as LLM)).toThrow();
});
});
+87
View File
@@ -0,0 +1,87 @@
import { RerankerConfig } from "../types";
import { LLM, LLMResponse } from "../llms/base";
import { Reranker, RerankResult } from "./base";
const SYSTEM_PROMPT = `You are a relevance scoring assistant. Given a query and a document, score how relevant the document is to the query.
Score the relevance on a scale from 0.0 to 1.0, where:
- 1.0 = Perfectly relevant and directly answers the query
- 0.8-0.9 = Highly relevant with good information
- 0.6-0.7 = Moderately relevant with some useful information
- 0.4-0.5 = Slightly relevant with limited useful information
- 0.0-0.3 = Not relevant or no useful information
Respond with only a single numerical score between 0.0 and 1.0. Do not include any explanation or additional text.`;
const MAX_INPUT_LEN = 4000;
export class LLMReranker implements Reranker {
private llm: LLM;
private topK?: number;
constructor(config: RerankerConfig, llm: LLM) {
if (!llm) {
throw new Error(
"LLMReranker requires an LLM instance; RerankerFactory should always provide one for the llm_reranker provider.",
);
}
this.llm = llm;
this.topK = config.topK;
}
async rerank(
query: string,
documents: string[],
topK?: number,
): Promise<RerankResult[]> {
if (documents.length === 0) return [];
const scored = await Promise.all(
documents.map(async (document, index) => {
try {
const rerankScore = await this.score(query, document);
return { index, rerankScore };
} catch (e) {
console.warn(
`LLM reranking failed for a document, assigning neutral score: ${e}`,
);
return { index, rerankScore: 0.5 };
}
}),
);
scored.sort((a, b) => b.rerankScore - a.rerankScore);
const finalTopK = topK || this.topK;
return finalTopK ? scored.slice(0, finalTopK) : scored;
}
private async score(query: string, document: string): Promise<number> {
const safeQuery = query.slice(0, MAX_INPUT_LEN);
const safeDoc = document.slice(0, MAX_INPUT_LEN);
const userMessage = `Query: ${safeQuery}\n\nDocument: ${safeDoc}`;
const response = await this.llm.generateResponse([
{ role: "system", content: SYSTEM_PROMPT },
{ role: "user", content: userMessage },
]);
const text =
typeof response === "string"
? response
: ((response as LLMResponse)?.content ?? "");
return this.extractScore(text);
}
private extractScore(responseText: string): number {
const matches =
responseText.match(/-?\d+\.\d+/g) || responseText.match(/-?\d+/g);
if (matches && matches.length > 0) {
const score = parseFloat(matches[0]);
return Math.min(Math.max(score, 0.0), 1.0);
}
return 0.5;
}
}
@@ -0,0 +1,133 @@
const mockRerank = jest.fn();
jest.mock("zeroentropy", () => ({
ZeroEntropy: jest.fn().mockImplementation(() => ({
models: { rerank: mockRerank },
})),
}));
import { ZeroEntropy } from "zeroentropy";
import { ZeroEntropyReranker } from "./zeroentropy";
describe("ZeroEntropyReranker", () => {
beforeEach(() => {
mockRerank.mockReset();
(ZeroEntropy as unknown as jest.Mock).mockClear();
});
it("throws when no API key is provided or configured", () => {
const originalEnv = process.env.ZERO_ENTROPY_API_KEY;
delete process.env.ZERO_ENTROPY_API_KEY;
expect(() => new ZeroEntropyReranker({})).toThrow(
/Zero Entropy API key is required/,
);
if (originalEnv !== undefined)
process.env.ZERO_ENTROPY_API_KEY = originalEnv;
});
it("sends the query, documents, and default model to ZeroEntropy without a top_n parameter", async () => {
mockRerank.mockResolvedValue({ results: [] });
const reranker = new ZeroEntropyReranker({ apiKey: "key" });
await reranker.rerank("capital of US?", ["a", "b", "c"], 2);
expect(mockRerank).toHaveBeenCalledWith({
model: "zerank-1",
query: "capital of US?",
documents: ["a", "b", "c"],
});
});
it("maps ZeroEntropy's results (relevance_score) to {index, rerankScore}", async () => {
mockRerank.mockResolvedValue({
results: [
{ index: 2, relevance_score: 0.9 },
{ index: 0, relevance_score: 0.31 },
],
});
const reranker = new ZeroEntropyReranker({ apiKey: "key" });
const results = await reranker.rerank("q", ["x", "y", "z"]);
expect(results).toEqual([
{ index: 2, rerankScore: 0.9 },
{ index: 0, rerankScore: 0.31 },
]);
});
it("sorts unsorted API results by descending relevance score client-side", async () => {
mockRerank.mockResolvedValue({
results: [
{ index: 0, relevance_score: 0.2 },
{ index: 1, relevance_score: 0.9 },
{ index: 2, relevance_score: 0.5 },
],
});
const reranker = new ZeroEntropyReranker({ apiKey: "key" });
const results = await reranker.rerank("q", ["a", "b", "c"]);
expect(results.map((r) => r.index)).toEqual([1, 2, 0]);
expect(results.map((r) => r.rerankScore)).toEqual([0.9, 0.5, 0.2]);
});
it("slices to topK client-side after sorting", async () => {
mockRerank.mockResolvedValue({
results: [
{ index: 0, relevance_score: 0.2 },
{ index: 1, relevance_score: 0.9 },
{ index: 2, relevance_score: 0.5 },
],
});
const reranker = new ZeroEntropyReranker({ apiKey: "key" });
const results = await reranker.rerank("q", ["a", "b", "c"], 2);
expect(results).toEqual([
{ index: 1, rerankScore: 0.9 },
{ index: 2, rerankScore: 0.5 },
]);
});
it("uses a custom model when provided", async () => {
mockRerank.mockResolvedValue({ results: [] });
const reranker = new ZeroEntropyReranker({
apiKey: "key",
model: "zerank-1-small",
});
await reranker.rerank("q", ["a"]);
expect(mockRerank).toHaveBeenCalledWith(
expect.objectContaining({ model: "zerank-1-small" }),
);
});
it("returns an empty array without calling ZeroEntropy when there are no documents", async () => {
const reranker = new ZeroEntropyReranker({ apiKey: "key" });
const results = await reranker.rerank("q", []);
expect(results).toEqual([]);
expect(mockRerank).not.toHaveBeenCalled();
});
it("falls back to the original order with rerankScore 0.0 when the API call fails", async () => {
mockRerank.mockRejectedValue(new Error("zero entropy is down"));
const warnSpy = jest.spyOn(console, "warn").mockImplementation(() => {});
const reranker = new ZeroEntropyReranker({ apiKey: "key" });
const results = await reranker.rerank("q", ["a", "b", "c"]);
expect(results).toEqual([
{ index: 0, rerankScore: 0.0 },
{ index: 1, rerankScore: 0.0 },
{ index: 2, rerankScore: 0.0 },
]);
expect(warnSpy).toHaveBeenCalled();
warnSpy.mockRestore();
});
});
@@ -0,0 +1,58 @@
import { ZeroEntropy } from "zeroentropy";
import { RerankerConfig } from "../types";
import { Reranker, RerankResult } from "./base";
const DEFAULT_MODEL = "zerank-1";
export class ZeroEntropyReranker implements Reranker {
private client: ZeroEntropy;
private model: string;
private topK?: number;
constructor(config: RerankerConfig) {
const apiKey = config.apiKey || process.env.ZERO_ENTROPY_API_KEY;
if (!apiKey) {
throw new Error(
"Zero Entropy API key is required. Set ZERO_ENTROPY_API_KEY environment variable or pass apiKey in config.",
);
}
this.client = new ZeroEntropy({ apiKey });
this.model = config.model || DEFAULT_MODEL;
this.topK = config.topK;
}
async rerank(
query: string,
documents: string[],
topK?: number,
): Promise<RerankResult[]> {
if (documents.length === 0) return [];
try {
const response = await this.client.models.rerank({
model: this.model,
query,
documents,
});
const scored = response.results.map((result) => ({
index: result.index,
rerankScore: result.relevance_score,
}));
scored.sort((a, b) => b.rerankScore - a.rerankScore);
const finalTopK = topK || this.topK;
return finalTopK ? scored.slice(0, finalTopK) : scored;
} catch (e) {
console.warn(
`Zero Entropy reranking failed, falling back to original order: ${e}`,
);
const scored = documents.map((_, index) => ({
index,
rerankScore: 0.0,
}));
const finalTopK = topK || this.topK;
return finalTopK ? scored.slice(0, finalTopK) : scored;
}
}
}
+59
View File
@@ -60,6 +60,53 @@ export interface LLMConfig {
maxTokens?: number;
}
export interface RerankerConfig {
apiKey?: string;
/** The reranker model to use. Default varies by provider. */
model?: string;
/** Maximum number of documents to return after reranking. Default: unset (return all). */
topK?: number;
/** `cohere` only. Return document texts in the response. Default: `false`. */
returnDocuments?: boolean;
/** `cohere` only. Maximum number of chunks per document. Default: unset. */
maxChunksPerDoc?: number;
/**
* `sentence_transformer` / `huggingface` only. Transformers.js device, e.g.
* `"cpu"`, `"wasm"`, `"webgpu"`. Default: unset (auto-detect).
*/
device?: string;
/** `huggingface` only. Max token length per query-document pair. Default: `512`. */
maxLength?: number;
/**
* `sentence_transformer` / `huggingface` only. Sigmoid-normalize raw logits
* to `[0, 1]`. Default: `true`; set `false` to surface raw logits.
*/
normalize?: boolean;
/** No-op: a search reranks a small candidate set in one forward pass. */
batchSize?: number;
/** No-op in this runtime. */
showProgressBar?: boolean;
/**
* `llm_reranker` only. LLM provider used to build the scoring LLM when
* `llm` is not set. Default: `"openai"`.
*/
provider?: string;
/** `llm_reranker` only. Temperature for LLM generation. Default: `0.0`. */
temperature?: number;
/** `llm_reranker` only. Maximum tokens for the LLM response. Default: `100`. */
maxTokens?: number;
/**
* `llm_reranker` only. Nested LLM configuration. When set, it overrides the
* top-level `provider`/`model`/`temperature`/`maxTokens`/`apiKey`, which
* then only act as defaults for fields missing from `llm.config`.
*/
llm?: {
provider: string;
config: LLMConfig;
};
[key: string]: any;
}
export interface MemoryConfig {
version?: string;
embedder: {
@@ -74,6 +121,10 @@ export interface MemoryConfig {
provider: string;
config: LLMConfig;
};
reranker?: {
provider: string;
config: RerankerConfig;
};
historyStore?: HistoryStoreConfig;
disableHistory?: boolean;
historyDbPath?: string;
@@ -87,6 +138,8 @@ export interface MemoryItem {
createdAt?: string;
updatedAt?: string;
score?: number;
/** Relevance score added by the reranker, alongside (not replacing) `score`. */
rerankScore?: number;
metadata?: Record<string, any>;
attributedTo?: string;
}
@@ -156,5 +209,11 @@ export const MemoryConfigSchema = z.object({
config: z.record(z.string(), z.any()),
})
.optional(),
reranker: z
.object({
provider: z.string(),
config: z.record(z.string(), z.any()),
})
.optional(),
disableHistory: z.boolean().optional(),
});
+86
View File
@@ -0,0 +1,86 @@
jest.mock("zeroentropy", () => ({ ZeroEntropy: jest.fn() }));
jest.mock("@huggingface/transformers", () => ({
AutoModelForSequenceClassification: { from_pretrained: jest.fn() },
AutoTokenizer: { from_pretrained: jest.fn() },
}));
import { RerankerFactory } from "./factory";
import { CohereReranker } from "../rerankers/cohere";
import { LLMReranker } from "../rerankers/llm";
import { ZeroEntropyReranker } from "../rerankers/zeroentropy";
import { CrossEncoderReranker } from "../rerankers/cross_encoder";
describe("RerankerFactory", () => {
const originalOpenAiKey = process.env.OPENAI_API_KEY;
afterEach(() => {
if (originalOpenAiKey === undefined) {
delete process.env.OPENAI_API_KEY;
} else {
process.env.OPENAI_API_KEY = originalOpenAiKey;
}
});
it("creates a CohereReranker for provider 'cohere'", () => {
const reranker = RerankerFactory.create("cohere", { apiKey: "key" });
expect(reranker).toBeInstanceOf(CohereReranker);
});
it("matches the provider name case-insensitively", () => {
const reranker = RerankerFactory.create("Cohere", { apiKey: "key" });
expect(reranker).toBeInstanceOf(CohereReranker);
});
it("creates a ZeroEntropyReranker for provider 'zero_entropy'", () => {
const reranker = RerankerFactory.create("zero_entropy", {
apiKey: "key",
});
expect(reranker).toBeInstanceOf(ZeroEntropyReranker);
});
it("creates a CrossEncoderReranker for provider 'sentence_transformer'", () => {
const reranker = RerankerFactory.create("sentence_transformer", {});
expect(reranker).toBeInstanceOf(CrossEncoderReranker);
});
it("creates a CrossEncoderReranker for provider 'huggingface'", () => {
const reranker = RerankerFactory.create("huggingface", {});
expect(reranker).toBeInstanceOf(CrossEncoderReranker);
});
it("creates an LLMReranker for provider 'llm_reranker', building a default openai LLM from top-level config", () => {
const reranker = RerankerFactory.create("llm_reranker", {
apiKey: "key",
});
expect(reranker).toBeInstanceOf(LLMReranker);
});
it("creates an LLMReranker that builds its own LLM from a nested config.llm", () => {
const reranker = RerankerFactory.create("llm_reranker", {
llm: { provider: "openai", config: { apiKey: "x" } },
});
expect(reranker).toBeInstanceOf(LLMReranker);
});
it("prefers the nested llm.provider over the top-level provider when building the llm_reranker's LLM", () => {
// If the top-level `provider` were used instead of the nested one, this
// would throw ("Unsupported LLM provider: not-a-real-provider").
const reranker = RerankerFactory.create("llm_reranker", {
provider: "not-a-real-provider",
llm: { provider: "openai", config: { apiKey: "key" } },
});
expect(reranker).toBeInstanceOf(LLMReranker);
});
it("throws for the 'llm_reranker' provider when the default LLM has no API key available", () => {
delete process.env.OPENAI_API_KEY;
expect(() => RerankerFactory.create("llm_reranker", {})).toThrow();
});
it("throws for an unsupported provider", () => {
expect(() => RerankerFactory.create("banana", {})).toThrow(
/unsupported reranker provider/i,
);
});
});
+69
View File
@@ -12,8 +12,14 @@ import {
EmbeddingConfig,
HistoryStoreConfig,
LLMConfig,
RerankerConfig,
VectorStoreConfig,
} from "../types";
import { Reranker } from "../rerankers/base";
import { CohereReranker } from "../rerankers/cohere";
import { LLMReranker } from "../rerankers/llm";
import { ZeroEntropyReranker } from "../rerankers/zeroentropy";
import { CrossEncoderReranker } from "../rerankers/cross_encoder";
import { Embedder } from "../embeddings/base";
import { LLM } from "../llms/base";
import { VectorStore } from "../vector_stores/base";
@@ -186,6 +192,69 @@ export class VectorStoreFactory {
}
}
export class RerankerFactory {
static create(provider: string, config: RerankerConfig): Reranker {
switch (provider.toLowerCase()) {
case "cohere":
return new CohereReranker(config);
case "zero_entropy":
return new ZeroEntropyReranker(config);
case "sentence_transformer":
return new CrossEncoderReranker(
config,
"Xenova/ms-marco-MiniLM-L-6-v2",
);
case "huggingface":
return new CrossEncoderReranker(
config,
"Xenova/bge-reranker-base",
512,
);
case "llm_reranker": {
const llm = RerankerFactory.buildLLMRerankerLLM(config);
return new LLMReranker(config, llm);
}
default:
throw new Error(`Unsupported reranker provider: ${provider}`);
}
}
private static buildLLMRerankerLLM(config: RerankerConfig): LLM {
const nested = config.llm;
let llmProvider: string;
let llmConfig: LLMConfig;
if (nested) {
llmProvider = nested.provider || config.provider || "openai";
llmConfig = { ...(nested.config || {}) };
if (llmConfig.model === undefined) {
llmConfig.model = config.model ?? "gpt-4o-mini";
}
if (llmConfig.temperature === undefined) {
llmConfig.temperature = config.temperature ?? 0.0;
}
if (llmConfig.maxTokens === undefined) {
llmConfig.maxTokens = config.maxTokens ?? 100;
}
if (config.apiKey && llmConfig.apiKey === undefined) {
llmConfig.apiKey = config.apiKey;
}
} else {
llmProvider = config.provider || "openai";
llmConfig = {
model: config.model ?? "gpt-4o-mini",
temperature: config.temperature ?? 0.0,
maxTokens: config.maxTokens ?? 100,
};
if (config.apiKey) {
llmConfig.apiKey = config.apiKey;
}
}
return LLMFactory.create(llmProvider, llmConfig);
}
}
export class HistoryManagerFactory {
static create(provider: string, config: HistoryStoreConfig): HistoryManager {
switch (provider.toLowerCase()) {
+184
View File
@@ -0,0 +1,184 @@
/**
* Reranker integration tests for Memory.search().
*
* Verifies the per-search `rerank` flag: when a reranker is configured and
* `rerank: true` is passed, search reorders results by the reranker's output;
* otherwise results pass through unchanged. Failures degrade gracefully.
*/
/// <reference types="jest" />
import { Memory } from "../src/memory";
import { CohereReranker } from "../src/rerankers/cohere";
import type { RerankResult } from "../src/rerankers/base";
jest.setTimeout(15000);
jest.mock("../src/embeddings/google", () => ({
GoogleEmbedder: jest.fn(),
}));
jest.mock("../src/llms/google", () => ({
GoogleLLM: jest.fn(),
}));
jest.mock("../src/llms/openai", () => ({
OpenAILLM: jest.fn().mockImplementation(() => ({
generateResponse: jest
.fn()
.mockResolvedValue(JSON.stringify({ memory: [] })),
})),
}));
const mockEmbedding = new Array(1536).fill(0.1);
jest.mock("../src/embeddings/openai", () => ({
OpenAIEmbedder: jest.fn().mockImplementation(() => ({
embed: jest.fn().mockResolvedValue(mockEmbedding),
embedBatch: jest
.fn()
.mockImplementation((texts: string[]) =>
Promise.resolve(texts.map(() => mockEmbedding)),
),
embeddingDims: 1536,
})),
}));
function createMemory(config: Record<string, any> = {}): Memory {
return new Memory({
version: "v1.1",
embedder: {
provider: "openai",
config: { apiKey: "test-key", model: "text-embedding-3-small" },
},
vectorStore: {
provider: "memory",
config: {
collectionName: `test-rerank-${Date.now()}-${Math.random()}`,
dimension: 1536,
dbPath: ":memory:",
},
},
llm: {
provider: "openai",
config: { apiKey: "test-key", model: "gpt-5-mini" },
},
historyDbPath: ":memory:",
...config,
});
}
// Two semantic results whose natural (score-sorted) order is [alpha, bravo].
async function primeSearch(m: any) {
await m._ensureInitialized();
m.embedder = { embed: jest.fn().mockResolvedValue(mockEmbedding) };
m.vectorStore.search = jest.fn().mockResolvedValue([
{ id: "a", score: 0.9, payload: { data: "alpha" } },
{ id: "b", score: 0.8, payload: { data: "bravo" } },
]);
m.vectorStore.keywordSearch = jest.fn().mockResolvedValue(null);
}
describe("Memory.search reranking", () => {
it("reorders results by the reranker when rerank:true, adding rerankScore while preserving the original vector score", async () => {
const memory = createMemory();
const m = memory as any;
await primeSearch(m);
const rerank = jest
.fn<Promise<RerankResult[]>, [string, string[], number?]>()
.mockResolvedValue([
{ index: 1, rerankScore: 0.99 }, // bravo
{ index: 0, rerankScore: 0.4 }, // alpha
]);
m.reranker = { rerank };
const result = await m.search("what did i eat", {
filters: { user_id: "u1" },
rerank: true,
});
expect(rerank).toHaveBeenCalledWith(
"what did i eat",
["alpha", "bravo"],
expect.any(Number),
);
expect(result.results.map((r: any) => r.memory)).toEqual([
"bravo",
"alpha",
]);
expect(result.results[0].rerankScore).toBe(0.99);
expect(result.results[1].rerankScore).toBe(0.4);
// The original vector similarity `score` must survive reranking.
expect(result.results[0].score).toBe(0.8); // bravo's original vector score
expect(result.results[1].score).toBe(0.9); // alpha's original vector score
await memory.reset();
});
it("leaves results untouched and does not call the reranker when rerank is omitted", async () => {
const memory = createMemory();
const m = memory as any;
await primeSearch(m);
const rerank = jest.fn();
m.reranker = { rerank };
const result = await m.search("what did i eat", {
filters: { user_id: "u1" },
});
expect(rerank).not.toHaveBeenCalled();
expect(result.results.map((r: any) => r.memory)).toEqual([
"alpha",
"bravo",
]);
expect(result.results[0].rerankScore).toBeUndefined();
await memory.reset();
});
it("is a no-op (no throw) when rerank:true but no reranker is configured", async () => {
const memory = createMemory();
const m = memory as any;
await primeSearch(m);
const result = await m.search("what did i eat", {
filters: { user_id: "u1" },
rerank: true,
});
expect(result.results.map((r: any) => r.memory)).toEqual([
"alpha",
"bravo",
]);
await memory.reset();
});
it("falls back to the original results when the reranker throws", async () => {
const memory = createMemory();
const m = memory as any;
await primeSearch(m);
m.reranker = {
rerank: jest.fn().mockRejectedValue(new Error("provider down")),
};
const warnSpy = jest.spyOn(console, "warn").mockImplementation(() => {});
const result = await m.search("what did i eat", {
filters: { user_id: "u1" },
rerank: true,
});
expect(result.results.map((r: any) => r.memory)).toEqual([
"alpha",
"bravo",
]);
expect(warnSpy).toHaveBeenCalled();
warnSpy.mockRestore();
await memory.reset();
});
it("wires a reranker from config in the constructor", () => {
const memory = createMemory({
reranker: { provider: "cohere", config: { apiKey: "test-key" } },
});
expect((memory as any).reranker).toBeInstanceOf(CohereReranker);
});
});