feat(memory): add search score explanations (#5102)
Co-authored-by: kartik-mem0 <kartik.labhshetwar@mem0.ai>
This commit is contained in:
co-authored by
kartik-mem0
parent
d817aa9c12
commit
a44855af9e
@@ -1064,7 +1064,7 @@ export class Memory {
|
||||
: {};
|
||||
|
||||
await this._ensureInitialized();
|
||||
const { topK = 20, threshold = 0.1 } = config;
|
||||
const { topK = 20, threshold = 0.1, explain = false } = config;
|
||||
|
||||
await this._captureEvent("search", {
|
||||
query_length: query.length,
|
||||
@@ -1243,6 +1243,7 @@ export class Memory {
|
||||
entityBoosts,
|
||||
threshold ?? 0.1,
|
||||
topK,
|
||||
explain,
|
||||
);
|
||||
|
||||
// Step 9: Format results
|
||||
@@ -1275,6 +1276,7 @@ export class Memory {
|
||||
...(payload.user_id && { user_id: payload.user_id }),
|
||||
...(payload.agent_id && { agent_id: payload.agent_id }),
|
||||
...(payload.run_id && { run_id: payload.run_id }),
|
||||
...(scored.scoreDetails && { score_details: scored.scoreDetails }),
|
||||
};
|
||||
});
|
||||
|
||||
|
||||
@@ -17,6 +17,7 @@ export interface SearchMemoryOptions {
|
||||
topK?: number;
|
||||
filters?: SearchFilters;
|
||||
threshold?: number;
|
||||
explain?: boolean;
|
||||
}
|
||||
|
||||
export interface GetAllMemoryOptions {
|
||||
|
||||
@@ -56,10 +56,21 @@ export function normalizeBm25(
|
||||
return 1.0 / (1.0 + Math.exp(-steepness * (rawScore - midpoint)));
|
||||
}
|
||||
|
||||
export interface ScoreDetails {
|
||||
semanticScore: number;
|
||||
bm25Score: number;
|
||||
entityBoost: number;
|
||||
rawScore: number;
|
||||
maxPossibleScore: number;
|
||||
finalScore: number;
|
||||
threshold: number;
|
||||
}
|
||||
|
||||
export interface ScoredResult {
|
||||
id: string;
|
||||
score: number;
|
||||
payload: Record<string, any>;
|
||||
scoreDetails?: ScoreDetails;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -82,6 +93,7 @@ export interface ScoredResult {
|
||||
* @param entityBoosts - Map of memory ID to entity boost score.
|
||||
* @param threshold - Minimum semantic score to include a candidate.
|
||||
* @param topK - Maximum number of results to return.
|
||||
* @param explain - Include scoreDetails in each result when true.
|
||||
* @returns Sorted list of scored results, highest score first.
|
||||
*/
|
||||
export function scoreAndRank(
|
||||
@@ -94,6 +106,7 @@ export function scoreAndRank(
|
||||
entityBoosts: Record<string, number>,
|
||||
threshold: number,
|
||||
topK: number,
|
||||
explain: boolean = false,
|
||||
): ScoredResult[] {
|
||||
const hasBm25 = Object.keys(bm25Scores).length > 0;
|
||||
const hasEntity = Object.keys(entityBoosts).length > 0;
|
||||
@@ -126,11 +139,23 @@ export function scoreAndRank(
|
||||
const rawCombined = semanticScore + bm25Score + entityBoost;
|
||||
const combined = Math.min(rawCombined / maxPossible, 1.0);
|
||||
|
||||
scored.push({
|
||||
const entry: ScoredResult = {
|
||||
id: memIdStr,
|
||||
score: combined,
|
||||
payload: result.payload,
|
||||
});
|
||||
};
|
||||
if (explain) {
|
||||
entry.scoreDetails = {
|
||||
semanticScore,
|
||||
bm25Score,
|
||||
entityBoost,
|
||||
rawScore: rawCombined,
|
||||
maxPossibleScore: maxPossible,
|
||||
finalScore: combined,
|
||||
threshold,
|
||||
};
|
||||
}
|
||||
scored.push(entry);
|
||||
}
|
||||
|
||||
scored.sort((a, b) => b.score - a.score);
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
/// <reference types="jest" />
|
||||
|
||||
import { scoreAndRank } from "../src/utils/scoring";
|
||||
|
||||
describe("scoreAndRank", () => {
|
||||
const results = [
|
||||
{ id: "a", score: 0.8, payload: { data: "mem a" } },
|
||||
{ id: "b", score: 0.5, payload: { data: "mem b" } },
|
||||
];
|
||||
|
||||
it("omits scoreDetails by default", () => {
|
||||
const scored = scoreAndRank(results, {}, {}, 0.1, 10);
|
||||
expect(scored[0].scoreDetails).toBeUndefined();
|
||||
expect(scored[1].scoreDetails).toBeUndefined();
|
||||
});
|
||||
|
||||
it("omits scoreDetails when explain is false", () => {
|
||||
const scored = scoreAndRank(results, {}, {}, 0.1, 10, false);
|
||||
expect(scored[0].scoreDetails).toBeUndefined();
|
||||
});
|
||||
|
||||
it("includes scoreDetails when explain is true", () => {
|
||||
const bm25 = { a: 0.6 };
|
||||
const entity = { a: 0.3 };
|
||||
const scored = scoreAndRank(results, bm25, entity, 0.1, 10, true);
|
||||
|
||||
const details = scored[0].scoreDetails!;
|
||||
expect(details).toBeDefined();
|
||||
expect(details.semanticScore).toBe(0.8);
|
||||
expect(details.bm25Score).toBe(0.6);
|
||||
expect(details.entityBoost).toBe(0.3);
|
||||
expect(details.rawScore).toBeCloseTo(1.7);
|
||||
expect(details.maxPossibleScore).toBe(2.5);
|
||||
expect(details.finalScore).toBeCloseTo(0.68);
|
||||
expect(details.threshold).toBe(0.1);
|
||||
});
|
||||
|
||||
it("includes scoreDetails for results without bm25/entity signals", () => {
|
||||
const scored = scoreAndRank(results, {}, {}, 0.1, 10, true);
|
||||
|
||||
const details = scored[0].scoreDetails!;
|
||||
expect(details.semanticScore).toBe(0.8);
|
||||
expect(details.bm25Score).toBe(0);
|
||||
expect(details.entityBoost).toBe(0);
|
||||
expect(details.rawScore).toBe(0.8);
|
||||
expect(details.maxPossibleScore).toBe(1.0);
|
||||
expect(details.finalScore).toBe(0.8);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user