Deshraj YadavandClaude Fable 5
4d5f2e653c
- triggers.py: hard-drop rules now filter PER TURN, so a durable fact sitting between two progress lines survives instead of being dropped with them. The eval harness caught this: 3 mixed-window fixtures were being destroyed client-side before the extractor ever saw them. - widened flag rules for 13 plainly durable windows the gate was missing (standing preferences, X-over-Y decisions, stated rules, diagnoses, verified procedures); verified procedures moved from aggressive to balanced - eval/: 56 labeled fixtures from the audited v1 corpus plus offline and live runners with a --check regression gate - un-ignore integrations/mem0-agent/eval (root .gitignore excluded it silently) Offline scorecard: hard_drop_recall 1.000, extract_recall 1.000, flag_precision 1.000, 0 misclassified. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
581 lines
26 KiB
Python
581 lines
26 KiB
Python
#!/usr/bin/env python3
|
|
"""Write-gate evaluation harness.
|
|
|
|
v1 shipped with no way to measure extraction quality, so it degraded unnoticed for
|
|
three months: 20.5% of the corpus turned into near-duplicate heartbeats and organic
|
|
searches fell from 257/month to 75/month. Nobody noticed because nobody could look.
|
|
This harness is the thing that looks.
|
|
|
|
Two modes:
|
|
|
|
--offline (default, no network)
|
|
Runs every fixture through mem0_agent.triggers.classify and scores the client's
|
|
local rules: hard-drop recall, hard-drop precision, flag precision/recall and
|
|
per-type accuracy. Costs nothing, so it can run in CI on every commit.
|
|
|
|
--live --project-id P --org-id O
|
|
Replays the `exclude` and `extract` fixtures against a SCRATCH project through
|
|
mem0_agent.api.Api with infer=True, polls until extraction lands, reads back and
|
|
scores what the platform actually stored. This measures the half of the gate that
|
|
lives in the custom instructions and cannot be unit-tested.
|
|
|
|
Both modes print a scorecard and write eval/last_report.json. `--check` turns the run
|
|
into a gate: it exits non-zero if hard-drop recall drops below 0.95 or extract recall
|
|
below 0.80, so no edit to the trigger rules or the custom instructions ships unmeasured.
|
|
|
|
Classifier contract (offline mode)
|
|
----------------------------------
|
|
`mem0_agent.triggers.classify(window)` is expected to take the message-window list and
|
|
return a decision. The adapter below accepts every reasonable shape -- a bool, a string,
|
|
a (decision, type) pair, a dict, or an object with attributes -- because the module is
|
|
built in parallel with this one. If the module is missing, offline scoring is skipped
|
|
with a clear message rather than crashing; if it exists but returns something
|
|
unreadable, that is reported as a contract mismatch, not as a score.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import os
|
|
import re
|
|
import sys
|
|
import time
|
|
import uuid
|
|
from pathlib import Path
|
|
from typing import Any, Callable
|
|
|
|
EVAL_DIR = Path(__file__).resolve().parent
|
|
sys.path.insert(0, str(EVAL_DIR))
|
|
sys.path.insert(0, str(EVAL_DIR.parent / "src"))
|
|
|
|
import fixtures as fx # noqa: E402
|
|
|
|
REPORT_PATH = EVAL_DIR / "last_report.json"
|
|
|
|
# --check gates. Raise them only when the measured score has been above the new bar for
|
|
# a while; lowering one is a decision that belongs in a PR description.
|
|
THRESHOLDS: dict[str, float] = {
|
|
"hard_drop_recall": 0.95,
|
|
"extract_recall": 0.80,
|
|
}
|
|
|
|
# Text that must never appear inside a stored memory. If one of these survives the gate
|
|
# the heartbeat class is leaking again, which is precisely how v1 rotted.
|
|
NOISE_PATTERNS = [
|
|
re.compile(r"\beta\b[^.]{0,20}\b\d", re.I),
|
|
re.compile(r"\bepoch\b", re.I),
|
|
re.compile(r"\d+\s*%\s*(complete|done)", re.I),
|
|
re.compile(r"chunks?\s+processed", re.I),
|
|
re.compile(r"\bpid\s*\d+", re.I),
|
|
re.compile(r"\bgradient\s+norm\b", re.I),
|
|
re.compile(r"\bqueue\s+depth\b", re.I),
|
|
re.compile(r"\belapsed\s+\d\d:\d\d", re.I),
|
|
]
|
|
|
|
|
|
def noise_leaks(text: str) -> list[str]:
|
|
return [p.pattern for p in NOISE_PATTERNS if p.search(text or "")]
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# classifier adapter
|
|
# ---------------------------------------------------------------------------
|
|
# Three outcomes, not two. "drop" is a hard drop -- mechanical noise the client refuses
|
|
# to send at any level. "skip" is "nothing worth storing right now", which is a soft miss:
|
|
# harmless on an exclude window, a lost memory on an extract one. Collapsing the two
|
|
# would hide exactly the regression this harness exists to catch.
|
|
_DROP_WORDS = {"drop", "noise", "ignore", "suppress", "reject", "block", "hard_drop"}
|
|
_SKIP_WORDS = {"skip", "none", "no", "defer", "wait", "noop", "no_trigger"}
|
|
_SEND_WORDS = {"send", "capture", "flag", "flagged", "store", "keep", "extract", "accept", "allow",
|
|
"pass", "yes"}
|
|
DECISIONS = ("drop", "skip", "send", "unknown")
|
|
|
|
|
|
class ContractMismatch(RuntimeError):
|
|
"""classify() exists but neither its signature nor its return value is readable."""
|
|
|
|
|
|
def load_classifier() -> tuple[Callable | None, str]:
|
|
"""Import lazily. triggers.py is written by a parallel workstream and may not exist."""
|
|
try:
|
|
from mem0_agent import triggers # type: ignore
|
|
except ImportError as e:
|
|
return None, f"mem0_agent.triggers is not available yet ({e})"
|
|
except Exception as e: # a broken module is a different problem than a missing one
|
|
return None, f"mem0_agent.triggers failed to import: {type(e).__name__}: {e}"
|
|
fn = getattr(triggers, "classify", None)
|
|
if not callable(fn):
|
|
return None, "mem0_agent.triggers exists but has no callable classify()"
|
|
return fn, ""
|
|
|
|
|
|
def _invoke(fn: Callable, window: list[dict], level: str | None = None) -> Any:
|
|
"""Try the plausible call shapes, most likely first."""
|
|
attempts: tuple[Callable[[], Any], ...] = ()
|
|
if level:
|
|
attempts += (lambda: fn(window, level), lambda: fn(window, level=level))
|
|
attempts += (
|
|
lambda: fn(window),
|
|
lambda: fn(messages=window),
|
|
lambda: fn(window=window),
|
|
lambda: fn("\n".join(m.get("content", "") for m in window)),
|
|
)
|
|
last: Exception | None = None
|
|
for call in attempts:
|
|
try:
|
|
return call()
|
|
except TypeError as e:
|
|
last = e
|
|
raise ContractMismatch(f"classify() rejected every call shape: {last}")
|
|
|
|
|
|
def _type_of(value: Any) -> str | None:
|
|
if isinstance(value, str) and value.lower() in _known_types():
|
|
return value.lower()
|
|
return None
|
|
|
|
|
|
def _known_types() -> set[str]:
|
|
try:
|
|
from mem0_agent.config.project_config import TYPES
|
|
|
|
return set(TYPES)
|
|
except Exception:
|
|
return set(fx.counts_by_type()) | {"session_state"}
|
|
|
|
|
|
def _look(result: Any, names: tuple[str, ...]) -> Any:
|
|
for n in names:
|
|
if isinstance(result, dict):
|
|
if n in result:
|
|
return result[n]
|
|
elif hasattr(result, n):
|
|
return getattr(result, n)
|
|
return None
|
|
|
|
|
|
def normalize(result: Any) -> tuple[str, str | None]:
|
|
"""Map whatever classify() returned onto (decision, type|None). See DECISIONS."""
|
|
if result is None:
|
|
return "skip", None
|
|
if isinstance(result, bool):
|
|
return ("send" if result else "skip"), None
|
|
if isinstance(result, str):
|
|
low = result.strip().lower()
|
|
if low in _known_types():
|
|
return "send", low
|
|
if low in _DROP_WORDS:
|
|
return "drop", None
|
|
if low in _SKIP_WORDS:
|
|
return "skip", None
|
|
if low in _SEND_WORDS:
|
|
return "send", None
|
|
return "unknown", None
|
|
if isinstance(result, (tuple, list)):
|
|
if not result:
|
|
return "unknown", None
|
|
decision, _ = normalize(result[0])
|
|
mtype = _type_of(result[1]) if len(result) > 1 else None
|
|
if decision == "unknown" and mtype:
|
|
decision = "send"
|
|
return decision, mtype
|
|
|
|
mtype = _type_of(_look(result, ("type", "mtype", "memory_type", "kind")))
|
|
# A string verdict is the most expressive shape, so it wins over the booleans:
|
|
# `flagged=False` cannot tell a hard drop apart from a soft skip.
|
|
verdict = _look(result, ("action", "decision", "verdict", "outcome", "status", "result"))
|
|
if isinstance(verdict, str):
|
|
decision, vtype = normalize(verdict)
|
|
if decision != "unknown":
|
|
return decision, (mtype or vtype)
|
|
drop = _look(result, ("drop", "dropped", "is_drop", "should_drop"))
|
|
if isinstance(drop, bool) and drop:
|
|
return "drop", mtype
|
|
send = _look(result, ("send", "capture", "should_capture", "flag", "flagged", "should_send", "keep"))
|
|
if isinstance(send, bool):
|
|
return ("send" if send else ("skip" if drop is False else "drop")), mtype
|
|
if isinstance(drop, bool):
|
|
return "send", mtype
|
|
if mtype:
|
|
return "send", mtype
|
|
return "unknown", None
|
|
|
|
|
|
def reason_of(result: Any) -> str:
|
|
"""Whatever the classifier called this rule -- the most useful column in the report."""
|
|
raw = _look(result, ("reason", "rule", "why", "trigger", "explanation"))
|
|
return str(raw) if isinstance(raw, (str, int)) else ""
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# scoring
|
|
# ---------------------------------------------------------------------------
|
|
def _ratio(num: int, den: int) -> float | None:
|
|
return round(num / den, 4) if den else None
|
|
|
|
|
|
def score_offline(rows: list[dict]) -> dict[str, Any]:
|
|
"""rows: {"id","label","expect_type","decision","got_type"}."""
|
|
drops = [r for r in rows if r["label"] == "drop"]
|
|
extracts = [r for r in rows if r["label"] == "extract"]
|
|
excludes = [r for r in rows if r["label"] == "exclude"]
|
|
dropped = [r for r in rows if r["decision"] == "drop"]
|
|
flagged = [r for r in rows if r["decision"] == "send"]
|
|
|
|
per_type: dict[str, dict[str, int]] = {}
|
|
for r in extracts:
|
|
want = r["expect_type"] or "unspecified"
|
|
bucket = per_type.setdefault(want, {"n": 0, "correct": 0, "typed": 0})
|
|
bucket["n"] += 1
|
|
if r["got_type"]:
|
|
bucket["typed"] += 1
|
|
if r["got_type"] == want:
|
|
bucket["correct"] += 1
|
|
typed = sum(b["typed"] for b in per_type.values())
|
|
correct = sum(b["correct"] for b in per_type.values())
|
|
|
|
return {
|
|
# The metric that keeps the corpus clean: mechanical noise must never be sent.
|
|
# capture.py only forwards windows whose action is "flag", so both "drop" and
|
|
# "skip" satisfy the contract -- the label says "never sent", not "matched a
|
|
# rule named drop".
|
|
"hard_drop_recall": _ratio(sum(1 for r in drops if r["decision"] in ("drop", "skip")), len(drops)),
|
|
# Advisory, not gated: how much of that containment comes from an explicit drop
|
|
# rule rather than from no flag rule happening to match. Noise held back only by
|
|
# the absence of a flag rule leaks the day someone adds one.
|
|
"hard_drop_explicit": _ratio(sum(1 for r in drops if r["decision"] == "drop"), len(drops)),
|
|
# Of everything hard-dropped, how much was safe to drop. A hard drop on an
|
|
# `extract` window is the one unrecoverable error -- the memory is gone and
|
|
# nothing logs a miss. Hard-dropping an `exclude` window is not counted against
|
|
# this: those are meant to be discarded, and doing it locally is simply cheaper.
|
|
"hard_drop_precision": _ratio(sum(1 for r in dropped if r["label"] != "extract"), len(dropped)),
|
|
# The metric that keeps the gate useful: durable knowledge must survive locally.
|
|
"extract_recall": _ratio(sum(1 for r in extracts if r["decision"] == "send"), len(extracts)),
|
|
# Of everything forwarded to the platform, how much was worth forwarding.
|
|
"flag_precision": _ratio(sum(1 for r in flagged if r["label"] == "extract"), len(flagged)),
|
|
# Extract windows lost to a soft "skip" rather than a hard drop. Same lost
|
|
# memory, different fix: a missing flag rule, not an over-broad drop rule.
|
|
"extract_skipped": sum(1 for r in extracts if r["decision"] == "skip"),
|
|
"extract_hard_dropped": sum(1 for r in extracts if r["decision"] == "drop"),
|
|
# Exclude windows the client suppressed locally -- free wins, not required, since
|
|
# the platform instructions are the designated owner of that class.
|
|
"exclude_suppressed_early": _ratio(
|
|
sum(1 for r in excludes if r["decision"] in ("drop", "skip")), len(excludes)),
|
|
"type_accuracy": _ratio(correct, typed),
|
|
"type_coverage": _ratio(typed, len(extracts)),
|
|
"per_type": per_type,
|
|
"unreadable": sum(1 for r in rows if r["decision"] == "unknown"),
|
|
}
|
|
|
|
|
|
def run_offline(level: str | None = None) -> dict[str, Any]:
|
|
fn, why = load_classifier()
|
|
if fn is None:
|
|
return {"mode": "offline", "skipped": True, "reason": why, "scores": {}, "rows": [],
|
|
"level": level}
|
|
|
|
rows: list[dict] = []
|
|
try:
|
|
for f in fx.FIXTURES:
|
|
try:
|
|
raw = _invoke(fn, f["window"], level)
|
|
decision, got_type = normalize(raw)
|
|
reason, err = reason_of(raw), ""
|
|
except ContractMismatch:
|
|
raise
|
|
except Exception as e: # a fixture that blows up the classifier is a real finding
|
|
decision, got_type, reason, err = "unknown", None, "", f"{type(e).__name__}: {e}"
|
|
rows.append({
|
|
"id": f["id"], "label": f["label"], "expect_type": f["expect_type"],
|
|
"decision": decision, "got_type": got_type, "reason": reason, "error": err,
|
|
})
|
|
except ContractMismatch as e:
|
|
return {"mode": "offline", "skipped": True, "reason": str(e), "scores": {}, "rows": [],
|
|
"level": level}
|
|
|
|
return {"mode": "offline", "skipped": False, "reason": "", "level": level,
|
|
"scores": score_offline(rows), "rows": rows}
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# live mode
|
|
# ---------------------------------------------------------------------------
|
|
def _memory_type(mem: dict) -> str | None:
|
|
"""Read the type from metadata. NEVER from categories: categorization lags ~3.9h
|
|
(median) and is 0% for memories under an hour old, so a fresh read sees nothing."""
|
|
meta = mem.get("metadata") or {}
|
|
return meta.get("type") if isinstance(meta, dict) else None
|
|
|
|
|
|
def score_live(rows: list[dict]) -> dict[str, Any]:
|
|
extracts = [r for r in rows if r["label"] == "extract"]
|
|
excludes = [r for r in rows if r["label"] == "exclude"]
|
|
per_type: dict[str, dict[str, int]] = {}
|
|
for r in extracts:
|
|
want = r["expect_type"] or "unspecified"
|
|
bucket = per_type.setdefault(want, {"n": 0, "stored": 0, "typed_correct": 0})
|
|
bucket["n"] += 1
|
|
if r["stored"] >= 1:
|
|
bucket["stored"] += 1
|
|
if want in (r["got_types"] or []):
|
|
bucket["typed_correct"] += 1
|
|
return {
|
|
"extract_recall": _ratio(sum(1 for r in extracts if r["stored"] >= 1), len(extracts)),
|
|
"exclude_suppression": _ratio(sum(1 for r in excludes if r["stored"] == 0), len(excludes)),
|
|
"type_match": _ratio(sum(b["typed_correct"] for b in per_type.values()),
|
|
sum(b["n"] for b in per_type.values())),
|
|
"noise_leak_count": sum(len(r["leaks"]) for r in rows),
|
|
"memories_written": sum(r["stored"] for r in rows),
|
|
"per_type": per_type,
|
|
}
|
|
|
|
|
|
def run_live(args: argparse.Namespace) -> dict[str, Any]:
|
|
from mem0_agent.api import Api, results_of # noqa: PLC0415
|
|
from mem0_agent.config.filters import all_in_scope # noqa: PLC0415
|
|
from mem0_agent.config.project_config import POLICY_VERSION # noqa: PLC0415
|
|
|
|
key = args.api_key or os.environ.get("MEM0_API_KEY")
|
|
if not key:
|
|
raise SystemExit("live mode needs an API key: --api-key or MEM0_API_KEY")
|
|
|
|
runid = args.run_id or uuid.uuid4().hex[:8]
|
|
app_id = f"eval-{runid}"
|
|
api = Api(key, org_id=args.org_id, project_id=args.project_id, strict=True)
|
|
|
|
classifier, why = load_classifier()
|
|
if classifier is None:
|
|
print(f"note: {why}; metadata.type will be stamped from expect_type (self-stamped, "
|
|
f"so type_match measures the round trip only)")
|
|
|
|
targets = [f for f in fx.FIXTURES if f["label"] in ("exclude", "extract")]
|
|
print(f"live run {runid}: project={args.project_id} app_id={app_id} fixtures={len(targets)}")
|
|
|
|
sent: list[dict] = []
|
|
for f in targets:
|
|
uid = f"eval-{runid}-{f['id']}"
|
|
mtype = f["expect_type"]
|
|
stamped_by = "expect_type"
|
|
if classifier is not None:
|
|
try:
|
|
decision, got = normalize(_invoke(classifier, f["window"], args.level))
|
|
if got:
|
|
mtype, stamped_by = got, "classifier"
|
|
except Exception:
|
|
pass
|
|
meta = {"session_id": f"eval-{runid}", "editor": "eval-harness",
|
|
"policy": POLICY_VERSION, "fixture": f["id"]}
|
|
if mtype:
|
|
meta["type"] = mtype
|
|
t0 = time.time()
|
|
status, body = api.add(f["window"], user_id=uid, app_id=app_id, infer=True, metadata=meta)
|
|
ok = status in (200, 201, 202)
|
|
print(f" sent {f['id']:34s} {status} {round(time.time() - t0, 2)}s"
|
|
+ ("" if ok else f" <- {body}"))
|
|
sent.append({"id": f["id"], "label": f["label"], "expect_type": f["expect_type"],
|
|
"user_id": uid, "add_status": status, "stamped_type": mtype,
|
|
"stamped_by": stamped_by, "accepted": ok})
|
|
|
|
def read(user_id: str) -> list[dict]:
|
|
status, body = api.get_all(all_in_scope(user_id, app_id), page_size=100)
|
|
return results_of(body) if status == 200 else []
|
|
|
|
# Poll rather than sleep: extraction landed anywhere from 20s to 5min in validation,
|
|
# so any fixed wait is either wrong or wasteful. Only the extract fixtures have a
|
|
# target to poll for; a zero read on an exclude fixture is only meaningful once the
|
|
# settle window has passed, so those are read once at the end.
|
|
started = time.time()
|
|
deadline = started + args.timeout
|
|
pending = {r["id"]: r["user_id"] for r in sent if r["label"] == "extract" and r["accepted"]}
|
|
landed: dict[str, list[dict]] = {}
|
|
interval = 5.0
|
|
print(f"polling for extraction (timeout {args.timeout:.0f}s, settle {args.min_settle:.0f}s)...")
|
|
while pending and time.time() < deadline:
|
|
time.sleep(min(interval, max(1.0, deadline - time.time())))
|
|
interval = min(interval * 1.4, 20.0)
|
|
for fid, uid in list(pending.items()):
|
|
mems = read(uid)
|
|
if mems:
|
|
landed[fid] = mems
|
|
pending.pop(fid, None)
|
|
print(f" t+{int(time.time() - started)}s extract landed "
|
|
f"{len(landed)}/{len(landed) + len(pending)}")
|
|
remaining_settle = args.min_settle - (time.time() - started)
|
|
if remaining_settle > 0:
|
|
print(f" settling {int(remaining_settle)}s before the suppression read...")
|
|
time.sleep(remaining_settle)
|
|
|
|
# Final read for everything: extract fixtures may have gained a second memory since
|
|
# they first landed, and the exclude fixtures are read here for the first time.
|
|
found = {r["id"]: read(r["user_id"]) for r in sent}
|
|
|
|
rows: list[dict] = []
|
|
for r in sent:
|
|
mems = found.get(r["id"], [])
|
|
texts = [m.get("memory") or "" for m in mems]
|
|
rows.append({**r,
|
|
"stored": len(mems),
|
|
"got_types": sorted({t for t in (_memory_type(m) for m in mems) if t}),
|
|
"memories": [{"id": m.get("id"), "text": t, "type": _memory_type(m),
|
|
"categories": m.get("categories")} for m, t in zip(mems, texts)],
|
|
"leaks": [lk for t in texts for lk in noise_leaks(t)]})
|
|
|
|
report = {"mode": "live", "skipped": False, "reason": "", "run_id": runid,
|
|
"project_id": args.project_id, "app_id": app_id,
|
|
"scores": score_live(rows), "rows": rows}
|
|
|
|
if args.cleanup:
|
|
deleted = 0
|
|
for r in rows:
|
|
status, _ = api.delete_all(user_id=r["user_id"], app_id=app_id)
|
|
deleted += r["stored"] if status in (200, 202, 204) else 0
|
|
report["cleanup"] = {"deleted_scopes": len(rows), "deleted_memories": deleted}
|
|
print(f"cleanup: removed {deleted} memories across {len(rows)} scopes")
|
|
else:
|
|
report["cleanup"] = {"skipped": True,
|
|
"hint": f"delete with app_id={app_id} / user_id prefix eval-{runid}-"}
|
|
return report
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# reporting
|
|
# ---------------------------------------------------------------------------
|
|
def print_scorecard(report: dict[str, Any]) -> None:
|
|
counts = fx.counts()
|
|
print()
|
|
print("=" * 72)
|
|
title = f" mem0-agent write gate -- {report['mode']} scorecard"
|
|
if report.get("level"):
|
|
title += f" (level={report['level']})"
|
|
print(title)
|
|
print("=" * 72)
|
|
print(f" coverage: {fx.coverage_line()}")
|
|
|
|
if report.get("skipped"):
|
|
print(f" SKIPPED: {report['reason']}")
|
|
print("=" * 72)
|
|
return
|
|
|
|
scores = report["scores"]
|
|
for name, value in scores.items():
|
|
if name == "per_type" or isinstance(value, dict):
|
|
continue
|
|
gate = THRESHOLDS.get(name)
|
|
shown = "n/a" if value is None else f"{value:.3f}" if isinstance(value, float) else str(value)
|
|
mark = ""
|
|
if gate is not None:
|
|
mark = " FAIL" if (value is None or value < gate) else " ok"
|
|
shown += f" (min {gate:.2f}){mark}"
|
|
print(f" {name:24s} {shown}")
|
|
|
|
per_type = scores.get("per_type") or {}
|
|
if per_type:
|
|
print(" per expected type:")
|
|
for t, b in sorted(per_type.items()):
|
|
body = " ".join(f"{k}={v}" for k, v in b.items())
|
|
print(f" {t:14s} {body}")
|
|
|
|
if report["mode"] == "offline":
|
|
soft = [r for r in report["rows"] if r["label"] == "drop" and r["decision"] == "skip"]
|
|
if soft:
|
|
print(f" noise contained only by the absence of a flag rule ({len(soft)}, advisory):")
|
|
for r in soft:
|
|
print(f" {r['id']:38s} {r.get('reason', '')}")
|
|
bad = [r for r in report["rows"]
|
|
if (r["label"] == "drop" and r["decision"] not in ("drop", "skip"))
|
|
or (r["label"] == "extract" and r["decision"] != "send")
|
|
or r["decision"] == "unknown"]
|
|
if bad:
|
|
print(f" misclassified ({len(bad)}):")
|
|
for r in bad:
|
|
want = "drop" if r["label"] == "drop" else "send"
|
|
detail = r.get("error") or r.get("reason") or ""
|
|
print(f" {r['id']:38s} want={want:5s} got={r['decision']:8s} {detail}")
|
|
else:
|
|
for r in report["rows"]:
|
|
want = "0" if r["label"] == "exclude" else ">=1"
|
|
ok = (r["stored"] == 0) if r["label"] == "exclude" else (r["stored"] >= 1)
|
|
print(f" [{'PASS' if ok else 'FAIL'}] {r['id']:34s} stored={r['stored']} want={want} "
|
|
f"types={r['got_types']}")
|
|
for m in r["memories"]:
|
|
print(f" -> {(m['text'] or '')[:96]}")
|
|
if scores.get("noise_leak_count"):
|
|
print(f" NOISE LEAK: {scores['noise_leak_count']} stored memories match heartbeat patterns")
|
|
print(f" labels: {counts}")
|
|
print("=" * 72)
|
|
|
|
|
|
def check(report: dict[str, Any]) -> tuple[bool, list[str]]:
|
|
"""A gate that cannot measure must not pass."""
|
|
if report.get("skipped"):
|
|
return False, [f"nothing was measured: {report['reason']}"]
|
|
failures = []
|
|
scores = report["scores"]
|
|
applicable = {k: v for k, v in THRESHOLDS.items() if k in scores}
|
|
if not applicable:
|
|
return False, [f"no gated metric present in {report['mode']} scores"]
|
|
for name, floor in applicable.items():
|
|
value = scores.get(name)
|
|
if value is None:
|
|
failures.append(f"{name} not measured (threshold {floor})")
|
|
elif value < floor:
|
|
failures.append(f"{name}={value:.3f} below threshold {floor}")
|
|
return (not failures), failures
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
p = argparse.ArgumentParser(description="mem0-agent write-gate evaluation")
|
|
mode = p.add_mutually_exclusive_group()
|
|
mode.add_argument("--offline", action="store_true", help="score the local trigger rules (default)")
|
|
mode.add_argument("--live", action="store_true", help="replay fixtures against a scratch project")
|
|
p.add_argument("--project-id", help="SCRATCH project id (required for --live)")
|
|
p.add_argument("--org-id", help="org id (required for --live)")
|
|
p.add_argument("--api-key", help="defaults to $MEM0_API_KEY")
|
|
p.add_argument("--run-id", help="override the generated run id")
|
|
p.add_argument("--timeout", type=float, default=360.0, help="live: max seconds to poll (default 360)")
|
|
p.add_argument("--min-settle", type=float, default=60.0,
|
|
help="live: minimum seconds before a zero read counts as suppression (default 60)")
|
|
p.add_argument("--level", help="capture aggressiveness passed to classify() "
|
|
"(conservative|balanced|aggressive); default is the module's own")
|
|
p.add_argument("--cleanup", action="store_true", help="live: delete everything this run wrote")
|
|
p.add_argument("--check", action="store_true", help="exit non-zero if scores regress below thresholds")
|
|
p.add_argument("--report", default=str(REPORT_PATH), help=f"report path (default {REPORT_PATH})")
|
|
args = p.parse_args(argv)
|
|
|
|
if args.live:
|
|
# A live run writes real memories. It must never be able to land in production
|
|
# by omission, so both ids are required and neither has a default.
|
|
missing = [n for n, v in (("--project-id", args.project_id), ("--org-id", args.org_id)) if not v]
|
|
if missing:
|
|
p.error("live mode requires " + " and ".join(missing)
|
|
+ " -- point them at a scratch project, never a production one")
|
|
report = run_live(args)
|
|
else:
|
|
report = run_offline(args.level)
|
|
|
|
report["thresholds"] = THRESHOLDS
|
|
report["fixture_counts"] = fx.counts()
|
|
report["fixture_counts_by_type"] = fx.counts_by_type()
|
|
report["generated_at"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
|
|
|
|
passed, failures = check(report)
|
|
report["check"] = {"passed": passed, "failures": failures}
|
|
|
|
print_scorecard(report)
|
|
Path(args.report).write_text(json.dumps(report, indent=1))
|
|
print(f"report written to {args.report}")
|
|
|
|
if args.check:
|
|
if failures:
|
|
print("CHECK FAILED:")
|
|
for f in failures:
|
|
print(f" - {f}")
|
|
return 1
|
|
print("CHECK PASSED")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|