Deshraj Yadav
3ea00cf88c
- pack.py: single-call context pack, budgeted, sanitized, cached, with served-id tracking that turns a citation into POSITIVE feedback - assist.py: error-signature lookup (never raw stdout, which returned 0 results in v1) - maintain.py: transactional consolidation (add merged -> verify -> delete sources) - transcript.py: JSONL parsing into non-overlapping windows via a cursor - cli.py: the twelve commands the hooks invoke; detached assist queues a block that the next prompt hook drains, keeping the hot path network-free - hooks generated from one spec; plugin manifest, MCP wiring, five skills Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
91 lines
3.3 KiB
Python
91 lines
3.3 KiB
Python
"""The fixture set is the yardstick for the write gate, so the yardstick itself is tested.
|
|
|
|
A malformed fixture silently changes what the harness measures -- a typo'd label would
|
|
quietly move a window out of the drop class and inflate hard-drop recall forever. These
|
|
tests are cheap and they run without the network.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import importlib.util
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from mem0_agent.config.project_config import TYPES
|
|
|
|
REQUIRED_KEYS = {"id", "window", "label", "expect_type", "note"}
|
|
VALID_LABELS = {"drop", "exclude", "extract"}
|
|
MIN_FIXTURES = 40
|
|
|
|
|
|
def _load_fixtures():
|
|
"""eval/ is a script directory, not a package -- load the module by path."""
|
|
path = Path(__file__).resolve().parents[1] / "eval" / "fixtures.py"
|
|
spec = importlib.util.spec_from_file_location("eval_fixtures", path)
|
|
assert spec and spec.loader, f"cannot load {path}"
|
|
module = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(module)
|
|
return module
|
|
|
|
|
|
fx = _load_fixtures()
|
|
|
|
|
|
def test_fixture_count():
|
|
assert len(fx.FIXTURES) >= MIN_FIXTURES, f"need at least {MIN_FIXTURES} fixtures for a meaningful score"
|
|
|
|
|
|
def test_all_labels_represented():
|
|
counts = fx.counts()
|
|
assert set(counts) == VALID_LABELS
|
|
for label, n in counts.items():
|
|
assert n >= 5, f"label {label!r} has only {n} fixtures; too few to score"
|
|
|
|
|
|
def test_ids_unique():
|
|
ids = [f["id"] for f in fx.FIXTURES]
|
|
dupes = {i for i in ids if ids.count(i) > 1}
|
|
assert not dupes, f"duplicate fixture ids: {sorted(dupes)}"
|
|
|
|
|
|
@pytest.mark.parametrize("fixture", fx.FIXTURES, ids=lambda f: f["id"])
|
|
def test_fixture_schema(fixture):
|
|
assert REQUIRED_KEYS <= set(fixture), f"missing keys: {sorted(REQUIRED_KEYS - set(fixture))}"
|
|
assert isinstance(fixture["id"], str) and fixture["id"]
|
|
assert fixture["label"] in VALID_LABELS
|
|
assert isinstance(fixture["note"], str) and fixture["note"].strip(), "every fixture must say why it exists"
|
|
|
|
|
|
@pytest.mark.parametrize("fixture", fx.FIXTURES, ids=lambda f: f["id"])
|
|
def test_window_shape(fixture):
|
|
window = fixture["window"]
|
|
assert isinstance(window, list) and window, "window must be a non-empty list of messages"
|
|
for msg in window:
|
|
assert set(msg) == {"role", "content"}, f"message keys must be role/content, got {sorted(msg)}"
|
|
assert msg["role"] in {"user", "assistant"}
|
|
assert isinstance(msg["content"], str) and msg["content"].strip()
|
|
|
|
|
|
@pytest.mark.parametrize("fixture", fx.FIXTURES, ids=lambda f: f["id"])
|
|
def test_expect_type(fixture):
|
|
want = fixture["expect_type"]
|
|
if fixture["label"] == "extract":
|
|
assert want in TYPES, f"extract fixtures need an expect_type from TYPES, got {want!r}"
|
|
else:
|
|
assert want is None, f"{fixture['label']} fixtures must not claim a type, got {want!r}"
|
|
|
|
|
|
def test_extract_covers_every_durable_type():
|
|
from mem0_agent.config.project_config import DURABLE_TYPES
|
|
|
|
covered = set(fx.counts_by_type())
|
|
assert set(DURABLE_TYPES) <= covered, f"no extract fixture for {sorted(set(DURABLE_TYPES) - covered)}"
|
|
|
|
|
|
def test_helpers_agree():
|
|
assert sum(fx.counts().values()) == len(fx.FIXTURES)
|
|
assert sum(fx.counts_by_type().values()) == len(fx.by_label("extract"))
|
|
assert fx.get("e01_pref_test_output_first") is not None
|
|
assert fx.get("nope_not_a_fixture") is None
|