"""The gate, tested against the classes of noise v1 actually stored. The three verbatim strings below are real records pulled from the polluted v1 corpus -- the training-heartbeat cluster (119 near-duplicates), the chunk- progress cluster, and the file-inventory class. If any of them stops being dropped, the regression is the one that mattered most. """ from __future__ import annotations import pytest from mem0_agent.triggers import ( LEVELS, TriggerResult, classify, repo_content_reason, shape_signature, ) # -------------------------------------------------------------------------- # helpers # -------------------------------------------------------------------------- def u(text: str) -> dict: return {"role": "user", "content": text} def a(text: str) -> dict: return {"role": "assistant", "content": text} # -------------------------------------------------------------------------- # real-corpus hard drops # -------------------------------------------------------------------------- TRAINING_HEARTBEAT = ( "Task notification (task-id bukn4vw5n): v4 train metrics at epoch 0.7381/2 " "(37% complete) with loss 0.4727, gradient norm 0.4716, ETA 124 minutes." ) CHUNK_PROGRESS = ( "Progress for task bnzbd1uay: 218 of 928 chunks processed (23% complete), " "approximately 5,141 synthetic memories generated, 11 chunk failures, " "ETA about 55 minutes." ) FILE_INVENTORY = ( "I modified VERSION, chat.py, agent.py, types.py, chunking.py, the slack adapter, " "the router, the tests, and several web components in this session." ) REAL_CORPUS_NOISE = [TRAINING_HEARTBEAT, CHUNK_PROGRESS, FILE_INVENTORY] @pytest.mark.parametrize("text", REAL_CORPUS_NOISE) @pytest.mark.parametrize("level", LEVELS) def test_real_corpus_noise_is_dropped_at_every_level(text, level): result = classify([a(text)], level) assert result.action == "drop", f"{level}: {result}" assert result.mtype is None @pytest.mark.parametrize( "text", [ "Still running the backfill, no changes since the last update.", "Heartbeat: the job is still running, will report back in 10 minutes.", "Continuing to process the queue; nothing to report yet.", "Step 4 of 12 of the ingest pipeline, 60% complete.", "Elapsed: 42 minutes, ETA about 3 hours for the remaining shards.", "Status update: 4,102 records processed and 3 retries so far.", ], ) def test_heartbeat_and_progress_phrasing_is_dropped(text): assert classify([a(text)], "aggressive").action == "drop" def test_tool_only_turns_are_dropped(): window = [ {"role": "assistant", "content": "", "tool_calls": [{"name": "Read"}]}, {"role": "tool", "content": '{"path": "src/app.py", "lines": 120}'}, {"role": "tool_result", "content": "ok"}, ] result = classify(window, "aggressive") assert result.action == "drop" assert result.reason == "tool_only" def test_transcript_tool_only_flag_is_honored(): """transcript.py hands us {role, content, tool_only}; a whole window of those is noise.""" window = [ {"role": "assistant", "content": "", "tool_only": True}, {"role": "user", "content": "", "tool_only": True}, ] assert classify(window, "aggressive").reason == "tool_only" def test_subagent_transcript_is_dropped_even_when_it_reads_like_a_decision(): window = [ {"role": "subagent", "content": "We decided to go with Kuzu because the graph fits in memory."}, ] assert classify(window, "aggressive").reason == "subagent_transcript" flagged = {"role": "assistant", "content": "We decided to use Kuzu because it is embedded.", "subagent": True} assert classify([flagged], "aggressive").action == "drop" def test_same_shape_repeated_inside_one_window_is_dropped(): window = [ a("Shard 1 of the export finished cleanly with no retries at all today"), a("Shard 2 of the export finished cleanly with no retries at all today"), a("Shard 3 of the export finished cleanly with no retries at all today"), ] assert classify(window, "aggressive").action == "drop" def test_repeat_of_a_recently_seen_shape_is_dropped(): window = [u("Let's go with Postgres instead of DynamoDB because the access patterns are relational.")] first = classify(window, "balanced") assert first.action == "flag" again = classify(window, "balanced", recent_shapes=[shape_signature(window)]) assert again.action == "drop" assert again.reason == "repeated_shape" def test_shape_signature_ignores_ids_counts_and_percentages(): one = [a("Training run alpha: 12 of 40 steps done, 30% complete, ETA 9 minutes.")] two = [a("Training run alpha: 31 of 40 steps done, 77% complete, ETA 4 minutes.")] other = [a("The release branch is cut and the changelog has been written.")] assert shape_signature(one) == shape_signature(two) assert shape_signature(one) != shape_signature(other) # -------------------------------------------------------------------------- # repo content -- client-side omission is the only enforcement # -------------------------------------------------------------------------- CLAUDE_MD_PASTE = """Here are the contents of our CLAUDE.md so you have the project rules: # AGENTS.md ## Repository Structure This is a polyglot monorepo containing Python and TypeScript packages. ## Coding Standards - Python source files use snake_case.py - Ruff is the single linting and formatting tool """ MARKDOWN_DUMP = """# Mem0 ## Installation pip install mem0ai ## Quickstart from mem0 import Memory ## License Apache-2.0 """ CONFIG_PASTE = """This is our pyproject.toml: ```toml [tool.ruff] line-length = 120 target-version = "py310" ``` """ @pytest.mark.parametrize("text", [CLAUDE_MD_PASTE, MARKDOWN_DUMP, CONFIG_PASTE]) @pytest.mark.parametrize("level", LEVELS) def test_repo_content_is_always_dropped(text, level): result = classify([u(text)], level) assert result.action == "drop", result assert result.reason.startswith("repo_content:") assert repo_content_reason([u(text)]) is not None def test_repo_content_detector_leaves_genuine_prose_alone(): genuine = [ u("Always name migration files with a UTC timestamp prefix - that's the rule here."), u("Remember this: I want the linter run before you tell me a task is done."), a("The root cause was that latest_only defaults to false on that endpoint."), ] for turn in genuine: assert repo_content_reason([turn]) is None def test_repo_content_beats_a_convention_sounding_paste(): """A pasted rule reads exactly like a stated rule; only the paste is dropped.""" pasted = u("Here are the contents of our CLAUDE.md:\n\n# Rules\n\nTests must be added for every fix.") stated = u("Tests must be added for every fix - that's the rule here, even for one-liners.") assert classify([pasted], "balanced").action == "drop" assert classify([stated], "balanced").action == "flag" # -------------------------------------------------------------------------- # flag rules and their types # -------------------------------------------------------------------------- @pytest.mark.parametrize( "window,level,mtype", [ ( [u("Remember this: I always want the linter run before you tell me a task is done.")], "conservative", "preference", ), ( [u("Don't forget that I review diffs top-down, so keep the summary at the end.")], "conservative", "preference", ), ( [a("I'll add a helper for that."), u("No, actually, stop doing that - inline it instead.")], "conservative", "preference", ), ( [u("I told you already: don't run the full suite on every save.")], "conservative", "preference", ), ( [ a("Postgres or DynamoDB for the event log?"), u("Let's go with Postgres because our access patterns are relational."), ], "balanced", "decision", ), ( [a("The retry loop silently swallowed the 429, so the root cause was the missing backoff.")], "balanced", "insight", ), ( [u("Always name migration files with a UTC timestamp prefix - that's the rule here.")], "balanced", "convention", ), ( [ u( "1. bump the version in pyproject\n" "2. tag the commit\n" "3. dispatch the publish workflow\n" "I verified those steps end to end on the last release." ) ], "aggressive", "runbook", ), ( [ u("Finish the pgvector migration and run the suite."), a("Schema migrated and the embedding column is backfilled."), a("All tests are passing now, so the migration is complete."), ], "aggressive", "insight", ), ], ) def test_genuine_windows_are_flagged_with_the_right_type(window, level, mtype): result = classify(window, level) assert result.action == "flag", result assert result.mtype == mtype, result # --- stated standing preferences: the highest-value thing this plugin captures --- STANDING_PREFERENCES = [ # Found by integration testing: a closed verb list after "stop" missed this entirely. "Stop dumping the whole diff at me every time. Show me the failing test output first, " "then the fix. That's how I want it from now on.", "stop showing me the full output", "I prefer rebase over merge for this repo.", "From now on, run the type checker before you hand a task back.", "I'd rather you asked before touching the lockfile.", "Please always put the summary at the end, going forward.", ] @pytest.mark.parametrize("text", STANDING_PREFERENCES) @pytest.mark.parametrize("level", LEVELS) def test_standing_preferences_are_flagged_at_every_level(text, level): result = classify([u(text)], level) assert result.action == "flag", result assert result.mtype == "preference", result @pytest.mark.parametrize("text", STANDING_PREFERENCES) def test_standing_preference_phrasing_from_the_assistant_is_not_a_preference(text): """scope=user is load-bearing: the assistant's own narration is not the user's rule.""" assert classify([a(text)], "balanced").mtype != "preference" @pytest.mark.parametrize("text", REAL_CORPUS_NOISE) @pytest.mark.parametrize("level", LEVELS) def test_noise_still_drops_after_widening_the_preference_rules(text, level): """Hard drops run before every flag rule, including from a user turn.""" assert classify([u(text)], level).action == "drop" assert classify([u(text + " Do this every time, from now on.")], level).action == "drop" def test_one_off_instructions_are_not_preferences(): """'skip tests for now' is a task instruction; only standing rules get stored.""" for text in ( "Skip the tests for now and just get the build green.", "You do it this time, I'm out of patience.", "Just run it yourself and paste what you get.", ): assert classify([u(text)], "balanced").action == "skip", text def test_remember_intent_takes_the_type_from_the_phrasing(): window = [u("Remember that we decided to use Kuzu for the graph store because it is embedded.")] result = classify(window, "conservative") assert result == TriggerResult("flag", "decision", "remember_intent") def test_assistant_chatter_is_not_a_user_preference(): """Attribution: the assistant saying 'remember this' must not create a preference.""" window = [a("Remember this: I always run the linter before finishing a task.")] assert classify(window, "conservative").action != "flag" # -------------------------------------------------------------------------- # level gating # -------------------------------------------------------------------------- DECISION_WINDOW = [ a("Should the event log go in Postgres or DynamoDB?"), u("Let's go with Postgres because our access patterns are relational."), ] RUNBOOK_WINDOW = [ u( "1. bump the version in pyproject\n" "2. tag the commit\n" "3. dispatch the publish workflow\n" "I verified those steps end to end on the last release." ) ] def test_decision_is_gated_to_balanced_and_up(): assert classify(DECISION_WINDOW, "conservative").action == "skip" assert classify(DECISION_WINDOW, "balanced") == TriggerResult("flag", "decision", "decision_language") assert classify(DECISION_WINDOW, "aggressive").mtype == "decision" def test_verified_runbook_is_captured_from_balanced_up(): """A procedure the user says they verified is durable knowledge, not a stretch goal. `aggressive` is for completed goals and procedures the assistant merely proposes.""" assert classify(RUNBOOK_WINDOW, "conservative").action == "skip" assert classify(RUNBOOK_WINDOW, "balanced").mtype == "runbook" assert classify(RUNBOOK_WINDOW, "aggressive").mtype == "runbook" def test_explicit_intent_survives_the_most_conservative_level(): window = [u("Remember this: I always want the linter run before you tell me a task is done.")] assert classify(window, "conservative").action == "flag" def test_unknown_level_falls_back_to_balanced(): assert classify(DECISION_WINDOW, "nonsense").mtype == "decision" # -------------------------------------------------------------------------- # nothing-to-store # -------------------------------------------------------------------------- @pytest.mark.parametrize( "window", [ [], [u("Now look at the retry helper in the client and tell me what it does.")], [a("Sure, I'll take a look at that file and report what I find.")], ], ) def test_ordinary_working_turns_are_skipped(window): assert classify(window, "balanced").action == "skip"