refactor(integrations): shared agent plugin runtimes and native adapters (#7203)

integrations/codex-plugin/core/telemetry.py

@@ -0,0 +1,397 @@
+#!/usr/bin/env python3
+"""Anonymous usage telemetry for Mem0 agent plugins.
+
+Hooks run on a 3-6 second budget and fire on every tool call, so recording never
+touches the network: `record` appends one JSON line to a local spool and returns.
+A detached `python3 telemetry.py` drains the spool in one batched PostHog request,
+started once per session and again from the flush worker that is already detached.
+
+Pure stdlib, matching the rest of the plugin. Opt out with MEM0_TELEMETRY=false.
+
+Never sends prompts, memory text, queries, file paths, repository names, or API
+keys: only event names, durations, counts, coarse outcomes, and salted hashes.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import platform
+import subprocess
+import sys
+import time
+import urllib.error
+import urllib.request
+import uuid
+from pathlib import Path
+from typing import Any
+
+import memory_core
+
+_harness: str = "generic"
+_source_tag: str = "MEM0_PLUGIN"
+_PRIVATE_KEYS = {
+    "apikey",
+    "authorization",
+    "password",
+    "query",
+    "secret",
+    "prompt",
+    "token",
+    "text",
+    "memory",
+    "message",
+    "error",
+    "path",
+    "cwd",
+    "userid",
+    "agentid",
+    "runid",
+    "repoid",
+    "repositoryid",
+    "projectid",
+    "appid",
+    "filters",
+}
+
+
+def init(harness: str = "generic", source_tag: str = "") -> None:
+    global _harness, _source_tag
+    _harness = harness
+    _source_tag = source_tag or f"MEM0_{harness.upper().replace('-', '_')}_PLUGIN"
+
+POSTHOG_API_KEY = "phc_hgJkUVJFYtmaJqrvf6CYN67TIQ8yhXAkWzUn9AMU4yX"
+POSTHOG_CAPTURE_URL = "https://us.i.posthog.com/i/v0/e/"
+POSTHOG_BATCH_URL = "https://us.i.posthog.com/batch/"
+EVENT_PREFIX = "code"
+SPOOL_LIMIT_BYTES = 256 * 1024
+BATCH_SIZE = 100
+SEND_TIMEOUT = 5
+CLAIM_STALE_SECONDS = 120
+CLAIM_EXPIRY_SECONDS = 7 * 24 * 60 * 60
+
+
+def is_enabled() -> bool:
+    """Whether telemetry is switched on for this process."""
+    return os.environ.get("MEM0_TELEMETRY", "true").strip().lower() not in {
+        "false",
+        "0",
+        "no",
+        "off",
+    }
+
+
+def _digest(value: str, length: int = 16) -> str:
+    return hashlib.sha256(value.encode("utf-8")).hexdigest()[:length]
+
+
+def _safe_value(value: Any) -> Any:
+    if isinstance(value, str):
+        return memory_core.redact(value)
+    if isinstance(value, dict):
+        return {
+            key: _safe_value(item)
+            for key, item in value.items()
+            if "".join(character for character in str(key).lower() if character.isalnum())
+            not in _PRIVATE_KEYS
+        }
+    if isinstance(value, (list, tuple)):
+        return [_safe_value(item) for item in value]
+    if value is None or isinstance(value, (bool, int, float)):
+        return value
+    return memory_core.redact(value)
+
+
+def _spool_path() -> Path:
+    return memory_core.data_dir() / "telemetry.jsonl"
+
+
+def _identity_path() -> Path:
+    return memory_core.data_dir() / "telemetry-identity.json"
+
+
+def _read_identity() -> dict[str, str]:
+    try:
+        value = json.loads(_identity_path().read_text(encoding="utf-8"))
+    except (OSError, json.JSONDecodeError):
+        return {}
+    return value if isinstance(value, dict) else {}
+
+
+def _write_identity(identity: dict[str, str]) -> None:
+    path = _identity_path()
+    temporary = path.with_suffix(f".{os.getpid()}.tmp")
+    try:
+        path.parent.mkdir(parents=True, exist_ok=True)
+        temporary.write_text(json.dumps(identity), encoding="utf-8")
+        temporary.replace(path)
+    except OSError:
+        try:
+            temporary.unlink()
+        except OSError:
+            pass
+
+
+def anonymous_id(identity: dict[str, str] | None = None) -> str:
+    """Per-machine anonymous identifier, created and persisted on first use."""
+    identity = _read_identity() if identity is None else identity
+    existing = identity.get("anonymous_id")
+    if existing:
+        return existing
+    created = f"code-anon-{uuid.uuid4().hex}"
+    identity["anonymous_id"] = created
+    _write_identity(identity)
+    return created
+
+
+def is_first_run() -> bool:
+    """Whether this machine has never recorded a plugin event before."""
+    return not _identity_path().exists()
+
+
+def record(
+    event: str,
+    *,
+    repo: Any = None,
+    session_id: str | None = None,
+    **properties: Any,
+) -> None:
+    """Append one event to the local spool. Never blocks and never raises."""
+    if not is_enabled():
+        return
+    try:
+        spool = _spool_path()
+        try:
+            if spool.stat().st_size > SPOOL_LIMIT_BYTES:
+                return
+        except OSError:
+            pass
+        properties = _safe_value(properties)
+        properties.update(
+            harness=_harness,
+            plugin_version=memory_core.PLUGIN_VERSION,
+            os=sys.platform,
+            python_version=platform.python_version(),
+        )
+        if repo is not None:
+            properties["repo_hash"] = _digest(getattr(repo, "identity", ""))
+        if session_id:
+            properties["session_hash"] = _digest(session_id)
+        line = json.dumps(
+            {
+                "event": f"{EVENT_PREFIX}.{event}",
+                "timestamp": memory_core.utc_now(),
+                "properties": {
+                    key: value for key, value in properties.items() if value is not None
+                },
+            },
+            separators=(",", ":"),
+            default=str,
+        )
+        spool.parent.mkdir(parents=True, exist_ok=True)
+        with spool.open("a", encoding="utf-8") as handle:
+            handle.write(line + "\n")
+    except Exception:
+        pass
+
+
+def error_kind(exc: BaseException | str) -> str:
+    """Coarse, content-free label for a failure, safe to send."""
+    text = exc if isinstance(exc, str) else f"{type(exc).__name__}: {exc}"
+    lowered = text.lower()
+    if "timed out" in lowered or "timeout" in lowered:
+        return "timeout"
+    if "401" in lowered or "403" in lowered or "unauthor" in lowered or "forbidden" in lowered:
+        return "auth"
+    if "429" in lowered or "rate limit" in lowered:
+        return "rate-limited"
+    if any(code in lowered for code in ("500", "502", "503", "504")):
+        return "server-error"
+    if "400" in lowered or "422" in lowered:
+        return "bad-request"
+    if isinstance(exc, str):
+        return "other"
+    if isinstance(exc, urllib.error.URLError):
+        return "network"
+    return type(exc).__name__
+
+
+def spawn_flush() -> bool:
+    """Start the detached sender that drains the spool."""
+    if not is_enabled():
+        return False
+    try:
+        if not _spool_path().exists() and not any(
+            memory_core.data_dir().glob("telemetry-*.sending")
+        ):
+            return False
+        subprocess.Popen(
+            [sys.executable, str(Path(__file__).resolve())],
+            stdin=subprocess.DEVNULL,
+            stdout=subprocess.DEVNULL,
+            stderr=subprocess.DEVNULL,
+            close_fds=True,
+            **memory_core.detached_process_kwargs(),
+        )
+        return True
+    except Exception:
+        return False
+
+
+def _claim_spool() -> Path | None:
+    """Rename the spool aside so exactly one sender owns each batch."""
+    directory = memory_core.data_dir()
+    claim = directory / f"telemetry-{os.getpid()}-{uuid.uuid4().hex[:8]}.sending"
+    spool = _spool_path()
+    try:
+        spool.replace(claim)
+        return claim
+    except OSError:
+        pass
+    now = time.time()
+    for orphan in sorted(directory.glob("telemetry-*.sending")):
+        try:
+            age = now - orphan.stat().st_mtime
+        except OSError:
+            continue
+        if age > CLAIM_EXPIRY_SECONDS:
+            try:
+                orphan.unlink()
+            except OSError:
+                pass
+            continue
+        if age < CLAIM_STALE_SECONDS:
+            continue
+        try:
+            orphan.replace(claim)
+            return claim
+        except OSError:
+            continue
+    return None
+
+
+def _resolve_email(key: str) -> str:
+    """Trade the API key for the account email so events join other Mem0 surfaces."""
+    url = os.environ.get("MEM0_API_URL", memory_core.DEFAULT_API_URL).rstrip("/") + "/v1/ping/"
+    request = urllib.request.Request(
+        url, headers={"Authorization": f"Token {key}", "Content-Type": "application/json"}
+    )
+    try:
+        with urllib.request.urlopen(request, timeout=SEND_TIMEOUT) as response:
+            payload = json.loads(response.read().decode("utf-8"))
+    except Exception:
+        return ""
+    email = payload.get("user_email") if isinstance(payload, dict) else ""
+    return email if isinstance(email, str) else ""
+
+
+def _post(payload: dict[str, Any], url: str) -> bool:
+    request = urllib.request.Request(
+        url,
+        data=json.dumps(payload, default=str).encode("utf-8"),
+        headers={"Content-Type": "application/json"},
+    )
+    try:
+        with urllib.request.urlopen(request, timeout=SEND_TIMEOUT):
+            return True
+    except Exception:
+        return False
+
+
+def resolve_distinct_id() -> tuple[str, str]:
+    """Return the PostHog distinct id and the anonymous id it replaced, if any."""
+    identity = _read_identity()
+    email = identity.get("email", "")
+    if email:
+        return email, ""
+    key = memory_core.api_key()
+    if not key:
+        return anonymous_id(identity), ""
+    email = _resolve_email(key)
+    if not email:
+        return anonymous_id(identity), ""
+    previous = identity.get("anonymous_id", "")
+    identity["email"] = email
+    _write_identity(identity)
+    return email, previous
+
+
+def flush() -> int:
+    """Drain claimed spools to PostHog and return the number of events sent."""
+    if not is_enabled():
+        return 0
+    claim = _claim_spool()
+    if claim is None:
+        return 0
+    try:
+        lines = claim.read_text(encoding="utf-8").splitlines()
+    except OSError:
+        return 0
+    events = []
+    for line in lines:
+        try:
+            value = json.loads(line)
+        except json.JSONDecodeError:
+            continue
+        if isinstance(value, dict) and value.get("event"):
+            events.append(value)
+    if not events:
+        try:
+            claim.unlink()
+        except OSError:
+            pass
+        return 0
+
+    distinct_id, aliased_anonymous_id = resolve_distinct_id()
+    if aliased_anonymous_id:
+        _post(
+            {
+                "api_key": POSTHOG_API_KEY,
+                "event": "$identify",
+                "distinct_id": distinct_id,
+                "properties": {
+                    "$anon_distinct_id": aliased_anonymous_id,
+                    "$lib": "posthog-python",
+                },
+            },
+            POSTHOG_CAPTURE_URL,
+        )
+
+    sent = 0
+    for start in range(0, len(events), BATCH_SIZE):
+        batch = [
+            {
+                "event": event["event"],
+                "distinct_id": distinct_id,
+                "timestamp": event.get("timestamp"),
+                "properties": {
+                    "source": _source_tag,
+                    "language": "python",
+                    "$process_person_profile": False,
+                    "$lib": "posthog-python",
+                    **(event.get("properties") or {}),
+                },
+            }
+            for event in events[start : start + BATCH_SIZE]
+        ]
+        if not _post({"api_key": POSTHOG_API_KEY, "batch": batch}, POSTHOG_BATCH_URL):
+            return sent
+        sent += len(batch)
+    try:
+        claim.unlink()
+    except OSError:
+        pass
+    return sent
+
+
+def main() -> int:
+    flush()
+    return 0
+
+
+if __name__ == "__main__":
+    try:
+        raise SystemExit(main())
+    except Exception:
+        raise SystemExit(0)