diff --git a/docs/components/vectordbs/dbs/databricks.mdx b/docs/components/vectordbs/dbs/databricks.mdx index db0056e86..890b1edcb 100644 --- a/docs/components/vectordbs/dbs/databricks.mdx +++ b/docs/components/vectordbs/dbs/databricks.mdx @@ -6,7 +6,8 @@ description: "Use Databricks Vector Search as a serverless vector store in Mem0 ### Usage -```python + +```python Python import os from mem0 import Memory @@ -36,10 +37,44 @@ messages = [ m.add(messages, user_id="alice", metadata={"category": "movies"}) ``` +```typescript TypeScript +// Requires the Databricks SQL driver (peer dependency): pnpm add @databricks/sql +import { Memory } from 'mem0ai/oss'; + +const config = { + vectorStore: { + provider: 'databricks', + config: { + workspaceUrl: 'https://your-workspace.databricks.com', + // SQL warehouse HTTP path, used for index writes (required) + httpPath: '/sql/1.0/warehouses/your-warehouse-id', + accessToken: 'your-access-token', + catalog: 'your_catalog', + schema: 'your_schema', + tableName: 'your_table', + collectionName: 'your_index_name', + embeddingModelDims: 1536, + }, + }, +}; + +const memory = new Memory(config); +const messages = [ + {"role": "user", "content": "I'm planning to watch a movie tonight. Any recommendations?"}, + {"role": "assistant", "content": "How about thriller movies? They can be quite engaging."}, + {"role": "user", "content": "I'm not a big fan of thriller movies but I love sci-fi movies."}, + {"role": "assistant", "content": "Got it! I'll avoid thriller recommendations and suggest sci-fi movies in the future."} +] +await memory.add(messages, { userId: "alice", metadata: { category: "movies" } }); +``` + + ### Config Here are the parameters available for configuring Databricks Vector Search: + + | Parameter | Description | Default Value | | --- | --- | --- | | `workspace_url` | The URL of your Databricks workspace | **Required** | @@ -60,6 +95,32 @@ Here are the parameters available for configuring Databricks Vector Search: | `pipeline_type` | Sync pipeline type: `TRIGGERED` or `CONTINUOUS` | `TRIGGERED` | | `warehouse_name` | Databricks SQL warehouse name (if using SQL warehouse) | `None` | | `query_type` | Query type: `ANN` or `HYBRID` | `ANN` | + + +| Parameter | Description | Default Value | +| --- | --- | --- | +| `workspaceUrl` | The URL of your Databricks workspace (or pass `host`) | **Required** | +| `httpPath` | SQL warehouse HTTP path, used for index writes | **Required** | +| `accessToken` | Personal Access Token for authentication | `None` | +| `clientId` | Service principal client ID (alternative to `accessToken`) | `None` | +| `clientSecret` | Service principal client secret (required with `clientId`) | `None` | +| `endpointName` | Name of the Vector Search endpoint | `mem0_vector_search` | +| `endpointType` | Type of endpoint (`STANDARD` or `STORAGE_OPTIMIZED`) | `STANDARD` | +| `pipelineType` | Delta Sync pipeline type: `TRIGGERED` or `CONTINUOUS` | `TRIGGERED` | +| `queryType` | Query type: `ANN` or `HYBRID` | `ANN` | +| `catalog` | Unity Catalog catalog name | `main` | +| `schema` | Unity Catalog schema name | `default` | +| `collectionName` | Vector Search index name | `mem0` | +| `tableName` | Source Delta table name | falls back to `collectionName` | +| `embeddingModelDims` | Dimension of self-managed embeddings | `1536` | +| `syncPollIntervalMs` | Poll interval while waiting for a `TRIGGERED` sync | `1000` | +| `syncTimeoutMs` | Timeout while waiting for an index sync | `300000` | + + +The TypeScript provider uses `DELTA_SYNC` indexes with self-managed embeddings: pass vectors directly. `DIRECT_ACCESS` indexes, Databricks-computed embeddings (`embedding_model_endpoint_name`), and Azure AD auth are Python-only today. It writes to the index through a SQL warehouse, so `httpPath` is required, and `@databricks/sql` must be installed as a peer dependency. + + + ### Authentication diff --git a/mem0-ts/package.json b/mem0-ts/package.json index 497a1e1f9..a8d02c6d1 100644 --- a/mem0-ts/package.json +++ b/mem0-ts/package.json @@ -109,12 +109,13 @@ }, "peerDependencies": { "@anthropic-ai/sdk": "^0.40.1", - "@aws-sdk/client-neptune-graph": "3.966.0", + "@aws-sdk/client-neptune-graph": ">=3.0.0 <3.968.0", "@aws-sdk/client-s3vectors": "3.967.0", "@mochow/mochow-sdk-node": "^2.1.5", "@azure/identity": "^4.0.0", "@azure/search-documents": "^12.0.0", "@cloudflare/workers-types": "^4.20250504.0", + "@databricks/sql": "^1.16.0", "@google-cloud/aiplatform": "^6.8.0", "@google/genai": "^1.40.0", "@huggingface/transformers": "^3.0.0 || ^4.0.0", @@ -161,6 +162,12 @@ }, "@aws-sdk/client-bedrock-runtime": { "optional": true + }, + "@databricks/sql": { + "optional": true + }, + "@aws-sdk/client-neptune-graph": { + "optional": true } }, "engines": { @@ -198,7 +205,8 @@ "@modelcontextprotocol/sdk": "^1.25.4", "esbuild": ">=0.28.1", "undici@<6.27.0": ">=6.27.0 <8.0.0", - "@aws-sdk/client-bedrock-runtime": "3.967.0" + "@aws-sdk/client-bedrock-runtime": "3.967.0", + "@aws-sdk/client-neptune-graph": "3.966.0" } } } diff --git a/mem0-ts/pnpm-lock.yaml b/mem0-ts/pnpm-lock.yaml index 54b3d643a..5b9791705 100644 --- a/mem0-ts/pnpm-lock.yaml +++ b/mem0-ts/pnpm-lock.yaml @@ -28,6 +28,7 @@ overrides: esbuild: '>=0.28.1' undici@<6.27.0: '>=6.27.0 <8.0.0' '@aws-sdk/client-bedrock-runtime': 3.967.0 + '@aws-sdk/client-neptune-graph': 3.966.0 importers: @@ -54,6 +55,9 @@ importers: '@cloudflare/workers-types': specifier: ^4.20250504.0 version: 4.20260611.1 + '@databricks/sql': + specifier: ^1.16.0 + version: 1.16.0 '@elastic/elasticsearch': specifier: ^9.0.0 version: 9.4.2 @@ -128,13 +132,13 @@ importers: version: 0.3.0 mongodb: specifier: ^7.0.0 - version: 7.2.0 + version: 7.2.0(socks@2.8.9) mysql2: specifier: ^3.0.0 version: 3.22.5(@types/node@22.19.21) natural: specifier: ^8.0.1 - version: 8.1.1(@opentelemetry/api@1.9.1) + version: 8.1.1(@opentelemetry/api@1.9.1)(socks@2.8.9) ollama: specifier: ^0.5.14 version: 0.5.18 @@ -208,6 +212,15 @@ importers: packages: + '@75lb/deep-merge@1.1.4': + resolution: {integrity: sha512-Fjmi8VSxoGF80wn8HSHTPfUKLSJ+aiQE7rXyrFV6apP8Xyj8v4bqdC7BBoklxjt671gUqP0yREwpDlUP5R/VyQ==} + engines: {node: '>=12.17'} + peerDependencies: + '@75lb/nature': latest + peerDependenciesMeta: + '@75lb/nature': + optional: true + '@anthropic-ai/sdk@0.40.1': resolution: {integrity: sha512-DJMWm8lTEM9Lk/MSFL+V+ugF7jKOn0M2Ujvb5fN8r2nY14aHbGPZ1k6sgjL+tpJ3VuOGJNG+4R83jEpOuYPv8w==} @@ -756,6 +769,58 @@ packages: '@dabh/diagnostics@2.0.8': resolution: {integrity: sha512-R4MSXTVnuMzGD7bzHdW2ZhhdPC/igELENcq5IjEverBvq5hn1SXCWcsi6eSsdWP0/Ur+SItRRjAktmdoX/8R/Q==} + '@databricks/databricks-sql-kernel-darwin-arm64@0.2.0': + resolution: {integrity: sha512-dSZJD1uileOqRDfs5KsW6m43PAl3GqzQMcETbRT1Zjw2FRLQDtLKTA3+VWmlUUHhIz583WeTps91Sjee79cXbA==} + engines: {node: '>=18.0.0'} + cpu: [arm64] + os: [darwin] + + '@databricks/databricks-sql-kernel-darwin-x64@0.2.0': + resolution: {integrity: sha512-JEe7TqLrXrAoN3SWsi7NgjAo6V+7jEvBwLswwhl0qUp1MoPrZT+y5Kf49vuHPlh61vFgi2F6NlCNfdB/8wpihg==} + engines: {node: '>=18.0.0'} + cpu: [x64] + os: [darwin] + + '@databricks/databricks-sql-kernel-linux-arm64-gnu@0.2.0': + resolution: {integrity: sha512-meJ2JF5w4qEiaiI9TNwg2iMgasadqeKSDAw3r3hWTEdzKwUY7saMTFNcjumwR8R3cQMCFJfp3YBOwKS5/hvN9A==} + engines: {node: '>=18.0.0'} + cpu: [arm64] + os: [linux] + + '@databricks/databricks-sql-kernel-linux-arm64-musl@0.2.0': + resolution: {integrity: sha512-YTCPs11w6purXCQwJBCh99oHBwTEW4q46OWxO9Rnddo1wJd8EPBa3Misw08URoUVuVuEfYGy5YtSvuPfkkj8Vw==} + engines: {node: '>=18.0.0'} + cpu: [arm64] + os: [linux] + + '@databricks/databricks-sql-kernel-linux-x64-gnu@0.2.0': + resolution: {integrity: sha512-wIEJX2mtoCc/KGOzzbxXwOR5aUB5dBSUG1Ypp+JLI28XsZB2eCLcP4jyAQ+Uklxn+SU1hICuok+30t/7wSvKUg==} + engines: {node: '>=18.0.0'} + cpu: [x64] + os: [linux] + + '@databricks/databricks-sql-kernel-linux-x64-musl@0.2.0': + resolution: {integrity: sha512-0wtWdOxYh7BfeklDpI0tpTk/ljdjJmCQsNFPLy2C7EnK60J3sroYzTaFqgn8Qqc7T03VHZl/6GG1pXC3zPuU1g==} + engines: {node: '>=18.0.0'} + cpu: [x64] + os: [linux] + + '@databricks/databricks-sql-kernel-win32-arm64-msvc@0.2.0': + resolution: {integrity: sha512-gS4y1hIIDitr1d9bW6yWEkn4wMW7abinDwFFVUJtYsWkFS+rMq/8CwAYioqNprXQP+XEQS7fLiLYq/BYsQB3Aw==} + engines: {node: '>=18.0.0'} + cpu: [arm64] + os: [win32] + + '@databricks/databricks-sql-kernel-win32-x64-msvc@0.2.0': + resolution: {integrity: sha512-zl6KtfB02eVsBNZEQ/kf/dhmesUqbVGwhSFb5oKlmtyj/1kah9R1FRKBVKTLt3v5n9LnjBnomV2s+DRxRSh4rA==} + engines: {node: '>=18.0.0'} + cpu: [x64] + os: [win32] + + '@databricks/sql@1.16.0': + resolution: {integrity: sha512-kaSBjYUT0rDYsrouvPakwRjs030KKvImOTHRG95EsYOLkNN+/YvJ8JgQ0vQjvgT3NUREoGys+SQ4ZzqwHuwoCA==} + engines: {node: '>=14.0.0'} + '@datastructures-js/deque@1.0.8': resolution: {integrity: sha512-PSBhJ2/SmeRPRHuBv7i/fHWIdSC3JTyq56qb+Rq0wjOagi0/fdV5/B/3Md5zFZus/W6OkSPMaxMKKMNMrSmubg==} @@ -1763,6 +1828,9 @@ packages: resolution: {integrity: sha512-V/1hRKLSCJ0zEL+9QFRBUtivvePfOsaAYQmC0HhFNSHC2F3xFs4jSF3YhkLmzex6E4V4FGvmBDOP72D/53NnZA==} engines: {node: '>=20.0.0'} + '@tootallnate/quickjs-emscripten@0.23.0': + resolution: {integrity: sha512-C5Mc6rdnsaJDjO3UpGW/CQTHtCKaYlScZTly4JIu97Jxo/odCiH0ITnDXSJPTOrEKk/ycSZ0AOgTmkDtkOsvIA==} + '@tsconfig/node10@1.0.12': resolution: {integrity: sha512-UCYBaeFvM11aU2y3YPZ//O5Rhj+xKyzy7mvcIoAjASbigy8mHMryP5cK7dgjlz2hWxh1g5pLw084E0a/wlUSFQ==} @@ -1793,6 +1861,12 @@ packages: '@types/better-sqlite3@7.6.13': resolution: {integrity: sha512-NMv9ASNARoKksWtsq/SHakpYAYnhBrQgGD8zkLYk/jaK8jUGn08CfEdTRgYhMypUQAfzSP8W6gNLe0q19/t4VA==} + '@types/command-line-args@5.2.0': + resolution: {integrity: sha512-UuKzKpJJ/Ief6ufIaIzr3A/0XnluX7RvFgwkV89Yzvm77wCh1kFaFmqN8XEnGcN62EuHdedQjEMb8mYxFLGPyA==} + + '@types/command-line-usage@5.0.2': + resolution: {integrity: sha512-n7RlEEJ+4x4TS7ZQddTmNSxP+zziEG0TNsMfiRIxcIVXt71ENJ9ojeXmGO3wPoTdn7pJcU2xc3CJYMktNT6DPg==} + '@types/estree@1.0.9': resolution: {integrity: sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg==} @@ -1820,6 +1894,9 @@ packages: '@types/node@18.19.130': resolution: {integrity: sha512-GRaXQx6jGfL8sKfaIDD6OupbIHBr9jv7Jnaml9tB7l4v068PAOXqfcujMMo5PhbIs6ggR1XODELqahT2R8v0fg==} + '@types/node@20.3.0': + resolution: {integrity: sha512-cumHmIAf6On83X7yP+LrsEyUOf/YlociZelmpRYaGFydoaPdxdt80MAbu6vWerQT2COCp2nPvHdsbD7tHn/YlQ==} + '@types/node@22.19.21': resolution: {integrity: sha512-VMeFBSCKQKmm2swI2kW51SFusDqekC6q9trBCvJ/JliDchFSuoYYKN7yVNjPthP1HKZcx3U1gI/wTcEBjEFKTA==} @@ -1829,6 +1906,9 @@ packages: '@types/normalize-package-data@2.4.4': resolution: {integrity: sha512-37i+OaWTh9qeK4LSHPsyRC7NahnGotNuZvjLSgcPzblpHB3rrCJxAOgI5gCdKm7coonsaX1Of0ILiTcnZjbfxA==} + '@types/pad-left@2.1.1': + resolution: {integrity: sha512-Xd22WCRBydkGSApl5Bw0PhAOHKSVjNL3E3AwzKaps96IMraPqy5BvZIsBVK6JLwdybUzjHnuWVwpDd0JjTfHXA==} + '@types/pg@8.11.0': resolution: {integrity: sha512-sDAlRiBNthGjNFfvt0k6mtotoVYVQ63pA8R4EMWka7crawSR60waVYR0HAgmPRs/e2YaeJTD/43OoZ3PFw80pw==} @@ -1952,6 +2032,10 @@ packages: anynum@1.0.1: resolution: {integrity: sha512-N6//FLET/tXYNM/F6ABca1oH6fWB+KlTt909Le28WMDBk8oaT4vY17DCrwg2MvmuqUKt3Ni4N5dGJ/EoBgcO6A==} + apache-arrow@13.0.0: + resolution: {integrity: sha512-3gvCX0GDawWz6KFNC28p65U+zGh/LZ6ZNKWNu74N6CQlKzxeoWHpi4CgEQsgRSEMuyrIIXi1Ea2syja7dwcHvw==} + hasBin: true + apparatus@0.0.10: resolution: {integrity: sha512-KLy/ugo33KZA7nugtQ7O0E1c8kQ52N3IvD/XgIh4w/Nr28ypfkwDfA67F1ev4N1m5D+BOk1+b2dEJDfpj/VvZg==} engines: {node: '>=0.2.6'} @@ -1962,6 +2046,18 @@ packages: argparse@2.0.1: resolution: {integrity: sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==} + array-back@3.1.0: + resolution: {integrity: sha512-TkuxA4UCOvxuDK6NZYXCalszEzj+TLszyASooky+i742l9TqsOdYCMJJupxRic61hwquNtppB3hgcuq9SVSH1Q==} + engines: {node: '>=6'} + + array-back@6.2.3: + resolution: {integrity: sha512-SGDvmg6QTYiTxCBkYVmThcoa67uLl35pyzRHdpCGBOcqFy6BtwnphoFPk7LhJshD+Yk1Kt35WGWeZPTgwR4Fhw==} + engines: {node: '>=12.17'} + + ast-types@0.13.4: + resolution: {integrity: sha512-x1FCFnFifvYDDzTaLII71vG5uvDwgtmDTEVWAxrgeiR8VjMONcCXJx7E+USjDtHlwFmt9MysbqgF9b9Vjr6w+w==} + engines: {node: '>=4'} + async-limiter@1.0.1: resolution: {integrity: sha512-csOlWGAcRFJaI6m+F2WKdnMKr4HhdhFVBk0H/QbJFMCr+uO2kwohwXQPxw/9OCxp05r5ghVBFSyioixx3gfkNQ==} @@ -2024,6 +2120,10 @@ packages: engines: {node: '>=6.0.0'} hasBin: true + basic-ftp@5.3.1: + resolution: {integrity: sha512-bopVNp6ugyA150DDuZfPFdt1KZ5a94ZDiwX4hMgZDzF+GttD80lEy8kj98kbyhLXnPvhtIo93mdnLIjpCAeeOw==} + engines: {node: '>=10.0.0'} + better-sqlite3@12.10.0: resolution: {integrity: sha512-CyzaZRQKyHkB2ZInfTTl2nvT33EbDpjkLEbE8/Zck3Ll6O0qqvuGdrJ45HgtH+HykRg88ITY3AdreBGN70aBSQ==} engines: {node: 20.x || 22.x || 23.x || 24.x || 25.x || 26.x} @@ -2142,6 +2242,10 @@ packages: resolution: {integrity: sha512-HritfMGq9V7SuESeSodHvArs0mLuMk7uh+7hQK2lqdvXrvm50aWxb4RPxkK3mPDdsgHjJ427xNRFITMH2ei+Sw==} engines: {node: '>=18'} + chalk-template@0.4.0: + resolution: {integrity: sha512-/ghrgmhfY8RaSdeo43hNXxpoHAtxdbskUHjPpfqUWGttFgycUhYPGx3YZBCnUCvOa7Doivn1IZec3DEGFoMgLg==} + engines: {node: '>=12'} + chalk@4.1.2: resolution: {integrity: sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==} engines: {node: '>=10'} @@ -2277,10 +2381,22 @@ packages: resolution: {integrity: sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==} engines: {node: '>= 0.8'} + command-line-args@5.2.1: + resolution: {integrity: sha512-H4UfQhZyakIjC74I9d34fGYDwk3XpSr17QhEd0Q3I9Xq1CETHo4Hcuo87WyWHpAF1aSLjLRf5lD9ZGX2qStUvg==} + engines: {node: '>=4.0.0'} + + command-line-usage@7.0.1: + resolution: {integrity: sha512-NCyznE//MuTjwi3y84QVUGEOT+P5oto1e1Pk/jFPVdPPfsG03qpTIl3yw6etR+v73d0lXsoojRpvbru2sqePxQ==} + engines: {node: '>=12.20.0'} + commander@4.1.1: resolution: {integrity: sha512-NOKm8xhkzAjzFx8B2v5OAHT+u5pRQc2UCa2Vq9jYL/31o2wi9mxBA7LIFs3sV5VSC49z6pEhfbMULvShKj26WA==} engines: {node: '>= 6'} + commander@9.5.0: + resolution: {integrity: sha512-KRs7WVDKg86PWiuAqhDrAQnTXZKraVcCc6vFdL14qrZ/DcWwuRo7VoiYXalXO7S5GKpqYiVEwCbgFDfxNHKJBQ==} + engines: {node: ^12.20.0 || >=14} + compromise@14.15.1: resolution: {integrity: sha512-9F3UkUaEU1PPz2fgStkE/TI4tk++0wHxS8xfWq9PQWL/v28dy8bEcPVVSLh3dISIRD7PEhJ8YTzHRKF8y9tnLA==} engines: {node: '>=12.0.0'} @@ -2320,10 +2436,17 @@ packages: crypt@0.0.2: resolution: {integrity: sha512-mCxBlsHFYh9C+HVpiEacem8FEBnMXgU9gy4zmNC+SXAZNB/1idgp/aulFJ4FgCi7GPEVbfyng092GqL2k2rmow==} + cuint@0.2.2: + resolution: {integrity: sha512-d4ZVpCW31eWwCMe1YT3ur7mUDnTXbgwyzaL320DrcRT45rfjYxkt5QWLrmOJ+/UEAI2+fQgKe/fCjR8l4TpRgw==} + data-uri-to-buffer@4.0.1: resolution: {integrity: sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==} engines: {node: '>= 12'} + data-uri-to-buffer@6.0.2: + resolution: {integrity: sha512-7hvf7/GW8e86rW0ptuwS3OcBGDjIi6SZva7hCyWC0yYry2cOPmLIjXAUHI6DK2HsnwJd9ifmt57i8eV2n4YNpw==} + engines: {node: '>= 14'} + dayjs@1.11.21: resolution: {integrity: sha512-98IT+HOahAisibz/yjKbzuOBwYcjJ7BCLPzARyHiyEBmRz4fatF+KPJszEHXsGYjUG234aH/cOjW1wwTbKUZlA==} @@ -2368,6 +2491,10 @@ packages: resolution: {integrity: sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A==} engines: {node: '>= 0.4'} + define-lazy-prop@2.0.0: + resolution: {integrity: sha512-Ds09qNh8yw3khSjiJjiUInaGX9xlqZDY7JVryGxdxV7NPeuqQfplOpQ66yJFZut3jLa5zOwkXw1g9EI2uKh4Og==} + engines: {node: '>=8'} + define-lazy-prop@3.0.0: resolution: {integrity: sha512-N+MeXYoqr3pOgn8xfyRPREN7gHakLYjhsHhWGT3fWAiL4IkAt0iDw14QiiEm2bE30c5XX5q0FtAA3CK5f9/BUg==} engines: {node: '>=12'} @@ -2376,6 +2503,10 @@ packages: resolution: {integrity: sha512-8QmQKqEASLd5nx0U1B1okLElbUuuttJ/AnYmRXbbbGDWh6uS208EjD4Xqq/I9wK7u0v6O08XhTWnt5XtEbR6Dg==} engines: {node: '>= 0.4'} + degenerator@5.0.1: + resolution: {integrity: sha512-TllpMR/t0M5sqCXfj85i4XaAzxmS5tVA16dqvdkMwGmzI+dXLXnw3J+3Vdv7VKw+ThlTMboK6i9rnZ6Nntj5CQ==} + engines: {node: '>= 14'} + delayed-stream@1.0.0: resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==} engines: {node: '>=0.4.0'} @@ -2489,6 +2620,24 @@ packages: resolution: {integrity: sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==} engines: {node: '>=10'} + escodegen@2.1.0: + resolution: {integrity: sha512-2NlIDTwUWJN0mRPQOdtQBzbUHvdGY2P1VXSyU83Q3xKxM7WHX2Ql8dKq782Q9TgQUNOLEzEYu9bzLNj1q88I5w==} + engines: {node: '>=6.0'} + hasBin: true + + esprima@4.0.1: + resolution: {integrity: sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==} + engines: {node: '>=4'} + hasBin: true + + estraverse@5.3.0: + resolution: {integrity: sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==} + engines: {node: '>=4.0'} + + esutils@2.0.3: + resolution: {integrity: sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==} + engines: {node: '>=0.10.0'} + event-target-shim@5.0.1: resolution: {integrity: sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==} engines: {node: '>=6'} @@ -2568,6 +2717,10 @@ packages: resolution: {integrity: sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==} engines: {node: '>=8'} + find-replace@3.0.0: + resolution: {integrity: sha512-6Tb2myMioCAgv5kfvP5/PkZZ/ntTpVK39fHY7WkWBgvbeE+VHd/tZuZ4mrC+bxh4cfOZeYKVPaJIZtZXV7GNCQ==} + engines: {node: '>=4.0.0'} + find-up@4.1.0: resolution: {integrity: sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==} engines: {node: '>=8'} @@ -2579,6 +2732,9 @@ packages: resolution: {integrity: sha512-5z2nZxrnKxk+jLq5TyD0xbPXI2I18FF+knIZVG55e0CXWgXF/F4SpCBsiW7JTBPwghqXsC66T2yctnVT/sMO0g==} hasBin: true + flatbuffers@23.5.26: + resolution: {integrity: sha512-vE+SI9vrJDwi1oETtTIFldC/o9GsVKRM+s6EL0nQgxXlYV1Vc4Tk30hj4xGICftInKQKj1F3up2n8UbIVobISQ==} + flatbuffers@25.9.23: resolution: {integrity: sha512-MI1qs7Lo4Syw0EOzUl0xjs2lsoeqFku44KpngfIduHBYvzm8h2+7K8YMQh1JtVVVrUvhLpNwqVi4DERegUJhPQ==} @@ -2677,6 +2833,10 @@ packages: resolution: {integrity: sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==} engines: {node: '>=10'} + get-uri@6.0.5: + resolution: {integrity: sha512-b1O07XYq8eRuVzBNgJLstU6FYc1tS6wnMtF1I1D9lE8LxZSOGZ7LhxN54yPP6mGw5f2CkXY2BQUL9Fx41qvcIg==} + engines: {node: '>= 14'} + github-from-package@0.0.0: resolution: {integrity: sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==} @@ -2845,6 +3005,10 @@ packages: resolution: {integrity: sha512-4rTJX6Q5wTYEvxboXi8DsEiUo+OvqJGtLYOSGm37KpdRXsG5XJjbVtYKGJpPSWP+QT7rWscA4vsrdmzbEbenpw==} engines: {node: '>=18.12.0'} + ip-address@10.2.0: + resolution: {integrity: sha512-/+S6j4E9AHvW9SWMSEY9Xfy66O5PWvVEJ08O0y5JGyEKQpojb0K0GKpz/v5HJ/G0vi3D2sjGK78119oXZeE0qA==} + engines: {node: '>= 12'} + is-arrayish@0.2.1: resolution: {integrity: sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==} @@ -2859,6 +3023,11 @@ packages: resolution: {integrity: sha512-evOr8xfXKxE6qSR0hSXL2r3sd7ALj8+7jQEUvPYcm5sgZFdJ+AYzT6yNmJenvIYQBgIGwfwz08sL8zoL7yq2BA==} engines: {node: '>= 0.4'} + is-docker@2.2.1: + resolution: {integrity: sha512-F+i2BKsFrH66iaUFc0woD8sLy8getkwTwtOBjvs56Cx4CgJDeKQeqfz8wAYiSb8JOprWhHH5p77PbmYCvvUuXQ==} + engines: {node: '>=8'} + hasBin: true + is-docker@3.0.0: resolution: {integrity: sha512-eljcgEDlEns/7AXFosB5K/2nCM4P7FQPkGc/DWLy5rmFEWvZayGrik1d9/QIY5nJ4f9YsVvBkA6kJpHn9rISdQ==} engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} @@ -2899,6 +3068,10 @@ packages: is-unsafe@1.0.1: resolution: {integrity: sha512-CLK2+VdgERgD96EYm5lUQssZYlRg2tkZnbsxZoacmSiRxiFJ4Nk4SzjCl+Ur+v3kXIY9dTIdb3IH22y1mZ56LA==} + is-wsl@2.2.0: + resolution: {integrity: sha512-fKzAra0rGJUUBwGBgNkHZuToZcn+TtXHpeCgmkMJMMYx1sQDYaCSyjJBSCa2nH1DGm7s3n1oBnohoVTBaN7Lww==} + engines: {node: '>=8'} + is-wsl@3.1.1: resolution: {integrity: sha512-e6rvdUCiQCAuumZslxRJWR/Doq4VpPR82kqclvcS0efgt430SlGIk05vdCN58+VrzgtIcfNODjozVielycD4Sw==} engines: {node: '>=16'} @@ -3067,6 +3240,9 @@ packages: node-notifier: optional: true + jose@4.15.9: + resolution: {integrity: sha512-1vUQX+IdDMVPj4k8kOxgUqlcK518yluMuGZwqlr44FS1ppZB/5GWh4rZG89erpOBOJjU/OBsnCVFfapsRz6nEA==} + joycon@3.1.1: resolution: {integrity: sha512-34wB/Y7MW7bzjKRjUKTa46I2Z7eV62Rkhva+KkopW7Qvv/OSWBqvkSY7vusOPrNuZcUG3tApvdVgNB8POj3SPw==} engines: {node: '>=10'} @@ -3089,6 +3265,10 @@ packages: json-bigint@1.0.0: resolution: {integrity: sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ==} + json-bignum@0.0.3: + resolution: {integrity: sha512-2WHyXj3OfHSgNyuzDbSxI1w2jgw5gkWSWhS7Qg4bWXx1nLk3jnbwfUeS0PSba3IzpTUWdHxBieELUzXRjQB2zg==} + engines: {node: '>=0.8'} + json-parse-even-better-errors@2.3.1: resolution: {integrity: sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==} @@ -3214,6 +3394,9 @@ packages: lodash.once@4.1.1: resolution: {integrity: sha512-Sb487aTOCr9drQVL8pIxOzVhafOjZN9UU54hiN8PU3uAiSV7lx1yYNpbNmex2PK6dSJoNTSJUUswT651yww3Mg==} + lodash@4.18.1: + resolution: {integrity: sha512-dMInicTPVE8d1e5otfwmmjlxkZoUpiVLwyeTdUsi/Caj/gfzzblBcCE5sRHV/AsjuCmxWrte2TNGSYuCeCq+0Q==} + logform@2.7.0: resolution: {integrity: sha512-TFYA4jnP7PVbmlBIfhlSe+WKxs9dklXMTEGcBCIvLhE/Tn3H6Gk1norupVW7m5Cnd4bLcr08AytbyV/xj7f/kQ==} engines: {node: '>= 12.0.0'} @@ -3230,6 +3413,14 @@ packages: lru-cache@5.1.1: resolution: {integrity: sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==} + lru-cache@6.0.0: + resolution: {integrity: sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==} + engines: {node: '>=10'} + + lru-cache@7.18.3: + resolution: {integrity: sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA==} + engines: {node: '>=12'} + lru-cache@9.1.2: resolution: {integrity: sha512-ERJq3FOzJTxBbFjZ7iDs+NiK4VI9Wz+RdrrAB8dio1oV+YvdPzUEE4QNiT2VD51DkIbCYRUUzCRkssXCHqSnKQ==} engines: {node: 14 || >=16.14} @@ -3238,6 +3429,10 @@ packages: resolution: {integrity: sha512-DqC6n3QQ77zdFpCMASA1a3Jlb64Hv2N2DciFGkO/4L9+q/IpIAuRlKOvCXabtRW6cQf8usbmM6BE/TOPysCdIA==} engines: {bun: '>=1.0.0', deno: '>=1.30.0', node: '>=8.0.0'} + lz4@0.6.5: + resolution: {integrity: sha512-KSZcJU49QZOlJSItaeIU3p8WoAvkTmD9fJqeahQXNu1iQ/kR0/mQLdbrK8JY9MY8f6AhJoMrihp1nu1xDbscSQ==} + engines: {node: '>= 0.10'} + magic-string@0.30.21: resolution: {integrity: sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ==} @@ -3390,6 +3585,9 @@ packages: resolution: {integrity: sha512-Tz09sEL2EEuv5fFowm419c1+a/jSMiBjI9gHxVLrVdbUkkNUUfjsVYs9pVZu5oCon/kmRh9TfLEObFtkVxmY0w==} engines: {node: '>=8.0.0'} + nan@2.28.0: + resolution: {integrity: sha512-fTsDz99OTq2sVePhGdp4qQhggZFtKr64ZNVyVajRKtMOkJxYekplBh577PiJB12v/D3s2E5cGtOI45LWp6rnLQ==} + napi-build-utils@2.0.0: resolution: {integrity: sha512-GEbrYkbfF7MoNaoh2iGG84Mnf/WZfB0GdGEsM8wz7Expx/LlWf5U8t9nvJKXSp3qr5IsEbK04cBGhol/KwOsWA==} @@ -3403,6 +3601,10 @@ packages: neo-async@2.6.2: resolution: {integrity: sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==} + netmask@2.1.1: + resolution: {integrity: sha512-eonl3sLUha+S1GzTPxychyhnUzKyeQkZ7jLjKrBagJgPla13F+uQ71HgpFefyHgqrjEbCPkDArxYsjY8/+gLKA==} + engines: {node: '>= 0.4.0'} + nice-grpc-client-middleware-retry@3.1.15: resolution: {integrity: sha512-fXfNNdtjCQzc3O/w3WsK1AOU+xdE1V7FCm4FEQWS/UUVsV1616S2oryvWqsZAF8aOvuMir8lFtNmgH3DeP+PBg==} @@ -3462,6 +3664,10 @@ packages: resolution: {integrity: sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==} engines: {node: '>=0.10.0'} + object-hash@2.2.0: + resolution: {integrity: sha512-gScRMn0bS5fH+IuwyIFgnh9zBdo4DV+6GhygmWM9HyNJSgS0hScp1f5vjtm7oIIOiT9trXrShAkLFSc2IqKNgw==} + engines: {node: '>= 6'} + object-hash@3.0.0: resolution: {integrity: sha512-RSn9F68PjH9HqtltsSnqYC1XXoWe9Bju5+213R98cNGttag9q9yAOTzdbsqvIa7aNm5WffBZFpWYr2aWrklWAw==} engines: {node: '>= 6'} @@ -3473,6 +3679,10 @@ packages: obuf@1.1.2: resolution: {integrity: sha512-PX1wu0AmAdPqOL1mWhqmlOd8kOIZQwGZw6rh7uby9fTc5lhaOWFLX3I6R1hrF9k3zUY40e6igsLGkDXK92LJNg==} + oidc-token-hash@5.2.0: + resolution: {integrity: sha512-6gj2m8cJZ+iSW8bm0FXdGF0YhIQbKrfP4yWTNzxc31U6MOjfEmB1rHvlYvxI1B7t7BCi1F2vYTT6YhtQRG4hxw==} + engines: {node: ^10.13.0 || >=12.0.0} + ollama@0.5.18: resolution: {integrity: sha512-lTFqTf9bo7Cd3hpF6CviBe/DEhewjoZYd9N/uCe7O20qYTvGqrNOFOBDj3lbZgFWHUgDv5EeyusYxsZSLS8nvg==} @@ -3510,6 +3720,10 @@ packages: resolution: {integrity: sha512-YgBpdJHPyQ2UE5x+hlSXcnejzAvD0b22U2OuAP+8OnlJT+PjWPxtgmGqKKc+RgTM63U9gN0YzrYc71R2WT/hTA==} engines: {node: '>=18'} + open@8.4.2: + resolution: {integrity: sha512-7x81NCL719oNbsq/3mh+hVrAWmFuEYUqrq/Iw3kUzH8ReypT9QQ0BLoJS7/G9k6N81XjW4qHWtjWwe/9eLy1EQ==} + engines: {node: '>=12'} + openai@4.104.0: resolution: {integrity: sha512-p99EFNsA/yX6UhVO93f5kJsDRLAg+CTA2RBqdHK4RtK8u5IJw32Hyb2dTGKbnnFmnuoBv5r7Z2CURI9sGZpSuA==} hasBin: true @@ -3522,6 +3736,9 @@ packages: zod: optional: true + openid-client@5.7.1: + resolution: {integrity: sha512-jDBPgSVfTnkIh71Hg9pRvtJc6wTwqjRkN88+gCFtYWrlP4Yx2Dsrow8uPi3qLr/aeymPF3o2+dS+wOpglK04ew==} + p-finally@1.0.0: resolution: {integrity: sha512-LICb2p9CB7FS+0eR1oqWnHhp0FljGLZCWBE9aix0Uye9W8LTQPwMTYVGWQWIw9RdQiDg4+epXQODwIYJtSJaow==} engines: {node: '>=4'} @@ -3554,12 +3771,24 @@ packages: resolution: {integrity: sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==} engines: {node: '>=6'} + pac-proxy-agent@7.2.0: + resolution: {integrity: sha512-TEB8ESquiLMc0lV8vcd5Ql/JAKAoyzHFXaStwjkzpOpC5Yv+pIzLfHvjTSdf3vpa2bMiUQrg9i6276yn8666aA==} + engines: {node: '>= 14'} + + pac-resolver@7.0.1: + resolution: {integrity: sha512-5NPgf87AT2STgwa2ntRMr45jTKrYBGkVU36yT0ig/n/GMAa3oPqhZfIQ2kMEimReg0+t9kZViDVZ83qfVUlckg==} + engines: {node: '>= 14'} + package-json-from-dist@1.0.1: resolution: {integrity: sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==} packet-reader@1.0.0: resolution: {integrity: sha512-HAKu/fG3HpHFO0AA8WE8q2g+gBJaZ9MG7fcKk+IJPLTGAD6Psw4443l+9DGRbOIh3/aXr7Phy0TjilYivJo5XQ==} + pad-left@2.1.0: + resolution: {integrity: sha512-HJxs9K9AztdIQIAIa/OIazRAUW/L6B9hbQDxO4X07roW3eo9XqZc2ur9bn1StH9CnbbI9EgvejHQX7CBpCF1QA==} + engines: {node: '>=0.10.0'} + pako@2.2.0: resolution: {integrity: sha512-zJq6RP/5q+TO2OpFV3FHzlPnFjmkb7Nc99a5SNjJE+uu/PkpChs+NIZSSzbBoD+6kjiISXjfYdwj1ZRQ81dz/w==} @@ -3761,6 +3990,13 @@ packages: resolution: {integrity: sha512-+k0vdJKNdW+Vu+dYe8tZA/VvQb6XKNWexC6URwBFXxNnjLJz9nQJCemGyNgRAWD+B7+nGNc9qMPGwcD7s4nzUw==} engines: {node: '>=12.0.0'} + proxy-agent@6.5.0: + resolution: {integrity: sha512-TmatMXdr2KlRiA2CyDu8GqR8EjahTG3aY3nXjdzFyoZbmB8hrBsTyMezhULIXKnC0jpfjlmiZ3+EaCzoInSu/A==} + engines: {node: '>= 14'} + + proxy-from-env@1.1.0: + resolution: {integrity: sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==} + proxy-from-env@2.1.0: resolution: {integrity: sha512-cJ+oHTW1VAEa8cJslgmUZrc+sjRKgAKl3Zyse6+PV38hZe/V6Z14TbCuXcan9F9ghlz4QrFr2c92TNF82UkYHA==} engines: {node: '>=10'} @@ -3831,6 +4067,10 @@ packages: resolution: {integrity: sha512-LDsoVvb/CpoV9EN3FXvgvSHNJWuCIzl9MiO3ppOevuGLpSGJhwfQjpEwfFJcQvNSddHADDdZaWx0HnmMxRXG7g==} engines: {node: '>= 18.19.0'} + repeat-string@1.6.1: + resolution: {integrity: sha512-PV0dzCYDNfRi1jCDbJzpW7jNNDRuCOG/jI5ctQcGKt/clZD+YcPS3yIlWuTJMmESC8aevCFmWJy5wjAFgNqN6w==} + engines: {node: '>=0.10'} + require-directory@2.1.1: resolution: {integrity: sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==} engines: {node: '>=0.10.0'} @@ -3955,9 +4195,21 @@ packages: resolution: {integrity: sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==} engines: {node: '>=8'} + smart-buffer@4.2.0: + resolution: {integrity: sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==} + engines: {node: '>= 6.0.0', npm: '>= 3.0.0'} + snappyjs@0.7.0: resolution: {integrity: sha512-u5iEEXkMe2EInQio6Wv9LWHOQYRDbD2O9hzS27GpT/lwfIQhTCnHCTqedqHIHe9ZcvQo+9au6vngQayipz1NYw==} + socks-proxy-agent@8.0.5: + resolution: {integrity: sha512-HehCEsotFqbPW9sJ8WVYB6UbmIMv7kUUORIF2Nncq4VQvBfNBLibW9YZR5dlYCSUhwcD628pRllm7n+E+YTzJw==} + engines: {node: '>= 14'} + + socks@2.8.9: + resolution: {integrity: sha512-LJhUYUvItdQ0LkJTmPeaEObWXAqFyfmP85x0tch/ez9cahmhlBBLbIqDFnvBnUJGagb0JbIQrkBs1wJ+yRYpEw==} + engines: {node: '>= 10.0.0', npm: '>= 3.0.0'} + source-map-support@0.5.13: resolution: {integrity: sha512-SHSKFHadjVA5oR4PPqhtAVdcBWwRYVd6g6cAXnIbRiIwc2EhPrTuKUBdSLvlEKyIP3GCf89fltvcZiP9MMFA1w==} @@ -4012,6 +4264,10 @@ packages: stream-events@1.0.5: resolution: {integrity: sha512-E1GUzBSgvct8Jsb3v2X15pjzN1tYebtbLaMg+eBOUOAxgbLoSbT2NS91ckc5lJD1KfLjId+jXJRgo0qnV5Nerg==} + stream-read-all@3.0.1: + resolution: {integrity: sha512-EWZT9XOceBPlVJRrYcykW8jyRSZYbkb/0ZK36uLEmoWVO5gxBOnntNTseNzfREsqxqdfEGQrD8SXQ3QWbBmq8A==} + engines: {node: '>=10'} + stream-shift@1.0.3: resolution: {integrity: sha512-76ORR0DO1o1hlKwTbi/DM3EXWGf3ZJYO8cXX5RJwnul2DEg2oyoZyjLNoQM8WsvZiFKCRfC1O0J7iCvie3RZmQ==} @@ -4088,6 +4344,11 @@ packages: resolution: {integrity: sha512-yUT0ukFkFEt4nb+NY+n2ag51aS/u9UHXoZw+A4jgD77/jzZsBoSDHuqysrVCBC4CYR4TYvUJq54ONpXgDBH8tA==} engines: {node: '>=0.2.6'} + table-layout@3.0.2: + resolution: {integrity: sha512-rpyNZYRw+/C+dYkcQ3Pr+rLxW4CfHpXjPDnG7lYhdRoUcZTUt+KEsX+94RGp/aVp/MQU35JCITv2T/beY4m+hw==} + engines: {node: '>=12.17'} + hasBin: true + tar-fs@2.1.4: resolution: {integrity: sha512-mDAjwmZdh7LTT6pNleZ05Yt65HC3E+NiQzl672vQG38jIrehtJk/J3mNwIg+vShQPcLF/LV7CMnDW6vjj6sfYQ==} @@ -4117,6 +4378,10 @@ packages: thenify@3.3.1: resolution: {integrity: sha512-RVZSIV5IG10Hk3enotrhvz0T9em6cyHBLkH/YAZuKqd8hRkKhSfCGIcP2KUY0EPxndzANBmNllzWPwak+bheSw==} + thrift@0.16.0: + resolution: {integrity: sha512-W8DpGyTPlIaK3f+e1XOCLxefaUWXtrOXAaVIDbfYhmVyriYeAKgsBVFNJUV1F9SQ2SPt2sG44AZQxSGwGj/3VA==} + engines: {node: '>= 10.18.0'} + thrift@0.23.0: resolution: {integrity: sha512-j7F1ls8JogClU88Ta/pwD/OzYuiFeD6Z5GoWw7ip+jcDhcNYFKgfXYEsyLXYpiNfJO94fbLnITGxyfZ19YzA6Q==} engines: {node: '>= 10.18.0'} @@ -4256,6 +4521,14 @@ packages: engines: {node: '>=14.17'} hasBin: true + typical@4.0.0: + resolution: {integrity: sha512-VAH4IvQ7BDFYglMd7BPRDfLgxZZX4O4TFcRDA6EN5X7erNJJq+McIEp8np9aVtxrCJ6qx4GTYVfOWNjcqwZgRw==} + engines: {node: '>=8'} + + typical@7.3.0: + resolution: {integrity: sha512-ya4mg/30vm+DOWfBg4YK3j2WD6TWtRkCbasOJr40CseYENzCUby/7rIvXA99JGsQHeNxLbnXdyLLxKSv3tauFw==} + engines: {node: '>=12.17'} + ufo@1.6.4: resolution: {integrity: sha512-JFNbkD1Svwe0KvGi8GOeLcP4kAWQ609twvCdcHxq1oSL8svv39ZuSvajcD8B+5D0eL4+s1Is2D/O6KN3qcTeRA==} @@ -4370,6 +4643,10 @@ packages: wordwrap@1.0.0: resolution: {integrity: sha512-gvVzJFlPycKc5dZN4yPkP8w7Dc37BtP1yczEneOb4uq34pXZcvrtRTmWV8W+Ume+XCxKgbjM+nevkyFPMybd4Q==} + wordwrapjs@5.1.1: + resolution: {integrity: sha512-0yweIbkINJodk27gX9LBGMzyQdBDan3s/dEAiwBOj+Mf0PPyWL6/rikalkv8EeD0E8jm4o5RXEOrFTP3NXbhJg==} + engines: {node: '>=12.17'} + wrap-ansi@7.0.0: resolution: {integrity: sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==} engines: {node: '>=10'} @@ -4423,6 +4700,9 @@ packages: xxhash-wasm@1.1.0: resolution: {integrity: sha512-147y/6YNh+tlp6nd/2pWq38i9h6mz/EuQ6njIrmW8D1BS5nCqs0P6DG+m6zTGnNz5I+uhZ0SHxBs9BsPrwcKDA==} + xxhashjs@0.2.2: + resolution: {integrity: sha512-AkTuIuVTET12tpsVIQo+ZU6f/qDmKuRUcjaqR+OIvm+aCBsZ95i7UVY5WJ9TMsSaZ0DA2WxoZ4acu0sPH+OKAw==} + y18n@5.0.8: resolution: {integrity: sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==} engines: {node: '>=10'} @@ -4470,6 +4750,11 @@ packages: snapshots: + '@75lb/deep-merge@1.1.4': + dependencies: + lodash: 4.18.1 + typical: 7.3.0 + '@anthropic-ai/sdk@0.40.1': dependencies: '@types/node': 18.19.130 @@ -5720,6 +6005,60 @@ snapshots: enabled: 2.0.0 kuler: 2.0.0 + '@databricks/databricks-sql-kernel-darwin-arm64@0.2.0': + optional: true + + '@databricks/databricks-sql-kernel-darwin-x64@0.2.0': + optional: true + + '@databricks/databricks-sql-kernel-linux-arm64-gnu@0.2.0': + optional: true + + '@databricks/databricks-sql-kernel-linux-arm64-musl@0.2.0': + optional: true + + '@databricks/databricks-sql-kernel-linux-x64-gnu@0.2.0': + optional: true + + '@databricks/databricks-sql-kernel-linux-x64-musl@0.2.0': + optional: true + + '@databricks/databricks-sql-kernel-win32-arm64-msvc@0.2.0': + optional: true + + '@databricks/databricks-sql-kernel-win32-x64-msvc@0.2.0': + optional: true + + '@databricks/sql@1.16.0': + dependencies: + apache-arrow: 13.0.0 + commander: 9.5.0 + flatbuffers: 23.5.26 + node-fetch: 2.7.0 + node-int64: 0.4.0 + open: 8.4.2 + openid-client: 5.7.1 + proxy-agent: 6.5.0 + thrift: 0.16.0 + uuid: 11.1.1 + winston: 3.19.0 + optionalDependencies: + '@databricks/databricks-sql-kernel-darwin-arm64': 0.2.0 + '@databricks/databricks-sql-kernel-darwin-x64': 0.2.0 + '@databricks/databricks-sql-kernel-linux-arm64-gnu': 0.2.0 + '@databricks/databricks-sql-kernel-linux-arm64-musl': 0.2.0 + '@databricks/databricks-sql-kernel-linux-x64-gnu': 0.2.0 + '@databricks/databricks-sql-kernel-linux-x64-musl': 0.2.0 + '@databricks/databricks-sql-kernel-win32-arm64-msvc': 0.2.0 + '@databricks/databricks-sql-kernel-win32-x64-msvc': 0.2.0 + lz4: 0.6.5 + transitivePeerDependencies: + - '@75lb/nature' + - bufferutil + - encoding + - supports-color + - utf-8-validate + '@datastructures-js/deque@1.0.8': {} '@elastic/elasticsearch@9.4.2': @@ -6755,6 +7094,8 @@ snapshots: '@supabase/realtime-js': 2.108.1 '@supabase/storage-js': 2.108.1 + '@tootallnate/quickjs-emscripten@0.23.0': {} + '@tsconfig/node10@1.0.12': {} '@tsconfig/node12@1.0.11': {} @@ -6793,6 +7134,10 @@ snapshots: dependencies: '@types/node': 22.19.21 + '@types/command-line-args@5.2.0': {} + + '@types/command-line-usage@5.0.2': {} + '@types/estree@1.0.9': {} '@types/graceful-fs@4.1.9': @@ -6827,6 +7172,8 @@ snapshots: dependencies: undici-types: 5.26.5 + '@types/node@20.3.0': {} + '@types/node@22.19.21': dependencies: undici-types: 6.21.0 @@ -6837,6 +7184,8 @@ snapshots: '@types/normalize-package-data@2.4.4': {} + '@types/pad-left@2.1.1': {} + '@types/pg@8.11.0': dependencies: '@types/node': 22.19.21 @@ -6965,6 +7314,21 @@ snapshots: anynum@1.0.1: {} + apache-arrow@13.0.0: + dependencies: + '@types/command-line-args': 5.2.0 + '@types/command-line-usage': 5.0.2 + '@types/node': 20.3.0 + '@types/pad-left': 2.1.1 + command-line-args: 5.2.1 + command-line-usage: 7.0.1 + flatbuffers: 23.5.26 + json-bignum: 0.0.3 + pad-left: 2.1.0 + tslib: 2.8.1 + transitivePeerDependencies: + - '@75lb/nature' + apparatus@0.0.10: dependencies: sylvester: 0.0.21 @@ -6973,6 +7337,14 @@ snapshots: argparse@2.0.1: {} + array-back@3.1.0: {} + + array-back@6.2.3: {} + + ast-types@0.13.4: + dependencies: + tslib: 2.8.1 + async-limiter@1.0.1: {} async@3.2.6: {} @@ -7058,6 +7430,8 @@ snapshots: baseline-browser-mapping@2.10.35: {} + basic-ftp@5.3.1: {} + better-sqlite3@12.10.0: dependencies: bindings: 1.5.0 @@ -7168,6 +7542,10 @@ snapshots: adm-zip: 0.5.17 long: 5.2.5 + chalk-template@0.4.0: + dependencies: + chalk: 4.1.2 + chalk@4.1.2: dependencies: ansi-styles: 4.3.0 @@ -7292,8 +7670,26 @@ snapshots: dependencies: delayed-stream: 1.0.0 + command-line-args@5.2.1: + dependencies: + array-back: 3.1.0 + find-replace: 3.0.0 + lodash.camelcase: 4.3.0 + typical: 4.0.0 + + command-line-usage@7.0.1: + dependencies: + array-back: 6.2.3 + chalk-template: 0.4.0 + table-layout: 3.0.2 + typical: 7.3.0 + transitivePeerDependencies: + - '@75lb/nature' + commander@4.1.1: {} + commander@9.5.0: {} + compromise@14.15.1: dependencies: efrt: 2.7.0 @@ -7344,8 +7740,13 @@ snapshots: crypt@0.0.2: {} + cuint@0.2.2: + optional: true + data-uri-to-buffer@4.0.1: {} + data-uri-to-buffer@6.0.2: {} + dayjs@1.11.21: {} debug@4.4.3(supports-color@5.5.0): @@ -7377,6 +7778,8 @@ snapshots: es-errors: 1.3.0 gopd: 1.2.0 + define-lazy-prop@2.0.0: {} + define-lazy-prop@3.0.0: {} define-properties@1.2.1: @@ -7385,6 +7788,12 @@ snapshots: has-property-descriptors: 1.0.2 object-keys: 1.1.1 + degenerator@5.0.1: + dependencies: + ast-types: 0.13.4 + escodegen: 2.1.0 + esprima: 4.0.1 + delayed-stream@1.0.0: {} denque@2.1.0: {} @@ -7499,6 +7908,20 @@ snapshots: escape-string-regexp@4.0.0: {} + escodegen@2.1.0: + dependencies: + esprima: 4.0.1 + estraverse: 5.3.0 + esutils: 2.0.3 + optionalDependencies: + source-map: 0.6.1 + + esprima@4.0.1: {} + + estraverse@5.3.0: {} + + esutils@2.0.3: {} + event-target-shim@5.0.1: {} eventemitter3@4.0.7: {} @@ -7590,6 +8013,10 @@ snapshots: dependencies: to-regex-range: 5.0.1 + find-replace@3.0.0: + dependencies: + array-back: 3.1.0 + find-up@4.1.0: dependencies: locate-path: 5.0.0 @@ -7608,6 +8035,8 @@ snapshots: kolorist: 1.8.0 read-pkg: 8.1.0 + flatbuffers@23.5.26: {} + flatbuffers@25.9.23: {} fn.name@1.1.0: {} @@ -7701,6 +8130,14 @@ snapshots: get-stream@6.0.1: {} + get-uri@6.0.5: + dependencies: + basic-ftp: 5.3.1 + data-uri-to-buffer: 6.0.2 + debug: 4.4.3(supports-color@5.5.0) + transitivePeerDependencies: + - supports-color + github-from-package@0.0.0: {} glob-parent@5.1.2: @@ -7923,6 +8360,8 @@ snapshots: transitivePeerDependencies: - supports-color + ip-address@10.2.0: {} + is-arrayish@0.2.1: {} is-binary-path@2.1.0: @@ -7935,6 +8374,8 @@ snapshots: dependencies: hasown: 2.0.4 + is-docker@2.2.1: {} + is-docker@3.0.0: {} is-extglob@2.1.1: {} @@ -7959,6 +8400,10 @@ snapshots: is-unsafe@1.0.1: {} + is-wsl@2.2.0: + dependencies: + is-docker: 2.2.1 + is-wsl@3.1.1: dependencies: is-inside-container: 1.0.0 @@ -8325,6 +8770,8 @@ snapshots: - supports-color - ts-node + jose@4.15.9: {} + joycon@3.1.1: {} js-tiktoken@1.0.21: @@ -8343,6 +8790,8 @@ snapshots: dependencies: bignumber.js: 9.3.1 + json-bignum@0.0.3: {} + json-parse-even-better-errors@2.3.1: {} json-parse-even-better-errors@3.0.2: {} @@ -8435,6 +8884,8 @@ snapshots: lodash.once@4.1.1: {} + lodash@4.18.1: {} + logform@2.7.0: dependencies: '@colors/colors': 1.6.0 @@ -8454,10 +8905,24 @@ snapshots: dependencies: yallist: 3.1.1 + lru-cache@6.0.0: + dependencies: + yallist: 4.0.0 + + lru-cache@7.18.3: {} + lru-cache@9.1.2: {} lru.min@1.1.4: {} + lz4@0.6.5: + dependencies: + buffer: 5.7.1 + cuint: 0.2.2 + nan: 2.28.0 + xxhashjs: 0.2.2 + optional: true + magic-string@0.30.21: dependencies: '@jridgewell/sourcemap-codec': 1.5.5 @@ -8546,16 +9011,18 @@ snapshots: '@types/whatwg-url': 13.0.0 whatwg-url: 14.2.0 - mongodb@7.2.0: + mongodb@7.2.0(socks@2.8.9): dependencies: '@mongodb-js/saslprep': 1.4.11 bson: 7.2.0 mongodb-connection-string-url: 7.0.1 + optionalDependencies: + socks: 2.8.9 - mongoose@9.7.0: + mongoose@9.7.0(socks@2.8.9): dependencies: kareem: 3.3.0 - mongodb: 7.2.0 + mongodb: 7.2.0(socks@2.8.9) mpath: 0.9.0 mquery: 6.0.0 ms: 2.1.3 @@ -8599,18 +9066,21 @@ snapshots: dependencies: lru.min: 1.1.4 + nan@2.28.0: + optional: true + napi-build-utils@2.0.0: {} natural-compare@1.4.0: {} - natural@8.1.1(@opentelemetry/api@1.9.1): + natural@8.1.1(@opentelemetry/api@1.9.1)(socks@2.8.9): dependencies: afinn-165: 2.0.2 afinn-165-financialmarketnews: 3.0.0 apparatus: 0.0.10 dotenv: 17.4.2 memjs: 1.3.2 - mongoose: 9.7.0 + mongoose: 9.7.0(socks@2.8.9) pg: 8.21.0 redis: 5.12.1(@opentelemetry/api@1.9.1) safe-stable-stringify: 2.5.0 @@ -8633,6 +9103,8 @@ snapshots: neo-async@2.6.2: {} + netmask@2.1.1: {} + nice-grpc-client-middleware-retry@3.1.15: dependencies: abort-controller-x: 0.5.0 @@ -8695,12 +9167,16 @@ snapshots: object-assign@4.1.1: {} + object-hash@2.2.0: {} + object-hash@3.0.0: {} object-keys@1.1.1: {} obuf@1.1.2: {} + oidc-token-hash@5.2.0: {} + ollama@0.5.18: dependencies: whatwg-fetch: 3.6.20 @@ -8751,6 +9227,12 @@ snapshots: is-inside-container: 1.0.0 wsl-utils: 0.1.0 + open@8.4.2: + dependencies: + define-lazy-prop: 2.0.0 + is-docker: 2.2.1 + is-wsl: 2.2.0 + openai@4.104.0(ws@5.2.5)(zod@3.25.76): dependencies: '@types/node': 18.19.130 @@ -8766,6 +9248,13 @@ snapshots: transitivePeerDependencies: - encoding + openid-client@5.7.1: + dependencies: + jose: 4.15.9 + lru-cache: 6.0.0 + object-hash: 2.2.0 + oidc-token-hash: 5.2.0 + p-finally@1.0.0: {} p-limit@2.3.0: @@ -8796,10 +9285,32 @@ snapshots: p-try@2.2.0: {} + pac-proxy-agent@7.2.0: + dependencies: + '@tootallnate/quickjs-emscripten': 0.23.0 + agent-base: 7.1.4 + debug: 4.4.3(supports-color@5.5.0) + get-uri: 6.0.5 + http-proxy-agent: 7.0.2 + https-proxy-agent: 7.0.6 + pac-resolver: 7.0.1 + socks-proxy-agent: 8.0.5 + transitivePeerDependencies: + - supports-color + + pac-resolver@7.0.1: + dependencies: + degenerator: 5.0.1 + netmask: 2.1.1 + package-json-from-dist@1.0.1: {} packet-reader@1.0.0: {} + pad-left@2.1.0: + dependencies: + repeat-string: 1.6.1 + pako@2.2.0: {} parse-json@5.2.0: @@ -8996,6 +9507,21 @@ snapshots: '@types/node': 22.19.21 long: 5.3.2 + proxy-agent@6.5.0: + dependencies: + agent-base: 7.1.4 + debug: 4.4.3(supports-color@5.5.0) + http-proxy-agent: 7.0.2 + https-proxy-agent: 7.0.6 + lru-cache: 7.18.3 + pac-proxy-agent: 7.2.0 + proxy-from-env: 1.1.0 + socks-proxy-agent: 8.0.5 + transitivePeerDependencies: + - supports-color + + proxy-from-env@1.1.0: {} + proxy-from-env@2.1.0: {} pstree.remy@1.1.8: {} @@ -9075,6 +9601,8 @@ snapshots: - '@node-rs/xxhash' - '@opentelemetry/api' + repeat-string@1.6.1: {} + require-directory@2.1.1: {} resolve-cwd@3.0.0: @@ -9232,8 +9760,23 @@ snapshots: slash@3.0.0: {} + smart-buffer@4.2.0: {} + snappyjs@0.7.0: {} + socks-proxy-agent@8.0.5: + dependencies: + agent-base: 7.1.4 + debug: 4.4.3(supports-color@5.5.0) + socks: 2.8.9 + transitivePeerDependencies: + - supports-color + + socks@2.8.9: + dependencies: + ip-address: 10.2.0 + smart-buffer: 4.2.0 + source-map-support@0.5.13: dependencies: buffer-from: 1.1.2 @@ -9281,6 +9824,8 @@ snapshots: dependencies: stubs: 3.0.0 + stream-read-all@3.0.1: {} + stream-shift@1.0.3: {} string-length@4.0.2: @@ -9354,6 +9899,18 @@ snapshots: sylvester@0.0.21: {} + table-layout@3.0.2: + dependencies: + '@75lb/deep-merge': 1.1.4 + array-back: 6.2.3 + command-line-args: 5.2.1 + command-line-usage: 7.0.1 + stream-read-all: 3.0.1 + typical: 7.3.0 + wordwrapjs: 5.1.1 + transitivePeerDependencies: + - '@75lb/nature' + tar-fs@2.1.4: dependencies: chownr: 1.1.4 @@ -9402,6 +9959,17 @@ snapshots: dependencies: any-promise: 1.3.0 + thrift@0.16.0: + dependencies: + browser-or-node: 1.3.0 + isomorphic-ws: 4.0.1(ws@5.2.5) + node-int64: 0.4.0 + q: 1.5.1 + ws: 5.2.5 + transitivePeerDependencies: + - bufferutil + - utf-8-validate + thrift@0.23.0: dependencies: browser-or-node: 1.3.0 @@ -9533,6 +10101,10 @@ snapshots: typescript@5.5.4: {} + typical@4.0.0: {} + + typical@7.3.0: {} + ufo@1.6.4: {} uglify-js@3.19.3: @@ -9649,6 +10221,8 @@ snapshots: wordwrap@1.0.0: {} + wordwrapjs@5.1.1: {} + wrap-ansi@7.0.0: dependencies: ansi-styles: 4.3.0 @@ -9684,6 +10258,11 @@ snapshots: xxhash-wasm@1.1.0: {} + xxhashjs@0.2.2: + dependencies: + cuint: 0.2.2 + optional: true + y18n@5.0.8: {} yallist@3.1.1: {} diff --git a/mem0-ts/src/oss/src/config/manager.ts b/mem0-ts/src/oss/src/config/manager.ts index c4ad83116..b2803d1b2 100644 --- a/mem0-ts/src/oss/src/config/manager.ts +++ b/mem0-ts/src/oss/src/config/manager.ts @@ -60,9 +60,14 @@ export class ConfigManager { })(), }, vectorStore: { - provider: + // Every factory already matches the provider case-insensitively, so a capitalized + // name constructs the right store -- but the `provider === "memory"` comparisons that + // pick per-provider entity-store settings do not. Normalize once, here, so those + // comparisons cannot silently miss. + provider: ( userConfig.vectorStore?.provider || - DEFAULT_MEMORY_CONFIG.vectorStore.provider, + DEFAULT_MEMORY_CONFIG.vectorStore.provider + ).toLowerCase(), config: (() => { const defaultConf = DEFAULT_MEMORY_CONFIG.vectorStore.config; const userConf = userConfig.vectorStore?.config; diff --git a/mem0-ts/src/oss/src/index.ts b/mem0-ts/src/oss/src/index.ts index 84c6316e6..b883c6355 100644 --- a/mem0-ts/src/oss/src/index.ts +++ b/mem0-ts/src/oss/src/index.ts @@ -36,6 +36,7 @@ export * from "./vector_stores/langchain"; export * from "./vector_stores/vectorize"; export * from "./vector_stores/azure_ai_search"; export * from "./vector_stores/pgvector"; +export * from "./vector_stores/databricks"; export * from "./vector_stores/neptune_analytics"; export * from "./vector_stores/elasticsearch"; export * from "./vector_stores/upstash_vector"; diff --git a/mem0-ts/src/oss/src/memory/index.ts b/mem0-ts/src/oss/src/memory/index.ts index 23a6708e8..4e48dc181 100644 --- a/mem0-ts/src/oss/src/memory/index.ts +++ b/mem0-ts/src/oss/src/memory/index.ts @@ -7,6 +7,7 @@ import { Message, SearchFilters, SearchResult, + VectorStoreConfig, } from "../types"; import { EmbedderFactory, @@ -286,18 +287,24 @@ export class Memory { private async getEntityStore(): Promise { if (!this._entityStore) { + const entityProvider = this.config.vectorStore.provider; const entityCollectionName = `${this.collectionName}_entities`; - const entityConfig = { + const entityConfig: VectorStoreConfig = { ...this.config.vectorStore.config, collectionName: entityCollectionName, }; // For file-based stores (memory/SQLite), always use a separate DB for entities - if (this.config.vectorStore.provider === "memory") { + if (entityProvider === "memory") { const basePath = entityConfig.dbPath || getDefaultVectorStoreDbPath(); entityConfig.dbPath = basePath.replace(/\.db$/, "_entities.db"); } + if (entityProvider === "databricks") { + entityConfig.tableName = entityConfig.tableName + ? `${entityConfig.tableName}_entities` + : entityCollectionName; + } this._entityStore = VectorStoreFactory.create( - this.config.vectorStore.provider, + entityProvider, entityConfig, ); await this._entityStore.initialize(); @@ -1747,7 +1754,7 @@ export class Memory { await this.db.reset(); // Check provider before attempting deleteCol - if (this.config.vectorStore.provider.toLowerCase() !== "langchain") { + if (this.config.vectorStore.provider !== "langchain") { try { await this.vectorStore.deleteCol(); } catch (e) { diff --git a/mem0-ts/src/oss/src/utils/factory.ts b/mem0-ts/src/oss/src/utils/factory.ts index 2bed7f484..fcafde537 100644 --- a/mem0-ts/src/oss/src/utils/factory.ts +++ b/mem0-ts/src/oss/src/utils/factory.ts @@ -55,6 +55,7 @@ import { HuggingFaceEmbedder } from "../embeddings/huggingface"; import { LangchainVectorStore } from "../vector_stores/langchain"; import { AzureAISearch } from "../vector_stores/azure_ai_search"; import { PGVector } from "../vector_stores/pgvector"; +import { DatabricksVectorStore } from "../vector_stores/databricks"; import { NeptuneAnalyticsVectorStore } from "../vector_stores/neptune_analytics"; import { VertexAIEmbedder } from "../embeddings/vertexai"; import { ElasticsearchDB } from "../vector_stores/elasticsearch"; @@ -173,6 +174,8 @@ export class VectorStoreFactory { return new VertexAIVectorSearch(config as any); case "pgvector": return new PGVector(config as any); + case "databricks": + return new DatabricksVectorStore(config as any); case "neptune": case "neptune-analytics": return new NeptuneAnalyticsVectorStore(config as any); diff --git a/mem0-ts/src/oss/src/vector_stores/databricks.ts b/mem0-ts/src/oss/src/vector_stores/databricks.ts new file mode 100644 index 000000000..627e522ee --- /dev/null +++ b/mem0-ts/src/oss/src/vector_stores/databricks.ts @@ -0,0 +1,1627 @@ +import axios, { AxiosInstance } from "axios"; +import { VectorStore } from "./base"; +import { SearchFilters, VectorStoreConfig, VectorStoreResult } from "../types"; + +const SAFE_IDENTIFIER_RE = /^[A-Za-z_][A-Za-z0-9_]{0,127}$/; +const DEFAULT_PAGE_SIZE = 100; +const MAX_QUERY_RESULTS = 10_000; +const MAX_FULL_TEXT_RESULTS = 200; +const DEFAULT_SYNC_POLL_INTERVAL_MS = 1000; +const DEFAULT_SYNC_TIMEOUT_MS = 5 * 60 * 1000; +const DATABRICKS_SERVER_FILTER_KEYS = new Set([ + "memory_id", + "user_id", + "agent_id", + "run_id", +]); + +interface DatabricksConfig extends VectorStoreConfig { + workspaceUrl?: string; + host?: string; + httpPath: string; + accessToken?: string; + clientId?: string; + clientSecret?: string; + endpointName?: string; + endpointType?: "STANDARD" | "STORAGE_OPTIMIZED"; + pipelineType?: "TRIGGERED" | "CONTINUOUS"; + queryType?: "ANN" | "HYBRID"; + catalog?: string; + schema?: string; + tableName?: string; + embeddingModelDims?: number; + syncPollIntervalMs?: number; + syncTimeoutMs?: number; + sqlClient?: DatabricksSqlClientLike; + httpClient?: DatabricksHttpClientLike; +} + +interface DatabricksSqlClientLike { + connect(options: Record): Promise; + openSession(): Promise; + close?(): Promise; +} + +interface DatabricksSqlSessionLike { + executeStatement(statement: string): Promise; + close?(): Promise; +} + +interface DatabricksSqlOperationLike { + fetchAll(): Promise>>; + close?(): Promise; +} + +interface DatabricksHttpClientLike { + get(url: string, config?: Record): Promise<{ data: any }>; + post( + url: string, + data?: any, + config?: Record, + ): Promise<{ data: any }>; + delete(url: string, config?: Record): Promise<{ data: any }>; +} + +interface DatabricksVector { + id: string; + payload: Record; +} + +function buildDatabricksAuthorizationDetails( + indexName: string, + operation: "ReadVectorIndex" | "WriteVectorIndex", +): string { + return JSON.stringify([ + { + type: "unity_catalog_permission", + securable_type: "table", + securable_object_name: indexName, + operation, + }, + ]); +} + +function validateIdentifier( + name: string, + label: string = "identifier", +): string { + if (!SAFE_IDENTIFIER_RE.test(name)) { + throw new Error( + `Invalid ${label} '${name}': only letters, digits, and underscores are allowed, ` + + `must start with a letter or underscore, and be at most 128 characters.`, + ); + } + return name; +} + +function extractHostAndWorkspaceUrl(config: DatabricksConfig): { + host: string; + workspaceUrl: string; +} { + if (config.workspaceUrl) { + const url = new URL(config.workspaceUrl); + return { + host: url.host, + workspaceUrl: `${url.protocol}//${url.host}`, + }; + } + + if (config.host) { + return { + host: config.host, + workspaceUrl: `https://${config.host}`, + }; + } + + throw new Error( + "Databricks vector store requires either workspaceUrl or host.", + ); +} + +// `Memory.search()` hands keywordSearch() an already-lemmatized query, so BM25 has to score +// against lemmatized text. Same contract as baidu.ts / pgvector.ts. +function lemmatizedText(payload: Record): string { + const data = typeof payload.data === "string" ? payload.data : ""; + return typeof payload.textLemmatized === "string" && + payload.textLemmatized.length > 0 + ? payload.textLemmatized + : data; +} + +function formatSqlValue(value: any): string { + if (value === null || value === undefined) { + return "NULL"; + } + if (typeof value === "boolean") { + return value ? "TRUE" : "FALSE"; + } + if (typeof value === "number") { + if (!Number.isFinite(value)) { + throw new Error("Databricks vector store only accepts finite numbers."); + } + return String(value); + } + if (Array.isArray(value)) { + return `array(${value.map((entry) => formatSqlValue(entry)).join(", ")})`; + } + const json = + typeof value === "string" ? value : JSON.stringify(value ?? {}) || "{}"; + // Databricks/Spark SQL treats backslash as an escape char in string literals, + // so a trailing "\" would consume the closing quote (breaking out of the + // literal) and any backslash would be dropped on write. Escape backslashes + // before doubling quotes so the literal is injection-safe and round-trips. + const escaped = json.replace(/\\/g, "\\\\").replace(/'/g, "''"); + return `'${escaped}'`; +} + +function extractRowValue(row: Record, keys: string[]): any { + for (const key of keys) { + if (key in row) { + return row[key]; + } + const upper = key.toUpperCase(); + if (upper in row) { + return row[upper]; + } + } + return undefined; +} + +function isPlainObject(value: any): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function normalizeFilterValue(value: any): any { + if (Array.isArray(value)) { + return value.map((entry) => normalizeFilterValue(entry)); + } + return value; +} + +function isDatabricksServerFilterKey(key: string): boolean { + return DATABRICKS_SERVER_FILTER_KEYS.has(key); +} + +function mergeDatabricksFilters( + target: Record, + source: Record, +): boolean { + for (const [key, value] of Object.entries(source)) { + if ( + key in target && + JSON.stringify(target[key]) !== JSON.stringify(value) + ) { + return false; + } + target[key] = value; + } + return true; +} + +function buildSimpleDatabricksFilter( + key: string, + value: any, +): Record | null { + if (!isDatabricksServerFilterKey(key)) { + return null; + } + + const safeKey = validateIdentifier(key, "filter key"); + + if (!isPlainObject(value)) { + return { + [safeKey]: normalizeFilterValue(value), + }; + } + + const entries = Object.entries(value); + if (entries.length !== 1) { + return null; + } + + const [operator, operand] = entries[0]; + switch (operator) { + case "eq": + return { [safeKey]: normalizeFilterValue(operand) }; + case "ne": + return { [`${safeKey} NOT`]: normalizeFilterValue(operand) }; + case "gt": + return { [`${safeKey} >`]: normalizeFilterValue(operand) }; + case "gte": + return { [`${safeKey} >=`]: normalizeFilterValue(operand) }; + case "lt": + return { [`${safeKey} <`]: normalizeFilterValue(operand) }; + case "lte": + return { [`${safeKey} <=`]: normalizeFilterValue(operand) }; + case "in": + return Array.isArray(operand) + ? { [safeKey]: normalizeFilterValue(operand) } + : null; + case "nin": + return Array.isArray(operand) && operand.length === 1 + ? { [`${safeKey} NOT`]: normalizeFilterValue(operand[0]) } + : null; + default: + return null; + } +} + +function buildStandardDatabricksFiltersFromClauses( + clauses: Array<[string, any]>, +): Record | undefined { + const result: Record = {}; + + for (const [key, value] of clauses) { + const translated = buildSimpleDatabricksFilter(key, value); + if (!translated) { + continue; + } + if (!mergeDatabricksFilters(result, translated)) { + return undefined; + } + } + + return Object.keys(result).length > 0 ? result : undefined; +} + +function buildStorageOptimizedDatabricksFilterClause( + key: string, + value: any, +): string | null { + if (!isDatabricksServerFilterKey(key)) { + return null; + } + + const safeKey = validateIdentifier(key, "filter key"); + + if (!isPlainObject(value)) { + if (Array.isArray(value)) { + return value.length > 0 + ? `${safeKey} IN (${value + .map((entry) => formatSqlValue(normalizeFilterValue(entry))) + .join(", ")})` + : null; + } + return `${safeKey} = ${formatSqlValue(normalizeFilterValue(value))}`; + } + + const entries = Object.entries(value); + if (entries.length !== 1) { + return null; + } + + const [operator, operand] = entries[0]; + switch (operator) { + case "eq": + return `${safeKey} = ${formatSqlValue(normalizeFilterValue(operand))}`; + case "ne": + // SQL three-valued logic: `col != x` is NULL (excluded) when col IS NULL, + // but the local matcher keeps rows with a missing field for `ne`. Match it. + return `(${safeKey} IS NULL OR ${safeKey} != ${formatSqlValue(normalizeFilterValue(operand))})`; + case "gt": + return `${safeKey} > ${formatSqlValue(normalizeFilterValue(operand))}`; + case "gte": + return `${safeKey} >= ${formatSqlValue(normalizeFilterValue(operand))}`; + case "lt": + return `${safeKey} < ${formatSqlValue(normalizeFilterValue(operand))}`; + case "lte": + return `${safeKey} <= ${formatSqlValue(normalizeFilterValue(operand))}`; + case "in": + return Array.isArray(operand) && operand.length > 0 + ? `${safeKey} IN (${operand + .map((entry) => formatSqlValue(normalizeFilterValue(entry))) + .join(", ")})` + : null; + case "nin": + // Same NULL-handling as `ne` above. + return Array.isArray(operand) && operand.length > 0 + ? `(${safeKey} IS NULL OR ${safeKey} NOT IN (${operand + .map((entry) => formatSqlValue(normalizeFilterValue(entry))) + .join(", ")}))` + : null; + default: + return null; + } +} + +function collectConjunctiveDatabricksFilters( + filters?: SearchFilters, +): Array<[string, any]> { + if (!filters || Object.keys(filters).length === 0) { + return []; + } + + const clauses: Array<[string, any]> = []; + + for (const [key, value] of Object.entries(filters)) { + if (key === "$and") { + if (!Array.isArray(value)) { + continue; + } + for (const entry of value) { + if (!isPlainObject(entry)) { + continue; + } + clauses.push(...collectConjunctiveDatabricksFilters(entry)); + } + continue; + } + + if (key === "$or" || key === "$not") { + continue; + } + + clauses.push([key, value]); + } + + return clauses; +} + +// True iff `filters` has no `$or`/`$not` at any nesting depth (recursing through +// `$and`), i.e. every entry is something collectConjunctiveDatabricksFilters can see. +function isFullyConjunctiveDatabricksFilter(filters?: SearchFilters): boolean { + if (!filters || Object.keys(filters).length === 0) { + return true; + } + + for (const [key, value] of Object.entries(filters)) { + if (key === "$and") { + if (!Array.isArray(value)) { + return false; + } + if ( + value.some( + (entry) => + !isPlainObject(entry) || + !isFullyConjunctiveDatabricksFilter(entry as SearchFilters), + ) + ) { + return false; + } + continue; + } + + if (key === "$or" || key === "$not") { + return false; + } + } + + return true; +} + +function buildDatabricksServerFilters( + endpointType: "STANDARD" | "STORAGE_OPTIMIZED", + filters?: SearchFilters, +): { filters?: string; filters_json?: string } { + const clauses = collectConjunctiveDatabricksFilters(filters); + + if (clauses.length === 0) { + return {}; + } + + if (endpointType === "STORAGE_OPTIMIZED") { + const translatedClauses = clauses + .map(([key, value]) => + buildStorageOptimizedDatabricksFilterClause(key, value), + ) + .filter((clause): clause is string => Boolean(clause)); + + return translatedClauses.length > 0 + ? { filters: translatedClauses.join(" AND ") } + : {}; + } + + const translatedFilters = buildStandardDatabricksFiltersFromClauses(clauses); + + return translatedFilters + ? { filters_json: JSON.stringify(translatedFilters) } + : {}; +} + +export class DatabricksVectorStore implements VectorStore { + private readonly host: string; + private readonly workspaceUrl: string; + private readonly httpPath: string; + private readonly accessToken?: string; + private readonly clientId?: string; + private readonly clientSecret?: string; + private readonly endpointName: string; + private readonly endpointType: "STANDARD" | "STORAGE_OPTIMIZED"; + private readonly pipelineType: "TRIGGERED" | "CONTINUOUS"; + private readonly queryType: "ANN" | "HYBRID"; + private readonly dimension: number; + private readonly catalog: string; + private readonly schema: string; + private readonly tableName: string; + private readonly indexName: string; + private readonly fullTableName: string; + private readonly fullIndexName: string; + private readonly syncPollIntervalMs: number; + private readonly syncTimeoutMs: number; + private sqlClient: DatabricksSqlClientLike | null; + private readonly httpClient: DatabricksHttpClientLike; + private session?: DatabricksSqlSessionLike; + private _sessionPromise?: Promise; + private _initPromise?: Promise; + private sqlModulePromise?: Promise<{ + DBSQLClient: new () => DatabricksSqlClientLike; + }>; + private indexSyncWait: Promise | null = null; + private indexSyncRunning = false; + private indexSyncQueued = false; + private indexSyncError: unknown = null; + private readonly oauthTokens = new Map< + string, + { accessToken: string; expiresAt: number } + >(); + + constructor(config: DatabricksConfig) { + const { host, workspaceUrl } = extractHostAndWorkspaceUrl(config); + + this.host = host; + this.workspaceUrl = workspaceUrl; + this.httpPath = config.httpPath; + this.accessToken = config.accessToken; + this.clientId = config.clientId; + this.clientSecret = config.clientSecret; + this.endpointName = config.endpointName || "mem0_vector_search"; + this.endpointType = config.endpointType || "STANDARD"; + this.pipelineType = config.pipelineType || "TRIGGERED"; + this.queryType = config.queryType || "ANN"; + this.dimension = config.embeddingModelDims || config.dimension || 1536; + this.catalog = validateIdentifier(config.catalog || "main", "catalog"); + this.schema = validateIdentifier(config.schema || "default", "schema"); + this.indexName = validateIdentifier( + config.collectionName || "mem0", + "collectionName", + ); + this.tableName = validateIdentifier( + config.tableName || this.indexName, + "tableName", + ); + this.fullTableName = `${this.catalog}.${this.schema}.${this.tableName}`; + this.fullIndexName = `${this.catalog}.${this.schema}.${this.indexName}`; + this.syncPollIntervalMs = + config.syncPollIntervalMs ?? DEFAULT_SYNC_POLL_INTERVAL_MS; + this.syncTimeoutMs = config.syncTimeoutMs ?? DEFAULT_SYNC_TIMEOUT_MS; + this.sqlClient = config.sqlClient ?? null; + this.httpClient = config.httpClient || this.createHttpClient(); + + if ( + this.endpointType === "STORAGE_OPTIMIZED" && + this.pipelineType !== "TRIGGERED" + ) { + throw new Error( + "Databricks storage-optimized endpoints only support TRIGGERED pipelineType.", + ); + } + + if ( + this.endpointType === "STORAGE_OPTIMIZED" && + this.dimension % 16 !== 0 + ) { + throw new Error( + "Databricks storage-optimized endpoints require dimensions divisible by 16.", + ); + } + + this.initialize().catch(console.error); + } + + async initialize(): Promise { + if (!this._initPromise) { + this._initPromise = this._doInitialize().catch((error) => { + // A failed init (e.g. a cold/auto-suspended warehouse at startup) must not be cached + // forever -- clear it so the next public call retries instead of replaying the + // rejection. Every step is idempotent (CREATE ... IF NOT EXISTS / ensure*), so a + // retry is safe. + this._initPromise = undefined; + throw error; + }); + } + return this._initPromise; + } + + private async _doInitialize(): Promise { + await this.executeSql( + `CREATE SCHEMA IF NOT EXISTS ${this.catalog}.${this.schema}`, + ); + await this.executeSql(` + CREATE TABLE IF NOT EXISTS ${this.fullTableName} ( + memory_id STRING, + embedding ARRAY, + payload STRING, + text_lemmatized STRING, + user_id STRING, + agent_id STRING, + run_id STRING + ) USING DELTA + TBLPROPERTIES ('delta.enableChangeDataFeed' = 'true') + `); + await this.ensureChangeDataFeedEnabled(); + await this.executeSql(` + CREATE TABLE IF NOT EXISTS ${this.catalog}.${this.schema}.memory_migrations ( + user_id STRING + ) USING DELTA + `); + + await this.ensureEndpointExists(); + await this.waitForEndpointReadiness(); + await this.ensureIndexExists(); + await this.waitForIndexReadiness(); + } + + async insert( + vectors: number[][], + ids: string[], + payloads: Record[], + ): Promise { + await this.initialize(); + + const values = vectors.map((vector, index) => { + this.assertVectorDimension(vector, "Vector"); + const payload = payloads[index] || {}; + const sessionValues = this.extractSessionValues(payload); + return `( + ${formatSqlValue(ids[index])}, + ${formatSqlValue(vector)}, + ${formatSqlValue(JSON.stringify(payload))}, + ${formatSqlValue(lemmatizedText(payload))}, + ${formatSqlValue(sessionValues.user_id)}, + ${formatSqlValue(sessionValues.agent_id)}, + ${formatSqlValue(sessionValues.run_id)} + )`; + }); + + await this.executeSql(` + INSERT INTO ${this.fullTableName} + (memory_id, embedding, payload, text_lemmatized, user_id, agent_id, run_id) + VALUES ${values.join(", ")} + `); + this.requestIndexSync(); + } + + async search( + query: number[], + topK: number = 5, + filters?: SearchFilters, + ): Promise { + await this.initialize(); + this.assertVectorDimension(query, "Query"); + + if (this.queryType === "HYBRID") { + throw new Error( + "Databricks HYBRID search requires query_text, but search() only receives query vectors.", + ); + } + + const requestFilters = buildDatabricksServerFilters( + this.endpointType, + filters, + ); + + return this.queryIndex( + { + columns: ["memory_id", "payload"], + query_type: this.queryType, + query_vector: query, + ...requestFilters, + }, + filters, + topK, + ); + } + + async keywordSearch( + query: string, + topK: number = 5, + filters?: SearchFilters, + ): Promise { + await this.initialize(); + const requestFilters = buildDatabricksServerFilters( + this.endpointType, + filters, + ); + + // ponytail: `text_lemmatized` is synced so BM25 has clean text to score, but this query + // does not scope matching to it -- the `query_columns` parameter that would is Beta and + // gated behind the Full-Text Search public preview. Until it is GA the serialized + // `payload` column is also matchable, which adds noise (hashes, JSON keys) to ranking. + // Upgrade path: add `query_columns: ["text_lemmatized"]` once the preview ships. + return this.queryIndex( + { + columns: ["memory_id", "payload"], + query_type: "FULL_TEXT", + query_text: query, + ...requestFilters, + }, + filters, + topK, + ); + } + + async get(vectorId: string): Promise { + await this.initialize(); + + const rows = await this.executeSql(` + SELECT memory_id, payload + FROM ${this.fullTableName} + WHERE memory_id = ${formatSqlValue(vectorId)} + LIMIT 1 + `); + const row = rows[0]; + + if (!row) { + return null; + } + + return { + id: String(extractRowValue(row, ["memory_id"]) || vectorId), + payload: this.parsePayload(extractRowValue(row, ["payload"])), + }; + } + + async update( + vectorId: string, + vector: number[], + payload: Record, + ): Promise { + await this.initialize(); + this.assertVectorDimension(vector, "Vector"); + + // Session columns (user_id/agent_id/run_id) are the scope keys that search/list filter on and + // are written once at insert(). Mirror Python's `excluded_keys`: update() must never rewrite + // them from the payload -- a partial payload that omits these keys would otherwise SET them to + // NULL and hide the row from every session-scoped search and list(). + await this.executeSql(` + UPDATE ${this.fullTableName} + SET embedding = ${formatSqlValue(vector)}, + payload = ${formatSqlValue(JSON.stringify(payload || {}))}, + text_lemmatized = ${formatSqlValue(lemmatizedText(payload || {}))} + WHERE memory_id = ${formatSqlValue(vectorId)} + `); + this.requestIndexSync(); + } + + async delete(vectorId: string): Promise { + await this.initialize(); + + await this.executeSql(` + DELETE FROM ${this.fullTableName} + WHERE memory_id = ${formatSqlValue(vectorId)} + `); + this.requestIndexSync(); + } + + async deleteCol(): Promise { + await this.initialize(); + + try { + await this.httpClient.delete( + `/indexes/${encodeURIComponent(this.fullIndexName)}`, + ); + } catch (error: any) { + if (error?.response?.status !== 404) { + throw error; + } + } + + await this.executeSql(`DROP TABLE IF EXISTS ${this.fullTableName}`); + } + + async list( + filters?: SearchFilters, + topK: number = 100, + ): Promise<[VectorStoreResult[], number]> { + // `limit` below is interpolated directly into the SQL string (not a bound parameter, + // and not passed through formatSqlValue()), so a non-integer or non-positive topK must + // be rejected here rather than reaching the query -- otherwise a caller that skips + // TypeScript's compile-time check (e.g. anything passing user input straight through) + // could inject arbitrary SQL via the LIMIT clause. + if (!Number.isSafeInteger(topK) || topK <= 0) { + // isSafeInteger (not isInteger): an unsafe/huge integer like 1e21 stringifies as "1e+21", + // which is meaningless as a LIMIT and would slip past a plain integer check. + throw new Error( + `Databricks vector store: topK must be a positive integer, got ${topK}`, + ); + } + + await this.initialize(); + + // Push the SQL-translatable conjunctive filters (session keys) into a WHERE + // clause so a filtered list does not pull the whole table to the client. + // filterVector below still enforces the complete filter, so this clause is a + // best-effort narrowing: untranslatable filters ($or/$not/metadata) yield an + // empty clause and fall back to a bounded scan (see LIMIT below) + local filtering. + const conjunctiveFilters = collectConjunctiveDatabricksFilters(filters); + const clauses = conjunctiveFilters.map(([key, value]) => + buildStorageOptimizedDatabricksFilterClause(key, value), + ); + const whereClause = clauses + .filter((clause): clause is string => Boolean(clause)) + .join(" AND "); + + // The WHERE clause fully expresses `filters` (so filterVector below is a + // no-op) only if there's no $or/$not anywhere AND every entry translated. + const isFullyExpressed = + isFullyConjunctiveDatabricksFilter(filters) && + clauses.every((clause) => clause !== null); + // ponytail: 10k scan ceiling when the filter can't be fully pushed down; paginate if a user hits it. + const limit = isFullyExpressed ? topK : MAX_QUERY_RESULTS; + + const rows = await this.executeSql(` + SELECT memory_id, payload + FROM ${this.fullTableName} + ${whereClause ? `WHERE ${whereClause}` : ""} + LIMIT ${limit} + `); + const results: VectorStoreResult[] = []; + + for (const row of rows) { + const item: DatabricksVector = { + id: String(extractRowValue(row, ["memory_id"])), + payload: this.parsePayload(extractRowValue(row, ["payload"])), + }; + if (!this.filterVector(item, filters)) { + continue; + } + results.push({ + id: item.id, + payload: item.payload, + }); + } + + return [results.slice(0, topK), results.length]; + } + + async getUserId(): Promise { + await this.initialize(); + + const rows = await this.executeSql(` + SELECT user_id + FROM ${this.catalog}.${this.schema}.memory_migrations + LIMIT 1 + `); + const existing = rows[0] + ? extractRowValue(rows[0], ["user_id"]) + : undefined; + + if (typeof existing === "string" && existing.length > 0) { + return existing; + } + + const userId = + Math.random().toString(36).substring(2, 15) + + Math.random().toString(36).substring(2, 15); + await this.setUserId(userId); + return userId; + } + + async setUserId(userId: string): Promise { + await this.initialize(); + + await this.executeSql( + `DELETE FROM ${this.catalog}.${this.schema}.memory_migrations`, + ); + await this.executeSql(` + INSERT INTO ${this.catalog}.${this.schema}.memory_migrations (user_id) + VALUES (${formatSqlValue(userId)}) + `); + } + + private async ensureEndpointExists(): Promise { + try { + await this.httpClient.get( + `/endpoints/${encodeURIComponent(this.endpointName)}`, + ); + } catch (error: any) { + if (error?.response?.status !== 404) { + throw error; + } + + await this.httpClient.post("/endpoints", { + name: this.endpointName, + endpoint_type: this.endpointType, + }); + } + } + + private async ensureIndexExists(): Promise { + try { + await this.httpClient.get( + `/indexes/${encodeURIComponent(this.fullIndexName)}`, + ); + } catch (error: any) { + if (error?.response?.status !== 404) { + throw error; + } + + await this.httpClient.post("/indexes", { + name: this.fullIndexName, + endpoint_name: this.endpointName, + primary_key: "memory_id", + index_type: "DELTA_SYNC", + delta_sync_index_spec: { + source_table: this.fullTableName, + pipeline_type: this.pipelineType, + // ponytail: `payload` stays synced because query results are hydrated straight + // from it. Drop it here and hydrate by memory_id over SQL if index size bites. + columns_to_sync: + this.endpointType === "STANDARD" + ? [ + "memory_id", + "payload", + "text_lemmatized", + "user_id", + "agent_id", + "run_id", + ] + : undefined, + embedding_vector_columns: [ + { + name: "embedding", + embedding_dimension: this.dimension, + }, + ], + }, + }); + } + } + + private async ensureChangeDataFeedEnabled(): Promise { + if (this.endpointType !== "STANDARD") { + return; + } + + await this.executeSql(` + ALTER TABLE ${this.fullTableName} + SET TBLPROPERTIES ('delta.enableChangeDataFeed' = 'true') + `); + } + + private createHttpClient(): AxiosInstance { + const baseURL = `${this.workspaceUrl}/api/2.0/vector-search`; + + if (this.accessToken) { + return axios.create({ + baseURL, + headers: { + Authorization: `Bearer ${this.accessToken}`, + }, + }); + } + + if (!this.clientId || !this.clientSecret) { + throw new Error( + "Databricks vector store requires accessToken or clientId/clientSecret when httpClient is not provided.", + ); + } + + const baseClient = axios.create({ baseURL }); + return { + get: async (url: string, config?: Record) => + baseClient.get(url, await this.withOAuthHeaders("GET", url, config)), + post: async (url: string, data?: any, config?: Record) => + baseClient.post( + url, + data, + await this.withOAuthHeaders("POST", url, config), + ), + delete: async (url: string, config?: Record) => + baseClient.delete( + url, + await this.withOAuthHeaders("DELETE", url, config), + ), + } as DatabricksHttpClientLike as AxiosInstance; + } + + private async withOAuthHeaders( + method: "GET" | "POST" | "DELETE", + url: string, + config?: Record, + ): Promise> { + const token = await this.getOAuthAccessToken( + this.getOAuthAuthorizationDetails(method, url), + ); + return { + ...(config || {}), + headers: { + ...(config?.headers || {}), + Authorization: `Bearer ${token}`, + }, + }; + } + + private getOAuthAuthorizationDetails( + method: "GET" | "POST" | "DELETE", + url: string, + ): string | undefined { + if ( + method === "POST" && + (url.endsWith("/query") || url.endsWith("/query-next-page")) + ) { + return buildDatabricksAuthorizationDetails( + this.fullIndexName, + "ReadVectorIndex", + ); + } + + if ( + method === "POST" && + (url.endsWith("/sync") || + url.endsWith("/upsert-data") || + url.endsWith("/delete-data")) + ) { + return buildDatabricksAuthorizationDetails( + this.fullIndexName, + "WriteVectorIndex", + ); + } + + return undefined; + } + + private async getOAuthAccessToken( + authorizationDetails?: string, + ): Promise { + const cacheKey = authorizationDetails || "__management__"; + const cached = this.oauthTokens.get(cacheKey); + if (cached && Date.now() < cached.expiresAt - 60_000) { + return cached.accessToken; + } + + if (!this.clientId || !this.clientSecret) { + throw new Error( + "Databricks vector store requires clientId/clientSecret for OAuth token refresh.", + ); + } + + const formData = new URLSearchParams({ + grant_type: "client_credentials", + scope: "all-apis", + }); + if (authorizationDetails) { + formData.set("authorization_details", authorizationDetails); + } + + const response = await axios.post( + `${this.workspaceUrl}/oidc/v1/token`, + formData, + { + auth: { + username: this.clientId, + password: this.clientSecret, + }, + headers: { + "Content-Type": "application/x-www-form-urlencoded", + }, + }, + ); + + const token = response?.data?.access_token; + if (typeof token !== "string" || token.length === 0) { + throw new Error( + "Databricks OAuth token response did not include access_token.", + ); + } + + const expiresInSeconds = Number(response?.data?.expires_in ?? 3600); + this.oauthTokens.set(cacheKey, { + accessToken: token, + expiresAt: Date.now() + Math.max(1, expiresInSeconds) * 1000, + }); + return token; + } + + private async getSession(): Promise { + if (this.session) { + return this.session; + } + + if (!this._sessionPromise) { + this._sessionPromise = this.openSession(); + } + + try { + this.session = await this._sessionPromise; + } catch (error) { + // The first connection attempt failed (cold/auto-suspended warehouse, or a token that + // expired before first use). This runs before executeSql()'s own try/catch, so drop the + // rejected promise here too -- otherwise the next call replays it forever. + this.resetSession(); + throw error; + } + return this.session; + } + + /** + * `@databricks/sql` is an OPTIONAL peer, and `index.ts` re-exports this module eagerly, so a + * top-level `import` makes `import "mem0ai/oss"` throw MODULE_NOT_FOUND for every user who + * never installed the driver -- including everyone who uses a different vector store. Load it + * on the first SQL call instead, the way milvus.ts and baidu.ts load theirs. + * + * This MUST be a dynamic `import()`, never `require()`: tsup/esbuild rewrite `require()` in + * the published ESM bundle (`dist/oss/index.mjs`) into a `__require` shim that throws + * `Dynamic require of "..." is not supported`, so every ESM consumer would hit a dead + * provider even with the driver installed. + */ + private async getSqlModule(): Promise<{ + DBSQLClient: new () => DatabricksSqlClientLike; + }> { + if (!this.sqlModulePromise) { + this.sqlModulePromise = import("@databricks/sql").then( + (mod) => + mod as unknown as { DBSQLClient: new () => DatabricksSqlClientLike }, + (err) => { + // Let a later call retry rather than caching the rejection forever. + this.sqlModulePromise = undefined; + const detail = err instanceof Error ? err.message : String(err); + throw new Error( + "The '@databricks/sql' package is required to use the Databricks vector store. " + + `Install it with: npm install @databricks/sql (original error: ${detail})`, + ); + }, + ); + } + return this.sqlModulePromise; + } + + private async getSqlClient(): Promise { + if (!this.sqlClient) { + const { DBSQLClient } = await this.getSqlModule(); + this.sqlClient = new DBSQLClient(); + } + return this.sqlClient; + } + + private async openSession(): Promise { + const sqlClient = await this.getSqlClient(); + const connected = await sqlClient.connect(this.buildSqlConnectionOptions()); + return connected.openSession(); + } + + private buildSqlConnectionOptions(): Record { + const base = { + host: this.host, + path: this.httpPath, + } as Record; + + if (this.clientId && this.clientSecret) { + return { + ...base, + authType: "databricks-oauth", + oauthClientId: this.clientId, + oauthClientSecret: this.clientSecret, + }; + } + + if (!this.accessToken) { + throw new Error( + "Databricks vector store requires accessToken or clientId/clientSecret for SQL connections.", + ); + } + + return { + ...base, + token: this.accessToken, + }; + } + + /** + * Drop the cached session/session-promise so the next `getSession()` opens a fresh one. + * Called after any SQL failure below -- a stale session (expired token, dropped socket) + * would otherwise keep being handed out and keep failing forever. + */ + private resetSession(): void { + this.session = undefined; + this._sessionPromise = undefined; + } + + private async executeSql( + statement: string, + ): Promise>> { + const session = await this.getSession(); + + let operation; + try { + operation = await session.executeStatement(statement); + } catch (error) { + // ponytail: no retry here -- statement could be a non-idempotent write, so we only + // clear the bad session and let the caller's own retry (if any) reconnect. + this.resetSession(); + throw error; + } + + try { + return await operation.fetchAll(); + } catch (error) { + this.resetSession(); + throw error; + } finally { + if (typeof operation.close === "function") { + await operation.close(); + } + } + } + + /** + * A TRIGGERED pipeline only ingests new rows when it is explicitly synced, so every write + * must request one. Writers fire the sync and return; only readers wait for it to land. + * A sync already in flight absorbs later writes -- deleteAll()'s N sequential deletes + * collapse into far fewer than N pipeline runs. + */ + private requestIndexSync(): void { + if (this.pipelineType !== "TRIGGERED") { + return; + } + + this.indexSyncQueued = true; + if (this.indexSyncRunning) { + // A drain is live and has not yet re-checked the queue, so it will pick this up. + return; + } + this.indexSyncRunning = true; + this.indexSyncWait = this.drainIndexSync(); + } + + private async drainIndexSync(): Promise { + try { + // Re-checking the flag and clearing `indexSyncRunning` both happen with no `await` + // between them, so a write can never land on a drain that has stopped looking. + while (this.indexSyncQueued) { + this.indexSyncQueued = false; + await this.httpClient.post( + `/indexes/${encodeURIComponent(this.fullIndexName)}/sync`, + ); + await this.waitForIndexReadiness(); + } + this.indexSyncError = null; + } catch (error) { + // Surfaced to the next reader: the writer that scheduled this has already returned. + this.indexSyncError = error; + } finally { + this.indexSyncRunning = false; + } + } + + private async awaitIndexSync(): Promise { + while (this.indexSyncWait) { + const wait = this.indexSyncWait; + await wait; + // Only retire the promise we actually awaited; a newer drain may have replaced it, + // and that one still owes us its rows. Clearing unconditionally would drop it. + if (this.indexSyncWait === wait) { + this.indexSyncWait = null; + } + } + if (this.indexSyncError) { + const error = this.indexSyncError; + this.indexSyncError = null; + throw error; + } + } + + private async waitForEndpointReadiness(): Promise { + const deadline = Date.now() + this.syncTimeoutMs; + + while (Date.now() <= deadline) { + const response = await this.httpClient.get( + `/endpoints/${encodeURIComponent(this.endpointName)}`, + ); + const state = + response?.data?.endpoint_status?.state ?? response?.data?.state; + + if (state === "ONLINE") { + return; + } + + if (typeof state !== "string") { + throw new Error( + "Databricks endpoint status did not report a state during initialization.", + ); + } + + if (this.syncPollIntervalMs > 0) { + await new Promise((resolve) => + setTimeout(resolve, this.syncPollIntervalMs), + ); + } + } + + throw new Error( + `Timed out waiting for Databricks endpoint ${this.endpointName} to become ready.`, + ); + } + + private async waitForIndexReadiness(): Promise { + const deadline = Date.now() + this.syncTimeoutMs; + + while (Date.now() <= deadline) { + const response = await this.httpClient.get( + `/indexes/${encodeURIComponent(this.fullIndexName)}`, + ); + const ready = response?.data?.status?.ready; + + if (ready === true) { + return; + } + + if (ready !== false) { + throw new Error( + "Databricks index status did not report a readiness flag after sync.", + ); + } + + if (this.syncPollIntervalMs > 0) { + await new Promise((resolve) => + setTimeout(resolve, this.syncPollIntervalMs), + ); + } + } + + throw new Error( + `Timed out waiting for Databricks index ${this.fullIndexName} to become ready after sync.`, + ); + } + + private shouldPaginateForLocalFiltering(filters?: SearchFilters): boolean { + if (!filters || Object.keys(filters).length === 0) { + return false; + } + + for (const [key, value] of Object.entries(filters)) { + if (key === "$and") { + if (!Array.isArray(value)) { + return true; + } + if ( + value.some( + (entry) => + !isPlainObject(entry) || + this.shouldPaginateForLocalFiltering(entry as SearchFilters), + ) + ) { + return true; + } + continue; + } + + if (key === "$or" || key === "$not") { + return true; + } + + const translated = + this.endpointType === "STORAGE_OPTIMIZED" + ? buildStorageOptimizedDatabricksFilterClause(key, value) + : buildSimpleDatabricksFilter(key, value); + + if (!translated) { + return true; + } + } + + return false; + } + + private extractNextPageToken(responseData: any): string | undefined { + const token = + responseData?.next_page_token ?? responseData?.result?.next_page_token; + return typeof token === "string" && token.length > 0 ? token : undefined; + } + + private async queryIndex( + requestBody: Record, + filters: SearchFilters | undefined, + topK: number, + ): Promise { + // Read-after-write: writers only request the sync, the reader is what blocks on it. + // get()/list() skip this -- they read the source Delta table, which is always current. + await this.awaitIndexSync(); + const requiresLocalFilteringPagination = + this.shouldPaginateForLocalFiltering(filters); + const queryResultCap = + requestBody.query_type === "FULL_TEXT" + ? MAX_FULL_TEXT_RESULTS + : MAX_QUERY_RESULTS; + const response = await this.httpClient.post( + `/indexes/${encodeURIComponent(this.fullIndexName)}/query`, + { + ...requestBody, + num_results: Math.min( + requiresLocalFilteringPagination + ? queryResultCap + : Math.max(topK, DEFAULT_PAGE_SIZE), + queryResultCap, + ), + }, + ); + + const results = this.normalizeQueryResults( + response.data, + ["memory_id", "payload"], + filters, + ); + if (results.length >= topK) { + return results.slice(0, topK); + } + + let nextPageToken = this.extractNextPageToken(response.data); + while (nextPageToken && results.length < topK) { + const nextPage = await this.httpClient.post( + `/indexes/${encodeURIComponent(this.fullIndexName)}/query-next-page`, + { + endpoint_name: this.endpointName, + page_token: nextPageToken, + }, + ); + results.push( + ...this.normalizeQueryResults( + nextPage.data, + ["memory_id", "payload"], + filters, + ), + ); + nextPageToken = this.extractNextPageToken(nextPage.data); + } + + return results.slice(0, topK); + } + + private normalizeQueryResults( + responseData: any, + fallbackColumns: string[], + filters: SearchFilters | undefined, + ): VectorStoreResult[] { + const resultData = responseData?.result || responseData; + const dataArray = Array.isArray(resultData?.data_array) + ? resultData.data_array + : []; + const manifestColumns = Array.isArray(resultData?.manifest?.columns) + ? resultData.manifest.columns.map((column: any) => + typeof column === "string" ? column : column.name, + ) + : fallbackColumns; + const results: VectorStoreResult[] = []; + + for (const row of dataArray) { + const rowDict = this.rowToObject(row, manifestColumns); + const item: DatabricksVector = { + id: String(rowDict.memory_id || rowDict.id || rowDict.vector_id), + payload: this.parsePayload(rowDict.payload), + }; + if (!this.filterVector(item, filters)) { + continue; + } + const rawScore = + rowDict.score ?? + (Array.isArray(row) && row.length > manifestColumns.length + ? row[row.length - 1] + : undefined); + results.push({ + id: item.id, + payload: item.payload, + score: + rawScore === undefined || rawScore === null + ? undefined + : Number(rawScore), + }); + } + + return results; + } + + private rowToObject(row: any, columns: string[]): Record { + if (!Array.isArray(row)) { + return row || {}; + } + + return Object.fromEntries( + columns.map((column, index) => [column, row[index]]), + ); + } + + private parsePayload(rawValue: any): Record { + if (typeof rawValue === "string") { + try { + const parsed = JSON.parse(rawValue); + if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) { + return parsed; + } + } catch { + return {}; + } + } + + if (rawValue && typeof rawValue === "object" && !Array.isArray(rawValue)) { + return rawValue; + } + + return {}; + } + + private extractSessionValues(payload: Record): { + user_id: any; + agent_id: any; + run_id: any; + } { + return { + user_id: payload.user_id, + agent_id: payload.agent_id, + run_id: payload.run_id, + }; + } + + private assertVectorDimension(vector: number[], label: string): void { + if (vector.length !== this.dimension) { + throw new Error( + `${label} dimension mismatch. Expected ${this.dimension}, got ${vector.length}`, + ); + } + for (const value of vector) { + if (!Number.isFinite(value)) { + throw new Error( + `${label} values must be finite numbers for Databricks vector search.`, + ); + } + } + } + + private matchFieldCondition( + vector: DatabricksVector, + key: string, + value: any, + ): boolean { + const fieldValue = key === "memory_id" ? vector.id : vector.payload[key]; + + if (typeof value !== "object" || value === null) { + if (value === "*") { + return true; + } + return fieldValue === value; + } + + if (Array.isArray(value)) { + return value.includes(fieldValue); + } + + let sawOperator = false; + + if ("eq" in value) { + sawOperator = true; + if (fieldValue !== value.eq) { + return false; + } + } + if ("ne" in value) { + sawOperator = true; + if (fieldValue === value.ne) { + return false; + } + } + if ("gt" in value) { + sawOperator = true; + if (!(fieldValue > value.gt)) { + return false; + } + } + if ("gte" in value) { + sawOperator = true; + if (!(fieldValue >= value.gte)) { + return false; + } + } + if ("lt" in value) { + sawOperator = true; + if (!(fieldValue < value.lt)) { + return false; + } + } + if ("lte" in value) { + sawOperator = true; + if (!(fieldValue <= value.lte)) { + return false; + } + } + if ("in" in value) { + sawOperator = true; + if (!Array.isArray(value.in) || !value.in.includes(fieldValue)) { + return false; + } + } + if ("nin" in value) { + sawOperator = true; + if (Array.isArray(value.nin) && value.nin.includes(fieldValue)) { + return false; + } + } + if ("contains" in value) { + sawOperator = true; + if ( + typeof fieldValue !== "string" || + !fieldValue.includes(value.contains) + ) { + return false; + } + } + if ("icontains" in value) { + sawOperator = true; + if ( + typeof fieldValue !== "string" || + !fieldValue.toLowerCase().includes(value.icontains.toLowerCase()) + ) { + return false; + } + } + + return sawOperator ? true : fieldValue === value; + } + + private filterVector( + vector: DatabricksVector, + filters?: SearchFilters, + ): boolean { + if (!filters || Object.keys(filters).length === 0) { + return true; + } + + const keyMap: Record = { + $and: "AND", + $or: "OR", + $not: "NOT", + }; + const normalized: Record = {}; + for (const [key, value] of Object.entries(filters)) { + const normalizedKey = keyMap[key] || key; + if (!(normalizedKey in normalized)) { + normalized[normalizedKey] = value; + } + } + + for (const [key, value] of Object.entries(normalized)) { + if (key === "AND") { + if (!Array.isArray(value)) { + throw new Error( + `AND filter value must be a list of filter dicts, got ${typeof value}`, + ); + } + if ( + !value.every((entry: SearchFilters) => + this.filterVector(vector, entry), + ) + ) { + return false; + } + } else if (key === "OR") { + if (!Array.isArray(value)) { + throw new Error( + `OR filter value must be a list of filter dicts, got ${typeof value}`, + ); + } + if ( + !value.some((entry: SearchFilters) => + this.filterVector(vector, entry), + ) + ) { + return false; + } + } else if (key === "NOT") { + if (!Array.isArray(value)) { + throw new Error( + `NOT filter value must be a list of filter dicts, got ${typeof value}`, + ); + } + if ( + !value.every( + (entry: SearchFilters) => !this.filterVector(vector, entry), + ) + ) { + return false; + } + } else if (!this.matchFieldCondition(vector, key, value)) { + return false; + } + } + + return true; + } +} diff --git a/mem0-ts/src/oss/src/vector_stores/neptune_analytics.ts b/mem0-ts/src/oss/src/vector_stores/neptune_analytics.ts index 9349af63f..79f6e5c9a 100644 --- a/mem0-ts/src/oss/src/vector_stores/neptune_analytics.ts +++ b/mem0-ts/src/oss/src/vector_stores/neptune_analytics.ts @@ -1,10 +1,10 @@ -import { - ExecuteQueryCommand, - NeptuneGraphClient, -} from "@aws-sdk/client-neptune-graph"; import { VectorStore } from "./base"; import { SearchFilters, VectorStoreConfig, VectorStoreResult } from "../types"; +/** + * The `@aws-sdk/client-neptune-graph` dependency is loaded on first use via dynamic + * `import()` so the package stays optional (mirrors `aws_bedrock.ts`). + */ interface NeptuneAnalyticsConfig extends VectorStoreConfig { graphIdentifier?: string; endpoint?: string; @@ -14,7 +14,14 @@ interface NeptuneAnalyticsConfig extends VectorStoreConfig { } interface NeptuneGraphClientLike { - send(command: ExecuteQueryCommand): Promise; + send(command: any): Promise; +} + +interface NeptuneSDK { + NeptuneGraphClient: new ( + config: Record, + ) => NeptuneGraphClientLike; + ExecuteQueryCommand: new (input: Record) => any; } interface NeptuneExecuteQueryOutput { @@ -33,7 +40,10 @@ interface WhereClauseResult { } export class NeptuneAnalyticsVectorStore implements VectorStore { - private readonly client: NeptuneGraphClientLike; + private clientConfig: Record; + private clientOverride?: NeptuneGraphClientLike; + private sdkPromise?: Promise; + private clientPromise?: Promise; private readonly graphIdentifier: string; private readonly collectionName: string; private readonly collectionLabel: string; @@ -54,8 +64,8 @@ export class NeptuneAnalyticsVectorStore implements VectorStore { this.userLabelExpr = this.escapeLabel(this.userLabel); this.userNodeId = "mem0-user"; this.dimension = config.dimension || 1536; - this.client = - config.client || new NeptuneGraphClient(this.buildClientConfig(config)); + this.clientConfig = this.buildClientConfig(config); + this.clientOverride = config.client; void this.initialize().catch(console.error); } @@ -171,6 +181,18 @@ export class NeptuneAnalyticsVectorStore implements VectorStore { const hasPayload = !!payload && Object.keys(payload).length > 0; const hasVector = vector.length > 0; + // ponytail: a combined update writes the payload before the embedding, and Neptune's vector + // index isn't transactional -- if the upsert below fails, the new payload would otherwise be + // left committed against the stale embedding (searches would match the old vector but return + // the new metadata). Capture the prior node so a failed upsert can be restored; this is + // best-effort compensation, not a rollback. Only needed when both writes happen -- a + // payload-only or vector-only update can't desync. + // The restore assumes a single writer per vectorId -- concurrent updates to the same node can + // interleave and clobber each other's compensation. AWS advises against concurrent same-vertex + // writes to the Neptune Analytics vector index for exactly this reason. + const priorResult = + hasPayload && hasVector ? await this.get(vectorId) : null; + if (hasPayload) { const properties = this.buildStoredPayload(payload); await this.executeQuery( @@ -187,20 +209,45 @@ export class NeptuneAnalyticsVectorStore implements VectorStore { } if (hasVector) { - const updateResults = await this.executeQuery( - ` - MATCH (n:${this.collectionLabelExpr} {\`~id\`: $vectorId}) - WITH n, $embedding AS embedding - CALL neptune.algo.vectors.upsert(n, embedding) - YIELD success - RETURN success - `, - { - vectorId, - embedding: vector, - }, - ); - this.assertSuccessfulResults(updateResults, "Update"); + try { + const updateResults = await this.executeQuery( + ` + MATCH (n:${this.collectionLabelExpr} {\`~id\`: $vectorId}) + WITH n, $embedding AS embedding + CALL neptune.algo.vectors.upsert(n, embedding) + YIELD success + RETURN success + `, + { + vectorId, + embedding: vector, + }, + ); + this.assertSuccessfulResults(updateResults, "Update"); + } catch (error) { + if (priorResult) { + try { + await this.executeQuery( + ` + MATCH (n:${this.collectionLabelExpr} {\`~id\`: $vectorId}) + SET n = $properties + RETURN n + `, + { + vectorId, + properties: priorResult.payload, + }, + ); + } catch (restoreError) { + // Do not mask the original failure with a compensation failure. + console.error( + "Neptune Analytics: failed to restore prior payload after a failed update upsert", + restoreError, + ); + } + } + throw error; + } } } @@ -985,12 +1032,58 @@ export class NeptuneAnalyticsVectorStore implements VectorStore { } } + /** + * Load the optional AWS SDK on first use. + * + * This MUST be a dynamic `import()`, never `require()`: tsup/esbuild rewrite + * `require()` in the published ESM bundle (`dist/oss/index.mjs`) into a + * `__require` shim that throws `Dynamic require of "..." is not supported`, + * so every ESM consumer would hit a dead provider even with the SDK installed. + */ + private async getSDK(): Promise { + if (!this.sdkPromise) { + this.sdkPromise = import("@aws-sdk/client-neptune-graph").then( + (sdk) => sdk as unknown as NeptuneSDK, + (err) => { + // Let a later call retry rather than caching the rejection forever. + this.sdkPromise = undefined; + const detail = err instanceof Error ? err.message : String(err); + throw new Error( + "The '@aws-sdk/client-neptune-graph' package is required to use the Neptune Analytics vector store. " + + `Install it with: npm install @aws-sdk/client-neptune-graph (original error: ${detail})`, + ); + }, + ); + } + return this.sdkPromise; + } + + /** Memoized Neptune client; an injected `config.client` short-circuits the SDK. */ + private async getClient(): Promise { + if (this.clientOverride) return this.clientOverride; + if (!this.clientPromise) { + this.clientPromise = this.getSDK() + .then( + ({ NeptuneGraphClient }) => new NeptuneGraphClient(this.clientConfig), + ) + .catch((err) => { + // Mirror getSDK(): drop the rejected promise so a later call retries rather than + // replaying a cached rejection forever (a rejected promise is still truthy, so the + // `!this.clientPromise` guard above would otherwise never re-enter). + this.clientPromise = undefined; + throw err; + }); + } + return this.clientPromise; + } + private async executeQuery( queryString: string, parameters: Record = {}, ): Promise { - const response = await this.client.send( - new ExecuteQueryCommand({ + const [client, sdk] = await Promise.all([this.getClient(), this.getSDK()]); + const response = await client.send( + new sdk.ExecuteQueryCommand({ graphIdentifier: this.graphIdentifier, language: "OPEN_CYPHER", queryString, diff --git a/mem0-ts/src/oss/tests/config-manager.test.ts b/mem0-ts/src/oss/tests/config-manager.test.ts index 331384465..7c1856ad8 100644 --- a/mem0-ts/src/oss/tests/config-manager.test.ts +++ b/mem0-ts/src/oss/tests/config-manager.test.ts @@ -522,6 +522,34 @@ describe("ConfigManager", () => { expect(cfg.embedder.config).not.toHaveProperty("embedding_dims"); }); }); + + describe("mergeConfig - vector store provider normalization", () => { + const baseLlm = { provider: "openai", config: { apiKey: "test-key" } }; + const baseEmbedder = { provider: "openai", config: { apiKey: "test-key" } }; + + it.each(["Memory", "DATABRICKS", "QdRaNt"])( + "lowercases the vector store provider %p", + (provider) => { + const config = ConfigManager.mergeConfig({ + embedder: baseEmbedder, + vectorStore: { provider, config: { collectionName: "test" } }, + llm: baseLlm, + }); + + expect(config.vectorStore.provider).toBe(provider.toLowerCase()); + }, + ); + + it("still falls back to the default provider when none is given", () => { + const config = ConfigManager.mergeConfig({ + embedder: baseEmbedder, + vectorStore: { config: { collectionName: "test" } } as any, + llm: baseLlm, + }); + + expect(config.vectorStore.provider).toBe("memory"); + }); + }); }); // ───────────────────────────────────────────────────────────────────────── diff --git a/mem0-ts/src/oss/tests/factory.unit.test.ts b/mem0-ts/src/oss/tests/factory.unit.test.ts index 0a3994303..706c178b3 100644 --- a/mem0-ts/src/oss/tests/factory.unit.test.ts +++ b/mem0-ts/src/oss/tests/factory.unit.test.ts @@ -183,6 +183,11 @@ jest.mock("../src/vector_stores/pgvector", () => ({ .fn() .mockImplementation((config) => ({ type: "pgvector", config })), })); +jest.mock("../src/vector_stores/databricks", () => ({ + DatabricksVectorStore: jest + .fn() + .mockImplementation((config) => ({ type: "databricks", config })), +})); jest.mock("../src/vector_stores/neptune_analytics", () => ({ NeptuneAnalyticsVectorStore: jest.fn().mockImplementation((config) => ({ type: "neptune-analytics", @@ -346,6 +351,7 @@ describe("VectorStoreFactory", () => { ["vectorize"], ["azure-ai-search"], ["pgvector"], + ["databricks"], ["neptune"], ["neptune-analytics"], ["upstash_vector"], diff --git a/mem0-ts/src/oss/tests/vector-stores-compat.test.ts b/mem0-ts/src/oss/tests/vector-stores-compat.test.ts index 6ab87294f..ee2438c5e 100644 --- a/mem0-ts/src/oss/tests/vector-stores-compat.test.ts +++ b/mem0-ts/src/oss/tests/vector-stores-compat.test.ts @@ -719,6 +719,1511 @@ describe("AzureAISearch – backward compat with mocked client", () => { }); }); +// ─────────────────────────────────────────────────────────────────────────── +// 6. Databricks — mock SQL + REST clients, test interface + idempotent init +// ─────────────────────────────────────────────────────────────────────────── +describe("Databricks – backward compat with mocked clients", () => { + let DatabricksVectorStore: any; + + beforeEach(() => { + jest.resetModules(); + + jest.doMock("@databricks/sql", () => { + let migrationUserId: string | undefined; + + const executeStatement = jest + .fn() + .mockImplementation(async (sql: string) => { + const normalized = sql.replace(/\s+/g, " ").trim(); + + if ( + normalized.startsWith("CREATE SCHEMA IF NOT EXISTS") || + normalized.startsWith("CREATE TABLE IF NOT EXISTS") || + normalized.startsWith("ALTER TABLE main.default.memories") + ) { + return { + fetchAll: jest.fn().mockResolvedValue([]), + close: jest.fn().mockResolvedValue(undefined), + }; + } + + if ( + normalized.startsWith( + "SELECT user_id FROM main.default.memory_migrations LIMIT 1", + ) + ) { + return { + fetchAll: jest + .fn() + .mockResolvedValue( + migrationUserId ? [{ user_id: migrationUserId }] : [], + ), + close: jest.fn().mockResolvedValue(undefined), + }; + } + + if ( + normalized.startsWith("DELETE FROM main.default.memory_migrations") + ) { + migrationUserId = undefined; + return { + fetchAll: jest.fn().mockResolvedValue([]), + close: jest.fn().mockResolvedValue(undefined), + }; + } + + if ( + normalized.startsWith( + "INSERT INTO main.default.memory_migrations (user_id)", + ) + ) { + const match = normalized.match(/VALUES \('([^']*)'\)/); + migrationUserId = match ? match[1] : undefined; + return { + fetchAll: jest.fn().mockResolvedValue([]), + close: jest.fn().mockResolvedValue(undefined), + }; + } + + return { + fetchAll: jest.fn().mockResolvedValue([]), + close: jest.fn().mockResolvedValue(undefined), + }; + }); + + const session = { + executeStatement, + close: jest.fn().mockResolvedValue(undefined), + }; + const client = { + connect: jest.fn().mockResolvedValue(undefined), + openSession: jest.fn().mockResolvedValue(session), + close: jest.fn().mockResolvedValue(undefined), + }; + + client.connect.mockResolvedValue(client); + + return { + DBSQLClient: jest.fn().mockImplementation(() => client), + __mockClient: client, + __mockSession: session, + }; + }); + + jest.doMock("axios", () => { + let endpointExists = false; + let endpointPendingReadyResponses = 0; + let indexExists = false; + let indexPendingReadyResponses = 0; + let syncPendingReadyResponses = 0; + let indexNeverReady = false; + let pagedQueryResponses: any[] = []; + + const defaultQueryResponse = { + result: { + manifest: { + columns: [{ name: "memory_id" }, { name: "payload" }], + }, + data_array: [ + ["id-1", JSON.stringify({ user_id: "u1", topic: "alpha" }), 0.98], + ["id-2", JSON.stringify({ user_id: "u2", topic: "beta" }), 0.75], + ], + }, + }; + + const httpClient = { + get: jest.fn().mockImplementation(async (url: string) => { + if (url === "/endpoints/mem0_vector_search") { + if (!endpointExists) { + const error: any = new Error("missing endpoint"); + error.response = { status: 404 }; + throw error; + } + if (endpointPendingReadyResponses > 0) { + endpointPendingReadyResponses -= 1; + return { + data: { + name: "mem0_vector_search", + endpoint_status: { state: "PROVISIONING" }, + }, + }; + } + return { + data: { + name: "mem0_vector_search", + endpoint_status: { state: "ONLINE" }, + }, + }; + } + + if (url === "/indexes/main.default.memories") { + if (!indexExists) { + const error: any = new Error("missing index"); + error.response = { status: 404 }; + throw error; + } + if (indexNeverReady) { + return { + data: { + name: "main.default.memories", + status: { ready: false }, + }, + }; + } + if (indexPendingReadyResponses > 0) { + indexPendingReadyResponses -= 1; + return { + data: { + name: "main.default.memories", + status: { ready: false }, + }, + }; + } + if (syncPendingReadyResponses > 0) { + syncPendingReadyResponses -= 1; + return { + data: { + name: "main.default.memories", + status: { ready: false }, + }, + }; + } + return { + data: { + name: "main.default.memories", + status: { ready: true }, + }, + }; + } + + throw new Error(`Unexpected Databricks GET ${url}`); + }), + post: jest.fn().mockImplementation(async (url: string, body: any) => { + if (url === "/endpoints") { + endpointExists = true; + endpointPendingReadyResponses = 1; + return { data: { endpoint: body.name } }; + } + + if (url === "/indexes") { + indexExists = true; + indexPendingReadyResponses = 1; + return { data: { index: body.name } }; + } + + if (url === "/indexes/main.default.memories/sync") { + syncPendingReadyResponses = 1; + return { data: { status: "queued" } }; + } + + if (url === "/indexes/main.default.memories/query") { + return { + data: + pagedQueryResponses.shift() ?? + JSON.parse(JSON.stringify(defaultQueryResponse)), + }; + } + + if (url === "/indexes/main.default.memories/query-next-page") { + return { + data: + pagedQueryResponses.shift() ?? + JSON.parse(JSON.stringify(defaultQueryResponse)), + }; + } + + throw new Error(`Unexpected Databricks POST ${url}`); + }), + delete: jest.fn().mockImplementation(async (url: string) => { + if (url === "/indexes/main.default.memories") { + indexExists = false; + return { data: {} }; + } + throw new Error(`Unexpected Databricks DELETE ${url}`); + }), + }; + + const create = jest.fn().mockReturnValue(httpClient); + const authPost = jest.fn().mockResolvedValue({ + data: { access_token: "oauth-token", expires_in: 3600 }, + }); + + return { + __esModule: true, + default: { create, post: authPost }, + create, + post: authPost, + __mockHttpClient: httpClient, + __setPagedQueryResponses: (responses: any[]) => { + pagedQueryResponses = responses.map((response) => + JSON.parse(JSON.stringify(response)), + ); + }, + __setIndexNeverReady: (value: boolean) => { + indexNeverReady = value; + }, + __mockAuthPost: authPost, + }; + }); + + DatabricksVectorStore = + require("../src/vector_stores/databricks").DatabricksVectorStore; + }); + + afterEach(() => { + jest.restoreAllMocks(); + jest.resetModules(); + }); + + it("implements full VectorStore interface", () => { + const store = new DatabricksVectorStore({ + workspaceUrl: "https://workspace.databricks.com", + httpPath: "/sql/1.0/warehouses/test", + accessToken: "dapi-test", + catalog: "main", + schema: "default", + collectionName: "memories", + dimension: 3, + syncPollIntervalMs: 0, + }); + expect(typeof store.insert).toBe("function"); + expect(typeof store.search).toBe("function"); + expect(typeof store.get).toBe("function"); + expect(typeof store.update).toBe("function"); + expect(typeof store.delete).toBe("function"); + expect(typeof store.deleteCol).toBe("function"); + expect(typeof store.list).toBe("function"); + expect(typeof store.getUserId).toBe("function"); + expect(typeof store.setUserId).toBe("function"); + expect(typeof store.initialize).toBe("function"); + }); + + it("initialize() is idempotent (same promise returned)", async () => { + const databricksSql = require("@databricks/sql"); + const axiosModule = require("axios"); + const store = new DatabricksVectorStore({ + workspaceUrl: "https://workspace.databricks.com", + httpPath: "/sql/1.0/warehouses/test", + accessToken: "dapi-test", + catalog: "main", + schema: "default", + collectionName: "memories", + dimension: 3, + syncPollIntervalMs: 0, + }); + + const p1 = store.initialize(); + const p2 = store.initialize(); + const p3 = store.initialize(); + await Promise.all([p1, p2, p3]); + + const clientInstance = databricksSql.DBSQLClient.mock.results[0].value; + const session = databricksSql.__mockSession; + const httpClient = axiosModule.__mockHttpClient; + expect(clientInstance.connect).toHaveBeenCalledTimes(1); + expect(clientInstance.openSession).toHaveBeenCalledTimes(1); + expect( + httpClient.get.mock.calls.filter( + ([url]: [string]) => url === "/endpoints/mem0_vector_search", + ).length, + ).toBeGreaterThan(1); + expect( + httpClient.get.mock.calls.filter( + ([url]: [string]) => url === "/indexes/main.default.memories", + ).length, + ).toBeGreaterThan(1); + expect(httpClient.post).toHaveBeenCalledWith("/endpoints", { + name: "mem0_vector_search", + endpoint_type: "STANDARD", + }); + expect(httpClient.post).toHaveBeenCalledWith("/indexes", { + name: "main.default.memories", + endpoint_name: "mem0_vector_search", + primary_key: "memory_id", + index_type: "DELTA_SYNC", + delta_sync_index_spec: { + source_table: "main.default.memories", + pipeline_type: "TRIGGERED", + columns_to_sync: [ + "memory_id", + "payload", + "text_lemmatized", + "user_id", + "agent_id", + "run_id", + ], + embedding_vector_columns: [ + { + name: "embedding", + embedding_dimension: 3, + }, + ], + }, + }); + expect( + session.executeStatement.mock.calls.some( + ([sql]: [string]) => + sql.includes("ALTER TABLE main.default.memories") && + sql.includes("delta.enableChangeDataFeed"), + ), + ).toBe(true); + }); + + it("retries loading @databricks/sql after a failed dynamic import instead of caching the rejection", async () => { + let shouldFail = true; + jest.doMock("@databricks/sql", () => { + if (shouldFail) { + throw new Error("Cannot find module '@databricks/sql'"); + } + const session = { + executeStatement: jest.fn().mockResolvedValue({ + fetchAll: jest.fn().mockResolvedValue([]), + close: jest.fn().mockResolvedValue(undefined), + }), + close: jest.fn().mockResolvedValue(undefined), + }; + const client = { + connect: jest.fn().mockResolvedValue(undefined), + openSession: jest.fn().mockResolvedValue(session), + close: jest.fn().mockResolvedValue(undefined), + }; + client.connect.mockResolvedValue(client); + return { DBSQLClient: jest.fn().mockImplementation(() => client) }; + }); + + const store = new DatabricksVectorStore({ + workspaceUrl: "https://workspace.databricks.com", + httpPath: "/sql/1.0/warehouses/test", + accessToken: "dapi-test", + catalog: "main", + schema: "default", + collectionName: "memories", + dimension: 3, + syncPollIntervalMs: 0, + }); + + await expect((store as any).getSqlModule()).rejects.toThrow( + "The '@databricks/sql' package is required to use the Databricks vector store. " + + "Install it with: npm install @databricks/sql (original error: Cannot find module '@databricks/sql')", + ); + + shouldFail = false; + const sqlModule = await (store as any).getSqlModule(); + expect(typeof sqlModule.DBSQLClient).toBe("function"); + }); + + it("executeSql(): reconnects on the next call after a session failure instead of reusing a dead session", async () => { + const databricksSql = require("@databricks/sql"); + const store = new DatabricksVectorStore({ + workspaceUrl: "https://workspace.databricks.com", + httpPath: "/sql/1.0/warehouses/test", + accessToken: "dapi-test", + catalog: "main", + schema: "default", + collectionName: "memories", + dimension: 3, + syncPollIntervalMs: 0, + }); + + await store.initialize(); + const session = databricksSql.__mockSession; + const openSessionSpy = jest.spyOn(store as any, "openSession"); + + session.executeStatement.mockRejectedValueOnce( + new Error("session expired"), + ); + await expect((store as any).executeSql("SELECT 1")).rejects.toThrow( + "session expired", + ); + expect(openSessionSpy).not.toHaveBeenCalled(); + + await (store as any).executeSql("SELECT 1"); + expect(openSessionSpy).toHaveBeenCalledTimes(1); + }); + + it("getSession(): retries after a failed connection attempt instead of caching the rejection", async () => { + const databricksSql = require("@databricks/sql"); + const store = new DatabricksVectorStore({ + workspaceUrl: "https://workspace.databricks.com", + httpPath: "/sql/1.0/warehouses/test", + accessToken: "dapi-test", + catalog: "main", + schema: "default", + collectionName: "memories", + dimension: 3, + syncPollIntervalMs: 0, + }); + + await store.initialize(); + // Simulate the warehouse dropping so the next getSession() must reconnect from scratch + // (this clears both the cached session and any in-flight session promise). + (store as any).resetSession(); + + let attempts = 0; + jest.spyOn(store as any, "openSession").mockImplementation(async () => { + attempts++; + if (attempts === 1) { + throw new Error("warehouse cold start"); + } + return databricksSql.__mockSession; + }); + + // First connection attempt fails (e.g. a cold/auto-suspended warehouse at startup)... + await expect((store as any).getSession()).rejects.toThrow( + "warehouse cold start", + ); + + // ...the next call must open a fresh session rather than replay the rejected promise. + // Without getSession()'s catch-and-reset, openSession would never be called a second time. + const session = await (store as any).getSession(); + expect(session).toBe(databricksSql.__mockSession); + expect(attempts).toBe(2); + }); + + it("supports service-principal credentials for REST and SQL setup", async () => { + const axiosModule = require("axios"); + const databricksSql = require("@databricks/sql"); + const store = new DatabricksVectorStore({ + workspaceUrl: "https://workspace.databricks.com", + httpPath: "/sql/1.0/warehouses/test", + clientId: "client-id", + clientSecret: "client-secret", + catalog: "main", + schema: "default", + collectionName: "memories", + dimension: 3, + syncPollIntervalMs: 0, + }); + + await store.initialize(); + await store.insert([[1, 0, 0]], ["id-1"], [{ user_id: "u1" }]); + await store.search([1, 0, 0], 1); + + expect(axiosModule.__mockAuthPost).toHaveBeenCalledWith( + "https://workspace.databricks.com/oidc/v1/token", + expect.any(URLSearchParams), + expect.objectContaining({ + auth: { + username: "client-id", + password: "client-secret", + }, + }), + ); + const authDetails = axiosModule.__mockAuthPost.mock.calls.map( + ([, params]: [string, URLSearchParams]) => + params.get("authorization_details"), + ); + const readDetails = JSON.stringify([ + { + type: "unity_catalog_permission", + securable_type: "table", + securable_object_name: "main.default.memories", + operation: "ReadVectorIndex", + }, + ]); + const writeDetails = JSON.stringify([ + { + type: "unity_catalog_permission", + securable_type: "table", + securable_object_name: "main.default.memories", + operation: "WriteVectorIndex", + }, + ]); + + expect(authDetails).toContain(null); + expect(authDetails).toContain(readDetails); + expect(authDetails).toContain(writeDetails); + expect(databricksSql.__mockClient.connect).toHaveBeenCalledWith( + expect.objectContaining({ + authType: "databricks-oauth", + oauthClientId: "client-id", + oauthClientSecret: "client-secret", + }), + ); + }); + + it("shapes Databricks SQL writes, syncs triggered indexes, and normalizes search results", async () => { + const databricksSql = require("@databricks/sql"); + const axiosModule = require("axios"); + const store = new DatabricksVectorStore({ + workspaceUrl: "https://workspace.databricks.com", + httpPath: "/sql/1.0/warehouses/test", + accessToken: "dapi-test", + catalog: "main", + schema: "default", + collectionName: "memories", + dimension: 3, + // Non-zero: a 0ms poll skips the sleep entirely, so a sync would drain within the + // microtask queue and never overlap the next write. 1ms models a real readiness wait. + syncPollIntervalMs: 1, + }); + + await store.initialize(); + await store.insert( + [[1, 0, 0]], + ["id-1"], + [{ user_id: "u1", topic: "alpha" }], + ); + + const session = databricksSql.__mockSession; + expect(session.executeStatement).toHaveBeenCalledWith( + expect.stringContaining("INSERT INTO main.default.memories"), + ); + expect(session.executeStatement).toHaveBeenCalledWith( + expect.stringContaining("'id-1'"), + ); + expect(session.executeStatement).toHaveBeenCalledWith( + expect.stringContaining("array(1, 0, 0)"), + ); + await store.update("id-1", [0, 1, 0], { user_id: "u1", topic: "beta" }); + await store.delete("id-1"); + + const results = await store.search([1, 0, 0], 5, { + user_id: "u1", + topic: "alpha", + }); + const httpClient = axiosModule.__mockHttpClient; + const syncPosts = httpClient.post.mock.calls.filter( + ([url]: [string]) => url === "/indexes/main.default.memories/sync", + ); + // insert/update/delete each request a sync, but writes that land while one is already + // in flight coalesce into it -- so three writes never cost three pipeline runs. + expect(syncPosts.length).toBeGreaterThanOrEqual(1); + expect(syncPosts.length).toBeLessThan(3); + expect( + httpClient.get.mock.calls.filter( + ([url]: [string]) => url === "/indexes/main.default.memories", + ).length, + ).toBeGreaterThan(4); + expect(httpClient.post).toHaveBeenCalledWith( + "/indexes/main.default.memories/query", + expect.objectContaining({ + columns: ["memory_id", "payload"], + filters_json: JSON.stringify({ user_id: "u1" }), + query_type: "ANN", + query_vector: [1, 0, 0], + }), + ); + expect(results).toEqual([ + { + id: "id-1", + payload: { user_id: "u1", topic: "alpha" }, + score: 0.98, + }, + ]); + }); + + it("keeps metadata-only filters local for standard endpoints", async () => { + const axiosModule = require("axios"); + const store = new DatabricksVectorStore({ + workspaceUrl: "https://workspace.databricks.com", + httpPath: "/sql/1.0/warehouses/test", + accessToken: "dapi-test", + catalog: "main", + schema: "default", + collectionName: "memories", + dimension: 3, + syncPollIntervalMs: 0, + }); + + const results = await store.search([1, 0, 0], 5, { topic: "alpha" }); + const queryCall = axiosModule.__mockHttpClient.post.mock.calls.find( + ([url]: [string]) => url === "/indexes/main.default.memories/query", + ); + + expect(queryCall?.[1]).toEqual( + expect.objectContaining({ + columns: ["memory_id", "payload"], + query_type: "ANN", + query_vector: [1, 0, 0], + }), + ); + expect(queryCall?.[1]).not.toHaveProperty("filters_json"); + expect(results).toEqual([ + { + id: "id-1", + payload: { user_id: "u1", topic: "alpha" }, + score: 0.98, + }, + ]); + }); + + it("applies multiple local metadata operators on the same field", async () => { + const axiosModule = require("axios"); + axiosModule.__setPagedQueryResponses([ + { + result: { + manifest: { + columns: [{ name: "memory_id" }, { name: "payload" }], + }, + data_array: [ + [ + "id-1", + JSON.stringify({ user_id: "u1", importance: 1.0, topic: "high" }), + 0.99, + ], + [ + "id-2", + JSON.stringify({ + user_id: "u1", + importance: 0.75, + topic: "within-range", + }), + 0.98, + ], + ], + }, + }, + ]); + + const store = new DatabricksVectorStore({ + workspaceUrl: "https://workspace.databricks.com", + httpPath: "/sql/1.0/warehouses/test", + accessToken: "dapi-test", + catalog: "main", + schema: "default", + collectionName: "memories", + dimension: 3, + syncPollIntervalMs: 0, + }); + + const results = await store.search([1, 0, 0], 5, { + importance: { gte: 0.5, lte: 0.9 }, + }); + const queryCall = axiosModule.__mockHttpClient.post.mock.calls.find( + ([url]: [string]) => url === "/indexes/main.default.memories/query", + ); + + expect(queryCall?.[1]).not.toHaveProperty("filters_json"); + expect(results).toEqual([ + { + id: "id-2", + payload: { user_id: "u1", importance: 0.75, topic: "within-range" }, + score: 0.98, + }, + ]); + }); + + it("paginates when local-only filters need matches beyond the first page", async () => { + const axiosModule = require("axios"); + axiosModule.__setPagedQueryResponses([ + { + result: { + manifest: { + columns: [{ name: "memory_id" }, { name: "payload" }], + }, + data_array: [ + ["id-2", JSON.stringify({ user_id: "u2", topic: "beta" }), 0.99], + ], + next_page_token: "page-2", + }, + }, + { + result: { + manifest: { + columns: [{ name: "memory_id" }, { name: "payload" }], + }, + data_array: [ + ["id-1", JSON.stringify({ user_id: "u1", topic: "alpha" }), 0.98], + ], + }, + }, + ]); + + const store = new DatabricksVectorStore({ + workspaceUrl: "https://workspace.databricks.com", + httpPath: "/sql/1.0/warehouses/test", + accessToken: "dapi-test", + catalog: "main", + schema: "default", + collectionName: "memories", + dimension: 3, + syncPollIntervalMs: 0, + }); + + const results = await store.search([1, 0, 0], 1, { topic: "alpha" }); + expect( + axiosModule.__mockHttpClient.post.mock.calls.some( + ([url]: [string]) => + url === "/indexes/main.default.memories/query-next-page", + ), + ).toBe(true); + expect(results).toEqual([ + { + id: "id-1", + payload: { user_id: "u1", topic: "alpha" }, + score: 0.98, + }, + ]); + }); + + it("paginates across Databricks result pages when topK exceeds the first page", async () => { + const axiosModule = require("axios"); + axiosModule.__setPagedQueryResponses([ + { + result: { + manifest: { + columns: [{ name: "memory_id" }, { name: "payload" }], + }, + data_array: [ + ["id-1", JSON.stringify({ user_id: "u1", topic: "alpha" }), 0.99], + ], + next_page_token: "page-2", + }, + }, + { + result: { + manifest: { + columns: [{ name: "memory_id" }, { name: "payload" }], + }, + data_array: [ + ["id-2", JSON.stringify({ user_id: "u2", topic: "beta" }), 0.98], + ], + }, + }, + ]); + + const store = new DatabricksVectorStore({ + workspaceUrl: "https://workspace.databricks.com", + httpPath: "/sql/1.0/warehouses/test", + accessToken: "dapi-test", + catalog: "main", + schema: "default", + collectionName: "memories", + dimension: 3, + syncPollIntervalMs: 0, + }); + + const results = await store.search([1, 0, 0], 1001); + const queryCall = axiosModule.__mockHttpClient.post.mock.calls.find( + ([url]: [string]) => url === "/indexes/main.default.memories/query", + ); + + expect(queryCall?.[1]).toEqual( + expect.objectContaining({ + num_results: 1001, + }), + ); + expect( + axiosModule.__mockHttpClient.post.mock.calls.some( + ([url]: [string]) => + url === "/indexes/main.default.memories/query-next-page", + ), + ).toBe(true); + expect(results).toEqual([ + { + id: "id-1", + payload: { user_id: "u1", topic: "alpha" }, + score: 0.99, + }, + { + id: "id-2", + payload: { user_id: "u2", topic: "beta" }, + score: 0.98, + }, + ]); + }); + + it("caps full-text local-fallback requests at Databricks' 200-result limit", async () => { + const axiosModule = require("axios"); + const store = new DatabricksVectorStore({ + workspaceUrl: "https://workspace.databricks.com", + httpPath: "/sql/1.0/warehouses/test", + accessToken: "dapi-test", + catalog: "main", + schema: "default", + collectionName: "memories", + dimension: 3, + syncPollIntervalMs: 0, + }); + + const results = await store.keywordSearch("alpha", 5, { topic: "alpha" }); + const queryCall = axiosModule.__mockHttpClient.post.mock.calls.find( + ([url]: [string]) => url === "/indexes/main.default.memories/query", + ); + + expect(queryCall?.[1]).toEqual( + expect.objectContaining({ + num_results: 200, + query_type: "FULL_TEXT", + query_text: "alpha", + }), + ); + expect(results).toEqual([ + { + id: "id-1", + payload: { user_id: "u1", topic: "alpha" }, + score: 0.98, + }, + ]); + }); + + it("uses SQL-like filters for storage-optimized endpoints", async () => { + const axiosModule = require("axios"); + const store = new DatabricksVectorStore({ + workspaceUrl: "https://workspace.databricks.com", + httpPath: "/sql/1.0/warehouses/test", + accessToken: "dapi-test", + catalog: "main", + schema: "default", + collectionName: "memories", + dimension: 16, + endpointType: "STORAGE_OPTIMIZED", + }); + const query = new Array(16).fill(0); + query[0] = 1; + + const results = await store.search(query, 5, { + user_id: "u1", + topic: "alpha", + }); + const queryCall = axiosModule.__mockHttpClient.post.mock.calls.find( + ([url]: [string]) => url === "/indexes/main.default.memories/query", + ); + + expect(queryCall?.[1]).toEqual( + expect.objectContaining({ + columns: ["memory_id", "payload"], + filters: "user_id = 'u1'", + query_type: "ANN", + query_vector: query, + }), + ); + expect(queryCall?.[1]).not.toHaveProperty("filters_json"); + expect(results).toEqual([ + { + id: "id-1", + payload: { user_id: "u1", topic: "alpha" }, + score: 0.98, + }, + ]); + }); + + it("translates storage-optimized array shorthand filters into IN clauses", async () => { + const axiosModule = require("axios"); + const store = new DatabricksVectorStore({ + workspaceUrl: "https://workspace.databricks.com", + httpPath: "/sql/1.0/warehouses/test", + accessToken: "dapi-test", + catalog: "main", + schema: "default", + collectionName: "memories", + dimension: 16, + endpointType: "STORAGE_OPTIMIZED", + }); + const query = new Array(16).fill(0); + query[0] = 1; + + await store.search(query, 5, { + user_id: ["u1", "u2"], + }); + const queryCall = axiosModule.__mockHttpClient.post.mock.calls.find( + ([url]: [string]) => url === "/indexes/main.default.memories/query", + ); + + expect(queryCall?.[1]).toEqual( + expect.objectContaining({ + filters: "user_id IN ('u1', 'u2')", + }), + ); + expect(queryCall?.[1]).not.toHaveProperty("filters_json"); + }); + + it("falls back to local filtering for logical operators", async () => { + const axiosModule = require("axios"); + const store = new DatabricksVectorStore({ + workspaceUrl: "https://workspace.databricks.com", + httpPath: "/sql/1.0/warehouses/test", + accessToken: "dapi-test", + catalog: "main", + schema: "default", + collectionName: "memories", + dimension: 3, + syncPollIntervalMs: 0, + }); + + const orResults = await store.search([1, 0, 0], 5, { + $or: [{ user_id: "u1" }, { topic: "beta" }], + }); + const notResults = await store.search([1, 0, 0], 5, { + $not: [{ user_id: "u2" }], + }); + const queryCalls = axiosModule.__mockHttpClient.post.mock.calls.filter( + ([url]: [string]) => url === "/indexes/main.default.memories/query", + ); + + expect(queryCalls).toHaveLength(2); + expect(queryCalls[0][1]).not.toHaveProperty("filters_json"); + expect(queryCalls[1][1]).not.toHaveProperty("filters_json"); + expect(orResults.map((result: any) => result.id)).toEqual(["id-1", "id-2"]); + expect(notResults.map((result: any) => result.id)).toEqual(["id-1"]); + }); + + it("keeps memory_id filters aligned between Databricks and local fallback", async () => { + const axiosModule = require("axios"); + const store = new DatabricksVectorStore({ + workspaceUrl: "https://workspace.databricks.com", + httpPath: "/sql/1.0/warehouses/test", + accessToken: "dapi-test", + catalog: "main", + schema: "default", + collectionName: "memories", + dimension: 3, + syncPollIntervalMs: 0, + }); + + const results = await store.search([1, 0, 0], 5, { memory_id: "id-1" }); + const queryCall = axiosModule.__mockHttpClient.post.mock.calls.find( + ([url]: [string]) => url === "/indexes/main.default.memories/query", + ); + + expect(queryCall?.[1]).toEqual( + expect.objectContaining({ + filters_json: JSON.stringify({ memory_id: "id-1" }), + }), + ); + expect(results).toEqual([ + { + id: "id-1", + payload: { user_id: "u1", topic: "alpha" }, + score: 0.98, + }, + ]); + }); + + it("omits columns_to_sync for storage-optimized endpoints", async () => { + const axiosModule = require("axios"); + const store = new DatabricksVectorStore({ + workspaceUrl: "https://workspace.databricks.com", + httpPath: "/sql/1.0/warehouses/test", + accessToken: "dapi-test", + catalog: "main", + schema: "default", + collectionName: "memories", + dimension: 16, + endpointType: "STORAGE_OPTIMIZED", + }); + + await store.initialize(); + + expect(axiosModule.__mockHttpClient.post).toHaveBeenCalledWith("/indexes", { + name: "main.default.memories", + endpoint_name: "mem0_vector_search", + primary_key: "memory_id", + index_type: "DELTA_SYNC", + delta_sync_index_spec: { + source_table: "main.default.memories", + pipeline_type: "TRIGGERED", + columns_to_sync: undefined, + embedding_vector_columns: [ + { + name: "embedding", + embedding_dimension: 16, + }, + ], + }, + }); + }); + + it("rejects invalid storage-optimized embedding dimensions", () => { + expect( + () => + new DatabricksVectorStore({ + workspaceUrl: "https://workspace.databricks.com", + httpPath: "/sql/1.0/warehouses/test", + accessToken: "dapi-test", + catalog: "main", + schema: "default", + collectionName: "memories", + dimension: 3, + endpointType: "STORAGE_OPTIMIZED", + }), + ).toThrow("require dimensions divisible by 16"); + }); + + it("rejects HYBRID vector search without query text", async () => { + const axiosModule = require("axios"); + const store = new DatabricksVectorStore({ + workspaceUrl: "https://workspace.databricks.com", + httpPath: "/sql/1.0/warehouses/test", + accessToken: "dapi-test", + catalog: "main", + schema: "default", + collectionName: "memories", + dimension: 3, + queryType: "HYBRID", + syncPollIntervalMs: 0, + }); + + await expect(store.search([1, 0, 0], 5)).rejects.toThrow( + "Databricks HYBRID search requires query_text", + ); + expect( + axiosModule.__mockHttpClient.post.mock.calls.find( + ([url]: [string]) => url === "/indexes/main.default.memories/query", + ), + ).toBeUndefined(); + }); + + it("roundtrips migration user ids", async () => { + const store = new DatabricksVectorStore({ + workspaceUrl: "https://workspace.databricks.com", + httpPath: "/sql/1.0/warehouses/test", + accessToken: "dapi-test", + catalog: "main", + schema: "default", + collectionName: "memories", + dimension: 3, + }); + + await store.setUserId("custom-user"); + expect(await store.getUserId()).toBe("custom-user"); + }); + + it("escapes backslashes and quotes in SQL string literals (injection-safe)", async () => { + const databricksSql = require("@databricks/sql"); + const store = new DatabricksVectorStore({ + workspaceUrl: "https://workspace.databricks.com", + httpPath: "/sql/1.0/warehouses/test", + accessToken: "dapi-test", + catalog: "main", + schema: "default", + collectionName: "memories", + dimension: 3, + syncPollIntervalMs: 0, + }); + + await store.initialize(); + // The id/user_id carry a single quote and a backslash. Databricks SQL treats + // "\" as an escape char, so without doubling it, "\'" would break out of the + // literal (injection) and any backslash would be dropped on write. + await store.insert([[1, 0, 0]], ["a'b\\c"], [{ user_id: "x'y\\z" }]); + + const session = databricksSql.__mockSession; + const insertSql = session.executeStatement.mock.calls + .map((call: [string]) => call[0]) + .find((sql: string) => sql.includes("INSERT INTO main.default.memories")); + + expect(insertSql).toBeDefined(); + // Backslash doubled AND quote doubled: 'a''b\\c' + expect(insertSql).toContain("'a''b\\\\c'"); + // The un-escaped single-backslash form (the breakout vector) must be absent. + expect(insertSql).not.toContain("'a''b\\c'"); + }); + + it("pushes session filters into the SQL WHERE clause when listing", async () => { + const databricksSql = require("@databricks/sql"); + const store = new DatabricksVectorStore({ + workspaceUrl: "https://workspace.databricks.com", + httpPath: "/sql/1.0/warehouses/test", + accessToken: "dapi-test", + catalog: "main", + schema: "default", + collectionName: "memories", + dimension: 3, + syncPollIntervalMs: 0, + }); + + await store.initialize(); + const session = databricksSql.__mockSession; + session.executeStatement.mockClear(); + + await store.list({ user_id: "u1" }, 10); + + const listSql = session.executeStatement.mock.calls + .map((call: [string]) => call[0].replace(/\s+/g, " ").trim()) + .find((sql: string) => + sql.startsWith("SELECT memory_id, payload FROM main.default.memories"), + ); + + expect(listSql).toBeDefined(); + expect(listSql).toContain("WHERE user_id = 'u1'"); + }); + + it("list(): pushes null-safe ne/nin predicates into the WHERE clause", async () => { + const databricksSql = require("@databricks/sql"); + const store = new DatabricksVectorStore({ + workspaceUrl: "https://workspace.databricks.com", + httpPath: "/sql/1.0/warehouses/test", + accessToken: "dapi-test", + catalog: "main", + schema: "default", + collectionName: "memories", + dimension: 3, + syncPollIntervalMs: 0, + }); + + await store.initialize(); + const session = databricksSql.__mockSession; + + session.executeStatement.mockClear(); + await store.list({ agent_id: { ne: "a2" } }, 10); + const neSql = session.executeStatement.mock.calls + .map((call: [string]) => call[0].replace(/\s+/g, " ").trim()) + .find((sql: string) => + sql.startsWith("SELECT memory_id, payload FROM main.default.memories"), + ); + expect(neSql).toBeDefined(); + expect(neSql).toContain("(agent_id IS NULL OR agent_id != 'a2')"); + expect(neSql).not.toContain("WHERE agent_id != 'a2'"); + + session.executeStatement.mockClear(); + await store.list({ agent_id: { nin: ["a2", "a3"] } }, 10); + const ninSql = session.executeStatement.mock.calls + .map((call: [string]) => call[0].replace(/\s+/g, " ").trim()) + .find((sql: string) => + sql.startsWith("SELECT memory_id, payload FROM main.default.memories"), + ); + expect(ninSql).toBeDefined(); + expect(ninSql).toContain( + "(agent_id IS NULL OR agent_id NOT IN ('a2', 'a3'))", + ); + expect(ninSql).not.toContain("WHERE agent_id NOT IN ('a2', 'a3')"); + }); + + it("list(): bounds the SQL scan with LIMIT topK when the filter fully pushes down", async () => { + const databricksSql = require("@databricks/sql"); + const store = new DatabricksVectorStore({ + workspaceUrl: "https://workspace.databricks.com", + httpPath: "/sql/1.0/warehouses/test", + accessToken: "dapi-test", + catalog: "main", + schema: "default", + collectionName: "memories", + dimension: 3, + syncPollIntervalMs: 0, + }); + + await store.initialize(); + const session = databricksSql.__mockSession; + + session.executeStatement.mockClear(); + await store.list({ user_id: "u1" }); + const defaultTopKSql = session.executeStatement.mock.calls + .map((call: [string]) => call[0].replace(/\s+/g, " ").trim()) + .find((sql: string) => + sql.startsWith("SELECT memory_id, payload FROM main.default.memories"), + ); + expect(defaultTopKSql).toContain("LIMIT 100"); + + session.executeStatement.mockClear(); + await store.list({ user_id: "u1" }, 7); + const customTopKSql = session.executeStatement.mock.calls + .map((call: [string]) => call[0].replace(/\s+/g, " ").trim()) + .find((sql: string) => + sql.startsWith("SELECT memory_id, payload FROM main.default.memories"), + ); + expect(customTopKSql).toContain("LIMIT 7"); + }); + + it("list(): rejects a topK that isn't a positive integer before it reaches SQL", async () => { + const store = new DatabricksVectorStore({ + workspaceUrl: "https://workspace.databricks.com", + httpPath: "/sql/1.0/warehouses/test", + accessToken: "dapi-test", + catalog: "main", + schema: "default", + collectionName: "memories", + dimension: 3, + syncPollIntervalMs: 0, + }); + + for (const badTopK of [ + 0, + -1, + 2.5, + NaN, + "10; DROP TABLE t" as unknown as number, + ]) { + await expect(store.list({ user_id: "u1" }, badTopK)).rejects.toThrow( + `Databricks vector store: topK must be a positive integer, got ${badTopK}`, + ); + } + }); + + it("list(): falls back to the 10k scan ceiling (no WHERE) when $or can't be pushed down", async () => { + const databricksSql = require("@databricks/sql"); + const store = new DatabricksVectorStore({ + workspaceUrl: "https://workspace.databricks.com", + httpPath: "/sql/1.0/warehouses/test", + accessToken: "dapi-test", + catalog: "main", + schema: "default", + collectionName: "memories", + dimension: 3, + syncPollIntervalMs: 0, + }); + + await store.initialize(); + const session = databricksSql.__mockSession; + session.executeStatement.mockClear(); + + await store.list({ $or: [{ user_id: "u1" }, { topic: "beta" }] }); + + const listSql = session.executeStatement.mock.calls + .map((call: [string]) => call[0].replace(/\s+/g, " ").trim()) + .find((sql: string) => + sql.startsWith("SELECT memory_id, payload FROM main.default.memories"), + ); + + expect(listSql).toBeDefined(); + expect(listSql).not.toContain("WHERE"); + expect(listSql).toContain("LIMIT 10000"); + }); + + it("provisions a dedicated text_lemmatized column for BM25 keyword search", async () => { + const databricksSql = require("@databricks/sql"); + const store = new DatabricksVectorStore({ + workspaceUrl: "https://workspace.databricks.com", + httpPath: "/sql/1.0/warehouses/test", + accessToken: "dapi-test", + catalog: "main", + schema: "default", + collectionName: "memories", + dimension: 3, + syncPollIntervalMs: 0, + }); + + await store.initialize(); + + const session = databricksSql.__mockSession; + const createSql = session.executeStatement.mock.calls + .map((call: [string]) => call[0].replace(/\s+/g, " ").trim()) + .find((sql: string) => + sql.startsWith("CREATE TABLE IF NOT EXISTS main.default.memories"), + ); + + expect(createSql).toBeDefined(); + expect(createSql).toContain("text_lemmatized STRING"); + }); + + it("writes lemmatized text on insert and refreshes it on update", async () => { + const databricksSql = require("@databricks/sql"); + const store = new DatabricksVectorStore({ + workspaceUrl: "https://workspace.databricks.com", + httpPath: "/sql/1.0/warehouses/test", + accessToken: "dapi-test", + catalog: "main", + schema: "default", + collectionName: "memories", + dimension: 3, + syncPollIntervalMs: 0, + }); + + await store.initialize(); + const session = databricksSql.__mockSession; + session.executeStatement.mockClear(); + + await store.insert( + [[1, 0, 0]], + ["id-1"], + [{ data: "running shoes", textLemmatized: "run shoe" }], + ); + + const insertSql = session.executeStatement.mock.calls + .map((call: [string]) => call[0].replace(/\s+/g, " ").trim()) + .find((sql: string) => sql.startsWith("INSERT INTO")); + + expect(insertSql).toContain( + "(memory_id, embedding, payload, text_lemmatized, user_id, agent_id, run_id)", + ); + expect(insertSql).toContain("'run shoe'"); + + session.executeStatement.mockClear(); + await store.update("id-1", [0, 1, 0], { + data: "walking boots", + textLemmatized: "walk boot", + }); + + const updateSql = session.executeStatement.mock.calls + .map((call: [string]) => call[0].replace(/\s+/g, " ").trim()) + .find((sql: string) => sql.startsWith("UPDATE")); + + expect(updateSql).toContain("text_lemmatized = 'walk boot'"); + }); + + it("falls back to payload.data when textLemmatized is absent", async () => { + const databricksSql = require("@databricks/sql"); + const store = new DatabricksVectorStore({ + workspaceUrl: "https://workspace.databricks.com", + httpPath: "/sql/1.0/warehouses/test", + accessToken: "dapi-test", + catalog: "main", + schema: "default", + collectionName: "memories", + dimension: 3, + syncPollIntervalMs: 0, + }); + + await store.initialize(); + const session = databricksSql.__mockSession; + session.executeStatement.mockClear(); + + await store.insert([[1, 0, 0]], ["id-1"], [{ data: "plain text" }]); + + const insertSql = session.executeStatement.mock.calls + .map((call: [string]) => call[0].replace(/\s+/g, " ").trim()) + .find((sql: string) => sql.startsWith("INSERT INTO")); + + expect(insertSql).toContain("'plain text'"); + }); + + it("coalesces triggered-index syncs across sequential deletes", async () => { + const axiosModule = require("axios"); + const store = new DatabricksVectorStore({ + workspaceUrl: "https://workspace.databricks.com", + httpPath: "/sql/1.0/warehouses/test", + accessToken: "dapi-test", + catalog: "main", + schema: "default", + collectionName: "memories", + dimension: 3, + syncPollIntervalMs: 1, + }); + + await store.initialize(); + const httpClient = axiosModule.__mockHttpClient; + httpClient.post.mockClear(); + + // deleteAll() awaits each deleteMemory in turn -- exactly this shape. + for (let index = 0; index < 8; index += 1) { + await store.delete(`id-${index}`); + } + await store.search([1, 0, 0], 5); + + const syncPosts = httpClient.post.mock.calls.filter( + ([url]: [string]) => url === "/indexes/main.default.memories/sync", + ); + + expect(syncPosts.length).toBeGreaterThanOrEqual(1); + expect(syncPosts.length).toBeLessThan(8); + }); + + it("starts one sync for concurrent writers, and still drains the second write", async () => { + const axiosModule = require("axios"); + const store = new DatabricksVectorStore({ + workspaceUrl: "https://workspace.databricks.com", + httpPath: "/sql/1.0/warehouses/test", + accessToken: "dapi-test", + catalog: "main", + schema: "default", + collectionName: "memories", + dimension: 3, + syncPollIntervalMs: 1, + }); + + await store.initialize(); + const httpClient = axiosModule.__mockHttpClient; + httpClient.post.mockClear(); + + const syncPosts = () => + httpClient.post.mock.calls.filter( + ([url]: [string]) => url === "/indexes/main.default.memories/sync", + ).length; + + await Promise.all([ + store.insert([[1, 0, 0]], ["id-a"], [{ data: "alpha" }]), + store.insert([[0, 1, 0]], ["id-b"], [{ data: "beta" }]), + ]); + + // Both writers ran before any sync settled. Exactly one drain may own the pipeline -- + // if both had raced past the guard they would each have POSTed their own sync. + expect(syncPosts()).toBe(1); + + // The second writer's rows are not lost: the live drain re-checks the queue and syncs again. + await store.search([1, 0, 0], 5); + expect(syncPosts()).toBe(2); + }); + + it("does not block writes on index readiness, and surfaces sync failures on the next read", async () => { + const axiosModule = require("axios"); + const store = new DatabricksVectorStore({ + workspaceUrl: "https://workspace.databricks.com", + httpPath: "/sql/1.0/warehouses/test", + accessToken: "dapi-test", + catalog: "main", + schema: "default", + collectionName: "memories", + dimension: 3, + syncPollIntervalMs: 1, + syncTimeoutMs: 10, + }); + + await store.initialize(); + // Every readiness poll from here on reports not-ready, so the sync can only time out. + axiosModule.__setIndexNeverReady(true); + + // The write still resolves: it schedules the sync, it does not wait for it. + await expect( + store.insert([[1, 0, 0]], ["id-1"], [{ data: "alpha" }]), + ).resolves.toBeUndefined(); + + // The reader is what pays -- and inherits the deferred failure. + await expect(store.search([1, 0, 0], 5)).rejects.toThrow(); + + // The error is consumed once, not latched forever. + axiosModule.__setIndexNeverReady(false); + await expect(store.search([1, 0, 0], 5)).resolves.toBeDefined(); + }); + + it("update() does not null out session columns on a partial payload", async () => { + const databricksSql = require("@databricks/sql"); + const store = new DatabricksVectorStore({ + workspaceUrl: "https://workspace.databricks.com", + httpPath: "/sql/1.0/warehouses/test", + accessToken: "dapi-test", + catalog: "main", + schema: "default", + collectionName: "memories", + dimension: 3, + syncPollIntervalMs: 1, + }); + + await store.initialize(); + await store.insert( + [[1, 0, 0]], + ["id-1"], + [{ user_id: "u1", topic: "alpha" }], + ); + // Partial payload -- omits user_id/agent_id/run_id, the way a caller updating just one + // field would. Mirrors Python's `excluded_keys`: these must survive untouched. + await store.update("id-1", [0, 1, 0], { topic: "beta" }); + + const session = databricksSql.__mockSession; + const updateSql = session.executeStatement.mock.calls + .map((call: [string]) => call[0]) + .find((sql: string) => sql.includes("UPDATE")); + + expect(updateSql).toBeDefined(); + expect(updateSql).not.toMatch(/\buser_id\s*=/); + expect(updateSql).not.toMatch(/\bagent_id\s*=/); + expect(updateSql).not.toMatch(/\brun_id\s*=/); + expect(updateSql).toMatch(/embedding\s*=/); + expect(updateSql).toMatch(/payload\s*=/); + }); + + it("list() rejects an unsafe-integer topK that would slip past Number.isInteger", async () => { + const store = new DatabricksVectorStore({ + workspaceUrl: "https://workspace.databricks.com", + httpPath: "/sql/1.0/warehouses/test", + accessToken: "dapi-test", + catalog: "main", + schema: "default", + collectionName: "memories", + dimension: 3, + syncPollIntervalMs: 0, + }); + + // 1e21 is Number.isInteger()-true but not representable exactly, and stringifies as + // "1e+21" -- meaningless (and unguarded) as a SQL LIMIT. + await expect(store.list({ user_id: "u1" }, 1e21)).rejects.toThrow( + /topK must be a positive integer/, + ); + // Sanity: a normal topK is still accepted (does not throw). + await expect(store.list({ user_id: "u1" }, 10)).resolves.toBeDefined(); + }); +}); + // ─────────────────────────────────────────────────────────────────────────── // Cassandra — mock client, test interface + idempotent init // ─────────────────────────────────────────────────────────────────────────── @@ -1533,8 +3038,39 @@ describe("S3 Vectors – backward compat with mocked client", () => { // 6. Neptune Analytics — mock NeptuneGraph client, test interface + init // ─────────────────────────────────────────────────────────────────────────── describe("Neptune Analytics – backward compat with mocked client", () => { + // `@aws-sdk/client-neptune-graph` is an optional peer loaded via dynamic `import()` + // (see neptune_analytics.ts). Register a default virtual mock so every test that + // exercises `executeQuery()` -- even ones that inject their own `config.client` -- + // can resolve `sdk.ExecuteQueryCommand` without the real package being installed. + // Tests that care about the AWS client constructor itself override this locally + // with their own `jest.doMock(..., { virtual: true })` before requiring the module. + beforeEach(() => { + jest.resetModules(); + jest.doMock( + "@aws-sdk/client-neptune-graph", + () => ({ + ExecuteQueryCommand: class ExecuteQueryCommand { + input: any; + + constructor(input: any) { + this.input = input; + } + }, + NeptuneGraphClient: class NeptuneGraphClient { + constructor(public config: any) {} + + async send() { + throw new Error( + "Neptune Analytics tests must inject a mock `client`; the default virtual SDK has no server to talk to.", + ); + } + }, + }), + { virtual: true }, + ); + }); + afterEach(() => { - jest.dontMock("@aws-sdk/client-neptune-graph"); jest.resetModules(); }); @@ -2034,29 +3570,34 @@ describe("Neptune Analytics – backward compat with mocked client", () => { expect(mockClient.send).not.toHaveBeenCalled(); }); - it("passes custom HTTPS endpoints to the AWS client when graphIdentifier is provided", () => { + it("passes custom HTTPS endpoints to the AWS client when graphIdentifier is provided", async () => { jest.resetModules(); - const neptuneGraphClient = jest.fn().mockReturnValue({ - send: jest.fn(), - }); + const send = jest + .fn() + .mockResolvedValue(createMockResponse({ results: [] })); + const neptuneGraphClient = jest.fn().mockReturnValue({ send }); - jest.doMock("@aws-sdk/client-neptune-graph", () => ({ - ExecuteQueryCommand: class ExecuteQueryCommand { - input: any; + jest.doMock( + "@aws-sdk/client-neptune-graph", + () => ({ + ExecuteQueryCommand: class ExecuteQueryCommand { + input: any; - constructor(input: any) { - this.input = input; - } - }, - NeptuneGraphClient: neptuneGraphClient, - })); + constructor(input: any) { + this.input = input; + } + }, + NeptuneGraphClient: neptuneGraphClient, + }), + { virtual: true }, + ); const { NeptuneAnalyticsVectorStore, } = require("../src/vector_stores/neptune_analytics"); - new NeptuneAnalyticsVectorStore({ + const store = new NeptuneAnalyticsVectorStore({ graphIdentifier: "g-1234567890", endpoint: "https://example.us-east-1.neptune-graph.amazonaws.com", collectionName: "test", @@ -2066,6 +3607,9 @@ describe("Neptune Analytics – backward compat with mocked client", () => { maxAttempts: 3, }); + // The client is now constructed lazily on first use, not in the constructor. + await store.search([1, 2, 3], 1); + expect(neptuneGraphClient).toHaveBeenCalledWith({ endpoint: "https://example.us-east-1.neptune-graph.amazonaws.com", maxAttempts: 3, @@ -2077,18 +3621,22 @@ describe("Neptune Analytics – backward compat with mocked client", () => { it("rejects HTTPS endpoints without an explicit graphIdentifier", () => { jest.resetModules(); - jest.doMock("@aws-sdk/client-neptune-graph", () => ({ - ExecuteQueryCommand: class ExecuteQueryCommand { - input: any; + jest.doMock( + "@aws-sdk/client-neptune-graph", + () => ({ + ExecuteQueryCommand: class ExecuteQueryCommand { + input: any; - constructor(input: any) { - this.input = input; - } - }, - NeptuneGraphClient: jest.fn().mockReturnValue({ - send: jest.fn(), + constructor(input: any) { + this.input = input; + } + }, + NeptuneGraphClient: jest.fn().mockReturnValue({ + send: jest.fn(), + }), }), - })); + { virtual: true }, + ); const { NeptuneAnalyticsVectorStore, @@ -2114,16 +3662,20 @@ describe("Neptune Analytics – backward compat with mocked client", () => { .mockResolvedValue(createMockResponse({ results: [] })); const neptuneGraphClient = jest.fn().mockReturnValue({ send }); - jest.doMock("@aws-sdk/client-neptune-graph", () => ({ - ExecuteQueryCommand: class ExecuteQueryCommand { - input: any; + jest.doMock( + "@aws-sdk/client-neptune-graph", + () => ({ + ExecuteQueryCommand: class ExecuteQueryCommand { + input: any; - constructor(input: any) { - this.input = input; - } - }, - NeptuneGraphClient: neptuneGraphClient, - })); + constructor(input: any) { + this.input = input; + } + }, + NeptuneGraphClient: neptuneGraphClient, + }), + { virtual: true }, + ); const { NeptuneAnalyticsVectorStore, @@ -2144,6 +3696,109 @@ describe("Neptune Analytics – backward compat with mocked client", () => { expect(send.mock.calls[0][0].input.graphIdentifier).toBe("g-1234567890"); }); + it("does not construct the Neptune client or load the SDK until the first query", async () => { + jest.resetModules(); + + let clientConstructions = 0; + let sdkLoads = 0; + jest.doMock( + "@aws-sdk/client-neptune-graph", + () => { + sdkLoads++; + return { + ExecuteQueryCommand: class ExecuteQueryCommand { + input: any; + + constructor(input: any) { + this.input = input; + } + }, + NeptuneGraphClient: class NeptuneGraphClient { + constructor(public config: any) { + clientConstructions++; + } + + async send() { + return createMockResponse({ results: [] }); + } + }, + }; + }, + { virtual: true }, + ); + + const { + NeptuneAnalyticsVectorStore, + } = require("../src/vector_stores/neptune_analytics"); + + const store = new NeptuneAnalyticsVectorStore({ + graphIdentifier: "g-1234567890", + collectionName: "test", + dimension: 3, + }); + + expect(clientConstructions).toBe(0); + expect(sdkLoads).toBe(0); + + await store.search([1, 2, 3], 1); + + expect(sdkLoads).toBe(1); + expect(clientConstructions).toBe(1); + }); + + it("retries constructing the Neptune client after a failed SDK load instead of caching the rejection", async () => { + jest.resetModules(); + + let shouldFail = true; + jest.doMock( + "@aws-sdk/client-neptune-graph", + () => { + if (shouldFail) { + throw new Error("Cannot find module '@aws-sdk/client-neptune-graph'"); + } + return { + ExecuteQueryCommand: class ExecuteQueryCommand { + input: any; + + constructor(input: any) { + this.input = input; + } + }, + NeptuneGraphClient: class NeptuneGraphClient { + constructor(public config: any) {} + + async send() { + return createMockResponse({ results: [] }); + } + }, + }; + }, + { virtual: true }, + ); + + const { + NeptuneAnalyticsVectorStore, + } = require("../src/vector_stores/neptune_analytics"); + + const store = new NeptuneAnalyticsVectorStore({ + graphIdentifier: "g-1234567890", + collectionName: "test", + dimension: 3, + }); + + // First use fails because the optional SDK can't be loaded... + await expect((store as any).getClient()).rejects.toThrow( + "The '@aws-sdk/client-neptune-graph' package is required", + ); + + // ...but a later call must retry rather than replay the cached rejection. Without + // getClient()'s catch-and-reset this second call would throw the same stale error. + shouldFail = false; + const client = await (store as any).getClient(); + expect(client).toBeDefined(); + expect(typeof client.send).toBe("function"); + }); + it("shapes Neptune write requests and normalizes search results", async () => { const { NeptuneAnalyticsVectorStore, @@ -2592,7 +4247,7 @@ describe("Neptune Analytics – backward compat with mocked client", () => { expect(await freshStore.getUserId()).toBe("persisted-user"); }); - it("throws when Neptune rejects an update upsert", async () => { + it("rolls back the payload when Neptune rejects an update upsert", async () => { const { NeptuneAnalyticsVectorStore, } = require("../src/vector_stores/neptune_analytics"); @@ -2613,15 +4268,15 @@ describe("Neptune Analytics – backward compat with mocked client", () => { store.update("id-1", [3, 2, 1], { data: "beta", user_id: "u1" }), ).rejects.toThrow("Update failed in Neptune Analytics"); - // The payload write runs before the vector upsert, so it is already - // durable by the time the vector step rejects — the caller's new - // metadata must not be silently dropped just because the embedding - // failed to update afterward. + // Neptune's vector index isn't transactional: the payload write already lands by the + // time the vector step rejects. Without compensation the new metadata would be stuck + // pointing at the stale embedding, so a failed update rolls the payload back to what + // it was before the call instead of leaving the two desynced. const afterFailedUpsert = await store.get("id-1"); expect(afterFailedUpsert).not.toBeNull(); expect(afterFailedUpsert!.payload).toEqual( expect.objectContaining({ - data: "beta", + data: "alpha", user_id: "u1", }), ); @@ -2873,7 +4528,7 @@ describe("Vectorize – backward compat with mocked client", () => { }); // ─────────────────────────────────────────────────────────────────────────── -// 7. LangchainVectorStore — mock Langchain client, verify no-op init +// 8. LangchainVectorStore — mock Langchain client, verify no-op init // ─────────────────────────────────────────────────────────────────────────── describe("LangchainVectorStore – backward compat", () => { it("implements full VectorStore interface", () => { @@ -3515,6 +5170,92 @@ describe("Memory class – backward compat with all providers", () => { expect(mockVStore2.initialize).toHaveBeenCalled(); }); + it("derives a separate Databricks entity table when provider casing differs", async () => { + const primaryStore = createMockVectorStore(); + const entityStore = createMockVectorStore(); + mockVectorStoreFactory.create + .mockReturnValueOnce(primaryStore) + .mockReturnValueOnce(entityStore); + + const mem = new MemoryClass({ + embedder: { provider: "openai", config: { apiKey: "k" } }, + vectorStore: { + provider: "Databricks", + config: { + workspaceUrl: "https://workspace.databricks.com", + httpPath: "/sql/1.0/warehouses/test", + accessToken: "dapi-test", + collectionName: "memories", + tableName: "memory_rows", + dimension: 1536, + }, + }, + llm: { provider: "openai", config: { apiKey: "k" } }, + disableHistory: true, + }); + + await mem.getAll({ filters: { user_id: "u1" } }); + await (mem as any).getEntityStore(); + + expect(mockVectorStoreFactory.create).toHaveBeenNthCalledWith( + 2, + "databricks", + expect.objectContaining({ + collectionName: "memories_entities", + tableName: "memory_rows_entities", + }), + ); + expect(entityStore.initialize).toHaveBeenCalled(); + }); + + // optional-peers.test.ts proves no optional peer is imported at module scope. This asserts the + // other half for Databricks: the driver is still loaded lazily (memoized dynamic import, with + // retry-on-failure via getSqlModule()) rather than dropped entirely or made static/eager. + it("still loads the optional @databricks/sql driver lazily", () => { + const source = fs.readFileSync( + path.join(__dirname, "../src/vector_stores/databricks.ts"), + "utf8", + ); + expect(source).toContain('import("@databricks/sql")'); + }); + + it("sends file-based entities to their own DB when provider casing differs", async () => { + const primaryStore = createMockVectorStore(); + const entityStore = createMockVectorStore(); + mockVectorStoreFactory.create + .mockReturnValueOnce(primaryStore) + .mockReturnValueOnce(entityStore); + + const mem = new MemoryClass({ + embedder: { provider: "openai", config: { apiKey: "k" } }, + vectorStore: { + provider: "Memory", + config: { + collectionName: "memories", + dbPath: "/tmp/mem0-casing.db", + dimension: 1536, + }, + }, + llm: { provider: "openai", config: { apiKey: "k" } }, + disableHistory: true, + }); + + await mem.getAll({ filters: { user_id: "u1" } }); + await (mem as any).getEntityStore(); + + // Un-normalized, the `=== "memory"` check misses, dbPath is left alone, and entities land + // in the very file they are supposed to be split out of. + expect(mockVectorStoreFactory.create).toHaveBeenNthCalledWith( + 2, + "memory", + expect.objectContaining({ + collectionName: "memories_entities", + dbPath: "/tmp/mem0-casing_entities.db", + }), + ); + expect(entityStore.initialize).toHaveBeenCalled(); + }); + it("propagates init error to public methods", async () => { const failingEmbedder = { embed: jest.fn().mockRejectedValue(new Error("Embedder unreachable")), diff --git a/mem0-ts/tsup.config.ts b/mem0-ts/tsup.config.ts index 777308270..a98af7eb4 100644 --- a/mem0-ts/tsup.config.ts +++ b/mem0-ts/tsup.config.ts @@ -32,6 +32,7 @@ const external = [ "@azure/identity", "cloudflare", "@cloudflare/workers-types", + "@databricks/sql", "@langchain/core", "fastembed", "compromise", diff --git a/mem0/vector_stores/neptune_analytics.py b/mem0/vector_stores/neptune_analytics.py index 905b5de57..4c9868beb 100644 --- a/mem0/vector_stores/neptune_analytics.py +++ b/mem0/vector_stores/neptune_analytics.py @@ -232,6 +232,20 @@ class NeptuneAnalyticsVector(VectorStoreBase): vector (Optional[List[float]]): New embedding vector. payload (Optional[Dict]): New metadata to replace existing payload. """ + # ponytail: a combined update writes the payload before the embedding, and Neptune's + # vector index isn't transactional -- if the upsert below fails, the new payload would + # otherwise be left committed against the stale embedding. Capture the prior properties + # so a failed upsert can be restored; this is best-effort compensation, not a rollback. + # Only needed when both writes happen -- a payload-only or vector-only update can't desync. + # The restore assumes a single writer per vector_id -- concurrent updates to the same node + # can interleave and clobber each other's compensation. AWS advises against concurrent + # same-vertex writes to the Neptune Analytics vector index for exactly this reason. + prior_properties = None + if payload and vector: + prior = self.get(vector_id) + if prior is not None: + prior_properties = dict(prior.payload or {}) + prior_properties[self._FIELD_LABEL] = self.collection_name if payload: # Replace payload @@ -242,9 +256,9 @@ class NeptuneAnalyticsVector(VectorStoreBase): "vector_id": vector_id } query_string_embedding = f""" - MATCH (n :{self.collection_name}) - WHERE id(n) = $vector_id - SET n = $properties + MATCH (n :{self.collection_name}) + WHERE id(n) = $vector_id + SET n = $properties """ self.execute_query(query_string_embedding, para_payload) @@ -254,14 +268,40 @@ class NeptuneAnalyticsVector(VectorStoreBase): "vector_id": vector_id } query_string_embedding = f""" - MATCH (n :{self.collection_name}) - WHERE id(n) = $vector_id - WITH $embedding as embedding, n as n - CALL neptune.algo.vectors.upsert(n, embedding) - YIELD success - RETURN success + MATCH (n :{self.collection_name}) + WHERE id(n) = $vector_id + WITH $embedding as embedding, n as n + CALL neptune.algo.vectors.upsert(n, embedding) + YIELD success + RETURN success """ - self.execute_query(query_string_embedding, para_embedding) + try: + result = self.execute_query(query_string_embedding, para_embedding) + # A soft {"success": False} row desyncs the payload from the embedding just as + # much as a thrown error, so treat it as a failure and let the rollback below fire. + # Mirrors the TS store's assertSuccessfulResults() check (Python's + # _process_success_message only logs, so it cannot drive the rollback). + for row in result or []: + if "success" in row and row["success"] is not True: + raise RuntimeError(f"Neptune Analytics update upsert reported failure for {vector_id}") + except Exception: + if prior_properties is not None: + try: + restore_query = f""" + MATCH (n :{self.collection_name}) + WHERE id(n) = $vector_id + SET n = $properties + """ + self.execute_query( + restore_query, + {"properties": prior_properties, "vector_id": vector_id}, + ) + except Exception: + logger.error( + f"Neptune Analytics: failed to restore prior payload for {vector_id} " + "after a failed vector upsert" + ) + raise diff --git a/tests/vector_stores/test_neptune_analytics.py b/tests/vector_stores/test_neptune_analytics.py index 41cd841df..82f1e3169 100644 --- a/tests/vector_stores/test_neptune_analytics.py +++ b/tests/vector_stores/test_neptune_analytics.py @@ -275,9 +275,116 @@ class TestNeptuneAnalyticsVectorInitValidation: def test_rejects_injection_payload_in_init(self, payload, monkeypatch): from mem0.vector_stores.neptune_analytics import NeptuneAnalyticsVector monkeypatch.setattr("mem0.vector_stores.neptune_analytics.NeptuneAnalyticsGraph", lambda *args, **kwargs: None) - + with pytest.raises(ValueError, match="Invalid collection_name"): NeptuneAnalyticsVector( endpoint="neptune-graph://test", collection_name=payload ) + + +class _FakeNeptuneGraph: + """Minimal stand-in for `NeptuneAnalyticsGraph.query()` so update()'s compensation + path can be exercised without a real Neptune Analytics endpoint.""" + + def __init__(self): + self.nodes = {} + self.fail_next_upsert = False + self.soft_fail_next_upsert = False + self.get_call_count = 0 + + def query(self, query_string, params=None): + params = params or {} + + if "UNWIND $rows" in query_string: + rows = params["rows"] + if "CALL neptune.algo.vectors.upsert" in query_string: + return [{"success": True} for _ in rows] + for row in rows: + self.nodes[row["node_id"]] = dict(row["properties"]) + return [] + + if "CALL neptune.algo.vectors.upsert" in query_string: + if self.fail_next_upsert: + self.fail_next_upsert = False + raise RuntimeError("simulated Neptune upsert failure") + if self.soft_fail_next_upsert: + self.soft_fail_next_upsert = False + return [{"success": False}] + return [{"success": True}] + + if "SET n = $properties" in query_string: + self.nodes[params["vector_id"]] = dict(params["properties"]) + return [] + + if "RETURN n" in query_string and "node_id" in params: + self.get_call_count += 1 + vector_id = params["node_id"] + if vector_id not in self.nodes: + return [] + return [{"n": {"~id": vector_id, "~properties": dict(self.nodes[vector_id])}}] + + if "DETACH DELETE n" in query_string: + self.nodes.pop(params.get("node_id"), None) + return [] + + return [] + + +class TestNeptuneAnalyticsUpdateRollback: + """update() must not leave a payload committed against a stale embedding when the + vector upsert step fails. See the compensation logic in `NeptuneAnalyticsVector.update()`.""" + + def _make_vec(self, monkeypatch): + monkeypatch.setattr("mem0.vector_stores.neptune_analytics.NeptuneAnalyticsGraph", lambda *args, **kwargs: None) + vec = NeptuneAnalyticsVector(endpoint="neptune-graph://test", collection_name="rollback") + vec.graph = _FakeNeptuneGraph() + return vec + + def test_restores_prior_payload_when_upsert_fails(self, monkeypatch): + vec = self._make_vec(monkeypatch) + vec.insert(vectors=[[0.1, 0.2]], ids=["A"], payloads=[{"data": "alpha", "user_id": "u1"}]) + + vec.graph.fail_next_upsert = True + with pytest.raises(RuntimeError): + vec.update("A", vector=[0.9, 0.9], payload={"data": "beta", "user_id": "u1"}) + + restored = vec.get("A") + assert restored.payload["data"] == "alpha" + assert restored.payload["user_id"] == "u1" + + def test_does_not_snapshot_prior_state_for_a_vector_only_update(self, monkeypatch): + """Only a combined payload+vector update can desync -- a vector-only update has + nothing to roll back to, so it must skip the extra get() snapshot entirely.""" + vec = self._make_vec(monkeypatch) + vec.insert(vectors=[[0.1, 0.2]], ids=["A"], payloads=[{"data": "alpha", "user_id": "u1"}]) + + vec.graph.fail_next_upsert = True + calls_before = vec.graph.get_call_count + with pytest.raises(RuntimeError): + vec.update("A", vector=[0.9, 0.9]) + + assert vec.graph.get_call_count == calls_before + + def test_succeeds_normally_when_upsert_does_not_fail(self, monkeypatch): + vec = self._make_vec(monkeypatch) + vec.insert(vectors=[[0.1, 0.2]], ids=["A"], payloads=[{"data": "alpha", "user_id": "u1"}]) + + vec.update("A", vector=[0.9, 0.9], payload={"data": "beta", "user_id": "u1"}) + + updated = vec.get("A") + assert updated.payload["data"] == "beta" + + def test_rolls_back_on_soft_upsert_failure(self, monkeypatch): + """A soft {"success": False} row desyncs the payload from the embedding just as much as a + thrown error, so update() must treat it as a failure and roll the payload back too.""" + vec = self._make_vec(monkeypatch) + vec.insert(vectors=[[0.1, 0.2]], ids=["A"], payloads=[{"data": "alpha", "user_id": "u1"}]) + + vec.graph.soft_fail_next_upsert = True + with pytest.raises(RuntimeError): + vec.update("A", vector=[0.9, 0.9], payload={"data": "beta", "user_id": "u1"}) + + restored = vec.get("A") + assert restored.payload["data"] == "alpha" + assert restored.payload["user_id"] == "u1"