fix(vector-stores): Supabase 1000-row cap, RLS init probe, and col_info crash (#6695)

This commit is contained in:
Kartik
2026-07-31 20:36:41 +05:30
committed by GitHub
parent 8d45fb3c9a
commit 29fa41558c
5 changed files with 277 additions and 93 deletions
@@ -115,6 +115,27 @@ $$;
Go to [Supabase](https://supabase.com/dashboard/projects) and run the above SQL migrations in the SQL Editor. Go to [Supabase](https://supabase.com/dashboard/projects) and run the above SQL migrations in the SQL Editor.
### Row Level Security
Tables created through the Supabase dashboard have Row Level Security (RLS) enabled by default with no policies attached. With RLS on and no policies, the TypeScript SDK's queries return zero rows with an HTTP 200 (no error is raised), which looks like an empty memory store rather than a permissions problem. If you use the SQL migrations above (via the SQL Editor), RLS is left in its default off state and this does not apply.
If your table has RLS enabled, add policies for the key your app uses (the example below grants full access to the `service_role` key; scope it down for anon/authenticated keys as needed):
```sql
alter table memories enable row level security;
create policy "Allow service role full access to memories"
on memories
for all
to service_role
using (true)
with check (true);
```
### PostgREST Row Limits
Supabase's PostgREST layer caps the number of rows returned by a single request at `db-max-rows` (1000 by default), for both `.select()` queries and RPC function calls like `match_vectors`. Requesting a `topK` above this limit for `search()` or `list()` will not raise an error, results are capped at `db-max-rows` instead. The TypeScript `list()` method paginates internally to work around this, but `search()` cannot since `match_vectors` has no offset parameter; it logs a warning when it detects a truncated result. Raise `db-max-rows` in your Supabase project settings if you need more than 1000 results per search.
### Config ### Config
Here are the parameters available for configuring Supabase: Here are the parameters available for configuring Supabase:
+51 -46
View File
@@ -31,6 +31,8 @@ interface SupabaseConfig extends VectorStoreConfig {
metadataColumnName?: string; metadataColumnName?: string;
} }
const POSTGREST_MAX_ROWS = 1000;
/* /*
SQL Migration to run in Supabase SQL Editor: SQL Migration to run in Supabase SQL Editor:
@@ -123,34 +125,19 @@ export class SupabaseDB implements VectorStore {
private async _doInitialize(): Promise<void> { private async _doInitialize(): Promise<void> {
await this.ensureClient(); await this.ensureClient();
try { try {
// Verify table exists and vector operations work by attempting a test insert const { error: probeError } = await this.client
const testVector = Array(1536).fill(0);
// First try to delete any existing test vector
try {
await this.client.from(this.tableName).delete().eq("id", "test_vector");
} catch {
// Ignore delete errors - table might not exist yet
}
// Try to insert the test vector
const { error: insertError } = await this.client
.from(this.tableName) .from(this.tableName)
.insert({ .select(this.embeddingColumnName)
id: "test_vector", .limit(1);
[this.embeddingColumnName]: testVector,
[this.metadataColumnName]: {},
})
.select();
// If we get a duplicate key error, that's actually fine - it means the table exists if (probeError) {
if (insertError && insertError.code !== "23505") { console.error("Table probe error:", probeError);
console.error("Test insert error:", insertError);
throw new Error( throw new Error(
`Vector operations failed. Please ensure: `Vector operations failed. Please ensure:
1. The vector extension is enabled 1. The vector extension is enabled
2. The table "${this.tableName}" exists with correct schema 2. The table "${this.tableName}" exists with correct schema
3. The match_vectors function is created 3. The match_vectors function is created
4. Row Level Security policies allow the configured Supabase key to read the table
RUN THE FOLLOWING SQL IN YOUR SUPABASE SQL EDITOR: RUN THE FOLLOWING SQL IN YOUR SUPABASE SQL EDITOR:
@@ -205,13 +192,6 @@ See the SQL migration instructions in the code comments.`,
); );
} }
// Clean up test vector - ignore errors here too
try {
await this.client.from(this.tableName).delete().eq("id", "test_vector");
} catch {
// Ignore delete errors
}
console.log("Connected to Supabase successfully"); console.log("Connected to Supabase successfully");
} catch (error) { } catch (error) {
console.error("Error during Supabase initialization:", error); console.error("Error during Supabase initialization:", error);
@@ -270,6 +250,13 @@ See the SQL migration instructions in the code comments.`,
if (!data) return []; if (!data) return [];
const results = data as VectorSearchResult[]; const results = data as VectorSearchResult[];
if (topK > POSTGREST_MAX_ROWS && results.length === POSTGREST_MAX_ROWS) {
console.warn(
`Supabase search requested topK=${topK} but match_vectors returned exactly the PostgREST row cap of ${POSTGREST_MAX_ROWS}; results were likely truncated.`,
);
}
return results.map((result) => ({ return results.map((result) => ({
id: result.id, id: result.id,
payload: result.metadata, payload: result.metadata,
@@ -364,27 +351,45 @@ See the SQL migration instructions in the code comments.`,
): Promise<[VectorStoreResult[], number]> { ): Promise<[VectorStoreResult[], number]> {
await this.initialize(); await this.initialize();
try { try {
let query = this.client const results: VectorStoreResult[] = [];
.from(this.tableName) let totalCount = 0;
.select("*", { count: "exact" }) let offset = 0;
.limit(topK);
if (filters) { while (results.length < topK) {
Object.entries(filters).forEach(([key, value]) => { const to = Math.min(offset + POSTGREST_MAX_ROWS, topK) - 1;
query = query.eq(`${this.metadataColumnName}->>${key}`, value);
}); let query = this.client
.from(this.tableName)
.select("*", { count: "exact" })
.order("id", { ascending: true });
if (filters) {
Object.entries(filters).forEach(([key, value]) => {
query = query.eq(`${this.metadataColumnName}->>${key}`, value);
});
}
const { data, error, count } = await query.range(offset, to);
if (error) throw error;
totalCount = count ?? totalCount;
const page = (data ?? []).map((item: VectorData) => ({
id: item.id,
payload: item[this.metadataColumnName],
}));
results.push(...page);
const requestedPageSize = to - offset + 1;
if (page.length < requestedPageSize) break;
offset += page.length;
if (results.length >= totalCount) break;
} }
const { data, error, count } = await query; return [results, totalCount];
if (error) throw error;
const results = data.map((item: VectorData) => ({
id: item.id,
payload: item[this.metadataColumnName],
}));
return [results, count || 0];
} catch (error) { } catch (error) {
console.error("Error listing vectors:", error); console.error("Error listing vectors:", error);
throw error; throw error;
@@ -531,34 +531,43 @@ describe("Redis – backward compat with mocked client", () => {
// ─────────────────────────────────────────────────────────────────────────── // ───────────────────────────────────────────────────────────────────────────
describe("Supabase – backward compat with mocked client", () => { describe("Supabase – backward compat with mocked client", () => {
let SupabaseDB: any; let SupabaseDB: any;
let mockClient: any;
let mockTableApi: any;
let mockSelectBuilder: any;
beforeEach(() => { beforeEach(() => {
jest.resetModules(); jest.resetModules();
jest.doMock("@supabase/supabase-js", () => { mockSelectBuilder = {
const mockClient = { limit: jest.fn().mockResolvedValue({ data: [{}], error: null }),
from: jest.fn().mockReturnValue({ range: jest.fn().mockResolvedValue({ data: [], error: null, count: 0 }),
insert: jest.fn().mockReturnValue({ maybeSingle: jest.fn().mockResolvedValue({ data: null, error: null }),
select: jest.fn().mockReturnValue({ error: null }), };
}), mockSelectBuilder.eq = jest.fn().mockReturnValue(mockSelectBuilder);
select: jest.fn().mockReturnValue({ mockSelectBuilder.order = jest.fn().mockReturnValue(mockSelectBuilder);
eq: jest.fn().mockReturnValue({ data: [], error: null }),
}), mockTableApi = {
delete: jest.fn().mockReturnValue({ select: jest.fn().mockReturnValue(mockSelectBuilder),
eq: jest.fn().mockReturnValue({ error: null }), insert: jest.fn().mockReturnValue({ error: null }),
}), delete: jest.fn().mockReturnValue({
update: jest.fn().mockReturnValue({ eq: jest.fn().mockReturnValue({ error: null }),
eq: jest.fn().mockReturnValue({ error: null }), neq: jest.fn().mockReturnValue({ error: null }),
}), }),
upsert: jest.fn().mockReturnValue({ error: null }), update: jest.fn().mockReturnValue({
}), eq: jest.fn().mockReturnValue({ error: null }),
rpc: jest.fn().mockResolvedValue({ data: [], error: null }), }),
}; upsert: jest.fn().mockReturnValue({ error: null }),
return { };
createClient: jest.fn().mockReturnValue(mockClient),
__mockClient: mockClient, mockClient = {
}; from: jest.fn().mockReturnValue(mockTableApi),
}); rpc: jest.fn().mockResolvedValue({ data: [], error: null }),
};
jest.doMock("@supabase/supabase-js", () => ({
createClient: jest.fn().mockReturnValue(mockClient),
__mockClient: mockClient,
}));
SupabaseDB = require("../src/vector_stores/supabase").SupabaseDB; SupabaseDB = require("../src/vector_stores/supabase").SupabaseDB;
}); });
@@ -598,7 +607,86 @@ describe("Supabase – backward compat with mocked client", () => {
const p1 = store.initialize(); const p1 = store.initialize();
const p2 = store.initialize(); const p2 = store.initialize();
await Promise.all([p1, p2]); await Promise.all([p1, p2]);
// No crash = idempotent (Supabase init runs test insert only once) // No crash = idempotent (Supabase init runs a read-only probe query only once)
});
it("_doInitialize probes with a non-destructive select, not an insert", async () => {
const store = new SupabaseDB({
supabaseUrl: "https://example.supabase.co",
supabaseKey: "fake-key",
tableName: "memories",
collectionName: "test",
});
await store.initialize();
expect(mockTableApi.insert).not.toHaveBeenCalled();
expect(mockTableApi.select).toHaveBeenCalledWith("embedding");
expect(mockSelectBuilder.limit).toHaveBeenCalledWith(1);
});
it("list() paginates via .range() across multiple pages when count exceeds page size", async () => {
const totalRows = 1500;
const rangeMock = jest
.fn()
.mockImplementation((from: number, to: number) => {
const remaining = totalRows - from;
const rowCount = Math.max(0, Math.min(to - from + 1, remaining));
const data = Array.from({ length: rowCount }, (_, i) => ({
id: `id-${from + i}`,
metadata: { index: from + i },
}));
return Promise.resolve({ data, error: null, count: totalRows });
});
const listBuilder: any = {
limit: jest.fn().mockResolvedValue({ data: [{}], error: null }),
range: rangeMock,
};
listBuilder.eq = jest.fn().mockReturnValue(listBuilder);
listBuilder.order = jest.fn().mockReturnValue(listBuilder);
mockTableApi.select.mockReturnValue(listBuilder);
const store = new SupabaseDB({
supabaseUrl: "https://example.supabase.co",
supabaseKey: "fake-key",
tableName: "memories",
collectionName: "test",
});
const [results, count] = await store.list(undefined, totalRows);
expect(count).toBe(totalRows);
expect(results).toHaveLength(totalRows);
expect(rangeMock.mock.calls.length).toBeGreaterThan(1);
expect(listBuilder.order).toHaveBeenCalledWith("id", { ascending: true });
expect(new Set(results.map((r) => r.id)).size).toBe(totalRows);
});
it("search() warns when results are truncated at the PostgREST row cap", async () => {
const cappedResults = Array.from({ length: 1000 }, (_, i) => ({
id: `id-${i}`,
similarity: 0.9,
metadata: {},
}));
const warnSpy = jest.spyOn(console, "warn").mockImplementation(() => {});
const store = new SupabaseDB({
supabaseUrl: "https://example.supabase.co",
supabaseKey: "fake-key",
tableName: "memories",
collectionName: "test",
});
mockClient.rpc.mockResolvedValue({ data: cappedResults, error: null });
const results = await store.search([0.1, 0.2, 0.3], 5000);
expect(results).toHaveLength(1000);
expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining("5000"));
expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining("1000"));
warnSpy.mockRestore();
}); });
it("constructor does not emit an unhandled rejection when init fails", async () => { it("constructor does not emit an unhandled rejection when init fails", async () => {
@@ -606,14 +694,15 @@ describe("Supabase – backward compat with mocked client", () => {
jest.doMock("@supabase/supabase-js", () => { jest.doMock("@supabase/supabase-js", () => {
const failing = { const failing = {
from: jest.fn().mockReturnValue({ from: jest.fn().mockReturnValue({
insert: jest.fn().mockReturnValue({ select: jest.fn().mockReturnValue({
select: jest.fn().mockResolvedValue({ limit: jest.fn().mockResolvedValue({
error: { code: "42P01", message: "no table" }, data: null,
error: {
code: "42501",
message: "permission denied for table memories",
},
}), }),
}), }),
delete: jest.fn().mockReturnValue({
eq: jest.fn().mockResolvedValue({ error: null }),
}),
}), }),
}; };
return { createClient: jest.fn().mockReturnValue(failing) }; return { createClient: jest.fn().mockReturnValue(failing) };
+30 -8
View File
@@ -14,6 +14,8 @@ from mem0.vector_stores.base import VectorStoreBase
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
VECS_MAX_QUERY_LIMIT = 1000
class OutputData(BaseModel): class OutputData(BaseModel):
id: Optional[str] id: Optional[str]
@@ -131,11 +133,21 @@ class Supabase(VectorStoreBase):
List[OutputData]: Search results List[OutputData]: Search results
""" """
filters = self._preprocess_filters(filters) filters = self._preprocess_filters(filters)
limit = top_k
if limit > VECS_MAX_QUERY_LIMIT:
logger.warning(
f"Requested top_k={top_k} exceeds the vecs query limit of {VECS_MAX_QUERY_LIMIT}; "
f"capping to {VECS_MAX_QUERY_LIMIT}."
)
limit = VECS_MAX_QUERY_LIMIT
results = self.collection.query( results = self.collection.query(
data=vectors, limit=top_k, filters=filters, include_metadata=True, include_value=True data=vectors, limit=limit, filters=filters, include_metadata=True, include_value=True
) )
return [OutputData(id=str(result[0]), score=max(0.0, 1.0 - float(result[1])), payload=result[2]) for result in results] return [
OutputData(id=str(result[0]), score=max(0.0, 1.0 - float(result[1])), payload=result[2])
for result in results
]
def delete(self, vector_id: str): def delete(self, vector_id: str):
""" """
@@ -201,12 +213,15 @@ class Supabase(VectorStoreBase):
Returns: Returns:
Dict: Collection information including name and configuration Dict: Collection information including name and configuration
""" """
info = self.collection.describe()
return { return {
"name": info.name, "name": self.collection.name,
"count": info.vectors, "count": len(self.collection),
"dimension": info.dimension, "dimension": self.collection.dimension,
"index": {"method": info.index_method, "metric": info.distance_metric}, "index": {
"name": self.collection.index,
"measure": self.index_measure.value,
"is_indexed": self.collection.is_indexed_for_measure(self.index_measure),
},
} }
def list(self, filters: Optional[dict] = None, top_k: int = 100) -> List[OutputData]: def list(self, filters: Optional[dict] = None, top_k: int = 100) -> List[OutputData]:
@@ -222,8 +237,15 @@ class Supabase(VectorStoreBase):
""" """
filters = self._preprocess_filters(filters) filters = self._preprocess_filters(filters)
query = [0] * self.embedding_model_dims query = [0] * self.embedding_model_dims
limit = top_k
if limit > VECS_MAX_QUERY_LIMIT:
logger.warning(
f"Requested top_k={top_k} exceeds the vecs query limit of {VECS_MAX_QUERY_LIMIT}; "
f"capping to {VECS_MAX_QUERY_LIMIT}."
)
limit = VECS_MAX_QUERY_LIMIT
ids = self.collection.query( ids = self.collection.query(
data=query, limit=top_k, filters=filters, include_metadata=True, include_value=False data=query, limit=limit, filters=filters, include_metadata=True, include_value=False
) )
ids = [id[0] for id in ids] ids = [id[0] for id in ids]
records = self.collection.fetch(ids=ids) records = self.collection.fetch(ids=ids)
+55 -8
View File
@@ -1,9 +1,10 @@
from unittest.mock import Mock, patch import logging
from unittest.mock import MagicMock, Mock, patch
import pytest import pytest
from mem0.configs.vector_stores.supabase import IndexMeasure, IndexMethod from mem0.configs.vector_stores.supabase import IndexMeasure, IndexMethod
from mem0.vector_stores.supabase import Supabase from mem0.vector_stores.supabase import VECS_MAX_QUERY_LIMIT, Supabase
@pytest.fixture @pytest.fixture
@@ -14,13 +15,12 @@ def mock_vecs_client():
@pytest.fixture @pytest.fixture
def mock_collection(): def mock_collection():
collection = Mock() collection = MagicMock()
collection.name = "test_collection" collection.name = "test_collection"
collection.vectors = 100
collection.dimension = 1536 collection.dimension = 1536
collection.index_method = "hnsw" collection.index = "ix_vector_cosine_ops_hnsw_m16_efc64_abc1234"
collection.distance_metric = "cosine_distance" collection.is_indexed_for_measure.return_value = True
collection.describe.return_value = collection collection.__len__.return_value = 100
return collection return collection
@@ -141,10 +141,57 @@ def test_col_info(supabase_instance, mock_collection):
"name": "test_collection", "name": "test_collection",
"count": 100, "count": 100,
"dimension": 1536, "dimension": 1536,
"index": {"method": "hnsw", "metric": "cosine_distance"}, "index": {
"name": "ix_vector_cosine_ops_hnsw_m16_efc64_abc1234",
"measure": "cosine_distance",
"is_indexed": True,
},
} }
def test_search_clamps_top_k_to_vecs_limit(supabase_instance, mock_collection, caplog):
mock_collection.query.return_value = []
with caplog.at_level(logging.WARNING, logger="mem0.vector_stores.supabase"):
supabase_instance.search(query="", vectors=[[0.1, 0.2, 0.3]], top_k=5000)
mock_collection.query.assert_called_once_with(
data=[[0.1, 0.2, 0.3]], limit=VECS_MAX_QUERY_LIMIT, filters=None, include_metadata=True, include_value=True
)
assert "5000" in caplog.text
assert str(VECS_MAX_QUERY_LIMIT) in caplog.text
def test_search_does_not_clamp_top_k_within_limit(supabase_instance, mock_collection):
mock_collection.query.return_value = []
supabase_instance.search(query="", vectors=[[0.1, 0.2, 0.3]], top_k=5)
mock_collection.query.assert_called_once_with(
data=[[0.1, 0.2, 0.3]], limit=5, filters=None, include_metadata=True, include_value=True
)
def test_list_clamps_top_k_to_vecs_limit(supabase_instance, mock_collection, caplog):
mock_collection.query.return_value = []
mock_collection.fetch.return_value = []
with caplog.at_level(logging.WARNING, logger="mem0.vector_stores.supabase"):
supabase_instance.list(top_k=5000)
_, kwargs = mock_collection.query.call_args
assert kwargs["limit"] == VECS_MAX_QUERY_LIMIT
assert "5000" in caplog.text
assert str(VECS_MAX_QUERY_LIMIT) in caplog.text
def test_col_info_does_not_raise(supabase_instance):
info = supabase_instance.col_info()
assert isinstance(info, dict)
assert info["name"] == "test_collection"
def test_preprocess_filters(supabase_instance): def test_preprocess_filters(supabase_instance):
# Test single filter # Test single filter
single_filter = {"category": "test"} single_filter = {"category": "test"}