fix(vector-stores): Supabase 1000-row cap, RLS init probe, and col_info crash (#6695)

This commit is contained in:
Kartik
2026-07-31 20:36:41 +05:30
committed by GitHub
parent 8d45fb3c9a
commit 29fa41558c
5 changed files with 277 additions and 93 deletions
@@ -115,6 +115,27 @@ $$;
Go to [Supabase](https://supabase.com/dashboard/projects) and run the above SQL migrations in the SQL Editor.
### Row Level Security
Tables created through the Supabase dashboard have Row Level Security (RLS) enabled by default with no policies attached. With RLS on and no policies, the TypeScript SDK's queries return zero rows with an HTTP 200 (no error is raised), which looks like an empty memory store rather than a permissions problem. If you use the SQL migrations above (via the SQL Editor), RLS is left in its default off state and this does not apply.
If your table has RLS enabled, add policies for the key your app uses (the example below grants full access to the `service_role` key; scope it down for anon/authenticated keys as needed):
```sql
alter table memories enable row level security;
create policy "Allow service role full access to memories"
on memories
for all
to service_role
using (true)
with check (true);
```
### PostgREST Row Limits
Supabase's PostgREST layer caps the number of rows returned by a single request at `db-max-rows` (1000 by default), for both `.select()` queries and RPC function calls like `match_vectors`. Requesting a `topK` above this limit for `search()` or `list()` will not raise an error, results are capped at `db-max-rows` instead. The TypeScript `list()` method paginates internally to work around this, but `search()` cannot since `match_vectors` has no offset parameter; it logs a warning when it detects a truncated result. Raise `db-max-rows` in your Supabase project settings if you need more than 1000 results per search.
### Config
Here are the parameters available for configuring Supabase:
+37 -32
View File
@@ -31,6 +31,8 @@ interface SupabaseConfig extends VectorStoreConfig {
metadataColumnName?: string;
}
const POSTGREST_MAX_ROWS = 1000;
/*
SQL Migration to run in Supabase SQL Editor:
@@ -123,34 +125,19 @@ export class SupabaseDB implements VectorStore {
private async _doInitialize(): Promise<void> {
await this.ensureClient();
try {
// Verify table exists and vector operations work by attempting a test insert
const testVector = Array(1536).fill(0);
// First try to delete any existing test vector
try {
await this.client.from(this.tableName).delete().eq("id", "test_vector");
} catch {
// Ignore delete errors - table might not exist yet
}
// Try to insert the test vector
const { error: insertError } = await this.client
const { error: probeError } = await this.client
.from(this.tableName)
.insert({
id: "test_vector",
[this.embeddingColumnName]: testVector,
[this.metadataColumnName]: {},
})
.select();
.select(this.embeddingColumnName)
.limit(1);
// If we get a duplicate key error, that's actually fine - it means the table exists
if (insertError && insertError.code !== "23505") {
console.error("Test insert error:", insertError);
if (probeError) {
console.error("Table probe error:", probeError);
throw new Error(
`Vector operations failed. Please ensure:
1. The vector extension is enabled
2. The table "${this.tableName}" exists with correct schema
3. The match_vectors function is created
4. Row Level Security policies allow the configured Supabase key to read the table
RUN THE FOLLOWING SQL IN YOUR SUPABASE SQL EDITOR:
@@ -205,13 +192,6 @@ See the SQL migration instructions in the code comments.`,
);
}
// Clean up test vector - ignore errors here too
try {
await this.client.from(this.tableName).delete().eq("id", "test_vector");
} catch {
// Ignore delete errors
}
console.log("Connected to Supabase successfully");
} catch (error) {
console.error("Error during Supabase initialization:", error);
@@ -270,6 +250,13 @@ See the SQL migration instructions in the code comments.`,
if (!data) return [];
const results = data as VectorSearchResult[];
if (topK > POSTGREST_MAX_ROWS && results.length === POSTGREST_MAX_ROWS) {
console.warn(
`Supabase search requested topK=${topK} but match_vectors returned exactly the PostgREST row cap of ${POSTGREST_MAX_ROWS}; results were likely truncated.`,
);
}
return results.map((result) => ({
id: result.id,
payload: result.metadata,
@@ -364,10 +351,17 @@ See the SQL migration instructions in the code comments.`,
): Promise<[VectorStoreResult[], number]> {
await this.initialize();
try {
const results: VectorStoreResult[] = [];
let totalCount = 0;
let offset = 0;
while (results.length < topK) {
const to = Math.min(offset + POSTGREST_MAX_ROWS, topK) - 1;
let query = this.client
.from(this.tableName)
.select("*", { count: "exact" })
.limit(topK);
.order("id", { ascending: true });
if (filters) {
Object.entries(filters).forEach(([key, value]) => {
@@ -375,16 +369,27 @@ See the SQL migration instructions in the code comments.`,
});
}
const { data, error, count } = await query;
const { data, error, count } = await query.range(offset, to);
if (error) throw error;
const results = data.map((item: VectorData) => ({
totalCount = count ?? totalCount;
const page = (data ?? []).map((item: VectorData) => ({
id: item.id,
payload: item[this.metadataColumnName],
}));
results.push(...page);
return [results, count || 0];
const requestedPageSize = to - offset + 1;
if (page.length < requestedPageSize) break;
offset += page.length;
if (results.length >= totalCount) break;
}
return [results, totalCount];
} catch (error) {
console.error("Error listing vectors:", error);
throw error;
@@ -531,34 +531,43 @@ describe("Redis – backward compat with mocked client", () => {
// ───────────────────────────────────────────────────────────────────────────
describe("Supabase – backward compat with mocked client", () => {
let SupabaseDB: any;
let mockClient: any;
let mockTableApi: any;
let mockSelectBuilder: any;
beforeEach(() => {
jest.resetModules();
jest.doMock("@supabase/supabase-js", () => {
const mockClient = {
from: jest.fn().mockReturnValue({
insert: jest.fn().mockReturnValue({
select: jest.fn().mockReturnValue({ error: null }),
}),
select: jest.fn().mockReturnValue({
eq: jest.fn().mockReturnValue({ data: [], error: null }),
}),
mockSelectBuilder = {
limit: jest.fn().mockResolvedValue({ data: [{}], error: null }),
range: jest.fn().mockResolvedValue({ data: [], error: null, count: 0 }),
maybeSingle: jest.fn().mockResolvedValue({ data: null, error: null }),
};
mockSelectBuilder.eq = jest.fn().mockReturnValue(mockSelectBuilder);
mockSelectBuilder.order = jest.fn().mockReturnValue(mockSelectBuilder);
mockTableApi = {
select: jest.fn().mockReturnValue(mockSelectBuilder),
insert: jest.fn().mockReturnValue({ error: null }),
delete: jest.fn().mockReturnValue({
eq: jest.fn().mockReturnValue({ error: null }),
neq: jest.fn().mockReturnValue({ error: null }),
}),
update: jest.fn().mockReturnValue({
eq: jest.fn().mockReturnValue({ error: null }),
}),
upsert: jest.fn().mockReturnValue({ error: null }),
}),
};
mockClient = {
from: jest.fn().mockReturnValue(mockTableApi),
rpc: jest.fn().mockResolvedValue({ data: [], error: null }),
};
return {
jest.doMock("@supabase/supabase-js", () => ({
createClient: jest.fn().mockReturnValue(mockClient),
__mockClient: mockClient,
};
});
}));
SupabaseDB = require("../src/vector_stores/supabase").SupabaseDB;
});
@@ -598,7 +607,86 @@ describe("Supabase – backward compat with mocked client", () => {
const p1 = store.initialize();
const p2 = store.initialize();
await Promise.all([p1, p2]);
// No crash = idempotent (Supabase init runs test insert only once)
// No crash = idempotent (Supabase init runs a read-only probe query only once)
});
it("_doInitialize probes with a non-destructive select, not an insert", async () => {
const store = new SupabaseDB({
supabaseUrl: "https://example.supabase.co",
supabaseKey: "fake-key",
tableName: "memories",
collectionName: "test",
});
await store.initialize();
expect(mockTableApi.insert).not.toHaveBeenCalled();
expect(mockTableApi.select).toHaveBeenCalledWith("embedding");
expect(mockSelectBuilder.limit).toHaveBeenCalledWith(1);
});
it("list() paginates via .range() across multiple pages when count exceeds page size", async () => {
const totalRows = 1500;
const rangeMock = jest
.fn()
.mockImplementation((from: number, to: number) => {
const remaining = totalRows - from;
const rowCount = Math.max(0, Math.min(to - from + 1, remaining));
const data = Array.from({ length: rowCount }, (_, i) => ({
id: `id-${from + i}`,
metadata: { index: from + i },
}));
return Promise.resolve({ data, error: null, count: totalRows });
});
const listBuilder: any = {
limit: jest.fn().mockResolvedValue({ data: [{}], error: null }),
range: rangeMock,
};
listBuilder.eq = jest.fn().mockReturnValue(listBuilder);
listBuilder.order = jest.fn().mockReturnValue(listBuilder);
mockTableApi.select.mockReturnValue(listBuilder);
const store = new SupabaseDB({
supabaseUrl: "https://example.supabase.co",
supabaseKey: "fake-key",
tableName: "memories",
collectionName: "test",
});
const [results, count] = await store.list(undefined, totalRows);
expect(count).toBe(totalRows);
expect(results).toHaveLength(totalRows);
expect(rangeMock.mock.calls.length).toBeGreaterThan(1);
expect(listBuilder.order).toHaveBeenCalledWith("id", { ascending: true });
expect(new Set(results.map((r) => r.id)).size).toBe(totalRows);
});
it("search() warns when results are truncated at the PostgREST row cap", async () => {
const cappedResults = Array.from({ length: 1000 }, (_, i) => ({
id: `id-${i}`,
similarity: 0.9,
metadata: {},
}));
const warnSpy = jest.spyOn(console, "warn").mockImplementation(() => {});
const store = new SupabaseDB({
supabaseUrl: "https://example.supabase.co",
supabaseKey: "fake-key",
tableName: "memories",
collectionName: "test",
});
mockClient.rpc.mockResolvedValue({ data: cappedResults, error: null });
const results = await store.search([0.1, 0.2, 0.3], 5000);
expect(results).toHaveLength(1000);
expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining("5000"));
expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining("1000"));
warnSpy.mockRestore();
});
it("constructor does not emit an unhandled rejection when init fails", async () => {
@@ -606,14 +694,15 @@ describe("Supabase – backward compat with mocked client", () => {
jest.doMock("@supabase/supabase-js", () => {
const failing = {
from: jest.fn().mockReturnValue({
insert: jest.fn().mockReturnValue({
select: jest.fn().mockResolvedValue({
error: { code: "42P01", message: "no table" },
select: jest.fn().mockReturnValue({
limit: jest.fn().mockResolvedValue({
data: null,
error: {
code: "42501",
message: "permission denied for table memories",
},
}),
}),
delete: jest.fn().mockReturnValue({
eq: jest.fn().mockResolvedValue({ error: null }),
}),
}),
};
return { createClient: jest.fn().mockReturnValue(failing) };
+30 -8
View File
@@ -14,6 +14,8 @@ from mem0.vector_stores.base import VectorStoreBase
logger = logging.getLogger(__name__)
VECS_MAX_QUERY_LIMIT = 1000
class OutputData(BaseModel):
id: Optional[str]
@@ -131,11 +133,21 @@ class Supabase(VectorStoreBase):
List[OutputData]: Search results
"""
filters = self._preprocess_filters(filters)
limit = top_k
if limit > VECS_MAX_QUERY_LIMIT:
logger.warning(
f"Requested top_k={top_k} exceeds the vecs query limit of {VECS_MAX_QUERY_LIMIT}; "
f"capping to {VECS_MAX_QUERY_LIMIT}."
)
limit = VECS_MAX_QUERY_LIMIT
results = self.collection.query(
data=vectors, limit=top_k, filters=filters, include_metadata=True, include_value=True
data=vectors, limit=limit, filters=filters, include_metadata=True, include_value=True
)
return [OutputData(id=str(result[0]), score=max(0.0, 1.0 - float(result[1])), payload=result[2]) for result in results]
return [
OutputData(id=str(result[0]), score=max(0.0, 1.0 - float(result[1])), payload=result[2])
for result in results
]
def delete(self, vector_id: str):
"""
@@ -201,12 +213,15 @@ class Supabase(VectorStoreBase):
Returns:
Dict: Collection information including name and configuration
"""
info = self.collection.describe()
return {
"name": info.name,
"count": info.vectors,
"dimension": info.dimension,
"index": {"method": info.index_method, "metric": info.distance_metric},
"name": self.collection.name,
"count": len(self.collection),
"dimension": self.collection.dimension,
"index": {
"name": self.collection.index,
"measure": self.index_measure.value,
"is_indexed": self.collection.is_indexed_for_measure(self.index_measure),
},
}
def list(self, filters: Optional[dict] = None, top_k: int = 100) -> List[OutputData]:
@@ -222,8 +237,15 @@ class Supabase(VectorStoreBase):
"""
filters = self._preprocess_filters(filters)
query = [0] * self.embedding_model_dims
limit = top_k
if limit > VECS_MAX_QUERY_LIMIT:
logger.warning(
f"Requested top_k={top_k} exceeds the vecs query limit of {VECS_MAX_QUERY_LIMIT}; "
f"capping to {VECS_MAX_QUERY_LIMIT}."
)
limit = VECS_MAX_QUERY_LIMIT
ids = self.collection.query(
data=query, limit=top_k, filters=filters, include_metadata=True, include_value=False
data=query, limit=limit, filters=filters, include_metadata=True, include_value=False
)
ids = [id[0] for id in ids]
records = self.collection.fetch(ids=ids)
+55 -8
View File
@@ -1,9 +1,10 @@
from unittest.mock import Mock, patch
import logging
from unittest.mock import MagicMock, Mock, patch
import pytest
from mem0.configs.vector_stores.supabase import IndexMeasure, IndexMethod
from mem0.vector_stores.supabase import Supabase
from mem0.vector_stores.supabase import VECS_MAX_QUERY_LIMIT, Supabase
@pytest.fixture
@@ -14,13 +15,12 @@ def mock_vecs_client():
@pytest.fixture
def mock_collection():
collection = Mock()
collection = MagicMock()
collection.name = "test_collection"
collection.vectors = 100
collection.dimension = 1536
collection.index_method = "hnsw"
collection.distance_metric = "cosine_distance"
collection.describe.return_value = collection
collection.index = "ix_vector_cosine_ops_hnsw_m16_efc64_abc1234"
collection.is_indexed_for_measure.return_value = True
collection.__len__.return_value = 100
return collection
@@ -141,10 +141,57 @@ def test_col_info(supabase_instance, mock_collection):
"name": "test_collection",
"count": 100,
"dimension": 1536,
"index": {"method": "hnsw", "metric": "cosine_distance"},
"index": {
"name": "ix_vector_cosine_ops_hnsw_m16_efc64_abc1234",
"measure": "cosine_distance",
"is_indexed": True,
},
}
def test_search_clamps_top_k_to_vecs_limit(supabase_instance, mock_collection, caplog):
mock_collection.query.return_value = []
with caplog.at_level(logging.WARNING, logger="mem0.vector_stores.supabase"):
supabase_instance.search(query="", vectors=[[0.1, 0.2, 0.3]], top_k=5000)
mock_collection.query.assert_called_once_with(
data=[[0.1, 0.2, 0.3]], limit=VECS_MAX_QUERY_LIMIT, filters=None, include_metadata=True, include_value=True
)
assert "5000" in caplog.text
assert str(VECS_MAX_QUERY_LIMIT) in caplog.text
def test_search_does_not_clamp_top_k_within_limit(supabase_instance, mock_collection):
mock_collection.query.return_value = []
supabase_instance.search(query="", vectors=[[0.1, 0.2, 0.3]], top_k=5)
mock_collection.query.assert_called_once_with(
data=[[0.1, 0.2, 0.3]], limit=5, filters=None, include_metadata=True, include_value=True
)
def test_list_clamps_top_k_to_vecs_limit(supabase_instance, mock_collection, caplog):
mock_collection.query.return_value = []
mock_collection.fetch.return_value = []
with caplog.at_level(logging.WARNING, logger="mem0.vector_stores.supabase"):
supabase_instance.list(top_k=5000)
_, kwargs = mock_collection.query.call_args
assert kwargs["limit"] == VECS_MAX_QUERY_LIMIT
assert "5000" in caplog.text
assert str(VECS_MAX_QUERY_LIMIT) in caplog.text
def test_col_info_does_not_raise(supabase_instance):
info = supabase_instance.col_info()
assert isinstance(info, dict)
assert info["name"] == "test_collection"
def test_preprocess_filters(supabase_instance):
# Test single filter
single_filter = {"category": "test"}