fix: add auto_refresh option for OpenSearch Serverless compatibility (#3893)
Co-authored-by: Zaid Malhis <Malhis@users.noreply.github.com>
This commit is contained in:
@@ -18,6 +18,12 @@ class OpenSearchConfig(BaseModel):
|
||||
"RequestsHttpConnection", description="Connection class for OpenSearch"
|
||||
)
|
||||
pool_maxsize: int = Field(20, description="Maximum number of connections in the pool")
|
||||
auto_refresh: bool = Field(
|
||||
False,
|
||||
description="Automatically refresh index after insert operations to make documents "
|
||||
"immediately searchable. Disabled by default for OpenSearch Serverless compatibility. "
|
||||
"OpenSearch automatically refreshes indices every ~1 second, so most users don't need this.",
|
||||
)
|
||||
|
||||
@model_validator(mode="before")
|
||||
@classmethod
|
||||
|
||||
@@ -39,6 +39,8 @@ class OpenSearchDB(VectorStoreBase):
|
||||
|
||||
self.collection_name = config.collection_name
|
||||
self.embedding_model_dims = config.embedding_model_dims
|
||||
self.auto_refresh = config.auto_refresh
|
||||
|
||||
self.create_col(self.collection_name, self.embedding_model_dims)
|
||||
|
||||
def create_index(self) -> None:
|
||||
@@ -148,8 +150,6 @@ class OpenSearchDB(VectorStoreBase):
|
||||
}
|
||||
try:
|
||||
self.client.index(index=self.collection_name, body=body)
|
||||
# Force refresh to make documents immediately searchable for tests
|
||||
self.client.indices.refresh(index=self.collection_name)
|
||||
|
||||
results.append(
|
||||
OutputData(
|
||||
@@ -162,6 +162,14 @@ class OpenSearchDB(VectorStoreBase):
|
||||
logger.error(f"Error inserting vector {id_}: {e}", exc_info=True)
|
||||
raise
|
||||
|
||||
# Refresh once after the full batch (not per document) if explicitly enabled.
|
||||
# Disabled by default for Serverless compatibility: OpenSearch Serverless does not
|
||||
# support the indices.refresh() API, and refreshing per document would cause a
|
||||
# cluster-level I/O stall on every insert.
|
||||
# See: https://docs.aws.amazon.com/opensearch-service/latest/developerguide/serverless-genref.html
|
||||
if self.auto_refresh:
|
||||
self.client.indices.refresh(index=self.collection_name)
|
||||
|
||||
return results
|
||||
|
||||
def search(
|
||||
|
||||
Reference in New Issue
Block a user