diff --git a/openmemory/api/app/routers/memories.py b/openmemory/api/app/routers/memories.py index f73aa702b..a19309953 100644 --- a/openmemory/api/app/routers/memories.py +++ b/openmemory/api/app/routers/memories.py @@ -260,6 +260,8 @@ async def create_memory( # Process Qdrant response if isinstance(qdrant_response, dict) and 'results' in qdrant_response: + created_memories = [] + for result in qdrant_response['results']: if result['event'] == 'ADD': # Get the Qdrant-generated ID @@ -294,9 +296,17 @@ async def create_memory( ) db.add(history) - db.commit() + created_memories.append(memory) + + # Commit all changes at once + if created_memories: + db.commit() + for memory in created_memories: db.refresh(memory) - return memory + + # Return the first memory (for API compatibility) + # but all memories are now saved to the database + return created_memories[0] except Exception as qdrant_error: logging.warning(f"Qdrant operation failed: {qdrant_error}.") # Return a json response with the error diff --git a/openmemory/api/app/utils/memory.py b/openmemory/api/app/utils/memory.py index 921e4dffb..a4f557fe6 100644 --- a/openmemory/api/app/utils/memory.py +++ b/openmemory/api/app/utils/memory.py @@ -135,14 +135,111 @@ def reset_memory_client(): def get_default_memory_config(): """Get default memory client configuration with sensible defaults.""" + # Detect vector store based on environment variables + vector_store_config = { + "collection_name": "openmemory", + "host": "mem0_store", + } + + # Check for different vector store configurations based on environment variables + if os.environ.get('CHROMA_HOST') and os.environ.get('CHROMA_PORT'): + vector_store_provider = "chroma" + vector_store_config.update({ + "host": os.environ.get('CHROMA_HOST'), + "port": int(os.environ.get('CHROMA_PORT')) + }) + elif os.environ.get('QDRANT_HOST') and os.environ.get('QDRANT_PORT'): + vector_store_provider = "qdrant" + vector_store_config.update({ + "host": os.environ.get('QDRANT_HOST'), + "port": int(os.environ.get('QDRANT_PORT')) + }) + elif os.environ.get('WEAVIATE_CLUSTER_URL') or (os.environ.get('WEAVIATE_HOST') and os.environ.get('WEAVIATE_PORT')): + vector_store_provider = "weaviate" + # Prefer an explicit cluster URL if provided; otherwise build from host/port + cluster_url = os.environ.get('WEAVIATE_CLUSTER_URL') + if not cluster_url: + weaviate_host = os.environ.get('WEAVIATE_HOST') + weaviate_port = int(os.environ.get('WEAVIATE_PORT')) + cluster_url = f"http://{weaviate_host}:{weaviate_port}" + vector_store_config = { + "collection_name": "openmemory", + "cluster_url": cluster_url + } + elif os.environ.get('REDIS_URL'): + vector_store_provider = "redis" + vector_store_config = { + "collection_name": "openmemory", + "redis_url": os.environ.get('REDIS_URL') + } + elif os.environ.get('PG_HOST') and os.environ.get('PG_PORT'): + vector_store_provider = "pgvector" + vector_store_config.update({ + "host": os.environ.get('PG_HOST'), + "port": int(os.environ.get('PG_PORT')), + "dbname": os.environ.get('PG_DB', 'mem0'), + "user": os.environ.get('PG_USER', 'mem0'), + "password": os.environ.get('PG_PASSWORD', 'mem0') + }) + elif os.environ.get('MILVUS_HOST') and os.environ.get('MILVUS_PORT'): + vector_store_provider = "milvus" + # Construct the full URL as expected by MilvusDBConfig + milvus_host = os.environ.get('MILVUS_HOST') + milvus_port = int(os.environ.get('MILVUS_PORT')) + milvus_url = f"http://{milvus_host}:{milvus_port}" + + vector_store_config = { + "collection_name": "openmemory", + "url": milvus_url, + "token": os.environ.get('MILVUS_TOKEN', ''), # Always include, empty string for local setup + "db_name": os.environ.get('MILVUS_DB_NAME', ''), + "embedding_model_dims": 1536, + "metric_type": "COSINE" # Using COSINE for better semantic similarity + } + elif os.environ.get('ELASTICSEARCH_HOST') and os.environ.get('ELASTICSEARCH_PORT'): + vector_store_provider = "elasticsearch" + # Construct the full URL with scheme since Elasticsearch client expects it + elasticsearch_host = os.environ.get('ELASTICSEARCH_HOST') + elasticsearch_port = int(os.environ.get('ELASTICSEARCH_PORT')) + # Use http:// scheme since we're not using SSL + full_host = f"http://{elasticsearch_host}" + + vector_store_config.update({ + "host": full_host, + "port": elasticsearch_port, + "user": os.environ.get('ELASTICSEARCH_USER', 'elastic'), + "password": os.environ.get('ELASTICSEARCH_PASSWORD', 'changeme'), + "verify_certs": False, + "use_ssl": False, + "embedding_model_dims": 1536 + }) + elif os.environ.get('OPENSEARCH_HOST') and os.environ.get('OPENSEARCH_PORT'): + vector_store_provider = "opensearch" + vector_store_config.update({ + "host": os.environ.get('OPENSEARCH_HOST'), + "port": int(os.environ.get('OPENSEARCH_PORT')) + }) + elif os.environ.get('FAISS_PATH'): + vector_store_provider = "faiss" + vector_store_config = { + "collection_name": "openmemory", + "path": os.environ.get('FAISS_PATH'), + "embedding_model_dims": 1536, + "distance_strategy": "cosine" + } + else: + # Default fallback to Qdrant + vector_store_provider = "qdrant" + vector_store_config.update({ + "port": 6333, + }) + + print(f"Auto-detected vector store: {vector_store_provider} with config: {vector_store_config}") + return { "vector_store": { - "provider": "qdrant", - "config": { - "collection_name": "openmemory", - "host": "mem0_store", - "port": 6333, - } + "provider": vector_store_provider, + "config": vector_store_config }, "llm": { "provider": "openai", @@ -242,6 +339,9 @@ def get_memory_client(custom_instructions: str = None): # Fix Ollama URLs for Docker if needed if config["embedder"].get("provider") == "ollama": config["embedder"] = _fix_ollama_urls(config["embedder"]) + + if "vector_store" in mem0_config and mem0_config["vector_store"] is not None: + config["vector_store"] = mem0_config["vector_store"] else: print("No configuration found in database, using defaults") diff --git a/openmemory/compose/chroma.yml b/openmemory/compose/chroma.yml new file mode 100644 index 000000000..3961fff2c --- /dev/null +++ b/openmemory/compose/chroma.yml @@ -0,0 +1,11 @@ +services: + mem0_store: + image: ghcr.io/chroma-core/chroma:latest + restart: unless-stopped + environment: + - CHROMA_SERVER_HOST=0.0.0.0 + - CHROMA_SERVER_HTTP_PORT=8000 + ports: + - "8000:8000" + volumes: + - mem0_storage:/data \ No newline at end of file diff --git a/openmemory/compose/elasticsearch.yml b/openmemory/compose/elasticsearch.yml new file mode 100644 index 000000000..22e909e95 --- /dev/null +++ b/openmemory/compose/elasticsearch.yml @@ -0,0 +1,15 @@ +services: + mem0_store: + image: docker.elastic.co/elasticsearch/elasticsearch:8.13.4 + restart: unless-stopped + environment: + - discovery.type=single-node + - xpack.security.enabled=false + - ES_JAVA_OPTS=-Xms512m -Xmx512m + ulimits: + memlock: { soft: -1, hard: -1 } + nofile: { soft: 65536, hard: 65536 } + ports: + - "9200:9200" + volumes: + - mem0_storage:/usr/share/elasticsearch/data \ No newline at end of file diff --git a/openmemory/compose/faiss.yml b/openmemory/compose/faiss.yml new file mode 100644 index 000000000..3b2c94173 --- /dev/null +++ b/openmemory/compose/faiss.yml @@ -0,0 +1,3 @@ +services: + # FAISS is a local file-based vector store, so no separate container is needed + # Data will be persisted through volume mounts in the main application diff --git a/openmemory/compose/milvus.yml b/openmemory/compose/milvus.yml new file mode 100644 index 000000000..013ba1e32 --- /dev/null +++ b/openmemory/compose/milvus.yml @@ -0,0 +1,43 @@ +services: + etcd: + image: quay.io/coreos/etcd:v3.5.5 + restart: unless-stopped + environment: + - ETCD_AUTO_COMPACTION_MODE=revision + - ETCD_QUOTA_BACKEND_BYTES=4294967296 + - ETCD_SNAPSHOT_COUNT=50000 + - ETCD_LISTEN_CLIENT_URLS=http://0.0.0.0:2379 + - ETCD_ADVERTISE_CLIENT_URLS=http://etcd:2379 + - ETCD_LISTEN_PEER_URLS=http://0.0.0.0:2380 + - ETCD_INITIAL_ADVERTISE_PEER_URLS=http://etcd:2380 + - ETCD_INITIAL_CLUSTER=default=http://etcd:2380 + - ETCD_NAME=default + - ETCD_DATA_DIR=/etcd + volumes: + - ./data/milvus/etcd:/etcd + + minio: + image: minio/minio:RELEASE.2023-10-25T06-33-25Z + restart: unless-stopped + command: server /minio_data + environment: + - MINIO_ACCESS_KEY=minioadmin + - MINIO_SECRET_KEY=minioadmin + volumes: + - ./data/milvus/minio:/minio_data + + mem0_store: + image: milvusdb/milvus:v2.4.7 + restart: unless-stopped + command: ["milvus", "run", "standalone"] + depends_on: + - etcd + - minio + environment: + - ETCD_ENDPOINTS=etcd:2379 + - MINIO_ADDRESS=minio:9000 + ports: + - "19530:19530" + - "9091:9091" + volumes: + - ./data/milvus/milvus:/var/lib/milvus \ No newline at end of file diff --git a/openmemory/compose/opensearch.yml b/openmemory/compose/opensearch.yml new file mode 100644 index 000000000..8b2e7100e --- /dev/null +++ b/openmemory/compose/opensearch.yml @@ -0,0 +1,19 @@ +services: + mem0_store: + image: opensearchproject/opensearch:2.13.0 + restart: unless-stopped + user: "1000:1000" + environment: + - discovery.type=single-node + - plugins.security.disabled=true + - OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m + - OPENSEARCH_INITIAL_ADMIN_PASSWORD=Openmemory123! + - bootstrap.memory_lock=true + ulimits: + memlock: { soft: -1, hard: -1 } + nofile: { soft: 65536, hard: 65536 } + ports: + - "9200:9200" + - "9600:9600" + volumes: + - mem0_storage:/usr/share/opensearch/data \ No newline at end of file diff --git a/openmemory/compose/pgvector.yml b/openmemory/compose/pgvector.yml new file mode 100644 index 000000000..8cb956008 --- /dev/null +++ b/openmemory/compose/pgvector.yml @@ -0,0 +1,12 @@ +services: + mem0_store: + image: pgvector/pgvector:pg16 + restart: unless-stopped + environment: + - POSTGRES_DB=mem0 + - POSTGRES_USER=mem0 + - POSTGRES_PASSWORD=mem0 + ports: + - "5432:5432" + volumes: + - mem0_storage:/var/lib/postgresql/data \ No newline at end of file diff --git a/openmemory/compose/qdrant.yml b/openmemory/compose/qdrant.yml new file mode 100644 index 000000000..422f4a4d8 --- /dev/null +++ b/openmemory/compose/qdrant.yml @@ -0,0 +1,8 @@ +services: + mem0_store: + image: qdrant/qdrant:latest + restart: unless-stopped + ports: + - "6333:6333" + volumes: + - mem0_storage:/mem0/storage \ No newline at end of file diff --git a/openmemory/compose/redis.yml b/openmemory/compose/redis.yml new file mode 100644 index 000000000..1ca2ef803 --- /dev/null +++ b/openmemory/compose/redis.yml @@ -0,0 +1,13 @@ +services: + mem0_store: + image: redis/redis-stack-server:latest + restart: unless-stopped + ports: + - "6379:6379" + volumes: + - mem0_storage:/var/lib/redis-stack + command: > + redis-stack-server + --appendonly yes + --appendfsync everysec + --save 900 1 300 10 60 10000 \ No newline at end of file diff --git a/openmemory/compose/weaviate.yml b/openmemory/compose/weaviate.yml new file mode 100644 index 000000000..6eab1b8bc --- /dev/null +++ b/openmemory/compose/weaviate.yml @@ -0,0 +1,14 @@ +services: + mem0_store: + image: semitechnologies/weaviate:latest + restart: unless-stopped + environment: + - QUERY_DEFAULTS_LIMIT=25 + - AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED=true + - PERSISTENCE_DATA_PATH=/var/lib/weaviate + - CLUSTER_HOSTNAME=node1 + - WEAVIATE_CLUSTER_URL=http://mem0_store:8080 + ports: + - "8080:8080" + volumes: + - mem0_storage:/var/lib/weaviate \ No newline at end of file diff --git a/openmemory/run.sh b/openmemory/run.sh index 029b7d4b9..ca15322e7 100644 --- a/openmemory/run.sh +++ b/openmemory/run.sh @@ -54,36 +54,325 @@ export NEXT_PUBLIC_API_URL export NEXT_PUBLIC_USER_ID="$USER" export FRONTEND_PORT -# Create docker-compose.yml file -echo "📝 Creating docker-compose.yml..." -cat > docker-compose.yml < docker-compose.yml + + # Extract services from the compose file and replace volume name + # First get everything except the last volumes section + tail -n +2 "$compose_file" | sed '/^volumes:/,$d' | sed "s/mem0_storage/${volume_name}/g" >> docker-compose.yml + + # Add a newline to ensure proper YAML formatting + echo "" >> docker-compose.yml + + # Add the openmemory-mcp service + cat >> docker-compose.yml <> docker-compose.yml <> docker-compose.yml <> docker-compose.yml <> docker-compose.yml <> docker-compose.yml <> docker-compose.yml <> docker-compose.yml <> docker-compose.yml <> docker-compose.yml <> docker-compose.yml <> docker-compose.yml <=1.9.1" || echo "⚠️ Failed to install qdrant packages" + ;; + chroma) + docker exec openmemory-openmemory-mcp-1 pip install "chromadb>=0.4.24" || echo "⚠️ Failed to install chroma packages" + ;; + weaviate) + docker exec openmemory-openmemory-mcp-1 pip install "weaviate-client>=4.4.0,<4.15.0" || echo "⚠️ Failed to install weaviate packages" + ;; + faiss) + docker exec openmemory-openmemory-mcp-1 pip install "faiss-cpu>=1.7.4" || echo "⚠️ Failed to install faiss packages" + ;; + pgvector) + docker exec openmemory-openmemory-mcp-1 pip install "vecs>=0.4.0" "psycopg>=3.2.8" || echo "⚠️ Failed to install pgvector packages" + ;; + redis) + docker exec openmemory-openmemory-mcp-1 pip install "redis>=5.0.0,<6.0.0" "redisvl>=0.1.0,<1.0.0" || echo "⚠️ Failed to install redis packages" + ;; + elasticsearch) + docker exec openmemory-openmemory-mcp-1 pip install "elasticsearch>=8.0.0,<9.0.0" || echo "⚠️ Failed to install elasticsearch packages" + ;; + milvus) + docker exec openmemory-openmemory-mcp-1 pip install "pymilvus>=2.4.0,<2.6.0" || echo "⚠️ Failed to install milvus packages" + ;; + *) + echo "⚠️ Unknown vector store: $vector_store. Installing default qdrant packages." + docker exec openmemory-openmemory-mcp-1 pip install "qdrant-client>=1.9.1" || echo "⚠️ Failed to install qdrant packages" + ;; + esac +} # Start services echo "🚀 Starting backend services..." docker compose up -d +# Wait for container to be ready before installing packages +echo "⏳ Waiting for container to be ready..." +for i in {1..30}; do + if docker exec openmemory-openmemory-mcp-1 python -c "import sys; print('ready')" >/dev/null 2>&1; then + break + fi + sleep 1 +done + +# Install vector store specific packages +install_vector_store_packages "$VECTOR_STORE" + +# If a specific vector store is selected, seed the backend config accordingly +if [ "$VECTOR_STORE" = "milvus" ]; then + echo "⏳ Waiting for API to be ready at ${NEXT_PUBLIC_API_URL}..." + for i in {1..60}; do + if curl -fsS "${NEXT_PUBLIC_API_URL}/api/v1/config" >/dev/null 2>&1; then + break + fi + sleep 1 + done + + echo "🧩 Configuring vector store (milvus) in backend..." + curl -fsS -X PUT "${NEXT_PUBLIC_API_URL}/api/v1/config/mem0/vector_store" \ + -H 'Content-Type: application/json' \ + -d "{\"provider\":\"milvus\",\"config\":{\"collection_name\":\"openmemory\",\"embedding_model_dims\":${EMBEDDING_DIMS},\"url\":\"http://mem0_store:19530\",\"token\":\"\",\"db_name\":\"\",\"metric_type\":\"COSINE\"}}" >/dev/null || true +elif [ "$VECTOR_STORE" = "weaviate" ]; then + echo "⏳ Waiting for API to be ready at ${NEXT_PUBLIC_API_URL}..." + for i in {1..60}; do + if curl -fsS "${NEXT_PUBLIC_API_URL}/api/v1/config" >/dev/null 2>&1; then + break + fi + sleep 1 + done + + echo "🧩 Configuring vector store (weaviate) in backend..." + curl -fsS -X PUT "${NEXT_PUBLIC_API_URL}/api/v1/config/mem0/vector_store" \ + -H 'Content-Type: application/json' \ + -d "{\"provider\":\"weaviate\",\"config\":{\"collection_name\":\"openmemory\",\"embedding_model_dims\":${EMBEDDING_DIMS},\"cluster_url\":\"http://mem0_store:8080\"}}" >/dev/null || true +elif [ "$VECTOR_STORE" = "redis" ]; then + echo "⏳ Waiting for API to be ready at ${NEXT_PUBLIC_API_URL}..." + for i in {1..60}; do + if curl -fsS "${NEXT_PUBLIC_API_URL}/api/v1/config" >/dev/null 2>&1; then + break + fi + sleep 1 + done + + echo "🧩 Configuring vector store (redis) in backend..." + curl -fsS -X PUT "${NEXT_PUBLIC_API_URL}/api/v1/config/mem0/vector_store" \ + -H 'Content-Type: application/json' \ + -d "{\"provider\":\"redis\",\"config\":{\"collection_name\":\"openmemory\",\"embedding_model_dims\":${EMBEDDING_DIMS},\"redis_url\":\"redis://mem0_store:6379\"}}" >/dev/null || true +elif [ "$VECTOR_STORE" = "pgvector" ]; then + echo "⏳ Waiting for API to be ready at ${NEXT_PUBLIC_API_URL}..." + for i in {1..60}; do + if curl -fsS "${NEXT_PUBLIC_API_URL}/api/v1/config" >/dev/null 2>&1; then + break + fi + sleep 1 + done + + echo "🧩 Configuring vector store (pgvector) in backend..." + curl -fsS -X PUT "${NEXT_PUBLIC_API_URL}/api/v1/config/mem0/vector_store" \ + -H 'Content-Type: application/json' \ + -d "{\"provider\":\"pgvector\",\"config\":{\"collection_name\":\"openmemory\",\"embedding_model_dims\":${EMBEDDING_DIMS},\"dbname\":\"mem0\",\"user\":\"mem0\",\"password\":\"mem0\",\"host\":\"mem0_store\",\"port\":5432,\"diskann\":false,\"hnsw\":true}}" >/dev/null || true +elif [ "$VECTOR_STORE" = "qdrant" ]; then + echo "⏳ Waiting for API to be ready at ${NEXT_PUBLIC_API_URL}..." + for i in {1..60}; do + if curl -fsS "${NEXT_PUBLIC_API_URL}/api/v1/config" >/dev/null 2>&1; then + break + fi + sleep 1 + done + + echo "🧩 Configuring vector store (qdrant) in backend..." + curl -fsS -X PUT "${NEXT_PUBLIC_API_URL}/api/v1/config/mem0/vector_store" \ + -H 'Content-Type: application/json' \ + -d "{\"provider\":\"qdrant\",\"config\":{\"collection_name\":\"openmemory\",\"embedding_model_dims\":${EMBEDDING_DIMS},\"host\":\"mem0_store\",\"port\":6333}}" >/dev/null || true +elif [ "$VECTOR_STORE" = "chroma" ]; then + echo "⏳ Waiting for API to be ready at ${NEXT_PUBLIC_API_URL}..." + for i in {1..60}; do + if curl -fsS "${NEXT_PUBLIC_API_URL}/api/v1/config" >/dev/null 2>&1; then + break + fi + sleep 1 + done + + echo "🧩 Configuring vector store (chroma) in backend..." + curl -fsS -X PUT "${NEXT_PUBLIC_API_URL}/api/v1/config/mem0/vector_store" \ + -H 'Content-Type: application/json' \ + -d "{\"provider\":\"chroma\",\"config\":{\"collection_name\":\"openmemory\",\"host\":\"mem0_store\",\"port\":8000}}" >/dev/null || true +elif [ "$VECTOR_STORE" = "elasticsearch" ]; then + echo "⏳ Waiting for API to be ready at ${NEXT_PUBLIC_API_URL}..." + for i in {1..60}; do + if curl -fsS "${NEXT_PUBLIC_API_URL}/api/v1/config" >/dev/null 2>&1; then + break + fi + sleep 1 + done + + echo "🧩 Configuring vector store (elasticsearch) in backend..." + curl -fsS -X PUT "${NEXT_PUBLIC_API_URL}/api/v1/config/mem0/vector_store" \ + -H 'Content-Type: application/json' \ + -d "{\"provider\":\"elasticsearch\",\"config\":{\"collection_name\":\"openmemory\",\"embedding_model_dims\":${EMBEDDING_DIMS},\"host\":\"http://mem0_store\",\"port\":9200,\"user\":\"elastic\",\"password\":\"changeme\",\"verify_certs\":false,\"use_ssl\":false}}" >/dev/null || true +elif [ "$VECTOR_STORE" = "faiss" ]; then + echo "⏳ Waiting for API to be ready at ${NEXT_PUBLIC_API_URL}..." + for i in {1..60}; do + if curl -fsS "${NEXT_PUBLIC_API_URL}/api/v1/config" >/dev/null 2>&1; then + break + fi + sleep 1 + done + + echo "🧩 Configuring vector store (faiss) in backend..." + curl -fsS -X PUT "${NEXT_PUBLIC_API_URL}/api/v1/config/mem0/vector_store" \ + -H 'Content-Type: application/json' \ + -d "{\"provider\":\"faiss\",\"config\":{\"collection_name\":\"openmemory\",\"embedding_model_dims\":${EMBEDDING_DIMS},\"path\":\"/tmp/faiss\",\"distance_strategy\":\"cosine\"}}" >/dev/null || true +fi + # Start the frontend echo "🚀 Starting frontend on port $FRONTEND_PORT..." docker run -d \ @@ -108,4 +397,4 @@ elif command -v start > /dev/null; then start "$URL" # Windows (if run via Git Bash or similar) else echo "⚠️ Could not detect a method to open the browser. Please open $URL manually." -fi +fi \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index d2c78fad4..9fbb96107 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,6 +43,10 @@ vector_stores = [ "pymochow>=2.2.9", "databricks-sdk>=0.63.0", "azure-identity>=1.24.0", + "redis>=5.0.0,<6.0.0", + "redisvl>=0.1.0,<1.0.0", + "elasticsearch>=8.0.0,<9.0.0", + "pymilvus>=2.4.0,<2.6.0", ] llms = [ "groq>=0.3.0", @@ -58,7 +62,7 @@ extras = [ "boto3>=1.34.0", "langchain-community>=0.0.0", "sentence-transformers>=5.0.0", - "elasticsearch>=8.0.0", + "elasticsearch>=8.0.0,<9.0.0", "opensearch-py>=2.0.0", "langchain-memgraph>=0.1.0", ]