[Bug fix] Fix issues related to logging configuration (#1318)

Co-authored-by: codeflash-ai[bot] <148906541+codeflash-ai[bot]@users.noreply.github.com>

.github/workflows/ci.yml

@@ -39,7 +39,7 @@ jobs:
           path: .venv
           key: venv-${{ runner.os }}-${{ hashFiles('**/poetry.lock') }}
       - name: Install dependencies
-        run: poetry install --all-extras
+        run: make install_all
         if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
       - name: Lint with ruff
         run: make lint

.gitignore

@@ -165,6 +165,7 @@ cython_debug/
 # Database
 db
 test-db
+!embedchain/core/db/
 
 .vscode
 .idea/

Makefile

@@ -11,6 +11,7 @@ install:
 
 install_all:
 	poetry install --all-extras
+	poetry run pip install pinecone-text pinecone-client
 
 install_es:
 	poetry install --extras elasticsearch
@@ -37,6 +38,12 @@ clean:
 lint:
 	poetry run ruff .
 
+build:
+	poetry build
+
+publish:
+	poetry publish
+
 # for example: make test file=tests/test_factory.py
 test:
 	poetry run pytest $(file)

README.md

@@ -2,10 +2,6 @@
   <img src="docs/logo/dark.svg" width="400px" alt="Embedchain Logo">
 </p>
 
-<p align="center">
-  <a href="https://runacap.com/ross-index/q3-2023/" target="_blank" rel="noopener"><img style="width: 260px; height: 56px" src="https://runacap.com/wp-content/uploads/2023/10/ROSS_badge_black_Q3_2023.svg" alt="ROSS Index - Fastest Growing Open-Source Startups in Q3 2023 | Runa Capital" width="260" height="56"/></a>
-</p>
-
 <p align="center">
   <a href="https://pypi.org/project/embedchain/">
     <img src="https://img.shields.io/pypi/v/embedchain" alt="PyPI">
@@ -32,14 +28,11 @@
 
 <hr />
 
-
-> ### Checkout our latest [Sadhguru AI app](https://sadhguru-ai.streamlit.app/) built using Embedchain.
-
 ## What is Embedchain?
 
-Embedchain is an Open Source RAG Framework that makes it easy to create and deploy AI apps. At its core, Embedchain follows the design principle of being *"Conventional but Configurable"* to serve both software engineers and machine learning engineers.
+Embedchain is an Open Source Framework for personalizing LLM responses. It makes it easy to create and deploy personalized AI apps. At its core, Embedchain follows the design principle of being *"Conventional but Configurable"* to serve both software engineers and machine learning engineers.
 
-Embedchain streamlines the creation of RAG applications, offering a seamless process for managing various types of unstructured data. It efficiently segments data into manageable chunks, generates relevant embeddings, and stores them in a vector database for optimized retrieval. With a suite of diverse APIs, it enables users to extract contextual information, find precise answers, or engage in interactive chat conversations, all tailored to their own data.
+Embedchain streamlines the creation of personalized LLM applications, offering a seamless process for managing various types of unstructured data. It efficiently segments data into manageable chunks, generates relevant embeddings, and stores them in a vector database for optimized retrieval. With a suite of diverse APIs, it enables users to extract contextual information, find precise answers, or engage in interactive chat conversations, all tailored to their own data.
 
 ## 🔧 Quick install
 
@@ -64,18 +57,18 @@ For example, you can create an Elon Musk bot using the following code:
 
 ```python
 import os
-from embedchain import Pipeline as App
+from embedchain import App
 
 # Create a bot instance
-os.environ["OPENAI_API_KEY"] = "YOUR API KEY"
-elon_bot = App()
+os.environ["OPENAI_API_KEY"] = "<YOUR_API_KEY>"
+app = App()
 
 # Embed online resources
-elon_bot.add("https://en.wikipedia.org/wiki/Elon_Musk")
-elon_bot.add("https://www.forbes.com/profile/elon-musk")
+app.add("https://en.wikipedia.org/wiki/Elon_Musk")
+app.add("https://www.forbes.com/profile/elon-musk")
 
-# Query the bot
-elon_bot.query("How many companies does Elon Musk run and name those?")
+# Query the app
+app.query("How many companies does Elon Musk run and name those?")
 # Answer: Elon Musk currently runs several companies. As of my knowledge, he is the CEO and lead designer of SpaceX, the CEO and product architect of Tesla, Inc., the CEO and founder of Neuralink, and the CEO and founder of The Boring Company. However, please note that this information may change over time, so it's always good to verify the latest updates.
 ```
 

configs/ollama.yaml

@@ -5,6 +5,7 @@ llm:
     temperature: 0.5
     top_p: 1
     stream: true
+    base_url: http://localhost:11434
 
 embedder:
   provider: huggingface

configs/vllm.yaml

@@ -0,0 +1,14 @@
+llm:
+  provider: vllm
+  config:
+    model: 'meta-llama/Llama-2-70b-hf'
+    temperature: 0.5
+    top_p: 1
+    top_k: 10
+    stream: true
+    trust_remote_code: true
+
+embedder:
+  provider: huggingface
+  config:
+    model: 'BAAI/bge-small-en-v1.5'

docs/_snippets/get-help.mdx

@@ -2,7 +2,7 @@
   <Card title="Talk to founders" icon="calendar" href="https://cal.com/taranjeetio/ec">
   Schedule a call
   </Card>
-  <Card title="Slack" icon="slack" href="https://join.slack.com/t/embedchain/shared_invite/zt-22uwz3c46-Zg7cIh5rOBteT_xe1jwLDw" color="#4A154B">
+  <Card title="Slack" icon="slack" href="https://embedchain.ai/slack" color="#4A154B">
     Join our slack community
   </Card>
   <Card title="Discord" icon="discord" href="https://discord.gg/6PzXDgEjG5" color="#7289DA">

docs/_snippets/missing-data-source-tip.mdx

@@ -4,7 +4,7 @@
   <Card title="Google Form" icon="file" href="https://forms.gle/NDRCKsRpUHsz2Wcm8" color="#7387d0">
     Fill out this form
   </Card>
-  <Card title="Slack" icon="slack" href="https://join.slack.com/t/embedchain/shared_invite/zt-22uwz3c46-Zg7cIh5rOBteT_xe1jwLDw" color="#4A154B">
+  <Card title="Slack" icon="slack" href="https://embedchain.ai/slack" color="#4A154B">
     Let us know on our slack community
   </Card>
   <Card title="Discord" icon="discord" href="https://discord.gg/6PzXDgEjG5" color="#7289DA">

docs/_snippets/missing-llm-tip.mdx

@@ -1,7 +1,7 @@
 <p>If you can't find the specific LLM you need, no need to fret. We're continuously expanding our support for additional LLMs, and you can help us prioritize by opening an issue on our GitHub or simply reaching out to us on our Slack or Discord community.</p>
 
 <CardGroup cols={2}>
-  <Card title="Slack" icon="slack" href="https://join.slack.com/t/embedchain/shared_invite/zt-22uwz3c46-Zg7cIh5rOBteT_xe1jwLDw" color="#4A154B">
+  <Card title="Slack" icon="slack" href="https://embedchain.ai/slack" color="#4A154B">
     Let us know on our slack community
   </Card>
   <Card title="Discord" icon="discord" href="https://discord.gg/6PzXDgEjG5" color="#7289DA">

docs/_snippets/missing-vector-db-tip.mdx

@@ -1,9 +1,9 @@
 
 
-<p>If you can't find the specific vector database, please feel free to request through one of the following channels and help us prioritize.</p>
+<p>If you can't find specific feature or run into issues, please feel free to reach out through one of the following channels.</p>
 
 <CardGroup cols={2}>
-  <Card title="Slack" icon="slack" href="https://join.slack.com/t/embedchain/shared_invite/zt-22uwz3c46-Zg7cIh5rOBteT_xe1jwLDw" color="#4A154B">
+  <Card title="Slack" icon="slack" href="https://embedchain.ai/slack" color="#4A154B">
     Let us know on our slack community
   </Card>
   <Card title="Discord" icon="discord" href="https://discord.gg/6PzXDgEjG5" color="#7289DA">

docs/api-reference/advanced/configuration.mdx

@@ -8,7 +8,7 @@ You can configure different components of your app (`llm`, `embedding model`, or
 
 
 <Tip>
-Embedchain applications are configurable using YAML file, JSON file or by directly passing the config dictionary. Checkout the [docs here](/api-reference/pipeline/overview#usage) on how to use other formats.
+Embedchain applications are configurable using YAML file, JSON file or by directly passing the config dictionary. Checkout the [docs here](/api-reference/app/overview#usage) on how to use other formats.
 </Tip>
 
 <CodeGroup>
@@ -198,11 +198,12 @@ Alright, let's dive into what each key means in the yaml config above:
         - `max_tokens` (Integer): Controls how many tokens are used in the response.
         - `top_p` (Float): Controls the diversity of word selection. A higher value (closer to 1) makes word selection more diverse.
         - `stream` (Boolean): Controls if the response is streamed back to the user (set to false).
-        - `prompt` (String): A prompt for the model to follow when generating responses, requires $context and $query variables.
+        - `prompt` (String): A prompt for the model to follow when generating responses, requires `$context` and `$query` variables.
         - `system_prompt` (String): A system prompt for the model to follow when generating responses, in this case, it's set to the style of William Shakespeare.
-        -  `stream` (Boolean): Controls if the response is streamed back to the user (set to false).
+        - `stream` (Boolean): Controls if the response is streamed back to the user (set to false).
         - `number_documents` (Integer): Number of documents to pull from the vectordb as context, defaults to 1
         - `api_key` (String): The API key for the language model.
+        - `model_kwargs` (Dict): Keyword arguments to pass to the language model. Used for `aws_bedrock` provider, since it requires different arguments for each model.
 3. `vectordb` Section:
     - `provider` (String): The provider for the vector database, set to 'chroma'. You can find the full list of vector database providers in [our docs](/components/vector-databases).
     - `config`:
@@ -214,7 +215,11 @@ Alright, let's dive into what each key means in the yaml config above:
     - `provider` (String): The provider for the embedder, set to 'openai'. You can find the full list of embedding model providers in [our docs](/components/embedding-models).
     - `config`:
         - `model` (String): The specific model used for text embedding, 'text-embedding-ada-002'.
+        - `vector_dimension` (Integer): The vector dimension of the embedding model. [Defaults](https://github.com/embedchain/embedchain/blob/e572b5a3dc1b66f1e9b3357d11a88c63b5ce06e3/embedchain/models/vector_dimensions.py)
         - `api_key` (String): The API key for the embedding model.
+        - `deployment_name` (String): The deployment name for the embedding model.
+        - `title` (String): The title for the embedding model for Google Embedder.
+        - `task_type` (String): The task type for the embedding model for Google Embedder.
 5. `chunker` Section:
     - `chunk_size` (Integer): The size of each chunk of text that is sent to the language model.
     - `chunk_overlap` (Integer): The amount of overlap between each chunk of text.

docs/api-reference/app/chat.mdx

@@ -129,3 +129,18 @@ app.chat("What is the net worth of Bill Gates?", session_id="user2")
 app.chat("What was my last question", session_id="user1")
 # 'Your last question was "What is the net worth of Elon Musk?"'
 ```
+
+### With custom context window
+
+If you want to customize the context window that you want to use during chat (default context window is 3 document chunks), you can do using the following code snippet:
+
+```python with custom chunks size
+from embedchain import App
+from embedchain.config import BaseLlmConfig
+
+app = App()
+app.add("https://www.forbes.com/profile/elon-musk")
+
+query_config = BaseLlmConfig(number_documents=5)
+app.chat("What is the net worth of Elon Musk?", config=query_config)
+```

docs/api-reference/app/delete.mdx

@@ -0,0 +1,48 @@
+---
+title: 🗑 delete
+---
+
+## Delete Document
+
+`delete()` method allows you to delete a document previously added to the app.
+
+### Usage
+
+```python
+from embedchain import App
+
+app = App()
+
+forbes_doc_id = app.add("https://www.forbes.com/profile/elon-musk")
+wiki_doc_id = app.add("https://en.wikipedia.org/wiki/Elon_Musk")
+
+app.delete(forbes_doc_id)   # deletes the forbes document
+```
+
+<Note>
+    If you do not have the document id, you can use `app.db.get()` method to get the document and extract the `hash` key from `metadatas` dictionary object, which serves as the document id.
+</Note>
+
+
+## Delete Chat Session History
+
+`delete_session_chat_history()` method allows you to delete all previous messages in a chat history.
+
+### Usage
+
+```python
+from embedchain import App
+
+app = App()
+
+app.add("https://www.forbes.com/profile/elon-musk")
+
+app.chat("What is the net worth of Elon Musk?")
+
+app.delete_session_chat_history()
+```
+
+<Note>
+    `delete_session_chat_history(session_id="session_1")` method also accepts `session_id` optional param for deleting chat history of a specific session.
+    It assumes the default session if no `session_id` is provided.
+</Note>

docs/api-reference/app/deploy.mdx

@@ -0,0 +1,5 @@
+---
+title: 🚀 deploy
+---
+
+The `deploy()` method is currently available on an invitation-only basis. To request access, please submit your information via the provided [Google Form](https://forms.gle/vigN11h7b4Ywat668). We will review your request and respond promptly.

docs/api-reference/app/evaluate.mdx

@@ -0,0 +1,41 @@
+---
+title: '📝 evaluate'
+---
+
+`evaluate()` method is used to evaluate the performance of a RAG app. You can find the signature below:
+
+### Parameters
+
+<ParamField path="question" type="Union[str, list[str]]">
+    A question or a list of questions to evaluate your app on.
+</ParamField>
+<ParamField path="metrics" type="Optional[list[Union[BaseMetric, str]]]" optional>
+    The metrics to evaluate your app on. Defaults to all metrics: `["context_relevancy", "answer_relevancy", "groundedness"]`
+</ParamField>
+<ParamField path="num_workers" type="int" optional>
+    Specify the number of threads to use for parallel processing.
+</ParamField>
+
+### Returns
+
+<ResponseField name="metrics" type="dict">
+    Returns the metrics you have chosen to evaluate your app on as a dictionary.
+</ResponseField>
+
+## Usage
+
+```python
+from embedchain import App
+
+app = App()
+
+# add data source
+app.add("https://www.forbes.com/profile/elon-musk")
+
+# run evaluation
+app.evaluate("what is the net worth of Elon Musk?")
+# {'answer_relevancy': 0.958019958036268, 'context_relevancy': 0.12903225806451613}
+
+# or
+# app.evaluate(["what is the net worth of Elon Musk?", "which companies does Elon Musk own?"])
+```

docs/api-reference/app/get.mdx

@@ -0,0 +1,33 @@
+---
+title: 📄 get
+---
+
+## Get data sources
+
+`get_data_sources()` returns a list of all the data sources added in the app.
+
+
+### Usage
+
+```python
+from embedchain import App
+
+app = App()
+
+app.add("https://www.forbes.com/profile/elon-musk")
+app.add("https://en.wikipedia.org/wiki/Elon_Musk")
+
+data_sources = app.get_data_sources()
+# [
+#   {
+#       'data_type': 'web_page',
+#       'data_value': 'https://en.wikipedia.org/wiki/Elon_Musk',
+#       'metadata': 'null'
+#   },
+#   {
+#       'data_type': 'web_page',
+#       'data_value': 'https://www.forbes.com/profile/elon-musk',
+#       'metadata': 'null'
+#   }
+# ]
+```

docs/api-reference/app/overview.mdx

@@ -1,34 +1,34 @@
 ---
-title: "Pipeline"
+title: "App"
 ---
 
-Create a RAG pipeline object on Embedchain. This is the main entrypoint for a developer to interact with Embedchain APIs. A pipeline configures the llm, vector database, embedding model, and retrieval strategy of your choice.
+Create a RAG app object on Embedchain. This is the main entrypoint for a developer to interact with Embedchain APIs. An app configures the llm, vector database, embedding model, and retrieval strategy of your choice.
 
 ### Attributes
 
 <ParamField path="local_id" type="str">
-    Pipeline ID
+    App ID
 </ParamField>
 <ParamField path="name" type="str" optional>
-    Name of the pipeline
+    Name of the app
 </ParamField>
 <ParamField path="config" type="BaseConfig">
-    Configuration of the pipeline
+    Configuration of the app
 </ParamField>
 <ParamField path="llm" type="BaseLlm">
-    Configured LLM for the RAG pipeline
+    Configured LLM for the RAG app
 </ParamField>
 <ParamField path="db" type="BaseVectorDB">
-    Configured vector database for the RAG pipeline
+    Configured vector database for the RAG app
 </ParamField>
 <ParamField path="embedding_model" type="BaseEmbedder">
-    Configured embedding model for the RAG pipeline
+    Configured embedding model for the RAG app
 </ParamField>
 <ParamField path="chunker" type="ChunkerConfig">
     Chunker configuration
 </ParamField>
 <ParamField path="client" type="Client" optional>
-    Client object (used to deploy a pipeline to Embedchain platform)
+    Client object (used to deploy an app to Embedchain platform)
 </ParamField>
 <ParamField path="logger" type="logging.Logger">
     Logger object
@@ -36,7 +36,7 @@ Create a RAG pipeline object on Embedchain. This is the main entrypoint for a de
 
 ## Usage
 
-You can create an embedchain pipeline instance using the following methods:
+You can create an app instance using the following methods:
 
 ### Default setting
 
@@ -127,4 +127,4 @@ app = App.from_config(config_path="config.json")
 }
 ```
 
-</CodeGroup>
+</CodeGroup>

docs/api-reference/app/search.mdx

@@ -0,0 +1,111 @@
+---
+title: '🔍 search'
+---
+
+`.search()` enables you to uncover the most pertinent context by performing a semantic search across your data sources based on a given query. Refer to the function signature below:
+
+### Parameters
+
+<ParamField path="query" type="str">
+    Question
+</ParamField>
+<ParamField path="num_documents" type="int" optional>
+    Number of relevant documents to fetch. Defaults to `3`
+</ParamField>
+<ParamField path="where" type="dict" optional>
+    Key value pair for metadata filtering.
+</ParamField>
+<ParamField path="raw_filter" type="dict" optional>
+    Pass raw filter query based on your vector database.
+    Currently, `raw_filter` param is only supported for Pinecone vector database.
+</ParamField>
+
+### Returns
+
+<ResponseField name="answer" type="dict">
+    Return list of dictionaries that contain the relevant chunk and their source information.
+</ResponseField>
+
+## Usage
+
+### Basic
+
+Refer to the following example on how to use the search api:
+
+```python Code example
+from embedchain import App
+
+app = App()
+app.add("https://www.forbes.com/profile/elon-musk")
+
+context = app.search("What is the net worth of Elon?", num_documents=2)
+print(context)
+```
+
+### Advanced
+
+#### Metadata filtering using `where` params
+
+Here is an advanced example of `search()` API with metadata filtering on pinecone database:
+
+```python
+import os
+
+from embedchain import App
+
+os.environ["PINECONE_API_KEY"] = "xxx"
+
+config = {
+    "vectordb": {
+        "provider": "pinecone",
+        "config": {
+            "metric": "dotproduct",
+            "vector_dimension": 1536,
+            "index_name": "ec-test",
+            "serverless_config": {"cloud": "aws", "region": "us-west-2"},
+        },
+    }
+}
+
+app = App.from_config(config=config)
+
+app.add("https://www.forbes.com/profile/bill-gates", metadata={"type": "forbes", "person": "gates"})
+app.add("https://en.wikipedia.org/wiki/Bill_Gates", metadata={"type": "wiki", "person": "gates"})
+
+results = app.search("What is the net worth of Bill Gates?", where={"person": "gates"})
+print("Num of search results: ", len(results))
+```
+
+#### Metadata filtering using `raw_filter` params
+
+Following is an example of metadata filtering by passing the raw filter query that pinecone vector database follows:
+
+```python
+import os
+
+from embedchain import App
+
+os.environ["PINECONE_API_KEY"] = "xxx"
+
+config = {
+    "vectordb": {
+        "provider": "pinecone",
+        "config": {
+            "metric": "dotproduct",
+            "vector_dimension": 1536,
+            "index_name": "ec-test",
+            "serverless_config": {"cloud": "aws", "region": "us-west-2"},
+        },
+    }
+}
+
+app = App.from_config(config=config)
+
+app.add("https://www.forbes.com/profile/bill-gates", metadata={"year": 2022, "person": "gates"})
+app.add("https://en.wikipedia.org/wiki/Bill_Gates", metadata={"year": 2024, "person": "gates"})
+
+print("Filter with person: gates and year > 2023")
+raw_filter = {"$and": [{"person": "gates"}, {"year": {"$gt": 2023}}]}
+results = app.search("What is the net worth of Bill Gates?", raw_filter=raw_filter)
+print("Num of search results: ", len(results))
+```

docs/api-reference/pipeline/delete.mdx

@@ -1,19 +0,0 @@
----
-title: 🗑 delete
----
-
-`delete_chat_history()` method allows you to delete all previous messages in a chat history.
-
-## Usage
-
-```python
-from embedchain import App
-
-app = App()
-
-app.add("https://www.forbes.com/profile/elon-musk")
-
-app.chat("What is the net worth of Elon Musk?")
-
-app.delete_chat_history()
-```

docs/api-reference/pipeline/deploy.mdx

@@ -1,31 +0,0 @@
----
-title: 🚀 deploy
----
-
-Using the `deploy()` method, Embedchain allows developers to easily launch their LLM-powered applications on the [Embedchain Platform](https://app.embedchain.ai). This platform facilitates seamless access to your data's context via a free and user-friendly REST API. Once your pipeline is deployed, you can update your data sources at any time.
-
-The `deploy()` method not only deploys your pipeline but also efficiently manages LLMs, vector databases, embedding models, and data syncing, enabling you to focus on querying, chatting, or searching without the hassle of infrastructure management.
-
-## Usage
-
-```python
-from embedchain import App
-
-# Initialize app
-app = App()
-
-# Add data source
-app.add("https://www.forbes.com/profile/elon-musk")
-
-# Deploy your pipeline to Embedchain Platform
-app.deploy()
-
-# 🔑 Enter your Embedchain API key. You can find the API key at https://app.embedchain.ai/settings/keys/
-# ec-xxxxxx
-
-# 🛠️ Creating pipeline on the platform...
-# 🎉🎉🎉 Pipeline created successfully! View your pipeline: https://app.embedchain.ai/pipelines/xxxxx
-
-# 🛠️ Adding data to your pipeline...
-# ✅ Data of type: web_page, value: https://www.forbes.com/profile/elon-musk added successfully.
-```

docs/api-reference/pipeline/search.mdx

@@ -1,57 +0,0 @@
----
-title: '🔍 search'
----
-
-`.search()` enables you to uncover the most pertinent context by performing a semantic search across your data sources based on a given query. Refer to the function signature below:
-
-### Parameters
-
-<ParamField path="query" type="str">
-    Question
-</ParamField>
-<ParamField path="num_documents" type="int" optional>
-    Number of relevant documents to fetch. Defaults to `3`
-</ParamField>
-
-### Returns
-
-<ResponseField name="answer" type="dict">
-    Return list of dictionaries that contain the relevant chunk and their source information.
-</ResponseField>
-
-## Usage
-
-Refer to the following example on how to use the search api:
-
-```python Code example
-from embedchain import App
-
-# Initialize app
-app = App()
-
-# Add data source
-app.add("https://www.forbes.com/profile/elon-musk")
-
-# Get relevant context using semantic search
-context = app.search("What is the net worth of Elon?", num_documents=2)
-print(context)
-# Context:
-# [
-#     {
-#         'context': 'Elon Musk PROFILEElon MuskCEO, Tesla$221.9BReal Time Net Worth ...',
-#         'metadata': {
-#             'source': 'https://www.forbes.com/profile/elon-musk',
-#             'document_id': 'some_document_id',
-#             'score': 0.404,
-#         }
-#     },
-#     {
-#         'context': 'company, which is now called X.Wealth HistoryHOVER TO REVEAL NET WORTH ...',
-#         'metadata': {
-#             'source': 'https://www.forbes.com/profile/elon-musk',
-#             'document_id': 'some_document_id',
-#             'score': 0.435,
-#         }
-#     }
-# ]
-```

docs/community/connect-with-us.mdx

@@ -8,7 +8,7 @@ We believe in building a vibrant and supportive community around embedchain. The
   <Card title="Twitter" icon="twitter" href="https://twitter.com/embedchain">
     Follow us on Twitter
   </Card>
-  <Card title="Slack" icon="slack" href="https://join.slack.com/t/embedchain/shared_invite/zt-22uwz3c46-Zg7cIh5rOBteT_xe1jwLDw" color="#4A154B">
+  <Card title="Slack" icon="slack" href="https://embedchain.ai/slack" color="#4A154B">
     Join our slack community
   </Card>
   <Card title="Discord" icon="discord" href="https://discord.gg/6PzXDgEjG5" color="#7289DA">

docs/components/data-sources/csv.mdx

@@ -2,18 +2,27 @@
 title: '📊 CSV'
 ---
 
-To add any csv file, use the data_type as `csv`. `csv` allows remote urls and conventional file paths. Headers are included for each line, so if you have an `age` column, `18` will be added as `age: 18`. Eg:
+You can load any csv file from your local file system or through a URL. Headers are included for each line, so if you have an `age` column, `18` will be added as `age: 18`.
+
+## Usage
+
+### Load from a local file
 
 ```python
 from embedchain import App
-
 app = App()
-app.add('https://people.sc.fsu.edu/~jburkardt/data/csv/airtravel.csv', data_type="csv")
-# Or add using the local file path
-# app.add('/path/to/file.csv', data_type="csv")
-
-app.query("Summarize the air travel data")
-# Answer: The air travel data shows the number of flights for the months of July in the years 1958, 1959, and 1960. In July 1958, there were 491 flights, in July 1959 there were 548 flights, and in July 1960 there were 622 flights.
+app.add('/path/to/file.csv', data_type='csv')
 ```
 
-Note: There is a size limit allowed for csv file beyond which it can throw error. This limit is set by the LLMs. Please consider chunking large csv files into smaller csv files.
+### Load from URL
+
+```python
+from embedchain import App
+app = App()
+app.add('https://people.sc.fsu.edu/~jburkardt/data/csv/airtravel.csv', data_type="csv")
+```
+
+<Note>
+There is a size limit allowed for csv file beyond which it can throw error. This limit is set by the LLMs. Please consider chunking large csv files into smaller csv files.
+</Note>
+

docs/components/data-sources/custom.mdx

@@ -7,11 +7,12 @@ When we say "custom", we mean that you can customize the loader and chunker to y
 ```python
 from embedchain import App
 import your_loader
-import your_chunker
+from my_module import CustomLoader
+from my_module import CustomChunker
 
 app = App()
-loader = your_loader()
-chunker = your_chunker()
+loader = CustomLoader()
+chunker = CustomChunker()
 
 app.add("source", data_type="custom", loader=loader, chunker=chunker)
 ```

docs/components/data-sources/data-type-handling.mdx

@@ -37,7 +37,14 @@ Create a local index:
 ```python
 from embedchain import App
 
-naval_chat_bot = App()
+config = {
+    "app": {
+        "config": {
+            "id": "app-1"
+        }
+    }
+}
+naval_chat_bot = App.from_config(config=config)
 naval_chat_bot.add("https://www.youtube.com/watch?v=3qHkcs3kG44")
 naval_chat_bot.add("https://navalmanack.s3.amazonaws.com/Eric-Jorgenson_The-Almanack-of-Naval-Ravikant_Final.pdf")
 ```
@@ -47,7 +54,14 @@ You can reuse the local index with the same code, but without adding new documen
 ```python
 from embedchain import App
 
-naval_chat_bot = App()
+config = {
+    "app": {
+        "config": {
+            "id": "app-1"
+        }
+    }
+}
+naval_chat_bot = App.from_config(config=config)
 print(naval_chat_bot.query("What unique capacity does Naval argue humans possess when it comes to understanding explanations or concepts?"))
 ```
 
@@ -58,7 +72,14 @@ You can reset the app by simply calling the `reset` method. This will delete the
 ```python
 from embedchain import App
 
-app = App()
+app = App()config = {
+    "app": {
+        "config": {
+            "id": "app-1"
+        }
+    }
+}
+naval_chat_bot = App.from_config(config=config)
 app.add("https://www.youtube.com/watch?v=3qHkcs3kG44")
 app.reset()
 ```

docs/components/data-sources/dropbox.mdx

@@ -10,9 +10,19 @@ Ensure your app has the following settings activated:
 
 - In the Permissions section, enable `files.content.read` and `files.metadata.read`.
 
+## Usage
+
+Install the `dropbox` pypi package:
+
+```bash
+pip install dropbox
+```
+
+Following is an example of how to use the dropbox loader:
+
 ```python
 import os
-from embedchain import Pipeline as App
+from embedchain import App
 
 os.environ["DROPBOX_ACCESS_TOKEN"] = "sl.xxx"
 os.environ["OPENAI_API_KEY"] = "sk-xxx"

docs/components/data-sources/github.mdx

@@ -29,11 +29,13 @@ response = app.query("What is Embedchain?")
 ```
 The `add` function of the app will accept any valid github query with qualifiers. It only supports loading github code, repository, issues and pull-requests.
 <Note>
-You must provide qualifiers `type:` and `repo:` in the query. The `type:` qualifier can be a combination of `code`, `repo`, `pr`, `issue`. The `repo:` qualifier must be a valid github repository name.
+You must provide qualifiers `type:` and `repo:` in the query. The `type:` qualifier can be a combination of `code`, `repo`, `pr`, `issue`, `branch`, `file`. The `repo:` qualifier must be a valid github repository name.
 </Note>
 
 <Card title="Valid queries" icon="lightbulb" iconType="duotone" color="#ca8b04">
     - `repo:embedchain/embedchain type:repo` - to load the repository
+    - `repo:embedchain/embedchain type:branch name:feature_test` - to load the branch of the repository
+    - `repo:embedchain/embedchain type:file path:README.md` - to load the specific file of the repository
     - `repo:embedchain/embedchain type:issue,pr` - to load the issues and pull-requests of the repository
     - `repo:embedchain/embedchain type:issue state:closed` - to load the closed issues of the repository
 </Card>

docs/components/data-sources/google-drive.mdx

@@ -0,0 +1,28 @@
+---
+title: 'Google Drive'
+---
+
+To use GoogleDriveLoader you must install the extra dependencies with `pip install --upgrade embedchain[googledrive]`.
+
+The data_type must be `google_drive`. Otherwise, it will be considered a regular web page.
+
+Google Drive requires the setup of credentials. This can be done by following the steps below:
+
+1. Go to the [Google Cloud Console](https://console.cloud.google.com/apis/credentials).
+2. Create a project if you don't have one already.
+3. Enable the [Google Drive API](https://console.cloud.google.com/flows/enableapi?apiid=drive.googleapis.com)
+4. [Authorize credentials for desktop app](https://developers.google.com/drive/api/quickstart/python#authorize_credentials_for_a_desktop_application)
+5. When done, you will be able to download the credentials in `json` format. Rename the downloaded file to `credentials.json` and save it in `~/.credentials/credentials.json`
+6. Set the environment variable `GOOGLE_APPLICATION_CREDENTIALS=~/.credentials/credentials.json`
+
+The first time you use the loader, you will be prompted to enter your Google account credentials.
+
+
+```python
+from embedchain import App
+
+app = App()
+
+url = "https://drive.google.com/drive/u/0/folders/xxx-xxx"
+app.add(url, data_type="google_drive")
+```

docs/components/data-sources/overview.mdx

@@ -5,34 +5,35 @@ title: Overview
 Embedchain comes with built-in support for various data sources. We handle the complexity of loading unstructured data from these data sources, allowing you to easily customize your app through a user-friendly interface.
 
 <CardGroup cols={4}>
-  <Card title="📰 PDF file" href="/components/data-sources/pdf-file"></Card>
-  <Card title="📊 CSV file" href="/components/data-sources/csv"></Card>
-  <Card title="📃 JSON file" href="/components/data-sources/json"></Card>
-  <Card title="📝 Text" href="/components/data-sources/text"></Card>
-  <Card title="📁 Directory/ Folder" href="/components/data-sources/directory"></Card>
-  <Card title="🌐 HTML Web page" href="/components/data-sources/web-page"></Card>
-  <Card title="📽️ Youtube Channel" href="/components/data-sources/youtube-channel"></Card>
-  <Card title="📺 Youtube Video" href="/components/data-sources/youtube-video"></Card>
-  <Card title="📚 Docs website" href="/components/data-sources/docs-site"></Card>
-  <Card title="📝 MDX file" href="/components/data-sources/mdx"></Card>
-  <Card title="📄 DOCX file" href="/components/data-sources/docx"></Card>
-  <Card title="📓 Notion" href="/components/data-sources/notion"></Card>
-  <Card title="🗺️ Sitemap" href="/components/data-sources/sitemap"></Card>
-  <Card title="🧾 XML file" href="/components/data-sources/xml"></Card>
-  <Card title="❓💬 Q&A pair" href="/components/data-sources/qna"></Card>
-  <Card title="🙌 OpenAPI" href="/components/data-sources/openapi"></Card>
-  <Card title="📬 Gmail" href="/components/data-sources/gmail"></Card>
-  <Card title="📝 Github" href="/components/data-sources/github"></Card>
-  <Card title="🐘 Postgres" href="/components/data-sources/postgres"></Card>
-  <Card title="🐬 MySQL" href="/components/data-sources/mysql"></Card>
-  <Card title="🤖 Slack" href="/components/data-sources/slack"></Card>
-  <Card title="💬 Discord" href="/components/data-sources/discord"></Card>
-  <Card title="🗨️ Discourse" href="/components/data-sources/discourse"></Card>
-  <Card title="📝 Substack" href="/components/data-sources/substack"></Card>
-  <Card title="🐝 Beehiiv" href="/components/data-sources/beehiiv"></Card>
-  <Card title="💾 Dropbox" href="/components/data-sources/dropbox"></Card>
-  <Card title="🖼️ Image" href="/components/data-sources/image"></Card>
-  <Card title="⚙️ Custom" href="/components/data-sources/custom"></Card>
+  <Card title="PDF file" href="/components/data-sources/pdf-file"></Card>
+  <Card title="CSV file" href="/components/data-sources/csv"></Card>
+  <Card title="JSON file" href="/components/data-sources/json"></Card>
+  <Card title="Text" href="/components/data-sources/text"></Card>
+  <Card title="Directory" href="/components/data-sources/directory"></Card>
+  <Card title="Web page" href="/components/data-sources/web-page"></Card>
+  <Card title="Youtube Channel" href="/components/data-sources/youtube-channel"></Card>
+  <Card title="Youtube Video" href="/components/data-sources/youtube-video"></Card>
+  <Card title="Docs website" href="/components/data-sources/docs-site"></Card>
+  <Card title="MDX file" href="/components/data-sources/mdx"></Card>
+  <Card title="DOCX file" href="/components/data-sources/docx"></Card>
+  <Card title="Notion" href="/components/data-sources/notion"></Card>
+  <Card title="Sitemap" href="/components/data-sources/sitemap"></Card>
+  <Card title="XML file" href="/components/data-sources/xml"></Card>
+  <Card title="Q&A pair" href="/components/data-sources/qna"></Card>
+  <Card title="OpenAPI" href="/components/data-sources/openapi"></Card>
+  <Card title="Gmail" href="/components/data-sources/gmail"></Card>
+  <Card title="Google Drive" href="/components/data-sources/google-drive"></Card>
+  <Card title="GitHub" href="/components/data-sources/github"></Card>
+  <Card title="Postgres" href="/components/data-sources/postgres"></Card>
+  <Card title="MySQL" href="/components/data-sources/mysql"></Card>
+  <Card title="Slack" href="/components/data-sources/slack"></Card>
+  <Card title="Discord" href="/components/data-sources/discord"></Card>
+  <Card title="Discourse" href="/components/data-sources/discourse"></Card>
+  <Card title="Substack" href="/components/data-sources/substack"></Card>
+  <Card title="Beehiiv" href="/components/data-sources/beehiiv"></Card>
+  <Card title="Dropbox" href="/components/data-sources/dropbox"></Card>
+  <Card title="Image" href="/components/data-sources/image"></Card>
+  <Card title="Custom" href="/components/data-sources/custom"></Card>
 </CardGroup>
 
 <br/ >

docs/components/data-sources/pdf-file.mdx

@@ -1,18 +1,28 @@
 ---
-title: '📰 PDF file'
+title: '📰 PDF'
 ---
 
-To add any pdf file, use the data_type as `pdf_file`. Eg:
+You can load any pdf file from your local file system or through a URL.
+
+## Usage
+
+### Load from a local file
 
 ```python
 from embedchain import App
-
 app = App()
+app.add('/path/to/file.pdf', data_type='pdf_file')
+```
 
+### Load from URL
+
+```python
+from embedchain import App
+app = App()
 app.add('https://arxiv.org/pdf/1706.03762.pdf', data_type='pdf_file')
 app.query("What is the paper 'attention is all you need' about?", citations=True)
 # Answer: The paper "Attention Is All You Need" proposes a new network architecture called the Transformer, which is based solely on attention mechanisms. It suggests that complex recurrent or convolutional neural networks can be replaced with a simpler architecture that connects the encoder and decoder through attention. The paper discusses how this approach can improve sequence transduction models, such as neural machine translation.
-# Contexts: 
+# Contexts:
 # [
 #     (
 #         'Provided proper attribution is ...',
@@ -23,25 +33,11 @@ app.query("What is the paper 'attention is all you need' about?", citations=True
 #             ...
 #         }
 #     ),
-#     (
-#         'Attention Visualizations Input ...',
-#         {
-#             'page': 12,
-#             'url': 'https://arxiv.org/pdf/1706.03762.pdf',
-#             'score': 0.41679039679873736,
-#             ...
-#         }
-#     ),
-#     (
-#         'sequence learning ...',
-#         {
-#             'page': 10,
-#             'url': 'https://arxiv.org/pdf/1706.03762.pdf',
-#             'score': 0.4188303600897153,
-#             ...
-#         }
-#     )
 # ]
 ```
 
-Note that we do not support password protected pdfs.
+We also store the page number under the key `page` with each chunk that helps understand where the answer is coming from. You can fetch the `page` key while during retrieval (refer to the example given above).
+
+<Note>
+Note that we do not support password protected pdf files.
+</Note>

docs/components/data-sources/youtube-channel.mdx

@@ -2,15 +2,17 @@
 title: '📽️ Youtube Channel'
 ---
 
-To add all the videos from a youtube channel to your app, use the data_type as `youtube_channel`.
+## Setup
 
-<Note>
 Make sure you have all the required packages installed before using this data type. You can install them by running the following command in your terminal.
 
 ```bash
-pip install -u "embedchain[youtube]"
+pip install -U "embedchain[youtube]"
 ```
-</Note>
+
+## Usage
+
+To add all the videos from a youtube channel to your app, use the data_type as `youtube_channel`.
 
 ```python
 from embedchain import App

docs/components/data-sources/youtube-video.mdx

@@ -2,6 +2,16 @@
 title: '📺 Youtube Video'
 ---
 
+## Setup
+
+Make sure you have all the required packages installed before using this data type. You can install them by running the following command in your terminal.
+
+```bash
+pip install -U "embedchain[youtube]"
+```
+
+## Usage
+
 To add any youtube video to your app, use the data_type as `youtube_video`. Eg:
 
 ```python

docs/components/embedding-models.mdx

@@ -13,6 +13,8 @@ Embedchain supports several embedding models from the following providers:
   <Card title="GPT4All" href="#gpt4all"></Card>
   <Card title="Hugging Face" href="#hugging-face"></Card>
   <Card title="Vertex AI" href="#vertex-ai"></Card>
+  <Card title="NVIDIA AI" href="#nvidia-ai"></Card>
+  <Card title="Cohere" href="#cohere"></Card>
 </CardGroup>
 
 ## OpenAI
@@ -40,7 +42,27 @@ app.query("What is OpenAI?")
 embedder:
   provider: openai
   config:
-    model: 'text-embedding-ada-002'
+    model: 'text-embedding-3-small'
+```
+
+</CodeGroup>
+
+* OpenAI announced two new embedding models: `text-embedding-3-small` and `text-embedding-3-large`. Embedchain supports both these models. Below you can find YAML config for both:
+
+<CodeGroup>
+
+```yaml text-embedding-3-small.yaml
+embedder:
+  provider: openai
+  config:
+    model: 'text-embedding-3-small'
+```
+
+```yaml text-embedding-3-large.yaml
+embedder:
+  provider: openai
+  config:
+    model: 'text-embedding-3-large'
 ```
 
 </CodeGroup>
@@ -200,3 +222,139 @@ embedder:
 ```
 
 </CodeGroup>
+
+## NVIDIA AI
+
+[NVIDIA AI Foundation Endpoints](https://www.nvidia.com/en-us/ai-data-science/foundation-models/) let you quickly use NVIDIA's AI models, such as Mixtral 8x7B, Llama 2 etc, through our API. These models are available in the [NVIDIA NGC catalog](https://catalog.ngc.nvidia.com/ai-foundation-models), fully optimized and ready to use on NVIDIA's AI platform. They are designed for high speed and easy customization, ensuring smooth performance on any accelerated setup.
+
+
+### Usage
+
+In order to use embedding models and LLMs from NVIDIA AI, create an account on [NVIDIA NGC Service](https://catalog.ngc.nvidia.com/).
+
+Generate an API key from their dashboard. Set the API key as `NVIDIA_API_KEY` environment variable. Note that the `NVIDIA_API_KEY` will start with `nvapi-`.
+
+Below is an example of how to use LLM model and embedding model from NVIDIA AI:
+
+<CodeGroup>
+
+```python main.py
+import os
+from embedchain import App
+
+os.environ['NVIDIA_API_KEY'] = 'nvapi-xxxx'
+
+config = {
+    "app": {
+        "config": {
+            "id": "my-app",
+        },
+    },
+    "llm": {
+        "provider": "nvidia",
+        "config": {
+            "model": "nemotron_steerlm_8b",
+        },
+    },
+    "embedder": {
+        "provider": "nvidia",
+        "config": {
+            "model": "nvolveqa_40k",
+            "vector_dimension": 1024,
+        },
+    },
+}
+
+app = App.from_config(config=config)
+
+app.add("https://www.forbes.com/profile/elon-musk")
+answer = app.query("What is the net worth of Elon Musk today?")
+# Answer: The net worth of Elon Musk is subject to fluctuations based on the market value of his holdings in various companies.
+# As of March 1, 2024, his net worth is estimated to be approximately $210 billion. However, this figure can change rapidly due to stock market fluctuations and other factors.
+# Additionally, his net worth may include other assets such as real estate and art, which are not reflected in his stock portfolio.
+```
+</CodeGroup>
+
+
+## Cohere
+
+To use embedding models and LLMs from COHERE, create an account on [COHERE](https://dashboard.cohere.com/welcome/login?redirect_uri=%2Fapi-keys).
+
+Generate an API key from their dashboard. Set the API key as `COHERE_API_KEY` environment variable.
+
+Once you have obtained the key, you can use it like this:
+
+<CodeGroup>
+
+```python main.py
+import os
+from embedchain import App
+
+os.environ['COHERE_API_KEY'] = 'xxx'
+
+# load embedding model configuration from config.yaml file
+app = App.from_config(config_path="config.yaml")
+```
+
+```yaml config.yaml
+embedder:
+  provider: cohere
+  config:
+    model: 'embed-english-light-v3.0'
+```
+
+</CodeGroup>
+
+* Cohere has few embedding models: `embed-english-v3.0`, `embed-multilingual-v3.0`, `embed-multilingual-light-v3.0`, `embed-english-v2.0`, `embed-english-light-v2.0` and `embed-multilingual-v2.0`. Embedchain supports all these models. Below you can find YAML config for all:
+
+<CodeGroup>
+
+```yaml embed-english-v3.0.yaml
+embedder:
+  provider: cohere
+  config:
+    model: 'embed-english-v3.0'
+    vector_dimension: 1024
+```
+
+```yaml embed-multilingual-v3.0.yaml
+embedder:
+  provider: cohere
+  config:
+    model: 'embed-multilingual-v3.0'
+    vector_dimension: 1024
+```
+
+```yaml embed-multilingual-light-v3.0.yaml
+embedder:
+  provider: cohere
+  config:
+    model: 'embed-multilingual-light-v3.0'
+    vector_dimension: 384
+```
+
+```yaml embed-english-v2.0.yaml
+embedder:
+  provider: cohere
+  config:
+    model: 'embed-english-v2.0'
+    vector_dimension: 4096
+```
+
+```yaml embed-english-light-v2.0.yaml
+embedder:
+  provider: cohere
+  config:
+    model: 'embed-english-light-v2.0'
+    vector_dimension: 1024
+```
+
+```yaml embed-multilingual-v2.0.yaml
+embedder:
+  provider: cohere
+  config:
+    model: 'embed-multilingual-v2.0'
+    vector_dimension: 768
+```
+
+</CodeGroup>

docs/components/evaluation.mdx

@@ -0,0 +1,275 @@
+---
+title: 🔬 Evaluation
+---
+
+## Overview
+
+We provide out-of-the-box evaluation metrics for your RAG application. You can use them to evaluate your RAG applications and compare against different settings of your production RAG application.
+
+Currently, we provide support for following evaluation metrics:
+
+<CardGroup cols={3}>
+    <Card title="Context Relevancy" href="#context_relevancy"></Card>
+    <Card title="Answer Relevancy" href="#answer_relevancy"></Card>
+    <Card title="Groundedness" href="#groundedness"></Card>
+    <Card title="Custom Metric" href="#custom_metric"></Card>
+</CardGroup>
+
+## Quickstart
+
+Here is a basic example of running evaluation:
+
+```python example.py
+from embedchain import App
+
+app = App()
+
+# Add data sources
+app.add("https://www.forbes.com/profile/elon-musk")
+
+# Run evaluation
+app.evaluate(["What is the net worth of Elon Musk?", "How many companies Elon Musk owns?"])
+# {'answer_relevancy': 0.9987286412340826, 'groundedness': 1.0, 'context_relevancy': 0.3571428571428571}
+```
+
+Under the hood, Embedchain does the following:
+
+1. Runs semantic search in the vector database and fetches context
+2. LLM call with question, context to fetch the answer
+3. Run evaluation on following metrics: `context relevancy`, `groundedness`, and `answer relevancy` and return result
+
+## Advanced Usage
+
+We use OpenAI's `gpt-4` model as default LLM model for automatic evaluation. Hence, we require you to set `OPENAI_API_KEY` as an environment variable.
+
+### Step-1: Create dataset
+
+In order to evaluate your RAG application, you have to setup a dataset. A data point in the dataset consists of `questions`, `contexts`, `answer`. Here is an example of how to create a dataset for evaluation:
+
+```python
+from embedchain.utils.eval import EvalData
+
+data = [
+    {
+        "question": "What is the net worth of Elon Musk?",
+        "contexts": [
+            "Elon Musk PROFILEElon MuskCEO, ...",
+            "a Twitter poll on whether the journalists' ...",
+            "2016 and run by Jared Birchall.[335]...",
+        ],
+        "answer": "As of the information provided, Elon Musk's net worth is $241.6 billion.",
+    },
+    {
+        "question": "which companies does Elon Musk own?",
+        "contexts": [
+            "of December 2023[update], ...",
+            "ThielCofounderView ProfileTeslaHolds ...",
+            "Elon Musk PROFILEElon MuskCEO, ...",
+        ],
+        "answer": "Elon Musk owns several companies, including Tesla, SpaceX, Neuralink, and The Boring Company.",
+    },
+]
+
+dataset = []
+
+for d in data:
+    eval_data = EvalData(question=d["question"], contexts=d["contexts"], answer=d["answer"])
+    dataset.append(eval_data)
+```
+
+### Step-2: Run evaluation
+
+Once you have created your dataset, you can run evaluation on the dataset by picking the metric you want to run evaluation on.
+
+For example, you can run evaluation on context relevancy metric using the following code:
+
+```python
+from embedchain.evaluation.metrics import ContextRelevance
+metric = ContextRelevance()
+score = metric.evaluate(dataset)
+print(score)
+```
+
+You can choose a different metric or write your own to run evaluation on. You can check the following links:
+
+- [Context Relevancy](#context_relevancy)
+- [Answer relenvancy](#answer_relevancy)
+- [Groundedness](#groundedness)
+- [Build your own metric](#custom_metric)
+
+## Metrics
+
+### Context Relevancy <a id="context_relevancy"></a>
+
+Context relevancy is a metric to determine "how relevant the context is to the question". We use OpenAI's `gpt-4` model to determine the relevancy of the context. We achieve this by prompting the model with the question and the context and asking it to return relevant sentences from the context. We then use the following formula to determine the score:
+
+```
+context_relevance_score = num_relevant_sentences_in_context / num_of_sentences_in_context
+```
+
+#### Examples
+
+You can run the context relevancy evaluation with the following simple code:
+
+```python
+from embedchain.evaluation.metrics import ContextRelevance
+
+metric = ContextRelevance()
+score = metric.evaluate(dataset)  # 'dataset' is definted in the create dataset section
+print(score)
+# 0.27975528364849833
+```
+
+In the above example, we used sensible defaults for the evaluation. However, you can also configure the evaluation metric as per your needs using the `ContextRelevanceConfig` class.
+
+Here is a more advanced example of how to pass a custom evaluation config for evaluating on context relevance metric:
+
+```python
+from embedchain.config.evaluation.base import ContextRelevanceConfig
+from embedchain.evaluation.metrics import ContextRelevance
+
+eval_config = ContextRelevanceConfig(model="gpt-4", api_key="sk-xxx", language="en")
+metric = ContextRelevance(config=eval_config)
+metric.evaluate(dataset)
+```
+
+#### `ContextRelevanceConfig`
+
+<ParamField path="model" type="str" optional>
+    The model to use for the evaluation. Defaults to `gpt-4`. We only support openai's models for now.
+</ParamField>
+<ParamField path="api_key" type="str" optional>
+    The openai api key to use for the evaluation. Defaults to `None`. If not provided, we will use the `OPENAI_API_KEY` environment variable.
+</ParamField>
+<ParamField path="language" type="str" optional>
+    The language of the dataset being evaluated. We need this to determine the understand the context provided in the dataset. Defaults to `en`.
+</ParamField>
+<ParamField path="prompt" type="str" optional>
+    The prompt to extract the relevant sentences from the context. Defaults to `CONTEXT_RELEVANCY_PROMPT`, which can be found at `embedchain.config.evaluation.base` path.
+</ParamField>
+
+
+### Answer Relevancy <a id="answer_relevancy"></a>
+
+Answer relevancy is a metric to determine how relevant the answer is to the question. We prompt the model with the answer and asking it to generate questions from the answer. We then use the cosine similarity between the generated questions and the original question to determine the score.
+
+```
+answer_relevancy_score = mean(cosine_similarity(generated_questions, original_question))
+```
+
+#### Examples
+
+You can run the answer relevancy evaluation with the following simple code:
+
+```python
+from embedchain.evaluation.metrics import AnswerRelevance
+
+metric = AnswerRelevance()
+score = metric.evaluate(dataset)
+print(score)
+# 0.9505334177461916
+```
+
+In the above example, we used sensible defaults for the evaluation. However, you can also configure the evaluation metric as per your needs using the `AnswerRelevanceConfig` class. Here is a more advanced example where you can provide your own evaluation config:
+
+```python
+from embedchain.config.evaluation.base import AnswerRelevanceConfig
+from embedchain.evaluation.metrics import AnswerRelevance
+
+eval_config = AnswerRelevanceConfig(
+    model='gpt-4',
+    embedder="text-embedding-ada-002",
+    api_key="sk-xxx",
+    num_gen_questions=2
+)
+metric = AnswerRelevance(config=eval_config)
+score = metric.evaluate(dataset)
+```
+
+#### `AnswerRelevanceConfig`
+
+<ParamField path="model" type="str" optional>
+    The model to use for the evaluation. Defaults to `gpt-4`. We only support openai's models for now.
+</ParamField>
+<ParamField path="embedder" type="str" optional>
+    The embedder to use for embedding the text. Defaults to `text-embedding-ada-002`. We only support openai's embedders for now.
+</ParamField>
+<ParamField path="api_key" type="str" optional>
+    The openai api key to use for the evaluation. Defaults to `None`. If not provided, we will use the `OPENAI_API_KEY` environment variable.
+</ParamField>
+<ParamField path="num_gen_questions" type="int" optional>
+    The number of questions to generate for each answer. We use the generated questions to compare the similarity with the original question to determine the score. Defaults to `1`.
+</ParamField>
+<ParamField path="prompt" type="str" optional>
+    The prompt to extract the `num_gen_questions` number of questions from the provided answer. Defaults to `ANSWER_RELEVANCY_PROMPT`, which can be found at `embedchain.config.evaluation.base` path.
+</ParamField>
+
+## Groundedness <a id="groundedness"></a>
+
+Groundedness is a metric to determine how grounded the answer is to the context. We use OpenAI's `gpt-4` model to determine the groundedness of the answer. We achieve this by prompting the model with the answer and asking it to generate claims from the answer. We then again prompt the model with the context and the generated claims to determine the verdict on the claims. We then use the following formula to determine the score:
+
+```
+groundedness_score = (sum of all verdicts) / (total # of claims)
+```
+
+You can run the groundedness evaluation with the following simple code:
+
+```python
+from embedchain.evaluation.metrics import Groundedness
+metric = Groundedness()
+score = metric.evaluate(dataset)    # dataset from above
+print(score)
+# 1.0
+```
+
+In the above example, we used sensible defaults for the evaluation. However, you can also configure the evaluation metric as per your needs using the `GroundednessConfig` class. Here is a more advanced example where you can configure the evaluation config:
+
+```python
+from embedchain.config.evaluation.base import GroundednessConfig
+from embedchain.evaluation.metrics import Groundedness
+
+eval_config = GroundednessConfig(model='gpt-4', api_key="sk-xxx")
+metric = Groundedness(config=eval_config)
+score = metric.evaluate(dataset)
+```
+
+
+#### `GroundednessConfig`
+
+<ParamField path="model" type="str" optional>
+    The model to use for the evaluation. Defaults to `gpt-4`. We only support openai's models for now.
+</ParamField>
+<ParamField path="api_key" type="str" optional>
+    The openai api key to use for the evaluation. Defaults to `None`. If not provided, we will use the `OPENAI_API_KEY` environment variable.
+</ParamField>
+<ParamField path="answer_claims_prompt" type="str" optional>
+    The prompt to extract the claims from the provided answer. Defaults to `GROUNDEDNESS_ANSWER_CLAIMS_PROMPT`, which can be found at `embedchain.config.evaluation.base` path.
+</ParamField>
+<ParamField path="claims_inference_prompt" type="str" optional>
+    The prompt to get verdicts on the claims from the answer from the given context. Defaults to `GROUNDEDNESS_CLAIMS_INFERENCE_PROMPT`, which can be found at `embedchain.config.evaluation.base` path.
+</ParamField>
+
+## Custom <a id="custom_metric"></a>
+
+You can also create your own evaluation metric by extending the `BaseMetric` class. You can find the source code for the existing metrics at `embedchain.evaluation.metrics` path.
+
+<Note>
+You must provide the `name` of your custom metric in the `__init__` method of your class. This name will be used to identify your metric in the evaluation report.
+</Note>
+
+```python
+from typing import Optional
+
+from embedchain.config.base_config import BaseConfig
+from embedchain.evaluation.metrics import BaseMetric
+from embedchain.utils.eval import EvalData
+
+class MyCustomMetric(BaseMetric):
+    def __init__(self, config: Optional[BaseConfig] = None):
+        super().__init__(name="my_custom_metric")
+
+    def evaluate(self, dataset: list[EvalData]):
+        score = 0.0
+        # write your evaluation logic here
+        return score
+```

docs/components/introduction.mdx

@@ -9,4 +9,5 @@ You can configure following components
 * [Data Source](/components/data-sources/overview)
 * [LLM](/components/llms)
 * [Embedding Model](/components/embedding-models)
-* [Vector Database](/components/vector-databases)
+* [Vector Database](/components/vector-databases)
+* [Evaluation](/components/evaluation)

docs/components/llms.mdx

@@ -14,11 +14,16 @@ Embedchain comes with built-in support for various popular large language models
   <Card title="Cohere" href="#cohere"></Card>
   <Card title="Together" href="#together"></Card>
   <Card title="Ollama" href="#ollama"></Card>
+  <Card title="vLLM" href="#vllm"></Card>
   <Card title="GPT4All" href="#gpt4all"></Card>
   <Card title="JinaChat" href="#jinachat"></Card>
   <Card title="Hugging Face" href="#hugging-face"></Card>
   <Card title="Llama2" href="#llama2"></Card>
   <Card title="Vertex AI" href="#vertex-ai"></Card>
+  <Card title="Mistral AI" href="#mistral-ai"></Card>
+  <Card title="AWS Bedrock" href="#aws-bedrock"></Card>
+  <Card title="Groq" href="#groq"></Card>
+  <Card title="NVIDIA AI" href="#nvidia-ai"></Card>
 </CardGroup>
 
 ## OpenAI
@@ -65,125 +70,75 @@ llm:
 </CodeGroup>
 
 ### Function Calling
-To enable [function calling](https://platform.openai.com/docs/guides/function-calling) in your application using embedchain and OpenAI, you need to pass functions into `OpenAILlm` class as an array of functions. Here are several ways in which you can achieve that:
+Embedchain supports OpenAI [Function calling](https://platform.openai.com/docs/guides/function-calling) with a single function. It accepts inputs in accordance with the [Langchain interface](https://python.langchain.com/docs/modules/model_io/chat/function_calling#legacy-args-functions-and-function_call).
 
-Examples:
-<Accordion title="Using Pydantic Models">
+<Accordion title="Pydantic Model">
   ```python
-import os
-from embedchain import App
-from embedchain.llm.openai import OpenAILlm
-import requests
-from pydantic import BaseModel, Field, ValidationError, field_validator
+  from pydantic import BaseModel
 
-os.environ["OPENAI_API_KEY"] = "sk-xxx"
+  class multiply(BaseModel):
+      """Multiply two integers together."""
 
-class QA(BaseModel):
-    """
-    A question and answer pair.
-    """
-
-    question: str = Field(
-        ..., description="The question.", example="What is a mountain?"
-    )
-    answer: str = Field(
-        ..., description="The answer.", example="A mountain is a hill."
-    )
-    person_who_is_asking: str = Field(
-        ..., description="The person who is asking the question.", example="John"
-    )
-
-    @field_validator("question")
-    def question_must_end_with_a_question_mark(cls, v):
-        """
-        Validate that the question ends with a question mark.
-        """
-        if not v.endswith("?"):
-            raise ValueError("question must end with a question mark")
-        return v
-
-    @field_validator("answer")
-    def answer_must_end_with_a_period(cls, v):
-        """
-        Validate that the answer ends with a period.
-        """
-        if not v.endswith("."):
-            raise ValueError("answer must end with a period")
-        return v
-
-llm = OpenAILlm(config=None,functions=[QA])
-app = App(llm=llm)
-
-result = app.query("Hey I am Sid. What is a mountain? A mountain is a hill.")
-
-print(result)
+      a: int = Field(..., description="First integer")
+      b: int = Field(..., description="Second integer")
   ```
-  </Accordion>
-  
-  <Accordion title="Using OpenAI JSON schema">
+</Accordion>
+
+<Accordion title="Python function">
+  ```python
+  def multiply(a: int, b: int) -> int:
+      """Multiply two integers together.
+
+      Args:
+          a: First integer
+          b: Second integer
+      """
+      return a * b
+  ```
+</Accordion>
+<Accordion title="OpenAI tool dictionary">
+  ```python
+  multiply = {
+    "type": "function",
+    "function": {
+      "name": "multiply",
+      "description": "Multiply two integers together.",
+      "parameters": {
+        "type": "object",
+        "properties": {
+          "a": {
+            "description": "First integer",
+            "type": "integer"
+          },
+          "b": {
+            "description": "Second integer",
+            "type": "integer"
+          }
+        },
+        "required": [
+          "a",
+          "b"
+        ]
+      }
+    }
+  }
+  ```
+</Accordion>
+
+With any of the previous inputs, the OpenAI LLM can be queried to provide the appropriate arguments for the function.
+
 ```python
 import os
 from embedchain import App
 from embedchain.llm.openai import OpenAILlm
-import requests
-from pydantic import BaseModel, Field, ValidationError, field_validator
 
 os.environ["OPENAI_API_KEY"] = "sk-xxx"
 
-json_schema = {
-    "name": "get_qa",
-    "description": "A question and answer pair and the user who is asking the question.",
-    "parameters": {
-        "type": "object",
-        "properties": {
-            "question": {"type": "string", "description": "The question."},
-            "answer": {"type": "string", "description": "The answer."},
-            "person_who_is_asking": {
-                "type": "string",
-                "description": "The person who is asking the question.",
-            }
-        },
-        "required": ["question", "answer", "person_who_is_asking"],
-    },
-}
-
-llm = OpenAILlm(config=None,functions=[json_schema])
+llm = OpenAILlm(tools=multiply)
 app = App(llm=llm)
 
-result = app.query("Hey I am Sid. What is a mountain? A mountain is a hill.")
-
-print(result)
-  ```
-  </Accordion>
-  <Accordion title="Using actual python functions">
-  ```python
-import os
-from embedchain import App
-from embedchain.llm.openai import OpenAILlm
-import requests
-from pydantic import BaseModel, Field, ValidationError, field_validator
-
-os.environ["OPENAI_API_KEY"] = "sk-xxx"
-
-def find_info_of_pokemon(pokemon: str):
-    """
-    Find the information of the given pokemon.
-    Args:
-        pokemon: The pokemon.
-    """
-    req = requests.get(f"https://pokeapi.co/api/v2/pokemon/{pokemon}")
-    if req.status_code == 404:
-        raise ValueError("pokemon not found")
-    return req.json()
-
-llm = OpenAILlm(config=None,functions=[find_info_of_pokemon])
-app = App(llm=llm)
-
-result = app.query("Tell me more about the pokemon pikachu.")
-
-print(result)
+result = app.query("What is the result of 125 multiplied by fifteen?")
 ```
-</Accordion>
 
 ## Google AI
 
@@ -249,7 +204,7 @@ app = App.from_config(config_path="config.yaml")
 llm:
   provider: azure_openai
   config:
-    model: gpt-35-turbo
+    model: gpt-3.5-turbo
     deployment_name: your_llm_deployment_name
     temperature: 0.5
     max_tokens: 1000
@@ -393,6 +348,35 @@ llm:
 
 </CodeGroup>
 
+
+## vLLM
+
+Setup vLLM by following instructions given in [their docs](https://docs.vllm.ai/en/latest/getting_started/installation.html).
+
+<CodeGroup>
+
+```python main.py
+import os
+from embedchain import App
+
+# load llm configuration from config.yaml file
+app = App.from_config(config_path="config.yaml")
+```
+
+```yaml config.yaml
+llm:
+  provider: vllm
+  config:
+    model: 'meta-llama/Llama-2-70b-hf'
+    temperature: 0.5
+    top_p: 1
+    top_k: 10
+    stream: true
+    trust_remote_code: true
+```
+
+</CodeGroup>
+
 ## GPT4ALL
 
 Install related dependencies using the following command:
@@ -468,7 +452,15 @@ pip install --upgrade 'embedchain[huggingface-hub]'
 
 First, set `HUGGINGFACE_ACCESS_TOKEN` in environment variable which you can obtain from [their platform](https://huggingface.co/settings/tokens).
 
-Once you have the token, load the app using the config yaml file:
+You can load the LLMs from Hugging Face using three ways:
+
+- [Hugging Face Hub](#hugging-face-hub)
+- [Hugging Face Local Pipelines](#hugging-face-local-pipelines)
+- [Hugging Face Inference Endpoint](#hugging-face-inference-endpoint)
+
+### Hugging Face Hub
+
+To load the model from Hugging Face Hub, use the following code:
 
 <CodeGroup>
 
@@ -478,22 +470,78 @@ from embedchain import App
 
 os.environ["HUGGINGFACE_ACCESS_TOKEN"] = "xxx"
 
-# load llm configuration from config.yaml file
-app = App.from_config(config_path="config.yaml")
-```
+config = {
+  "app": {"config": {"id": "my-app"}},
+  "llm": {
+      "provider": "huggingface",
+      "config": {
+          "model": "bigscience/bloom-1b7",
+          "top_p": 0.5,
+          "max_length": 200,
+          "temperature": 0.1,
+      },
+  },
+}
 
-```yaml config.yaml
-llm:
-  provider: huggingface
-  config:
-    model: 'google/flan-t5-xxl'
-    temperature: 0.5
-    max_tokens: 1000
-    top_p: 0.5
-    stream: false
+app = App.from_config(config=config)
 ```
 </CodeGroup>
 
+### Hugging Face Local Pipelines
+
+If you want to load the locally downloaded model from Hugging Face, you can do so by following the code provided below:
+
+<CodeGroup>
+```python main.py
+from embedchain import App
+
+config = {
+  "app": {"config": {"id": "my-app"}},
+  "llm": {
+      "provider": "huggingface",
+      "config": {
+          "model": "Trendyol/Trendyol-LLM-7b-chat-v0.1",
+          "local": True,  # Necessary if you want to run model locally
+          "top_p": 0.5,
+          "max_tokens": 1000,
+          "temperature": 0.1,
+      },
+  }
+}
+app = App.from_config(config=config)
+```
+</CodeGroup>
+
+### Hugging Face Inference Endpoint
+
+You can also use [Hugging Face Inference Endpoints](https://huggingface.co/docs/inference-endpoints/index#-inference-endpoints) to access custom endpoints. First, set the `HUGGINGFACE_ACCESS_TOKEN` as above.
+
+Then, load the app using the config yaml file:
+
+<CodeGroup>
+
+```python main.py
+from embedchain import App
+
+config = {
+  "app": {"config": {"id": "my-app"}},
+  "llm": {
+      "provider": "huggingface",
+      "config": {
+        "endpoint": "https://api-inference.huggingface.co/models/gpt2",
+        "model_params": {"temprature": 0.1, "max_new_tokens": 100}
+      },
+  },
+}
+app = App.from_config(config=config)
+
+```
+</CodeGroup>
+
+Currently only supports `text-generation` and `text2text-generation` for now [[ref](https://api.python.langchain.com/en/latest/llms/langchain_community.llms.huggingface_endpoint.HuggingFaceEndpoint.html?highlight=huggingfaceendpoint#)].
+
+See langchain's [hugging face endpoint](https://python.langchain.com/docs/integrations/chat/huggingface#huggingfaceendpoint) for more information. 
+
 ## Llama2
 
 Llama2 is integrated through [Replicate](https://replicate.com/).  Set `REPLICATE_API_TOKEN` in environment variable which you can obtain from [their platform](https://replicate.com/account/api-tokens).
@@ -547,5 +595,194 @@ llm:
 ```
 </CodeGroup>
 
+
+## Mistral AI
+
+Obtain the Mistral AI api key from their [console](https://console.mistral.ai/).
+
+<CodeGroup>
+ 
+ ```python main.py
+os.environ["MISTRAL_API_KEY"] = "xxx"
+
+app = App.from_config(config_path="config.yaml")
+
+app.add("https://www.forbes.com/profile/elon-musk")
+
+response = app.query("what is the net worth of Elon Musk?")
+# As of January 16, 2024, Elon Musk's net worth is $225.4 billion.
+
+response = app.chat("which companies does elon own?")
+# Elon Musk owns Tesla, SpaceX, Boring Company, Twitter, and X.
+
+response = app.chat("what question did I ask you already?")
+# You have asked me several times already which companies Elon Musk owns, specifically Tesla, SpaceX, Boring Company, Twitter, and X.
+```
+  
+```yaml config.yaml
+llm:
+  provider: mistralai
+  config:
+    model: mistral-tiny
+    temperature: 0.5
+    max_tokens: 1000
+    top_p: 1
+embedder:
+  provider: mistralai
+  config:
+    model: mistral-embed
+```
+</CodeGroup>
+
+
+## AWS Bedrock
+
+### Setup
+- Before using the AWS Bedrock LLM, make sure you have the appropriate model access from [Bedrock Console](https://us-east-1.console.aws.amazon.com/bedrock/home?region=us-east-1#/modelaccess).
+- You will also need to authenticate the `boto3` client by using a method in the [AWS documentation](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html#configuring-credentials)
+- You can optionally export an `AWS_REGION`
+
+
+### Usage
+
+<CodeGroup>
+
+```python main.py
+import os
+from embedchain import App
+
+os.environ["AWS_ACCESS_KEY_ID"] = "xxx"
+os.environ["AWS_SECRET_ACCESS_KEY"] = "xxx"
+os.environ["AWS_REGION"] = "us-west-2"
+
+app = App.from_config(config_path="config.yaml")
+```
+
+```yaml config.yaml
+llm:
+  provider: aws_bedrock
+  config:
+    model: amazon.titan-text-express-v1
+    # check notes below for model_kwargs
+    model_kwargs:
+      temperature: 0.5
+      topP: 1
+      maxTokenCount: 1000
+```
+</CodeGroup>
+
+<br />
+<Note>
+  The model arguments are different for each providers. Please refer to the [AWS Bedrock Documentation](https://us-east-1.console.aws.amazon.com/bedrock/home?region=us-east-1#/providers) to find the appropriate arguments for your model.
+</Note>
+
 <br/ >
+
+## Groq
+
+[Groq](https://groq.com/) is the creator of the world's first Language Processing Unit (LPU), providing exceptional speed performance for AI workloads running on their LPU Inference Engine.
+
+
+### Usage
+
+In order to use LLMs from Groq, go to their [platform](https://console.groq.com/keys) and get the API key.
+
+Set the API key as `GROQ_API_KEY` environment variable or pass in your app configuration to use the model as given below in the example.
+
+<CodeGroup>
+
+```python main.py
+import os
+from embedchain import App
+
+# Set your API key here or pass as the environment variable
+groq_api_key = "gsk_xxxx"
+
+config = {
+    "llm": {
+        "provider": "groq",
+        "config": {
+            "model": "mixtral-8x7b-32768",
+            "api_key": groq_api_key,
+            "stream": True
+        }
+    }
+}
+
+app = App.from_config(config=config)
+# Add your data source here
+app.add("https://docs.embedchain.ai/sitemap.xml", data_type="sitemap")
+app.query("Write a poem about Embedchain")
+
+# In the realm of data, vast and wide,
+# Embedchain stands with knowledge as its guide.
+# A platform open, for all to try,
+# Building bots that can truly fly.
+
+# With REST API, data in reach,
+# Deployment a breeze, as easy as a speech.
+# Updating data sources, anytime, anyday,
+# Embedchain's power, never sway.
+
+# A knowledge base, an assistant so grand,
+# Connecting to platforms, near and far.
+# Discord, WhatsApp, Slack, and more,
+# Embedchain's potential, never a bore.
+```
+</CodeGroup>
+
+## NVIDIA AI
+
+[NVIDIA AI Foundation Endpoints](https://www.nvidia.com/en-us/ai-data-science/foundation-models/) let you quickly use NVIDIA's AI models, such as Mixtral 8x7B, Llama 2 etc, through our API. These models are available in the [NVIDIA NGC catalog](https://catalog.ngc.nvidia.com/ai-foundation-models), fully optimized and ready to use on NVIDIA's AI platform. They are designed for high speed and easy customization, ensuring smooth performance on any accelerated setup.
+
+
+### Usage
+
+In order to use LLMs from NVIDIA AI, create an account on [NVIDIA NGC Service](https://catalog.ngc.nvidia.com/).
+
+Generate an API key from their dashboard. Set the API key as `NVIDIA_API_KEY` environment variable. Note that the `NVIDIA_API_KEY` will start with `nvapi-`.
+
+Below is an example of how to use LLM model and embedding model from NVIDIA AI:
+
+<CodeGroup>
+
+```python main.py
+import os
+from embedchain import App
+
+os.environ['NVIDIA_API_KEY'] = 'nvapi-xxxx'
+
+config = {
+    "app": {
+        "config": {
+            "id": "my-app",
+        },
+    },
+    "llm": {
+        "provider": "nvidia",
+        "config": {
+            "model": "nemotron_steerlm_8b",
+        },
+    },
+    "embedder": {
+        "provider": "nvidia",
+        "config": {
+            "model": "nvolveqa_40k",
+            "vector_dimension": 1024,
+        },
+    },
+}
+
+app = App.from_config(config=config)
+
+app.add("https://www.forbes.com/profile/elon-musk")
+answer = app.query("What is the net worth of Elon Musk today?")
+# Answer: The net worth of Elon Musk is subject to fluctuations based on the market value of his holdings in various companies.
+# As of March 1, 2024, his net worth is estimated to be approximately $210 billion. However, this figure can change rapidly due to stock market fluctuations and other factors.
+# Additionally, his net worth may include other assets such as real estate and art, which are not reflected in his stock portfolio.
+```
+</CodeGroup>
+
+<br/ >
+
 <Snippet file="missing-llm-tip.mdx" />

docs/components/vector-databases.mdx

@@ -17,216 +17,4 @@ Utilizing a vector database alongside Embedchain is a seamless process. All you
   <Card title="Weaviate" href="#weaviate"></Card>
 </CardGroup>
 
-## ChromaDB
-
-<CodeGroup>
-
-```python main.py
-from embedchain import App
-
-# load chroma configuration from yaml file
-app = App.from_config(config_path="config1.yaml")
-```
-
-```yaml config1.yaml
-vectordb:
-  provider: chroma
-  config:
-    collection_name: 'my-collection'
-    dir: db
-    allow_reset: true
-```
-
-```yaml config2.yaml
-vectordb:
-  provider: chroma
-  config:
-    collection_name: 'my-collection'
-    host: localhost
-    port: 5200
-    allow_reset: true
-```
-
-</CodeGroup>
-
-
-## Elasticsearch
-
-Install related dependencies using the following command:
-
-```bash
-pip install --upgrade 'embedchain[elasticsearch]'
-```
-
-<Note>
-You can configure the Elasticsearch connection by providing either `es_url` or `cloud_id`. If you are using the Elasticsearch Service on Elastic Cloud, you can find the `cloud_id` on the [Elastic Cloud dashboard](https://cloud.elastic.co/deployments).
-</Note>
-
-You can authorize the connection to Elasticsearch by providing either `basic_auth`, `api_key`, or `bearer_auth`.
-
-<CodeGroup>
-
-```python main.py
-from embedchain import App
-
-# load elasticsearch configuration from yaml file
-app = App.from_config(config_path="config.yaml")
-```
-
-```yaml config.yaml
-vectordb:
-  provider: elasticsearch
-  config:
-    collection_name: 'es-index'
-    cloud_id: 'deployment-name:xxxx'
-    basic_auth:
-      - elastic
-      - <your_password>
-    verify_certs: false
-```
-</CodeGroup>
-
-## OpenSearch
-
-Install related dependencies using the following command:
-
-```bash
-pip install --upgrade 'embedchain[opensearch]'
-```
-
-<CodeGroup>
-
-```python main.py
-from embedchain import App
-
-# load opensearch configuration from yaml file
-app = App.from_config(config_path="config.yaml")
-```
-
-```yaml config.yaml
-vectordb:
-  provider: opensearch
-  config:
-    collection_name: 'my-app'
-    opensearch_url: 'https://localhost:9200'
-    http_auth:
-      - admin
-      - admin
-    vector_dimension: 1536
-    use_ssl: false
-    verify_certs: false
-```
-
-</CodeGroup>
-
-## Zilliz
-
-Install related dependencies using the following command:
-
-```bash
-pip install --upgrade 'embedchain[milvus]'
-```
-
-Set the Zilliz environment variables `ZILLIZ_CLOUD_URI` and `ZILLIZ_CLOUD_TOKEN` which you can find it on their [cloud platform](https://cloud.zilliz.com/).
-
-<CodeGroup>
-
-```python main.py
-import os
-from embedchain import App
-
-os.environ['ZILLIZ_CLOUD_URI'] = 'https://xxx.zillizcloud.com'
-os.environ['ZILLIZ_CLOUD_TOKEN'] = 'xxx'
-
-# load zilliz configuration from yaml file
-app = App.from_config(config_path="config.yaml")
-```
-
-```yaml config.yaml
-vectordb:
-  provider: zilliz
-  config:
-    collection_name: 'zilliz_app'
-    uri: https://xxxx.api.gcp-region.zillizcloud.com
-    token: xxx
-    vector_dim: 1536
-    metric_type: L2
-```
-
-</CodeGroup>
-
-## LanceDB
-
-_Coming soon_
-
-## Pinecone
-
-Install pinecone related dependencies using the following command:
-
-```bash
-pip install --upgrade 'embedchain[pinecone]'
-```
-
-In order to use Pinecone as vector database, set the environment variables `PINECONE_API_KEY` and `PINECONE_ENV` which you can find on [Pinecone dashboard](https://app.pinecone.io/).
-
-<CodeGroup>
-
-```python main.py
-from embedchain import App
-
-# load pinecone configuration from yaml file
-app = App.from_config(config_path="config.yaml")
-```
-
-```yaml config.yaml
-vectordb:
-  provider: pinecone
-  config:
-    metric: cosine
-    vector_dimension: 1536
-    collection_name: my-pinecone-index
-```
-
-</CodeGroup>
-
-## Qdrant
-
-In order to use Qdrant as a vector database, set the environment variables `QDRANT_URL` and `QDRANT_API_KEY` which you can find on [Qdrant Dashboard](https://cloud.qdrant.io/).
-
-<CodeGroup>
-```python main.py
-from embedchain import App
-
-# load qdrant configuration from yaml file
-app = App.from_config(config_path="config.yaml")
-```
-
-```yaml config.yaml
-vectordb:
-  provider: qdrant
-  config:
-    collection_name: my_qdrant_index
-```
-</CodeGroup>
-
-## Weaviate
-
-In order to use Weaviate as a vector database, set the environment variables `WEAVIATE_ENDPOINT` and `WEAVIATE_API_KEY` which you can find on [Weaviate dashboard](https://console.weaviate.cloud/dashboard).
-
-<CodeGroup>
-```python main.py
-from embedchain import App
-
-# load weaviate configuration from yaml file
-app = App.from_config(config_path="config.yaml")
-```
-
-```yaml config.yaml
-vectordb:
-  provider: weaviate
-  config:
-    collection_name: my_weaviate_index
-```
-</CodeGroup>
-
 <Snippet file="missing-vector-db-tip.mdx" />

docs/components/vector-databases/chromadb.mdx

@@ -0,0 +1,35 @@
+---
+title: ChromaDB
+---
+
+<CodeGroup>
+
+```python main.py
+from embedchain import App
+
+# load chroma configuration from yaml file
+app = App.from_config(config_path="config1.yaml")
+```
+
+```yaml config1.yaml
+vectordb:
+  provider: chroma
+  config:
+    collection_name: 'my-collection'
+    dir: db
+    allow_reset: true
+```
+
+```yaml config2.yaml
+vectordb:
+  provider: chroma
+  config:
+    collection_name: 'my-collection'
+    host: localhost
+    port: 5200
+    allow_reset: true
+```
+
+</CodeGroup>
+
+<Snippet file="missing-vector-db-tip.mdx" />

docs/components/vector-databases/elasticsearch.mdx

@@ -0,0 +1,39 @@
+---
+title: Elasticsearch
+---
+
+Install related dependencies using the following command:
+
+```bash
+pip install --upgrade 'embedchain[elasticsearch]'
+```
+
+<Note>
+You can configure the Elasticsearch connection by providing either `es_url` or `cloud_id`. If you are using the Elasticsearch Service on Elastic Cloud, you can find the `cloud_id` on the [Elastic Cloud dashboard](https://cloud.elastic.co/deployments).
+</Note>
+
+You can authorize the connection to Elasticsearch by providing either `basic_auth`, `api_key`, or `bearer_auth`.
+
+<CodeGroup>
+
+```python main.py
+from embedchain import App
+
+# load elasticsearch configuration from yaml file
+app = App.from_config(config_path="config.yaml")
+```
+
+```yaml config.yaml
+vectordb:
+  provider: elasticsearch
+  config:
+    collection_name: 'es-index'
+    cloud_id: 'deployment-name:xxxx'
+    basic_auth:
+      - elastic
+      - <your_password>
+    verify_certs: false
+```
+</CodeGroup>
+
+<Snippet file="missing-vector-db-tip.mdx" />

docs/components/vector-databases/opensearch.mdx

@@ -0,0 +1,36 @@
+---
+title: OpenSearch
+---
+
+Install related dependencies using the following command:
+
+```bash
+pip install --upgrade 'embedchain[opensearch]'
+```
+
+<CodeGroup>
+
+```python main.py
+from embedchain import App
+
+# load opensearch configuration from yaml file
+app = App.from_config(config_path="config.yaml")
+```
+
+```yaml config.yaml
+vectordb:
+  provider: opensearch
+  config:
+    collection_name: 'my-app'
+    opensearch_url: 'https://localhost:9200'
+    http_auth:
+      - admin
+      - admin
+    vector_dimension: 1536
+    use_ssl: false
+    verify_certs: false
+```
+
+</CodeGroup>
+
+<Snippet file="missing-vector-db-tip.mdx" />

docs/components/vector-databases/pinecone.mdx

@@ -0,0 +1,109 @@
+---
+title: Pinecone
+---
+
+## Overview
+
+Install pinecone related dependencies using the following command:
+
+```bash
+pip install --upgrade 'pinecone-client pinecone-text'
+```
+
+In order to use Pinecone as vector database, set the environment variable `PINECONE_API_KEY` which you can find on [Pinecone dashboard](https://app.pinecone.io/).
+
+<CodeGroup>
+
+```python main.py
+from embedchain import App
+
+# Load pinecone configuration from yaml file
+app = App.from_config(config_path="pod_config.yaml")
+# Or
+app = App.from_config(config_path="serverless_config.yaml")
+```
+
+```yaml pod_config.yaml
+vectordb:
+  provider: pinecone
+  config:
+    metric: cosine
+    vector_dimension: 1536
+    index_name: my-pinecone-index
+    pod_config:
+      environment: gcp-starter
+      metadata_config:
+        indexed:
+          - "url"
+          - "hash"
+```
+
+```yaml serverless_config.yaml
+vectordb:
+  provider: pinecone
+  config:
+    metric: cosine
+    vector_dimension: 1536
+    index_name: my-pinecone-index
+    serverless_config:
+      cloud: aws
+      region: us-west-2
+```
+
+</CodeGroup>
+
+<br />
+<Note>
+You can find more information about Pinecone configuration [here](https://docs.pinecone.io/docs/manage-indexes#create-a-pod-based-index).
+You can also optionally provide `index_name` as a config param in yaml file to specify the index name. If not provided, the index name will be `{collection_name}-{vector_dimension}`.
+</Note>
+
+## Usage
+
+### Hybrid search
+
+Here is an example of how you can do hybrid search using Pinecone as a vector database through Embedchain.
+
+```python
+import os
+
+from embedchain import App
+
+config = {
+    'app': {
+        "config": {
+            "id": "ec-docs-hybrid-search"
+        }
+    },
+    'vectordb': {
+        'provider': 'pinecone',
+        'config': {
+            'metric': 'dotproduct',
+            'vector_dimension': 1536,
+            'index_name': 'my-index',
+            'serverless_config': {
+                'cloud': 'aws',
+                'region': 'us-west-2'
+            },
+            'hybrid_search': True, # Remember to set this for hybrid search
+        }
+    }
+}
+
+# Initialize app
+app = App.from_config(config=config)
+
+# Add documents
+app.add("/path/to/file.pdf", data_type="pdf_file", namespace="my-namespace")
+
+# Query
+app.query("<YOUR QUESTION HERE>", namespace="my-namespace")
+
+# Chat
+app.chat("<YOUR QUESTION HERE>", namespace="my-namespace")
+```
+
+Under the hood, Embedchain fetches the relevant chunks from the documents you added by doing hybrid search on the pinecone index.
+If you have questions on how pinecone hybrid search works, please refer to their [offical documentation here](https://docs.pinecone.io/docs/hybrid-search).
+
+<Snippet file="missing-vector-db-tip.mdx" />

docs/components/vector-databases/qdrant.mdx

@@ -0,0 +1,23 @@
+---
+title: Qdrant
+---
+
+In order to use Qdrant as a vector database, set the environment variables `QDRANT_URL` and `QDRANT_API_KEY` which you can find on [Qdrant Dashboard](https://cloud.qdrant.io/).
+
+<CodeGroup>
+```python main.py
+from embedchain import App
+
+# load qdrant configuration from yaml file
+app = App.from_config(config_path="config.yaml")
+```
+
+```yaml config.yaml
+vectordb:
+  provider: qdrant
+  config:
+    collection_name: my_qdrant_index
+```
+</CodeGroup>
+
+<Snippet file="missing-vector-db-tip.mdx" />

docs/components/vector-databases/weaviate.mdx

@@ -0,0 +1,24 @@
+---
+title: Weaviate
+---
+
+
+In order to use Weaviate as a vector database, set the environment variables `WEAVIATE_ENDPOINT` and `WEAVIATE_API_KEY` which you can find on [Weaviate dashboard](https://console.weaviate.cloud/dashboard).
+
+<CodeGroup>
+```python main.py
+from embedchain import App
+
+# load weaviate configuration from yaml file
+app = App.from_config(config_path="config.yaml")
+```
+
+```yaml config.yaml
+vectordb:
+  provider: weaviate
+  config:
+    collection_name: my_weaviate_index
+```
+</CodeGroup>
+
+<Snippet file="missing-vector-db-tip.mdx" />

docs/components/vector-databases/zilliz.mdx

@@ -0,0 +1,39 @@
+---
+title: Zilliz
+---
+
+Install related dependencies using the following command:
+
+```bash
+pip install --upgrade 'embedchain[milvus]'
+```
+
+Set the Zilliz environment variables `ZILLIZ_CLOUD_URI` and `ZILLIZ_CLOUD_TOKEN` which you can find it on their [cloud platform](https://cloud.zilliz.com/).
+
+<CodeGroup>
+
+```python main.py
+import os
+from embedchain import App
+
+os.environ['ZILLIZ_CLOUD_URI'] = 'https://xxx.zillizcloud.com'
+os.environ['ZILLIZ_CLOUD_TOKEN'] = 'xxx'
+
+# load zilliz configuration from yaml file
+app = App.from_config(config_path="config.yaml")
+```
+
+```yaml config.yaml
+vectordb:
+  provider: zilliz
+  config:
+    collection_name: 'zilliz_app'
+    uri: https://xxxx.api.gcp-region.zillizcloud.com
+    token: xxx
+    vector_dim: 1536
+    metric_type: L2
+```
+
+</CodeGroup>
+
+<Snippet file="missing-vector-db-tip.mdx" />

docs/deployment/embedchain_ai.mdx

@@ -5,31 +5,10 @@ description: 'Deploy your RAG application to embedchain.ai platform'
 
 ## Deploy on Embedchain Platform
 
-Embedchain enables developers to deploy their LLM-powered apps in production using the [Embedchain platform](https://app.embedchain.ai). The platform offers free access to context on your data through its REST API. Once the pipeline is deployed, you can update your data sources anytime after deployment.
+Embedchain enables developers to deploy their LLM-powered apps in production using the Embedchain platform. The platform offers free access to context on your data through its REST API. Once the pipeline is deployed, you can update your data sources anytime after deployment.
 
-See the example below on how to use the deploy your app (for free):
+Deployment to Embedchain Platform is currently available on an invitation-only basis. To request access, please submit your information via the provided [Google Form](https://forms.gle/vigN11h7b4Ywat668). We will review your request and respond promptly.
 
-```python
-from embedchain import App
-
-# Initialize app
-app = App()
-
-# Add data source
-app.add("https://www.forbes.com/profile/elon-musk")
-
-# Deploy your pipeline to Embedchain Platform
-app.deploy()
-
-# 🔑 Enter your Embedchain API key. You can find the API key at https://app.embedchain.ai/settings/keys/
-# ec-xxxxxx
-
-# 🛠️ Creating pipeline on the platform...
-# 🎉🎉🎉 Pipeline created successfully! View your pipeline: https://app.embedchain.ai/pipelines/xxxxx
-
-# 🛠️ Adding data to your pipeline...
-# ✅ Data of type: web_page, value: https://www.forbes.com/profile/elon-musk added successfully.
-```
 
 ## Seeking help?
 

docs/deployment/railway.mdx

@@ -0,0 +1,86 @@
+---
+title: 'Railway.app'
+description: 'Deploy your RAG application to railway.app'
+---
+
+It's easy to host your Embedchain-powered apps and APIs on railway.
+
+Follow the instructions given below to deploy your first application quickly:
+
+## Step-1: Create RAG app
+
+```bash Install embedchain
+pip install embedchain
+```
+
+<Tip>
+**Create a full stack app using Embedchain CLI**
+
+To use your hosted embedchain RAG app, you can easily set up a FastAPI server that can be used anywhere.
+To easily set up a FastAPI server, check out [Get started with Full stack](https://docs.embedchain.ai/get-started/full-stack) page.
+
+Hosting this server on railway is super easy!
+
+</Tip>
+
+## Step-2: Set up your project
+
+### With Docker
+
+You can create a `Dockerfile` in the root of the project, with all the instructions. However, this method is sometimes slower in deployment.
+
+### Without Docker
+
+By default, Railway uses Python 3.7. Embedchain requires the python version to be >3.9 in order to install.
+
+To fix this, create a `.python-version` file in the root directory of your project and specify the correct version
+
+```bash .python-version
+3.10
+```
+
+You also need to create a `requirements.txt` file to specify the requirements.
+
+```bash requirements.txt
+python-dotenv
+embedchain
+fastapi==0.108.0
+uvicorn==0.25.0
+embedchain
+beautifulsoup4
+sentence-transformers
+```
+
+## Step-3: Deploy to Railway 🚀
+
+1. Go to https://railway.app and create an account.
+2. Create a project by clicking on the "Start a new project" button
+
+### With Github
+
+Select `Empty Project` or `Deploy from Github Repo`. 
+
+You should be all set!
+
+### Without Github
+
+You can also use the railway CLI to deploy your apps from the terminal, if you don't want to connect a git repository.
+
+To do this, just run this command in your terminal
+
+```bash Install and set up railway CLI
+npm i -g @railway/cli
+railway login
+railway link [projectID]
+```
+
+Finally, run `railway up` to deploy your app.
+```bash Deploy
+railway up
+```
+
+## Seeking help?
+
+If you run into issues with deployment, please feel free to reach out to us via any of the following methods:
+
+<Snippet file="get-help.mdx" />

docs/examples/nextjs-assistant.mdx

@@ -0,0 +1,124 @@
+Fork the Embedchain repo on [Github](https://github.com/embedchain/embedchain) to create your own NextJS discord and slack bot powered by Embedchain.
+
+If you run into problems with forking, please refer to [github docs](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/working-with-forks/fork-a-repo) for forking a repo.
+
+We will work from the `examples/nextjs` folder so change your current working directory by running the command - `cd <your_forked_repo>/examples/nextjs`
+
+# Installation
+
+First, lets start by install all the required packages and dependencies.
+
+- Install all the required python packages by running ```pip install -r requirements.txt```
+
+- We will use [Fly.io](https://fly.io/) to deploy our embedchain app, discord and slack bot. Follow the step one to install [Fly.io CLI](https://docs.embedchain.ai/deployment/fly_io#step-1-install-flyctl-command-line)
+
+# Developement
+
+## Embedchain App
+
+First, we need an Embedchain app powered with the knowledge of NextJS. We have already created an embedchain app using FastAPI in `ec_app` folder for you. Feel free to ingest data of your choice to power the App.
+
+<Note>
+Navigate to `ec_app` folder and create `.env` file in this folder and set your OpenAI API key as shown in `.env.example` file. If you want to use other open-source models, feel free to use the app config in `app.py`. More details for using custom configuration for Embedchain app is [available here](https://docs.embedchain.ai/api-reference/advanced/configuration).
+</Note>
+
+Before running the ec commands to develope the app, open `fly.toml` file and update the `name` variable to something unique. This is important as `fly.io` requires users to provide a globally unique deployment app names.
+
+Now, we need to launch this application with fly.io. You can see your app on [fly.io dashboard](https://fly.io/dashboard). Run the following command to launch your app on fly.io:
+```bash
+fly launch --no-deploy
+```
+
+To run the app in development, run the following command:
+
+```bash
+ec dev
+```
+
+Run `ec deploy` to deploy your app on Fly.io. Once you deploy your app, save the endpoint on which our discord and slack bot will send requests.
+
+
+## Discord bot
+
+For discord bot, you will need to create the bot on discord developer portal and get the discord bot token and your discord bot name.
+
+While keeping in mind the following note, create the discord bot by following the instructions from our [discord bot docs](https://docs.embedchain.ai/examples/discord_bot) and get discord bot token.
+
+<Note>
+You do not need to set `OPENAI_API_KEY` to run this discord bot. Follow the remaining instructions to create a discord bot app. We recommend you to give the following sets of bot permissions to run the discord bot without errors:
+
+```
+(General Permissions)
+Read Message/View Channels
+
+(Text Permissions)
+Send Messages
+Create Public Thread
+Create Private Thread
+Send Messages in Thread
+Manage Threads
+Embed Links
+Read Message History
+```
+</Note>
+
+Once you have your discord bot token and discord app name. Navigate to `nextjs_discord` folder and create `.env` file and define your discord bot token, discord bot name and endpoint of your embedchain app as shown in `.env.example` file.
+
+To run the app in development:
+
+```bash
+python app.py
+```
+
+Before deploying the app, open `fly.toml` file and update the `name` variable to something unique. This is important as `fly.io` requires users to provide a globally unique deployment app names.
+
+Now, we need to launch this application with fly.io. You can see your app on [fly.io dashboard](https://fly.io/dashboard). Run the following command to launch your app on fly.io:
+```bash
+fly launch --no-deploy
+```
+
+Run `ec deploy` to deploy your app on Fly.io. Once you deploy your app, your discord bot will be live!
+
+
+## Slack bot
+
+For Slack bot, you will need to create the bot on slack developer portal and get the slack bot token and slack app token.
+
+### Setup
+
+- Create a workspace on Slack if you don't have one already by clicking [here](https://slack.com/intl/en-in/).
+- Create a new App on your Slack account by going [here](https://api.slack.com/apps).
+- Select `From Scratch`, then enter the Bot Name and select your workspace.
+- Go to `App Credentials` section on the `Basic Information` tab from the left sidebar, create your app token and save it in your `.env` file as `SLACK_APP_TOKEN`.
+- Go to `Socket Mode` tab from the left sidebar and enable the socket mode to listen to slack message from your workspace.
+- (Optional) Under the `App Home` tab you can change your App display name and default name.
+- Navigate to `Event Subscription` tab, and enable the event subscription so that we can listen to slack events.
+- Once you enable the event subscription, you will need to subscribe to bot events to authorize the bot to listen to app mention events of the bot. Do that by tapping on `Add Bot User Event` button and select `app_mention`.
+- On the left Sidebar, go to `OAuth and Permissions` and add the following scopes under `Bot Token Scopes`:
+```text
+app_mentions:read
+channels:history
+channels:read
+chat:write
+emoji:read
+reactions:write
+reactions:read
+```
+- Now select the option `Install to Workspace` and after it's done, copy the `Bot User OAuth Token` and set it in your `.env` file as `SLACK_BOT_TOKEN`.
+
+Once you have your slack bot token and slack app token. Navigate to `nextjs_slack` folder and create `.env` file and define your slack bot token, slack app token and endpoint of your embedchain app as shown in `.env.example` file.
+
+To run the app in development:
+
+```bash
+python app.py
+```
+
+Before deploying the app, open `fly.toml` file and update the `name` variable to something unique. This is important as `fly.io` requires users to provide a globally unique deployment app names.
+
+Now, we need to launch this application with fly.io. You can see your app on [fly.io dashboard](https://fly.io/dashboard). Run the following command to launch your app on fly.io:
+```bash
+fly launch --no-deploy
+```
+
+Run `ec deploy` to deploy your app on Fly.io. Once you deploy your app, your slack bot will be live!

docs/examples/showcase.mdx

@@ -20,7 +20,7 @@ Embedchain community has been super active in creating demos on top of Embedchai
 - [Create Instant ChatBot 🤖 using embedchain](https://databutton.com/v/h3e680h9) by Avra, ([Tweet](https://twitter.com/Avra_b/status/1674704745154641920/))
 - [JOBO 🤖 — The AI-driven sidekick to craft your resume](https://try-jobo.com/) by Enrico Willemse, ([LinkedIn Post](https://www.linkedin.com/posts/enrico-willemse_jobai-gptfun-embedchain-activity-7090340080879374336-ueLB/))
 - [Explore Your Knowledge Base: Interactive chats over various forms of documents](https://chatdocs.dkedar.com/) by Kedar Dabhadkar, ([LinkedIn Post](https://www.linkedin.com/posts/dkedar7_machinelearning-llmops-activity-7092524836639424513-2O3L/))
-- [Chatbot trained on 1000+ videos of Ester hicks the co-author behind the famous book Secret](https://ask-abraham.thoughtseed.repl.co) by Mohan Kumar
+- [Chatbot trained on 1000+ videos of Ester hicks the co-author behind the famous book Secret](https://askabraham.tokenofme.io/) by Mohan Kumar
 
 
 ## Templates

docs/examples/slack-AI.mdx

@@ -0,0 +1,67 @@
+[Embedchain Examples Repo](https://github.com/embedchain/examples) contains code on how to build your own Slack AI to chat with the unstructured data lying in your slack channels.
+
+![Slack AI Demo](/images/slack-ai.png)
+
+## Getting started
+
+Create a Slack AI involves 3 steps
+
+* Create slack user
+* Set environment variables
+* Run the app locally
+
+### Step 1: Create Slack user token
+
+Follow the steps given below to fetch your slack user token to get data through Slack APIs:
+
+1. Create a workspace on Slack if you don’t have one already by clicking [here](https://slack.com/intl/en-in/).
+2. Create a new App on your Slack account by going [here](https://api.slack.com/apps).
+3. Select `From Scratch`, then enter the App Name and select your workspace.
+4. Navigate to `OAuth & Permissions` tab from the left sidebar and go to the `scopes` section. Add the following scopes under `User Token Scopes`:
+
+    ```
+    # Following scopes are needed for reading channel history
+    channels:history
+    channels:read
+
+    # Following scopes are needed to fetch list of channels from slack
+    groups:read
+    mpim:read
+    im:read
+    ```
+
+5. Click on the `Install to Workspace` button under `OAuth Tokens for Your Workspace` section in the same page and install the app in your slack workspace.
+6. After installing the app you will see the `User OAuth Token`, save that token as you will need to configure it as `SLACK_USER_TOKEN` for this demo.
+
+### Step 2: Set environment variables
+
+Navigate to `api` folder and set your `HUGGINGFACE_ACCESS_TOKEN` and `SLACK_USER_TOKEN` in `.env.example` file. Then rename the `.env.example` file to `.env`.
+
+
+<Note>
+By default, we use `Mixtral` model from Hugging Face. However, if you prefer to use OpenAI model, then set `OPENAI_API_KEY` instead of `HUGGINGFACE_ACCESS_TOKEN` along with `SLACK_USER_TOKEN` in `.env` file, and update the code in `api/utils/app.py` file to use OpenAI model instead of Hugging Face model.
+</Note>
+
+### Step 3: Run app locally
+
+Follow the instructions given below to run app locally based on your development setup (with docker or without docker):
+
+#### With docker
+
+```bash
+docker-compose build
+ec start --docker
+```
+
+#### Without docker
+
+```bash
+ec install-reqs
+ec start
+```
+
+Finally, you will have the Slack AI frontend running on http://localhost:3000. You can also access the REST APIs on http://localhost:8000.
+
+## Credits
+
+This demo was built using the Embedchain's [full stack demo template](https://docs.embedchain.ai/get-started/full-stack). Follow the instructions [given here](https://docs.embedchain.ai/get-started/full-stack) to create your own full stack RAG application.

docs/get-started/deployment.mdx

@@ -9,6 +9,7 @@ After successfully setting up and testing your RAG app locally, the next step is
   <Card title="Fly.io" href="/deployment/fly_io"></Card>
   <Card title="Modal.com" href="/deployment/modal_com"></Card>
   <Card title="Render.com" href="/deployment/render_com"></Card>
+  <Card title="Railway.app" href="/deployment/railway"></Card>
   <Card title="Streamlit.io" href="/deployment/streamlit_io"></Card>
   <Card title="Gradio.app" href="/deployment/gradio_app"></Card>
   <Card title="Huggingface.co" href="/deployment/huggingface_spaces"></Card>

docs/get-started/full-stack.mdx

@@ -2,34 +2,80 @@
 title: '💻 Full stack'
 ---
 
-Embedchain provides a clean and simple cli utility that lets you create full-stack RAG applications locally with a single comand. 
+Get started with full-stack RAG applications using Embedchain's easy-to-use CLI tool. Set up everything with just a few commands, whether you prefer Docker or not.
 
-## Prerequisite
+## Prerequisites
+
+Choose your setup method:
+
+* [Without docker](#without-docker)
+* [With Docker](#with-docker)
+
+### Without Docker
+
+Ensure these are installed:
 
-Make sure that you have installed the following:
 - Embedchain python package (`pip install embedchain`)
 - [Node.js](https://docs.npmjs.com/downloading-and-installing-node-js-and-npm) and [Yarn](https://classic.yarnpkg.com/lang/en/docs/install/)
 
-## Get started
+### With Docker
+
+Install Docker from [Docker's official website](https://docs.docker.com/engine/install/).
+
+## Quick Start Guide
+
+### Install the package
+
+Before proceeding, make sure you have the Embedchain package installed.
+
+```bash
+pip install embedchain -U
+```
+
+### Setting Up
 
 For the purpose of the demo, you have to set `OPENAI_API_KEY` to start with but you can choose any llm by changing the configuration easily.
 
-Now run the following command:
+### Installation Commands
 
-```bash
-ec runserver
+<CodeGroup>
+
+```bash without docker
+ec create-app my-app
+cd my-app
+ec start
 ```
 
-Once you run this command, Embedchain does the following:
+```bash with docker
+ec create-app my-app --docker
+cd my-app
+ec start --docker
+```
 
-1. Fetch full stack template that uses FastAPI for backend, and Next.JS template for frontend
-2. Install necessary requirements
-3. Launch the frontend and backend server for you to interact with.
+</CodeGroup>
 
-Once you are done, visit `http://localhost:3000` and you will see a chat UI as shown below.
+### What Happens Next?
+
+1. Embedchain fetches a full stack template (FastAPI backend, Next.JS frontend).
+2. Installs required components.
+3. Launches both frontend and backend servers.
+
+### See It In Action
+
+Open http://localhost:3000 to view the chat UI.
 
 ![full stack example](/images/fullstack.png)
 
-You can navigate to [Embedchain admin panel] where  you can see the chunks created for your documents that you ingested for your RAG application. Below is a screenshot for the same:
+### Admin Panel
+
+Check out the Embedchain admin panel to see the document chunks for your RAG application.
 
 ![full stack chunks](/images/fullstack-chunks.png)
+
+### API Server
+
+If you want to access the API server, you can do so at http://localhost:8000/docs.
+
+![API Server](/images/fullstack-api-server.png)
+
+You can customize the UI and code as per your requirements.

docs/get-started/introduction.mdx

@@ -4,21 +4,21 @@ title: 📚 Introduction
 
 ## What is Embedchain?
 
-Embedchain is an Open Source RAG Framework that makes it easy to create and deploy AI apps. At its core, Embedchain follows the design principle of being *"Conventional but Configurable"* to serve both software engineers and machine learning engineers.
+Embedchain is an Open Source Framework that makes it easy to create and deploy personalized AI apps. At its core, Embedchain follows the design principle of being *"Conventional but Configurable"* to serve both software engineers and machine learning engineers.
 
-Embedchain streamlines the creation of RAG applications, offering a seamless process for managing various types of unstructured data. It efficiently segments data into manageable chunks, generates relevant embeddings, and stores them in a vector database for optimized retrieval. With a suite of diverse APIs, it enables users to extract contextual information, find precise answers, or engage in interactive chat conversations, all tailored to their own data.
+Embedchain streamlines the creation of personalized LLM applications, offering a seamless process for managing various types of unstructured data. It efficiently segments data into manageable chunks, generates relevant embeddings, and stores them in a vector database for optimized retrieval. With a suite of diverse APIs, it enables users to extract contextual information, find precise answers, or engage in interactive chat conversations, all tailored to their own data.
 
 ## Who is Embedchain for?
 
 Embedchain is designed for a diverse range of users, from AI professionals like Data Scientists and Machine Learning Engineers to those just starting their AI journey, including college students, independent developers, and hobbyists. Essentially, it's for anyone with an interest in AI, regardless of their expertise level.
 
-Our APIs are user-friendly yet adaptable, enabling beginners to effortlessly create LLM-powered applications with as few as 4 lines of code. At the same time, we offer extensive customization options for every aspect of the RAG pipeline. This includes the choice of LLMs, vector databases, loaders and chunkers, retrieval strategies, re-ranking, and more.
+Our APIs are user-friendly yet adaptable, enabling beginners to effortlessly create LLM-powered applications with as few as 4 lines of code. At the same time, we offer extensive customization options for every aspect of building a personalized AI application. This includes the choice of LLMs, vector databases, loaders and chunkers, retrieval strategies, re-ranking, and more.
 
 Our platform's clear and well-structured abstraction layers ensure that users can tailor the system to meet their specific needs, whether they're crafting a simple project or a complex, nuanced AI application.
 
 ## Why Use Embedchain?
 
-Developing a robust and efficient RAG (Retrieval-Augmented Generation) pipeline for production use presents numerous complexities, such as:
+Developing a personalized AI application for production use presents numerous complexities, such as:
 
 - Integrating and indexing data from diverse sources.
 - Determining optimal data chunking methods for each source.
@@ -48,11 +48,11 @@ When a user asks a question, whether for chatting, searching, or querying, Embed
 2. **Document Retrieval**: These embeddings are then used to find related documents in the database.
 3. **Answer Generation**: The related documents are used by the LLM to craft a precise answer.
 
-With Embedchain, you don’t have to worry about the complexities of building a RAG pipeline. It offers an easy-to-use interface for developing applications with any kind of data.
+With Embedchain, you don’t have to worry about the complexities of building a personalized AI application. It offers an easy-to-use interface for developing applications with any kind of data.
 
 ## Getting started
 
-Checkout our [quickstart guide](/get-started/quickstart) to start your first RAG application.
+Checkout our [quickstart guide](/get-started/quickstart) to start your first AI application.
 
 ## Support
 

docs/get-started/quickstart.mdx

@@ -1,11 +1,11 @@
 ---
 title: '⚡ Quickstart'
-description: '💡 Create a RAG app on your own data in a minute'
+description: '💡 Create an AI app on your own data in a minute'
 ---
 
 ## Installation
 
-First install the python package.
+First install the Python package:
 
 ```bash
 pip install embedchain
@@ -31,41 +31,47 @@ This section gives a quickstart example of using Mistral as the Open source LLM
 We are using Mistral hosted at Hugging Face, so will you need a Hugging Face token to run this example. Its *free* and you can create one [here](https://huggingface.co/docs/hub/security-tokens).
 
 <CodeGroup>
-```python quickstart.py
+```python huggingface_demo.py
 import os
-# replace this with your HF key
+# Replace this with your HF token
 os.environ["HUGGINGFACE_ACCESS_TOKEN"] = "hf_xxxx"
 
 from embedchain import App
-app = App.from_config("mistral.yaml")
+
+config = {
+  'llm': {
+    'provider': 'huggingface',
+    'config': {
+      'model': 'mistralai/Mistral-7B-Instruct-v0.2',
+      'top_p': 0.5
+    }
+  },
+  'embedder': {
+    'provider': 'huggingface',
+    'config': {
+      'model': 'sentence-transformers/all-mpnet-base-v2'
+    }
+  }
+}
+app = App.from_config(config=config)
 app.add("https://www.forbes.com/profile/elon-musk")
 app.add("https://en.wikipedia.org/wiki/Elon_Musk")
 app.query("What is the net worth of Elon Musk today?")
 # Answer: The net worth of Elon Musk today is $258.7 billion.
 ```
-```yaml mistral.yaml
-llm:
-  provider: huggingface
-  config:
-    model: 'mistralai/Mistral-7B-v0.1'
-    top_p: 0.5
-embedder:
-  provider: huggingface
-  config:
-    model: 'sentence-transformers/all-mpnet-base-v2'
-```
 </CodeGroup>
 
 ## Paid Models
 
 In this section, we will use both LLM and embedding model from OpenAI.
 
-```python quickstart.py
+```python openai_demo.py
 import os
-# replace this with your OpenAI key
+from embedchain import App
+
+# Replace this with your OpenAI key
 os.environ["OPENAI_API_KEY"] = "sk-xxxx"
 
-from embedchain import App
 app = App()
 app.add("https://www.forbes.com/profile/elon-musk")
 app.add("https://en.wikipedia.org/wiki/Elon_Musk")
@@ -80,4 +86,4 @@ Now that you have created your first app, you can follow any of the links:
 * [Introduction](/get-started/introduction)
 * [Customization](/components/introduction)
 * [Use cases](/use-cases/introduction)
-* [Deployment](/get-started/deployment)
+* [Deployment](/get-started/deployment)

docs/integration/langsmith.mdx

@@ -5,22 +5,30 @@ description: 'Integrate with Langsmith to debug and monitor your LLM app'
 
 Embedchain now supports integration with [LangSmith](https://www.langchain.com/langsmith).
 
-To use langsmith, you need to do the following steps
+To use LangSmith, you need to do the following steps.
 
-1. Have an account on langsmith and keep the environment variables in handy
-2. Set the environments variables in your app so that embedchain has context about it.
+1. Have an account on LangSmith and keep the environment variables in handy
+2. Set the environment variables in your app so that embedchain has context about it.
 3. Just use embedchain and everything will be logged to LangSmith, so that you can better test and monitor your application.
 
-Lets cover each step in detail.
+Let's cover each step in detail.
 
-* First make sure that you a LangSmith account created and have all the necessary variables handy. LangSmith has a [good documentation](https://docs.smith.langchain.com/) on how to get started with their service.
 
-* Once you have the account setup, we will need the following environment variables
+* First make sure that you have created a LangSmith account and have all the necessary variables handy. LangSmith has a [good documentation](https://docs.smith.langchain.com/) on how to get started with their service.
+
+* Once you have setup the account, we will need the following environment variables
 
 ```bash
+# Setting environment variable for LangChain Tracing V2 integration.
 export LANGCHAIN_TRACING_V2=true
+
+# Setting the API endpoint for LangChain.
 export LANGCHAIN_ENDPOINT=https://api.smith.langchain.com
+
+# Replace '<your-api-key>' with your LangChain API key.
 export LANGCHAIN_API_KEY=<your-api-key>
+
+# Replace '<your-project>' with your LangChain project name, or it defaults to "default".
 export LANGCHAIN_PROJECT=<your-project>  # if not specified, defaults to "default"
 ```
 
@@ -29,20 +37,32 @@ If you are using Python, you can use the following code to set environment varia
 ```python
 import os
 
+# Setting environment variable for LangChain Tracing V2 integration.
 os.environ['LANGCHAIN_TRACING_V2'] = 'true'
+
+# Setting the API endpoint for LangChain.
 os.environ['LANGCHAIN_ENDPOINT'] = 'https://api.smith.langchain.com'
-os.environ['LANGCHAIN_API_KEY'] = <your-api-key>
-os.environ['LANGCHAIN_PROJECT] = <your-project>
+
+# Replace '<your-api-key>' with your LangChain API key.
+os.environ['LANGCHAIN_API_KEY'] = '<your-api-key>'
+
+# Replace '<your-project>' with your LangChain project name.
+os.environ['LANGCHAIN_PROJECT'] = '<your-project>'
 ```
 
-* Now create an app using embedchain and everything will be automatically visible in the LangSmith
+* Now create an app using Embedchain and everything will be automatically visible in the LangSmith
 
 
 ```python
 from embedchain import App
 
+# Initialize EmbedChain application.
 app = App()
+
+# Add data to your app
 app.add("https://en.wikipedia.org/wiki/Elon_Musk")
+
+# Query your app
 app.query("How many companies did Elon found?")
 ```
 

docs/mint.json

@@ -88,9 +88,8 @@
       "pages": [
         "components/introduction",
         {
-          "group": "Data sources",
+          "group": "🗂️ Data sources",
           "pages": [
-
             "components/data-sources/overview",
             {
               "group": "Data types",
@@ -129,9 +128,21 @@
             "components/data-sources/data-type-handling"
           ]
         },
+        {
+          "group": "🗄️ Vector databases",
+          "pages": [
+                "components/vector-databases/chromadb",
+                "components/vector-databases/elasticsearch",
+                "components/vector-databases/pinecone",
+                "components/vector-databases/opensearch",
+                "components/vector-databases/qdrant",
+                "components/vector-databases/weaviate",
+                "components/vector-databases/zilliz"
+          ]
+        },
         "components/llms",
-        "components/vector-databases",
-        "components/embedding-models"
+        "components/embedding-models",
+        "components/evaluation"
       ]
     },
     {
@@ -141,6 +152,7 @@
         "deployment/fly_io",
         "deployment/modal_com",
         "deployment/render_com",
+        "deployment/railway",
         "deployment/streamlit_io",
         "deployment/gradio_app",
         "deployment/huggingface_spaces",
@@ -174,7 +186,9 @@
         },
         "examples/full_stack",
         "examples/openai-assistant",
-        "examples/opensource-assistant"
+        "examples/opensource-assistant",
+        "examples/nextjs-assistant",
+        "examples/slack-AI"
       ]
     },
     {
@@ -196,17 +210,19 @@
     {
       "group": "API Reference",
       "pages": [
-        "api-reference/pipeline/overview",
+        "api-reference/app/overview",
         {
-          "group": "Pipeline methods",
+          "group": "App methods",
           "pages": [
-            "api-reference/pipeline/add",
-            "api-reference/pipeline/query",
-            "api-reference/pipeline/chat",
-            "api-reference/pipeline/search",
-            "api-reference/pipeline/deploy",
-            "api-reference/pipeline/reset",
-            "api-reference/pipeline/delete"
+            "api-reference/app/add",
+            "api-reference/app/query",
+            "api-reference/app/chat",
+            "api-reference/app/search",
+            "api-reference/app/get",
+            "api-reference/app/evaluate",
+            "api-reference/app/deploy",
+            "api-reference/app/reset",
+            "api-reference/app/delete"
           ]
         },
         "api-reference/store/openai-assistant",
@@ -234,7 +250,7 @@
   "footerSocials": {
     "website": "https://embedchain.ai",
     "github": "https://github.com/embedchain/embedchain",
-    "slack": "https://join.slack.com/t/embedchain/shared_invite/zt-22uwz3c46-Zg7cIh5rOBteT_xe1jwLDw",
+    "slack": "https://embedchain.ai/slack",
     "discord": "https://discord.gg/6PzXDgEjG5",
     "twitter": "https://twitter.com/embedchain",
     "linkedin": "https://www.linkedin.com/company/embedchain"

docs/platform/faq.mdx

@@ -1,3 +0,0 @@
----
-title: 'FAQs'
----

docs/platform/overview.mdx

@@ -1,3 +0,0 @@
----
-title: 'Overview'
----

docs/platform/quickstart.mdx

@@ -1,3 +0,0 @@
----
-title: 'Quickstart'
----

docs/platform/roadmap.mdx

@@ -1,3 +0,0 @@
----
-title: 'Roadmap'
----

docs/platform/security.mdx

@@ -1,3 +0,0 @@
----
-title: 'Security'
----

embedchain-js/README.md

@@ -178,7 +178,7 @@ await app.addLocal("qna_pair", ["Question", "Answer"]);
 
 ## Testing
 
-Before you consume valueable tokens, you should make sure that the embedding you have done works and that it's receiving the correct document from the database.
+Before you consume valuable tokens, you should make sure that the embedding you have done works and that it's receiving the correct document from the database.
 
 For this you can use the `dryRun` method.
 

embedchain/__init__.py

@@ -7,4 +7,4 @@ from embedchain.client import Client  # noqa: F401
 from embedchain.pipeline import Pipeline  # noqa: F401
 
 # Setup the user directory if doesn't exist already
-Client.setup_dir()
+Client.setup()

embedchain/alembic.ini

@@ -0,0 +1,116 @@
+# A generic, single database configuration.
+
+[alembic]
+# path to migration scripts
+script_location = embedchain:migrations
+
+# template used to generate migration file names; The default value is %%(rev)s_%%(slug)s
+# Uncomment the line below if you want the files to be prepended with date and time
+# see https://alembic.sqlalchemy.org/en/latest/tutorial.html#editing-the-ini-file
+# for all available tokens
+# file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s
+
+# sys.path path, will be prepended to sys.path if present.
+# defaults to the current working directory.
+prepend_sys_path = .
+
+# timezone to use when rendering the date within the migration file
+# as well as the filename.
+# If specified, requires the python>=3.9 or backports.zoneinfo library.
+# Any required deps can installed by adding `alembic[tz]` to the pip requirements
+# string value is passed to ZoneInfo()
+# leave blank for localtime
+# timezone =
+
+# max length of characters to apply to the
+# "slug" field
+# truncate_slug_length = 40
+
+# set to 'true' to run the environment during
+# the 'revision' command, regardless of autogenerate
+# revision_environment = false
+
+# set to 'true' to allow .pyc and .pyo files without
+# a source .py file to be detected as revisions in the
+# versions/ directory
+# sourceless = false
+
+# version location specification; This defaults
+# to alembic/versions.  When using multiple version
+# directories, initial revisions must be specified with --version-path.
+# The path separator used here should be the separator specified by "version_path_separator" below.
+# version_locations = %(here)s/bar:%(here)s/bat:alembic/versions
+
+# version path separator; As mentioned above, this is the character used to split
+# version_locations. The default within new alembic.ini files is "os", which uses os.pathsep.
+# If this key is omitted entirely, it falls back to the legacy behavior of splitting on spaces and/or commas.
+# Valid values for version_path_separator are:
+#
+# version_path_separator = :
+# version_path_separator = ;
+# version_path_separator = space
+version_path_separator = os  # Use os.pathsep. Default configuration used for new projects.
+
+# set to 'true' to search source files recursively
+# in each "version_locations" directory
+# new in Alembic version 1.10
+# recursive_version_locations = false
+
+# the output encoding used when revision files
+# are written from script.py.mako
+# output_encoding = utf-8
+
+sqlalchemy.url = driver://user:pass@localhost/dbname
+
+
+[post_write_hooks]
+# post_write_hooks defines scripts or Python functions that are run
+# on newly generated revision scripts.  See the documentation for further
+# detail and examples
+
+# format using "black" - use the console_scripts runner, against the "black" entrypoint
+# hooks = black
+# black.type = console_scripts
+# black.entrypoint = black
+# black.options = -l 79 REVISION_SCRIPT_FILENAME
+
+# lint with attempts to fix using "ruff" - use the exec runner, execute a binary
+# hooks = ruff
+# ruff.type = exec
+# ruff.executable = %(here)s/.venv/bin/ruff
+# ruff.options = --fix REVISION_SCRIPT_FILENAME
+
+# Logging configuration
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = WARN
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S

embedchain/app.py

@@ -1,34 +1,42 @@
 import ast
+import concurrent.futures
 import json
 import logging
 import os
-import sqlite3
-import uuid
-from typing import Any, Dict, Optional
+from typing import Any, Optional, Union
 
 import requests
 import yaml
+from tqdm import tqdm
 
-from embedchain.cache import (Config, ExactMatchEvaluation,
-                              SearchDistanceEvaluation, cache,
-                              gptcache_data_manager, gptcache_pre_function)
+from embedchain.cache import (
+    Config,
+    ExactMatchEvaluation,
+    SearchDistanceEvaluation,
+    cache,
+    gptcache_data_manager,
+    gptcache_pre_function,
+)
 from embedchain.client import Client
 from embedchain.config import AppConfig, CacheConfig, ChunkerConfig
-from embedchain.constants import SQLITE_PATH
+from embedchain.core.db.database import get_session, init_db, setup_engine
+from embedchain.core.db.models import DataSource
 from embedchain.embedchain import EmbedChain
 from embedchain.embedder.base import BaseEmbedder
 from embedchain.embedder.openai import OpenAIEmbedder
+from embedchain.evaluation.base import BaseMetric
+from embedchain.evaluation.metrics import AnswerRelevance, ContextRelevance, Groundedness
 from embedchain.factory import EmbedderFactory, LlmFactory, VectorDBFactory
 from embedchain.helpers.json_serializable import register_deserializable
 from embedchain.llm.base import BaseLlm
 from embedchain.llm.openai import OpenAILlm
 from embedchain.telemetry.posthog import AnonymousTelemetry
-from embedchain.utils import validate_config
+from embedchain.utils.evaluation import EvalData, EvalMetric
+from embedchain.utils.misc import validate_config
 from embedchain.vectordb.base import BaseVectorDB
 from embedchain.vectordb.chroma import ChromaDB
 
-# Setup the user directory if doesn't exist already
-Client.setup_dir()
+logger = logging.getLogger(__name__)
 
 
 @register_deserializable
@@ -48,10 +56,10 @@ class App(EmbedChain):
         embedding_model: BaseEmbedder = None,
         llm: BaseLlm = None,
         config_data: dict = None,
-        log_level=logging.WARN,
         auto_deploy: bool = False,
         chunker: ChunkerConfig = None,
         cache_config: CacheConfig = None,
+        log_level: int = logging.WARN,
     ):
         """
         Initialize a new `App` instance.
@@ -66,8 +74,6 @@ class App(EmbedChain):
         :type llm: BaseLlm, optional
         :param config_data: Config dictionary, defaults to None
         :type config_data: dict, optional
-        :param log_level: Log level to use, defaults to logging.WARN
-        :type log_level: int, optional
         :param auto_deploy: Whether to deploy the pipeline automatically, defaults to False
         :type auto_deploy: bool, optional
         :raises Exception: If an error occurs while creating the pipeline
@@ -81,22 +87,22 @@ class App(EmbedChain):
         if name and config:
             raise Exception("Cannot provide both name and config. Please provide only one of them.")
 
-        logging.basicConfig(level=log_level, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s")
-        self.logger = logging.getLogger(__name__)
+        # Initialize the metadata db for the app
+        setup_engine(database_uri=os.environ.get("EMBEDCHAIN_DB_URI"))
+        init_db()
+
         self.auto_deploy = auto_deploy
         # Store the dict config as an attribute to be able to send it
         self.config_data = config_data if (config_data and validate_config(config_data)) else None
         self.client = None
         # pipeline_id from the backend
         self.id = None
-        self.chunker = None
-        if chunker:
-            self.chunker = ChunkerConfig(**chunker)
+        self.chunker = ChunkerConfig(**chunker) if chunker else None
         self.cache_config = cache_config
 
         self.config = config or AppConfig()
         self.name = self.config.name
-        self.config.id = self.local_id = str(uuid.uuid4()) if self.config.id is None else self.config.id
+        self.config.id = self.local_id = "default-app-id" if self.config.id is None else self.config.id
 
         if id is not None:
             # Init client first since user is trying to fetch the pipeline
@@ -114,6 +120,9 @@ class App(EmbedChain):
         self.llm = llm or OpenAILlm()
         self._init_db()
 
+        # Session for the metadata db
+        self.db_session = get_session()
+
         # If cache_config is provided, initializing the cache ...
         if self.cache_config is not None:
             self._init_cache()
@@ -121,27 +130,6 @@ class App(EmbedChain):
         # Send anonymous telemetry
         self._telemetry_props = {"class": self.__class__.__name__}
         self.telemetry = AnonymousTelemetry(enabled=self.config.collect_metrics)
-
-        # Establish a connection to the SQLite database
-        self.connection = sqlite3.connect(SQLITE_PATH, check_same_thread=False)
-        self.cursor = self.connection.cursor()
-
-        # Create the 'data_sources' table if it doesn't exist
-        self.cursor.execute(
-            """
-            CREATE TABLE IF NOT EXISTS data_sources (
-                pipeline_id TEXT,
-                hash TEXT,
-                type TEXT,
-                value TEXT,
-                metadata TEXT,
-                is_uploaded INTEGER DEFAULT 0,
-                PRIMARY KEY (pipeline_id, hash)
-            )
-        """
-        )
-        self.connection.commit()
-        # Send anonymous telemetry
         self.telemetry.capture(event_name="init", properties=self._telemetry_props)
 
         self.user_asks = []
@@ -244,30 +232,6 @@ class App(EmbedChain):
         r.raise_for_status()
         return r.json()
 
-    def search(self, query, num_documents=3):
-        """
-        Search for similar documents related to the query in the vector database.
-        """
-        # Send anonymous telemetry
-        self.telemetry.capture(event_name="search", properties=self._telemetry_props)
-
-        # TODO: Search will call the endpoint rather than fetching the data from the db itself when deploy=True.
-        if self.id is None:
-            where = {"app_id": self.local_id}
-            context = self.db.query(
-                query,
-                n_results=num_documents,
-                where=where,
-                citations=True,
-            )
-            result = []
-            for c in context:
-                result.append({"context": c[0], "metadata": c[1]})
-            return result
-        else:
-            # Make API call to the backend to get the results
-            NotImplementedError("Search is not implemented yet for the prod mode.")
-
     def _upload_file_to_presigned_url(self, presigned_url, file_path):
         try:
             with open(file_path, "rb") as file:
@@ -275,7 +239,7 @@ class App(EmbedChain):
                 response.raise_for_status()
                 return response.status_code == 200
         except Exception as e:
-            self.logger.exception(f"Error occurred during file upload: {str(e)}")
+            logger.exception(f"Error occurred during file upload: {str(e)}")
             print("❌ Error occurred during file upload!")
             return False
 
@@ -309,7 +273,7 @@ class App(EmbedChain):
                 metadata = {"file_path": data_value, "s3_key": s3_key}
                 data_value = presigned_url
             else:
-                self.logger.error(f"File upload failed for hash: {data_hash}")
+                logger.error(f"File upload failed for hash: {data_hash}")
                 return False
         else:
             if data_type == "qna_pair":
@@ -325,20 +289,14 @@ class App(EmbedChain):
             return False
 
     def _mark_data_as_uploaded(self, data_hash):
-        self.cursor.execute(
-            "UPDATE data_sources SET is_uploaded = 1 WHERE hash = ? AND pipeline_id = ?",
-            (data_hash, self.local_id),
-        )
-        self.connection.commit()
+        self.db_session.query(DataSource).filter_by(hash=data_hash, app_id=self.local_id).update({"is_uploaded": 1})
 
     def get_data_sources(self):
-        db_data = self.cursor.execute("SELECT * FROM data_sources WHERE pipeline_id = ?", (self.local_id,)).fetchall()
-
-        data_sources = []
-        for data in db_data:
-            data_sources.append({"data_type": data[2], "data_value": data[3], "metadata": data[4]})
-
-        return data_sources
+        data_sources = self.db_session.query(DataSource).filter_by(app_id=self.local_id).all()
+        results = []
+        for row in data_sources:
+            results.append({"data_type": row.type, "data_value": row.value, "metadata": row.meta_data})
+        return results
 
     def deploy(self):
         if self.client is None:
@@ -347,14 +305,11 @@ class App(EmbedChain):
         pipeline_data = self._create_pipeline()
         self.id = pipeline_data["id"]
 
-        results = self.cursor.execute(
-            "SELECT * FROM data_sources WHERE pipeline_id = ? AND is_uploaded = 0", (self.local_id,)  # noqa:E501
-        ).fetchall()
-
+        results = self.db_session.query(DataSource).filter_by(app_id=self.local_id, is_uploaded=0).all()
         if len(results) > 0:
             print("🛠️ Adding data to your pipeline...")
         for result in results:
-            data_hash, data_type, data_value = result[1], result[2], result[3]
+            data_hash, data_type, data_value = result.hash, result.data_type, result.data_value
             self._process_and_upload_data(data_hash, data_type, data_value)
 
         # Send anonymous telemetry
@@ -364,23 +319,23 @@ class App(EmbedChain):
     def from_config(
         cls,
         config_path: Optional[str] = None,
-        config: Optional[Dict[str, Any]] = None,
+        config: Optional[dict[str, Any]] = None,
         auto_deploy: bool = False,
         yaml_path: Optional[str] = None,
     ):
         """
-        Instantiate a Pipeline object from a configuration.
+        Instantiate a App object from a configuration.
 
         :param config_path: Path to the YAML or JSON configuration file.
         :type config_path: Optional[str]
         :param config: A dictionary containing the configuration.
-        :type config: Optional[Dict[str, Any]]
-        :param auto_deploy: Whether to deploy the pipeline automatically, defaults to False
+        :type config: Optional[dict[str, Any]]
+        :param auto_deploy: Whether to deploy the app automatically, defaults to False
         :type auto_deploy: bool, optional
         :param yaml_path: (Deprecated) Path to the YAML configuration file. Use config_path instead.
         :type yaml_path: Optional[str]
-        :return: An instance of the Pipeline class.
-        :rtype: Pipeline
+        :return: An instance of the App class.
+        :rtype: App
         """
         # Backward compatibility for yaml_path
         if yaml_path and not config_path:
@@ -393,7 +348,7 @@ class App(EmbedChain):
 
         if config_path:
             file_extension = os.path.splitext(config_path)[1]
-            with open(config_path, "r") as file:
+            with open(config_path, "r", encoding="UTF-8") as file:
                 if file_extension in [".yaml", ".yml"]:
                     config_data = yaml.safe_load(file)
                 elif file_extension == ".json":
@@ -403,18 +358,16 @@ class App(EmbedChain):
         elif config and isinstance(config, dict):
             config_data = config
         else:
-            logging.error(
+            logger.error(
                 "Please provide either a config file path (YAML or JSON) or a config dictionary. Falling back to defaults because no config is provided.",  # noqa: E501
             )
             config_data = {}
 
-        try:
-            validate_config(config_data)
-        except Exception as e:
-            raise Exception(f"Error occurred while validating the config. Error: {str(e)}")
+        # Validate the config
+        validate_config(config_data)
 
         app_config_data = config_data.get("app", {}).get("config", {})
-        db_config_data = config_data.get("vectordb", {})
+        vector_db_config_data = config_data.get("vectordb", {})
         embedding_model_config_data = config_data.get("embedding_model", config_data.get("embedder", {}))
         llm_config_data = config_data.get("llm", {})
         chunker_config_data = config_data.get("chunker", {})
@@ -422,10 +375,14 @@ class App(EmbedChain):
 
         app_config = AppConfig(**app_config_data)
 
-        db_provider = db_config_data.get("provider", "chroma")
-        db = VectorDBFactory.create(db_provider, db_config_data.get("config", {}))
+        vector_db_provider = vector_db_config_data.get("provider", "chroma")
+        vector_db = VectorDBFactory.create(vector_db_provider, vector_db_config_data.get("config", {}))
 
         if llm_config_data:
+            # Initialize the metadata db for the app here since llmfactory needs it for initialization of
+            # the llm memory
+            setup_engine(database_uri=os.environ.get("EMBEDCHAIN_DB_URI"))
+            init_db()
             llm_provider = llm_config_data.get("provider", "openai")
             llm = LlmFactory.create(llm_provider, llm_config_data.get("config", {}))
         else:
@@ -441,17 +398,113 @@ class App(EmbedChain):
         else:
             cache_config = None
 
-        # Send anonymous telemetry
-        event_properties = {"init_type": "config_data"}
-        AnonymousTelemetry().capture(event_name="init", properties=event_properties)
-
         return cls(
             config=app_config,
             llm=llm,
-            db=db,
+            db=vector_db,
             embedding_model=embedding_model,
             config_data=config_data,
             auto_deploy=auto_deploy,
             chunker=chunker_config_data,
             cache_config=cache_config,
         )
+
+    def _eval(self, dataset: list[EvalData], metric: Union[BaseMetric, str]):
+        """
+        Evaluate the app on a dataset for a given metric.
+        """
+        metric_str = metric.name if isinstance(metric, BaseMetric) else metric
+        eval_class_map = {
+            EvalMetric.CONTEXT_RELEVANCY.value: ContextRelevance,
+            EvalMetric.ANSWER_RELEVANCY.value: AnswerRelevance,
+            EvalMetric.GROUNDEDNESS.value: Groundedness,
+        }
+
+        if metric_str in eval_class_map:
+            return eval_class_map[metric_str]().evaluate(dataset)
+
+        # Handle the case for custom metrics
+        if isinstance(metric, BaseMetric):
+            return metric.evaluate(dataset)
+        else:
+            raise ValueError(f"Invalid metric: {metric}")
+
+    def evaluate(
+        self,
+        questions: Union[str, list[str]],
+        metrics: Optional[list[Union[BaseMetric, str]]] = None,
+        num_workers: int = 4,
+    ):
+        """
+        Evaluate the app on a question.
+
+        param: questions: A question or a list of questions to evaluate.
+        type: questions: Union[str, list[str]]
+        param: metrics: A list of metrics to evaluate. Defaults to all metrics.
+        type: metrics: Optional[list[Union[BaseMetric, str]]]
+        param: num_workers: Number of workers to use for parallel processing.
+        type: num_workers: int
+        return: A dictionary containing the evaluation results.
+        rtype: dict
+        """
+        if "OPENAI_API_KEY" not in os.environ:
+            raise ValueError("Please set the OPENAI_API_KEY environment variable with permission to use `gpt4` model.")
+
+        queries, answers, contexts = [], [], []
+        if isinstance(questions, list):
+            with concurrent.futures.ThreadPoolExecutor(max_workers=num_workers) as executor:
+                future_to_data = {executor.submit(self.query, q, citations=True): q for q in questions}
+                for future in tqdm(
+                    concurrent.futures.as_completed(future_to_data),
+                    total=len(future_to_data),
+                    desc="Getting answer and contexts for questions",
+                ):
+                    question = future_to_data[future]
+                    queries.append(question)
+                    answer, context = future.result()
+                    answers.append(answer)
+                    contexts.append(list(map(lambda x: x[0], context)))
+        else:
+            answer, context = self.query(questions, citations=True)
+            queries = [questions]
+            answers = [answer]
+            contexts = [list(map(lambda x: x[0], context))]
+
+        metrics = metrics or [
+            EvalMetric.CONTEXT_RELEVANCY.value,
+            EvalMetric.ANSWER_RELEVANCY.value,
+            EvalMetric.GROUNDEDNESS.value,
+        ]
+
+        logger.info(f"Collecting data from {len(queries)} questions for evaluation...")
+        dataset = []
+        for q, a, c in zip(queries, answers, contexts):
+            dataset.append(EvalData(question=q, answer=a, contexts=c))
+
+        logger.info(f"Evaluating {len(dataset)} data points...")
+        result = {}
+        with concurrent.futures.ThreadPoolExecutor(max_workers=num_workers) as executor:
+            future_to_metric = {executor.submit(self._eval, dataset, metric): metric for metric in metrics}
+            for future in tqdm(
+                concurrent.futures.as_completed(future_to_metric),
+                total=len(future_to_metric),
+                desc="Evaluating metrics",
+            ):
+                metric = future_to_metric[future]
+                if isinstance(metric, BaseMetric):
+                    result[metric.name] = future.result()
+                else:
+                    result[metric] = future.result()
+
+        if self.config.collect_metrics:
+            telemetry_props = self._telemetry_props
+            metrics_names = []
+            for metric in metrics:
+                if isinstance(metric, BaseMetric):
+                    metrics_names.append(metric.name)
+                else:
+                    metrics_names.append(metric)
+            telemetry_props["metrics"] = metrics_names
+            self.telemetry.capture(event_name="evaluate", properties=telemetry_props)
+
+        return result

embedchain/bots/discord.py

@@ -17,6 +17,8 @@ except ModuleNotFoundError:
     ) from None
 
 
+logger = logging.getLogger(__name__)
+
 intents = discord.Intents.default()
 intents.message_content = True
 client = discord.Client(intents=intents)
@@ -37,7 +39,7 @@ class DiscordBot(BaseBot):
             self.add(data)
             response = f"Added data from: {data}"
         except Exception:
-            logging.exception(f"Failed to add data {data}.")
+            logger.exception(f"Failed to add data {data}.")
             response = "Some error occurred while adding data."
         return response
 
@@ -45,7 +47,7 @@ class DiscordBot(BaseBot):
         try:
             response = self.query(message)
         except Exception:
-            logging.exception(f"Failed to query {message}.")
+            logger.exception(f"Failed to query {message}.")
             response = "An error occurred. Please try again!"
         return response
 
@@ -60,7 +62,7 @@ class DiscordBot(BaseBot):
 async def query_command(interaction: discord.Interaction, question: str):
     await interaction.response.defer()
     member = client.guilds[0].get_member(client.user.id)
-    logging.info(f"User: {member}, Query: {question}")
+    logger.info(f"User: {member}, Query: {question}")
     try:
         answer = discord_bot.ask_bot(question)
         if args.include_question:
@@ -70,20 +72,20 @@ async def query_command(interaction: discord.Interaction, question: str):
         await interaction.followup.send(response)
     except Exception as e:
         await interaction.followup.send("An error occurred. Please try again!")
-        logging.error("Error occurred during 'query' command:", e)
+        logger.error("Error occurred during 'query' command:", e)
 
 
 @tree.command(name="add", description="add new content to the embedchain database")
 async def add_command(interaction: discord.Interaction, url_or_text: str):
     await interaction.response.defer()
     member = client.guilds[0].get_member(client.user.id)
-    logging.info(f"User: {member}, Add: {url_or_text}")
+    logger.info(f"User: {member}, Add: {url_or_text}")
     try:
         response = discord_bot.add_data(url_or_text)
         await interaction.followup.send(response)
     except Exception as e:
         await interaction.followup.send("An error occurred. Please try again!")
-        logging.error("Error occurred during 'add' command:", e)
+        logger.error("Error occurred during 'add' command:", e)
 
 
 @tree.command(name="ping", description="Simple ping pong command")
@@ -96,7 +98,7 @@ async def on_app_command_error(interaction: discord.Interaction, error: discord.
     if isinstance(error, commands.CommandNotFound):
         await interaction.followup.send("Invalid command. Please refer to the documentation for correct syntax.")
     else:
-        logging.error("Error occurred during command execution:", error)
+        logger.error("Error occurred during command execution:", error)
 
 
 @client.event
@@ -104,8 +106,8 @@ async def on_ready():
     # TODO: Sync in admin command, to not hit rate limits.
     # This might be overkill for most users, and it would require to set a guild or user id, where sync is allowed.
     await tree.sync()
-    logging.debug("Command tree synced")
-    logging.info(f"Logged in as {client.user.name}")
+    logger.debug("Command tree synced")
+    logger.info(f"Logged in as {client.user.name}")
 
 
 def start_command():

embedchain/bots/poe.py

@@ -1,7 +1,7 @@
 import argparse
 import logging
 import os
-from typing import List, Optional
+from typing import Optional
 
 from embedchain.helpers.json_serializable import register_deserializable
 
@@ -53,7 +53,7 @@ class PoeBot(BaseBot, PoeBot):
         answer = self.handle_message(last_message, history)
         yield self.text_event(answer)
 
-    def handle_message(self, message, history: Optional[List[str]] = None):
+    def handle_message(self, message, history: Optional[list[str]] = None):
         if message.startswith("/add "):
             response = self.add_data(message)
         else:
@@ -70,7 +70,7 @@ class PoeBot(BaseBot, PoeBot):
     #         response = "Some error occurred while adding data."
     #     return response
 
-    def ask_bot(self, message, history: List[str]):
+    def ask_bot(self, message, history: list[str]):
         try:
             self.app.llm.set_history(history=history)
             response = self.query(message)

embedchain/bots/slack.py

@@ -19,6 +19,8 @@ except ModuleNotFoundError:
     ) from None
 
 
+logger = logging.getLogger(__name__)
+
 SLACK_BOT_TOKEN = os.environ.get("SLACK_BOT_TOKEN")
 
 
@@ -42,10 +44,10 @@ class SlackBot(BaseBot):
                     try:
                         response = self.chat_bot.chat(question)
                         self.send_slack_message(message["channel"], response)
-                        logging.info("Query answered successfully!")
+                        logger.info("Query answered successfully!")
                     except Exception as e:
                         self.send_slack_message(message["channel"], "An error occurred. Please try again!")
-                        logging.error("Error occurred during 'query' command:", e)
+                        logger.error("Error occurred during 'query' command:", e)
                 elif text.startswith("add"):
                     _, data_type, url_or_text = text.split(" ", 2)
                     if url_or_text.startswith("<") and url_or_text.endswith(">"):
@@ -55,10 +57,10 @@ class SlackBot(BaseBot):
                         self.send_slack_message(message["channel"], f"Added {data_type} : {url_or_text}")
                     except ValueError as e:
                         self.send_slack_message(message["channel"], f"Error: {str(e)}")
-                        logging.error("Error occurred during 'add' command:", e)
+                        logger.error("Error occurred during 'add' command:", e)
                     except Exception as e:
                         self.send_slack_message(message["channel"], f"Failed to add {data_type} : {url_or_text}")
-                        logging.error("Error occurred during 'add' command:", e)
+                        logger.error("Error occurred during 'add' command:", e)
 
     def send_slack_message(self, channel, message):
         response = self.client.chat_postMessage(channel=channel, text=message)
@@ -68,7 +70,7 @@ class SlackBot(BaseBot):
         app = Flask(__name__)
 
         def signal_handler(sig, frame):
-            logging.info("\nGracefully shutting down the SlackBot...")
+            logger.info("\nGracefully shutting down the SlackBot...")
             sys.exit(0)
 
         signal.signal(signal.SIGINT, signal_handler)

embedchain/bots/whatsapp.py

@@ -8,6 +8,8 @@ from embedchain.helpers.json_serializable import register_deserializable
 
 from .base import BaseBot
 
+logger = logging.getLogger(__name__)
+
 
 @register_deserializable
 class WhatsAppBot(BaseBot):
@@ -35,7 +37,7 @@ class WhatsAppBot(BaseBot):
             self.add(data)
             response = f"Added data from: {data}"
         except Exception:
-            logging.exception(f"Failed to add data {data}.")
+            logger.exception(f"Failed to add data {data}.")
             response = "Some error occurred while adding data."
         return response
 
@@ -43,7 +45,7 @@ class WhatsAppBot(BaseBot):
         try:
             response = self.query(message)
         except Exception:
-            logging.exception(f"Failed to query {message}.")
+            logger.exception(f"Failed to query {message}.")
             response = "An error occurred. Please try again!"
         return response
 
@@ -51,7 +53,7 @@ class WhatsAppBot(BaseBot):
         app = self.flask.Flask(__name__)
 
         def signal_handler(sig, frame):
-            logging.info("\nGracefully shutting down the WhatsAppBot...")
+            logger.info("\nGracefully shutting down the WhatsAppBot...")
             sys.exit(0)
 
         signal.signal(signal.SIGINT, signal_handler)

embedchain/cache.py

@@ -1,6 +1,6 @@
 import logging
 import os  # noqa: F401
-from typing import Any, Dict
+from typing import Any
 
 from gptcache import cache  # noqa: F401
 from gptcache.adapter.adapter import adapt  # noqa: F401
@@ -14,8 +14,10 @@ from gptcache.similarity_evaluation.distance import \
 from gptcache.similarity_evaluation.exact_match import \
     ExactMatchEvaluation  # noqa: F401
 
+logger = logging.getLogger(__name__)
 
-def gptcache_pre_function(data: Dict[str, Any], **params: Dict[str, Any]):
+
+def gptcache_pre_function(data: dict[str, Any], **params: dict[str, Any]):
     return data["input_query"]
 
 
@@ -24,12 +26,12 @@ def gptcache_data_manager(vector_dimension):
 
 
 def gptcache_data_convert(cache_data):
-    logging.info("[Cache] Cache hit, returning cache data...")
+    logger.info("[Cache] Cache hit, returning cache data...")
     return cache_data
 
 
 def gptcache_update_cache_callback(llm_data, update_cache_func, *args, **kwargs):
-    logging.info("[Cache] Cache missed, updating cache...")
+    logger.info("[Cache] Cache missed, updating cache...")
     update_cache_func(Answer(llm_data, CacheDataType.STR))
     return llm_data
 

embedchain/chunkers/base_chunker.py

@@ -6,6 +6,8 @@ from embedchain.config.add_config import ChunkerConfig
 from embedchain.helpers.json_serializable import JSONSerializable
 from embedchain.models.data_type import DataType
 
+logger = logging.getLogger(__name__)
+
 
 class BaseChunker(JSONSerializable):
     def __init__(self, text_splitter):
@@ -17,7 +19,7 @@ class BaseChunker(JSONSerializable):
         """
         Loads data and chunks it.
 
-        :param loader: The loader which's `load_data` method is used to create
+        :param loader: The loader whose `load_data` method is used to create
         the raw data.
         :param src: The data to be handled by the loader. Can be a URL for
         remote sources or local content for local loaders.
@@ -25,9 +27,9 @@ class BaseChunker(JSONSerializable):
         """
         documents = []
         chunk_ids = []
-        idMap = {}
+        id_map = {}
         min_chunk_size = config.min_chunk_size if config is not None else 1
-        logging.info(f"[INFO] Skipping chunks smaller than {min_chunk_size} characters")
+        logger.info(f"Skipping chunks smaller than {min_chunk_size} characters")
         data_result = loader.load_data(src)
         data_records = data_result["data"]
         doc_id = data_result["doc_id"]
@@ -39,21 +41,24 @@ class BaseChunker(JSONSerializable):
         for data in data_records:
             content = data["content"]
 
-            meta_data = data["meta_data"]
+            metadata = data["meta_data"]
             # add data type to meta data to allow query using data type
-            meta_data["data_type"] = self.data_type.value
-            meta_data["doc_id"] = doc_id
-            url = meta_data["url"]
+            metadata["data_type"] = self.data_type.value
+            metadata["doc_id"] = doc_id
+
+            # TODO: Currently defaulting to the src as the url. This is done intentianally since some
+            # of the data types like 'gmail' loader doesn't have the url in the meta data.
+            url = metadata.get("url", src)
 
             chunks = self.get_chunks(content)
             for chunk in chunks:
                 chunk_id = hashlib.sha256((chunk + url).encode()).hexdigest()
                 chunk_id = f"{app_id}--{chunk_id}" if app_id is not None else chunk_id
-                if idMap.get(chunk_id) is None and len(chunk) >= min_chunk_size:
-                    idMap[chunk_id] = True
+                if id_map.get(chunk_id) is None and len(chunk) >= min_chunk_size:
+                    id_map[chunk_id] = True
                     chunk_ids.append(chunk_id)
                     documents.append(chunk)
-                    metadatas.append(meta_data)
+                    metadatas.append(metadata)
         return {
             "documents": documents,
             "ids": chunk_ids,
@@ -77,5 +82,6 @@ class BaseChunker(JSONSerializable):
 
         # TODO: This should be done during initialization. This means it has to be done in the child classes.
 
-    def get_word_count(self, documents):
+    @staticmethod
+    def get_word_count(documents) -> int:
         return sum([len(document.split(" ")) for document in documents])

embedchain/chunkers/google_drive.py

@@ -0,0 +1,22 @@
+from typing import Optional
+
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+
+from embedchain.chunkers.base_chunker import BaseChunker
+from embedchain.config.add_config import ChunkerConfig
+from embedchain.helpers.json_serializable import register_deserializable
+
+
+@register_deserializable
+class GoogleDriveChunker(BaseChunker):
+    """Chunker for google drive folder."""
+
+    def __init__(self, config: Optional[ChunkerConfig] = None):
+        if config is None:
+            config = ChunkerConfig(chunk_size=1000, chunk_overlap=0, length_function=len)
+        text_splitter = RecursiveCharacterTextSplitter(
+            chunk_size=config.chunk_size,
+            chunk_overlap=config.chunk_overlap,
+            length_function=config.length_function,
+        )
+        super().__init__(text_splitter)

embedchain/cli.py

@@ -1,35 +1,33 @@
 import json
 import os
-import re
 import shutil
 import signal
 import subprocess
 import sys
 import tempfile
+import time
 import zipfile
 from pathlib import Path
 
 import click
-import pkg_resources
 import requests
 from rich.console import Console
 
 from embedchain.telemetry.posthog import AnonymousTelemetry
+from embedchain.utils.cli import (deploy_fly, deploy_gradio_app,
+                                  deploy_hf_spaces, deploy_modal,
+                                  deploy_render, deploy_streamlit,
+                                  get_pkg_path_from_name, setup_fly_io_app,
+                                  setup_gradio_app, setup_hf_app,
+                                  setup_modal_com_app, setup_render_com_app,
+                                  setup_streamlit_io_app)
 
 console = Console()
-
-
-@click.group()
-def cli():
-    pass
-
-
-anonymous_telemetry = AnonymousTelemetry()
-
-
 api_process = None
 ui_process = None
 
+anonymous_telemetry = AnonymousTelemetry()
+
 
 def signal_handler(sig, frame):
     """Signal handler to catch termination signals and kill server processes."""
@@ -44,110 +42,135 @@ def signal_handler(sig, frame):
     sys.exit(0)
 
 
-def get_pkg_path_from_name(template: str):
-    try:
-        # Determine the installation location of the embedchain package
-        package_path = pkg_resources.resource_filename("embedchain", "")
-    except ImportError:
-        console.print("❌ [bold red]Failed to locate the 'embedchain' package. Is it installed?[/bold red]")
+@click.group()
+def cli():
+    pass
+
+
+@cli.command()
+@click.argument("app_name")
+@click.option("--docker", is_flag=True, help="Use docker to create the app.")
+@click.pass_context
+def create_app(ctx, app_name, docker):
+    if Path(app_name).exists():
+        console.print(
+            f"❌ [red]Directory '{app_name}' already exists. Try using a new directory name, or remove it.[/red]"
+        )
         return
 
-    # Construct the source path from the embedchain package
-    src_path = os.path.join(package_path, "deployment", template)
+    os.makedirs(app_name)
+    os.chdir(app_name)
 
-    if not os.path.exists(src_path):
-        console.print(f"❌ [bold red]Template '{template}' not found.[/bold red]")
+    # Step 1: Download the zip file
+    zip_url = "http://github.com/embedchain/ec-admin/archive/main.zip"
+    console.print(f"Creating a new embedchain app in [green]{Path().resolve()}[/green]\n")
+    try:
+        response = requests.get(zip_url)
+        response.raise_for_status()
+        with tempfile.NamedTemporaryFile(delete=False) as tmp_file:
+            tmp_file.write(response.content)
+            zip_file_path = tmp_file.name
+        console.print("✅ [bold green]Fetched template successfully.[/bold green]")
+    except requests.RequestException as e:
+        console.print(f"❌ [bold red]Failed to download zip file: {e}[/bold red]")
+        anonymous_telemetry.capture(event_name="ec_create_app", properties={"success": False})
         return
 
-    return src_path
-
-
-def setup_fly_io_app(extra_args):
-    fly_launch_command = ["fly", "launch", "--region", "sjc", "--no-deploy"] + list(extra_args)
+    # Step 2: Extract the zip file
     try:
-        console.print(f"🚀 [bold cyan]Running: {' '.join(fly_launch_command)}[/bold cyan]")
-        shutil.move(".env.example", ".env")
-        subprocess.run(fly_launch_command, check=True)
-        console.print("✅ [bold green]'fly launch' executed successfully.[/bold green]")
-    except subprocess.CalledProcessError as e:
-        console.print(f"❌ [bold red]An error occurred: {e}[/bold red]")
-    except FileNotFoundError:
-        console.print(
-            "❌ [bold red]'fly' command not found. Please ensure Fly CLI is installed and in your PATH.[/bold red]"
-        )
+        with zipfile.ZipFile(zip_file_path, "r") as zip_ref:
+            # Get the name of the root directory inside the zip file
+            root_dir = Path(zip_ref.namelist()[0])
+            for member in zip_ref.infolist():
+                # Build the path to extract the file to, skipping the root directory
+                target_file = Path(member.filename).relative_to(root_dir)
+                source_file = zip_ref.open(member, "r")
+                if member.is_dir():
+                    # Create directory if it doesn't exist
+                    os.makedirs(target_file, exist_ok=True)
+                else:
+                    with open(target_file, "wb") as file:
+                        # Write the file
+                        shutil.copyfileobj(source_file, file)
+            console.print("✅ [bold green]Extracted zip file successfully.[/bold green]")
+            anonymous_telemetry.capture(event_name="ec_create_app", properties={"success": True})
+    except zipfile.BadZipFile:
+        console.print("❌ [bold red]Error in extracting zip file. The file might be corrupted.[/bold red]")
+        anonymous_telemetry.capture(event_name="ec_create_app", properties={"success": False})
+        return
 
-
-def setup_modal_com_app(extra_args):
-    modal_setup_file = os.path.join(os.path.expanduser("~"), ".modal.toml")
-    if os.path.exists(modal_setup_file):
-        console.print(
-            """✅ [bold green]Modal setup already done. You can now install the dependencies by doing \n
-            `pip install -r requirements.txt`[/bold green]"""
-        )
+    if docker:
+        subprocess.run(["docker-compose", "build"], check=True)
     else:
-        modal_setup_cmd = ["modal", "setup"] + list(extra_args)
-        console.print(f"🚀 [bold cyan]Running: {' '.join(modal_setup_cmd)}[/bold cyan]")
-        subprocess.run(modal_setup_cmd, check=True)
-    shutil.move(".env.example", ".env")
-    console.print(
-        """Great! Now you can install the dependencies by doing: \n
-                  `pip install -r requirements.txt`\n
-                  \n
-                  To run your app locally:\n
-                  `ec dev`
-                  """
-    )
+        ctx.invoke(install_reqs)
 
 
-def setup_render_com_app():
-    render_setup_file = os.path.join(os.path.expanduser("~"), ".render/config.yaml")
-    if os.path.exists(render_setup_file):
-        console.print(
-            """✅ [bold green]Render setup already done. You can now install the dependencies by doing \n
-            `pip install -r requirements.txt`[/bold green]"""
-        )
-    else:
-        render_setup_cmd = ["render", "config", "init"]
-        console.print(f"🚀 [bold cyan]Running: {' '.join(render_setup_cmd)}[/bold cyan]")
-        subprocess.run(render_setup_cmd, check=True)
-    shutil.move(".env.example", ".env")
-    console.print(
-        """Great! Now you can install the dependencies by doing: \n
-                  `pip install -r requirements.txt`\n
-                  \n
-                  To run your app locally:\n
-                  `ec dev`
-                  """
-    )
+@cli.command()
+def install_reqs():
+    try:
+        console.print("Installing python requirements...\n")
+        time.sleep(2)
+        os.chdir("api")
+        subprocess.run(["pip", "install", "-r", "requirements.txt"], check=True)
+        os.chdir("..")
+        console.print("\n ✅ [bold green]Installed API requirements successfully.[/bold green]\n")
+    except Exception as e:
+        console.print(f"❌ [bold red]Failed to install API requirements: {e}[/bold red]")
+        anonymous_telemetry.capture(event_name="ec_install_reqs", properties={"success": False})
+        return
+
+    try:
+        os.chdir("ui")
+        subprocess.run(["yarn"], check=True)
+        console.print("\n✅ [bold green]Successfully installed frontend requirements.[/bold green]")
+        anonymous_telemetry.capture(event_name="ec_install_reqs", properties={"success": True})
+    except Exception as e:
+        console.print(f"❌ [bold red]Failed to install frontend requirements. Error: {e}[/bold red]")
+        anonymous_telemetry.capture(event_name="ec_install_reqs", properties={"success": False})
 
 
-def setup_streamlit_io_app():
-    # nothing needs to be done here
-    console.print("Great! Now you can install the dependencies by doing `pip install -r requirements.txt`")
+@cli.command()
+@click.option("--docker", is_flag=True, help="Run inside docker.")
+def start(docker):
+    if docker:
+        subprocess.run(["docker-compose", "up"], check=True)
+        return
 
+    # Set up signal handling
+    signal.signal(signal.SIGINT, signal_handler)
+    signal.signal(signal.SIGTERM, signal_handler)
 
-def setup_gradio_app():
-    # nothing needs to be done here
-    console.print("Great! Now you can install the dependencies by doing `pip install -r requirements.txt`")
+    # Step 1: Start the API server
+    try:
+        os.chdir("api")
+        api_process = subprocess.Popen(["python", "-m", "main"], stdout=None, stderr=None)
+        os.chdir("..")
+        console.print("✅ [bold green]API server started successfully.[/bold green]")
+    except Exception as e:
+        console.print(f"❌ [bold red]Failed to start the API server: {e}[/bold red]")
+        anonymous_telemetry.capture(event_name="ec_start", properties={"success": False})
+        return
 
+    # Sleep for 2 seconds to give the user time to read the message
+    time.sleep(2)
 
-def setup_hf_app():
-    subprocess.run(["pip", "install", "huggingface_hub[cli]"], check=True)
-    hf_setup_file = os.path.join(os.path.expanduser("~"), ".cache/huggingface/token")
-    if os.path.exists(hf_setup_file):
-        console.print(
-            """✅ [bold green]HuggingFace setup already done. You can now install the dependencies by doing \n
-            `pip install -r requirements.txt`[/bold green]"""
-        )
-    else:
-        console.print(
-            """🚀 [cyan]Running: huggingface-cli login \n
-                Please provide a [bold]WRITE[/bold] token so that we can directly deploy\n
-                your apps from the terminal.[/cyan]
-                """
-        )
-        subprocess.run(["huggingface-cli", "login"], check=True)
-    console.print("Great! Now you can install the dependencies by doing `pip install -r requirements.txt`")
+    # Step 2: Install UI requirements and start the UI server
+    try:
+        os.chdir("ui")
+        subprocess.run(["yarn"], check=True)
+        ui_process = subprocess.Popen(["yarn", "dev"])
+        console.print("✅ [bold green]UI server started successfully.[/bold green]")
+        anonymous_telemetry.capture(event_name="ec_start", properties={"success": True})
+    except Exception as e:
+        console.print(f"❌ [bold red]Failed to start the UI server: {e}[/bold red]")
+        anonymous_telemetry.capture(event_name="ec_start", properties={"success": False})
+
+    # Keep the script running until it receives a kill signal
+    try:
+        api_process.wait()
+        ui_process.wait()
+    except KeyboardInterrupt:
+        console.print("\n🛑 [bold yellow]Stopping server...[/bold yellow]")
 
 
 @cli.command()
@@ -172,7 +195,7 @@ def create(template, extra_args):
         setup_streamlit_io_app()
     elif template == "gradio.app":
         setup_gradio_app()
-    elif template == "hf/gradio.app" or template == "hf/streamlit.app":
+    elif template == "hf/gradio.app" or template == "hf/streamlit.io":
         setup_hf_app()
     else:
         raise ValueError(f"Unknown template '{template}'.")
@@ -269,7 +292,7 @@ def dev(debug, host, port):
         run_dev_modal_com()
     elif template == "render.com":
         run_dev_render_com(debug, host, port)
-    elif template == "streamlit.io" or template == "hf/streamlit.app":
+    elif template == "streamlit.io" or template == "hf/streamlit.io":
         run_dev_streamlit_io()
     elif template == "gradio.app" or template == "hf/gradio.app":
         run_dev_gradio()
@@ -277,141 +300,6 @@ def dev(debug, host, port):
         raise ValueError(f"Unknown template '{template}'.")
 
 
-def read_env_file(env_file_path):
-    """
-    Reads an environment file and returns a dictionary of key-value pairs.
-
-    Args:
-    env_file_path (str): The path to the .env file.
-
-    Returns:
-    dict: Dictionary of environment variables.
-    """
-    env_vars = {}
-    with open(env_file_path, "r") as file:
-        for line in file:
-            # Ignore comments and empty lines
-            if line.strip() and not line.strip().startswith("#"):
-                # Assume each line is in the format KEY=VALUE
-                key_value_match = re.match(r"(\w+)=(.*)", line.strip())
-                if key_value_match:
-                    key, value = key_value_match.groups()
-                    env_vars[key] = value
-    return env_vars
-
-
-def deploy_fly():
-    app_name = ""
-    with open("fly.toml", "r") as file:
-        for line in file:
-            if line.strip().startswith("app ="):
-                app_name = line.split("=")[1].strip().strip('"')
-
-    if not app_name:
-        console.print("❌ [bold red]App name not found in fly.toml[/bold red]")
-        return
-
-    env_vars = read_env_file(".env")
-    secrets_command = ["flyctl", "secrets", "set", "-a", app_name] + [f"{k}={v}" for k, v in env_vars.items()]
-
-    deploy_command = ["fly", "deploy"]
-    try:
-        # Set secrets
-        console.print(f"🔐 [bold cyan]Setting secrets for {app_name}[/bold cyan]")
-        subprocess.run(secrets_command, check=True)
-
-        # Deploy application
-        console.print(f"🚀 [bold cyan]Running: {' '.join(deploy_command)}[/bold cyan]")
-        subprocess.run(deploy_command, check=True)
-        console.print("✅ [bold green]'fly deploy' executed successfully.[/bold green]")
-
-    except subprocess.CalledProcessError as e:
-        console.print(f"❌ [bold red]An error occurred: {e}[/bold red]")
-    except FileNotFoundError:
-        console.print(
-            "❌ [bold red]'fly' command not found. Please ensure Fly CLI is installed and in your PATH.[/bold red]"
-        )
-
-
-def deploy_modal():
-    modal_deploy_cmd = ["modal", "deploy", "app"]
-    try:
-        console.print(f"🚀 [bold cyan]Running: {' '.join(modal_deploy_cmd)}[/bold cyan]")
-        subprocess.run(modal_deploy_cmd, check=True)
-        console.print("✅ [bold green]'modal deploy' executed successfully.[/bold green]")
-    except subprocess.CalledProcessError as e:
-        console.print(f"❌ [bold red]An error occurred: {e}[/bold red]")
-    except FileNotFoundError:
-        console.print(
-            "❌ [bold red]'modal' command not found. Please ensure Modal CLI is installed and in your PATH.[/bold red]"
-        )
-
-
-def deploy_streamlit():
-    streamlit_deploy_cmd = ["streamlit", "run", "app.py"]
-    try:
-        console.print(f"🚀 [bold cyan]Running: {' '.join(streamlit_deploy_cmd)}[/bold cyan]")
-        console.print(
-            """\n\n✅ [bold yellow]To deploy a streamlit app, you can directly it from the UI.\n
-        Click on the 'Deploy' button on the top right corner of the app.\n
-        For more information, please refer to https://docs.embedchain.ai/deployment/streamlit_io
-        [/bold yellow]
-                      \n\n"""
-        )
-        subprocess.run(streamlit_deploy_cmd, check=True)
-    except subprocess.CalledProcessError as e:
-        console.print(f"❌ [bold red]An error occurred: {e}[/bold red]")
-    except FileNotFoundError:
-        console.print(
-            """❌ [bold red]'streamlit' command not found.\n
-            Please ensure Streamlit CLI is installed and in your PATH.[/bold red]"""
-        )
-
-
-def deploy_render():
-    render_deploy_cmd = ["render", "blueprint", "launch"]
-
-    try:
-        console.print(f"🚀 [bold cyan]Running: {' '.join(render_deploy_cmd)}[/bold cyan]")
-        subprocess.run(render_deploy_cmd, check=True)
-        console.print("✅ [bold green]'render blueprint launch' executed successfully.[/bold green]")
-    except subprocess.CalledProcessError as e:
-        console.print(f"❌ [bold red]An error occurred: {e}[/bold red]")
-    except FileNotFoundError:
-        console.print(
-            "❌ [bold red]'render' command not found. Please ensure Render CLI is installed and in your PATH.[/bold red]"  # noqa:E501
-        )
-
-
-def deploy_gradio_app():
-    gradio_deploy_cmd = ["gradio", "deploy"]
-
-    try:
-        console.print(f"🚀 [bold cyan]Running: {' '.join(gradio_deploy_cmd)}[/bold cyan]")
-        subprocess.run(gradio_deploy_cmd, check=True)
-        console.print("✅ [bold green]'gradio deploy' executed successfully.[/bold green]")
-    except subprocess.CalledProcessError as e:
-        console.print(f"❌ [bold red]An error occurred: {e}[/bold red]")
-    except FileNotFoundError:
-        console.print(
-            "❌ [bold red]'gradio' command not found. Please ensure Gradio CLI is installed and in your PATH.[/bold red]"  # noqa:E501
-        )
-
-
-def deploy_hf_spaces(ec_app_name):
-    if not ec_app_name:
-        console.print("❌ [bold red]'name' not found in embedchain.json[/bold red]")
-        return
-    hf_spaces_deploy_cmd = ["huggingface-cli", "upload", ec_app_name, ".", ".", "--repo-type=space"]
-
-    try:
-        console.print(f"🚀 [bold cyan]Running: {' '.join(hf_spaces_deploy_cmd)}[/bold cyan]")
-        subprocess.run(hf_spaces_deploy_cmd, check=True)
-        console.print("✅ [bold green]'huggingface-cli upload' executed successfully.[/bold green]")
-    except subprocess.CalledProcessError as e:
-        console.print(f"❌ [bold red]An error occurred: {e}[/bold red]")
-
-
 @cli.command()
 def deploy():
     # Check for platform-specific files
@@ -437,99 +325,3 @@ def deploy():
         deploy_hf_spaces(ec_app_name)
     else:
         console.print("❌ [bold red]No recognized deployment platform found.[/bold red]")
-
-
-@cli.command()
-def runserver():
-    # Set up signal handling
-    signal.signal(signal.SIGINT, signal_handler)
-    signal.signal(signal.SIGTERM, signal_handler)
-
-    # Check if 'api' and 'ui' directories exist
-    if os.path.exists("api") and os.path.exists("ui"):
-        pass
-    else:
-        # Step 1: Download the zip file
-        zip_url = "http://github.com/embedchain/ec-admin/archive/main.zip"
-        try:
-            response = requests.get(zip_url)
-            response.raise_for_status()
-            with tempfile.NamedTemporaryFile(delete=False) as tmp_file:
-                tmp_file.write(response.content)
-                zip_file_path = tmp_file.name
-            console.print("✅ [bold green]Downloaded zip file successfully.[/bold green]")
-        except requests.RequestException as e:
-            console.print(f"❌ [bold red]Failed to download zip file: {e}[/bold red]")
-            return
-
-        # Step 2: Extract the zip file
-        try:
-            with zipfile.ZipFile(zip_file_path, "r") as zip_ref:
-                # Get the name of the root directory inside the zip file
-                root_dir = Path(zip_ref.namelist()[0])
-                for member in zip_ref.infolist():
-                    # Build the path to extract the file to, skipping the root directory
-                    target_file = Path(member.filename).relative_to(root_dir)
-                    source_file = zip_ref.open(member, "r")
-                    if member.is_dir():
-                        # Create directory if it doesn't exist
-                        os.makedirs(target_file, exist_ok=True)
-                    else:
-                        with open(target_file, "wb") as file:
-                            # Write the file
-                            shutil.copyfileobj(source_file, file)
-                console.print("✅ [bold green]Extracted zip file successfully.[/bold green]")
-        except zipfile.BadZipFile:
-            console.print("❌ [bold red]Error in extracting zip file. The file might be corrupted.[/bold red]")
-            return
-
-        # Step 3: Install API requirements
-        try:
-            os.chdir("api")
-            subprocess.run(["pip", "install", "-r", "requirements.txt"], check=True)
-            os.chdir("..")
-            console.print("✅ [bold green]Installed API requirements successfully.[/bold green]")
-        except Exception as e:
-            console.print(f"❌ [bold red]Failed to install API requirements: {e}[/bold red]")
-            return
-
-    # Step 4: Start the API server
-    try:
-        os.chdir("api")
-        api_process = subprocess.Popen(
-            ["uvicorn", "main:app", "--reload", "--host", "127.0.0.1", "--port", "8000"], stdout=None, stderr=None
-        )
-        os.chdir("..")
-        console.print("✅ [bold green]API server started successfully.[/bold green]")
-    except Exception as e:
-        console.print(f"❌ [bold red]Failed to start the API server: {e}[/bold red]")
-        return
-
-    # Step 5: Install UI requirements and start the UI server
-    try:
-        os.chdir("ui")
-        subprocess.run(["yarn"], check=True)
-        ui_process = subprocess.Popen(["yarn", "dev"], stdout=None, stderr=None)
-        console.print("✅ [bold green]UI server started successfully.[/bold green]")
-    except Exception as e:
-        console.print(f"❌ [bold red]Failed to start the UI server: {e}[/bold red]")
-
-    # Wait for the subprocesses to complete
-    api_process.wait()
-    ui_process.wait()
-
-    # Step 6: Install UI requirements and start the UI server
-    try:
-        os.chdir("ui")
-        subprocess.run(["yarn"], check=True)
-        subprocess.Popen(["yarn", "dev"])
-        console.print("✅ [bold green]UI server started successfully.[/bold green]")
-    except Exception as e:
-        console.print(f"❌ [bold red]Failed to start the UI server: {e}[/bold red]")
-
-    # Keep the script running until it receives a kill signal
-    try:
-        api_process.wait()
-        ui_process.wait()
-    except KeyboardInterrupt:
-        console.print("\n🛑 [bold yellow]Stopping server...[/bold yellow]")

embedchain/client.py

@@ -7,6 +7,8 @@ import requests
 
 from embedchain.constants import CONFIG_DIR, CONFIG_FILE
 
+logger = logging.getLogger(__name__)
+
 
 class Client:
     def __init__(self, api_key=None, host="https://apiv2.embedchain.ai"):
@@ -24,14 +26,14 @@ class Client:
         else:
             if "api_key" in self.config_data:
                 self.api_key = self.config_data["api_key"]
-                logging.info("API key loaded successfully!")
+                logger.info("API key loaded successfully!")
             else:
                 raise ValueError(
                     "You are not logged in. Please obtain an API key from https://app.embedchain.ai/settings/keys/"
                 )
 
     @classmethod
-    def setup_dir(self):
+    def setup(cls):
         """
         Loads the user id from the config file if it exists, otherwise generates a new
         one and saves it to the config file.
@@ -40,6 +42,7 @@ class Client:
         :rtype: str
         """
         os.makedirs(CONFIG_DIR, exist_ok=True)
+
         if os.path.exists(CONFIG_FILE):
             with open(CONFIG_FILE, "r") as f:
                 data = json.load(f)
@@ -53,7 +56,7 @@ class Client:
     @classmethod
     def load_config(cls):
         if not os.path.exists(CONFIG_FILE):
-            cls.setup_dir()
+            cls.setup()
 
         with open(CONFIG_FILE, "r") as config_file:
             return json.load(config_file)
@@ -63,7 +66,7 @@ class Client:
         with open(CONFIG_FILE, "w") as config_file:
             json.dump(self.config_data, config_file, indent=4)
 
-        logging.info("API key saved successfully!")
+        logger.info("API key saved successfully!")
 
     def clear(self):
         if "api_key" in self.config_data:
@@ -71,17 +74,17 @@ class Client:
             with open(CONFIG_FILE, "w") as config_file:
                 json.dump(self.config_data, config_file, indent=4)
             self.api_key = None
-            logging.info("API key deleted successfully!")
+            logger.info("API key deleted successfully!")
         else:
-            logging.warning("API key not found in the configuration file.")
+            logger.warning("API key not found in the configuration file.")
 
     def update(self, api_key):
         if self.check(api_key):
             self.api_key = api_key
             self.save()
-            logging.info("API key updated successfully!")
+            logger.info("API key updated successfully!")
         else:
-            logging.warning("Invalid API key provided. API key not updated.")
+            logger.warning("Invalid API key provided. API key not updated.")
 
     def check(self, api_key):
         validation_url = f"{self.host}/api/v1/accounts/api_keys/validate/"
@@ -89,8 +92,8 @@ class Client:
         if response.status_code == 200:
             return True
         else:
-            logging.warning(f"Response from API: {response.text}")
-            logging.warning("Invalid API key. Unable to validate.")
+            logger.warning(f"Response from API: {response.text}")
+            logger.warning("Invalid API key. Unable to validate.")
             return False
 
     def get(self):

embedchain/config/add_config.py

@@ -1,7 +1,8 @@
 import builtins
 import logging
+from collections.abc import Callable
 from importlib import import_module
-from typing import Callable, Optional
+from typing import Optional
 
 from embedchain.config.base_config import BaseConfig
 from embedchain.helpers.json_serializable import register_deserializable
@@ -26,7 +27,7 @@ class ChunkerConfig(BaseConfig):
         if self.min_chunk_size >= self.chunk_size:
             raise ValueError(f"min_chunk_size {min_chunk_size} should be less than chunk_size {chunk_size}")
         if self.min_chunk_size < self.chunk_overlap:
-            logging.warn(
+            logging.warning(
                 f"min_chunk_size {min_chunk_size} should be greater than chunk_overlap {chunk_overlap}, otherwise it is redundant."  # noqa:E501
             )
 
@@ -35,7 +36,8 @@ class ChunkerConfig(BaseConfig):
         else:
             self.length_function = length_function if length_function else len
 
-    def load_func(self, dotpath: str):
+    @staticmethod
+    def load_func(dotpath: str):
         if "." not in dotpath:
             return getattr(builtins, dotpath)
         else:

embedchain/config/base_app_config.py

@@ -5,6 +5,8 @@ from embedchain.config.base_config import BaseConfig
 from embedchain.helpers.json_serializable import JSONSerializable
 from embedchain.vectordb.base import BaseVectorDB
 
+logger = logging.getLogger(__name__)
+
 
 class BaseAppConfig(BaseConfig, JSONSerializable):
     """
@@ -36,29 +38,21 @@ class BaseAppConfig(BaseConfig, JSONSerializable):
         defaults to None
         :type collection_name: Optional[str], optional
         """
-        self._setup_logging(log_level)
         self.id = id
         self.collect_metrics = True if (collect_metrics is True or collect_metrics is None) else False
         self.collection_name = collection_name
 
         if db:
             self._db = db
-            logging.warning(
+            logger.warning(
                 "DEPRECATION WARNING: Please supply the database as the second parameter during app init. "
                 "Such as `app(config=config, db=db)`."
             )
 
         if collection_name:
-            logging.warning("DEPRECATION WARNING: Please supply the collection name to the database config.")
+            logger.warning("DEPRECATION WARNING: Please supply the collection name to the database config.")
         return
 
-    def _setup_logging(self, debug_level):
-        level = logging.WARNING  # Default level
-        if debug_level is not None:
-            level = getattr(logging, debug_level.upper(), None)
-            if not isinstance(level, int):
-                raise ValueError(f"Invalid log level: {debug_level}")
-
-        logging.basicConfig(format="%(asctime)s [%(name)s] [%(levelname)s] %(message)s", level=level)
-        self.logger = logging.getLogger(__name__)
-        return
+    def _setup_logging(self, log_level):
+        logger.basicConfig(format="%(asctime)s [%(name)s] [%(levelname)s] %(message)s", level=log_level)
+        self.logger = logger.getLogger(__name__)

embedchain/config/base_config.py

@@ -1,4 +1,4 @@
-from typing import Any, Dict
+from typing import Any
 
 from embedchain.helpers.json_serializable import JSONSerializable
 
@@ -12,10 +12,10 @@ class BaseConfig(JSONSerializable):
         """Initializes a configuration class for a class."""
         pass
 
-    def as_dict(self) -> Dict[str, Any]:
+    def as_dict(self) -> dict[str, Any]:
         """Return config object as a dict
 
         :return: config object as dict
-        :rtype: Dict[str, Any]
+        :rtype: dict[str, Any]
         """
         return vars(self)

embedchain/config/cache_config.py

@@ -1,4 +1,4 @@
-from typing import Any, Dict, Optional
+from typing import Any, Optional
 
 from embedchain.config.base_config import BaseConfig
 from embedchain.helpers.json_serializable import register_deserializable
@@ -10,12 +10,12 @@ class CacheSimilarityEvalConfig(BaseConfig):
     This is the evaluator to compare two embeddings according to their distance computed in embedding retrieval stage.
     In the retrieval stage, `search_result` is the distance used for approximate nearest neighbor search and have been
     put into `cache_dict`. `max_distance` is used to bound this distance to make it between [0-`max_distance`].
-    `positive` is used to indicate this distance is directly proportional to the similarity of two entites.
-    If `positive` is set `False`, `max_distance` will be used to substract this distance to get the final score.
+    `positive` is used to indicate this distance is directly proportional to the similarity of two entities.
+    If `positive` is set `False`, `max_distance` will be used to subtract this distance to get the final score.
 
     :param max_distance: the bound of maximum distance.
     :type max_distance: float
-    :param positive: if the larger distance indicates more similar of two entities, It is True. Otherwise it is False.
+    :param positive: if the larger distance indicates more similar of two entities, It is True. Otherwise, it is False.
     :type positive: bool
     """
 
@@ -29,7 +29,8 @@ class CacheSimilarityEvalConfig(BaseConfig):
         self.max_distance = max_distance
         self.positive = positive
 
-    def from_config(config: Optional[Dict[str, Any]]):
+    @staticmethod
+    def from_config(config: Optional[dict[str, Any]]):
         if config is None:
             return CacheSimilarityEvalConfig()
         else:
@@ -63,7 +64,8 @@ class CacheInitConfig(BaseConfig):
         self.similarity_threshold = similarity_threshold
         self.auto_flush = auto_flush
 
-    def from_config(config: Optional[Dict[str, Any]]):
+    @staticmethod
+    def from_config(config: Optional[dict[str, Any]]):
         if config is None:
             return CacheInitConfig()
         else:
@@ -83,7 +85,8 @@ class CacheConfig(BaseConfig):
         self.similarity_eval_config = similarity_eval_config
         self.init_config = init_config
 
-    def from_config(config: Optional[Dict[str, Any]]):
+    @staticmethod
+    def from_config(config: Optional[dict[str, Any]]):
         if config is None:
             return CacheConfig()
         else:

embedchain/config/embedder/base.py

@@ -6,7 +6,12 @@ from embedchain.helpers.json_serializable import register_deserializable
 @register_deserializable
 class BaseEmbedderConfig:
     def __init__(
-        self, model: Optional[str] = None, deployment_name: Optional[str] = None, api_key: Optional[str] = None
+        self,
+        model: Optional[str] = None,
+        deployment_name: Optional[str] = None,
+        vector_dimension: Optional[int] = None,
+        api_key: Optional[str] = None,
+        api_base: Optional[str] = None,
     ):
         """
         Initialize a new instance of an embedder config class.
@@ -18,4 +23,6 @@ class BaseEmbedderConfig:
         """
         self.model = model
         self.deployment_name = deployment_name
+        self.vector_dimension = vector_dimension
         self.api_key = api_key
+        self.api_base = api_base

embedchain/config/evaluation/__init__.py

@@ -0,0 +1,2 @@
+from .base import (AnswerRelevanceConfig, ContextRelevanceConfig,  # noqa: F401
+                   GroundednessConfig)

embedchain/config/evaluation/base.py

@@ -0,0 +1,92 @@
+from typing import Optional
+
+from embedchain.config.base_config import BaseConfig
+
+ANSWER_RELEVANCY_PROMPT = """
+Please provide $num_gen_questions questions from the provided answer.
+You must provide the complete question, if are not able to provide the complete question, return empty string ("").
+Please only provide one question per line without numbers or bullets to distinguish them.
+You must only provide the questions and no other text.
+
+$answer
+"""  # noqa:E501
+
+
+CONTEXT_RELEVANCY_PROMPT = """
+Please extract relevant sentences from the provided context that is required to answer the given question.
+If no relevant sentences are found, or if you believe the question cannot be answered from the given context, return the empty string ("").
+While extracting candidate sentences you're not allowed to make any changes to sentences from given context or make up any sentences.
+You must only provide sentences from the given context and nothing else.
+
+Context: $context
+Question: $question
+"""  # noqa:E501
+
+GROUNDEDNESS_ANSWER_CLAIMS_PROMPT = """
+Please provide one or more statements from each sentence of the provided answer.
+You must provide the symantically equivalent statements for each sentence of the answer.
+You must provide the complete statement, if are not able to provide the complete statement, return empty string ("").
+Please only provide one statement per line WITHOUT numbers or bullets.
+If the question provided is not being answered in the provided answer, return empty string ("").
+You must only provide the statements and no other text.
+
+$question
+$answer
+"""  # noqa:E501
+
+GROUNDEDNESS_CLAIMS_INFERENCE_PROMPT = """
+Given the context and the provided claim statements, please provide a verdict for each claim statement whether it can be completely infered from the given context or not.
+Use only "1" (yes), "0" (no) and "-1" (null) for "yes", "no" or "null" respectively.
+You must provide one verdict per line, ONLY WITH "1", "0" or "-1" as per your verdict to the given statement and nothing else.
+You must provide the verdicts in the same order as the claim statements.
+
+Contexts: 
+$context
+
+Claim statements: 
+$claim_statements
+"""  # noqa:E501
+
+
+class GroundednessConfig(BaseConfig):
+    def __init__(
+        self,
+        model: str = "gpt-4",
+        api_key: Optional[str] = None,
+        answer_claims_prompt: str = GROUNDEDNESS_ANSWER_CLAIMS_PROMPT,
+        claims_inference_prompt: str = GROUNDEDNESS_CLAIMS_INFERENCE_PROMPT,
+    ):
+        self.model = model
+        self.api_key = api_key
+        self.answer_claims_prompt = answer_claims_prompt
+        self.claims_inference_prompt = claims_inference_prompt
+
+
+class AnswerRelevanceConfig(BaseConfig):
+    def __init__(
+        self,
+        model: str = "gpt-4",
+        embedder: str = "text-embedding-ada-002",
+        api_key: Optional[str] = None,
+        num_gen_questions: int = 1,
+        prompt: str = ANSWER_RELEVANCY_PROMPT,
+    ):
+        self.model = model
+        self.embedder = embedder
+        self.api_key = api_key
+        self.num_gen_questions = num_gen_questions
+        self.prompt = prompt
+
+
+class ContextRelevanceConfig(BaseConfig):
+    def __init__(
+        self,
+        model: str = "gpt-4",
+        api_key: Optional[str] = None,
+        language: str = "en",
+        prompt: str = CONTEXT_RELEVANCY_PROMPT,
+    ):
+        self.model = model
+        self.api_key = api_key
+        self.language = language
+        self.prompt = prompt

embedchain/config/llm/base.py

@@ -1,45 +1,68 @@
 import logging
 import re
 from string import Template
-from typing import Any, Dict, List, Optional
+from typing import Any, Optional
 
 from embedchain.config.base_config import BaseConfig
 from embedchain.helpers.json_serializable import register_deserializable
 
+logger = logging.getLogger(__name__)
+
 DEFAULT_PROMPT = """
-  Use the following pieces of context to answer the query at the end.
-  If you don't know the answer, just say that you don't know, don't try to make up an answer.
+You are a Q&A expert system. Your responses must always be rooted in the context provided for each query. Here are some guidelines to follow:
 
-  $context
+1. Refrain from explicitly mentioning the context provided in your response.
+2. The context should silently guide your answers without being directly acknowledged.
+3. Do not use phrases such as 'According to the context provided', 'Based on the context, ...' etc.
 
-  Query: $query
+Context information:
+----------------------
+$context
+----------------------
 
-  Helpful Answer:
+Query: $query
+Answer:
 """  # noqa:E501
 
 DEFAULT_PROMPT_WITH_HISTORY = """
-  Use the following pieces of context to answer the query at the end.
-  If you don't know the answer, just say that you don't know, don't try to make up an answer.
-  I will provide you with our conversation history.
+You are a Q&A expert system. Your responses must always be rooted in the context provided for each query. You are also provided with the conversation history with the user. Make sure to use relevant context from conversation history as needed.
 
-  $context
+Here are some guidelines to follow:
 
-  History: $history
+1. Refrain from explicitly mentioning the context provided in your response.
+2. The context should silently guide your answers without being directly acknowledged.
+3. Do not use phrases such as 'According to the context provided', 'Based on the context, ...' etc.
 
-  Query: $query
+Context information:
+----------------------
+$context
+----------------------
 
-  Helpful Answer:
+Conversation history:
+----------------------
+$history
+----------------------
+
+Query: $query
+Answer:
 """  # noqa:E501
 
 DOCS_SITE_DEFAULT_PROMPT = """
-  Use the following pieces of context to answer the query at the end.
-  If you don't know the answer, just say that you don't know, don't try to make up an answer. Wherever possible, give complete code snippet. Dont make up any code snippet on your own.
+You are an expert AI assistant for developer support product. Your responses must always be rooted in the context provided for each query. Wherever possible, give complete code snippet. Dont make up any code snippet on your own.
 
-  $context
+Here are some guidelines to follow:
 
-  Query: $query
+1. Refrain from explicitly mentioning the context provided in your response.
+2. The context should silently guide your answers without being directly acknowledged.
+3. Do not use phrases such as 'According to the context provided', 'Based on the context, ...' etc.
 
-  Helpful Answer:
+Context information:
+----------------------
+$context
+----------------------
+
+Query: $query
+Answer:
 """  # noqa:E501
 
 DEFAULT_PROMPT_TEMPLATE = Template(DEFAULT_PROMPT)
@@ -68,10 +91,14 @@ class BaseLlmConfig(BaseConfig):
         stream: bool = False,
         deployment_name: Optional[str] = None,
         system_prompt: Optional[str] = None,
-        where: Dict[str, Any] = None,
+        where: dict[str, Any] = None,
         query_type: Optional[str] = None,
-        callbacks: Optional[List] = None,
+        callbacks: Optional[list] = None,
         api_key: Optional[str] = None,
+        base_url: Optional[str] = None,
+        endpoint: Optional[str] = None,
+        model_kwargs: Optional[dict[str, Any]] = None,
+        local: Optional[bool] = False,
     ):
         """
         Initializes a configuration class instance for the LLM.
@@ -104,15 +131,25 @@ class BaseLlmConfig(BaseConfig):
         :param system_prompt: System prompt string, defaults to None
         :type system_prompt: Optional[str], optional
         :param where: A dictionary of key-value pairs to filter the database results., defaults to None
-        :type where: Dict[str, Any], optional
+        :type where: dict[str, Any], optional
+        :param api_key: The api key of the custom endpoint, defaults to None
+        :type api_key: Optional[str], optional
+        :param endpoint: The api url of the custom endpoint, defaults to None
+        :type endpoint: Optional[str], optional
+        :param model_kwargs: A dictionary of key-value pairs to pass to the model, defaults to None
+        :type model_kwargs: Optional[Dict[str, Any]], optional
         :param callbacks: Langchain callback functions to use, defaults to None
-        :type callbacks: Optional[List], optional
+        :type callbacks: Optional[list], optional
+        :param query_type: The type of query to use, defaults to None
+        :type query_type: Optional[str], optional
+        :param local: If True, the model will be run locally, defaults to False (for huggingface provider)
+        :type local: Optional[bool], optional
         :raises ValueError: If the template is not valid as template should
         contain $context and $query (and optionally $history)
         :raises ValueError: Stream is not boolean
         """
         if template is not None:
-            logging.warning(
+            logger.warning(
                 "The `template` argument is deprecated and will be removed in a future version. "
                 + "Please use `prompt` instead."
             )
@@ -132,8 +169,12 @@ class BaseLlmConfig(BaseConfig):
         self.query_type = query_type
         self.callbacks = callbacks
         self.api_key = api_key
+        self.base_url = base_url
+        self.endpoint = endpoint
+        self.model_kwargs = model_kwargs
+        self.local = local
 
-        if type(prompt) is str:
+        if isinstance(prompt, str):
             prompt = Template(prompt)
 
         if self.validate_prompt(prompt):
@@ -146,24 +187,26 @@ class BaseLlmConfig(BaseConfig):
         self.stream = stream
         self.where = where
 
-    def validate_prompt(self, prompt: Template) -> bool:
+    @staticmethod
+    def validate_prompt(prompt: Template) -> Optional[re.Match[str]]:
         """
         validate the prompt
 
         :param prompt: the prompt to validate
         :type prompt: Template
         :return: valid (true) or invalid (false)
-        :rtype: bool
+        :rtype: Optional[re.Match[str]]
         """
         return re.search(query_re, prompt.template) and re.search(context_re, prompt.template)
 
-    def _validate_prompt_history(self, prompt: Template) -> bool:
+    @staticmethod
+    def _validate_prompt_history(prompt: Template) -> Optional[re.Match[str]]:
         """
         validate the prompt with history
 
         :param prompt: the prompt to validate
         :type prompt: Template
         :return: valid (true) or invalid (false)
-        :rtype: bool
+        :rtype: Optional[re.Match[str]]
         """
         return re.search(history_re, prompt.template)

embedchain/config/vectordb/elasticsearch.py

@@ -1,5 +1,5 @@
 import os
-from typing import Dict, List, Optional, Union
+from typing import Optional, Union
 
 from embedchain.config.vectordb.base import BaseVectorDbConfig
 from embedchain.helpers.json_serializable import register_deserializable
@@ -11,9 +11,9 @@ class ElasticsearchDBConfig(BaseVectorDbConfig):
         self,
         collection_name: Optional[str] = None,
         dir: Optional[str] = None,
-        es_url: Union[str, List[str]] = None,
+        es_url: Union[str, list[str]] = None,
         cloud_id: Optional[str] = None,
-        **ES_EXTRA_PARAMS: Dict[str, any],
+        **ES_EXTRA_PARAMS: dict[str, any],
     ):
         """
         Initializes a configuration class instance for an Elasticsearch client.
@@ -23,13 +23,13 @@ class ElasticsearchDBConfig(BaseVectorDbConfig):
         :param dir: Path to the database directory, where the database is stored, defaults to None
         :type dir: Optional[str], optional
         :param es_url: elasticsearch url or list of nodes url to be used for connection, defaults to None
-        :type es_url: Union[str, List[str]], optional
+        :type es_url: Union[str, list[str]], optional
         :param ES_EXTRA_PARAMS: extra params dict that can be passed to elasticsearch.
-        :type ES_EXTRA_PARAMS: Dict[str, Any], optional
+        :type ES_EXTRA_PARAMS: dict[str, Any], optional
         """
         if es_url and cloud_id:
             raise ValueError("Only one of `es_url` and `cloud_id` can be set.")
-        # self, es_url: Union[str, List[str]] = None, **ES_EXTRA_PARAMS: Dict[str, any]):
+        # self, es_url: Union[str, list[str]] = None, **ES_EXTRA_PARAMS: dict[str, any]):
         self.ES_URL = es_url or os.environ.get("ELASTICSEARCH_URL")
         self.CLOUD_ID = cloud_id or os.environ.get("ELASTICSEARCH_CLOUD_ID")
         if not self.ES_URL and not self.CLOUD_ID:

embedchain/config/vectordb/opensearch.py

@@ -1,4 +1,4 @@
-from typing import Dict, Optional, Tuple
+from typing import Optional
 
 from embedchain.config.vectordb.base import BaseVectorDbConfig
 from embedchain.helpers.json_serializable import register_deserializable
@@ -9,11 +9,11 @@ class OpenSearchDBConfig(BaseVectorDbConfig):
     def __init__(
         self,
         opensearch_url: str,
-        http_auth: Tuple[str, str],
+        http_auth: tuple[str, str],
         vector_dimension: int = 1536,
         collection_name: Optional[str] = None,
         dir: Optional[str] = None,
-        **extra_params: Dict[str, any],
+        **extra_params: dict[str, any],
     ):
         """
         Initializes a configuration class instance for an OpenSearch client.
@@ -23,7 +23,7 @@ class OpenSearchDBConfig(BaseVectorDbConfig):
         :param opensearch_url: URL of the OpenSearch domain
         :type opensearch_url: str, Eg, "http://localhost:9200"
         :param http_auth: Tuple of username and password
-        :type http_auth: Tuple[str, str], Eg, ("username", "password")
+        :type http_auth: tuple[str, str], Eg, ("username", "password")
         :param vector_dimension: Dimension of  the vector, defaults to 1536 (openai embedding model)
         :type vector_dimension: int, optional
         :param dir: Path to the database directory, where the database is stored, defaults to None

embedchain/config/vectordb/pinecone.py

@@ -1,4 +1,5 @@
-from typing import Dict, Optional
+import os
+from typing import Optional
 
 from embedchain.config.vectordb.base import BaseVectorDbConfig
 from embedchain.helpers.json_serializable import register_deserializable
@@ -8,13 +9,37 @@ from embedchain.helpers.json_serializable import register_deserializable
 class PineconeDBConfig(BaseVectorDbConfig):
     def __init__(
         self,
-        collection_name: Optional[str] = None,
-        dir: Optional[str] = None,
+        index_name: Optional[str] = None,
+        api_key: Optional[str] = None,
         vector_dimension: int = 1536,
         metric: Optional[str] = "cosine",
-        **extra_params: Dict[str, any],
+        pod_config: Optional[dict[str, any]] = None,
+        serverless_config: Optional[dict[str, any]] = None,
+        hybrid_search: bool = False,
+        bm25_encoder: any = None,
+        **extra_params: dict[str, any],
     ):
         self.metric = metric
+        self.api_key = api_key
+        self.index_name = index_name
         self.vector_dimension = vector_dimension
         self.extra_params = extra_params
-        super().__init__(collection_name=collection_name, dir=dir)
+        self.hybrid_search = hybrid_search
+        self.bm25_encoder = bm25_encoder
+        if pod_config is None and serverless_config is None:
+            # If no config is provided, use the default pod spec config
+            pod_environment = os.environ.get("PINECONE_ENV", "gcp-starter")
+            self.pod_config = {"environment": pod_environment, "metadata_config": {"indexed": ["*"]}}
+        else:
+            self.pod_config = pod_config
+        self.serverless_config = serverless_config
+
+        if self.pod_config and self.serverless_config:
+            raise ValueError("Only one of pod_config or serverless_config can be provided.")
+
+        if self.hybrid_search and self.metric != "dotproduct":
+            raise ValueError(
+                "Hybrid search is only supported with dotproduct metric in Pinecone. See full docs here: https://docs.pinecone.io/docs/hybrid-search#limitations"
+            )  # noqa:E501
+
+        super().__init__(collection_name=self.index_name, dir=None)

embedchain/config/vectordb/qdrant.py

@@ -1,4 +1,4 @@
-from typing import Dict, Optional
+from typing import Optional
 
 from embedchain.config.vectordb.base import BaseVectorDbConfig
 from embedchain.helpers.json_serializable import register_deserializable
@@ -7,18 +7,18 @@ from embedchain.helpers.json_serializable import register_deserializable
 @register_deserializable
 class QdrantDBConfig(BaseVectorDbConfig):
     """
-    Config to initialize an qdrant client.
-    :param url. qdrant url or list of nodes url to be used for connection
+    Config to initialize a qdrant client.
+    :param: url. qdrant url or list of nodes url to be used for connection
     """
 
     def __init__(
         self,
         collection_name: Optional[str] = None,
         dir: Optional[str] = None,
-        hnsw_config: Optional[Dict[str, any]] = None,
-        quantization_config: Optional[Dict[str, any]] = None,
+        hnsw_config: Optional[dict[str, any]] = None,
+        quantization_config: Optional[dict[str, any]] = None,
         on_disk: Optional[bool] = None,
-        **extra_params: Dict[str, any],
+        **extra_params: dict[str, any],
     ):
         """
         Initializes a configuration class instance for a qdrant client.
@@ -28,9 +28,9 @@ class QdrantDBConfig(BaseVectorDbConfig):
         :param dir: Path to the database directory, where the database is stored, defaults to None
         :type dir: Optional[str], optional
         :param hnsw_config: Params for HNSW index
-        :type hnsw_config: Optional[Dict[str, any]], defaults to None
+        :type hnsw_config: Optional[dict[str, any]], defaults to None
         :param quantization_config: Params for quantization, if None - quantization will be disabled
-        :type quantization_config: Optional[Dict[str, any]], defaults to None
+        :type quantization_config: Optional[dict[str, any]], defaults to None
         :param on_disk: If true - point`s payload will not be stored in memory.
                 It will be read from the disk every time it is requested.
                 This setting saves RAM by (slightly) increasing the response time.

embedchain/config/vectordb/weaviate.py

@@ -1,4 +1,4 @@
-from typing import Dict, Optional
+from typing import Optional
 
 from embedchain.config.vectordb.base import BaseVectorDbConfig
 from embedchain.helpers.json_serializable import register_deserializable
@@ -10,7 +10,7 @@ class WeaviateDBConfig(BaseVectorDbConfig):
         self,
         collection_name: Optional[str] = None,
         dir: Optional[str] = None,
-        **extra_params: Dict[str, any],
+        **extra_params: dict[str, any],
     ):
         self.extra_params = extra_params
         super().__init__(collection_name=collection_name, dir=dir)

embedchain/config/vectordb/zilliz.py

@@ -26,7 +26,7 @@ class ZillizDBConfig(BaseVectorDbConfig):
         :param uri: Cluster endpoint obtained from the Zilliz Console, defaults to None
         :type uri: Optional[str], optional
         :param token: API Key, if a Serverless Cluster, username:password, if a Dedicated Cluster, defaults to None
-        :type port: Optional[str], optional
+        :type token: Optional[str], optional
         """
         self.uri = uri or os.environ.get("ZILLIZ_CLOUD_URI")
         if not self.uri:

embedchain/constants.py

@@ -6,3 +6,6 @@ HOME_DIR = str(Path.home())
 CONFIG_DIR = os.path.join(HOME_DIR, ".embedchain")
 CONFIG_FILE = os.path.join(CONFIG_DIR, "config.json")
 SQLITE_PATH = os.path.join(CONFIG_DIR, "embedchain.db")
+
+# Set the environment variable for the database URI
+os.environ.setdefault("EMBEDCHAIN_DB_URI", f"sqlite:///{SQLITE_PATH}")

embedchain/core/db/database.py

@@ -0,0 +1,88 @@
+import os
+
+from alembic import command
+from alembic.config import Config
+from sqlalchemy import create_engine
+from sqlalchemy.engine.base import Engine
+from sqlalchemy.orm import Session as SQLAlchemySession
+from sqlalchemy.orm import scoped_session, sessionmaker
+
+from .models import Base
+
+
+class DatabaseManager:
+    def __init__(self, echo: bool = False):
+        self.database_uri = os.environ.get("EMBEDCHAIN_DB_URI")
+        self.echo = echo
+        self.engine: Engine = None
+        self._session_factory = None
+
+    def setup_engine(self) -> None:
+        """Initializes the database engine and session factory."""
+        if not self.database_uri:
+            raise RuntimeError("Database URI is not set. Set the EMBEDCHAIN_DB_URI environment variable.")
+        connect_args = {}
+        if self.database_uri.startswith("sqlite"):
+            connect_args["check_same_thread"] = False
+        self.engine = create_engine(self.database_uri, echo=self.echo, connect_args=connect_args)
+        self._session_factory = scoped_session(sessionmaker(bind=self.engine))
+        Base.metadata.bind = self.engine
+
+    def init_db(self) -> None:
+        """Creates all tables defined in the Base metadata."""
+        if not self.engine:
+            raise RuntimeError("Database engine is not initialized. Call setup_engine() first.")
+        Base.metadata.create_all(self.engine)
+
+    def get_session(self) -> SQLAlchemySession:
+        """Provides a session for database operations."""
+        if not self._session_factory:
+            raise RuntimeError("Session factory is not initialized. Call setup_engine() first.")
+        return self._session_factory()
+
+    def close_session(self) -> None:
+        """Closes the current session."""
+        if self._session_factory:
+            self._session_factory.remove()
+
+    def execute_transaction(self, transaction_block):
+        """Executes a block of code within a database transaction."""
+        session = self.get_session()
+        try:
+            transaction_block(session)
+            session.commit()
+        except Exception as e:
+            session.rollback()
+            raise e
+        finally:
+            self.close_session()
+
+
+# Singleton pattern to use throughout the application
+database_manager = DatabaseManager()
+
+
+# Convenience functions for backward compatibility and ease of use
+def setup_engine(database_uri: str, echo: bool = False) -> None:
+    database_manager.database_uri = database_uri
+    database_manager.echo = echo
+    database_manager.setup_engine()
+
+
+def alembic_upgrade() -> None:
+    """Upgrades the database to the latest version."""
+    alembic_config_path = os.path.join(os.path.dirname(__file__), "..", "..", "alembic.ini")
+    alembic_cfg = Config(alembic_config_path)
+    command.upgrade(alembic_cfg, "head")
+
+
+def init_db() -> None:
+    alembic_upgrade()
+
+
+def get_session() -> SQLAlchemySession:
+    return database_manager.get_session()
+
+
+def execute_transaction(transaction_block):
+    database_manager.execute_transaction(transaction_block)

embedchain/core/db/models.py

@@ -0,0 +1,31 @@
+import uuid
+
+from sqlalchemy import TIMESTAMP, Column, Integer, String, Text, func
+from sqlalchemy.orm import declarative_base
+
+Base = declarative_base()
+metadata = Base.metadata
+
+
+class DataSource(Base):
+    __tablename__ = "ec_data_sources"
+
+    id = Column(String, primary_key=True, default=lambda: str(uuid.uuid4()))
+    app_id = Column(Text, index=True)
+    hash = Column(Text, index=True)
+    type = Column(Text, index=True)
+    value = Column(Text)
+    meta_data = Column(Text, name="metadata")
+    is_uploaded = Column(Integer, default=0)
+
+
+class ChatHistory(Base):
+    __tablename__ = "ec_chat_history"
+
+    app_id = Column(String, primary_key=True)
+    id = Column(String, primary_key=True)
+    session_id = Column(String, primary_key=True, index=True)
+    question = Column(Text)
+    answer = Column(Text)
+    meta_data = Column(Text, name="metadata")
+    created_at = Column(TIMESTAMP, default=func.current_timestamp(), index=True)

embedchain/data_formatter/data_formatter.py

@@ -34,7 +34,8 @@ class DataFormatter(JSONSerializable):
         self.loader = self._get_loader(data_type=data_type, config=config.loader, loader=loader)
         self.chunker = self._get_chunker(data_type=data_type, config=config.chunker, chunker=chunker)
 
-    def _lazy_load(self, module_path: str):
+    @staticmethod
+    def _lazy_load(module_path: str):
         module_path, class_name = module_path.rsplit(".", 1)
         module = import_module(module_path)
         return getattr(module, class_name)
@@ -74,6 +75,7 @@ class DataFormatter(JSONSerializable):
             DataType.DISCORD: "embedchain.loaders.discord.DiscordLoader",
             DataType.RSSFEED: "embedchain.loaders.rss_feed.RSSFeedLoader",
             DataType.BEEHIIV: "embedchain.loaders.beehiiv.BeehiivLoader",
+            DataType.GOOGLE_DRIVE: "embedchain.loaders.google_drive.GoogleDriveLoader",
             DataType.DIRECTORY: "embedchain.loaders.directory_loader.DirectoryLoader",
             DataType.SLACK: "embedchain.loaders.slack.SlackLoader",
             DataType.DROPBOX: "embedchain.loaders.dropbox.DropboxLoader",
@@ -120,6 +122,7 @@ class DataFormatter(JSONSerializable):
             DataType.CUSTOM: "embedchain.chunkers.common_chunker.CommonChunker",
             DataType.RSSFEED: "embedchain.chunkers.rss_feed.RSSFeedChunker",
             DataType.BEEHIIV: "embedchain.chunkers.beehiiv.BeehiivChunker",
+            DataType.GOOGLE_DRIVE: "embedchain.chunkers.google_drive.GoogleDriveChunker",
             DataType.DIRECTORY: "embedchain.chunkers.common_chunker.CommonChunker",
             DataType.SLACK: "embedchain.chunkers.common_chunker.CommonChunker",
             DataType.DROPBOX: "embedchain.chunkers.common_chunker.CommonChunker",