Improve getting started page by adding steps (#904 )

Use either embedder or embedding_model as YAML key (#905 )
[Feature Improvement] Update JSON Loader to support loading data from more sources (#898 )
2023-11-03 13:40:40 -07:00 · 2023-11-03 13:40:16 -07:00 · 2023-11-03 10:00:27 -07:00 · 2023-11-03 09:40:48 -07:00 · 2023-11-03 01:08:27 -07:00 · 2023-11-03 00:42:55 -07:00
362 changed files with 44393 additions and 3198 deletions
@@ -1 +1 @@
-OPENAI_API_KEY=
+OPENAI_API_KEY="your-openai-api-key"
@@ -1,5 +1,8 @@
 blank_issues_enabled: true
 contact_links:
+  - name: 1-on-1 Session
+    url: https://cal.com/taranjeetio/ec
+    about: Speak directly with Taranjeet, the founder, to discuss issues, share feedback, or explore improvements for Embedchain
  - name: Discord
    url: https://discord.gg/6PzXDgEjG5
-    about: General community discussions
+    about: General community discussions
@@ -23,10 +23,10 @@ jobs:
        run: |
          curl -sSL https://install.python-poetry.org | python3 -
          echo "$HOME/.local/bin" >> $GITHUB_PATH
-  
+
      - name: Install dependencies
        run: poetry install
-  
+
      - name: Build a binary wheel and a source tarball
        run: poetry build

@@ -37,4 +37,4 @@ jobs:

      - name: Publish distribution 📦 to PyPI
        if: startsWith(github.ref, 'refs/tags')
-        uses: pypa/gh-action-pypi-publish@release/v1
+        uses: pypa/gh-action-pypi-publish@release/v1
@@ -19,10 +19,27 @@ jobs:
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install poetry
-        run: pip install poetry==1.4.2
+        uses: snok/install-poetry@v1
+        with:
+          version: 1.4.2
+          virtualenvs-create: true
+          virtualenvs-in-project: true
+      - name: Load cached venv
+        id: cached-poetry-dependencies
+        uses: actions/cache@v2
+        with:
+          path: .venv
+          key: venv-${{ runner.os }}-${{ hashFiles('**/poetry.lock') }}
      - name: Install dependencies
        run: poetry install --all-extras
+        if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
      - name: Lint with ruff
-        run: make ci_lint
-      - name: Test with pytest
-        run: make ci_test
+        run: make lint
+      - name: Run tests and generate coverage report
+        run: make coverage
+      - name: Upload coverage reports to Codecov
+        uses: codecov/codecov-action@v3
+        with:
+          file: coverage.xml
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
@@ -76,7 +76,6 @@ docs/_build/
 target/

 # Jupyter Notebook
-.ipynb_checkpoints

 # IPython
 profile_default/
@@ -165,9 +164,14 @@ cython_debug/

 # Database
 db
+test-db

 .vscode
-/poetry.lock
 .idea/

 .DS_Store
+
+notebooks/*.yaml
+.ipynb_checkpoints/
+
+!configs/*.yaml
@@ -1,10 +1,10 @@
 # Contributing to embedchain

-Let us make contributing easy, collaborative and fun.
+Let us make contribution easy, collaborative and fun.

 ## Submit your Contribution through PR

-To make a contribution, follow the following steps:
+To make a contribution, follow these steps:

 1. Fork and clone this repository
 2. Do the changes on your fork with dedicated feature branch `feature/f1`
@@ -35,7 +35,7 @@ poetry shell

 ### 📌 Pre-commit

-To ensure our standards, make sure to install pre-commit before star to contribute.
+To ensure our standards, make sure to install pre-commit before starting to contribute.

 ```bash
 pre-commit install
@@ -51,7 +51,7 @@ make lint

 Make sure that the linter does not report any errors or warnings before submitting a pull request.

-### Code Format with `black`
+### Code Formatting with `black`

 We use `black` to reformat the code by running the following command:

@@ -4,27 +4,41 @@ PIP := $(PYTHON) -m pip
 PROJECT_NAME := embedchain

 # Targets
-.PHONY: install format lint clean test ci_lint ci_test
+.PHONY: install format lint clean test ci_lint ci_test coverage

 install:
-	$(PIP) install --upgrade pip
-	$(PIP) install -e .[dev]
+	poetry install
+
+install_all:
+	poetry install --all-extras
+
+install_es:
+	poetry install --extras elasticsearch
+
+install_opensearch:
+	poetry install --extras opensearch
+
+install_milvus:
+	poetry install --extras milvus
+
+shell:
+	poetry shell
+
+py_shell:
+	poetry run python

 format:
 	$(PYTHON) -m black .
 	$(PYTHON) -m isort .

-lint:
-	$(PYTHON) -m ruff .
-
 clean:
 	rm -rf dist build *.egg-info

-test:
-	$(PYTHON) -m pytest
-
-ci_lint:
+lint:
 	poetry run ruff .

-ci_test:
-	poetry run pytest
+test:
+	poetry run pytest $(file)
+
+coverage:
+	poetry run pytest --cov=$(PROJECT_NAME) --cov-report=xml
@@ -1,24 +1,44 @@
 # embedchain

+<a href="https://runacap.com/ross-index/q3-2023/" target="_blank" rel="noopener"><img style="width: 260px; height: 56px" src="https://runacap.com/wp-content/uploads/2023/10/ROSS_badge_black_Q3_2023.svg" alt="ROSS Index - Fastest Growing Open-Source Startups in Q3 2023 | Runa Capital" width="260" height="56"/></a>
+
 [![PyPI](https://img.shields.io/pypi/v/embedchain)](https://pypi.org/project/embedchain/)
-[![Discord](https://dcbadge.vercel.app/api/server/6PzXDgEjG5?style=flat)](https://discord.gg/6PzXDgEjG5)
+[![Slack](https://img.shields.io/badge/slack-embedchain-brightgreen.svg?logo=slack)](https://join.slack.com/t/embedchain/shared_invite/zt-22uwz3c46-Zg7cIh5rOBteT_xe1jwLDw)
+[![Discord](https://dcbadge.vercel.app/api/server/6PzXDgEjG5?style=flat)](https://discord.gg/CUU9FPhRNt)
 [![Twitter](https://img.shields.io/twitter/follow/embedchain)](https://twitter.com/embedchain)
 [![Substack](https://img.shields.io/badge/Substack-%23006f5c.svg?logo=substack)](https://embedchain.substack.com/)
 [![Open in Colab](https://camo.githubusercontent.com/84f0493939e0c4de4e6dbe113251b4bfb5353e57134ffd9fcab6b8714514d4d1/68747470733a2f2f636f6c61622e72657365617263682e676f6f676c652e636f6d2f6173736574732f636f6c61622d62616467652e737667)](https://colab.research.google.com/drive/138lMWhENGeEu7Q1-6lNbNTHGLZXBBz_B?usp=sharing)
+[![codecov](https://codecov.io/gh/embedchain/embedchain/graph/badge.svg?token=EMRRHZXW1Q)](https://codecov.io/gh/embedchain/embedchain)

-Embedchain is a framework to easily create LLM powered bots over any dataset. If you want a javascript version, check out [embedchain-js](https://github.com/embedchain/embedchainjs)
+Embedchain is a Data Platform for LLMs - load, index, retrieve, and sync any unstructured data. Using embedchain, you can easily create LLM powered apps over any data. If you want a javascript version, check out [embedchain-js](https://github.com/embedchain/embedchain/tree/main/embedchain-js)
+
+## Community
+
+* Join embedchain community on slack by accepting [this invite](https://join.slack.com/t/embedchain/shared_invite/zt-22uwz3c46-Zg7cIh5rOBteT_xe1jwLDw)
+
+## 🤝 Schedule a 1-on-1 Session
+
+Book a [1-on-1 Session](https://cal.com/taranjeetio/ec) with Taranjeet, the founder, to discuss any issues, provide feedback, or explore how we can improve Embedchain for you.

 ## 🔧 Quick install

 ```bash
-pip install embedchain
+pip install --upgrade embedchain
 ```

+To run Embedchain as a REST API server run the following command:
+
+```bash
+docker run -d --name embedchain -p 8080:8080 embedchain/rest-api:latest
+```
+
+Navigate to http://0.0.0.0:8080/docs to interact with the API.
+
 ## 🔍 Demo

 Try out embedchain in your browser:

-[![Open in Colab](https://camo.githubusercontent.com/84f0493939e0c4de4e6dbe113251b4bfb5353e57134ffd9fcab6b8714514d4d1/68747470733a2f2f636f6c61622e72657365617263682e676f6f676c652e636f6d2f6173736574732f636f6c61622d62616467652e737667)](https://colab.research.google.com/drive/138lMWhENGeEu7Q1-6lNbNTHGLZXBBz_B?usp=sharing)
+[![Open in Colab](https://camo.githubusercontent.com/84f0493939e0c4de4e6dbe113251b4bfb5353e57134ffd9fcab6b8714514d4d1/68747470733a2f2f636f6c61622e72657365617263682e676f6f676c652e636f6d2f6173736574732f636f6c61622d62616467652e737667)](https://colab.research.google.com/drive/17ON1LPonnXAtLaZEebnOktstB_1cJJmh?usp=sharing)

 ## 📖 Documentation

@@ -26,17 +46,25 @@ The documentation for embedchain can be found at [docs.embedchain.ai](https://do

 ## 💻 Usage

-Embedchain empowers you to create chatbot models similar to ChatGPT, using your own evolving dataset.
+Embedchain empowers you to create ChatGPT like apps, on your own dynamic dataset.

-### Data Types Supported
+### Data types supported

 * Youtube video
 * PDF file
+* CSV file
 * Web page
+* MDX file
+* XML file
 * Sitemap
 * Doc file
-* Code documentation website loader
 * Notion
+* JSON file
+* OpenAPI specs
+* Code docs website
+* Unstructured file loader and many more
+
+You can find the full list of data types on [our documentation](https://docs.embedchain.ai/data-sources/).

 ### Queries

@@ -44,7 +72,7 @@ For example, you can use Embedchain to create an Elon Musk bot using the followi

 ```python
 import os
-from embedchain import App
+from embedchain import Pipeline as App

 # Create a bot instance
 os.environ["OPENAI_API_KEY"] = "YOUR API KEY"
@@ -52,21 +80,67 @@ elon_bot = App()

 # Embed online resources
 elon_bot.add("https://en.wikipedia.org/wiki/Elon_Musk")
-elon_bot.add("https://tesla.com/elon-musk")
-elon_bot.add("https://www.youtube.com/watch?v=MxZpaJK74Y4")
+elon_bot.add("https://www.forbes.com/profile/elon-musk")
+elon_bot.add("https://www.youtube.com/watch?v=RcYjXbSJBN8")

 # Query the bot
-elon_bot.query("How many companies does Elon Musk run?")
-# Answer: Elon Musk runs four companies: Tesla, SpaceX, Neuralink, and The Boring Company
+elon_bot.query("How many companies does Elon Musk run and name those?")
+# Answer: Elon Musk currently runs several companies. As of my knowledge, he is the CEO and lead designer of SpaceX, the CEO and product architect of Tesla, Inc., the CEO and founder of Neuralink, and the CEO and founder of The Boring Company. However, please note that this information may change over time, so it's always good to verify the latest updates.
+
+# (Optional): Deploy app to Embedchain Platform
+app.deploy()
+# 🔑 Enter your Embedchain API key. You can find the API key at https://app.embedchain.ai/settings/keys/
+# ec-xxxxxx
+
+# 🛠️ Creating pipeline on the platform...
+# 🎉🎉🎉 Pipeline created successfully! View your pipeline: https://app.embedchain.ai/pipelines/xxxxx
+
+# 🛠️ Adding data to your pipeline...
+# ✅ Data of type: web_page, value: https://www.forbes.com/profile/elon-musk added successfully.
 ```

+## Examples
+
+| LLM          | Google Colab  | Replit   |
+|--------------|---------------|----------|
+| OpenAI       | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/embedchain/embedchain/blob/main/notebooks/openai.ipynb)           | [![Try with Replit Badge](https://replit.com/badge?caption=Try%20with%20Replit&variant=small)](https://replit.com/@taranjeetio/openai#main.py)      |
+| Anthropic    | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/embedchain/embedchain/blob/main/notebooks/anthropic.ipynb)        | [![Try with Replit Badge](https://replit.com/badge?caption=Try%20with%20Replit&variant=small)](https://replit.com/@taranjeetio/anthropic#main.py)   |
+| Azure OpenAI | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/embedchain/embedchain/blob/main/notebooks/azure-openai.ipynb)     | [![Try with Replit Badge](https://replit.com/badge?caption=Try%20with%20Replit&variant=small)](https://replit.com/@taranjeetio/azureopenai#main.py) |
+| VertexAI     | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/embedchain/embedchain/blob/main/notebooks/vertex_ai.ipynb)        | [![Try with Replit Badge](https://replit.com/badge?caption=Try%20with%20Replit&variant=small)](https://replit.com/@taranjeetio/vertexai#main.py)    |
+| Cohere       | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/embedchain/embedchain/blob/main/notebooks/cohere.ipynb)           | [![Try with Replit Badge](https://replit.com/badge?caption=Try%20with%20Replit&variant=small)](https://replit.com/@taranjeetio/cohere#main.py)      |
+| Hugging Face | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/embedchain/embedchain/blob/main/notebooks/hugging_face_hub.ipynb) | [![Try with Replit Badge](https://replit.com/badge?caption=Try%20with%20Replit&variant=small)](https://replit.com/@taranjeetio/huggingface#main.py) |
+| JinaChat     | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/embedchain/embedchain/blob/main/notebooks/jina.ipynb)             | [![Try with Replit Badge](https://replit.com/badge?caption=Try%20with%20Replit&variant=small)](https://replit.com/@taranjeetio/jina#main.py)        |
+| GPT4All      | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/embedchain/embedchain/blob/main/notebooks/gpt4all.ipynb)          | [![Try with Replit Badge](https://replit.com/badge?caption=Try%20with%20Replit&variant=small)](https://replit.com/@taranjeetio/gpt4all#main.py)     |
+| Llama2       | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/embedchain/embedchain/blob/main/notebooks/llama2.ipynb)           | [![Try with Replit Badge](https://replit.com/badge?caption=Try%20with%20Replit&variant=small)](https://replit.com/@taranjeetio/llama2#main.py)      |
+
+| Embedding model     | Google Colab                                                                                                                                                                            | Replit                                                                                                                        |
+| ------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------- |
+| OpenAI       | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/embedchain/embedchain/blob/main/notebooks/openai.ipynb)           | [![Try with Replit Badge](https://replit.com/badge?caption=Try%20with%20Replit&variant=small)](https://replit.com/@taranjeetio/openai#main.py)      |
+| VertexAI     | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/embedchain/embedchain/blob/main/notebooks/vertex_ai.ipynb)        | [![Try with Replit Badge](https://replit.com/badge?caption=Try%20with%20Replit&variant=small)](https://replit.com/@taranjeetio/vertexai#main.py)    |
+| GPT4All      | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/embedchain/embedchain/blob/main/notebooks/gpt4all.ipynb)          | [![Try with Replit Badge](https://replit.com/badge?caption=Try%20with%20Replit&variant=small)](https://replit.com/@taranjeetio/gpt4all#main.py)     |
+| Hugging Face | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/embedchain/embedchain/blob/main/notebooks/hugging_face_hub.ipynb) | [![Try with Replit Badge](https://replit.com/badge?caption=Try%20with%20Replit&variant=small)](https://replit.com/@taranjeetio/huggingface#main.py) |
+
+| Vector DB     | Google Colab                                                                                                                                                                         | Replit                                                                                                                          |
+| ------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------- |
+| ChromaDB      | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/embedchain/embedchain/blob/main/notebooks/chromadb.ipynb)      | [![Try with Replit Badge](https://replit.com/badge?caption=Try%20with%20Replit&variant=small)](https://replit.com/@taranjeetio/chromadb#main.py)      |
+| Elasticsearch | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/embedchain/embedchain/blob/main/notebooks/elasticsearch.ipynb) | [![Try with Replit Badge](https://replit.com/badge?caption=Try%20with%20Replit&variant=small)](https://replit.com/@taranjeetio/elasticsearchdb#main.py) |
+| Opensearch    | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/embedchain/embedchain/blob/main/notebooks/opensearch.ipynb)    | [![Try with Replit Badge](https://replit.com/badge?caption=Try%20with%20Replit&variant=small)](https://replit.com/@taranjeetio/opensearchdb#main.py)    |
+| Pinecone      | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/embedchain/embedchain/blob/main/notebooks/pinecone.ipynb)      | [![Try with Replit Badge](https://replit.com/badge?caption=Try%20with%20Replit&variant=small)](https://replit.com/@taranjeetio/pineconedb#main.py)      |
+
 ## 🤝 Contributing

 Contributions are welcome! Please check out the issues on the repository, and feel free to open a pull request.
 For more information, please see the [contributing guidelines](CONTRIBUTING.md).

-For more refrence, please go through [Development Guide](https://docs.embedchain.ai/contribution/dev) and [Documentation Guide](https://docs.embedchain.ai/contribution/docs).
+For more reference, please go through [Development Guide](https://docs.embedchain.ai/contribution/dev) and [Documentation Guide](https://docs.embedchain.ai/contribution/docs).

+<a href="https://github.com/embedchain/embedchain/graphs/contributors">
+  <img src="https://contrib.rocks/image?repo=embedchain/embedchain" />
+</a>
+
+## Telemetry
+
+We collect anonymous usage metrics to enhance our package's quality and user experience. This includes data like feature usage frequency and system info, but never personal details. The data helps us prioritize improvements and ensure compatibility. If you wish to opt-out, set the `app.config.collect_metrics = False` in the code. We prioritize data security and don't share this data externally.

 ## Citation

@@ -74,8 +148,8 @@ If you utilize this repository, please consider citing it with:

 ```
@misc{embedchain,
-  author = {Taranjeet Singh},
-  title = {Embedchain: Framework to easily create LLM powered bots over any dataset},
+  author = {Taranjeet Singh, Deshraj Yadav},
+  title = {Embedchain: Data platform for LLMs - load, index, retrieve, and sync any unstructured data},
  year = {2023},
  publisher = {GitHub},
  journal = {GitHub repository},
@@ -0,0 +1,8 @@
+llm:
+  provider: anthropic
+  config:
+    model: 'claude-instant-1'
+    temperature: 0.5
+    max_tokens: 1000
+    top_p: 1
+    stream: false
@@ -0,0 +1,19 @@
+app:
+  config:
+    id: azure-openai-app
+
+llm:
+  provider: azure_openai
+  config:
+    model: gpt-35-turbo
+    deployment_name: your_llm_deployment_name
+    temperature: 0.5
+    max_tokens: 1000
+    top_p: 1
+    stream: false
+
+embedder:
+  provider: azure_openai
+  config:
+    model: text-embedding-ada-002
+    deployment_name: you_embedding_model_deployment_name
@@ -0,0 +1,26 @@
+app:
+  config:
+    id: 'my-app'
+    collection_name: 'my-app'
+
+llm:
+  provider: openai
+  config:
+    model: 'gpt-3.5-turbo'
+    temperature: 0.5
+    max_tokens: 1000
+    top_p: 1
+    stream: false
+
+vectordb:
+  provider: chroma
+  config:
+    collection_name: 'my-app'
+    dir: db
+    allow_reset: true
+
+embedder:
+  provider: openai
+  config:
+    model: 'text-embedding-ada-002'
+    deployment_name: null
@@ -0,0 +1,7 @@
+llm:
+  provider: cohere
+  config:
+    model: large
+    temperature: 0.5
+    max_tokens: 1000
+    top_p: 1
@@ -0,0 +1,35 @@
+app:
+  config:
+    id: 'full-stack-app'
+
+llm:
+  provider: openai
+  config:
+    model: 'gpt-3.5-turbo'
+    temperature: 0.5
+    max_tokens: 1000
+    top_p: 1
+    stream: false
+    template: |
+      Use the following pieces of context to answer the query at the end.
+      If you don't know the answer, just say that you don't know, don't try to make up an answer.
+
+      $context
+
+      Query: $query
+
+      Helpful Answer:
+    system_prompt: |
+      Act as William Shakespeare. Answer the following questions in the style of William Shakespeare.
+
+vectordb:
+  provider: chroma
+  config:
+    collection_name: 'my-collection-name'
+    dir: db
+    allow_reset: true
+
+embedder:
+  provider: openai
+  config:
+    model: 'text-embedding-ada-002'
@@ -0,0 +1,11 @@
+llm:
+  provider: gpt4all
+  config:
+    model: 'orca-mini-3b.ggmlv3.q4_0.bin'
+    temperature: 0.5
+    max_tokens: 1000
+    top_p: 1
+    stream: false
+
+embedder:
+  provider: gpt4all
@@ -0,0 +1,8 @@
+llm:
+  provider: huggingface
+  config:
+    model: 'google/flan-t5-xxl'
+    temperature: 0.5
+    max_tokens: 1000
+    top_p: 0.5
+    stream: false
@@ -0,0 +1,7 @@
+llm:
+  provider: jina
+  config:
+    temperature: 0.5
+    max_tokens: 1000
+    top_p: 1
+    stream: false
@@ -0,0 +1,8 @@
+llm:
+  provider: llama2
+  config:
+    model: 'a16z-infra/llama13b-v2-chat:df7690f1994d94e96ad9d568eac121aecf50684a0b0963b25a41cc40061269e5'
+    temperature: 0.5
+    max_tokens: 1000
+    top_p: 0.5
+    stream: false
@@ -0,0 +1,33 @@
+app:
+  config:
+    id: 'my-app'
+    log_level: 'WARN'
+    collect_metrics: true
+    collection_name: 'my-app'
+
+llm:
+  provider: openai
+  config:
+    model: 'gpt-3.5-turbo'
+    temperature: 0.5
+    max_tokens: 1000
+    top_p: 1
+    stream: false
+
+vectordb:
+  provider: opensearch
+  config:
+    opensearch_url: 'https://localhost:9200'
+    http_auth:
+      - admin
+      - admin
+    vector_dimension: 1536
+    collection_name: 'my-app'
+    use_ssl: false
+    verify_certs: false
+
+embedder:
+  provider: openai
+  config:
+    model: 'text-embedding-ada-002'
+    deployment_name: null
@@ -0,0 +1,26 @@
+app:
+  config:
+    id: 'open-source-app'
+    collection_name: 'open-source-app'
+    collect_metrics: false
+
+llm:
+  provider: gpt4all
+  config:
+    model: 'orca-mini-3b.ggmlv3.q4_0.bin'
+    temperature: 0.5
+    max_tokens: 1000
+    top_p: 1
+    stream: false
+
+vectordb:
+  provider: chroma
+  config:
+    collection_name: 'open-source-app'
+    dir: db
+    allow_reset: true
+
+embedder:
+  provider: gpt4all
+  config:
+    deployment_name: null
@@ -0,0 +1,6 @@
+vectordb:
+  provider: pinecone
+  config:
+    metric: cosine
+    vector_dimension: 1536
+    collection_name: my-pinecone-index
@@ -0,0 +1,26 @@
+pipeline:
+  config:
+    name: Example pipeline
+    id: pipeline-1  # Make sure that id is different every time you create a new pipeline
+
+vectordb:
+  provider: chroma
+  config:
+    collection_name: pipeline-1
+    dir: db
+    allow_reset: true
+
+llm:
+  provider: gpt4all
+  config:
+    model: 'orca-mini-3b.ggmlv3.q4_0.bin'
+    temperature: 0.5
+    max_tokens: 1000
+    top_p: 1
+    stream: false
+
+embedding_model:
+  provider: gpt4all
+  config:
+    model: 'all-MiniLM-L6-v2'
+    deployment_name: null
@@ -0,0 +1,6 @@
+llm:
+  provider: vertexai
+  config:
+    model: 'chat-bison'
+    temperature: 0.5
+    top_p: 0.5
@@ -0,0 +1,4 @@
+vectordb:
+  provider: weaviate
+  config:
+    collection_name: my_weaviate_index
@@ -0,0 +1,10 @@
+install:
+	npm i -g mintlify
+
+run_local:
+	mintlify dev
+
+troubleshoot:
+	mintlify install
+
+.PHONY: install run_local troubleshoot
@@ -0,0 +1,11 @@
+<CardGroup cols={3}>
+  <Card title="Slack" icon="slack" href="https://join.slack.com/t/embedchain/shared_invite/zt-22uwz3c46-Zg7cIh5rOBteT_xe1jwLDw" color="#4A154B">
+    Join our slack community
+  </Card>
+  <Card title="Discord" icon="discord" href="https://discord.gg/6PzXDgEjG5" color="#7289DA">
+    Join our discord community
+  </Card>
+  <Card title="Schedule a call" icon="calendar" href="https://cal.com/taranjeetio/ec">
+  Schedule a call with Embedchain founder
+  </Card>
+</CardGroup>
@@ -0,0 +1,18 @@
+<Tip>
+If you can't find the specific data source, please feel free to request through one of the following channels and help us prioritize.
+
+<CardGroup cols={2}>
+  <Card title="Slack" icon="slack" href="https://join.slack.com/t/embedchain/shared_invite/zt-22uwz3c46-Zg7cIh5rOBteT_xe1jwLDw" color="#4A154B">
+    Let us know on our slack community
+  </Card>
+  <Card title="Discord" icon="discord" href="https://discord.gg/6PzXDgEjG5" color="#7289DA">
+    Let us know on discord community
+  </Card>
+  <Card title="GitHub" icon="github" href="https://github.com/embedchain/embedchain/issues/new?assignees=&labels=&projects=&template=feature_request.yml" color="#181717">
+  Open an issue on our GitHub
+  </Card>
+  <Card title="Schedule a call" icon="calendar" href="https://cal.com/taranjeetio/ec">
+  Schedule a call with Embedchain founder
+  </Card>
+</CardGroup>
+</Tip>
@@ -0,0 +1,18 @@
+<Tip>
+If you can't find the specific LLM you need, no need to fret. We're continuously expanding our support for additional LLMs, and you can help us prioritize by opening an issue on our GitHub or simply reaching out to us on our Slack or Discord community.
+
+<CardGroup cols={2}>
+  <Card title="Slack" icon="slack" href="https://join.slack.com/t/embedchain/shared_invite/zt-22uwz3c46-Zg7cIh5rOBteT_xe1jwLDw" color="#4A154B">
+    Let us know on our slack community
+  </Card>
+  <Card title="Discord" icon="discord" href="https://discord.gg/6PzXDgEjG5" color="#7289DA">
+    Let us know on discord community
+  </Card>
+  <Card title="GitHub" icon="github" href="https://github.com/embedchain/embedchain/issues/new?assignees=&labels=&projects=&template=feature_request.yml" color="#181717">
+  Open an issue on our GitHub
+  </Card>
+  <Card title="Schedule a call" icon="calendar" href="https://cal.com/taranjeetio/ec">
+  Schedule a call with Embedchain founder
+  </Card>
+</CardGroup>
+</Tip>
@@ -0,0 +1,18 @@
+<Tip>
+If you can't find the specific vector database, please feel free to request through one of the following channels and help us prioritize.
+
+<CardGroup cols={2}>
+  <Card title="Slack" icon="slack" href="https://join.slack.com/t/embedchain/shared_invite/zt-22uwz3c46-Zg7cIh5rOBteT_xe1jwLDw" color="#4A154B">
+    Let us know on our slack community
+  </Card>
+  <Card title="Discord" icon="discord" href="https://discord.gg/6PzXDgEjG5" color="#7289DA">
+    Let us know on discord community
+  </Card>
+  <Card title="GitHub" icon="github" href="https://github.com/embedchain/embedchain/issues/new?assignees=&labels=&projects=&template=feature_request.yml" color="#181717">
+  Open an issue on our GitHub
+  </Card>
+  <Card title="Schedule a call" icon="calendar" href="https://cal.com/taranjeetio/ec">
+  Schedule a call with Embedchain founder
+  </Card>
+</CardGroup>
+</Tip>
@@ -1,25 +0,0 @@
---
-title: '➕ Adding Data'
---
-
-## Add Dataset
-
- This step assumes that you have already created an `app` instance by either using `App`, `OpenSourceApp` or `CustomApp`. We are calling our app instance as `naval_chat_bot` 🤖
-
- Now use `.add` method to add any dataset.
-
-```python
-# naval_chat_bot = App() or
-# naval_chat_bot = OpenSourceApp()
-
-# Embed Online Resources
-naval_chat_bot.add("https://www.youtube.com/watch?v=3qHkcs3kG44")
-naval_chat_bot.add("https://navalmanack.s3.amazonaws.com/Eric-Jorgenson_The-Almanack-of-Naval-Ravikant_Final.pdf")
-naval_chat_bot.add("https://nav.al/feedback")
-naval_chat_bot.add("https://nav.al/agi")
-
-# Embed Local Resources
-naval_chat_bot.add(("Who is Naval Ravikant?", "Naval Ravikant is an Indian-American entrepreneur and investor."))
-```
-
-The possible formats to add data can be found on the [Supported Data Formats](/advanced/data_types) page.
@@ -1,128 +0,0 @@
---
-title: '📱 App types'
---
-
-## App Types
-
-We have three types of App.
-
-### App
-
-```python
-from embedchain import App
-app = App()
-```
-
- `App` uses OpenAI's model, so these are paid models. 💸 You will be charged for embedding model usage and LLM usage.
- `App` uses OpenAI's embedding model to create embeddings for chunks and ChatGPT API as LLM to get answer given the relevant docs. Make sure that you have an OpenAI account and an API key. If you have don't have an API key, you can create one by visiting [this link](https://platform.openai.com/account/api-keys).
- `App` is opinionated. It uses the best embedding model and LLM on the market.
- Once you have the API key, set it in an environment variable called `OPENAI_API_KEY`
-
-```python
-import os
-os.environ["OPENAI_API_KEY"] = "sk-xxxx"
-```
-
-### Llama2App
-
-```python
-import os
-
-from embedchain import Llama2App
-
-os.environ['REPLICATE_API_TOKEN'] = "REPLICATE API TOKEN"
-
-zuck_bot = Llama2App()
-
-# Embed your data
-zuck_bot.add("https://www.youtube.com/watch?v=Ff4fRgnuFgQ")
-zuck_bot.add("https://en.wikipedia.org/wiki/Mark_Zuckerberg")
-
-# Nice, your bot is ready now. Start asking questions to your bot.
-zuck_bot.query("Who is Mark Zuckerberg?")
-# Answer: Mark Zuckerberg is an American internet entrepreneur and business magnate. He is the co-founder and CEO of Facebook. Born in 1984, he dropped out of Harvard University to focus on his social media platform, which has since grown to become one of the largest and most influential technology companies in the world.
-
-# Enable web search for your bot
-zuck_bot.online = True # enable internet access for the bot
-zuck_bot.query("Who owns the new threads app and when it was founded?")
-# Answer: Based on the context provided, the new Threads app is owned by Meta, the parent company of Facebook, Instagram, and WhatsApp.
-```
-
- `Llama2App` uses Replicate's LLM model, so these are paid models. You can get the `REPLICATE_API_TOKEN` by registering on [their website](https://replicate.com/account).
- `Llama2App` uses OpenAI's embedding model to create embeddings for chunks. Make sure that you have an OpenAI account and an API key. If you have don't have an API key, you can create one by visiting [this link](https://platform.openai.com/account/api-keys).
-
-
-### OpenSourceApp
-
-```python
-from embedchain import OpenSourceApp
-app = OpenSourceApp()
-```
-
- `OpenSourceApp` uses open source embedding and LLM model. It uses `all-MiniLM-L6-v2` from Sentence Transformers library as the embedding model and `gpt4all` as the LLM.
- Here there is no need to setup any api keys. You just need to install embedchain package and these will get automatically installed. 📦
- Once you have imported and instantiated the app, every functionality from here onwards is the same for either type of app. 📚
- `OpenSourceApp` is opinionated. It uses the best open source embedding model and LLM on the market.
- extra dependencies are required for this app type. Install them with `pip install embedchain[opensource]`.
-
-### CustomApp
-
-```python
-from embedchain import CustomApp
-from embedchain.config import CustomAppConfig
-from embedchain.models import Providers, EmbeddingFunctions
-
-config = CustomAppConfig(embedding_fn=EmbeddingFunctions.OPENAI, provider=Providers.OPENAI)
-app = CustomApp(config)
-```
-
- `CustomApp` is not opinionated.
- Configuration required. It's for advanced users who want to mix and match different embedding models and LLMs. Configuration required.
- while it's doing that, it's still providing abstractions through `Providers`.
- paid and free/open source providers included.
- Once you have imported and instantiated the app, every functionality from here onwards is the same for either type of app. 📚
- Following providers are available for an LLM
-    - OPENAI
-    - ANTHPROPIC
-    - VERTEX_AI
-    - GPT4ALL
-    - AZURE_OPENAI
- Following embedding functions are available for an embedding function
-    - OPENAI
-    - HUGGING_FACE
-    - VERTEX_AI
-    - GPT4ALL
-    - AZURE_OPENAI
-
-
-### PersonApp
-
-```python
-from embedchain import PersonApp
-naval_chat_bot = PersonApp("name_of_person_or_character") #Like "Yoda"
-```
-
- `PersonApp` uses OpenAI's model, so these are paid models. 💸 You will be charged for embedding model usage and LLM usage.
- `PersonApp` uses OpenAI's embedding model to create embeddings for chunks and ChatGPT API as LLM to get answer given the relevant docs. Make sure that you have an OpenAI account and an API key. If you have don't have an API key, you can create one by visiting [this link](https://platform.openai.com/account/api-keys).
- Once you have the API key, set it in an environment variable called `OPENAI_API_KEY`
-
-```python
-import os
-os.environ["OPENAI_API_KEY"] = "sk-xxxx"
-```
-
-#### Compatibility with other apps
-
- If there is any other app instance in your script or app, you can change the import as
-
-```python
-from embedchain import App as EmbedChainApp
-from embedchain import OpenSourceApp as EmbedChainOSApp
-from embedchain import PersonApp as EmbedChainPersonApp
-
-# or
-
-from embedchain import App as ECApp
-from embedchain import OpenSourceApp as ECOSApp
-from embedchain import PersonApp as ECPApp
-```
@@ -4,86 +4,72 @@ title: '⚙️ Custom configurations'

 Embedchain is made to work out of the box. However, for advanced users we're also offering configuration options. All of these configuration options are optional and have sane defaults.

-## Examples
+You can configure different components of your app (`llm`, `embedding model`, or `vector database`) through a simple yaml configuration that Embedchain offers. Here is a generic full-stack example of the yaml config:

-### General
+```yaml
+app:
+  config:
+    id: 'full-stack-app'

-Here's the readme example with configuration options.
+llm:
+  provider: openai
+  config:
+    model: 'gpt-3.5-turbo'
+    temperature: 0.5
+    max_tokens: 1000
+    top_p: 1
+    stream: false
+    template: |
+      Use the following pieces of context to answer the query at the end.
+      If you don't know the answer, just say that you don't know, don't try to make up an answer.

-```python
-import os
-from embedchain import App
-from embedchain.config import AppConfig, AddConfig, QueryConfig, ChunkerConfig
-from chromadb.utils import embedding_functions
+      $context

-# Example: set the log level for debugging
-config = AppConfig(log_level="DEBUG")
-naval_chat_bot = App(config)
+      Query: $query

-# Example: specify a custom collection name
-config = AppConfig(collection_name="naval_chat_bot")
-naval_chat_bot = App(config)
+      Helpful Answer:
+    system_prompt: |
+      Act as William Shakespeare. Answer the following questions in the style of William Shakespeare.

-# Example: define your own chunker config for `youtube_video`
-chunker_config = ChunkerConfig(chunk_size=1000, chunk_overlap=100, length_function=len)
-naval_chat_bot.add("https://www.youtube.com/watch?v=3qHkcs3kG44", AddConfig(chunker=chunker_config))
+vectordb:
+  provider: chroma
+  config:
+    collection_name: 'full-stack-app'
+    dir: db
+    allow_reset: true

-add_config = AddConfig()
-naval_chat_bot.add("https://navalmanack.s3.amazonaws.com/Eric-Jorgenson_The-Almanack-of-Naval-Ravikant_Final.pdf", config=add_config)
-naval_chat_bot.add("https://nav.al/feedback", config=add_config)
-naval_chat_bot.add("https://nav.al/agi", config=add_config)
-
-naval_chat_bot.add(("Who is Naval Ravikant?", "Naval Ravikant is an Indian-American entrepreneur and investor."), config=add_config)
-
-query_config = QueryConfig()
-print(naval_chat_bot.query("What unique capacity does Naval argue humans possess when it comes to understanding explanations or concepts?", config=query_config))
+embedder:
+  provider: openai
+  config:
+    model: 'text-embedding-ada-002'
 ```

-### Custom prompt template
+Alright, let's dive into what each key means in the yaml config above:

-Here's the example of using custom prompt template with `.query`
+1. `app` Section:
+    - `config`:
+        - `id` (String): The ID or name of your full-stack application.
+2. `llm` Section:
+    - `provider` (String): The provider for the language model, which is set to 'openai'. You can find the full list of llm providers in [our docs](/components/llms).
+    - `model` (String): The specific model being used, 'gpt-3.5-turbo'.
+    - `config`:
+        - `temperature` (Float): Controls the randomness of the model's output. A higher value (closer to 1) makes the output more random.
+        - `max_tokens` (Integer): Controls how many tokens are used in the response.
+        - `top_p` (Float): Controls the diversity of word selection. A higher value (closer to 1) makes word selection more diverse.
+        - `stream` (Boolean): Controls if the response is streamed back to the user (set to false).
+        - `template` (String): A custom template for the prompt that the model uses to generate responses.
+        - `system_prompt` (String): A system prompt for the model to follow when generating responses, in this case, it's set to the style of William Shakespeare.
+3. `vectordb` Section:
+    - `provider` (String): The provider for the vector database, set to 'chroma'. You can find the full list of vector database providers in [our docs](/components/vector-databases).
+    - `config`:
+        - `collection_name` (String): The initial collection name for the database, set to 'full-stack-app'.
+        - `dir` (String): The directory for the database, set to 'db'.
+        - `allow_reset` (Boolean): Indicates whether resetting the database is allowed, set to true.
+4. `embedder` Section:
+    - `provider` (String): The provider for the embedder, set to 'openai'. You can find the full list of embedding model providers in [our docs](/components/embedding-models).
+    - `config`:
+        - `model` (String): The specific model used for text embedding, 'text-embedding-ada-002'.

-```python
-from embedchain.config import QueryConfig
-from embedchain.embedchain import App
-from string import Template
-import wikipedia
+If you have questions about the configuration above, please feel free to reach out to us using one of the following methods:

-einstein_chat_bot = App()
-
-# Embed Wikipedia page
-page = wikipedia.page("Albert Einstein")
-einstein_chat_bot.add(page.content)
-
-# Example: use your own custom template with `$context` and `$query`
-einstein_chat_template = Template("""
-        You are Albert Einstein, a German-born theoretical physicist,
-        widely ranked among the greatest and most influential scientists of all time.
-
-        Use the following information about Albert Einstein to respond to
-        the human's query acting as Albert Einstein.
-        Context: $context
-
-        Keep the response brief. If you don't know the answer, just say that you don't know, don't try to make up an answer.
-
-        Human: $query
-        Albert Einstein:""")
-query_config = QueryConfig(template=einstein_chat_template, system_prompt="You are Albert Einstein.")
-queries = [
-        "Where did you complete your studies?",
-        "Why did you win nobel prize?",
-        "Why did you divorce your first wife?",
-]
-for query in queries:
-        response = einstein_chat_bot.query(query, config=query_config)
-        print("Query: ", query)
-        print("Response: ", response)
-
-# Output
-# Query:  Where did you complete your studies?
-# Response:  I completed my secondary education at the Argovian cantonal school in Aarau, Switzerland.
-# Query:  Why did you win nobel prize?
-# Response:  I won the Nobel Prize in Physics in 1921 for my services to Theoretical Physics, particularly for my discovery of the law of the photoelectric effect.
-# Query:  Why did you divorce your first wife?
-# Response:  We divorced due to living apart for five years.
-```
+<Snippet file="get-help.mdx" />
@@ -1,141 +0,0 @@
---
-title: '📋 Supported data formats'
---
-
-## Automatic data type detection
-The add method automatically tries to detect the data_type, based on your input for the source argument. So `app.add('https://www.youtube.com/watch?v=dQw4w9WgXcQ')` is enough to embed a YouTube video.
-
-This detection is implemented for all formats. It is based on factors such as whether it's a URL, a local file, the source data type, etc.
-
-### Debugging automatic detection
-
-
-Set `log_level=DEBUG` (in [AppConfig](http://localhost:3000/advanced/query_configuration#appconfig)) and make sure it's working as intended.
-
-Otherwise, you will not know when, for instance, an invalid filepath is interpreted as raw text instead.
-
-### Forcing a data type
-
-To omit any issues with the data type detection, you can **force** a data_type by adding it as a `add` method argument.
-The examples below show you the keyword to force the respective `data_type`.
-
-Forcing can also be used for edge cases, such as interpreting a sitemap as a web_page, for reading it's raw text instead of following links.
-
-## Remote Data Types
-
-<Tip>
-**Use local files in remote data types**
-
-Some data_types are meant for remote content and only work with URLs.
-You can pass local files by formatting the path using the `file:` [URI scheme](https://en.wikipedia.org/wiki/File_URI_scheme), e.g. `file:///info.pdf`.
-</Tip>
-
-### Youtube video
-
-To add any youtube video to your app, use the data_type (first argument to `.add()` method) as `youtube_video`. Eg:
-
-```python
-app.add('a_valid_youtube_url_here', data_type='youtube_video')
-```
-
-### PDF file
-
-To add any pdf file, use the data_type as `pdf_file`. Eg:
-
-```python
-app.add('a_valid_url_where_pdf_file_can_be_accessed', data_type='pdf_file')
-```
-
-Note that we do not support password protected pdfs.
-
-### Web page
-
-To add any web page, use the data_type as `web_page`. Eg:
-
-```python
-app.add('a_valid_web_page_url', data_type='web_page')
-```
-
-### Sitemap
-
-Add all web pages from an xml-sitemap. Filters non-text files. Use the data_type as `sitemap`. Eg:
-
-```python
-app.add('https://example.com/sitemap.xml', data_type='sitemap')
-```
-
-### Doc file
-
-To add any doc/docx file, use the data_type as `docx`. `docx` allows remote urls and conventional file paths. Eg:
-
-```python
-app.add('https://example.com/content/intro.docx', data_type="docx")
-app.add('content/intro.docx', data_type="docx")
-```
-
-### Code documentation website loader
-
-To add any code documentation website as a loader, use the data_type as `docs_site`. Eg:
-
-```python
-app.add("https://docs.embedchain.ai/", data_type="docs_site")
-```
-
-### Notion
-To use notion you must install the extra dependencies with `pip install embedchain[notion]`.
-
-To load a notion page, use the data_type as `notion`. Since it is hard to automatically detect, forcing this is advised.
-The next argument must **end** with the `notion page id`. The id is a 32-character string. Eg:
-
-```python
-app.add("cfbc134ca6464fc980d0391613959196", "notion")
-app.add("my-page-cfbc134ca6464fc980d0391613959196", "notion")
-app.add("https://www.notion.so/my-page-cfbc134ca6464fc980d0391613959196", "notion")
-```
-
-## Local Data Types
-
-### Text
-
-To supply your own text, use the data_type as `text` and enter a string. The text is not processed, this can be very versatile. Eg:
-
-```python
-app.add('Seek wealth, not money or status. Wealth is having assets that earn while you sleep. Money is how we transfer time and wealth. Status is your place in the social hierarchy.', data_type='text')
-```
-
-Note: This is not used in the examples because in most cases you will supply a whole paragraph or file, which did not fit.
-
-### QnA pair
-
-To supply your own QnA pair, use the data_type as `qna_pair` and enter a tuple. Eg:
-
-```python
-app.add(("Question", "Answer"), data_type="qna_pair")
-```
-
-## Reusing a vector database
-
-Default behavior is to create a persistent vector DB in the directory **./db**. You can split your application into two Python scripts: one to create a local vector DB and the other to reuse this local persistent vector DB. This is useful when you want to index hundreds of documents and separately implement a chat interface.
-
-Create a local index:
-
-```python
-from embedchain import App
-
-naval_chat_bot = App()
-naval_chat_bot.add("https://www.youtube.com/watch?v=3qHkcs3kG44")
-naval_chat_bot.add("https://navalmanack.s3.amazonaws.com/Eric-Jorgenson_The-Almanack-of-Naval-Ravikant_Final.pdf")
-```
-
-You can reuse the local index with the same code, but without adding new documents:
-
-```python
-from embedchain import App
-
-naval_chat_bot = App()
-print(naval_chat_bot.query("What unique capacity does Naval argue humans possess when it comes to understanding explanations or concepts?"))
-```
-
-## More formats (coming soon!)
-
- If you want to add any other format, please create an [issue](https://github.com/embedchain/embedchain/issues) and we will add it to the list of supported formats.
@@ -1,75 +0,0 @@
---
-title: '🤝 Interface types'
---
-
-## Interface Types
-
-The embedchain app exposes the following methods.
-
-### Query Interface
-
- This interface is like a question answering bot. It takes a question and gets the answer. It does not maintain context about the previous chats.❓
-
- To use this, call `.query()` function to get the answer for any query.
-
-```python
-print(naval_chat_bot.query("What unique capacity does Naval argue humans possess when it comes to understanding explanations or concepts?"))
-# answer: Naval argues that humans possess the unique capacity to understand explanations or concepts to the maximum extent possible in this physical reality.
-```
-
-### Chat Interface
-
- This interface is chat interface where it remembers previous conversation. Right now it remembers 5 conversation by default. 💬
-
- To use this, call `.chat` function to get the answer for any query.
-
-```python
-print(naval_chat_bot.chat("How to be happy in life?"))
-# answer: The most important trick to being happy is to realize happiness is a skill you develop and a choice you make. You choose to be happy, and then you work at it. It's just like building muscles or succeeding at your job. It's about recognizing the abundance and gifts around you at all times.
-
-print(naval_chat_bot.chat("who is naval ravikant?"))
-# answer: Naval Ravikant is an Indian-American entrepreneur and investor.
-
-print(naval_chat_bot.chat("what did the author say about happiness?"))
-# answer: The author, Naval Ravikant, believes that happiness is a choice you make and a skill you develop. He compares the mind to the body, stating that just as the body can be molded and changed, so can the mind. He emphasizes the importance of being present in the moment and not getting caught up in regrets of the past or worries about the future. By being present and grateful for where you are, you can experience true happiness.
-```
-
-#### Dry Run
-
-Dry Run is an option in the `query` and `chat` methods that allows the user to not send their constructed prompt to the LLM, to save money. It's used for [testing](/advanced/testing#dry-run).
-
-
-### Stream Response
-
- You can add config to your query method to stream responses like ChatGPT does. You would require a downstream handler to render the chunk in your desirable format. Supports both OpenAI model and OpenSourceApp. 📊
-
- To use this, instantiate a `QueryConfig` or `ChatConfig` object with `stream=True`. Then pass it to the `.chat()` or `.query()` method. The following example iterates through the chunks and prints them as they appear.
-
-```python
-app = App()
-query_config = QueryConfig(stream = True)
-resp = app.query("What unique capacity does Naval argue humans possess when it comes to understanding explanations or concepts?", query_config)
-
-for chunk in resp:
-    print(chunk, end="", flush=True)
-# answer: Naval argues that humans possess the unique capacity to understand explanations or concepts to the maximum extent possible in this physical reality.
-```
-
-### Other Methods
-
-#### Reset
-
-Resets the database and deletes all embeddings. Irreversible. Requires reinitialization afterwards.
-
-```python
-app.reset()
-```
-
-#### Count
-
-Counts the number of embeddings (chunks) in the database.
-
-```python
-print(app.count())
-# returns: 481
-```
@@ -1,77 +0,0 @@
---
-title: '🔍 Query configurations'
---
-
-## AppConfig
-
-| option    | description           | type                            | default                |
-|-----------|-----------------------|---------------------------------|------------------------|
-| log_level | log level             | string                          | WARNING                |
-| embedding_fn| embedding function    | chromadb.utils.embedding_functions | \{text-embedding-ada-002\} |
-| db        | vector database (experimental) | BaseVectorDB               | ChromaDB               |
-| collection_name | initial collection name for the database | string             | embedchain_store |
-| collect_metrics | collect anonymous telemetry data to improve embedchain | boolean | true |
-
-
-## AddConfig
-
-|option|description|type|default|
-|---|---|---|---|
-|chunker|chunker config|ChunkerConfig|Default values for chunker depends on the `data_type`. Please refer [ChunkerConfig](#chunker-config)|
-|loader|loader config|LoaderConfig|None|
-
-Yes, you are passing `ChunkerConfig` to `AddConfig`, like so:
-
-```python
-chunker_config = ChunkerConfig(chunk_size=100)
-add_config = AddConfig(chunker=chunker_config)
-app.add("lorem ipsum", config=add_config)
-```
-
-### ChunkerConfig
-
-|option|description|type|default|
-|---|---|---|---|
-|chunk_size|Maximum size of chunks to return|int|Default value for various `data_type` mentioned below|
-|chunk_overlap|Overlap in characters between chunks|int|Default value for various `data_type` mentioned below|
-|length_function|Function that measures the length of given chunks|typing.Callable|Default value for various `data_type` mentioned below|
-
-Default values of chunker config parameters for different `data_type`:
-
-|data_type|chunk_size|chunk_overlap|length_function|
-|---|---|---|---|
-|docx|1000|0|len|
-|text|300|0|len|
-|qna_pair|300|0|len|
-|web_page|500|0|len|
-|pdf_file|1000|0|len|
-|youtube_video|2000|0|len|
-|docs_site|500|50|len|
-|notion|300|0|len|
-
-### LoaderConfig
-
-_coming soon_
-
-## QueryConfig
-
-|option|description|type|default|
-|---|---|---|---|
-|number_documents|Absolute number of documents to pull from the database as context.|int|1
-|template|custom template for prompt. If history is used with query, $history has to be included as well.|Template|Template("Use the following pieces of context to answer the query at the end. If you don't know the answer, just say that you don't know, don't try to make up an answer. \$context Query: \$query Helpful Answer:")|
-|model|name of the model used.|string|depends on app type|
-|temperature|Controls the randomness of the model's output. Higher values (closer to 1) make output more random, lower values make it more deterministic.|float|0|
-|max_tokens|Controls how many tokens are used. Exact implementation (whether it counts prompt and/or response) depends on the model.|int|1000|
-|top_p|Controls the diversity of words. Higher values (closer to 1) make word selection more diverse, lower values make words less diverse.|float|1|
-|history|include conversation history from your client or database.|any (recommendation: list[str])|None|
-|stream|control if response is streamed back to the user.|bool|False|
-|deployment_name|t.b.a.|str|None|
-|system_prompt|System prompt string. Unused if none.|str|None|
-
-## ChatConfig
-
-All options for query and...
-
-_coming soon_
-
-`history` is not supported, as that is handled is handled automatically, the config option is not supported.
@@ -1,29 +0,0 @@
---
-title: '🧪 Testing'
---
-
-## Methods for testing
-
-### Dry Run
-
-Before you consume valueable tokens, you should make sure that the embedding you have done works and that it's receiving the correct document from the database.
-
-For this you can use the `dry_run` option in your `query` or `chat` method.
-
-Following the example above, add this to your script:
-
-```python
-print(naval_chat_bot.query('Can you tell me who Naval Ravikant is?', dry_run=True))
-
-'''
-Use the following pieces of context to answer the query at the end. If you don't know the answer, just say that you don't know, don't try to make up an answer.
-        Q: Who is Naval Ravikant?
-A: Naval Ravikant is an Indian-American entrepreneur and investor.
-        Query: Can you tell me who Naval Ravikant is?
-        Helpful Answer:
-'''
-```
-
-_The embedding is confirmed to work as expected. It returns the right document, even if the question is asked slightly different. No prompt tokens have been consumed._
-
-**The dry run will still consume tokens to embed your query, but it is only ~1/15 of the prompt.**
@@ -1,34 +0,0 @@
---
-title: '💾 Vector Database'
---
-
-We support `Chroma` and `Elasticsearch` as two vector database. 
-`Chroma` is used as a default database.
-
-### Elasticsearch
-In order to use `Elasticsearch` as vector database we need to use App type `CustomApp`. 
-```python
-import os
-from embedchain import CustomApp
-from embedchain.config import CustomAppConfig, ElasticsearchDBConfig
-from embedchain.models import Providers, EmbeddingFunctions, VectorDatabases
-
-os.environ["OPENAI_API_KEY"] = 'OPENAI_API_KEY'
-
-es_config = ElasticsearchDBConfig(
-	# elasticsearch url or list of nodes url with different hosts and ports.
-	es_url='http://localhost:9200',
-	# pass named parameters supported by Python Elasticsearch client
-	ca_certs="/path/to/http_ca.crt",
-	basic_auth=("username", "password")
-)
-config = CustomAppConfig(
-	embedding_fn=EmbeddingFunctions.OPENAI, 
-	provider=Providers.OPENAI, 
-	db_type=VectorDatabases.ELASTICSEARCH, 
-	es_config=es_config,
-)
-es_app = CustomApp(config)
-```
- Set `db_type=VectorDatabases.ELASTICSEARCH` and `es_config=ElasticsearchDBConfig(es_url='')` in `CustomAppConfig`.
- `ElasticsearchDBConfig` accepts `es_url` as elasticsearch url or as list of nodes url with different hosts and ports. Additionally we can pass named paramaters supported by Python Elasticsearch client.
@@ -0,0 +1,28 @@
+---
+title: 🤝 Connect with Us
+---
+
+We believe in building a vibrant and supportive community around embedchain. There are various channels through which you can connect with us, stay updated, and contribute to the ongoing discussions:
+
+<CardGroup cols={3}>
+  <Card title="Twitter" icon="twitter" href="https://twitter.com/embedchain">
+    Follow us on Twitter
+  </Card>
+  <Card title="Slack" icon="slack" href="https://join.slack.com/t/embedchain/shared_invite/zt-22uwz3c46-Zg7cIh5rOBteT_xe1jwLDw" color="#4A154B">
+    Join our slack community
+  </Card>
+  <Card title="Discord" icon="discord" href="https://discord.gg/6PzXDgEjG5" color="#7289DA">
+    Join our discord community
+  </Card>
+  <Card title="LinkedIn" icon="linkedin" href="https://www.linkedin.com/company/embedchain/">
+  Connect with us on LinkedIn
+  </Card>
+  <Card title="Schedule a call" icon="calendar" href="https://cal.com/taranjeetio/ec">
+  Schedule a call with Embedchain founder
+  </Card>
+  <Card title="Newsletter" icon="message" href="https://embedchain.substack.com/">
+  Subscribe to our newsletter
+  </Card>
+</CardGroup>
+
+We look forward to connecting with you and seeing how we can create amazing things together!
@@ -8,6 +8,7 @@ Embedchain community has been super active in creating demos on top of Embedchai

 ### Open Source

+- [My GSoC23 bot- Streamlit chat](https://github.com/lucifertrj/EmbedChain_GSoC23_BOT) by Tarun Jain
 - [Discord Bot for LLM chat](https://github.com/Reidond/discord_bots_playground/tree/c8b0c36541e4b393782ee506804c4b6962426dd6/python/chat-channel-bot) by Reidond
 - [EmbedChain-Streamlit-Docker App](https://github.com/amjadraza/embedchain-streamlit-app) by amjadraza
 - [Harry Potter Philosphers Stone Bot](https://github.com/vinayak-kempawad/Harry_Potter_Philosphers_Stone_Bot/) by Vinayak Kempawad, ([LinkedIn post](https://www.linkedin.com/feed/update/urn:li:activity:7080907532155686912/))
@@ -38,6 +39,9 @@ Embedchain community has been super active in creating demos on top of Embedchai
 - [Understanding what the LLM framework embedchain does](https://zenn.dev/hijikix/articles/4bc8d60156a436) by Daisuke Hashimoto
 - [In bed with GPT and Node.js](https://dev.to/worldlinetech/in-bed-with-gpt-and-nodejs-4kh2) by Raphaël Semeteys, ([LinkedIn Post](https://www.linkedin.com/posts/raphaelsemeteys_in-bed-with-gpt-and-nodejs-activity-7088113552326029313-nn87/))
 - [Using Embedchain — A powerful LangChain Python wrapper to build Chat Bots even faster!⚡](https://medium.com/@avra42/using-embedchain-a-powerful-langchain-python-wrapper-to-build-chat-bots-even-faster-35c12994a360) by Avra, ([Tweet](https://twitter.com/Avra_b/status/1686767751560310784/))
+- [What is the Embedchain library?](https://jahaniwww.com/%da%a9%d8%aa%d8%a7%d8%a8%d8%ae%d8%a7%d9%86%d9%87-embedchain/) by Ali Jahani, ([LinkedIn Post](https://www.linkedin.com/posts/ajahani_aepaetaeqaexaggahyaeu-aetaexaesabraeaaeqaepaeu-activity-7097605202135904256-ppU-/))
+- [LangChain is Nice, But Have You Tried EmbedChain ?](https://medium.com/thoughts-on-machine-learning/langchain-is-nice-but-have-you-tried-embedchain-215a34421cde) by FS Ndzomga, ([Tweet](https://twitter.com/ndzfs/status/1695583640372035951/))
+- [Simplest Method to Build a Custom Chatbot with GPT-3.5 (via Embedchain)](https://www.ainewsletter.today/p/simplest-method-to-build-a-custom) by Arjun, ([Tweet](https://twitter.com/aiguy_arjun/status/1696393808467091758/))

 ### LinkedIn

@@ -50,6 +54,7 @@ Embedchain community has been super active in creating demos on top of Embedchai
 - [About embedchain](https://www.linkedin.com/feed/update/urn:li:activity:7080984218914189312/) by Morris Lee
 - [How to use Embedchain](https://www.linkedin.com/posts/nehaabansal_github-embedchainembedchain-framework-activity-7085830340136595456-kbW5/) by Neha Bansal
 - [Youtube/Webpage summary for Energy Study](https://www.linkedin.com/posts/bar%C4%B1%C5%9F-sanl%C4%B1-34b82715_enerji-python-activity-7082735341563977730-Js0U/) by Barış Sanlı, ([Tweet](https://twitter.com/barissanli/status/1676968784979193857/)) 
+- [Demo: How to use Embedchain? (Contains Collab Notebook link)](https://www.linkedin.com/posts/liorsinclair_embedchain-is-getting-a-lot-of-traction-because-activity-7103044695995424768-RckT/) by Lior Sinclair

 ### Twitter

@@ -65,6 +70,7 @@ Embedchain community has been super active in creating demos on top of Embedchai

 ## Videos

+- [Embedchain in one shot](https://www.youtube.com/watch?v=vIhDh7H73Ww&t=82s) by AI with Tarun
 - [embedChain Create LLM powered bots over any dataset Python Demo Tesla Neurallink Chatbot Example](https://www.youtube.com/watch?v=bJqAn22a6Gc) by Rithesh Sreenivasan
 - [Embedchain - NEW 🔥 Langchain BABY to build LLM Bots](https://www.youtube.com/watch?v=qj_GNQ06I8o) by 1littlecoder
 - [EmbedChain -- NEW!: Build LLM-Powered Bots with Any Dataset](https://www.youtube.com/watch?v=XmaBezzGHu4) by DataInsightEdge
@@ -79,9 +85,31 @@ Embedchain community has been super active in creating demos on top of Embedchai
 - [AI ChatBot in 5 lines Python Code](https://www.youtube.com/watch?v=zjWvLJLksv8) by Data Engineering
 - [Interview with Karl Marx](https://www.youtube.com/watch?v=5Y4Tscwj1xk) by Alexander Ray Williams
 - [Vlog where we try to build a bot based on our content on the internet](https://www.youtube.com/watch?v=I2w8CWM3bx4) by DV, ([Tweet](https://twitter.com/dvcoolster/status/1688387017544261632))
+- [CHAT with ANY ONLINE RESOURCES using EMBEDCHAIN|STREAMLIT with MEMORY |All OPENSOURCE](https://www.youtube.com/watch?v=TqQIHWoWTDQ&pp=ygUKZW1iZWRjaGFpbg%3D%3D) by DataInsightEdge
+- [Build POWERFUL LLM Bots EASILY with Your Own Data - Embedchain - Langchain 2.0? (Tutorial)](https://www.youtube.com/watch?v=jE24Y_GasE8) by WorldofAI, ([Tweet](https://twitter.com/intheworldofai/status/1696229166922780737))
+- [Embedchain: An AI knowledge base assistant for customizing enterprise private data, which can be connected to discord, whatsapp, slack, tele and other terminals (with gradio to build a request interface) in Chinese](https://www.youtube.com/watch?v=5RZzCJRk-d0) by AIGC LINK
+- [Embedchain Introduction](https://www.youtube.com/watch?v=Jet9zAqyggI) by Fahd Mirza 

 ## Mentions

 ### Github repos

+- [Awesome-LLM](https://github.com/Hannibal046/Awesome-LLM)
+- [awesome-chatgpt-api](https://github.com/reorx/awesome-chatgpt-api)
+- [awesome-langchain](https://github.com/kyrolabs/awesome-langchain)
+- [Awesome-Prompt-Engineering](https://github.com/promptslab/Awesome-Prompt-Engineering)
+- [awesome-chatgpt](https://github.com/eon01/awesome-chatgpt)
+- [Awesome-LLMOps](https://github.com/tensorchord/Awesome-LLMOps)
+- [awesome-generative-ai](https://github.com/filipecalegario/awesome-generative-ai)
+- [awesome-gpt](https://github.com/formulahendry/awesome-gpt)
 - [awesome-ChatGPT-repositories](https://github.com/taishi-i/awesome-ChatGPT-repositories)
+- [awesome-gpt-prompt-engineering](https://github.com/snwfdhmp/awesome-gpt-prompt-engineering)
+- [awesome-chatgpt](https://github.com/awesome-chatgpt/awesome-chatgpt)
+- [awesome-llm-and-aigc](https://github.com/sjinzh/awesome-llm-and-aigc)
+- [awesome-compbio-chatgpt](https://github.com/csbl-br/awesome-compbio-chatgpt)
+- [Awesome-LLM4Tool](https://github.com/OpenGVLab/Awesome-LLM4Tool)
+
+## Meetups
+
+- [Dash and ChatGPT: Future of AI-enabled apps 30/08/23](https://go.plotly.com/dash-chatgpt)
+- [Pie & AI: Bangalore - Build end-to-end LLM app using Embedchain 01/09/23](https://www.eventbrite.com/e/pie-ai-bangalore-build-end-to-end-llm-app-using-embedchain-tickets-698045722547)
@@ -0,0 +1,173 @@
+---
+title: 🧩 Embedding models
+---
+
+## Overview
+
+Embedchain supports several embedding models from the following providers:
+
+<CardGroup cols={4}>
+  <Card title="OpenAI" href="#openai"></Card>
+  <Card title="Azure OpenAI" href="#azure-openai"></Card>
+  <Card title="GPT4All" href="#gpt4all"></Card>
+  <Card title="Hugging Face" href="#hugging-face"></Card>
+  <Card title="Vertex AI" href="#vertex-ai"></Card>
+</CardGroup>
+
+## OpenAI
+
+To use OpenAI embedding function, you have to set the `OPENAI_API_KEY` environment variable. You can obtain the OpenAI API key from the [OpenAI Platform](https://platform.openai.com/account/api-keys).
+
+Once you have obtained the key, you can use it like this:
+
+<CodeGroup>
+
+```python main.py
+import os
+from embedchain import Pipeline as App
+
+os.environ['OPENAI_API_KEY'] = 'xxx'
+
+# load embedding model configuration from config.yaml file
+app = App.from_config(yaml_path="config.yaml")
+
+app.add("https://en.wikipedia.org/wiki/OpenAI")
+app.query("What is OpenAI?")
+```
+
+```yaml config.yaml
+embedder:
+  provider: openai
+  config:
+    model: 'text-embedding-ada-002'
+```
+
+</CodeGroup>
+
+## Azure OpenAI
+
+To use Azure OpenAI embedding model, you have to set some of the azure openai related environment variables as given in the code block below:
+
+<CodeGroup>
+
+```python main.py
+import os
+from embedchain import Pipeline as App
+
+os.environ["OPENAI_API_TYPE"] = "azure"
+os.environ["OPENAI_API_BASE"] = "https://xxx.openai.azure.com/"
+os.environ["OPENAI_API_KEY"] = "xxx"
+os.environ["OPENAI_API_VERSION"] = "xxx"
+
+app = App.from_config(yaml_path="config.yaml")
+```
+
+```yaml config.yaml
+llm:
+  provider: azure_openai
+  config:
+    model: gpt-35-turbo
+    deployment_name: your_llm_deployment_name
+    temperature: 0.5
+    max_tokens: 1000
+    top_p: 1
+    stream: false
+
+embedder:
+  provider: azure_openai
+  config:
+    model: text-embedding-ada-002
+    deployment_name: you_embedding_model_deployment_name
+```
+</CodeGroup>
+
+You can find the list of models and deployment name on the [Azure OpenAI Platform](https://oai.azure.com/portal).
+
+## GPT4ALL
+
+GPT4All supports generating high quality embeddings of arbitrary length documents of text using a CPU optimized contrastively trained Sentence Transformer.
+
+<CodeGroup>
+
+```python main.py
+from embedchain import Pipeline as App
+
+# load embedding model configuration from config.yaml file
+app = App.from_config(yaml_path="config.yaml")
+```
+
+```yaml config.yaml
+llm:
+  provider: gpt4all
+  config:
+    model: 'orca-mini-3b.ggmlv3.q4_0.bin'
+    temperature: 0.5
+    max_tokens: 1000
+    top_p: 1
+    stream: false
+
+embedder:
+  provider: gpt4all
+```
+
+</CodeGroup>
+
+## Hugging Face
+
+Hugging Face supports generating embeddings of arbitrary length documents of text using Sentence Transformer library. Example of how to generate embeddings using hugging face is given below:
+
+<CodeGroup>
+
+```python main.py
+from embedchain import Pipeline as App
+
+# load embedding model configuration from config.yaml file
+app = App.from_config(yaml_path="config.yaml")
+```
+
+```yaml config.yaml
+llm:
+  provider: huggingface
+  config:
+    model: 'google/flan-t5-xxl'
+    temperature: 0.5
+    max_tokens: 1000
+    top_p: 0.5
+    stream: false
+
+embedder:
+  provider: huggingface
+  config:
+    model: 'sentence-transformers/all-mpnet-base-v2'
+```
+
+</CodeGroup>
+
+## Vertex AI
+
+Embedchain supports Google's VertexAI embeddings model through a simple interface. You just have to pass the `model_name` in the config yaml and it would work out of the box.
+
+<CodeGroup>
+
+```python main.py
+from embedchain import Pipeline as App
+
+# load embedding model configuration from config.yaml file
+app = App.from_config(yaml_path="config.yaml")
+```
+
+```yaml config.yaml
+llm:
+  provider: vertexai
+  config:
+    model: 'chat-bison'
+    temperature: 0.5
+    top_p: 0.5
+
+embedder:
+  provider: vertexai
+  config:
+    model: 'textembedding-gecko'
+```
+
+</CodeGroup>
@@ -0,0 +1,325 @@
+---
+title: 🤖 Large language models (LLMs)
+---
+
+## Overview
+
+Embedchain comes with built-in support for various popular large language models. We handle the complexity of integrating these models for you, allowing you to easily customize your language model interactions through a user-friendly interface.
+
+<CardGroup cols={4}>
+  <Card title="OpenAI" href="#openai"></Card>
+  <Card title="Azure OpenAI" href="#azure-openai"></Card>
+  <Card title="Anthropic" href="#anthropic"></Card>
+  <Card title="Cohere" href="#cohere"></Card>
+  <Card title="GPT4All" href="#gpt4all"></Card>
+  <Card title="JinaChat" href="#jinachat"></Card>
+  <Card title="Hugging Face" href="#hugging-face"></Card>
+  <Card title="Llama2" href="#llama2"></Card>
+  <Card title="Vertex AI" href="#vertex-ai"></Card>
+</CardGroup>
+
+## OpenAI
+
+To use OpenAI LLM models, you have to set the `OPENAI_API_KEY` environment variable. You can obtain the OpenAI API key from the [OpenAI Platform](https://platform.openai.com/account/api-keys).
+
+Once you have obtained the key, you can use it like this:
+
+```python
+import os
+from embedchain import Pipeline as App
+
+os.environ['OPENAI_API_KEY'] = 'xxx'
+
+app = App()
+app.add("https://en.wikipedia.org/wiki/OpenAI")
+app.query("What is OpenAI?")
+```
+
+If you are looking to configure the different parameters of the LLM, you can do so by loading the app using a [yaml config](https://github.com/embedchain/embedchain/blob/main/configs/chroma.yaml) file.
+
+<CodeGroup>
+
+```python main.py
+import os
+from embedchain import Pipeline as App
+
+os.environ['OPENAI_API_KEY'] = 'xxx'
+
+# load llm configuration from config.yaml file
+app = App.from_config(yaml_path="config.yaml")
+```
+
+```yaml config.yaml
+llm:
+  provider: openai
+  config:
+    model: 'gpt-3.5-turbo'
+    temperature: 0.5
+    max_tokens: 1000
+    top_p: 1
+    stream: false
+```
+
+</CodeGroup>
+
+
+## Azure OpenAI
+
+To use Azure OpenAI model, you have to set some of the azure openai related environment variables as given in the code block below:
+
+<CodeGroup>
+
+```python main.py
+import os
+from embedchain import Pipeline as App
+
+os.environ["OPENAI_API_TYPE"] = "azure"
+os.environ["OPENAI_API_BASE"] = "https://xxx.openai.azure.com/"
+os.environ["OPENAI_API_KEY"] = "xxx"
+os.environ["OPENAI_API_VERSION"] = "xxx"
+
+app = App.from_config(yaml_path="config.yaml")
+```
+
+```yaml config.yaml
+llm:
+  provider: azure_openai
+  config:
+    model: gpt-35-turbo
+    deployment_name: your_llm_deployment_name
+    temperature: 0.5
+    max_tokens: 1000
+    top_p: 1
+    stream: false
+
+embedder:
+  provider: azure_openai
+  config:
+    model: text-embedding-ada-002
+    deployment_name: you_embedding_model_deployment_name
+```
+</CodeGroup>
+
+You can find the list of models and deployment name on the [Azure OpenAI Platform](https://oai.azure.com/portal).
+
+## Anthropic
+
+To use anthropic's model, please set the `ANTHROPIC_API_KEY` which you find on their [Account Settings Page](https://console.anthropic.com/account/keys).
+
+<CodeGroup>
+
+```python main.py
+import os
+from embedchain import Pipeline as App
+
+os.environ["ANTHROPIC_API_KEY"] = "xxx"
+
+# load llm configuration from config.yaml file
+app = App.from_config(yaml_path="config.yaml")
+```
+
+```yaml config.yaml
+llm:
+  provider: anthropic
+  config:
+    model: 'claude-instant-1'
+    temperature: 0.5
+    max_tokens: 1000
+    top_p: 1
+    stream: false
+```
+
+</CodeGroup>
+
+## Cohere
+
+Install related dependencies using the following command:
+
+```bash
+pip install --upgrade 'embedchain[cohere]'
+```
+
+Set the `COHERE_API_KEY` as environment variable which you can find on their [Account settings page](https://dashboard.cohere.com/api-keys).
+
+Once you have the API key, you are all set to use it with Embedchain.
+
+<CodeGroup>
+
+```python main.py
+import os
+from embedchain import Pipeline as App
+
+os.environ["COHERE_API_KEY"] = "xxx"
+
+# load llm configuration from config.yaml file
+app = App.from_config(yaml_path="config.yaml")
+```
+
+```yaml config.yaml
+llm:
+  provider: cohere
+  config:
+    model: large
+    temperature: 0.5
+    max_tokens: 1000
+    top_p: 1
+```
+
+</CodeGroup>
+
+## GPT4ALL
+
+Install related dependencies using the following command:
+
+```bash
+pip install --upgrade 'embedchain[opensource]'
+```
+
+GPT4all is a free-to-use, locally running, privacy-aware chatbot. No GPU or internet required. You can use this with Embedchain using the following code:
+
+<CodeGroup>
+
+```python main.py
+from embedchain import Pipeline as App
+
+# load llm configuration from config.yaml file
+app = App.from_config(yaml_path="config.yaml")
+```
+
+```yaml config.yaml
+llm:
+  provider: gpt4all
+  config:
+    model: 'orca-mini-3b.ggmlv3.q4_0.bin'
+    temperature: 0.5
+    max_tokens: 1000
+    top_p: 1
+    stream: false
+
+embedder:
+  provider: gpt4all
+```
+</CodeGroup>
+
+
+## JinaChat
+
+First, set `JINACHAT_API_KEY` in environment variable which you can obtain from [their platform](https://chat.jina.ai/api).
+
+Once you have the key, load the app using the config yaml file:
+
+<CodeGroup>
+
+```python main.py
+import os
+from embedchain import Pipeline as App
+
+os.environ["JINACHAT_API_KEY"] = "xxx"
+# load llm configuration from config.yaml file
+app = App.from_config(yaml_path="config.yaml")
+```
+
+```yaml config.yaml
+llm:
+  provider: jina
+  config:
+    temperature: 0.5
+    max_tokens: 1000
+    top_p: 1
+    stream: false
+```
+</CodeGroup>
+
+
+## Hugging Face
+
+
+Install related dependencies using the following command:
+
+```bash
+pip install --upgrade 'embedchain[huggingface_hub]'
+```
+
+First, set `HUGGINGFACE_ACCESS_TOKEN` in environment variable which you can obtain from [their platform](https://huggingface.co/settings/tokens).
+
+Once you have the token, load the app using the config yaml file:
+
+<CodeGroup>
+
+```python main.py
+import os
+from embedchain import Pipeline as App
+
+os.environ["HUGGINGFACE_ACCESS_TOKEN"] = "xxx"
+
+# load llm configuration from config.yaml file
+app = App.from_config(yaml_path="config.yaml")
+```
+
+```yaml config.yaml
+llm:
+  provider: huggingface
+  config:
+    model: 'google/flan-t5-xxl'
+    temperature: 0.5
+    max_tokens: 1000
+    top_p: 0.5
+    stream: false
+```
+</CodeGroup>
+
+## Llama2
+
+Llama2 is integrated through [Replicate](https://replicate.com/).  Set `REPLICATE_API_TOKEN` in environment variable which you can obtain from [their platform](https://replicate.com/account/api-tokens).
+
+Once you have the token, load the app using the config yaml file:
+
+<CodeGroup>
+
+```python main.py
+import os
+from embedchain import Pipeline as App
+
+os.environ["REPLICATE_API_TOKEN"] = "xxx"
+
+# load llm configuration from config.yaml file
+app = App.from_config(yaml_path="config.yaml")
+```
+
+```yaml config.yaml
+llm:
+  provider: llama2
+  config:
+    model: 'a16z-infra/llama13b-v2-chat:df7690f1994d94e96ad9d568eac121aecf50684a0b0963b25a41cc40061269e5'
+    temperature: 0.5
+    max_tokens: 1000
+    top_p: 0.5
+    stream: false
+```
+</CodeGroup>
+
+## Vertex AI
+
+Setup Google Cloud Platform application credentials by following the instruction on [GCP](https://cloud.google.com/docs/authentication/external/set-up-adc). Once setup is done, use the following code to create an app using VertexAI as provider:
+
+<CodeGroup>
+
+```python main.py
+from embedchain import Pipeline as App
+
+# load llm configuration from config.yaml file
+app = App.from_config(yaml_path="config.yaml")
+```
+
+```yaml config.yaml
+llm:
+  provider: vertexai
+  config:
+    model: 'chat-bison'
+    temperature: 0.5
+    top_p: 0.5
+```
+</CodeGroup>
+
+<br/ >
+<Snippet file="missing-llm-tip.mdx" />
@@ -0,0 +1,224 @@
+---
+title: 🗄️ Vector databases
+---
+
+## Overview
+
+Utilizing a vector database alongside Embedchain is a seamless process. All you need to do is configure it within the YAML configuration file. We've provided examples for each supported database below:
+
+<CardGroup cols={4}>
+  <Card title="ChromaDB" href="#chromadb"></Card>
+  <Card title="Elasticsearch" href="#elasticsearch"></Card>
+  <Card title="OpenSearch" href="#opensearch"></Card>
+  <Card title="Zilliz" href="#zilliz"></Card>
+  <Card title="LanceDB" href="#lancedb"></Card>
+  <Card title="Pinecone" href="#pinecone"></Card>
+  <Card title="Qdrant" href="#qdrant"></Card>
+  <Card title="Weaviate" href="#weaviate"></Card>
+</CardGroup>
+
+## ChromaDB
+
+<CodeGroup>
+
+```python main.py
+from embedchain import Pipeline as App
+
+# load chroma configuration from yaml file
+app = App.from_config(yaml_path="config1.yaml")
+```
+
+```yaml config1.yaml
+vectordb:
+  provider: chroma
+  config:
+    collection_name: 'my-collection'
+    dir: db
+    allow_reset: true
+```
+
+```yaml config2.yaml
+vectordb:
+  provider: chroma
+  config:
+    collection_name: 'my-collection'
+    host: localhost
+    port: 5200
+    allow_reset: true
+```
+
+</CodeGroup>
+
+
+## Elasticsearch
+
+Install related dependencies using the following command:
+
+```bash
+pip install --upgrade 'embedchain[elasticsearch]'
+```
+
+<CodeGroup>
+
+```python main.py
+from embedchain import Pipeline as App
+
+# load elasticsearch configuration from yaml file
+app = App.from_config(yaml_path="config.yaml")
+```
+
+```yaml config.yaml
+vectordb:
+  provider: elasticsearch
+  config:
+    collection_name: 'es-index'
+    es_url: http://localhost:9200
+    allow_reset: true
+    api_key: xxx
+```
+</CodeGroup>
+
+## OpenSearch
+
+Install related dependencies using the following command:
+
+```bash
+pip install --upgrade 'embedchain[opensearch]'
+```
+
+<CodeGroup>
+
+```python main.py
+from embedchain import Pipeline as App
+
+# load opensearch configuration from yaml file
+app = App.from_config(yaml_path="config.yaml")
+```
+
+```yaml config.yaml
+vectordb:
+  provider: opensearch
+  config:
+    opensearch_url: 'https://localhost:9200'
+    http_auth:
+      - admin
+      - admin
+    vector_dimension: 1536
+    collection_name: 'my-app'
+    use_ssl: false
+    verify_certs: false
+```
+
+</CodeGroup>
+
+## Zilliz
+
+Install related dependencies using the following command:
+
+```bash
+pip install --upgrade 'embedchain[milvus]'
+```
+
+Set the Zilliz environment variables `ZILLIZ_CLOUD_URI` and `ZILLIZ_CLOUD_TOKEN` which you can find it on their [cloud platform](https://cloud.zilliz.com/).
+
+<CodeGroup>
+
+```python main.py
+import os
+from embedchain import Pipeline as App
+
+os.environ['ZILLIZ_CLOUD_URI'] = 'https://xxx.zillizcloud.com'
+os.environ['ZILLIZ_CLOUD_TOKEN'] = 'xxx'
+
+# load zilliz configuration from yaml file
+app = App.from_config(yaml_path="config.yaml")
+```
+
+```yaml config.yaml
+vectordb:
+  provider: zilliz
+  config:
+    collection_name: 'zilliz-app'
+    uri: https://xxxx.api.gcp-region.zillizcloud.com
+    token: xxx
+    vector_dim: 1536
+    metric_type: L2
+```
+
+</CodeGroup>
+
+## LanceDB
+
+_Coming soon_
+
+## Pinecone
+
+Install pinecone related dependencies using the following command:
+
+```bash
+pip install --upgrade 'embedchain[pinecone]'
+```
+
+In order to use Pinecone as vector database, set the environment variables `PINECONE_API_KEY` and `PINECONE_ENV` which you can find on [Pinecone dashboard](https://app.pinecone.io/).
+
+<CodeGroup>
+
+```python main.py
+from embedchain import Pipeline as App
+
+# load pinecone configuration from yaml file
+app = App.from_config(yaml_path="config.yaml")
+```
+
+```yaml config.yaml
+vectordb:
+  provider: pinecone
+  config:
+    metric: cosine
+    vector_dimension: 1536
+    collection_name: my-pinecone-index
+```
+
+</CodeGroup>
+
+## Qdrant
+
+In order to use Qdrant as a vector database, set the environment variables `QDRANT_URL` and `QDRANT_API_KEY` which you can find on [Qdrant Dashboard](https://cloud.qdrant.io/).
+
+<CodeGroup>
+```python main.py
+from embedchain import Pipeline as App
+
+# load qdrant configuration from yaml file
+app = App.from_config(yaml_path="config.yaml")
+```
+
+```yaml config.yaml
+vectordb:
+  provider: qdrant
+  config:
+    collection_name: my_qdrant_index
+```
+</CodeGroup>
+
+## Weaviate
+
+In order to use Weaviate as a vector database, set the environment variables `WEAVIATE_ENDPOINT` and `WEAVIATE_API_KEY` which you can find on [Weaviate dashboard](https://console.weaviate.cloud/dashboard).
+
+<CodeGroup>
+```python main.py
+from embedchain import Pipeline as App
+
+# load weaviate configuration from yaml file
+app = App.from_config(yaml_path="config.yaml")
+```
+
+```yaml config.yaml
+vectordb:
+  provider: weaviate
+  config:
+    collection_name: my_weaviate_index
+```
+</CodeGroup>
+
+<Snippet file="missing-vector-db-tip.mdx" />
@@ -35,14 +35,10 @@ embedchain is built on the following stack:

 ## Team

-### Author
+### Authors

 - Taranjeet Singh ([@taranjeetio](https://twitter.com/taranjeetio))
-
-### Maintainer
-
 - Deshraj Yadav ([@deshrajdry](https://twitter.com/taranjeetio))
- [cachho](https://github.com/cachho)

 ### Citation

@@ -50,8 +46,8 @@ If you utilize this repository, please consider citing it with:

 ```
@misc{embedchain,
-  author = {Taranjeet Singh},
-  title = {Embechain: Framework to easily create LLM powered bots over any dataset},
+  author = {Taranjeet Singh, Deshraj Yadav},
+  title = {Embechain: Data platform for LLMs - Load, index, retrieve and sync any unstructured data},
  year = {2023},
  publisher = {GitHub},
  journal = {GitHub repository},
@@ -0,0 +1,4 @@
+---
+title: '📋 Guidelines'
+url: https://github.com/embedchain/embedchain/blob/main/CONTRIBUTING.md
+---
@@ -0,0 +1,4 @@
+---
+title: ' 🟨 Javascript'
+url: https://github.com/embedchain/embedchain/tree/main/embedchain-js
+---
@@ -0,0 +1,4 @@
+---
+title: '🐍 Python'
+url: https://github.com/embedchain/embedchain
+---
@@ -0,0 +1,19 @@
+---
+title: '📊 CSV'
+---
+
+To add any csv file, use the data_type as `csv`. `csv` allows remote urls and conventional file paths. Headers are included for each line, so if you have an `age` column, `18` will be added as `age: 18`. Eg:
+
+```python
+from embedchain import Pipeline as App
+
+app = App()
+app.add('https://people.sc.fsu.edu/~jburkardt/data/csv/airtravel.csv', data_type="csv")
+# Or add using the local file path
+# app.add('/path/to/file.csv', data_type="csv")
+
+app.query("Summarize the air travel data")
+# Answer: The air travel data shows the number of flights for the months of July in the years 1958, 1959, and 1960. In July 1958, there were 491 flights, in July 1959 there were 548 flights, and in July 1960 there were 622 flights.
+```
+
+Note: There is a size limit allowed for csv file beyond which it can throw error. This limit is set by the LLMs. Please consider chunking large csv files into smaller csv files.
@@ -0,0 +1,52 @@
+---
+title: 'Data type handling'
+---
+
+## Automatic data type detection
+
+The add method automatically tries to detect the data_type, based on your input for the source argument. So `app.add('https://www.youtube.com/watch?v=dQw4w9WgXcQ')` is enough to embed a YouTube video.
+
+This detection is implemented for all formats. It is based on factors such as whether it's a URL, a local file, the source data type, etc.
+
+### Debugging automatic detection
+
+Set `log_level: DEBUG` in the config yaml to debug if the data type detection is done right or not. Otherwise, you will not know when, for instance, an invalid filepath is interpreted as raw text instead.
+
+### Forcing a data type
+
+To omit any issues with the data type detection, you can **force** a data_type by adding it as a `add` method argument.
+The examples below show you the keyword to force the respective `data_type`.
+
+Forcing can also be used for edge cases, such as interpreting a sitemap as a web_page, for reading its raw text instead of following links.
+
+## Remote data types
+
+<Tip>
+**Use local files in remote data types**
+
+Some data_types are meant for remote content and only work with URLs.
+You can pass local files by formatting the path using the `file:` [URI scheme](https://en.wikipedia.org/wiki/File_URI_scheme), e.g. `file:///info.pdf`.
+</Tip>
+
+## Reusing a vector database
+
+Default behavior is to create a persistent vector db in the directory **./db**. You can split your application into two Python scripts: one to create a local vector db and the other to reuse this local persistent vector db. This is useful when you want to index hundreds of documents and separately implement a chat interface.
+
+Create a local index:
+
+```python
+from embedchain import Pipeline as App
+
+naval_chat_bot = App()
+naval_chat_bot.add("https://www.youtube.com/watch?v=3qHkcs3kG44")
+naval_chat_bot.add("https://navalmanack.s3.amazonaws.com/Eric-Jorgenson_The-Almanack-of-Naval-Ravikant_Final.pdf")
+```
+
+You can reuse the local index with the same code, but without adding new documents:
+
+```python
+from embedchain import Pipeline as App
+
+naval_chat_bot = App()
+print(naval_chat_bot.query("What unique capacity does Naval argue humans possess when it comes to understanding explanations or concepts?"))
+```
@@ -0,0 +1,14 @@
+---
+title: '📚🌐 Code documentation'
+---
+
+To add any code documentation website as a loader, use the data_type as `docs_site`. Eg:
+
+```python
+from embedchain import Pipeline as App
+
+app = App()
+app.add("https://docs.embedchain.ai/", data_type="docs_site")
+app.query("What is Embedchain?")
+# Answer: Embedchain is a platform that utilizes various components, including paid/proprietary ones, to provide what is believed to be the best configuration available. It uses LLM (Language Model) providers such as OpenAI, Anthpropic, Vertex_AI, GPT4ALL, Azure_OpenAI, LLAMA2, JINA, and COHERE. Embedchain allows users to import and utilize these LLM providers for their applications.'
+```
@@ -0,0 +1,18 @@
+---
+title: '📄 Docx file'
+---
+
+### Docx file
+
+To add any doc/docx file, use the data_type as `docx`. `docx` allows remote urls and conventional file paths. Eg:
+
+```python
+from embedchain import Pipeline as App
+
+app = App()
+app.add('https://example.com/content/intro.docx', data_type="docx")
+# Or add file using the local file path on your system
+# app.add('content/intro.docx', data_type="docx")
+
+app.query("Summarize the docx data?")
+```
@@ -0,0 +1,35 @@
+---
+title: '📬 Gmail'
+---
+
+To use GmailLoader you must install the extra dependencies with `pip install --upgrade embedchain[gmail]`.
+
+The `source` must be a valid Gmail search query, you can refer `https://support.google.com/mail/answer/7190?hl=en` to build a query.
+
+To load Gmail messages, you MUST use the data_type as `gmail`. Otherwise the source will be detected as simple `text`.
+
+To use this you need to save `credentials.json` in the directory from where you will run the loader. Follow these steps to get the credentials
+
+1. Go to the [Google Cloud Console](https://console.cloud.google.com/apis/credentials).
+2. Create a project if you don't have one already.
+3. Create an `OAuth Consent Screen` in the project. You may need to select the `external` option.
+4. Make sure the consent screen is published.
+5. Enable the [Gmail API](https://console.cloud.google.com/apis/api/gmail.googleapis.com)
+6. Create credentials from the `Credentials` tab.
+7. Select the type `OAuth Client ID`.
+8. Choose the application type `Web application`. As a name you can choose `embedchain` or any other name as per your use case.
+9. Add an authorized redirect URI for `http://localhost:8080/`.
+10. You can leave everything else at default, finish the creation.
+11. When you are done, a modal opens where you can download the details in `json` format.
+12. Put the `.json` file in your current directory and rename it to `credentials.json`
+
+```python
+import os
+from embedchain.apps.app import App
+from embedchain.models.data_type import DataType
+app = App()
+
+query = "to: me label:inbox"
+app.add(query, data_type=DataType.GMAIL)
+app.query("Summarize my email conversations")
+```
@@ -0,0 +1,53 @@
+---
+title: '📃 JSON'
+---
+
+To add any json file, use the data_type as `json`. Headers are included for each line, so if you have an `age` column, `18` will be added as `age: 18`. Eg:
+
+Here are the supported sources for loading `json`:
+```
+1. URL - valid url to json file that ends with ".json" extension.
+2. Local file - valid url to local json file that ends with ".json" extension.
+3. String - valid json string (e.g. - app.add('{"foo": "bar"}'))
+```
+
+If you would like to add other data structures (e.x. list, dict etc.), do:
+```
+    import json
+    a = {"foo": "bar"}
+    valid_json_string_data = json.dumps(a, indent=0)
+
+    b = [{"foo": "bar"}]
+    valid_json_string_data = json.dumps(b, indent=0)
+```
+Example:
+```
+import os
+
+from embedchain.apps.app import App
+
+os.environ["OPENAI_API_KEY"] = "openai_api_key"
+
+app = App()
+
+response = app.query("What is the net worth of Elon Musk as of October 2023?")
+
+print(response)
+"I'm sorry, but I don't have access to real-time information or future predictions. Therefore, I don't know the net worth of Elon Musk as of October 2023."
+
+source_id = app.add("temp.json")
+
+response = app.query("What is the net worth of Elon Musk as of October 2023?")
+
+print(response)
+"As of October 2023, Elon Musk's net worth is $255.2 billion."
+```
+temp.json
+```
+{
+    "question": "What is your net worth, Elon Musk?",
+    "answer": "As of October 2023, Elon Musk's net worth is $255.2 billion, making him one of the wealthiest individuals in the world."
+}
+```
+
+
@@ -0,0 +1,14 @@
+---
+title: '📝 Mdx file'
+---
+
+To add any `.mdx` file to your app, use the data_type (first argument to `.add()` method) as `mdx`. Note that this supports support mdx file present on machine, so this should be a file path. Eg:
+
+```python
+from embedchain import Pipeline as App
+
+app = App()
+app.add('path/to/file.mdx', data_type='mdx')
+
+app.query("What are the docs about?")
+```
@@ -0,0 +1,20 @@
+---
+title: '📓 Notion'
+---
+
+To use notion you must install the extra dependencies with `pip install --upgrade embedchain[community]`.
+
+To load a notion page, use the data_type as `notion`. Since it is hard to automatically detect, it is advised to specify the `data_type` when adding a notion document.
+The next argument must **end** with the `notion page id`. The id is a 32-character string. Eg:
+
+```python
+from embedchain import Pipeline as App
+
+app = App()
+
+app.add("cfbc134ca6464fc980d0391613959196", data_type="notion")
+app.add("my-page-cfbc134ca6464fc980d0391613959196", data_type="notion")
+app.add("https://www.notion.so/my-page-cfbc134ca6464fc980d0391613959196", data_type="notion")
+
+app.query("Summarize the notion doc")
+```
@@ -0,0 +1,23 @@
+---
+title: 🙌 OpenAPI
+---
+
+To add any OpenAPI spec yaml file (currently the json file will be detected as JSON data type), use the data_type as 'openapi'. 'openapi' allows remote urls and conventional file paths. Headers are included for each line, so if you have an `age` column, `18` will be added as `age: 18`. Eg:
+
+```python
+from embedchain.apps.app import App
+import os
+
+os.environ["OPENAI_API_KEY"] = "sk-xxx"
+
+app = App()
+
+app.add("https://github.com/openai/openai-openapi/blob/master/openapi.yaml", data_type="openapi")
+# Or add using the local file path
+# app.add("configs/openai_openapi.yaml", data_type="openapi")
+
+response = app.query("What can OpenAI API endpoint do? Can you list the things it can learn from?")
+# Answer: The OpenAI API endpoint allows users to interact with OpenAI's models and perform various tasks such as generating text, answering questions, summarizing documents, translating languages, and more. The specific capabilities and tasks that the API can learn from may vary depending on the models and features provided by OpenAI. For more detailed information, it is recommended to refer to the OpenAI API documentation at https://platform.openai.com/docs/api-reference.
+```
+
+NOTE: The yaml file added to the App must have the required OpenAPI fields otherwise the adding OpenAPI spec will fail. Please refer to [OpenAPI Spec Doc](https://spec.openapis.org/oas/v3.1.0)
@@ -0,0 +1,27 @@
+---
+title: Overview
+---
+
+Embedchain comes with built-in support for various data sources. We handle the complexity of loading unstructured data from these data sources, allowing you to easily customize your app through a user-friendly interface.
+
+<CardGroup cols={4}>
+  <Card title="📊 csv" href="/data-sources/csv"></Card>
+  <Card title="📃 JSON" href="/data-sources/json"></Card>
+  <Card title="📚🌐 docs site" href="/data-sources/docs-site"></Card>
+  <Card title="📄 docx" href="/data-sources/docx"></Card>
+  <Card title="📝 mdx" href="/data-sources/mdx"></Card>
+  <Card title="📓 notion" href="/data-sources/notion"></Card>
+  <Card title="📰 pdf" href="/data-sources/pdf-file"></Card>
+  <Card title="❓💬 q&a pair" href="/data-sources/qna"></Card>
+  <Card title="🗺️ sitemap" href="/data-sources/sitemap"></Card>
+  <Card title="📝 text" href="/data-sources/text"></Card>
+  <Card title="🌐📄 web page" href="/data-sources/web-page"></Card>
+  <Card title="🧾 xml" href="/data-sources/xml"></Card>
+  <Card title="🙌 OpenAPI" href="/data-sources/openapi"></Card>
+  <Card title="🎥📺 youtube video" href="/data-sources/youtube-video"></Card>
+  <Card title="📬 Gmail" href="/data-sources/gmail"></Card>
+</CardGroup>
+
+<br/ >
+
+<Snippet file="missing-data-source-tip.mdx" />
@@ -0,0 +1,17 @@
+---
+title: '📰 PDF file'
+---
+
+To add any pdf file, use the data_type as `pdf_file`. Eg:
+
+```python
+from embedchain import Pipeline as App
+
+app = App()
+
+app.add('https://arxiv.org/pdf/1706.03762.pdf', data_type='pdf_file')
+app.query("What is the paper 'attention is all you need' about?")
+# Answer: The paper "Attention Is All You Need" proposes a new network architecture called the Transformer, which is based solely on attention mechanisms. It suggests moving away from complex recurrent or convolutional neural networks and instead using attention mechanisms to connect the encoder and decoder in sequence transduction models.
+```
+
+Note that we do not support password protected pdfs.
@@ -0,0 +1,13 @@
+---
+title: '❓💬 Queston and answer pair'
+---
+
+QnA pair is a local data type. To supply your own QnA pair, use the data_type as `qna_pair` and enter a tuple. Eg:
+
+```python
+from embedchain import Pipeline as App
+
+app = App()
+
+app.add(("Question", "Answer"), data_type="qna_pair")
+```
@@ -0,0 +1,13 @@
+---
+title: '🗺️ Sitemap'
+---
+
+Add all web pages from an xml-sitemap. Filters non-text files. Use the data_type as `sitemap`. Eg:
+
+```python
+from embedchain import Pipeline as App
+
+app = App()
+
+app.add('https://example.com/sitemap.xml', data_type='sitemap')
+```
@@ -0,0 +1,17 @@
+---
+title: '📝 Text'
+---
+
+### Text
+
+Text is a local data type. To supply your own text, use the data_type as `text` and enter a string. The text is not processed, this can be very versatile. Eg:
+
+```python
+from embedchain import Pipeline as App
+
+app = App()
+
+app.add('Seek wealth, not money or status. Wealth is having assets that earn while you sleep. Money is how we transfer time and wealth. Status is your place in the social hierarchy.', data_type='text')
+```
+
+Note: This is not used in the examples because in most cases you will supply a whole paragraph or file, which did not fit.
@@ -0,0 +1,13 @@
+---
+title: '🌐📄 Web page'
+---
+
+To add any web page, use the data_type as `web_page`. Eg:
+
+```python
+from embedchain import Pipeline as App
+
+app = App()
+
+app.add('a_valid_web_page_url', data_type='web_page')
+```
@@ -0,0 +1,17 @@
+---
+title: '🧾 XML file'
+---
+
+### XML file
+
+To add any xml file, use the data_type as `xml`. Eg:
+
+```python
+from embedchain import Pipeline as App
+
+app = App()
+
+app.add('content/data.xml')
+```
+
+Note: Only the text content of the xml file will be added to the app. The tags will be ignored.
@@ -0,0 +1,13 @@
+---
+title: '🎥📺 Youtube video'
+---
+
+
+To add any youtube video to your app, use the data_type (first argument to `.add()` method) as `youtube_video`. Eg:
+
+```python
+from embedchain import Pipeline as App
+
+app = App()
+app.add('a_valid_youtube_url_here', data_type='youtube_video')
+```
@@ -1,91 +0,0 @@
---
-title: '🌍 API Server'
---
-
-The API Server based on Flask integrates the `embedchain` package, offering endpoints to add, query, and chat to engage in conversations with a chatbot using JSON requests.
-
-### 🐳 Docker Setup
-
- Open variables.env, and edit it to add your 🔑 `OPENAI_API_KEY`.
- To setup your api server using docker, run the following command inside this folder using your terminal.
-
-```bash
-docker-compose up --build
-```
-
-📝 Note: The build command might take a while to install all the packages depending on your system resources.
-
-### 🚀 Usage Instructions
-
- Your api server is running on [http://localhost:5000/](http://localhost:5000/)
- To use the api server, make an api call to the endpoints `/add`, `/query` and `/chat` using the json formats discussed below.
- To add data sources to the bot (/add):
-```json
-// Request
-{
-  "data_type": "your_data_type_here",
-  "url_or_text": "your_url_or_text_here"
-}
-
-// Response
-{
-  "data": "Added data_type: url_or_text"
-}
-```
- To ask queries from the bot (/query):
-```json
-// Request
-{
-  "question": "your_question_here"
-}
-
-// Response
-{
-  "data": "your_answer_here"
-}
-```
- To chat with the bot (/chat):
-```json
-// Request
-{
-  "question": "your_question_here"
-}
-
-// Response
-{
-  "data": "your_answer_here"
-}
-```
-
-### 📡 Curl Call Formats
-
- To add data sources to the bot (/add):
-```bash
-curl -X POST \
-     -H "Content-Type: application/json" \
-     -d '{
-         "data_type": "your_data_type_here",
-         "url_or_text": "your_url_or_text_here"
-     }' \
-     http://localhost:5000/add
-```
- To ask queries from the bot (/query):
-```bash
-curl -X POST \
-     -H "Content-Type: application/json" \
-     -d '{
-         "question": "your_question_here"
-     }' \
-     http://localhost:5000/query
-```
- To chat with the bot (/chat):
-```bash
-curl -X POST \
-     -H "Content-Type: application/json" \
-     -d '{
-         "question": "your_question_here"
-     }' \
-     http://localhost:5000/chat
-```
-
-🎉 Happy Chatting! 🎉
@@ -8,40 +8,54 @@ title: '🤖 Discord Bot'
 - Go to [https://discord.com/developers/applications/](https://discord.com/developers/applications/) and click on `New Application`.
 - Enter the name for your bot, accept the terms and click on `Create`. On the resulting page, enter the details of your bot as you like.
 - On the left sidebar, click on `Bot`. Under the heading `Privileged Gateway Intents`, toggle all 3 options to ON position. Save your changes.
- Now click on `Reset Token` and copy the token value. Set it as `DISCORD_BOT_TOKEN` in variables.env file.
+- Now click on `Reset Token` and copy the token value. Set it as `DISCORD_BOT_TOKEN` in .env file.
 - On the left sidebar, click on `OAuth2` and go to `General`.
 - Set `Authorization Method` to `In-app Authorization`. Under `Scopes` select `bot`.
 - Under `Bot Permissions` allow the following and then click on `Save Changes`.
 ```text
-Read Messages/View Channel (under General Permissions)
 Send Messages (under Text Permissions)
-Read Message History (under Text Permissions)
-Mention everyone (under Text Permissions)
 ```
 - Now under `OAuth2` and go to `URL Generator`. Under `Scopes` select `bot`.
 - Under `Bot Permissions` set the same permissions as above.
 - Now scroll down and copy the `Generated URL`. Paste it in a browser window and select the Server where you want to add the bot.
 - Click on `Continue` and authorize the bot.
- 🎉 The bot has been successfully added to your server.
+- 🎉 The bot has been successfully added to your server. But it's still offline.

-### 🐳 Docker Setup
+### Take the bot online
+
+1. Install embedchain python package:

- To setup your discord bot using docker, run the following command inside this folder using your terminal.
 ```bash
-docker-compose up --build
+pip install --upgrade "embedchain[discord]"
+```
+
+2. Launch your Discord bot:
+
+
+```bash
+python -m embedchain.bots.discord
+```
+
+If you prefer to see the question and not only the answer, run it with
+
+```bash
+python -m embedchain.bots.discord --include-question
 ```
-📝 Note: The build command might take a while to install all the packages depending on your system resources.

 ### 🚀 Usage Instructions

 - Go to the server where you have added your bot.
- You can add data sources to the bot using the command:
+- You can add data sources to the bot using the slash command:
 ```text
-/ec add <data_type> <url_or_text>
+/add <data_type> <url_or_text>
 ```
- You can ask your queries from the bot using the command:
+- You can ask your queries from the bot using the slash command:
 ```text
-/ec query <question>
+/query <question>
+```
+- You can chat with the bot using the slash command:
+```text
+/chat <question>
 ```
 📝 Note: To use the bot privately, you can message the bot directly by right clicking the bot and selecting `Message`.

@@ -2,6 +2,10 @@
 title: '🌐 Full Stack'
 ---

+The Full Stack app example can be found [here](https://github.com/embedchain/embedchain/tree/main/examples/full_stack).
+
+This guide will help you setup the full stack app on your local machine.
+
 ### 🐳 Docker Setup

 - To setup full stack app using docker, run the following command inside this folder using your terminal.
@@ -0,0 +1,59 @@
+---
+title: '🔮 Poe Bot'
+---
+
+### 🚀 Getting started
+
+1. Install embedchain python package:
+
+```bash
+pip install --upgrade "embedchain[poe]"
+```
+
+2. Create a free account on [Poe](https://www.poe.com?utm_source=embedchain).
+3. Click "Create Bot" button on top left.
+4. Give it a handle and an optional description.
+5. Select `Use API`.
+6. Under `API URL` enter your server or ngrok address. You can use your machine's public IP or DNS. Otherwise, employ a proxy server like [ngrok](https://ngrok.com/) to make your local bot accessible.
+7. Copy your api key and paste it in `.env` as `POE_API_KEY`.
+8. You will need to set `OPENAI_API_KEY` for generating embeddings and using LLM. Copy your OpenAI API key from [here](https://platform.openai.com/account/api-keys) and paste it in `.env` as `OPENAI_API_KEY`.
+9. Now create your bot using the following code snippet.
+
+```bash
+# make sure that you have set OPENAI_API_KEY and POE_API_KEY in .env file
+from embedchain.bots import PoeBot
+
+poe_bot = PoeBot()
+
+# add as many data sources as you want
+poe_bot.add("https://en.wikipedia.org/wiki/Adam_D%27Angelo")
+poe_bot.add("https://www.youtube.com/watch?v=pJQVAqmKua8")
+
+# start the bot
+# this start the poe bot server on port 8080 by default
+poe_bot.start()
+```
+
+10. You can paste the above in a file called `your_script.py` and then simply do
+
+```bash
+python your_script.py
+```
+
+Now your bot will start running at port `8080` by default.
+
+11. You can refer the [Supported Data formats](https://docs.embedchain.ai/advanced/data_types) section to refer the supported data types in embedchain.
+
+12. Click `Run check` to make sure your machine can be reached.
+13. Make sure your bot is private if that's what you want.
+14. Click `Create bot` at the bottom to finally create the bot
+15. Now your bot is created.
+
+### 💬 How to use
+
+- To ask the bot questions, just type your query in the Poe interface:
+```text
+<your-question-here>
+```
+
+- If you wish to add more data source to the bot, simply update your script and add as many `.add` as you like. You need to restart the server.
@@ -2,27 +2,25 @@
 title: '💼 Slack Bot'
 ---

-### 🖼️ Template Setup
+### 🖼️ Setup

- Fork [this](https://replit.com/@taranjeetio/EC-Slack-Bot-Template?v=1#README.md) replit template.
- Set your `OPENAI_API_KEY` in Secrets.
- Create a workspace on Slack if you don't have one already by clicking [here](https://slack.com/intl/en-in/).
- Create a new App on your Slack account by going [here](https://api.slack.com/apps).
- Select `From Scratch`, then enter the Bot Name and select your workspace.
- On the `Basic Information` page copy the `Signing Secret` and set it in your secrets as `SLACK_SIGNING_SECRET`.
- On the left Sidebar, go to `OAuth and Permissions` and add the following scopes under `Bot Token Scopes`:
+1. Create a workspace on Slack if you don't have one already by clicking [here](https://slack.com/intl/en-in/).
+2. Create a new App on your Slack account by going [here](https://api.slack.com/apps).
+3. Select `From Scratch`, then enter the Bot Name and select your workspace.
+4. On the left Sidebar, go to `OAuth and Permissions` and add the following scopes under `Bot Token Scopes`:
 ```text
 app_mentions:read
 channels:history
 channels:read
 chat:write
 ```
- Now select the option `Install to Workspace` and after it's done, copy the `Bot User OAuth Token` and set it in your secrets as `SLACK_BOT_TOKEN`.
- Start your replit container now by clicking on `Run`.
- On the Slack API website go to `Event Subscriptions` on the left Sidebar and turn on `Enable Events`.
- Copy the generated server URL in replit, append `/chat` at its end and paste it in `Request URL` box.
- After it gets verified, click on `Subscribe to bot events`, add `message.channels` Bot User Event and click on `Save Changes`.
- Now go to your workspace, click on the bot name in the Sidebar and then add the bot to any channel you want.
+5. Now select the option `Install to Workspace` and after it's done, copy the `Bot User OAuth Token` and set it in your secrets as `SLACK_BOT_TOKEN`.
+6. Run your bot now with `python3 -m embedchain.bots.slack`
+7. Expose your bot to the internet. Default port is `5000`, which can be changed by adding `port --8080` to the startup command. You can use your machine's public IP or DNS. Otherwise, employ a proxy server like [ngrok](https://ngrok.com/) to make your local bot accessible.
+8. On the Slack API website go to `Event Subscriptions` on the left Sidebar and turn on `Enable Events`.
+9. In `Request URL`, enter your server or ngrok address.
+10. After it gets verified, click on `Subscribe to bot events`, add `message.channels` Bot User Event and click on `Save Changes`.
+11. Now go to your workspace, right click on the bot name in the sidebar, click `view app details`, then `add this app to a channel`.

 ### 🚀 Usage Instructions

@@ -7,7 +7,7 @@ title: '💬 WhatsApp Bot'
 1. Install embedchain python package:

 ```bash
-pip install embedchain
+pip install --upgrade embedchain
 ```

 2. Launch your WhatsApp bot:
@@ -0,0 +1,123 @@
+---
+title: 🔎 Examples
+description: 'Collection of Google colab notebook and Replit links for users'
+---
+
+<table>
+  <thead>
+    <tr>
+      <th>LLM</th>
+      <th>Google Colab</th>
+      <th>Replit</th>
+    </tr>
+  </thead>
+  <tbody>
+    <tr>
+      <td className="align-middle">OpenAI</td>
+      <td className="align-middle"><a target="_blank" href="https://colab.research.google.com/github/embedchain/embedchain/blob/main/notebooks/openai.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" noZoom alt="Open In Colab"/></a></td>
+      <td className="align-middle"><a target="_blank" href="https://replit.com/@taranjeetio/openai#main.py"><img src="https://replit.com/badge?caption=Try%20with%20Replit&amp;variant=small" noZoom alt="Try with Replit Badge"/></a></td>
+    </tr>
+    <tr>
+      <td className="align-middle">Anthropic</td>
+      <td className="align-middle"><a target="_blank" href="https://colab.research.google.com/github/embedchain/embedchain/blob/main/notebooks/anthropic.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" noZoom alt="Open In Colab"/></a></td>
+      <td className="align-middle"><a target="_blank" href="https://replit.com/@taranjeetio/anthropic#main.py"><img src="https://replit.com/badge?caption=Try%20with%20Replit&amp;variant=small" noZoom alt="Try with Replit Badge"/></a></td>
+    </tr>
+    <tr>
+      <td className="align-middle">Azure OpenAI</td>
+      <td className="align-middle"><a target="_blank" href="https://colab.research.google.com/github/embedchain/embedchain/blob/main/notebooks/azure-openai.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" noZoom alt="Open In Colab"/></a></td>
+      <td className="align-middle"><a target="_blank" href="https://replit.com/@taranjeetio/azureopenai#main.py"><img src="https://replit.com/badge?caption=Try%20with%20Replit&amp;variant=small" noZoom alt="Try with Replit Badge"/></a></td>
+    </tr>
+    <tr>
+      <td className="align-middle">VertexAI</td>
+      <td className="align-middle"><a target="_blank" href="https://colab.research.google.com/github/embedchain/embedchain/blob/main/notebooks/vertex_ai.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" noZoom alt="Open In Colab"/></a></td>
+      <td className="align-middle"><a target="_blank" href="https://replit.com/@taranjeetio/vertexai#main.py"><img src="https://replit.com/badge?caption=Try%20with%20Replit&amp;variant=small" noZoom alt="Try with Replit Badge"/></a></td>
+    </tr>
+    <tr>
+      <td className="align-middle">Cohere</td>
+      <td className="align-middle"><a target="_blank" href="https://colab.research.google.com/github/embedchain/embedchain/blob/main/notebooks/cohere.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" noZoom alt="Open In Colab"/></a></td>
+      <td className="align-middle"><a target="_blank" href="https://replit.com/@taranjeetio/cohere#main.py"><img src="https://replit.com/badge?caption=Try%20with%20Replit&amp;variant=small" noZoom alt="Try with Replit Badge"/></a></td>
+    </tr>
+    <tr>
+      <td className="align-middle">Hugging Face</td>
+      <td className="align-middle"><a target="_blank" href="https://colab.research.google.com/github/embedchain/embedchain/blob/main/notebooks/hugging_face_hub.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" noZoom alt="Open In Colab"/></a></td>
+      <td className="align-middle"><a target="_blank" href="https://replit.com/@taranjeetio/huggingface#main.py"><img src="https://replit.com/badge?caption=Try%20with%20Replit&amp;variant=small" noZoom alt="Try with Replit Badge"/></a></td>
+    </tr>
+    <tr>
+      <td className="align-middle">JinaChat</td>
+      <td className="align-middle"><a target="_blank" href="https://colab.research.google.com/github/embedchain/embedchain/blob/main/notebooks/jina.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" noZoom alt="Open In Colab"/></a></td>
+      <td className="align-middle"><a target="_blank" href="https://replit.com/@taranjeetio/jina#main.py"><img src="https://replit.com/badge?caption=Try%20with%20Replit&amp;variant=small" noZoom alt="Try with Replit Badge"/></a></td>
+    </tr>
+    <tr>
+      <td className="align-middle">GPT4All</td>
+      <td className="align-middle"><a target="_blank" href="https://colab.research.google.com/github/embedchain/embedchain/blob/main/notebooks/gpt4all.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" noZoom alt="Open In Colab"/></a></td>
+      <td className="align-middle"><a target="_blank" href="https://replit.com/@taranjeetio/gpt4all#main.py"><img src="https://replit.com/badge?caption=Try%20with%20Replit&amp;variant=small" noZoom alt="Try with Replit Badge"/></a></td>
+    </tr>
+    <tr>
+      <td className="align-middle">Llama2</td>
+      <td className="align-middle"><a target="_blank" href="https://colab.research.google.com/github/embedchain/embedchain/blob/main/notebooks/llama2.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" noZoom alt="Open In Colab"/></a></td>
+      <td className="align-middle"><a target="_blank" href="https://replit.com/@taranjeetio/llama2#main.py"><img src="https://replit.com/badge?caption=Try%20with%20Replit&amp;variant=small" noZoom alt="Try with Replit Badge"/></a></td>
+    </tr>
+  </tbody>
+</table>
+<table>
+  <thead>
+    <tr>
+      <th>Embedding model</th>
+      <th>Google Colab</th>
+      <th>Replit</th>
+    </tr>
+  </thead>
+  <tbody>
+    <tr>
+      <td className="align-middle">OpenAI</td>
+      <td className="align-middle"><a target="_blank" href="https://colab.research.google.com/github/embedchain/embedchain/blob/main/notebooks/openai.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" noZoom alt="Open In Colab"/></a></td>
+      <td className="align-middle"><a target="_blank" href="https://replit.com/@taranjeetio/openai#main.py"><img src="https://replit.com/badge?caption=Try%20with%20Replit&amp;variant=small" noZoom alt="Try with Replit Badge"/></a></td>
+    </tr>
+    <tr>
+      <td className="align-middle">VertexAI</td>
+      <td className="align-middle"><a target="_blank" href="https://colab.research.google.com/github/embedchain/embedchain/blob/main/notebooks/vertex_ai.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" noZoom alt="Open In Colab"/></a></td>
+      <td className="align-middle"><a target="_blank" href="https://replit.com/@taranjeetio/vertexai#main.py"><img src="https://replit.com/badge?caption=Try%20with%20Replit&amp;variant=small" noZoom alt="Try with Replit Badge"/></a></td>
+    </tr>
+    <tr>
+      <td className="align-middle">GPT4All</td>
+      <td className="align-middle"><a target="_blank" href="https://colab.research.google.com/github/embedchain/embedchain/blob/main/notebooks/gpt4all.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" noZoom alt="Open In Colab"/></a></td>
+      <td className="align-middle"><a target="_blank" href="https://replit.com/@taranjeetio/gpt4all#main.py"><img src="https://replit.com/badge?caption=Try%20with%20Replit&amp;variant=small" noZoom alt="Try with Replit Badge"/></a></td>
+    </tr>
+    <tr>
+      <td className="align-middle">Hugging Face</td>
+      <td className="align-middle"><a target="_blank" href="https://colab.research.google.com/github/embedchain/embedchain/blob/main/notebooks/hugging_face_hub.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" noZoom alt="Open In Colab"/></a></td>
+      <td className="align-middle"><a target="_blank" href="https://replit.com/@taranjeetio/huggingface#main.py"><img src="https://replit.com/badge?caption=Try%20with%20Replit&amp;variant=small" noZoom alt="Try with Replit Badge"/></a></td>
+    </tr>
+  </tbody>
+</table>
+<table>
+  <thead>
+    <tr>
+      <th>Vector DB</th>
+      <th>Google Colab</th>
+      <th>Replit</th>
+    </tr>
+  </thead>
+  <tbody>
+    <tr>
+      <td className="align-middle">ChromaDB</td>
+      <td className="align-middle"><a target="_blank" href="https://colab.research.google.com/github/embedchain/embedchain/blob/main/notebooks/chromadb.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" noZoom alt="Open In Colab"/></a></td>
+      <td className="align-middle"><a target="_blank" href="https://replit.com/@taranjeetio/chromadb#main.py"><img src="https://replit.com/badge?caption=Try%20with%20Replit&amp;variant=small" noZoom alt="Try with Replit Badge"/></a></td>
+    </tr>
+    <tr>
+      <td className="align-middle">Elasticsearch</td>
+      <td className="align-middle"><a target="_blank" href="https://colab.research.google.com/github/embedchain/embedchain/blob/main/notebooks/elasticsearch.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" noZoom alt="Open In Colab"/></a></td>
+      <td className="align-middle"><a target="_blank" href="https://replit.com/@taranjeetio/elasticsearchdb#main.py"><img src="https://replit.com/badge?caption=Try%20with%20Replit&amp;variant=small" noZoom alt="Try with Replit Badge"/></a></td>
+    </tr>
+    <tr>
+      <td className="align-middle">Opensearch</td>
+      <td className="align-middle"><a target="_blank" href="https://colab.research.google.com/github/embedchain/embedchain/blob/main/notebooks/opensearch.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" noZoom alt="Open In Colab"/></a></td>
+      <td className="align-middle"><a target="_blank" href="https://replit.com/@taranjeetio/opensearchdb#main.py"><img src="https://replit.com/badge?caption=Try%20with%20Replit&amp;variant=small" noZoom alt="Try with Replit Badge"/></a></td>
+    </tr>
+    <tr>
+      <td className="align-middle">Pinecone</td>
+      <td className="align-middle"><a target="_blank" href="https://colab.research.google.com/github/embedchain/embedchain/blob/main/notebooks/pinecone.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" noZoom alt="Open In Colab"/></a></td>
+      <td className="align-middle"><a target="_blank" href="https://replit.com/@taranjeetio/pineconedb#main.py"><img src="https://replit.com/badge?caption=Try%20with%20Replit&amp;variant=small" noZoom alt="Try with Replit Badge"/></a></td>
+    </tr>
+  </tbody>
+</table>
@@ -0,0 +1,68 @@
+---
+title: ❓ FAQs
+description: 'Collections of all the frequently asked questions'
+---
+
+#### How to use GPT-4 as the LLM model?
+
+<CodeGroup>
+
+```python main.py
+import os
+from embedchain import Pipeline as App
+
+os.environ['OPENAI_API_KEY'] = 'xxx'
+
+# load llm configuration from gpt4.yaml file
+app = App.from_config(yaml_path="gpt4.yaml")
+```
+
+```yaml gpt4.yaml
+llm:
+  provider: openai
+  config:
+    model: 'gpt-4'
+    temperature: 0.5
+    max_tokens: 1000
+    top_p: 1
+    stream: false
+```
+
+</CodeGroup>
+
+#### I don't have OpenAI credits. How can I use some open source model?
+
+<CodeGroup>
+
+```python main.py
+import os
+from embedchain import Pipeline as App
+
+os.environ['OPENAI_API_KEY'] = 'xxx'
+
+# load llm configuration from opensource.yaml file
+app = App.from_config(yaml_path="opensource.yaml")
+```
+
+```yaml opensource.yaml
+llm:
+  provider: gpt4all
+  config:
+    model: 'orca-mini-3b.ggmlv3.q4_0.bin'
+    temperature: 0.5
+    max_tokens: 1000
+    top_p: 1
+    stream: false
+
+embedder:
+  provider: gpt4all
+  config:
+    model: 'all-MiniLM-L6-v2'
+```
+</CodeGroup>
+
+#### How to contact support?
+
+If docs aren't sufficient, please feel free to reach out to us using one of the following methods:
+
+<Snippet file="get-help.mdx" />
@@ -0,0 +1,131 @@
+---
+title: 📚 Introduction
+description: '📝 Embedchain is a Data Platform for LLMs - load, index, retrieve, and sync any unstructured data'
+---
+
+## 🌐 What is Embedchain?
+
+Embedchain simplifies data handling by automatically processing unstructured data, breaking it into chunks, generating embeddings, and storing it in a vector database.
+
+Through various APIs, you can obtain contextual information for queries, find answers to specific questions, and engage in chat conversations using your data.
+## 🔍 Search
+
+Embedchain lets you get most relevant context by doing semantic search over your data sources for a provided query. See the example below:
+
+```python
+from embedchain import Pipeline as App
+
+# Initialize app
+app = App()
+
+# Add data source
+app.add("https://www.forbes.com/profile/elon-musk")
+
+# Get relevant context using semantic search
+context = app.search("What is the net worth of Elon?", num_documents=2)
+print(context)
+# Context:
+# [
+#     {
+#         'context': 'Elon Musk PROFILEElon MuskCEO, Tesla$221.9BReal Time Net Worthas of 10/29/23Reflects change since 5 pm ET of prior trading day. 1 in the world todayPhoto by Martin Schoeller for ForbesAbout Elon MuskElon Musk cofounded six companies, including electric car maker Tesla, rocket producer SpaceX and tunneling startup Boring Company.He owns about 21% of Tesla between stock and options, but has pledged more than half his shares as collateral for personal loans of up to $3.5 billion.SpaceX, founded in',
+#         'source': 'https://www.forbes.com/profile/elon-musk',
+#         'document_id': 'some_document_id'
+#     },
+#     {
+#         'context': 'company, which is now called X.Wealth HistoryHOVER TO REVEAL NET WORTH BY YEARForbes Lists 1Forbes 400 (2023)The Richest Person In Every State (2023) 2Billionaires (2023) 1Innovative Leaders (2019) 25Powerful People (2018) 12Richest In Tech (2017)Global Game Changers (2016)More ListsPersonal StatsAge52Source of WealthTesla, SpaceX, Self MadeSelf-Made Score8Philanthropy Score1ResidenceAustin, TexasCitizenshipUnited StatesMarital StatusSingleChildren11EducationBachelor of Arts/Science, University',
+#         'source': 'https://www.forbes.com/profile/elon-musk',
+#         'document_id': 'some_document_id'
+#     }
+# ]
+```
+
+## ❓Query
+
+Embedchain empowers developers to ask questions and receive relevant answers through a user-friendly query API. Refer to the following example to learn how to utilize the query API:
+
+```python
+from embedchain import Pipeline as App
+
+# Initialize app
+app = App()
+
+# Add data source
+app.add("https://www.forbes.com/profile/elon-musk")
+
+# Get relevant answer for your query
+answer = app.query("What is the net worth of Elon?")
+print(answer)
+# Answer: The net worth of Elon Musk is $221.9 billion.
+```
+
+## 💬 Chat
+
+Embedchain allows easy chatting over your data sources using a user-friendly chat API. Check out the example below to understand how to use the chat API:
+
+```python
+from embedchain import Pipeline as App
+
+# Initialize app
+app = App()
+
+# Add data source
+app.add("https://www.forbes.com/profile/elon-musk")
+
+# Chat on your data using `.chat()`
+answer = app.chat("How much did Elon pay for Twitter?")
+print(answer)
+# Answer: Elon Musk paid $44 billion for Twitter.
+```
+
+## 🚀 Deploy
+
+Embedchain enables developers to deploy their LLM-powered apps in production using the Embedchain platform. The platform offers free access to context on your data through its REST API. Once the pipeline is deployed, you can update your data sources anytime after deployment.
+
+See the example below on how to use the deploy API:
+
+```python
+from embedchain import Pipeline as App
+
+# Initialize app
+app = App()
+
+# Add data source
+app.add("https://www.forbes.com/profile/elon-musk")
+
+# Deploy your pipeline to Embedchain Platform
+app.deploy()
+
+# 🔑 Enter your Embedchain API key. You can find the API key at https://app.embedchain.ai/settings/keys/
+# ec-xxxxxx
+
+# 🛠️ Creating pipeline on the platform...
+# 🎉🎉🎉 Pipeline created successfully! View your pipeline: https://app.embedchain.ai/pipelines/xxxxx
+
+# 🛠️ Adding data to your pipeline...
+# ✅ Data of type: web_page, value: https://www.forbes.com/profile/elon-musk added successfully.
+```
+
+## 🚀 How it works?
+
+Embedchain abstracts out the following steps from you to easily create LLM powered apps:
+
+1. Detect the data type and load data
+2. Create meaningful chunks
+3. Create embeddings for each chunk
+4. Store chunks in a vector database
+
+When a user asks a query, the following process happens to find the answer:
+
+1. Create an embedding for the query
+2. Find similar documents for the query from the vector database
+3. Pass the similar documents as context to LLM to get the final answer
+
+The process of loading the dataset and querying involves multiple steps, each with its own nuances:
+
+- How should I chunk the data? What is a meaningful chunk size?
+- How should I create embeddings for each chunk? Which embedding model should I use?
+- How should I store the chunks in a vector database? Which vector database should I use?
+- Should I store metadata along with the embeddings?
+- How should I find similar documents for a query? Which ranking model should I use?
+
+Embedchain takes care of all these nuances and provides a simple interface to create apps on any data.
@@ -0,0 +1,81 @@
+---
+title: '🚀 Quickstart'
+description: '💡 Start building LLM powered apps under 30 seconds'
+---
+
+Embedchain is a Data Platform for LLMs - load, index, retrieve, and sync any unstructured data. Using embedchain, you can easily create LLM powered apps over any data. 
+
+Install embedchain python package:
+
+```bash
+pip install embedchain
+```
+
+Creating an app involves 3 steps:
+
+<Steps>
+  <Step title="⚙️ Import app instance">
+```python
+from embedchain import Pipeline as App
+app = App()
+```
+  </Step>
+  <Step title="🗃️ Add data sources">
+```python
+# Add different data sources
+app.add("https://en.wikipedia.org/wiki/Elon_Musk")
+app.add("https://www.forbes.com/profile/elon-musk")
+# You can also add local data sources such as pdf, csv files etc.
+# app.add("/path/to/file.pdf")
+```
+  </Step>
+  <Step title="💬 Query or chat or search context on your data">
+```python
+app.query("What is the net worth of Elon Musk today?")
+# Answer: The net worth of Elon Musk today is $258.7 billion.
+```
+  </Step>
+  <Step title="🚀 (Optional) Deploy your pipeline to Embedchain Platform">
+```python
+app.deploy()
+# 🔑 Enter your Embedchain API key. You can find the API key at https://app.embedchain.ai/settings/keys/
+# ec-xxxxxx
+
+# 🛠️ Creating pipeline on the platform...
+# 🎉🎉🎉 Pipeline created successfully! View your pipeline: https://app.embedchain.ai/pipelines/xxxxx
+
+# 🛠️ Adding data to your pipeline...
+# ✅ Data of type: web_page, value: https://www.forbes.com/profile/elon-musk added successfully.
+```
+  </Step>
+</Steps>
+
+Putting it together, you can run your first app using the following code. Make sure to set the `OPENAI_API_KEY` 🔑 environment variable in the code.
+
+```python
+import os
+from embedchain import Pipeline as App
+
+os.environ["OPENAI_API_KEY"] = "xxx"
+app = App()
+
+# Add different data sources
+app.add("https://en.wikipedia.org/wiki/Elon_Musk")
+app.add("https://www.forbes.com/profile/elon-musk")
+# You can also add local data sources such as pdf, csv files etc.
+# app.add("/path/to/file.pdf")
+
+response = app.query("What is the net worth of Elon Musk today?")
+print(response)
+# Answer: The net worth of Elon Musk today is $258.7 billion.
+
+app.deploy()
+# 🔑 Enter your Embedchain API key. You can find the API key at https://app.embedchain.ai/settings/keys/
+# ec-xxxxxx
+
+# 🛠️ Creating pipeline on the platform...
+# 🎉🎉🎉 Pipeline created successfully! View your pipeline: https://app.embedchain.ai/pipelines/xxxxx
+
+# 🛠️ Adding data to your pipeline...
+# ✅ Data of type: web_page, value: https://www.forbes.com/profile/elon-musk added successfully.
+```
@@ -0,0 +1,51 @@
+---
+title: '🛠️ LangSmith'
+description: 'Integrate with Langsmith to debug and monitor your LLM app'
+---
+
+Embedchain now supports integration with [LangSmith](https://www.langchain.com/langsmith).
+
+To use langsmith, you need to do the following steps
+
+1. Have an account on langsmith and keep the environment variables in handy
+2. Set the environments variables in your app so that embedchain has context about it.
+3. Just use embedchain and everything will be logged to LangSmith, so that you can better test and monitor your application.
+
+Lets cover each step in detail.
+
+* First make sure that you a LangSmith account created and have all the necessary variables handy. LangSmith has a [good documentation](https://docs.smith.langchain.com/) on how to get started with their service.
+
+* Once you have the account setup, we will need the following environment variables
+
+```bash
+export LANGCHAIN_TRACING_V2=true
+export LANGCHAIN_ENDPOINT=https://api.smith.langchain.com
+export LANGCHAIN_API_KEY=<your-api-key>
+export LANGCHAIN_PROJECT=<your-project>  # if not specified, defaults to "default"
+```
+
+If you are using Python, you can use the following code to set environment variables
+
+```python
+import os
+
+os.environ['LANGCHAIN_TRACING_V2'] = 'true'
+os.environ['LANGCHAIN_ENDPOINT'] = 'https://api.smith.langchain.com'
+os.environ['LANGCHAIN_API_KEY'] = <your-api-key>
+os.environ['LANGCHAIN_PROJECT] = <your-project>
+```
+
+* Now create an app using embedchain and everything will be automatically visible in the LangSmith
+
+
+```python
+from embedchain import Pipeline as App
+
+app = App()
+app.add("https://en.wikipedia.org/wiki/Elon_Musk")
+app.query("How many companies did Elon found?")
+```
+
+* Now the entire log for this will be visible in langsmith.
+
+<img src="/images/langsmith.png"/>
@@ -1,56 +0,0 @@
---
-title: 📚 Introduction
-description: '📝 Embedchain is a framework to easily create LLM powered bots over any dataset.'
---
-
-## 🤔 What is Embedchain?
-
-Embedchain abstracts the entire process of loading a dataset, chunking it, creating embeddings, and storing it in a vector database.
-
-You can add a single or multiple datasets using the `.add` method. Then, simply use the `.query` method to find answers from the added datasets.
-
-If you want to create a Naval Ravikant bot with a YouTube video, a book in PDF format, two blog posts, and a question and answer pair, all you need to do is add the respective links. Embedchain will take care of the rest, creating a bot for you.
-
-```python
-from embedchain import App
-
-naval_chat_bot = App()
-# Embed Online Resources
-naval_chat_bot.add("https://www.youtube.com/watch?v=3qHkcs3kG44")
-naval_chat_bot.add("https://navalmanack.s3.amazonaws.com/Eric-Jorgenson_The-Almanack-of-Naval-Ravikant_Final.pdf")
-naval_chat_bot.add("https://nav.al/feedback")
-naval_chat_bot.add("https://nav.al/agi")
-
-# Embed Local Resources
-naval_chat_bot.add(("Who is Naval Ravikant?", "Naval Ravikant is an Indian-American entrepreneur and investor."))
-
-naval_chat_bot.query("What unique capacity does Naval argue humans possess when it comes to understanding explanations or concepts?")
-# Answer: Naval argues that humans possess the unique capacity to understand explanations or concepts to the maximum extent possible in this physical reality.
-```
-
-## 🚀 How it works?
-
-Creating a chat bot over any dataset involves the following steps:
-
-1. Detect the data type and load the data
-2. Create meaningful chunks
-3. Create embeddings for each chunk
-4. Store the chunks in a vector database
-
-When a user asks a query, the following process happens to find the answer:
-
-1. Create an embedding for the query
-2. Find similar documents for the query from the vector database
-3. Pass the similar documents as context to LLM to get the final answer.
-
-The process of loading the dataset and querying involves multiple steps, each with its own nuances:
-
- How should I chunk the data? What is a meaningful chunk size?
- How should I create embeddings for each chunk? Which embedding model should I use?
- How should I store the chunks in a vector database? Which vector database should I use?
- Should I store metadata along with the embeddings?
- How should I find similar documents for a query? Which ranking model should I use?
-
-Embedchain takes care of all these nuances and provides a simple interface to create bots over any dataset.
-
-In the first release, we make it easier for anyone to get a chatbot over any dataset up and running in less than a minute. Just create an app instance, add the datasets using the `.add` method, and use the `.query` method to get the relevant answers.
@@ -7,48 +7,162 @@
  },
  "favicon": "/favicon.png",
  "colors": {
-    "primary": "#12A7D3",
-    "light": "#81D7F7",
-    "dark": "#004E7A"
+    "primary": "#2B48EE",
+    "light": "#2B48EE",
+    "dark": "#2B48EE",
+    "background": {
+      "dark": "#020415"
+    }
+  },
+  "openapi": ["/rest-api.json"],
+  "metadata": {
+    "og:image": "/images/og.png",
+    "twitter:site": "@embedchain"
+  },
+  "topAnchor": {
+    "name": "Documentation",
+    "icon": "book-open"
  },
  "topbarLinks": [
    {
      "name": "Twitter",
      "url": "https://twitter.com/embedchain"
    },
+    {
+      "name": "Slack",
+      "url": "https://join.slack.com/t/embedchain/shared_invite/zt-22uwz3c46-Zg7cIh5rOBteT_xe1jwLDw"
+    },
    {
      "name": "Discord",
      "url": "https://discord.gg/6PzXDgEjG5"
    }
  ],
  "topbarCtaButton": {
-    "name": "GitHub",
-    "url": "https://embedchain.ai"
+    "name": "Get started",
+    "url": "https://app.embedchain.ai"
+  },
+  "primaryTab": {
+    "name": "Docs"
  },
  "navigation": [
    {
-      "group": "Getting started",
-      "pages": ["quickstart", "introduction"]
+      "group": "Get started",
+      "pages": [
+        "get-started/quickstart",
+        "get-started/introduction",
+        "get-started/faq",
+        "get-started/examples"
+      ]
+    },
+    {
+      "group": "Components",
+      "pages": [
+        "components/llms",
+        "components/embedding-models",
+        "components/vector-databases"
+      ]
+    },
+    {
+      "group": "Data sources",
+      "pages": [
+        "data-sources/overview",
+        {
+          "group": "Supported data sources",
+          "pages": [
+            "data-sources/csv",
+            "data-sources/json",
+            "data-sources/docs-site",
+            "data-sources/docx",
+            "data-sources/mdx",
+            "data-sources/notion",
+            "data-sources/pdf-file",
+            "data-sources/qna",
+            "data-sources/sitemap",
+            "data-sources/text",
+            "data-sources/web-page",
+            "data-sources/openapi",
+            "data-sources/youtube-video"
+          ]
+        },
+        "data-sources/data-type-handling"
+      ]
    },
    {
      "group": "Advanced",
-      "pages": ["advanced/app_types", "advanced/interface_types", "advanced/adding_data", "advanced/data_types", "advanced/query_configuration", "advanced/configuration", "advanced/testing", "advanced/vector_database", "advanced/showcase"]
+      "pages": ["advanced/configuration"]
+    },
+    {
+      "group": "REST API",
+      "pages": [
+        "rest-api/getting-started",
+        "rest-api/create",
+        "rest-api/get-all-apps",
+        "rest-api/add-data",
+        "rest-api/get-data",
+        "rest-api/query",
+        "rest-api/deploy",
+        "rest-api/delete",
+        "rest-api/check-status"
+      ]
    },
    {
      "group": "Examples",
-      "pages": ["examples/full_stack", "examples/api_server", "examples/discord_bot", "examples/slack_bot", "examples/telegram_bot", "examples/whatsapp_bot"]
+      "pages": [
+        "examples/full_stack",
+        "examples/discord_bot",
+        "examples/slack_bot",
+        "examples/telegram_bot",
+        "examples/whatsapp_bot",
+        "examples/poe_bot"
+      ]
    },
    {
-      "group": "Contribution Guidelines",
-      "pages": ["contribution/dev", "contribution/docs"]
+      "group": "Community",
+      "pages": ["community/connect-with-us", "community/showcase"]
+    },
+    {
+      "group": "Integrations",
+      "pages": ["integration/langsmith"]
+    },
+    {
+      "group": "Contribute",
+      "pages": [
+        "contribution/guidelines",
+        "contribution/dev",
+        "contribution/docs",
+        "contribution/python",
+        "contribution/javascript"
+      ]
+    },
+    {
+      "group": "Product",
+      "pages": ["product/release-notes"]
    }
-
  ],
  "footerSocials": {
-    "twitter": "https://twitter.com/embedchain",
+    "website": "https://embedchain.ai",
    "github": "https://github.com/embedchain/embedchain",
+    "slack": "https://join.slack.com/t/embedchain/shared_invite/zt-22uwz3c46-Zg7cIh5rOBteT_xe1jwLDw",
+    "discord": "https://discord.gg/6PzXDgEjG5",
+    "twitter": "https://twitter.com/embedchain",
    "linkedin": "https://www.linkedin.com/company/embedchain"
  },
-  "backgroundImage": "/background.png",
-  "isWhiteLabeled": true
+  "isWhiteLabeled": true,
+  "analytics": {
+    "posthog": {
+      "apiKey": "phc_PHQDA5KwztijnSojsxJ2c1DuJd52QCzJzT2xnSGvjN2",
+      "apiHost": "https://app.embedchain.ai/ingest"
+    }
+  },
+  "feedback": {
+    "suggestEdit": true,
+    "raiseIssue": true,
+    "thumbsRating": true
+  },
+  "search": {
+    "prompt": "✨ Search embedchain docs..."
+  },
+  "api": {
+    "baseUrl": "http://localhost:8080"
+  }
 }
@@ -0,0 +1,4 @@
+---
+title: ' 📜 Release Notes'
+url: https://github.com/embedchain/embedchain/releases
+---
@@ -1,35 +0,0 @@
---
-title: '🚀 Quickstart'
-description: '💡 Start building LLM powered bots under 30 seconds'
---
-
-Install embedchain python package:
-
-```bash
-pip install embedchain
-```
-
-Creating a chatbot involves 3 steps:
-
- ⚙️ Import the App instance
- 🗃️ Add Dataset
- 💬 Query or Chat on the dataset and get answers (Interface Types)
-
-Run your first bot in python using the following code. Make sure to set the `OPENAI_API_KEY` 🔑 environment variable in the code.
-
-```python
-import os
-
-from embedchain import App
-
-os.environ["OPENAI_API_KEY"] = "xxx"
-elon_musk_bot = App()
-
-# Embed Online Resources
-elon_musk_bot.add("https://en.wikipedia.org/wiki/Elon_Musk")
-elon_musk_bot.add("https://www.tesla.com/elon-musk")
-
-response = elon_musk_bot.query("How many companies does Elon Musk run?")
-print(response)
-# Answer: 'Elon Musk runs four companies: Tesla, SpaceX, Neuralink, and The Boring Company.'
-```
@@ -0,0 +1,427 @@
+{
+    "openapi": "3.1.0",
+    "info": {
+      "title": "Embedchain REST API",
+      "description": "This is the REST API for Embedchain.",
+      "license": {
+        "name": "Apache 2.0",
+        "url": "https://github.com/embedchain/embedchain/blob/main/LICENSE"
+      },
+      "version": "0.0.1"
+    },
+    "paths": {
+      "/ping": {
+        "get": {
+          "tags": ["Utility"],
+          "summary": "Check status",
+          "description": "Endpoint to check the status of the API",
+          "operationId": "check_status_ping_get",
+          "responses": {
+            "200": {
+              "description": "Successful Response",
+              "content": { "application/json": { "schema": {} } }
+            }
+          }
+        }
+      },
+      "/apps": {
+        "get": {
+          "tags": ["Apps"],
+          "summary": "Get all apps",
+          "description": "Get all applications",
+          "operationId": "get_all_apps_apps_get",
+          "responses": {
+            "200": {
+              "description": "Successful Response",
+              "content": { "application/json": { "schema": {} } }
+            }
+          }
+        }
+      },
+      "/create": {
+        "post": {
+          "tags": ["Apps"],
+          "summary": "Create app",
+          "description": "Create a new app using App ID",
+          "operationId": "create_app_using_default_config_create_post",
+          "parameters": [
+            {
+              "name": "app_id",
+              "in": "query",
+              "required": true,
+              "schema": { "type": "string", "title": "App Id" }
+            }
+          ],
+          "requestBody": {
+            "content": {
+              "multipart/form-data": {
+                "schema": {
+                  "allOf": [
+                    {
+                      "$ref": "#/components/schemas/Body_create_app_using_default_config_create_post"
+                    }
+                  ],
+                  "title": "Body"
+                }
+              }
+            }
+          },
+          "responses": {
+            "200": {
+              "description": "Successful Response",
+              "content": {
+                "application/json": {
+                  "schema": { "$ref": "#/components/schemas/DefaultResponse" }
+                }
+              }
+            },
+            "422": {
+              "description": "Validation Error",
+              "content": {
+                "application/json": {
+                  "schema": { "$ref": "#/components/schemas/HTTPValidationError" }
+                }
+              }
+            }
+          }
+        }
+      },
+      "/{app_id}/data": {
+        "get": {
+          "tags": ["Apps"],
+          "summary": "Get data",
+          "description": "Get all data sources for an app",
+          "operationId": "get_datasources_associated_with_app_id__app_id__data_get",
+          "parameters": [
+            {
+              "name": "app_id",
+              "in": "path",
+              "required": true,
+              "schema": { "type": "string", "title": "App Id" }
+            }
+          ],
+          "responses": {
+            "200": {
+              "description": "Successful Response",
+              "content": { "application/json": { "schema": {} } }
+            },
+            "422": {
+              "description": "Validation Error",
+              "content": {
+                "application/json": {
+                  "schema": { "$ref": "#/components/schemas/HTTPValidationError" }
+                }
+              }
+            }
+          }
+        }
+      },
+      "/{app_id}/add": {
+        "post": {
+          "tags": ["Apps"],
+          "summary": "Add data",
+          "description": "Add a data source to an app.",
+          "operationId": "add_datasource_to_an_app__app_id__add_post",
+          "parameters": [
+            {
+              "name": "app_id",
+              "in": "path",
+              "required": true,
+              "schema": { "type": "string", "title": "App Id" }
+            }
+          ],
+          "requestBody": {
+            "required": true,
+            "content": {
+              "application/json": {
+                "schema": { "$ref": "#/components/schemas/SourceApp" }
+              }
+            }
+          },
+          "responses": {
+            "200": {
+              "description": "Successful Response",
+              "content": {
+                "application/json": {
+                  "schema": { "$ref": "#/components/schemas/DefaultResponse" }
+                }
+              }
+            },
+            "422": {
+              "description": "Validation Error",
+              "content": {
+                "application/json": {
+                  "schema": { "$ref": "#/components/schemas/HTTPValidationError" }
+                }
+              }
+            }
+          }
+        }
+      },
+      "/{app_id}/query": {
+        "post": {
+          "tags": ["Apps"],
+          "summary": "Query app",
+          "description": "Query an app",
+          "operationId": "query_an_app__app_id__query_post",
+          "parameters": [
+            {
+              "name": "app_id",
+              "in": "path",
+              "required": true,
+              "schema": { "type": "string", "title": "App Id" }
+            }
+          ],
+          "requestBody": {
+            "required": true,
+            "content": {
+              "application/json": {
+                "schema": { "$ref": "#/components/schemas/QueryApp" }
+              }
+            }
+          },
+          "responses": {
+            "200": {
+              "description": "Successful Response",
+              "content": {
+                "application/json": {
+                  "schema": { "$ref": "#/components/schemas/DefaultResponse" }
+                }
+              }
+            },
+            "422": {
+              "description": "Validation Error",
+              "content": {
+                "application/json": {
+                  "schema": { "$ref": "#/components/schemas/HTTPValidationError" }
+                }
+              }
+            }
+          }
+        }
+      },
+      "/{app_id}/chat": {
+        "post": {
+          "tags": ["Apps"],
+          "summary": "Chat",
+          "description": "Chat with an app.\n\napp_id: The ID of the app. Use \"default\" for the default app.\n\nmessage: The message that you want to send to the app.",
+          "operationId": "chat_with_an_app__app_id__chat_post",
+          "parameters": [
+            {
+              "name": "app_id",
+              "in": "path",
+              "required": true,
+              "schema": { "type": "string", "title": "App Id" }
+            }
+          ],
+          "requestBody": {
+            "required": true,
+            "content": {
+              "application/json": {
+                "schema": { "$ref": "#/components/schemas/MessageApp" }
+              }
+            }
+          },
+          "responses": {
+            "200": {
+              "description": "Successful Response",
+              "content": {
+                "application/json": {
+                  "schema": { "$ref": "#/components/schemas/DefaultResponse" }
+                }
+              }
+            },
+            "422": {
+              "description": "Validation Error",
+              "content": {
+                "application/json": {
+                  "schema": { "$ref": "#/components/schemas/HTTPValidationError" }
+                }
+              }
+            }
+          }
+        }
+      },
+      "/{app_id}/deploy": {
+        "post": {
+          "tags": ["Apps"],
+          "summary": "Deploy App",
+          "description": "Deploy an existing app.",
+          "operationId": "deploy_app__app_id__deploy_post",
+          "parameters": [
+            {
+              "name": "app_id",
+              "in": "path",
+              "required": true,
+              "schema": { "type": "string", "title": "App Id" }
+            }
+          ],
+          "requestBody": {
+            "required": true,
+            "content": {
+              "application/json": {
+                "schema": { "$ref": "#/components/schemas/DeployAppRequest" }
+              }
+            }
+          },
+          "responses": {
+            "200": {
+              "description": "Successful Response",
+              "content": {
+                "application/json": {
+                  "schema": { "$ref": "#/components/schemas/DefaultResponse" }
+                }
+              }
+            },
+            "422": {
+              "description": "Validation Error",
+              "content": {
+                "application/json": {
+                  "schema": { "$ref": "#/components/schemas/HTTPValidationError" }
+                }
+              }
+            }
+          }
+        }
+      },
+      "/{app_id}/delete": {
+        "delete": {
+          "tags": ["Apps"],
+          "summary": "Delete app",
+          "description": "Delete an existing app",
+          "operationId": "delete_app__app_id__delete_delete",
+          "parameters": [
+            {
+              "name": "app_id",
+              "in": "path",
+              "required": true,
+              "schema": { "type": "string", "title": "App Id" }
+            }
+          ],
+          "responses": {
+            "200": {
+              "description": "Successful Response",
+              "content": {
+                "application/json": {
+                  "schema": { "$ref": "#/components/schemas/DefaultResponse" }
+                }
+              }
+            },
+            "422": {
+              "description": "Validation Error",
+              "content": {
+                "application/json": {
+                  "schema": { "$ref": "#/components/schemas/HTTPValidationError" }
+                }
+              }
+            }
+          }
+        }
+      }
+    },
+    "components": {
+      "schemas": {
+        "Body_create_app_using_default_config_create_post": {
+          "properties": {
+            "config": { "type": "string", "format": "binary", "title": "Config" }
+          },
+          "type": "object",
+          "title": "Body_create_app_using_default_config_create_post"
+        },
+        "DefaultResponse": {
+          "properties": { "response": { "type": "string", "title": "Response" } },
+          "type": "object",
+          "required": ["response"],
+          "title": "DefaultResponse"
+        },
+        "DeployAppRequest": {
+          "properties": {
+            "api_key": {
+              "type": "string",
+              "title": "Api Key",
+              "description": "The Embedchain API key for app deployments. You get the api key on the Embedchain platform by visiting [https://app.embedchain.ai](https://app.embedchain.ai)",
+              "default": ""
+            }
+          },
+          "type": "object",
+          "title": "DeployAppRequest",
+          "example":{
+            "api_key":"ec-xxx"
+         }
+        },
+        "HTTPValidationError": {
+          "properties": {
+            "detail": {
+              "items": { "$ref": "#/components/schemas/ValidationError" },
+              "type": "array",
+              "title": "Detail"
+            }
+          },
+          "type": "object",
+          "title": "HTTPValidationError"
+        },
+        "MessageApp": {
+          "properties": {
+            "message": {
+              "type": "string",
+              "title": "Message",
+              "description": "The message that you want to send to the App.",
+              "default": ""
+            }
+          },
+          "type": "object",
+          "title": "MessageApp"
+        },
+        "QueryApp": {
+          "properties": {
+            "query": {
+              "type": "string",
+              "title": "Query",
+              "description": "The query that you want to ask the App.",
+              "default": ""
+            }
+          },
+          "type": "object",
+          "title": "QueryApp",
+          "example":{
+            "query":"Who is Elon Musk?"
+         }
+        },
+        "SourceApp": {
+          "properties": {
+            "source": {
+              "type": "string",
+              "title": "Source",
+              "description": "The source that you want to add to the App.",
+              "default": ""
+            },
+            "data_type": {
+              "anyOf": [{ "type": "string" }, { "type": "null" }],
+              "title": "Data Type",
+              "description": "The type of data to add, remove it if you want Embedchain to detect it automatically.",
+              "default": ""
+            }
+          },
+          "type": "object",
+          "title": "SourceApp",
+          "example":{
+            "source":"https://en.wikipedia.org/wiki/Elon_Musk"
+         }
+        },
+        "ValidationError": {
+          "properties": {
+            "loc": {
+              "items": { "anyOf": [{ "type": "string" }, { "type": "integer" }] },
+              "type": "array",
+              "title": "Location"
+            },
+            "msg": { "type": "string", "title": "Message" },
+            "type": { "type": "string", "title": "Error Type" }
+          },
+          "type": "object",
+          "required": ["loc", "msg", "type"],
+          "title": "ValidationError"
+        }
+      }
+    }
+  }
@@ -0,0 +1,22 @@
+---
+openapi: post /{app_id}/add
+---
+
+<RequestExample>
+
+```bash Request
+curl --request POST \
+  --url http://localhost:8080/{app_id}/add \
+  -d "source=https://www.forbes.com/profile/elon-musk" \
+  -d "data_type=web_page"
+```
+
+</RequestExample>
+
+<ResponseExample>
+
+```json Response
+{ "response": "fec7fe91e6b2d732938a2ec2e32bfe3f" }
+```
+
+</ResponseExample>
@@ -0,0 +1,3 @@
+---
+openapi: post /{app_id}/chat
+---
@@ -0,0 +1,20 @@
+---
+openapi: get /ping
+---
+
+<RequestExample>
+
+```bash Request
+  curl --request GET \
+    --url http://localhost:8080/ping
+```
+
+</RequestExample>
+
+<ResponseExample>
+
+```json Response
+{ "ping": "pong" }
+```
+
+</ResponseExample>
@@ -0,0 +1,95 @@
+---
+openapi: post /create
+---
+
+<RequestExample>
+
+```bash Request
+curl --request POST \
+  --url http://localhost:8080/create?app_id=app1 \
+  -F "config=@/path/to/config.yaml"
+```
+
+</RequestExample>
+
+<ResponseExample>
+
+```json Response
+{ "response": "App created successfully. App ID: app1" }
+```
+
+</ResponseExample>
+
+By default we will use the opensource **gpt4all** model to get started. You can also specify your own config by uploading a config YAML file.
+
+For example, create a `config.yaml` file (adjust according to your requirements):
+
+```yaml
+app:
+  config:
+    id: "default-app"
+
+llm:
+  provider: openai
+  config:
+    model: "gpt-3.5-turbo"
+    temperature: 0.5
+    max_tokens: 1000
+    top_p: 1
+    stream: false
+    template: |
+      Use the following pieces of context to answer the query at the end.
+      If you don't know the answer, just say that you don't know, don't try to make up an answer.
+
+      $context
+
+      Query: $query
+
+      Helpful Answer:
+
+vectordb:
+  provider: chroma
+  config:
+    collection_name: "rest-api-app"
+    dir: db
+    allow_reset: true
+
+embedder:
+  provider: openai
+  config:
+    model: "text-embedding-ada-002"
+```
+
+To learn more about custom configurations, check out the [custom configurations docs](https://docs.embedchain.ai/advanced/configuration). To explore more examples of config yamls for embedchain, visit [embedchain/configs](https://github.com/embedchain/embedchain/tree/main/configs).
+
+Now, you can upload this config file in the request body.
+
+For example,
+
+```bash Request
+curl --request POST \
+  --url http://localhost:8080/create?app_id=my-app \
+  -F "config=@/path/to/config.yaml"
+```
+
+**Note:** To use custom models, an **API key** might be required. Refer to the table below to determine the necessary API key for your provider.
+
+| Keys                       | Providers                      |
+| -------------------------- | ------------------------------ |
+| `OPENAI_API_KEY `          | OpenAI, Azure OpenAI, Jina etc |
+| `OPENAI_API_TYPE`          | Azure OpenAI                   |
+| `OPENAI_API_BASE`          | Azure OpenAI                   |
+| `OPENAI_API_VERSION`       | Azure OpenAI                   |
+| `COHERE_API_KEY`           | Cohere                         |
+| `ANTHROPIC_API_KEY`        | Anthropic                      |
+| `JINACHAT_API_KEY`         | Jina                           |
+| `HUGGINGFACE_ACCESS_TOKEN` | Huggingface                    |
+| `REPLICATE_API_TOKEN`      | LLAMA2                         |
+
+To add env variables, you can simply run the docker command with the `-e` flag.
+
+For example,
+
+```bash
+docker run --name embedchain -p 8080:8080 -e OPENAI_API_KEY=<YOUR_OPENAI_API_KEY> embedchain/rest-api:latest
+```
@@ -0,0 +1,21 @@
+---
+openapi: delete /{app_id}/delete
+---
+
+
+<RequestExample>
+
+```bash Request
+  curl --request DELETE \
+    --url http://localhost:8080/{app_id}/delete
+```
+
+</RequestExample>
+
+<ResponseExample>
+
+```json Response
+{ "response": "App with id {app_id} deleted successfully." }
+```
+
+</ResponseExample>
@@ -0,0 +1,22 @@
+---
+openapi: post /{app_id}/deploy
+---
+
+
+<RequestExample>
+
+```bash Request
+curl --request POST \
+  --url http://localhost:8080/{app_id}/deploy \
+  -d "api_key=ec-xxxx"
+```
+
+</RequestExample>
+
+<ResponseExample>
+
+```json Response
+{ "response": "App deployed successfully." }
+```
+
+</ResponseExample>
@@ -0,0 +1,33 @@
+---
+openapi: get /apps
+---
+
+<RequestExample>
+
+```bash Request
+curl --request GET \
+  --url http://localhost:8080/apps
+```
+
+</RequestExample>
+
+<ResponseExample>
+
+```json Response
+{
+  "results": [
+    {
+      "config": "config1.yaml",
+      "id": 1,
+      "app_id": "app1"
+    },
+    {
+      "config": "config2.yaml",
+      "id": 2,
+      "app_id": "app2"
+    }
+  ]
+}
+```
+
+</ResponseExample>
@@ -0,0 +1,28 @@
+---
+openapi: get /{app_id}/data
+---
+
+<RequestExample>
+
+```bash Request
+curl --request GET \
+  --url http://localhost:8080/{app_id}/data
+```
+
+</RequestExample>
+
+<ResponseExample>
+
+```json Response
+{
+  "results": [
+    {
+      "data_type": "web_page",
+      "data_value": "https://www.forbes.com/profile/elon-musk/",
+      "metadata": "null"
+    }
+  ]
+}
+```
+
+</ResponseExample>
@@ -0,0 +1,294 @@
+---
+title: "🌍 Getting Started"
+---
+
+## Quickstart
+
+To use Embedchain as a REST API service, run the following command:
+
+```bash
+docker run --name embedchain -p 8080:8080 embedchain/rest-api:latest
+```
+
+Navigate to [http://localhost:8080/docs](http://localhost:8080/docs) to interact with the API. There is a full-fledged Swagger docs playground with all the information about the API endpoints.
+
+![Swagger Docs Screenshot](https://github.com/embedchain/embedchain/assets/73601258/299d81e5-a0df-407c-afc2-6fa2c4286844)
+
+## ⚡ Steps to get started
+
+<Steps>
+  <Step title="⚙️ Create an app">
+    <Tabs>
+      <Tab title="cURL">
+      ```bash
+      curl --request POST "http://localhost:8080/create?app_id=my-app" \
+       -H "accept: application/json"
+      ```
+      </Tab>
+      <Tab title="python">
+      ```python
+      import requests
+
+      url = "http://localhost:8080/create?app_id=my-app"
+
+      payload={}
+
+      response = requests.request("POST", url, data=payload)
+
+      print(response)
+      ```
+      </Tab>
+      <Tab title="javascript">
+      ```javascript
+      const data = fetch("http://localhost:8080/create?app_id=my-app", {
+        method: "POST",
+      }).then((res) => res.json());
+
+      console.log(data);
+      ```
+      </Tab>
+      <Tab title="go">
+      ```go
+      package main
+
+      import (
+        "fmt"
+        "net/http"
+        "io/ioutil"
+      )
+
+      func main() {
+
+        url := "http://localhost:8080/create?app_id=my-app"
+
+        payload := strings.NewReader("")
+
+        req, _ := http.NewRequest("POST", url, payload)
+
+        req.Header.Add("Content-Type", "application/json")
+
+        res, _ := http.DefaultClient.Do(req)
+
+        defer res.Body.Close()
+        body, _ := ioutil.ReadAll(res.Body)
+
+        fmt.Println(res)
+        fmt.Println(string(body))
+
+      }
+      ```
+      </Tab>
+    </Tabs>
+
+  </Step>
+  <Step title="🗃️ Add data sources">
+    <Tabs>
+      <Tab title="cURL">
+        ```bash
+        curl --request POST \
+          --url http://localhost:8080/my-app/add \
+          -d "source=https://www.forbes.com/profile/elon-musk" \
+          -d "data_type=web_page"
+          ```
+      </Tab>
+      <Tab title="python">
+        ```python
+        import requests
+
+        url = "http://localhost:8080/my-app/add"
+
+        payload = "source=https://www.forbes.com/profile/elon-musk&data_type=web_page"
+        headers = {}
+
+        response = requests.request("POST", url, headers=headers, data=payload)
+
+        print(response)
+        ```
+      </Tab>
+      <Tab title="javascript">
+        ```javascript
+        const data = fetch("http://localhost:8080/my-app/add", {
+          method: "POST",
+          body: "source=https://www.forbes.com/profile/elon-musk&data_type=web_page",
+        }).then((res) => res.json());
+
+        console.log(data);
+        ```
+        </Tab>
+      <Tab title="go">
+        ```go
+        package main
+
+        import (
+          "fmt"
+          "strings"
+          "net/http"
+          "io/ioutil"
+        )
+
+        func main() {
+
+          url := "http://localhost:8080/my-app/add"
+
+          payload := strings.NewReader("source=https://www.forbes.com/profile/elon-musk&data_type=web_page")
+
+          req, _ := http.NewRequest("POST", url, payload)
+
+          req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
+
+          res, _ := http.DefaultClient.Do(req)
+
+          defer res.Body.Close()
+          body, _ := ioutil.ReadAll(res.Body)
+
+          fmt.Println(res)
+          fmt.Println(string(body))
+
+        }
+        ```
+      </Tab>
+      </Tabs>
+
+  </Step>
+  <Step title="💬 Query on your data">
+    <Tabs>
+      <Tab title="cURL">
+        ```bash
+        curl --request POST \
+          --url http://localhost:8080/my-app/query \
+          -d "query=Who is Elon Musk?"
+        ```
+      </Tab>
+      <Tab title="python">
+        ```python
+        import requests
+
+        url = "http://localhost:8080/my-app/query"
+
+        payload = "query=Who is Elon Musk?"
+        headers = {}
+
+        response = requests.request("POST", url, headers=headers, data=payload)
+
+        print(response)
+        ```
+      </Tab>
+      <Tab title="javascript">
+        ```javascript
+        const data = fetch("http://localhost:8080/my-app/query", {
+          method: "POST",
+          body: "query=Who is Elon Musk?",
+        }).then((res) => res.json());
+
+        console.log(data);
+        ```
+        </Tab>
+        <Tab title="go">
+        ```go
+        package main
+
+        import (
+          "fmt"
+          "strings"
+          "net/http"
+          "io/ioutil"
+        )
+
+        func main() {
+
+          url := "http://localhost:8080/my-app/query"
+
+          payload := strings.NewReader("query=Who is Elon Musk?")
+
+          req, _ := http.NewRequest("POST", url, payload)
+
+          req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
+
+          res, _ := http.DefaultClient.Do(req)
+
+          defer res.Body.Close()
+          body, _ := ioutil.ReadAll(res.Body)
+
+          fmt.Println(res)
+          fmt.Println(string(body))
+
+        }
+        ```
+      </Tab>
+    </Tabs>
+
+  </Step>
+  <Step title="🚀 (Optional) Deploy your app to Embedchain Platform">
+    <Tabs>
+      <Tab title="cURL">
+        ```bash
+        curl --request POST \
+          --url http://localhost:8080/my-app/deploy \
+          -d "api_key=ec-xxxx"
+          ```
+      </Tab>
+      <Tab title="python">
+        ```python
+        import requests
+
+        url = "http://localhost:8080/my-app/deploy"
+
+        payload = "api_key=ec-xxxx"
+
+        response = requests.request("POST", url, data=payload)
+
+        print(response)
+        ```
+      </Tab>
+      <Tab title="javascript">
+        ```javascript
+        const data = fetch("http://localhost:8080/my-app/deploy", {
+          method: "POST",
+          body: "api_key=ec-xxxx",
+        }).then((res) => res.json());
+
+        console.log(data);
+        ```
+      </Tab>
+      <Tab title="go">
+        ```go
+        package main
+
+        import (
+          "fmt"
+          "strings"
+          "net/http"
+          "io/ioutil"
+        )
+
+        func main() {
+
+          url := "http://localhost:8080/my-app/deploy"
+
+          payload := strings.NewReader("api_key=ec-xxxx")
+
+          req, _ := http.NewRequest("POST", url, payload)
+
+          req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
+
+          res, _ := http.DefaultClient.Do(req)
+
+          defer res.Body.Close()
+          body, _ := ioutil.ReadAll(res.Body)
+
+          fmt.Println(res)
+          fmt.Println(string(body))
+
+        }
+        ```
+      </Tab>
+    </Tabs>
+
+  </Step>
+</Steps>
+
+And you're ready! 🎉
+
+If you run into issues, please feel free to contact us using below links:
+
+<Snippet file="get-help.mdx" />
@@ -0,0 +1,21 @@
+---
+openapi: post /{app_id}/query
+---
+
+<RequestExample>
+
+```bash Request
+curl --request POST \
+  --url http://localhost:8080/{app_id}/query \
+  -d "query=who is Elon Musk?"
+```
+
+</RequestExample>
+
+<ResponseExample>
+
+```json Response
+{ "response": "Net worth of Elon Musk is $218 Billion." }
+```
+
+</ResponseExample>
@@ -0,0 +1,2 @@
+node_modules
+dist
@@ -0,0 +1,56 @@
+{
+    // Configuration for JavaScript files
+    "extends": [
+      "airbnb-base",
+      "plugin:prettier/recommended"
+    ],
+    "rules": {
+      "prettier/prettier": [
+        "error",
+        {
+          "singleQuote": true,
+          "endOfLine": "auto"
+        }
+      ]
+    },
+    "overrides": [
+      // Configuration for TypeScript files
+      {
+        "files": ["**/*.ts", "**/__tests__/*.test.ts"],
+        "plugins": [
+          "@typescript-eslint",
+          "unused-imports",
+          "simple-import-sort"
+        ],
+        "extends": [
+          "airbnb-typescript",
+          "plugin:prettier/recommended"
+        ],
+        "parserOptions": {
+          "project": "./tsconfig.json"
+        },
+        "rules": {
+          "prettier/prettier": [
+            "error",
+            {
+              "singleQuote": true,
+              "endOfLine": "auto"
+            }
+          ],
+          "@typescript-eslint/comma-dangle": "off", // Avoid conflict rule between Eslint and Prettier
+          "@typescript-eslint/consistent-type-imports": "error", // Ensure `import type` is used when it's necessary
+          "import/prefer-default-export": "off", // Named export is easier to refactor automatically
+          "simple-import-sort/imports": "error", // Import configuration for `eslint-plugin-simple-import-sort`
+          "simple-import-sort/exports": "error", // Export configuration for `eslint-plugin-simple-import-sort`
+          "@typescript-eslint/no-unused-vars": "off",
+          "react/jsx-filename-extension": "off", // Gives error
+          "unused-imports/no-unused-imports": "error",
+          "unused-imports/no-unused-vars": [
+            "error",
+            { "argsIgnorePattern": "^_" }
+          ]
+        }
+      }
+    ]
+  }
+  
@@ -0,0 +1,47 @@
+name: Node.js Package
+
+on:
+  release:
+    types: [created]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 16
+      - run: npm ci
+      - run: npm test
+      - run: npm run build
+      - uses: actions/upload-artifact@v3
+        with:
+          name: dist
+          path: dist
+      - uses: actions/upload-artifact@v3
+        with:
+          name: types
+          path: types
+
+  publish-npm:
+    needs: build
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 16
+          registry-url: https://registry.npmjs.org/
+      - uses: actions/download-artifact@v3
+        with:
+          name: dist
+          path: dist
+      - uses: actions/download-artifact@v3
+        with:
+          name: types
+          path: types
+      - run: npm ci
+      - run: npm publish
+        env:
+          NODE_AUTH_TOKEN: ${{secrets.npm_token}}
@@ -0,0 +1,138 @@
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+lerna-debug.log*
+.pnpm-debug.log*
+
+# Diagnostic reports (https://nodejs.org/api/report.html)
+report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json
+
+# Runtime data
+pids
+*.pid
+*.seed
+*.pid.lock
+
+# Directory for instrumented libs generated by jscoverage/JSCover
+lib-cov
+
+# Coverage directory used by tools like istanbul
+coverage
+*.lcov
+
+# nyc test coverage
+.nyc_output
+
+# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
+.grunt
+
+# Bower dependency directory (https://bower.io/)
+bower_components
+
+# node-waf configuration
+.lock-wscript
+
+# Compiled binary addons (https://nodejs.org/api/addons.html)
+build/Release
+
+# Dependency directories
+node_modules/
+jspm_packages/
+
+# Snowpack dependency directory (https://snowpack.dev/)
+web_modules/
+
+# TypeScript cache
+*.tsbuildinfo
+
+# Optional npm cache directory
+.npm
+
+# Optional eslint cache
+.eslintcache
+
+# Optional stylelint cache
+.stylelintcache
+
+# Microbundle cache
+.rpt2_cache/
+.rts2_cache_cjs/
+.rts2_cache_es/
+.rts2_cache_umd/
+
+# Optional REPL history
+.node_repl_history
+
+# Output of 'npm pack'
+*.tgz
+
+# Yarn Integrity file
+.yarn-integrity
+
+# dotenv environment variable files
+.env
+.env.development.local
+.env.test.local
+.env.production.local
+.env.local
+
+# parcel-bundler cache (https://parceljs.org/)
+.cache
+.parcel-cache
+
+# Next.js build output
+.next
+out
+
+# Nuxt.js build / generate output
+.nuxt
+dist
+
+# Gatsby files
+.cache/
+# Comment in the public line in if your project uses Gatsby and not Next.js
+# https://nextjs.org/blog/next-9-1#public-directory-support
+# public
+
+# vuepress build output
+.vuepress/dist
+
+# vuepress v2.x temp and cache directory
+.temp
+.cache
+
+# Docusaurus cache and generated files
+.docusaurus
+
+# Serverless directories
+.serverless/
+
+# FuseBox cache
+.fusebox/
+
+# DynamoDB Local files
+.dynamodb/
+
+# TernJS port file
+.tern-port
+
+# Stores VSCode versions used for testing VSCode extensions
+.vscode-test
+
+# yarn v2
+.yarn/cache
+.yarn/unplugged
+.yarn/build-state.yml
+.yarn/install-state.gz
+.pnp.*
+
+.ideas.md
+.todos.md
+
+# Custom
+dist
+types
+build
@@ -0,0 +1,4 @@
+#!/bin/sh
+. "$(dirname "$0")/_/husky.sh"
+
+npx --no -- commitlint --edit $1
@@ -0,0 +1,5 @@
+#!/bin/sh
+. "$(dirname "$0")/_/husky.sh"
+
+# Disable concurent to run `check-types` after ESLint in lint-staged
+npx lint-staged --concurrent false
--- a/Show More
+++ b/Show More