[BugFix] Fix issue of chunks not getting embedded in opensearch index (#983)

.github/workflows/ci.yml

@@ -3,7 +3,15 @@ name: ci
 on:
   push:
     branches: [main]
+    paths:
+      - 'embedchain/**'
+      - 'tests/**'
+      - 'examples/**'
   pull_request:
+    paths:
+      - 'embedchain/**'
+      - 'tests/**'
+      - 'examples/**'
 
 jobs:
   build:

configs/chroma.yaml

@@ -23,4 +23,3 @@ embedder:
   provider: openai
   config:
     model: 'text-embedding-ada-002'
-    deployment_name: 'test-deployment'

docs/advanced/configuration.mdx

@@ -6,15 +6,16 @@ Embedchain is made to work out of the box. However, for advanced users we're als
 
 You can configure different components of your app (`llm`, `embedding model`, or `vector database`) through a simple yaml configuration that Embedchain offers. Here is a generic full-stack example of the yaml config:
 
-```yaml
+
+<Tip>
+Embedchain applications are configurable using YAML file, JSON file or by directly passing the config dictionary.
+</Tip>
+
+<CodeGroup>
+```yaml config.yaml
 app:
   config:
-    id: 'full-stack-app'
+    name: 'full-stack-app'
-
-chunker:
-  chunk_size: 100
-  chunk_overlap: 20
-  length_function: 'len'
 
 llm:
   provider: openai
@@ -47,38 +48,138 @@ embedder:
   provider: openai
   config:
     model: 'text-embedding-ada-002'
+
+chunker:
+  chunk_size: 2000
+  chunk_overlap: 100
+  length_function: 'len'
 ```
 
+```json config.json
+{
+  "app": {
+    "config": {
+      "name": "full-stack-app"
+    }
+  },
+  "llm": {
+    "provider": "openai",
+    "config": {
+      "model": "gpt-3.5-turbo",
+      "temperature": 0.5,
+      "max_tokens": 1000,
+      "top_p": 1,
+      "stream": false,
+      "template": "Use the following pieces of context to answer the query at the end.\nIf you don't know the answer, just say that you don't know, don't try to make up an answer.\n$context\n\nQuery: $query\n\nHelpful Answer:",
+      "system_prompt": "Act as William Shakespeare. Answer the following questions in the style of William Shakespeare."
+    }
+  },
+  "vectordb": {
+    "provider": "chroma",
+    "config": {
+      "collection_name": "full-stack-app",
+      "dir": "db",
+      "allow_reset": true
+    }
+  },
+  "embedder": {
+    "provider": "openai",
+    "config": {
+      "model": "text-embedding-ada-002"
+    }
+  },
+  "chunker": {
+    "chunk_size": 2000,
+    "chunk_overlap": 100,
+    "length_function": "len"
+  }
+}
+```
+
+```python config.py
+config = {
+    'app': {
+        'config': {
+            'name': 'full-stack-app'
+        }
+    },
+    'llm': {
+        'provider': 'openai',
+        'config': {
+            'model': 'gpt-3.5-turbo',
+            'temperature': 0.5,
+            'max_tokens': 1000,
+            'top_p': 1,
+            'stream': False,
+            'template': (
+                "Use the following pieces of context to answer the query at the end.\n"
+                "If you don't know the answer, just say that you don't know, don't try to make up an answer.\n"
+                "$context\n\nQuery: $query\n\nHelpful Answer:"
+            ),
+            'system_prompt': (
+                "Act as William Shakespeare. Answer the following questions in the style of William Shakespeare."
+            )
+        }
+    },
+    'vectordb': {
+        'provider': 'chroma',
+        'config': {
+            'collection_name': 'full-stack-app',
+            'dir': 'db',
+            'allow_reset': True
+        }
+    },
+    'embedder': {
+        'provider': 'openai',
+        'config': {
+            'model': 'text-embedding-ada-002'
+        }
+    },
+    'chunker': {
+        'chunk_size': 2000,
+        'chunk_overlap': 100,
+        'length_function': 'len'
+    }
+}
+```
+</CodeGroup>
+
 Alright, let's dive into what each key means in the yaml config above:
 
 1. `app` Section:
     - `config`:
-        - `id` (String): The ID or name of your full-stack application.
+        - `name` (String): The name of your full-stack application.
-2. `chunker` Section:
+        - `id` (String): The id of your full-stack application.
-    - `chunk_size` (Integer): The size of each chunk of text that is sent to the language model.
+        <Note>Only use this to reload already created apps. We recommend users to not create their own ids.</Note>
-    - `chunk_overlap` (Integer): The amount of overlap between each chunk of text.
+        - `collect_metrics` (Boolean): Indicates whether metrics should be collected for the app, defaults to `True`
-    - `length_function` (String): The function used to calculate the length of each chunk of text. In this case, it's set to 'len'. You can also use any function import directly as a string here.
+        - `log_level` (String): The log level for the app, defaults to `WARNING`
-3. `llm` Section:
+2. `llm` Section:
     - `provider` (String): The provider for the language model, which is set to 'openai'. You can find the full list of llm providers in [our docs](/components/llms).
-    - `model` (String): The specific model being used, 'gpt-3.5-turbo'.
     - `config`:
+        - `model` (String): The specific model being used, 'gpt-3.5-turbo'.
         - `temperature` (Float): Controls the randomness of the model's output. A higher value (closer to 1) makes the output more random.
         - `max_tokens` (Integer): Controls how many tokens are used in the response.
         - `top_p` (Float): Controls the diversity of word selection. A higher value (closer to 1) makes word selection more diverse.
         - `stream` (Boolean): Controls if the response is streamed back to the user (set to false).
         - `template` (String): A custom template for the prompt that the model uses to generate responses.
         - `system_prompt` (String): A system prompt for the model to follow when generating responses, in this case, it's set to the style of William Shakespeare.
-4. `vectordb` Section:
+        -  `stream` (Boolean): Controls if the response is streamed back to the user (set to false).
+        - `number_documents` (Integer): Number of documents to pull from the vectordb as context, defaults to 1
+3. `vectordb` Section:
     - `provider` (String): The provider for the vector database, set to 'chroma'. You can find the full list of vector database providers in [our docs](/components/vector-databases).
     - `config`:
-        - `collection_name` (String): The initial collection name for the database, set to 'full-stack-app'.
+        - `collection_name` (String): The initial collection name for the vectordb, set to 'full-stack-app'.
-        - `dir` (String): The directory for the database, set to 'db'.
+        - `dir` (String): The directory for the local database, set to 'db'.
-        - `allow_reset` (Boolean): Indicates whether resetting the database is allowed, set to true.
+        - `allow_reset` (Boolean): Indicates whether resetting the vectordb is allowed, set to true.
-5. `embedder` Section:
+    <Note>We recommend you to checkout vectordb specific config [here](https://docs.embedchain.ai/components/vector-databases)</Note>
+4. `embedder` Section:
     - `provider` (String): The provider for the embedder, set to 'openai'. You can find the full list of embedding model providers in [our docs](/components/embedding-models).
     - `config`:
         - `model` (String): The specific model used for text embedding, 'text-embedding-ada-002'.
-
+5. `chunker` Section:
+    - `chunk_size` (Integer): The size of each chunk of text that is sent to the language model.
+    - `chunk_overlap` (Integer): The amount of overlap between each chunk of text.
+    - `length_function` (String): The function used to calculate the length of each chunk of text. In this case, it's set to 'len'. You can also use any function import directly as a string here.
 If you have questions about the configuration above, please feel free to reach out to us using one of the following methods:
 
 <Snippet file="get-help.mdx" />

docs/components/embedding-models.mdx

@@ -29,7 +29,7 @@ from embedchain import Pipeline as App
 os.environ['OPENAI_API_KEY'] = 'xxx'
 
 # load embedding model configuration from config.yaml file
-app = App.from_config(yaml_path="config.yaml")
+app = App.from_config(config_path="config.yaml")
 
 app.add("https://en.wikipedia.org/wiki/OpenAI")
 app.query("What is OpenAI?")
@@ -55,11 +55,11 @@ import os
 from embedchain import Pipeline as App
 
 os.environ["OPENAI_API_TYPE"] = "azure"
-os.environ["OPENAI_API_BASE"] = "https://xxx.openai.azure.com/"
+os.environ["AZURE_OPENAI_ENDPOINT"] = "https://xxx.openai.azure.com/"
-os.environ["OPENAI_API_KEY"] = "xxx"
+os.environ["AZURE_OPENAI_API_KEY"] = "xxx"
 os.environ["OPENAI_API_VERSION"] = "xxx"
 
-app = App.from_config(yaml_path="config.yaml")
+app = App.from_config(config_path="config.yaml")
 ```
 
 ```yaml config.yaml
@@ -93,7 +93,7 @@ GPT4All supports generating high quality embeddings of arbitrary length document
 from embedchain import Pipeline as App
 
 # load embedding model configuration from config.yaml file
-app = App.from_config(yaml_path="config.yaml")
+app = App.from_config(config_path="config.yaml")
 ```
 
 ```yaml config.yaml
@@ -122,7 +122,7 @@ Hugging Face supports generating embeddings of arbitrary length documents of tex
 from embedchain import Pipeline as App
 
 # load embedding model configuration from config.yaml file
-app = App.from_config(yaml_path="config.yaml")
+app = App.from_config(config_path="config.yaml")
 ```
 
 ```yaml config.yaml
@@ -153,7 +153,7 @@ Embedchain supports Google's VertexAI embeddings model through a simple interfac
 from embedchain import Pipeline as App
 
 # load embedding model configuration from config.yaml file
-app = App.from_config(yaml_path="config.yaml")
+app = App.from_config(config_path="config.yaml")
 ```
 
 ```yaml config.yaml

docs/components/llms.mdx

@@ -46,7 +46,7 @@ from embedchain import Pipeline as App
 os.environ['OPENAI_API_KEY'] = 'xxx'
 
 # load llm configuration from config.yaml file
-app = App.from_config(yaml_path="config.yaml")
+app = App.from_config(config_path="config.yaml")
 ```
 
 ```yaml config.yaml
@@ -78,7 +78,7 @@ os.environ["OPENAI_API_BASE"] = "https://xxx.openai.azure.com/"
 os.environ["OPENAI_API_KEY"] = "xxx"
 os.environ["OPENAI_API_VERSION"] = "xxx"
 
-app = App.from_config(yaml_path="config.yaml")
+app = App.from_config(config_path="config.yaml")
 ```
 
 ```yaml config.yaml
@@ -115,7 +115,7 @@ from embedchain import Pipeline as App
 os.environ["ANTHROPIC_API_KEY"] = "xxx"
 
 # load llm configuration from config.yaml file
-app = App.from_config(yaml_path="config.yaml")
+app = App.from_config(config_path="config.yaml")
 ```
 
 ```yaml config.yaml
@@ -152,7 +152,7 @@ from embedchain import Pipeline as App
 os.environ["COHERE_API_KEY"] = "xxx"
 
 # load llm configuration from config.yaml file
-app = App.from_config(yaml_path="config.yaml")
+app = App.from_config(config_path="config.yaml")
 ```
 
 ```yaml config.yaml
@@ -183,7 +183,7 @@ GPT4all is a free-to-use, locally running, privacy-aware chatbot. No GPU or inte
 from embedchain import Pipeline as App
 
 # load llm configuration from config.yaml file
-app = App.from_config(yaml_path="config.yaml")
+app = App.from_config(config_path="config.yaml")
 ```
 
 ```yaml config.yaml
@@ -216,7 +216,7 @@ from embedchain import Pipeline as App
 
 os.environ["JINACHAT_API_KEY"] = "xxx"
 # load llm configuration from config.yaml file
-app = App.from_config(yaml_path="config.yaml")
+app = App.from_config(config_path="config.yaml")
 ```
 
 ```yaml config.yaml
@@ -253,7 +253,7 @@ from embedchain import Pipeline as App
 os.environ["HUGGINGFACE_ACCESS_TOKEN"] = "xxx"
 
 # load llm configuration from config.yaml file
-app = App.from_config(yaml_path="config.yaml")
+app = App.from_config(config_path="config.yaml")
 ```
 
 ```yaml config.yaml
@@ -283,7 +283,7 @@ from embedchain import Pipeline as App
 os.environ["REPLICATE_API_TOKEN"] = "xxx"
 
 # load llm configuration from config.yaml file
-app = App.from_config(yaml_path="config.yaml")
+app = App.from_config(config_path="config.yaml")
 ```
 
 ```yaml config.yaml
@@ -308,7 +308,7 @@ Setup Google Cloud Platform application credentials by following the instruction
 from embedchain import Pipeline as App
 
 # load llm configuration from config.yaml file
-app = App.from_config(yaml_path="config.yaml")
+app = App.from_config(config_path="config.yaml")
 ```
 
 ```yaml config.yaml

docs/components/vector-databases.mdx

@@ -25,7 +25,7 @@ Utilizing a vector database alongside Embedchain is a seamless process. All you
 from embedchain import Pipeline as App
 
 # load chroma configuration from yaml file
-app = App.from_config(yaml_path="config1.yaml")
+app = App.from_config(config_path="config1.yaml")
 ```
 
 ```yaml config1.yaml
@@ -64,7 +64,7 @@ pip install --upgrade 'embedchain[elasticsearch]'
 from embedchain import Pipeline as App
 
 # load elasticsearch configuration from yaml file
-app = App.from_config(yaml_path="config.yaml")
+app = App.from_config(config_path="config.yaml")
 ```
 
 ```yaml config.yaml
@@ -73,8 +73,11 @@ vectordb:
   config:
     collection_name: 'es-index'
     es_url: http://localhost:9200
-    allow_reset: true
+    http_auth:
+      - admin
+      - admin
     api_key: xxx
+    verify_certs: false
 ```
 </CodeGroup>
 
@@ -92,19 +95,19 @@ pip install --upgrade 'embedchain[opensearch]'
 from embedchain import Pipeline as App
 
 # load opensearch configuration from yaml file
-app = App.from_config(yaml_path="config.yaml")
+app = App.from_config(config_path="config.yaml")
 ```
 
 ```yaml config.yaml
 vectordb:
   provider: opensearch
   config:
+    collection_name: 'my-app'
     opensearch_url: 'https://localhost:9200'
     http_auth:
       - admin
       - admin
     vector_dimension: 1536
-    collection_name: 'my-app'
     use_ssl: false
     verify_certs: false
 ```
@@ -131,7 +134,7 @@ os.environ['ZILLIZ_CLOUD_URI'] = 'https://xxx.zillizcloud.com'
 os.environ['ZILLIZ_CLOUD_TOKEN'] = 'xxx'
 
 # load zilliz configuration from yaml file
-app = App.from_config(yaml_path="config.yaml")
+app = App.from_config(config_path="config.yaml")
 ```
 
 ```yaml config.yaml
@@ -167,7 +170,7 @@ In order to use Pinecone as vector database, set the environment variables `PINE
 from embedchain import Pipeline as App
 
 # load pinecone configuration from yaml file
-app = App.from_config(yaml_path="config.yaml")
+app = App.from_config(config_path="config.yaml")
 ```
 
 ```yaml config.yaml
@@ -190,7 +193,7 @@ In order to use Qdrant as a vector database, set the environment variables `QDRA
 from embedchain import Pipeline as App
 
 # load qdrant configuration from yaml file
-app = App.from_config(yaml_path="config.yaml")
+app = App.from_config(config_path="config.yaml")
 ```
 
 ```yaml config.yaml
@@ -210,7 +213,7 @@ In order to use Weaviate as a vector database, set the environment variables `WE
 from embedchain import Pipeline as App
 
 # load weaviate configuration from yaml file
-app = App.from_config(yaml_path="config.yaml")
+app = App.from_config(config_path="config.yaml")
 ```
 
 ```yaml config.yaml

docs/data-sources/data-type-handling.mdx

@@ -50,3 +50,15 @@ from embedchain import Pipeline as App
 naval_chat_bot = App()
 print(naval_chat_bot.query("What unique capacity does Naval argue humans possess when it comes to understanding explanations or concepts?"))
 ```
+
+## Resetting an app and vector database
+
+You can reset the app by simply calling the `reset` method. This will delete the vector database and all other app related files.
+
+```python
+from embedchain import Pipeline as App
+
+app = App()
+app.add("https://www.youtube.com/watch?v=3qHkcs3kG44")
+app.reset()
+```

docs/data-sources/discord.mdx

@@ -0,0 +1,28 @@
+---
+title: "💬 Discord"
+---
+
+To add any Discord channel messages to your app, just add the `channel_id` as the source and set the `data_type` to `discord`.
+
+<Note>
+    This loader requires a Discord bot token with read messages access.
+    To obtain the token, follow the instructions provided in this tutorial: 
+    <a href="https://www.writebots.com/discord-bot-token/">How to Get a Discord Bot Token?</a>.
+</Note>
+
+```python
+import os
+from embedchain import Pipeline as App
+
+# add your discord "BOT" token
+os.environ["DISCORD_TOKEN"] = "xxx"
+
+app = App()
+
+app.add("1177296711023075338", data_type="discord")
+
+response = app.query("What is Joe saying about Elon Musk?")
+
+print(response)
+# Answer: Joe is saying "Elon Musk is a genius".
+```

docs/data-sources/docs-site.mdx

@@ -1,5 +1,5 @@
 ---
-title: '📚🌐 Code documentation'
+title: '📚 Code documentation'
 ---
 
 To add any code documentation website as a loader, use the data_type as `docs_site`. Eg:

docs/data-sources/overview.mdx

@@ -5,25 +5,26 @@ title: Overview
 Embedchain comes with built-in support for various data sources. We handle the complexity of loading unstructured data from these data sources, allowing you to easily customize your app through a user-friendly interface.
 
 <CardGroup cols={4}>
-  <Card title="📊 csv" href="/data-sources/csv"></Card>
+  <Card title="📊 CSV" href="/data-sources/csv"></Card>
   <Card title="📃 JSON" href="/data-sources/json"></Card>
-  <Card title="📚🌐 docs site" href="/data-sources/docs-site"></Card>
+  <Card title="📚 docs site" href="/data-sources/docs-site"></Card>
   <Card title="📄 docx" href="/data-sources/docx"></Card>
   <Card title="📝 mdx" href="/data-sources/mdx"></Card>
-  <Card title="📓 notion" href="/data-sources/notion"></Card>
+  <Card title="📓 Notion" href="/data-sources/notion"></Card>
-  <Card title="📰 pdf" href="/data-sources/pdf-file"></Card>
+  <Card title="📰 PDF" href="/data-sources/pdf-file"></Card>
   <Card title="❓💬 q&a pair" href="/data-sources/qna"></Card>
   <Card title="🗺️ sitemap" href="/data-sources/sitemap"></Card>
   <Card title="📝 text" href="/data-sources/text"></Card>
-  <Card title="🌐📄 web page" href="/data-sources/web-page"></Card>
+  <Card title="🌐 web page" href="/data-sources/web-page"></Card>
   <Card title="🧾 xml" href="/data-sources/xml"></Card>
   <Card title="🙌 OpenAPI" href="/data-sources/openapi"></Card>
-  <Card title="📺 youtube video" href="/data-sources/youtube-video"></Card>
+  <Card title="📺 Youtube" href="/data-sources/youtube-video"></Card>
   <Card title="📬 Gmail" href="/data-sources/gmail"></Card>
   <Card title="🐘 Postgres" href="/data-sources/postgres"></Card>
   <Card title="🐬 MySQL" href="/data-sources/mysql"></Card>
   <Card title="🤖 Slack" href="/data-sources/slack"></Card>
   <Card title="🗨️ Discourse" href="/data-sources/discourse"></Card>
+  <Card title="💬 Discord" href="/data-sources/discord"></Card>
 </CardGroup>
 
 <br/ >

docs/data-sources/web-page.mdx

@@ -1,5 +1,5 @@
 ---
-title: '🌐📄 Web page'
+title: '🌐 Web page'
 ---
 
 To add any web page, use the data_type as `web_page`. Eg:

docs/data-sources/youtube-video.mdx

@@ -1,5 +1,5 @@
 ---
-title: '📺 Youtube video'
+title: '📺 Youtube'
 ---
 
 

docs/get-started/examples.mdx

@@ -1,8 +1,15 @@
 ---
 title: 🔎 Examples
-description: 'Collection of Google colab notebook and Replit links for users'
 ---
 
+# Explore awesome apps
+
+Check out the remarkable work accomplished using [Embedchain](https://app.embedchain.ai/custom-gpts/).
+
+## Collection of Google colab notebook and Replit links for users
+
+Get started with Embedchain by trying out the examples below. You can run the examples in your browser using Google Colab or Replit.
+
 <table>
   <thead>
     <tr>

docs/get-started/faq.mdx

@@ -2,13 +2,36 @@
 title: ❓ FAQs
 description: 'Collections of all the frequently asked questions'
 ---
-
+<AccordionGroup>
-#### Does Embedchain support OpenAI's Assistant APIs?
+<Accordion title="Does Embedchain support OpenAI's Assistant APIs?">
-
 Yes, it does. Please refer to the [OpenAI Assistant docs page](/get-started/openai-assistant).
+</Accordion>
+<Accordion title="How to use MistralAI language model?">
+Use the model provided on huggingface: `mistralai/Mistral-7B-v0.1`
+<CodeGroup>
+```python main.py
+import os
+from embedchain import Pipeline as App
 
-#### How to use `gpt-4-turbo` model released on OpenAI DevDay?
+os.environ["OPENAI_API_KEY"] = "sk-xxx"
+os.environ["HUGGINGFACE_ACCESS_TOKEN"] = "hf_your_token"
 
+app = App.from_config("huggingface.yaml")
+```
+```yaml huggingface.yaml
+llm:
+  provider: huggingface
+  config:
+    model: 'mistralai/Mistral-7B-v0.1'
+    temperature: 0.5
+    max_tokens: 1000
+    top_p: 0.5
+    stream: false
+```
+</CodeGroup>
+</Accordion>
+<Accordion title="How to use ChatGPT 4 turbo model released on OpenAI DevDay?">
+Use the model `gpt-4-turbo` provided my openai.
 <CodeGroup>
 
 ```python main.py
@@ -18,7 +41,7 @@ from embedchain import Pipeline as App
 os.environ['OPENAI_API_KEY'] = 'xxx'
 
 # load llm configuration from gpt4_turbo.yaml file
-app = App.from_config(yaml_path="gpt4_turbo.yaml")
+app = App.from_config(config_path="gpt4_turbo.yaml")
 ```
 
 ```yaml gpt4_turbo.yaml
@@ -31,12 +54,9 @@ llm:
     top_p: 1
     stream: false
 ```
-
 </CodeGroup>
-
+</Accordion>
-
+<Accordion title="How to use GPT-4 as the LLM model?">
-#### How to use GPT-4 as the LLM model?
-
 <CodeGroup>
 
 ```python main.py
@@ -46,7 +66,7 @@ from embedchain import Pipeline as App
 os.environ['OPENAI_API_KEY'] = 'xxx'
 
 # load llm configuration from gpt4.yaml file
-app = App.from_config(yaml_path="gpt4.yaml")
+app = App.from_config(config_path="gpt4.yaml")
 ```
 
 ```yaml gpt4.yaml
@@ -61,9 +81,8 @@ llm:
 ```
 
 </CodeGroup>
-
+</Accordion>
-#### I don't have OpenAI credits. How can I use some open source model?
+<Accordion title="I don't have OpenAI credits. How can I use some open source model?">
-
 <CodeGroup>
 
 ```python main.py
@@ -73,7 +92,7 @@ from embedchain import Pipeline as App
 os.environ['OPENAI_API_KEY'] = 'xxx'
 
 # load llm configuration from opensource.yaml file
-app = App.from_config(yaml_path="opensource.yaml")
+app = App.from_config(config_path="opensource.yaml")
 ```
 
 ```yaml opensource.yaml
@@ -93,8 +112,10 @@ embedder:
 ```
 </CodeGroup>
 
-#### How to contact support?
+</Accordion>
+</AccordionGroup>
 
+#### Need more help?
 If docs aren't sufficient, please feel free to reach out to us using one of the following methods:
 
 <Snippet file="get-help.mdx" />

docs/get-started/introduction.mdx

@@ -105,7 +105,7 @@ app.deploy()
 # ✅ Data of type: web_page, value: https://www.forbes.com/profile/elon-musk added successfully.
 ```
 
-## 🚀 How it works?
+## 🛠️ How it works?
 
 Embedchain abstracts out the following steps from you to easily create LLM powered apps:
 
@@ -129,3 +129,5 @@ The process of loading the dataset and querying involves multiple steps, each wi
 - How should I find similar documents for a query? Which ranking model should I use?
 
 Embedchain takes care of all these nuances and provides a simple interface to create apps on any data.
+
+## [🚀 Get started](https://docs.embedchain.ai/get-started/quickstart)

docs/get-started/quickstart.mdx

@@ -12,79 +12,73 @@ pip install embedchain
 ```
 
 <Tip>
-Embedchain now supports OpenAI's latest `gpt-4-turbo` model. Checkout the [docs here](/get-started/faq#how-to-use-gpt-4-turbo-model-released-on-openai-devday) on how to use it.
+Embedchain now supports OpenAI's latest `gpt-4-turbo` model. Checkout the [FAQs](/get-started/faq#how-to-use-gpt-4-turbo-model-released-on-openai-devday).
 </Tip>
 
 Creating an app involves 3 steps:
 
 <Steps>
   <Step title="⚙️ Import app instance">
-```python
+    ```python
-from embedchain import Pipeline as App
+    from embedchain import Pipeline as App
-app = App()
+    app = App()
-```
+    ```
+    <Accordion title="Customize your app by a simple YAML config" icon="gear-complex">
+      Embedchain provides a wide range of options to customize your app. You can customize the model, data sources, and much more.
+      Explore the custom configurations [here](https://docs.embedchain.ai/advanced/configuration).
+      ```python
+      from embedchain import Pipeline as App
+      app = App(yaml_config="config.yaml")
+      ```
+    </Accordion>
   </Step>
   <Step title="🗃️ Add data sources">
-```python
+    ```python
-# Add different data sources
+    app.add("https://en.wikipedia.org/wiki/Elon_Musk")
-app.add("https://en.wikipedia.org/wiki/Elon_Musk")
+    app.add("https://www.forbes.com/profile/elon-musk")
-app.add("https://www.forbes.com/profile/elon-musk")
+    # app.add("path/to/file/elon_musk.pdf")
-# You can also add local data sources such as pdf, csv files etc.
+    ```
-# app.add("/path/to/file.pdf")
+    <Accordion title="Embedchain supports adding data from many data sources." icon="files">
-```
+      Embedchain supports adding data from many data sources including web pages, PDFs, databases, and more.
+      Explore the list of supported [data sources](https://docs.embedchain.ai/data-sources/overview).
+    </Accordion>
   </Step>
-  <Step title="💬 Query or chat or search context on your data">
+  <Step title="💬 Ask questions, chat, or search through your data with ease">
-```python
+    ```python
-app.query("What is the net worth of Elon Musk today?")
+    app.query("What is the net worth of Elon Musk today?")
-# Answer: The net worth of Elon Musk today is $258.7 billion.
+    # Answer: The net worth of Elon Musk today is $258.7 billion.
-```
+    ```
+    <Accordion title="Want to chat with your app?" icon="face-thinking">
+      Embedchain provides a wide range of features to interact with your app. You can chat with your app, ask questions, search through your data, and much more.
+      ```python
+      app.chat("How many companies does Elon Musk run? Name those")
+      # Answer: Elon Musk runs 3 companies: Tesla, SpaceX, and Neuralink.
+      app.chat("What is his net worth today?")
+      # Answer: The net worth of Elon Musk today is $258.7 billion.
+      ```
+      To learn about other features, click [here](https://docs.embedchain.ai/get-started/introduction)
+    </Accordion>
   </Step>
-  <Step title="🚀 (Optional) Deploy your pipeline to Embedchain Platform">
+  <Step title="🚀 Seamlessly launch your App on the Embedchain Platform!">
-```python
+    ```python
-app.deploy()
+    app.deploy()
-# 🔑 Enter your Embedchain API key. You can find the API key at https://app.embedchain.ai/settings/keys/
+    # 🔑 Enter your Embedchain API key. You can find the API key at https://app.embedchain.ai/settings/keys/
-# ec-xxxxxx
+    # ec-xxxxxx
 
-# 🛠️ Creating pipeline on the platform...
+    # 🛠️ Creating pipeline on the platform...
-# 🎉🎉🎉 Pipeline created successfully! View your pipeline: https://app.embedchain.ai/pipelines/xxxxx
+    # 🎉🎉🎉 Pipeline created successfully! View your pipeline: https://app.embedchain.ai/pipelines/xxxxx
 
-# 🛠️ Adding data to your pipeline...
+    # 🛠️ Adding data to your pipeline...
-# ✅ Data of type: web_page, value: https://www.forbes.com/profile/elon-musk added successfully.
+    # ✅ Data of type: web_page, value: https://www.forbes.com/profile/elon-musk added successfully.
-```
+    ```
+    <Accordion title="Share your app with others" icon="laptop-mobile">
+      You can now share your app with others from our platform.
+      Access your app on our [platform](https://app.embedchain.ai/).
+    </Accordion>
   </Step>
 </Steps>
 
-Putting it together, you can run your first app using the following code. Make sure to set the `OPENAI_API_KEY` 🔑 environment variable in the code.
+Putting it together, you can run your first app using the following Google Colab. Make sure to set the `OPENAI_API_KEY` 🔑 environment variable in the code.
-
-```python
-import os
-from embedchain import Pipeline as App
-
-os.environ["OPENAI_API_KEY"] = "xxx"
-app = App()
-
-# Add different data sources
-app.add("https://en.wikipedia.org/wiki/Elon_Musk")
-app.add("https://www.forbes.com/profile/elon-musk")
-# You can also add local data sources such as pdf, csv files etc.
-# app.add("/path/to/file.pdf")
-
-response = app.query("What is the net worth of Elon Musk today?")
-print(response)
-# Answer: The net worth of Elon Musk today is $258.7 billion.
-
-app.deploy()
-# 🔑 Enter your Embedchain API key. You can find the API key at https://app.embedchain.ai/settings/keys/
-# ec-xxxxxx
-
-# 🛠️ Creating pipeline on the platform...
-# 🎉🎉🎉 Pipeline created successfully! View your pipeline: https://app.embedchain.ai/pipelines/xxxxx
-
-# 🛠️ Adding data to your pipeline...
-# ✅ Data of type: web_page, value: https://www.forbes.com/profile/elon-musk added successfully.
-```
-
-You can try it out yourself using the following Google Colab notebook:
 
 <a href="https://colab.research.google.com/drive/17ON1LPonnXAtLaZEebnOktstB_1cJJmh?usp=sharing">
     <img src="https://camo.githubusercontent.com/84f0493939e0c4de4e6dbe113251b4bfb5353e57134ffd9fcab6b8714514d4d1/68747470733a2f2f636f6c61622e72657365617263682e676f6f676c652e636f6d2f6173736574732f636f6c61622d62616467652e737667" alt="Open in Colab" />

docs/mint.json

@@ -89,7 +89,8 @@
             "data-sources/openapi",
             "data-sources/youtube-video",
             "data-sources/discourse",
-            "data-sources/substack"
+            "data-sources/substack",
+            "data-sources/discord"
           ]
         },
         "data-sources/data-type-handling"

embedchain/__init__.py

@@ -6,3 +6,6 @@ from embedchain.apps.app import App  # noqa: F401
 from embedchain.client import Client  # noqa: F401
 from embedchain.pipeline import Pipeline  # noqa: F401
 from embedchain.vectordb.chroma import ChromaDB  # noqa: F401
+
+# Setup the user directory if doesn't exist already
+Client.setup_dir()

embedchain/apps/app.py

@@ -2,7 +2,6 @@ from typing import Optional
 
 import yaml
 
-from embedchain.client import Client
 from embedchain.config import (AppConfig, BaseEmbedderConfig, BaseLlmConfig,
                                ChunkerConfig)
 from embedchain.config.vectordb.base import BaseVectorDbConfig
@@ -10,10 +9,10 @@ from embedchain.embedchain import EmbedChain
 from embedchain.embedder.base import BaseEmbedder
 from embedchain.embedder.openai import OpenAIEmbedder
 from embedchain.factory import EmbedderFactory, LlmFactory, VectorDBFactory
-from embedchain.helper.json_serializable import register_deserializable
+from embedchain.helpers.json_serializable import register_deserializable
 from embedchain.llm.base import BaseLlm
 from embedchain.llm.openai import OpenAILlm
-from embedchain.utils import validate_yaml_config
+from embedchain.utils import validate_config
 from embedchain.vectordb.base import BaseVectorDB
 from embedchain.vectordb.chroma import ChromaDB
 
@@ -68,9 +67,6 @@ class App(EmbedChain):
         :type system_prompt: Optional[str], optional
         :raises TypeError: LLM, database or embedder or their config is not a valid class instance.
         """
-        # Setup user directory if it doesn't exist already
-        Client.setup_dir()
-
         # Type check configs
         if config and not isinstance(config, AppConfig):
             raise TypeError(
@@ -134,14 +130,11 @@ class App(EmbedChain):
         :return: An instance of the App class.
         :rtype: App
         """
-        # Setup user directory if it doesn't exist already
-        Client.setup_dir()
-
         with open(yaml_path, "r") as file:
             config_data = yaml.safe_load(file)
 
         try:
-            validate_yaml_config(config_data)
+            validate_config(config_data)
         except Exception as e:
             raise Exception(f"❌ Error occurred while validating the YAML config. Error: {str(e)}")
 

embedchain/bots/base.py

@@ -3,8 +3,8 @@ from typing import Any
 from embedchain import Pipeline as App
 from embedchain.config import AddConfig, BaseLlmConfig, PipelineConfig
 from embedchain.embedder.openai import OpenAIEmbedder
-from embedchain.helper.json_serializable import (JSONSerializable,
+from embedchain.helpers.json_serializable import (JSONSerializable,
-                                                 register_deserializable)
+                                                  register_deserializable)
 from embedchain.llm.openai import OpenAILlm
 from embedchain.vectordb.chroma import ChromaDB
 

embedchain/bots/discord.py

@@ -2,7 +2,7 @@ import argparse
 import logging
 import os
 
-from embedchain.helper.json_serializable import register_deserializable
+from embedchain.helpers.json_serializable import register_deserializable
 
 from .base import BaseBot
 

embedchain/bots/poe.py

@@ -3,7 +3,7 @@ import logging
 import os
 from typing import List, Optional
 
-from embedchain.helper.json_serializable import register_deserializable
+from embedchain.helpers.json_serializable import register_deserializable
 
 from .base import BaseBot
 

embedchain/bots/slack.py

@@ -5,7 +5,7 @@ import signal
 import sys
 
 from embedchain import App
-from embedchain.helper.json_serializable import register_deserializable
+from embedchain.helpers.json_serializable import register_deserializable
 
 from .base import BaseBot
 

embedchain/bots/whatsapp.py

@@ -4,7 +4,7 @@ import logging
 import signal
 import sys
 
-from embedchain.helper.json_serializable import register_deserializable
+from embedchain.helpers.json_serializable import register_deserializable
 
 from .base import BaseBot
 

embedchain/chunkers/base_chunker.py

@@ -1,6 +1,6 @@
 import hashlib
 
-from embedchain.helper.json_serializable import JSONSerializable
+from embedchain.helpers.json_serializable import JSONSerializable
 from embedchain.models.data_type import DataType
 
 

embedchain/chunkers/common_chunker.py

@@ -4,7 +4,7 @@ from langchain.text_splitter import RecursiveCharacterTextSplitter
 
 from embedchain.chunkers.base_chunker import BaseChunker
 from embedchain.config.add_config import ChunkerConfig
-from embedchain.helper.json_serializable import register_deserializable
+from embedchain.helpers.json_serializable import register_deserializable
 
 
 @register_deserializable

embedchain/chunkers/discourse.py

@@ -4,7 +4,7 @@ from langchain.text_splitter import RecursiveCharacterTextSplitter
 
 from embedchain.chunkers.base_chunker import BaseChunker
 from embedchain.config.add_config import ChunkerConfig
-from embedchain.helper.json_serializable import register_deserializable
+from embedchain.helpers.json_serializable import register_deserializable
 
 
 @register_deserializable

embedchain/chunkers/docs_site.py

@@ -4,7 +4,7 @@ from langchain.text_splitter import RecursiveCharacterTextSplitter
 
 from embedchain.chunkers.base_chunker import BaseChunker
 from embedchain.config.add_config import ChunkerConfig
-from embedchain.helper.json_serializable import register_deserializable
+from embedchain.helpers.json_serializable import register_deserializable
 
 
 @register_deserializable

embedchain/chunkers/docx_file.py

@@ -4,7 +4,7 @@ from langchain.text_splitter import RecursiveCharacterTextSplitter
 
 from embedchain.chunkers.base_chunker import BaseChunker
 from embedchain.config.add_config import ChunkerConfig
-from embedchain.helper.json_serializable import register_deserializable
+from embedchain.helpers.json_serializable import register_deserializable
 
 
 @register_deserializable

embedchain/chunkers/gmail.py

@@ -4,7 +4,7 @@ from langchain.text_splitter import RecursiveCharacterTextSplitter
 
 from embedchain.chunkers.base_chunker import BaseChunker
 from embedchain.config.add_config import ChunkerConfig
-from embedchain.helper.json_serializable import register_deserializable
+from embedchain.helpers.json_serializable import register_deserializable
 
 
 @register_deserializable

embedchain/chunkers/json.py

@@ -4,7 +4,7 @@ from langchain.text_splitter import RecursiveCharacterTextSplitter
 
 from embedchain.chunkers.base_chunker import BaseChunker
 from embedchain.config.add_config import ChunkerConfig
-from embedchain.helper.json_serializable import register_deserializable
+from embedchain.helpers.json_serializable import register_deserializable
 
 
 @register_deserializable

embedchain/chunkers/mdx.py

@@ -4,7 +4,7 @@ from langchain.text_splitter import RecursiveCharacterTextSplitter
 
 from embedchain.chunkers.base_chunker import BaseChunker
 from embedchain.config.add_config import ChunkerConfig
-from embedchain.helper.json_serializable import register_deserializable
+from embedchain.helpers.json_serializable import register_deserializable
 
 
 @register_deserializable

embedchain/chunkers/mysql.py

@@ -4,7 +4,7 @@ from langchain.text_splitter import RecursiveCharacterTextSplitter
 
 from embedchain.chunkers.base_chunker import BaseChunker
 from embedchain.config.add_config import ChunkerConfig
-from embedchain.helper.json_serializable import register_deserializable
+from embedchain.helpers.json_serializable import register_deserializable
 
 
 @register_deserializable

embedchain/chunkers/notion.py

@@ -4,7 +4,7 @@ from langchain.text_splitter import RecursiveCharacterTextSplitter
 
 from embedchain.chunkers.base_chunker import BaseChunker
 from embedchain.config.add_config import ChunkerConfig
-from embedchain.helper.json_serializable import register_deserializable
+from embedchain.helpers.json_serializable import register_deserializable
 
 
 @register_deserializable

embedchain/chunkers/pdf_file.py

@@ -4,7 +4,7 @@ from langchain.text_splitter import RecursiveCharacterTextSplitter
 
 from embedchain.chunkers.base_chunker import BaseChunker
 from embedchain.config.add_config import ChunkerConfig
-from embedchain.helper.json_serializable import register_deserializable
+from embedchain.helpers.json_serializable import register_deserializable
 
 
 @register_deserializable

embedchain/chunkers/postgres.py

@@ -4,7 +4,7 @@ from langchain.text_splitter import RecursiveCharacterTextSplitter
 
 from embedchain.chunkers.base_chunker import BaseChunker
 from embedchain.config.add_config import ChunkerConfig
-from embedchain.helper.json_serializable import register_deserializable
+from embedchain.helpers.json_serializable import register_deserializable
 
 
 @register_deserializable

embedchain/chunkers/qna_pair.py

@@ -4,7 +4,7 @@ from langchain.text_splitter import RecursiveCharacterTextSplitter
 
 from embedchain.chunkers.base_chunker import BaseChunker
 from embedchain.config.add_config import ChunkerConfig
-from embedchain.helper.json_serializable import register_deserializable
+from embedchain.helpers.json_serializable import register_deserializable
 
 
 @register_deserializable

embedchain/chunkers/sitemap.py

@@ -4,7 +4,7 @@ from langchain.text_splitter import RecursiveCharacterTextSplitter
 
 from embedchain.chunkers.base_chunker import BaseChunker
 from embedchain.config.add_config import ChunkerConfig
-from embedchain.helper.json_serializable import register_deserializable
+from embedchain.helpers.json_serializable import register_deserializable
 
 
 @register_deserializable

embedchain/chunkers/slack.py

@@ -4,7 +4,7 @@ from langchain.text_splitter import RecursiveCharacterTextSplitter
 
 from embedchain.chunkers.base_chunker import BaseChunker
 from embedchain.config.add_config import ChunkerConfig
-from embedchain.helper.json_serializable import register_deserializable
+from embedchain.helpers.json_serializable import register_deserializable
 
 
 @register_deserializable

embedchain/chunkers/substack.py

@@ -4,7 +4,7 @@ from langchain.text_splitter import RecursiveCharacterTextSplitter
 
 from embedchain.chunkers.base_chunker import BaseChunker
 from embedchain.config.add_config import ChunkerConfig
-from embedchain.helper.json_serializable import register_deserializable
+from embedchain.helpers.json_serializable import register_deserializable
 
 
 @register_deserializable

embedchain/chunkers/text.py

@@ -4,7 +4,7 @@ from langchain.text_splitter import RecursiveCharacterTextSplitter
 
 from embedchain.chunkers.base_chunker import BaseChunker
 from embedchain.config.add_config import ChunkerConfig
-from embedchain.helper.json_serializable import register_deserializable
+from embedchain.helpers.json_serializable import register_deserializable
 
 
 @register_deserializable

embedchain/chunkers/unstructured_file.py

@@ -4,7 +4,7 @@ from langchain.text_splitter import RecursiveCharacterTextSplitter
 
 from embedchain.chunkers.base_chunker import BaseChunker
 from embedchain.config.add_config import ChunkerConfig
-from embedchain.helper.json_serializable import register_deserializable
+from embedchain.helpers.json_serializable import register_deserializable
 
 
 @register_deserializable

embedchain/chunkers/web_page.py

@@ -4,7 +4,7 @@ from langchain.text_splitter import RecursiveCharacterTextSplitter
 
 from embedchain.chunkers.base_chunker import BaseChunker
 from embedchain.config.add_config import ChunkerConfig
-from embedchain.helper.json_serializable import register_deserializable
+from embedchain.helpers.json_serializable import register_deserializable
 
 
 @register_deserializable

embedchain/chunkers/xml.py

@@ -4,7 +4,7 @@ from langchain.text_splitter import RecursiveCharacterTextSplitter
 
 from embedchain.chunkers.base_chunker import BaseChunker
 from embedchain.config.add_config import ChunkerConfig
-from embedchain.helper.json_serializable import register_deserializable
+from embedchain.helpers.json_serializable import register_deserializable
 
 
 @register_deserializable

embedchain/chunkers/youtube_video.py

@@ -4,7 +4,7 @@ from langchain.text_splitter import RecursiveCharacterTextSplitter
 
 from embedchain.chunkers.base_chunker import BaseChunker
 from embedchain.config.add_config import ChunkerConfig
-from embedchain.helper.json_serializable import register_deserializable
+from embedchain.helpers.json_serializable import register_deserializable
 
 
 @register_deserializable

embedchain/config/add_config.py

@@ -3,7 +3,7 @@ from importlib import import_module
 from typing import Callable, Optional
 
 from embedchain.config.base_config import BaseConfig
-from embedchain.helper.json_serializable import register_deserializable
+from embedchain.helpers.json_serializable import register_deserializable
 
 
 @register_deserializable

embedchain/config/apps/app_config.py

@@ -1,6 +1,6 @@
 from typing import Optional
 
-from embedchain.helper.json_serializable import register_deserializable
+from embedchain.helpers.json_serializable import register_deserializable
 
 from .base_app_config import BaseAppConfig
 

embedchain/config/apps/base_app_config.py

@@ -2,7 +2,7 @@ import logging
 from typing import Optional
 
 from embedchain.config.base_config import BaseConfig
-from embedchain.helper.json_serializable import JSONSerializable
+from embedchain.helpers.json_serializable import JSONSerializable
 from embedchain.vectordb.base import BaseVectorDB
 
 

embedchain/config/base_config.py

@@ -1,6 +1,6 @@
 from typing import Any, Dict
 
-from embedchain.helper.json_serializable import JSONSerializable
+from embedchain.helpers.json_serializable import JSONSerializable
 
 
 class BaseConfig(JSONSerializable):

embedchain/config/embedder/base.py

@@ -1,6 +1,6 @@
 from typing import Optional
 
-from embedchain.helper.json_serializable import register_deserializable
+from embedchain.helpers.json_serializable import register_deserializable
 
 
 @register_deserializable

embedchain/config/llm/base.py

@@ -1,9 +1,9 @@
 import re
 from string import Template
-from typing import Any, Dict, Optional
+from typing import Any, Dict, List, Optional
 
 from embedchain.config.base_config import BaseConfig
-from embedchain.helper.json_serializable import register_deserializable
+from embedchain.helpers.json_serializable import register_deserializable
 
 DEFAULT_PROMPT = """
   Use the following pieces of context to answer the query at the end.
@@ -68,6 +68,7 @@ class BaseLlmConfig(BaseConfig):
         system_prompt: Optional[str] = None,
         where: Dict[str, Any] = None,
         query_type: Optional[str] = None,
+        callbacks: Optional[List] = None,
     ):
         """
         Initializes a configuration class instance for the LLM.
@@ -98,6 +99,8 @@ class BaseLlmConfig(BaseConfig):
         :type system_prompt: Optional[str], optional
         :param where: A dictionary of key-value pairs to filter the database results., defaults to None
         :type where: Dict[str, Any], optional
+        :param callbacks: Langchain callback functions to use, defaults to None
+        :type callbacks: Optional[List], optional
         :raises ValueError: If the template is not valid as template should
         contain $context and $query (and optionally $history)
         :raises ValueError: Stream is not boolean
@@ -113,6 +116,7 @@ class BaseLlmConfig(BaseConfig):
         self.deployment_name = deployment_name
         self.system_prompt = system_prompt
         self.query_type = query_type
+        self.callbacks = callbacks
 
         if type(template) is str:
             template = Template(template)

embedchain/config/pipeline_config.py

@@ -1,6 +1,6 @@
 from typing import Optional
 
-from embedchain.helper.json_serializable import register_deserializable
+from embedchain.helpers.json_serializable import register_deserializable
 
 from .apps.base_app_config import BaseAppConfig
 

embedchain/config/vectordb/chroma.py

@@ -1,7 +1,7 @@
 from typing import Optional
 
 from embedchain.config.vectordb.base import BaseVectorDbConfig
-from embedchain.helper.json_serializable import register_deserializable
+from embedchain.helpers.json_serializable import register_deserializable
 
 
 @register_deserializable

embedchain/config/vectordb/elasticsearch.py

@@ -2,7 +2,7 @@ import os
 from typing import Dict, List, Optional, Union
 
 from embedchain.config.vectordb.base import BaseVectorDbConfig
-from embedchain.helper.json_serializable import register_deserializable
+from embedchain.helpers.json_serializable import register_deserializable
 
 
 @register_deserializable

embedchain/config/vectordb/opensearch.py

@@ -1,7 +1,7 @@
 from typing import Dict, Optional, Tuple
 
 from embedchain.config.vectordb.base import BaseVectorDbConfig
-from embedchain.helper.json_serializable import register_deserializable
+from embedchain.helpers.json_serializable import register_deserializable
 
 
 @register_deserializable

embedchain/config/vectordb/pinecone.py

@@ -1,7 +1,7 @@
 from typing import Dict, Optional
 
 from embedchain.config.vectordb.base import BaseVectorDbConfig
-from embedchain.helper.json_serializable import register_deserializable
+from embedchain.helpers.json_serializable import register_deserializable
 
 
 @register_deserializable

embedchain/config/vectordb/qdrant.py

@@ -1,7 +1,7 @@
 from typing import Dict, Optional
 
 from embedchain.config.vectordb.base import BaseVectorDbConfig
-from embedchain.helper.json_serializable import register_deserializable
+from embedchain.helpers.json_serializable import register_deserializable
 
 
 @register_deserializable

embedchain/config/vectordb/weaviate.py

@@ -1,7 +1,7 @@
 from typing import Dict, Optional
 
 from embedchain.config.vectordb.base import BaseVectorDbConfig
-from embedchain.helper.json_serializable import register_deserializable
+from embedchain.helpers.json_serializable import register_deserializable
 
 
 @register_deserializable

embedchain/config/vectordb/zilliz.py

@@ -2,7 +2,7 @@ import os
 from typing import Optional
 
 from embedchain.config.vectordb.base import BaseVectorDbConfig
-from embedchain.helper.json_serializable import register_deserializable
+from embedchain.helpers.json_serializable import register_deserializable
 
 
 @register_deserializable

embedchain/data_formatter/data_formatter.py

@@ -4,7 +4,7 @@ from typing import Any, Dict
 from embedchain.chunkers.base_chunker import BaseChunker
 from embedchain.config import AddConfig
 from embedchain.config.add_config import ChunkerConfig, LoaderConfig
-from embedchain.helper.json_serializable import JSONSerializable
+from embedchain.helpers.json_serializable import JSONSerializable
 from embedchain.loaders.base_loader import BaseLoader
 from embedchain.models.data_type import DataType
 
@@ -66,6 +66,7 @@ class DataFormatter(JSONSerializable):
             DataType.SUBSTACK: "embedchain.loaders.substack.SubstackLoader",
             DataType.GITHUB: "embedchain.loaders.github.GithubLoader",
             DataType.YOUTUBE_CHANNEL: "embedchain.loaders.youtube_channel.YoutubeChannelLoader",
+            DataType.DISCORD: "embedchain.loaders.discord.DiscordLoader",
         }
 
         custom_loaders = set(
@@ -118,6 +119,7 @@ class DataFormatter(JSONSerializable):
             DataType.SUBSTACK: "embedchain.chunkers.substack.SubstackChunker",
             DataType.GITHUB: "embedchain.chunkers.common_chunker.CommonChunker",
             DataType.YOUTUBE_CHANNEL: "embedchain.chunkers.common_chunker.CommonChunker",
+            DataType.DISCORD: "embedchain.chunkers.common_chunker.CommonChunker",
         }
 
         if data_type in chunker_classes:

embedchain/embedchain.py

@@ -13,7 +13,7 @@ from embedchain.config.apps.base_app_config import BaseAppConfig
 from embedchain.constants import SQLITE_PATH
 from embedchain.data_formatter import DataFormatter
 from embedchain.embedder.base import BaseEmbedder
-from embedchain.helper.json_serializable import JSONSerializable
+from embedchain.helpers.json_serializable import JSONSerializable
 from embedchain.llm.base import BaseLlm
 from embedchain.loaders.base_loader import BaseLoader
 from embedchain.models.data_type import (DataType, DirectDataType,

embedchain/embedder/base.py

@@ -3,12 +3,20 @@ from typing import Any, Callable, Optional
 from embedchain.config.embedder.base import BaseEmbedderConfig
 
 try:
-    from chromadb.api.types import Documents, Embeddings
+    from chromadb.api.types import Embeddable, EmbeddingFunction, Embeddings
 except RuntimeError:
     from embedchain.utils import use_pysqlite3
 
     use_pysqlite3()
-    from chromadb.api.types import Documents, Embeddings
+    from chromadb.api.types import Embeddable, EmbeddingFunction, Embeddings
+
+
+class EmbeddingFunc(EmbeddingFunction):
+    def __init__(self, embedding_fn: Callable[[list[str]], list[str]]):
+        self.embedding_fn = embedding_fn
+
+    def __call__(self, input: Embeddable) -> Embeddings:
+        return self.embedding_fn(input)
 
 
 class BaseEmbedder:
@@ -66,7 +74,4 @@ class BaseEmbedder:
         :rtype: Callable
         """
 
-        def embed_function(texts: Documents) -> Embeddings:
+        return EmbeddingFunc(embeddings.embed_documents)
-            return embeddings.embed_documents(texts)
-
-        return embed_function

embedchain/embedder/openai.py

@@ -2,7 +2,7 @@ import os
 from typing import Optional
 
 from chromadb.utils.embedding_functions import OpenAIEmbeddingFunction
-from langchain.embeddings import OpenAIEmbeddings
+from langchain.embeddings import AzureOpenAIEmbeddings
 
 from embedchain.config import BaseEmbedderConfig
 from embedchain.embedder.base import BaseEmbedder
@@ -17,7 +17,7 @@ class OpenAIEmbedder(BaseEmbedder):
             self.config.model = "text-embedding-ada-002"
 
         if self.config.deployment_name:
-            embeddings = OpenAIEmbeddings(deployment=self.config.deployment_name)
+            embeddings = AzureOpenAIEmbeddings(deployment=self.config.deployment_name)
             embedding_fn = BaseEmbedder._langchain_default_concept(embeddings)
         else:
             if os.getenv("OPENAI_API_KEY") is None and os.getenv("OPENAI_ORGANIZATION") is None:

embedchain/helpers/callbacks.py

@@ -0,0 +1,73 @@
+import queue
+from typing import Any, Dict, List, Union
+
+from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler
+from langchain.schema import LLMResult
+
+STOP_ITEM = "[END]"
+"""
+This is a special item that is used to signal the end of the stream.
+"""
+
+
+class StreamingStdOutCallbackHandlerYield(StreamingStdOutCallbackHandler):
+    """
+    This is a callback handler that yields the tokens as they are generated.
+    For a usage example, see the :func:`generate` function below.
+    """
+
+    q: queue.Queue
+    """
+    The queue to write the tokens to as they are generated.
+    """
+
+    def __init__(self, q: queue.Queue) -> None:
+        """
+        Initialize the callback handler.
+        q: The queue to write the tokens to as they are generated.
+        """
+        super().__init__()
+        self.q = q
+
+    def on_llm_start(self, serialized: Dict[str, Any], prompts: List[str], **kwargs: Any) -> None:
+        """Run when LLM starts running."""
+        with self.q.mutex:
+            self.q.queue.clear()
+
+    def on_llm_new_token(self, token: str, **kwargs: Any) -> None:
+        """Run on new LLM token. Only available when streaming is enabled."""
+        self.q.put(token)
+
+    def on_llm_end(self, response: LLMResult, **kwargs: Any) -> None:
+        """Run when LLM ends running."""
+        self.q.put(STOP_ITEM)
+
+    def on_llm_error(self, error: Union[Exception, KeyboardInterrupt], **kwargs: Any) -> None:
+        """Run when LLM errors."""
+        self.q.put("%s: %s" % (type(error).__name__, str(error)))
+        self.q.put(STOP_ITEM)
+
+
+def generate(rq: queue.Queue):
+    """
+    This is a generator that yields the items in the queue until it reaches the stop item.
+
+    Usage example:
+    ```
+    def askQuestion(callback_fn: StreamingStdOutCallbackHandlerYield):
+        llm = OpenAI(streaming=True, callbacks=[callback_fn])
+        return llm(prompt="Write a poem about a tree.")
+
+    @app.route("/", methods=["GET"])
+    def generate_output():
+        q = Queue()
+        callback_fn = StreamingStdOutCallbackHandlerYield(q)
+        threading.Thread(target=askQuestion, args=(callback_fn,)).start()
+        return Response(generate(q), mimetype="text/event-stream")
+    ```
+    """
+    while True:
+        result: str = rq.get()
+        if result == STOP_ITEM or result is None:
+            break
+        yield result

embedchain/llm/anthropic.py

@@ -3,7 +3,7 @@ import os
 from typing import Optional
 
 from embedchain.config import BaseLlmConfig
-from embedchain.helper.json_serializable import register_deserializable
+from embedchain.helpers.json_serializable import register_deserializable
 from embedchain.llm.base import BaseLlm
 
 

embedchain/llm/azure_openai.py

@@ -2,7 +2,7 @@ import logging
 from typing import Optional
 
 from embedchain.config import BaseLlmConfig
-from embedchain.helper.json_serializable import register_deserializable
+from embedchain.helpers.json_serializable import register_deserializable
 from embedchain.llm.base import BaseLlm
 
 

embedchain/llm/base.py

@@ -7,7 +7,7 @@ from embedchain.config import BaseLlmConfig
 from embedchain.config.llm.base import (DEFAULT_PROMPT,
                                         DEFAULT_PROMPT_WITH_HISTORY_TEMPLATE,
                                         DOCS_SITE_PROMPT_TEMPLATE)
-from embedchain.helper.json_serializable import JSONSerializable
+from embedchain.helpers.json_serializable import JSONSerializable
 from embedchain.memory.base import ECChatMemory
 from embedchain.memory.message import ChatMessage
 

embedchain/llm/cohere.py

@@ -5,7 +5,7 @@ from typing import Optional
 from langchain.llms import Cohere
 
 from embedchain.config import BaseLlmConfig
-from embedchain.helper.json_serializable import register_deserializable
+from embedchain.helpers.json_serializable import register_deserializable
 from embedchain.llm.base import BaseLlm
 
 

embedchain/llm/gpt4all.py

@@ -4,7 +4,7 @@ from langchain.callbacks.stdout import StdOutCallbackHandler
 from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler
 
 from embedchain.config import BaseLlmConfig
-from embedchain.helper.json_serializable import register_deserializable
+from embedchain.helpers.json_serializable import register_deserializable
 from embedchain.llm.base import BaseLlm
 
 

embedchain/llm/huggingface.py

@@ -1,11 +1,12 @@
 import importlib
+import logging
 import os
 from typing import Optional
 
 from langchain.llms import HuggingFaceHub
 
 from embedchain.config import BaseLlmConfig
-from embedchain.helper.json_serializable import register_deserializable
+from embedchain.helpers.json_serializable import register_deserializable
 from embedchain.llm.base import BaseLlm
 
 
@@ -42,9 +43,11 @@ class HuggingFaceLlm(BaseLlm):
         else:
             raise ValueError("`top_p` must be > 0.0 and < 1.0")
 
+        model = config.model or "google/flan-t5-xxl"
+        logging.info(f"Using HuggingFaceHub with model {model}")
         llm = HuggingFaceHub(
             huggingfacehub_api_token=os.environ["HUGGINGFACE_ACCESS_TOKEN"],
-            repo_id=config.model or "google/flan-t5-xxl",
+            repo_id=model,
             model_kwargs=model_kwargs,
         )
 

embedchain/llm/jina.py

@@ -5,7 +5,7 @@ from langchain.chat_models import JinaChat
 from langchain.schema import HumanMessage, SystemMessage
 
 from embedchain.config import BaseLlmConfig
-from embedchain.helper.json_serializable import register_deserializable
+from embedchain.helpers.json_serializable import register_deserializable
 from embedchain.llm.base import BaseLlm
 
 

embedchain/llm/llama2.py

@@ -5,7 +5,7 @@ from typing import Optional
 from langchain.llms import Replicate
 
 from embedchain.config import BaseLlmConfig
-from embedchain.helper.json_serializable import register_deserializable
+from embedchain.helpers.json_serializable import register_deserializable
 from embedchain.llm.base import BaseLlm
 
 

embedchain/llm/openai.py

@@ -4,7 +4,7 @@ from langchain.chat_models import ChatOpenAI
 from langchain.schema import HumanMessage, SystemMessage
 
 from embedchain.config import BaseLlmConfig
-from embedchain.helper.json_serializable import register_deserializable
+from embedchain.helpers.json_serializable import register_deserializable
 from embedchain.llm.base import BaseLlm
 
 
@@ -34,7 +34,8 @@ class OpenAILlm(BaseLlm):
             from langchain.callbacks.streaming_stdout import \
                 StreamingStdOutCallbackHandler
 
-            chat = ChatOpenAI(**kwargs, streaming=config.stream, callbacks=[StreamingStdOutCallbackHandler()])
+            callbacks = config.callbacks if config.callbacks else [StreamingStdOutCallbackHandler()]
+            chat = ChatOpenAI(**kwargs, streaming=config.stream, callbacks=callbacks)
         else:
             chat = ChatOpenAI(**kwargs)
         return chat(messages).content

embedchain/llm/vertex_ai.py

@@ -3,7 +3,7 @@ import logging
 from typing import Optional
 
 from embedchain.config import BaseLlmConfig
-from embedchain.helper.json_serializable import register_deserializable
+from embedchain.helpers.json_serializable import register_deserializable
 from embedchain.llm.base import BaseLlm
 
 

embedchain/loaders/base_loader.py

@@ -1,4 +1,4 @@
-from embedchain.helper.json_serializable import JSONSerializable
+from embedchain.helpers.json_serializable import JSONSerializable
 
 
 class BaseLoader(JSONSerializable):

embedchain/loaders/discord.py

@@ -0,0 +1,150 @@
+import hashlib
+import logging
+import os
+
+from embedchain.helpers.json_serializable import register_deserializable
+from embedchain.loaders.base_loader import BaseLoader
+
+
+@register_deserializable
+class DiscordLoader(BaseLoader):
+    """
+    Load data from a Discord Channel ID.
+    """
+
+    def __init__(self):
+        if not os.environ.get("DISCORD_TOKEN"):
+            raise ValueError("DISCORD_TOKEN is not set")
+
+        self.token = os.environ.get("DISCORD_TOKEN")
+
+    @staticmethod
+    def _format_message(message):
+        return {
+            "message_id": message.id,
+            "content": message.content,
+            "author": {
+                "id": message.author.id,
+                "name": message.author.name,
+                "discriminator": message.author.discriminator,
+            },
+            "created_at": message.created_at.isoformat(),
+            "attachments": [
+                {
+                    "id": attachment.id,
+                    "filename": attachment.filename,
+                    "size": attachment.size,
+                    "url": attachment.url,
+                    "proxy_url": attachment.proxy_url,
+                    "height": attachment.height,
+                    "width": attachment.width,
+                }
+                for attachment in message.attachments
+            ],
+            "embeds": [
+                {
+                    "title": embed.title,
+                    "type": embed.type,
+                    "description": embed.description,
+                    "url": embed.url,
+                    "timestamp": embed.timestamp.isoformat(),
+                    "color": embed.color,
+                    "footer": {
+                        "text": embed.footer.text,
+                        "icon_url": embed.footer.icon_url,
+                        "proxy_icon_url": embed.footer.proxy_icon_url,
+                    },
+                    "image": {
+                        "url": embed.image.url,
+                        "proxy_url": embed.image.proxy_url,
+                        "height": embed.image.height,
+                        "width": embed.image.width,
+                    },
+                    "thumbnail": {
+                        "url": embed.thumbnail.url,
+                        "proxy_url": embed.thumbnail.proxy_url,
+                        "height": embed.thumbnail.height,
+                        "width": embed.thumbnail.width,
+                    },
+                    "video": {
+                        "url": embed.video.url,
+                        "height": embed.video.height,
+                        "width": embed.video.width,
+                    },
+                    "provider": {
+                        "name": embed.provider.name,
+                        "url": embed.provider.url,
+                    },
+                    "author": {
+                        "name": embed.author.name,
+                        "url": embed.author.url,
+                        "icon_url": embed.author.icon_url,
+                        "proxy_icon_url": embed.author.proxy_icon_url,
+                    },
+                    "fields": [
+                        {
+                            "name": field.name,
+                            "value": field.value,
+                            "inline": field.inline,
+                        }
+                        for field in embed.fields
+                    ],
+                }
+                for embed in message.embeds
+            ],
+        }
+
+    def load_data(self, channel_id: str):
+        """Load data from a Discord Channel ID."""
+        import discord
+
+        messages = []
+
+        class DiscordClient(discord.Client):
+            async def on_ready(self) -> None:
+                logging.info("Logged on as {0}!".format(self.user))
+                try:
+                    channel = self.get_channel(int(channel_id))
+                    if not isinstance(channel, discord.TextChannel):
+                        raise ValueError(
+                            f"Channel {channel_id} is not a text channel. " "Only text channels are supported for now."
+                        )
+                    threads = {}
+
+                    for thread in channel.threads:
+                        threads[thread.id] = thread
+
+                    async for message in channel.history(limit=None):
+                        messages.append(DiscordLoader._format_message(message))
+                        if message.id in threads:
+                            async for thread_message in threads[message.id].history(limit=None):
+                                messages.append(DiscordLoader._format_message(thread_message))
+
+                except Exception as e:
+                    logging.error(e)
+                    await self.close()
+                finally:
+                    await self.close()
+
+        intents = discord.Intents.default()
+        intents.message_content = True
+        client = DiscordClient(intents=intents)
+        client.run(self.token)
+
+        meta_data = {
+            "url": channel_id,
+        }
+
+        messages = str(messages)
+
+        doc_id = hashlib.sha256((messages + channel_id).encode()).hexdigest()
+
+        return {
+            "doc_id": doc_id,
+            "data": [
+                {
+                    "content": messages,
+                    "meta_data": meta_data,
+                }
+            ],
+        }

embedchain/loaders/docs_site_loader.py

@@ -12,7 +12,7 @@ except ImportError:
     ) from None
 
 
-from embedchain.helper.json_serializable import register_deserializable
+from embedchain.helpers.json_serializable import register_deserializable
 from embedchain.loaders.base_loader import BaseLoader
 
 

embedchain/loaders/docx_file.py

@@ -6,7 +6,7 @@ except ImportError:
     raise ImportError(
         'Docx file requires extra dependencies. Install with `pip install --upgrade "embedchain[dataloaders]"`'
     ) from None
-from embedchain.helper.json_serializable import register_deserializable
+from embedchain.helpers.json_serializable import register_deserializable
 from embedchain.loaders.base_loader import BaseLoader
 
 

embedchain/loaders/local_qna_pair.py

@@ -1,6 +1,6 @@
 import hashlib
 
-from embedchain.helper.json_serializable import register_deserializable
+from embedchain.helpers.json_serializable import register_deserializable
 from embedchain.loaders.base_loader import BaseLoader
 
 

embedchain/loaders/local_text.py

@@ -1,6 +1,6 @@
 import hashlib
 
-from embedchain.helper.json_serializable import register_deserializable
+from embedchain.helpers.json_serializable import register_deserializable
 from embedchain.loaders.base_loader import BaseLoader
 
 

embedchain/loaders/mdx.py

@@ -1,6 +1,6 @@
 import hashlib
 
-from embedchain.helper.json_serializable import register_deserializable
+from embedchain.helpers.json_serializable import register_deserializable
 from embedchain.loaders.base_loader import BaseLoader
 
 

embedchain/loaders/notion.py

@@ -10,7 +10,7 @@ except ImportError:
     ) from None
 
 
-from embedchain.helper.json_serializable import register_deserializable
+from embedchain.helpers.json_serializable import register_deserializable
 from embedchain.loaders.base_loader import BaseLoader
 from embedchain.utils import clean_string
 

embedchain/loaders/pdf_file.py

@@ -6,7 +6,7 @@ except ImportError:
     raise ImportError(
         'PDF File requires extra dependencies. Install with `pip install --upgrade "embedchain[dataloaders]"`'
     ) from None
-from embedchain.helper.json_serializable import register_deserializable
+from embedchain.helpers.json_serializable import register_deserializable
 from embedchain.loaders.base_loader import BaseLoader
 from embedchain.utils import clean_string
 

embedchain/loaders/sitemap.py

@@ -13,7 +13,7 @@ except ImportError:
         'Sitemap requires extra dependencies. Install with `pip install --upgrade "embedchain[dataloaders]"`'
     ) from None
 
-from embedchain.helper.json_serializable import register_deserializable
+from embedchain.helpers.json_serializable import register_deserializable
 from embedchain.loaders.base_loader import BaseLoader
 from embedchain.loaders.web_page import WebPageLoader
 from embedchain.utils import is_readable

embedchain/loaders/substack.py

@@ -4,7 +4,7 @@ import time
 
 import requests
 
-from embedchain.helper.json_serializable import register_deserializable
+from embedchain.helpers.json_serializable import register_deserializable
 from embedchain.loaders.base_loader import BaseLoader
 from embedchain.utils import is_readable
 

embedchain/loaders/unstructured_file.py

@@ -1,6 +1,6 @@
 import hashlib
 
-from embedchain.helper.json_serializable import register_deserializable
+from embedchain.helpers.json_serializable import register_deserializable
 from embedchain.loaders.base_loader import BaseLoader
 from embedchain.utils import clean_string
 

embedchain/loaders/web_page.py

@@ -10,7 +10,7 @@ except ImportError:
         'Webpage requires extra dependencies. Install with `pip install --upgrade "embedchain[dataloaders]"`'
     ) from None
 
-from embedchain.helper.json_serializable import register_deserializable
+from embedchain.helpers.json_serializable import register_deserializable
 from embedchain.loaders.base_loader import BaseLoader
 from embedchain.utils import clean_string
 

embedchain/loaders/xml.py

@@ -6,7 +6,7 @@ except ImportError:
     raise ImportError(
         'XML file requires extra dependencies. Install with `pip install --upgrade "embedchain[dataloaders]"`'
     ) from None
-from embedchain.helper.json_serializable import register_deserializable
+from embedchain.helpers.json_serializable import register_deserializable
 from embedchain.loaders.base_loader import BaseLoader
 from embedchain.utils import clean_string
 

embedchain/loaders/youtube_video.py

@@ -6,7 +6,7 @@ except ImportError:
     raise ImportError(
         'YouTube video requires extra dependencies. Install with `pip install --upgrade "embedchain[dataloaders]"`'
     ) from None
-from embedchain.helper.json_serializable import register_deserializable
+from embedchain.helpers.json_serializable import register_deserializable
 from embedchain.loaders.base_loader import BaseLoader
 from embedchain.utils import clean_string
 

embedchain/memory/base.py

@@ -23,7 +23,7 @@ CHAT_MESSAGE_CREATE_TABLE_QUERY = """
 
 class ECChatMemory:
     def __init__(self) -> None:
-        with sqlite3.connect(SQLITE_PATH) as self.connection:
+        with sqlite3.connect(SQLITE_PATH, check_same_thread=False) as self.connection:
             self.cursor = self.connection.cursor()
 
             self.cursor.execute(CHAT_MESSAGE_CREATE_TABLE_QUERY)

embedchain/memory/message.py

@@ -1,7 +1,7 @@
 import logging
 from typing import Any, Dict, Optional
 
-from embedchain.helper.json_serializable import JSONSerializable
+from embedchain.helpers.json_serializable import JSONSerializable
 
 
 class BaseMessage(JSONSerializable):

embedchain/models/data_type.py

@@ -36,6 +36,7 @@ class IndirectDataType(Enum):
     SUBSTACK = "substack"
     GITHUB = "github"
     YOUTUBE_CHANNEL = "youtube_channel"
+    DISCORD = "discord"
 
 
 class SpecialDataType(Enum):
@@ -71,3 +72,4 @@ class DataType(Enum):
     SUBSTACK = IndirectDataType.SUBSTACK.value
     GITHUB = IndirectDataType.GITHUB.value
     YOUTUBE_CHANNEL = IndirectDataType.YOUTUBE_CHANNEL.value
+    DISCORD = IndirectDataType.DISCORD.value

embedchain/pipeline.py

@@ -4,6 +4,7 @@ import logging
 import os
 import sqlite3
 import uuid
+from typing import Any, Dict, Optional
 
 import requests
 import yaml
@@ -15,14 +16,17 @@ from embedchain.embedchain import EmbedChain
 from embedchain.embedder.base import BaseEmbedder
 from embedchain.embedder.openai import OpenAIEmbedder
 from embedchain.factory import EmbedderFactory, LlmFactory, VectorDBFactory
-from embedchain.helper.json_serializable import register_deserializable
+from embedchain.helpers.json_serializable import register_deserializable
 from embedchain.llm.base import BaseLlm
 from embedchain.llm.openai import OpenAILlm
 from embedchain.telemetry.posthog import AnonymousTelemetry
-from embedchain.utils import validate_yaml_config
+from embedchain.utils import validate_config
 from embedchain.vectordb.base import BaseVectorDB
 from embedchain.vectordb.chroma import ChromaDB
 
+# Setup the user directory if doesn't exist already
+Client.setup_dir()
+
 
 @register_deserializable
 class Pipeline(EmbedChain):
@@ -40,7 +44,7 @@ class Pipeline(EmbedChain):
         db: BaseVectorDB = None,
         embedding_model: BaseEmbedder = None,
         llm: BaseLlm = None,
-        yaml_path: str = None,
+        config_data: dict = None,
         log_level=logging.WARN,
         auto_deploy: bool = False,
         chunker: ChunkerConfig = None,
@@ -56,18 +60,15 @@ class Pipeline(EmbedChain):
         :type embedding_model: BaseEmbedder, optional
         :param llm: The LLM model used to calculate embeddings, defaults to None
         :type llm: BaseLlm, optional
-        :param yaml_path: Path to the YAML configuration file, defaults to None
+        :param config_data: Config dictionary, defaults to None
-        :type yaml_path: str, optional
+        :type config_data: dict, optional
         :param log_level: Log level to use, defaults to logging.WARN
         :type log_level: int, optional
         :param auto_deploy: Whether to deploy the pipeline automatically, defaults to False
         :type auto_deploy: bool, optional
         :raises Exception: If an error occurs while creating the pipeline
         """
-        # Setup user directory if it doesn't exist already
+        if id and config_data:
-        Client.setup_dir()
-
-        if id and yaml_path:
             raise Exception("Cannot provide both id and config. Please provide only one of them.")
 
         if id and name:
@@ -79,8 +80,8 @@ class Pipeline(EmbedChain):
         logging.basicConfig(level=log_level, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s")
         self.logger = logging.getLogger(__name__)
         self.auto_deploy = auto_deploy
-        # Store the yaml config as an attribute to be able to send it
+        # Store the dict config as an attribute to be able to send it
-        self.yaml_config = None
+        self.config_data = config_data if (config_data and validate_config(config_data)) else None
         self.client = None
         # pipeline_id from the backend
         self.id = None
@@ -92,11 +93,6 @@ class Pipeline(EmbedChain):
         self.name = self.config.name
         self.config.id = self.local_id = str(uuid.uuid4()) if self.config.id is None else self.config.id
 
-        if yaml_path:
-            with open(yaml_path, "r") as file:
-                config_data = yaml.safe_load(file)
-                self.yaml_config = config_data
-
         if id is not None:
             # Init client first since user is trying to fetch the pipeline
             # details from the platform
@@ -187,9 +183,9 @@ class Pipeline(EmbedChain):
         Create a pipeline on the platform.
         """
         print("🛠️ Creating pipeline on the platform...")
-        # self.yaml_config is a dict. Pass it inside the key 'yaml_config' to the backend
+        # self.config_data is a dict. Pass it inside the key 'yaml_config' to the backend
         payload = {
-            "yaml_config": json.dumps(self.yaml_config),
+            "yaml_config": json.dumps(self.config_data),
             "name": self.name,
             "local_id": self.local_id,
         }
@@ -346,27 +342,57 @@ class Pipeline(EmbedChain):
         self.telemetry.capture(event_name="deploy", properties=self._telemetry_props)
 
     @classmethod
-    def from_config(cls, yaml_path: str, auto_deploy: bool = False):
+    def from_config(
+        cls,
+        config_path: Optional[str] = None,
+        config: Optional[Dict[str, Any]] = None,
+        auto_deploy: bool = False,
+        yaml_path: Optional[str] = None,
+    ):
         """
-        Instantiate a Pipeline object from a YAML configuration file.
+        Instantiate a Pipeline object from a configuration.
 
-        :param yaml_path: Path to the YAML configuration file.
+        :param config_path: Path to the YAML or JSON configuration file.
-        :type yaml_path: str
+        :type config_path: Optional[str]
+        :param config: A dictionary containing the configuration.
+        :type config: Optional[Dict[str, Any]]
         :param auto_deploy: Whether to deploy the pipeline automatically, defaults to False
         :type auto_deploy: bool, optional
+        :param yaml_path: (Deprecated) Path to the YAML configuration file. Use config_path instead.
+        :type yaml_path: Optional[str]
         :return: An instance of the Pipeline class.
         :rtype: Pipeline
         """
-        # Setup user directory if it doesn't exist already
+        # Backward compatibility for yaml_path
-        Client.setup_dir()
+        if yaml_path and not config_path:
+            config_path = yaml_path
 
-        with open(yaml_path, "r") as file:
+        if config_path and config:
-            config_data = yaml.safe_load(file)
+            raise ValueError("Please provide only one of config_path or config.")
+
+        config_data = None
+
+        if config_path:
+            file_extension = os.path.splitext(config_path)[1]
+            with open(config_path, "r") as file:
+                if file_extension in [".yaml", ".yml"]:
+                    config_data = yaml.safe_load(file)
+                elif file_extension == ".json":
+                    config_data = json.load(file)
+                else:
+                    raise ValueError("config_path must be a path to a YAML or JSON file.")
+        elif config and isinstance(config, dict):
+            config_data = config
+        else:
+            logging.error(
+                "Please provide either a config file path (YAML or JSON) or a config dictionary. Falling back to defaults because no config is provided.",  # noqa: E501
+            )
+            config_data = {}
 
         try:
-            validate_yaml_config(config_data)
+            validate_config(config_data)
         except Exception as e:
-            raise Exception(f"❌ Error occurred while validating the YAML config. Error: {str(e)}")
+            raise Exception(f"Error occurred while validating the config. Error: {str(e)}")
 
         pipeline_config_data = config_data.get("app", {}).get("config", {})
         db_config_data = config_data.get("vectordb", {})
@@ -391,7 +417,7 @@ class Pipeline(EmbedChain):
         )
 
         # Send anonymous telemetry
-        event_properties = {"init_type": "yaml_config"}
+        event_properties = {"init_type": "config_data"}
         AnonymousTelemetry().capture(event_name="init", properties=event_properties)
 
         return cls(
@@ -399,7 +425,7 @@ class Pipeline(EmbedChain):
             llm=llm,
             db=db,
             embedding_model=embedding_model,
-            yaml_path=yaml_path,
+            config_data=config_data,
             auto_deploy=auto_deploy,
             chunker=chunker_config_data,
         )

embedchain/store/assistants.py

@@ -10,7 +10,7 @@ from typing import cast
 from openai import OpenAI
 from openai.types.beta.threads import MessageContentText, ThreadMessage
 
-from embedchain import Pipeline
+from embedchain import Client, Pipeline
 from embedchain.config import AddConfig
 from embedchain.data_formatter import DataFormatter
 from embedchain.models.data_type import DataType
@@ -19,6 +19,9 @@ from embedchain.utils import detect_datatype
 
 logging.basicConfig(level=logging.WARN)
 
+# Setup the user directory if doesn't exist already
+Client.setup_dir()
+
 
 class OpenAIAssistant:
     def __init__(
@@ -162,7 +165,7 @@ class AIAssistant:
         self.instructions = instructions
         self.assistant_id = assistant_id or str(uuid.uuid4())
         self.thread_id = thread_id or str(uuid.uuid4())
-        self.pipeline = Pipeline.from_config(yaml_path=yaml_path) if yaml_path else Pipeline()
+        self.pipeline = Pipeline.from_config(config_path=yaml_path) if yaml_path else Pipeline()
         self.pipeline.local_id = self.pipeline.config.id = self.thread_id
 
         if self.instructions:

embedchain/utils.py

@@ -355,7 +355,7 @@ def is_valid_json_string(source: str):
         return False
 
 
-def validate_yaml_config(config_data):
+def validate_config(config_data):
     schema = Schema(
         {
             Optional("app"): {

embedchain/vectordb/base.py

@@ -1,6 +1,6 @@
 from embedchain.config.vectordb.base import BaseVectorDbConfig
 from embedchain.embedder.base import BaseEmbedder
-from embedchain.helper.json_serializable import JSONSerializable
+from embedchain.helpers.json_serializable import JSONSerializable
 
 
 class BaseVectorDB(JSONSerializable):

embedchain/vectordb/chroma.py

@@ -6,7 +6,7 @@ from langchain.docstore.document import Document
 from tqdm import tqdm
 
 from embedchain.config import ChromaDbConfig
-from embedchain.helper.json_serializable import register_deserializable
+from embedchain.helpers.json_serializable import register_deserializable
 from embedchain.vectordb.base import BaseVectorDB
 
 try:

embedchain/vectordb/elasticsearch.py

@@ -10,7 +10,7 @@ except ImportError:
     ) from None
 
 from embedchain.config import ElasticsearchDBConfig
-from embedchain.helper.json_serializable import register_deserializable
+from embedchain.helpers.json_serializable import register_deserializable
 from embedchain.vectordb.base import BaseVectorDB