Bump package version to 0.1.27 (#994)

Co-authored-by: Deven Patel <deven298@yahoo.com>

docs/advanced/configuration.mdx

@@ -6,15 +6,16 @@ Embedchain is made to work out of the box. However, for advanced users we're als
 
 You can configure different components of your app (`llm`, `embedding model`, or `vector database`) through a simple yaml configuration that Embedchain offers. Here is a generic full-stack example of the yaml config:
 
-```yaml
+
+<Tip>
+Embedchain applications are configurable using YAML file, JSON file or by directly passing the config dictionary.
+</Tip>
+
+<CodeGroup>
+```yaml config.yaml
 app:
   config:
-    id: 'full-stack-app'
+    name: 'full-stack-app'
-
-chunker:
-  chunk_size: 100
-  chunk_overlap: 20
-  length_function: 'len'
 
 llm:
   provider: openai
@@ -47,38 +48,138 @@ embedder:
   provider: openai
   config:
     model: 'text-embedding-ada-002'
+
+chunker:
+  chunk_size: 2000
+  chunk_overlap: 100
+  length_function: 'len'
 ```
 
+```json config.json
+{
+  "app": {
+    "config": {
+      "name": "full-stack-app"
+    }
+  },
+  "llm": {
+    "provider": "openai",
+    "config": {
+      "model": "gpt-3.5-turbo",
+      "temperature": 0.5,
+      "max_tokens": 1000,
+      "top_p": 1,
+      "stream": false,
+      "template": "Use the following pieces of context to answer the query at the end.\nIf you don't know the answer, just say that you don't know, don't try to make up an answer.\n$context\n\nQuery: $query\n\nHelpful Answer:",
+      "system_prompt": "Act as William Shakespeare. Answer the following questions in the style of William Shakespeare."
+    }
+  },
+  "vectordb": {
+    "provider": "chroma",
+    "config": {
+      "collection_name": "full-stack-app",
+      "dir": "db",
+      "allow_reset": true
+    }
+  },
+  "embedder": {
+    "provider": "openai",
+    "config": {
+      "model": "text-embedding-ada-002"
+    }
+  },
+  "chunker": {
+    "chunk_size": 2000,
+    "chunk_overlap": 100,
+    "length_function": "len"
+  }
+}
+```
+
+```python config.py
+config = {
+    'app': {
+        'config': {
+            'name': 'full-stack-app'
+        }
+    },
+    'llm': {
+        'provider': 'openai',
+        'config': {
+            'model': 'gpt-3.5-turbo',
+            'temperature': 0.5,
+            'max_tokens': 1000,
+            'top_p': 1,
+            'stream': False,
+            'template': (
+                "Use the following pieces of context to answer the query at the end.\n"
+                "If you don't know the answer, just say that you don't know, don't try to make up an answer.\n"
+                "$context\n\nQuery: $query\n\nHelpful Answer:"
+            ),
+            'system_prompt': (
+                "Act as William Shakespeare. Answer the following questions in the style of William Shakespeare."
+            )
+        }
+    },
+    'vectordb': {
+        'provider': 'chroma',
+        'config': {
+            'collection_name': 'full-stack-app',
+            'dir': 'db',
+            'allow_reset': True
+        }
+    },
+    'embedder': {
+        'provider': 'openai',
+        'config': {
+            'model': 'text-embedding-ada-002'
+        }
+    },
+    'chunker': {
+        'chunk_size': 2000,
+        'chunk_overlap': 100,
+        'length_function': 'len'
+    }
+}
+```
+</CodeGroup>
+
 Alright, let's dive into what each key means in the yaml config above:
 
 1. `app` Section:
     - `config`:
-        - `id` (String): The ID or name of your full-stack application.
+        - `name` (String): The name of your full-stack application.
-2. `chunker` Section:
+        - `id` (String): The id of your full-stack application.
-    - `chunk_size` (Integer): The size of each chunk of text that is sent to the language model.
+        <Note>Only use this to reload already created apps. We recommend users to not create their own ids.</Note>
-    - `chunk_overlap` (Integer): The amount of overlap between each chunk of text.
+        - `collect_metrics` (Boolean): Indicates whether metrics should be collected for the app, defaults to `True`
-    - `length_function` (String): The function used to calculate the length of each chunk of text. In this case, it's set to 'len'. You can also use any function import directly as a string here.
+        - `log_level` (String): The log level for the app, defaults to `WARNING`
-3. `llm` Section:
+2. `llm` Section:
     - `provider` (String): The provider for the language model, which is set to 'openai'. You can find the full list of llm providers in [our docs](/components/llms).
-    - `model` (String): The specific model being used, 'gpt-3.5-turbo'.
     - `config`:
+        - `model` (String): The specific model being used, 'gpt-3.5-turbo'.
         - `temperature` (Float): Controls the randomness of the model's output. A higher value (closer to 1) makes the output more random.
         - `max_tokens` (Integer): Controls how many tokens are used in the response.
         - `top_p` (Float): Controls the diversity of word selection. A higher value (closer to 1) makes word selection more diverse.
         - `stream` (Boolean): Controls if the response is streamed back to the user (set to false).
         - `template` (String): A custom template for the prompt that the model uses to generate responses.
         - `system_prompt` (String): A system prompt for the model to follow when generating responses, in this case, it's set to the style of William Shakespeare.
-4. `vectordb` Section:
+        -  `stream` (Boolean): Controls if the response is streamed back to the user (set to false).
+        - `number_documents` (Integer): Number of documents to pull from the vectordb as context, defaults to 1
+3. `vectordb` Section:
     - `provider` (String): The provider for the vector database, set to 'chroma'. You can find the full list of vector database providers in [our docs](/components/vector-databases).
     - `config`:
-        - `collection_name` (String): The initial collection name for the database, set to 'full-stack-app'.
+        - `collection_name` (String): The initial collection name for the vectordb, set to 'full-stack-app'.
-        - `dir` (String): The directory for the database, set to 'db'.
+        - `dir` (String): The directory for the local database, set to 'db'.
-        - `allow_reset` (Boolean): Indicates whether resetting the database is allowed, set to true.
+        - `allow_reset` (Boolean): Indicates whether resetting the vectordb is allowed, set to true.
-5. `embedder` Section:
+    <Note>We recommend you to checkout vectordb specific config [here](https://docs.embedchain.ai/components/vector-databases)</Note>
+4. `embedder` Section:
     - `provider` (String): The provider for the embedder, set to 'openai'. You can find the full list of embedding model providers in [our docs](/components/embedding-models).
     - `config`:
         - `model` (String): The specific model used for text embedding, 'text-embedding-ada-002'.
-
+5. `chunker` Section:
+    - `chunk_size` (Integer): The size of each chunk of text that is sent to the language model.
+    - `chunk_overlap` (Integer): The amount of overlap between each chunk of text.
+    - `length_function` (String): The function used to calculate the length of each chunk of text. In this case, it's set to 'len'. You can also use any function import directly as a string here.
 If you have questions about the configuration above, please feel free to reach out to us using one of the following methods:
 
 <Snippet file="get-help.mdx" />

docs/components/embedding-models.mdx

@@ -29,7 +29,7 @@ from embedchain import Pipeline as App
 os.environ['OPENAI_API_KEY'] = 'xxx'
 
 # load embedding model configuration from config.yaml file
-app = App.from_config(yaml_path="config.yaml")
+app = App.from_config(config_path="config.yaml")
 
 app.add("https://en.wikipedia.org/wiki/OpenAI")
 app.query("What is OpenAI?")
@@ -59,7 +59,7 @@ os.environ["AZURE_OPENAI_ENDPOINT"] = "https://xxx.openai.azure.com/"
 os.environ["AZURE_OPENAI_API_KEY"] = "xxx"
 os.environ["OPENAI_API_VERSION"] = "xxx"
 
-app = App.from_config(yaml_path="config.yaml")
+app = App.from_config(config_path="config.yaml")
 ```
 
 ```yaml config.yaml
@@ -93,7 +93,7 @@ GPT4All supports generating high quality embeddings of arbitrary length document
 from embedchain import Pipeline as App
 
 # load embedding model configuration from config.yaml file
-app = App.from_config(yaml_path="config.yaml")
+app = App.from_config(config_path="config.yaml")
 ```
 
 ```yaml config.yaml
@@ -122,7 +122,7 @@ Hugging Face supports generating embeddings of arbitrary length documents of tex
 from embedchain import Pipeline as App
 
 # load embedding model configuration from config.yaml file
-app = App.from_config(yaml_path="config.yaml")
+app = App.from_config(config_path="config.yaml")
 ```
 
 ```yaml config.yaml
@@ -153,7 +153,7 @@ Embedchain supports Google's VertexAI embeddings model through a simple interfac
 from embedchain import Pipeline as App
 
 # load embedding model configuration from config.yaml file
-app = App.from_config(yaml_path="config.yaml")
+app = App.from_config(config_path="config.yaml")
 ```
 
 ```yaml config.yaml

docs/components/llms.mdx

@@ -46,7 +46,7 @@ from embedchain import Pipeline as App
 os.environ['OPENAI_API_KEY'] = 'xxx'
 
 # load llm configuration from config.yaml file
-app = App.from_config(yaml_path="config.yaml")
+app = App.from_config(config_path="config.yaml")
 ```
 
 ```yaml config.yaml
@@ -78,7 +78,7 @@ os.environ["OPENAI_API_BASE"] = "https://xxx.openai.azure.com/"
 os.environ["OPENAI_API_KEY"] = "xxx"
 os.environ["OPENAI_API_VERSION"] = "xxx"
 
-app = App.from_config(yaml_path="config.yaml")
+app = App.from_config(config_path="config.yaml")
 ```
 
 ```yaml config.yaml
@@ -115,7 +115,7 @@ from embedchain import Pipeline as App
 os.environ["ANTHROPIC_API_KEY"] = "xxx"
 
 # load llm configuration from config.yaml file
-app = App.from_config(yaml_path="config.yaml")
+app = App.from_config(config_path="config.yaml")
 ```
 
 ```yaml config.yaml
@@ -152,7 +152,7 @@ from embedchain import Pipeline as App
 os.environ["COHERE_API_KEY"] = "xxx"
 
 # load llm configuration from config.yaml file
-app = App.from_config(yaml_path="config.yaml")
+app = App.from_config(config_path="config.yaml")
 ```
 
 ```yaml config.yaml
@@ -183,7 +183,7 @@ GPT4all is a free-to-use, locally running, privacy-aware chatbot. No GPU or inte
 from embedchain import Pipeline as App
 
 # load llm configuration from config.yaml file
-app = App.from_config(yaml_path="config.yaml")
+app = App.from_config(config_path="config.yaml")
 ```
 
 ```yaml config.yaml
@@ -216,7 +216,7 @@ from embedchain import Pipeline as App
 
 os.environ["JINACHAT_API_KEY"] = "xxx"
 # load llm configuration from config.yaml file
-app = App.from_config(yaml_path="config.yaml")
+app = App.from_config(config_path="config.yaml")
 ```
 
 ```yaml config.yaml
@@ -253,7 +253,7 @@ from embedchain import Pipeline as App
 os.environ["HUGGINGFACE_ACCESS_TOKEN"] = "xxx"
 
 # load llm configuration from config.yaml file
-app = App.from_config(yaml_path="config.yaml")
+app = App.from_config(config_path="config.yaml")
 ```
 
 ```yaml config.yaml
@@ -283,7 +283,7 @@ from embedchain import Pipeline as App
 os.environ["REPLICATE_API_TOKEN"] = "xxx"
 
 # load llm configuration from config.yaml file
-app = App.from_config(yaml_path="config.yaml")
+app = App.from_config(config_path="config.yaml")
 ```
 
 ```yaml config.yaml
@@ -308,7 +308,7 @@ Setup Google Cloud Platform application credentials by following the instruction
 from embedchain import Pipeline as App
 
 # load llm configuration from config.yaml file
-app = App.from_config(yaml_path="config.yaml")
+app = App.from_config(config_path="config.yaml")
 ```
 
 ```yaml config.yaml

docs/components/vector-databases.mdx

@@ -25,7 +25,7 @@ Utilizing a vector database alongside Embedchain is a seamless process. All you
 from embedchain import Pipeline as App
 
 # load chroma configuration from yaml file
-app = App.from_config(yaml_path="config1.yaml")
+app = App.from_config(config_path="config1.yaml")
 ```
 
 ```yaml config1.yaml
@@ -64,7 +64,7 @@ pip install --upgrade 'embedchain[elasticsearch]'
 from embedchain import Pipeline as App
 
 # load elasticsearch configuration from yaml file
-app = App.from_config(yaml_path="config.yaml")
+app = App.from_config(config_path="config.yaml")
 ```
 
 ```yaml config.yaml
@@ -73,8 +73,11 @@ vectordb:
   config:
     collection_name: 'es-index'
     es_url: http://localhost:9200
-    allow_reset: true
+    http_auth:
+      - admin
+      - admin
     api_key: xxx
+    verify_certs: false
 ```
 </CodeGroup>
 
@@ -92,19 +95,19 @@ pip install --upgrade 'embedchain[opensearch]'
 from embedchain import Pipeline as App
 
 # load opensearch configuration from yaml file
-app = App.from_config(yaml_path="config.yaml")
+app = App.from_config(config_path="config.yaml")
 ```
 
 ```yaml config.yaml
 vectordb:
   provider: opensearch
   config:
+    collection_name: 'my-app'
     opensearch_url: 'https://localhost:9200'
     http_auth:
       - admin
       - admin
     vector_dimension: 1536
-    collection_name: 'my-app'
     use_ssl: false
     verify_certs: false
 ```
@@ -131,7 +134,7 @@ os.environ['ZILLIZ_CLOUD_URI'] = 'https://xxx.zillizcloud.com'
 os.environ['ZILLIZ_CLOUD_TOKEN'] = 'xxx'
 
 # load zilliz configuration from yaml file
-app = App.from_config(yaml_path="config.yaml")
+app = App.from_config(config_path="config.yaml")
 ```
 
 ```yaml config.yaml
@@ -167,7 +170,7 @@ In order to use Pinecone as vector database, set the environment variables `PINE
 from embedchain import Pipeline as App
 
 # load pinecone configuration from yaml file
-app = App.from_config(yaml_path="config.yaml")
+app = App.from_config(config_path="config.yaml")
 ```
 
 ```yaml config.yaml
@@ -190,7 +193,7 @@ In order to use Qdrant as a vector database, set the environment variables `QDRA
 from embedchain import Pipeline as App
 
 # load qdrant configuration from yaml file
-app = App.from_config(yaml_path="config.yaml")
+app = App.from_config(config_path="config.yaml")
 ```
 
 ```yaml config.yaml
@@ -210,7 +213,7 @@ In order to use Weaviate as a vector database, set the environment variables `WE
 from embedchain import Pipeline as App
 
 # load weaviate configuration from yaml file
-app = App.from_config(yaml_path="config.yaml")
+app = App.from_config(config_path="config.yaml")
 ```
 
 ```yaml config.yaml

docs/data-sources/custom.mdx

@@ -0,0 +1,41 @@
+---
+title: '⚙️ Custom'
+---
+
+When we say "custom", we mean that you can customize the loader and chunker to your needs. This is done by passing a custom loader and chunker to the `add` method.
+
+```python
+from embedchain import Pipeline as App
+import your_loader
+import your_chunker
+
+app = App()
+loader = your_loader()
+chunker = your_chunker()
+
+app.add("source", data_type="custom", loader=loader, chunker=chunker)
+```
+
+<Note>
+    The custom loader and chunker must be a class that inherits from the [`BaseLoader`](https://github.com/embedchain/embedchain/blob/main/embedchain/loaders/base_loader.py) and [`BaseChunker`](https://github.com/embedchain/embedchain/blob/main/embedchain/chunkers/base_chunker.py) classes respectively.
+</Note>
+
+<Note>
+    If the `data_type` is not a valid data type, the `add` method will fallback to the `custom` data type and expect a custom loader and chunker to be passed by the user.
+</Note>
+
+Example:
+
+```python
+from embedchain import Pipeline as App
+from embedchain.loaders.github import GithubLoader
+
+app = App()
+
+loader = GithubLoader(config={"token": "ghp_xxx"})
+
+app.add("repo:embedchain/embedchain type:repo", data_type="github", loader=loader)
+
+app.query("What is Embedchain?")
+# Answer: Embedchain is a Data Platform for Large Language Models (LLMs). It allows users to seamlessly load, index, retrieve, and sync unstructured data in order to build dynamic, LLM-powered applications. There is also a JavaScript implementation called embedchain-js available on GitHub.
+```

docs/data-sources/data-type-handling.mdx

@@ -50,3 +50,15 @@ from embedchain import Pipeline as App
 naval_chat_bot = App()
 print(naval_chat_bot.query("What unique capacity does Naval argue humans possess when it comes to understanding explanations or concepts?"))
 ```
+
+## Resetting an app and vector database
+
+You can reset the app by simply calling the `reset` method. This will delete the vector database and all other app related files.
+
+```python
+from embedchain import Pipeline as App
+
+app = App()
+app.add("https://www.youtube.com/watch?v=3qHkcs3kG44")
+app.reset()
+```

docs/data-sources/discord.mdx

@@ -0,0 +1,28 @@
+---
+title: "💬 Discord"
+---
+
+To add any Discord channel messages to your app, just add the `channel_id` as the source and set the `data_type` to `discord`.
+
+<Note>
+    This loader requires a Discord bot token with read messages access.
+    To obtain the token, follow the instructions provided in this tutorial: 
+    <a href="https://www.writebots.com/discord-bot-token/">How to Get a Discord Bot Token?</a>.
+</Note>
+
+```python
+import os
+from embedchain import Pipeline as App
+
+# add your discord "BOT" token
+os.environ["DISCORD_TOKEN"] = "xxx"
+
+app = App()
+
+app.add("1177296711023075338", data_type="discord")
+
+response = app.query("What is Joe saying about Elon Musk?")
+
+print(response)
+# Answer: Joe is saying "Elon Musk is a genius".
+```

docs/data-sources/docs-site.mdx

@@ -1,5 +1,5 @@
 ---
-title: '📚🌐 Code documentation'
+title: '📚 Code documentation'
 ---
 
 To add any code documentation website as a loader, use the data_type as `docs_site`. Eg:

docs/data-sources/github.mdx

@@ -0,0 +1,50 @@
+---
+title: 📝 Github
+---
+
+1. Setup the Github loader by configuring the Github account with username and personal access token (PAT). Check out [this](https://docs.github.com/en/enterprise-server@3.6/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens#creating-a-personal-access-token) link to learn how to create a PAT.
+```Python
+from embedchain.loaders.github import GithubLoader
+
+loader = GithubLoader(
+    config={
+        "token":"ghp_xxxx"
+        }
+    )
+```
+
+2. Once you setup the loader, you can create an app and load data using the above Github loader
+```Python
+import os
+from embedchain.pipeline import Pipeline as App
+
+os.environ["OPENAI_API_KEY"] = "sk-xxxx"
+
+app = App()
+
+app.add("repo:embedchain/embedchain type:repo", data_type="github", loader=loader)
+
+response = app.query("What is Embedchain?")
+# Answer: Embedchain is a Data Platform for Large Language Models (LLMs). It allows users to seamlessly load, index, retrieve, and sync unstructured data in order to build dynamic, LLM-powered applications. There is also a JavaScript implementation called embedchain-js available on GitHub.
+```
+The `add` function of the app will accept any valid github query with qualifiers. It only supports loading github code, repository, issues and pull-requests.
+<Note>
+You must provide qualifiers `type:` and `repo:` in the query. The `type:` qualifier can be a combination of `code`, `repo`, `pr`, `issue`. The `repo:` qualifier must be a valid github repository name.
+</Note>
+
+<Card title="Valid queries" icon="lightbulb" iconType="duotone" color="#ca8b04">
+    - `repo:embedchain/embedchain type:repo` - to load the repository
+    - `repo:embedchain/embedchain type:issue,pr` - to load the issues and pull-requests of the repository
+    - `repo:embedchain/embedchain type:issue state:closed` - to load the closed issues of the repository
+</Card>
+
+3. We automatically create a chunker to chunk your GitHub data, however if you wish to provide your own chunker class. Here is how you can do that:
+```Python
+from embedchain.chunkers.common_chunker import CommonChunker
+from embedchain.config.add_config import ChunkerConfig
+
+github_chunker_config = ChunkerConfig(chunk_size=2000, chunk_overlap=0, length_function=len)
+github_chunker = CommonChunker(config=github_chunker_config)
+
+app.add(load_query, data_type="github", loader=loader, chunker=github_chunker)
+```

docs/data-sources/mysql.mdx

@@ -21,7 +21,6 @@ For more details on how to setup with valid config, check MySQL [documentation](
 
 2. Once you setup the loader, you can create an app and load data using the above MySQL loader
 ```Python
-import os
 from embedchain.pipeline import Pipeline as App
 
 app = App()

docs/data-sources/overview.mdx

@@ -5,25 +5,28 @@ title: Overview
 Embedchain comes with built-in support for various data sources. We handle the complexity of loading unstructured data from these data sources, allowing you to easily customize your app through a user-friendly interface.
 
 <CardGroup cols={4}>
-  <Card title="📊 csv" href="/data-sources/csv"></Card>
+  <Card title="📊 CSV" href="/data-sources/csv"></Card>
   <Card title="📃 JSON" href="/data-sources/json"></Card>
-  <Card title="📚🌐 docs site" href="/data-sources/docs-site"></Card>
+  <Card title="📚 docs site" href="/data-sources/docs-site"></Card>
   <Card title="📄 docx" href="/data-sources/docx"></Card>
   <Card title="📝 mdx" href="/data-sources/mdx"></Card>
-  <Card title="📓 notion" href="/data-sources/notion"></Card>
+  <Card title="📓 Notion" href="/data-sources/notion"></Card>
-  <Card title="📰 pdf" href="/data-sources/pdf-file"></Card>
+  <Card title="📰 PDF" href="/data-sources/pdf-file"></Card>
   <Card title="❓💬 q&a pair" href="/data-sources/qna"></Card>
   <Card title="🗺️ sitemap" href="/data-sources/sitemap"></Card>
   <Card title="📝 text" href="/data-sources/text"></Card>
-  <Card title="🌐📄 web page" href="/data-sources/web-page"></Card>
+  <Card title="🌐 web page" href="/data-sources/web-page"></Card>
   <Card title="🧾 xml" href="/data-sources/xml"></Card>
   <Card title="🙌 OpenAPI" href="/data-sources/openapi"></Card>
-  <Card title="📺 youtube video" href="/data-sources/youtube-video"></Card>
+  <Card title="📺 Youtube" href="/data-sources/youtube-video"></Card>
   <Card title="📬 Gmail" href="/data-sources/gmail"></Card>
   <Card title="🐘 Postgres" href="/data-sources/postgres"></Card>
   <Card title="🐬 MySQL" href="/data-sources/mysql"></Card>
   <Card title="🤖 Slack" href="/data-sources/slack"></Card>
   <Card title="🗨️ Discourse" href="/data-sources/discourse"></Card>
+  <Card title="💬 Discord" href="/data-sources/discord"></Card>
+  <Card title="📝 Github" href="/data-sources/github"></Card>
+  <Card title="⚙️ Custom" href="/data-sources/custom"></Card>
 </CardGroup>
 
 <br/ >

docs/data-sources/web-page.mdx

@@ -1,5 +1,5 @@
 ---
-title: '🌐📄 Web page'
+title: '🌐 Web page'
 ---
 
 To add any web page, use the data_type as `web_page`. Eg:

docs/data-sources/youtube-video.mdx

@@ -1,5 +1,5 @@
 ---
-title: '📺 Youtube video'
+title: '📺 Youtube'
 ---
 
 

docs/get-started/examples.mdx

@@ -1,8 +1,15 @@
 ---
 title: 🔎 Examples
-description: 'Collection of Google colab notebook and Replit links for users'
 ---
 
+# Explore awesome apps
+
+Check out the remarkable work accomplished using [Embedchain](https://app.embedchain.ai/custom-gpts/).
+
+## Collection of Google colab notebook and Replit links for users
+
+Get started with Embedchain by trying out the examples below. You can run the examples in your browser using Google Colab or Replit.
+
 <table>
   <thead>
     <tr>

docs/get-started/faq.mdx

@@ -2,13 +2,36 @@
 title: ❓ FAQs
 description: 'Collections of all the frequently asked questions'
 ---
-
+<AccordionGroup>
-#### Does Embedchain support OpenAI's Assistant APIs?
+<Accordion title="Does Embedchain support OpenAI's Assistant APIs?">
-
 Yes, it does. Please refer to the [OpenAI Assistant docs page](/get-started/openai-assistant).
+</Accordion>
+<Accordion title="How to use MistralAI language model?">
+Use the model provided on huggingface: `mistralai/Mistral-7B-v0.1`
+<CodeGroup>
+```python main.py
+import os
+from embedchain import Pipeline as App
 
-#### How to use `gpt-4-turbo` model released on OpenAI DevDay?
+os.environ["OPENAI_API_KEY"] = "sk-xxx"
+os.environ["HUGGINGFACE_ACCESS_TOKEN"] = "hf_your_token"
 
+app = App.from_config("huggingface.yaml")
+```
+```yaml huggingface.yaml
+llm:
+  provider: huggingface
+  config:
+    model: 'mistralai/Mistral-7B-v0.1'
+    temperature: 0.5
+    max_tokens: 1000
+    top_p: 0.5
+    stream: false
+```
+</CodeGroup>
+</Accordion>
+<Accordion title="How to use ChatGPT 4 turbo model released on OpenAI DevDay?">
+Use the model `gpt-4-turbo` provided my openai.
 <CodeGroup>
 
 ```python main.py
@@ -18,7 +41,7 @@ from embedchain import Pipeline as App
 os.environ['OPENAI_API_KEY'] = 'xxx'
 
 # load llm configuration from gpt4_turbo.yaml file
-app = App.from_config(yaml_path="gpt4_turbo.yaml")
+app = App.from_config(config_path="gpt4_turbo.yaml")
 ```
 
 ```yaml gpt4_turbo.yaml
@@ -31,12 +54,9 @@ llm:
     top_p: 1
     stream: false
 ```
-
 </CodeGroup>
-
+</Accordion>
-
+<Accordion title="How to use GPT-4 as the LLM model?">
-#### How to use GPT-4 as the LLM model?
-
 <CodeGroup>
 
 ```python main.py
@@ -46,7 +66,7 @@ from embedchain import Pipeline as App
 os.environ['OPENAI_API_KEY'] = 'xxx'
 
 # load llm configuration from gpt4.yaml file
-app = App.from_config(yaml_path="gpt4.yaml")
+app = App.from_config(config_path="gpt4.yaml")
 ```
 
 ```yaml gpt4.yaml
@@ -61,9 +81,8 @@ llm:
 ```
 
 </CodeGroup>
-
+</Accordion>
-#### I don't have OpenAI credits. How can I use some open source model?
+<Accordion title="I don't have OpenAI credits. How can I use some open source model?">
-
 <CodeGroup>
 
 ```python main.py
@@ -73,7 +92,7 @@ from embedchain import Pipeline as App
 os.environ['OPENAI_API_KEY'] = 'xxx'
 
 # load llm configuration from opensource.yaml file
-app = App.from_config(yaml_path="opensource.yaml")
+app = App.from_config(config_path="opensource.yaml")
 ```
 
 ```yaml opensource.yaml
@@ -93,8 +112,10 @@ embedder:
 ```
 </CodeGroup>
 
-#### How to contact support?
+</Accordion>
+</AccordionGroup>
 
+#### Need more help?
 If docs aren't sufficient, please feel free to reach out to us using one of the following methods:
 
 <Snippet file="get-help.mdx" />

docs/get-started/introduction.mdx

@@ -105,7 +105,7 @@ app.deploy()
 # ✅ Data of type: web_page, value: https://www.forbes.com/profile/elon-musk added successfully.
 ```
 
-## 🚀 How it works?
+## 🛠️ How it works?
 
 Embedchain abstracts out the following steps from you to easily create LLM powered apps:
 
@@ -129,3 +129,5 @@ The process of loading the dataset and querying involves multiple steps, each wi
 - How should I find similar documents for a query? Which ranking model should I use?
 
 Embedchain takes care of all these nuances and provides a simple interface to create apps on any data.
+
+## [🚀 Get started](https://docs.embedchain.ai/get-started/quickstart)

docs/get-started/quickstart.mdx

@@ -12,79 +12,84 @@ pip install embedchain
 ```
 
 <Tip>
-Embedchain now supports OpenAI's latest `gpt-4-turbo` model. Checkout the [docs here](/get-started/faq#how-to-use-gpt-4-turbo-model-released-on-openai-devday) on how to use it.
+Embedchain now supports OpenAI's latest `gpt-4-turbo` model. Checkout the [FAQs](/get-started/faq#how-to-use-gpt-4-turbo-model-released-on-openai-devday).
 </Tip>
 
 Creating an app involves 3 steps:
 
 <Steps>
   <Step title="⚙️ Import app instance">
-```python
+    ```python
-from embedchain import Pipeline as App
+    from embedchain import Pipeline as App
-app = App()
+    app = App()
-```
+    ```
+    <Accordion title="Customize your app by a simple YAML config" icon="gear-complex">
+      Embedchain provides a wide range of options to customize your app. You can customize the model, data sources, and much more.
+      Explore the custom configurations [here](https://docs.embedchain.ai/advanced/configuration).
+      <CodeGroup>
+      ```python yaml_app.py
+      from embedchain import Pipeline as App
+      app = App.from_config(config_path="config.yaml")
+      ```
+      ```python json_app.py
+      from embedchain import Pipeline as App
+      app = App.from_config(config_path="config.json")
+      ```
+      ```python app.py
+      from embedchain import Pipeline as App
+      config = {} # Add your config here
+      app = App.from_config(config=config)
+      ```
+      </CodeGroup>
+    </Accordion>
   </Step>
   <Step title="🗃️ Add data sources">
-```python
+    ```python
-# Add different data sources
+    app.add("https://en.wikipedia.org/wiki/Elon_Musk")
-app.add("https://en.wikipedia.org/wiki/Elon_Musk")
+    app.add("https://www.forbes.com/profile/elon-musk")
-app.add("https://www.forbes.com/profile/elon-musk")
+    # app.add("path/to/file/elon_musk.pdf")
-# You can also add local data sources such as pdf, csv files etc.
+    ```
-# app.add("/path/to/file.pdf")
+    <Accordion title="Embedchain supports adding data from many data sources." icon="files">
-```
+      Embedchain supports adding data from many data sources including web pages, PDFs, databases, and more.
+      Explore the list of supported [data sources](https://docs.embedchain.ai/data-sources/overview).
+    </Accordion>
   </Step>
-  <Step title="💬 Query or chat or search context on your data">
+  <Step title="💬 Ask questions, chat, or search through your data with ease">
-```python
+    ```python
-app.query("What is the net worth of Elon Musk today?")
+    app.query("What is the net worth of Elon Musk today?")
-# Answer: The net worth of Elon Musk today is $258.7 billion.
+    # Answer: The net worth of Elon Musk today is $258.7 billion.
-```
+    ```
+    <Accordion title="Want to chat with your app?" icon="face-thinking">
+      Embedchain provides a wide range of features to interact with your app. You can chat with your app, ask questions, search through your data, and much more.
+      ```python
+      app.chat("How many companies does Elon Musk run? Name those")
+      # Answer: Elon Musk runs 3 companies: Tesla, SpaceX, and Neuralink.
+      app.chat("What is his net worth today?")
+      # Answer: The net worth of Elon Musk today is $258.7 billion.
+      ```
+      To learn about other features, click [here](https://docs.embedchain.ai/get-started/introduction)
+    </Accordion>
   </Step>
-  <Step title="🚀 (Optional) Deploy your pipeline to Embedchain Platform">
+  <Step title="🚀 Seamlessly launch your App on the Embedchain Platform!">
-```python
+    ```python
-app.deploy()
+    app.deploy()
-# 🔑 Enter your Embedchain API key. You can find the API key at https://app.embedchain.ai/settings/keys/
+    # 🔑 Enter your Embedchain API key. You can find the API key at https://app.embedchain.ai/settings/keys/
-# ec-xxxxxx
+    # ec-xxxxxx
 
-# 🛠️ Creating pipeline on the platform...
+    # 🛠️ Creating pipeline on the platform...
-# 🎉🎉🎉 Pipeline created successfully! View your pipeline: https://app.embedchain.ai/pipelines/xxxxx
+    # 🎉🎉🎉 Pipeline created successfully! View your pipeline: https://app.embedchain.ai/pipelines/xxxxx
 
-# 🛠️ Adding data to your pipeline...
+    # 🛠️ Adding data to your pipeline...
-# ✅ Data of type: web_page, value: https://www.forbes.com/profile/elon-musk added successfully.
+    # ✅ Data of type: web_page, value: https://www.forbes.com/profile/elon-musk added successfully.
-```
+    ```
+    <Accordion title="Share your app with others" icon="laptop-mobile">
+      You can now share your app with others from our platform.
+      Access your app on our [platform](https://app.embedchain.ai/).
+    </Accordion>
   </Step>
 </Steps>
 
-Putting it together, you can run your first app using the following code. Make sure to set the `OPENAI_API_KEY` 🔑 environment variable in the code.
+Putting it together, you can run your first app using the following Google Colab. Make sure to set the `OPENAI_API_KEY` 🔑 environment variable in the code.
-
-```python
-import os
-from embedchain import Pipeline as App
-
-os.environ["OPENAI_API_KEY"] = "xxx"
-app = App()
-
-# Add different data sources
-app.add("https://en.wikipedia.org/wiki/Elon_Musk")
-app.add("https://www.forbes.com/profile/elon-musk")
-# You can also add local data sources such as pdf, csv files etc.
-# app.add("/path/to/file.pdf")
-
-response = app.query("What is the net worth of Elon Musk today?")
-print(response)
-# Answer: The net worth of Elon Musk today is $258.7 billion.
-
-app.deploy()
-# 🔑 Enter your Embedchain API key. You can find the API key at https://app.embedchain.ai/settings/keys/
-# ec-xxxxxx
-
-# 🛠️ Creating pipeline on the platform...
-# 🎉🎉🎉 Pipeline created successfully! View your pipeline: https://app.embedchain.ai/pipelines/xxxxx
-
-# 🛠️ Adding data to your pipeline...
-# ✅ Data of type: web_page, value: https://www.forbes.com/profile/elon-musk added successfully.
-```
-
-You can try it out yourself using the following Google Colab notebook:
 
 <a href="https://colab.research.google.com/drive/17ON1LPonnXAtLaZEebnOktstB_1cJJmh?usp=sharing">
     <img src="https://camo.githubusercontent.com/84f0493939e0c4de4e6dbe113251b4bfb5353e57134ffd9fcab6b8714514d4d1/68747470733a2f2f636f6c61622e72657365617263682e676f6f676c652e636f6d2f6173736574732f636f6c61622d62616467652e737667" alt="Open in Colab" />

docs/mint.json

@@ -89,7 +89,8 @@
             "data-sources/openapi",
             "data-sources/youtube-video",
             "data-sources/discourse",
-            "data-sources/substack"
+            "data-sources/substack",
+            "data-sources/discord"
           ]
         },
         "data-sources/data-type-handling"

embedchain/apps/app.py

@@ -12,7 +12,7 @@ from embedchain.factory import EmbedderFactory, LlmFactory, VectorDBFactory
 from embedchain.helpers.json_serializable import register_deserializable
 from embedchain.llm.base import BaseLlm
 from embedchain.llm.openai import OpenAILlm
-from embedchain.utils import validate_yaml_config
+from embedchain.utils import validate_config
 from embedchain.vectordb.base import BaseVectorDB
 from embedchain.vectordb.chroma import ChromaDB
 
@@ -134,7 +134,7 @@ class App(EmbedChain):
             config_data = yaml.safe_load(file)
 
         try:
-            validate_yaml_config(config_data)
+            validate_config(config_data)
         except Exception as e:
             raise Exception(f"❌ Error occurred while validating the YAML config. Error: {str(e)}")
 

embedchain/chunkers/base_chunker.py

@@ -41,7 +41,6 @@ class BaseChunker(JSONSerializable):
             url = meta_data["url"]
 
             chunks = self.get_chunks(content)
-
             for chunk in chunks:
                 chunk_id = hashlib.sha256((chunk + url).encode()).hexdigest()
                 chunk_id = f"{app_id}--{chunk_id}" if app_id is not None else chunk_id

embedchain/chunkers/common_chunker.py

@@ -13,7 +13,7 @@ class CommonChunker(BaseChunker):
 
     def __init__(self, config: Optional[ChunkerConfig] = None):
         if config is None:
-            config = ChunkerConfig(chunk_size=1000, chunk_overlap=0, length_function=len)
+            config = ChunkerConfig(chunk_size=2000, chunk_overlap=0, length_function=len)
         text_splitter = RecursiveCharacterTextSplitter(
             chunk_size=config.chunk_size,
             chunk_overlap=config.chunk_overlap,

embedchain/data_formatter/data_formatter.py

@@ -1,5 +1,5 @@
 from importlib import import_module
-from typing import Any, Dict
+from typing import Optional
 
 from embedchain.chunkers.base_chunker import BaseChunker
 from embedchain.config import AddConfig
@@ -16,7 +16,13 @@ class DataFormatter(JSONSerializable):
     .add or .add_local method call
     """
 
-    def __init__(self, data_type: DataType, config: AddConfig, kwargs: Dict[str, Any]):
+    def __init__(
+        self,
+        data_type: DataType,
+        config: AddConfig,
+        loader: Optional[BaseLoader] = None,
+        chunker: Optional[BaseChunker] = None,
+    ):
         """
         Initialize a dataformatter, set data type and chunker based on datatype.
 
@@ -25,15 +31,15 @@ class DataFormatter(JSONSerializable):
         :param config: AddConfig instance with nested loader and chunker config attributes.
         :type config: AddConfig
         """
-        self.loader = self._get_loader(data_type=data_type, config=config.loader, kwargs=kwargs)
+        self.loader = self._get_loader(data_type=data_type, config=config.loader, loader=loader)
-        self.chunker = self._get_chunker(data_type=data_type, config=config.chunker, kwargs=kwargs)
+        self.chunker = self._get_chunker(data_type=data_type, config=config.chunker, chunker=chunker)
 
     def _lazy_load(self, module_path: str):
         module_path, class_name = module_path.rsplit(".", 1)
         module = import_module(module_path)
         return getattr(module, class_name)
 
-    def _get_loader(self, data_type: DataType, config: LoaderConfig, kwargs: Dict[str, Any]) -> BaseLoader:
+    def _get_loader(self, data_type: DataType, config: LoaderConfig, loader: Optional[BaseLoader]) -> BaseLoader:
         """
         Returns the appropriate data loader for the given data type.
 
@@ -64,26 +70,17 @@ class DataFormatter(JSONSerializable):
             DataType.GMAIL: "embedchain.loaders.gmail.GmailLoader",
             DataType.NOTION: "embedchain.loaders.notion.NotionLoader",
             DataType.SUBSTACK: "embedchain.loaders.substack.SubstackLoader",
-            DataType.GITHUB: "embedchain.loaders.github.GithubLoader",
             DataType.YOUTUBE_CHANNEL: "embedchain.loaders.youtube_channel.YoutubeChannelLoader",
+            DataType.DISCORD: "embedchain.loaders.discord.DiscordLoader",
         }
 
-        custom_loaders = set(
+        if data_type == DataType.CUSTOM or loader is not None:
-            [
+            loader_class: type = loader
-                DataType.POSTGRES,
+            if loader_class:
-                DataType.MYSQL,
+                return loader_class
-                DataType.SLACK,
+        elif data_type in loaders:
-                DataType.DISCOURSE,
-            ]
-        )
-
-        if data_type in loaders:
             loader_class: type = self._lazy_load(loaders[data_type])
             return loader_class()
-        elif data_type in custom_loaders:
-            loader_class: type = kwargs.get("loader", None)
-            if loader_class is not None:
-                return loader_class
 
         raise ValueError(
             f"Cant find the loader for {data_type}.\
@@ -91,7 +88,7 @@ class DataFormatter(JSONSerializable):
                         check `https://docs.embedchain.ai/data-sources/overview`."
         )
 
-    def _get_chunker(self, data_type: DataType, config: ChunkerConfig, kwargs: Dict[str, Any]) -> BaseChunker:
+    def _get_chunker(self, data_type: DataType, config: ChunkerConfig, chunker: Optional[BaseChunker]) -> BaseChunker:
         """Returns the appropriate chunker for the given data type (updated for lazy loading)."""
         chunker_classes = {
             DataType.YOUTUBE_VIDEO: "embedchain.chunkers.youtube_video.YoutubeVideoChunker",
@@ -111,27 +108,22 @@ class DataFormatter(JSONSerializable):
             DataType.OPENAPI: "embedchain.chunkers.openapi.OpenAPIChunker",
             DataType.GMAIL: "embedchain.chunkers.gmail.GmailChunker",
             DataType.NOTION: "embedchain.chunkers.notion.NotionChunker",
-            DataType.POSTGRES: "embedchain.chunkers.postgres.PostgresChunker",
-            DataType.MYSQL: "embedchain.chunkers.mysql.MySQLChunker",
-            DataType.SLACK: "embedchain.chunkers.slack.SlackChunker",
-            DataType.DISCOURSE: "embedchain.chunkers.discourse.DiscourseChunker",
             DataType.SUBSTACK: "embedchain.chunkers.substack.SubstackChunker",
-            DataType.GITHUB: "embedchain.chunkers.common_chunker.CommonChunker",
             DataType.YOUTUBE_CHANNEL: "embedchain.chunkers.common_chunker.CommonChunker",
+            DataType.DISCORD: "embedchain.chunkers.common_chunker.CommonChunker",
+            DataType.CUSTOM: "embedchain.chunkers.common_chunker.CommonChunker",
         }
 
-        if data_type in chunker_classes:
+        if chunker is not None:
-            if "chunker" in kwargs:
+            return chunker
-                chunker_class = kwargs.get("chunker")
+        elif data_type in chunker_classes:
-            else:
+            chunker_class = self._lazy_load(chunker_classes[data_type])
-                chunker_class = self._lazy_load(chunker_classes[data_type])
-
             chunker = chunker_class(config)
             chunker.set_data_type(data_type)
             return chunker
-        else:
+
-            raise ValueError(
+        raise ValueError(
-                f"Cant find the chunker for {data_type}.\
+            f"Cant find the chunker for {data_type}.\
-                    We recommend to pass the chunker to use data_type: {data_type},\
+                We recommend to pass the chunker to use data_type: {data_type},\
-                        check `https://docs.embedchain.ai/data-sources/overview`."
+                    check `https://docs.embedchain.ai/data-sources/overview`."
-            )
+        )

embedchain/embedchain.py

@@ -133,7 +133,9 @@ class EmbedChain(JSONSerializable):
         metadata: Optional[Dict[str, Any]] = None,
         config: Optional[AddConfig] = None,
         dry_run=False,
-        **kwargs: Dict[str, Any],
+        loader: Optional[BaseLoader] = None,
+        chunker: Optional[BaseChunker] = None,
+        **kwargs: Optional[Dict[str, Any]],
     ):
         """
         Adds the data from the given URL to the vector db.
@@ -178,10 +180,10 @@ class EmbedChain(JSONSerializable):
             try:
                 data_type = DataType(data_type)
             except ValueError:
-                raise ValueError(
+                logging.info(
-                    f"Invalid data_type: '{data_type}'.",
+                    f"Invalid data_type: '{data_type}', using `custom` instead.\n Check docs to pass the valid data type: `https://docs.embedchain.ai/data-sources/overview`"  # noqa: E501
-                    f"Please use one of the following: {[data_type.value for data_type in DataType]}",
+                )
-                ) from None
+                data_type = DataType.CUSTOM
 
         if not data_type:
             data_type = detect_datatype(source)
@@ -190,21 +192,11 @@ class EmbedChain(JSONSerializable):
         hash_object = hashlib.md5(str(source).encode("utf-8"))
         source_hash = hash_object.hexdigest()
 
-        # Check if the data hash already exists, if so, skip the addition
-        self.cursor.execute(
-            "SELECT 1 FROM data_sources WHERE hash = ? AND pipeline_id = ?", (source_hash, self.config.id)
-        )
-        existing_data = self.cursor.fetchone()
-
-        if existing_data:
-            print(f"Data with hash {source_hash} already exists. Skipping addition.")
-            return source_hash
-
         self.user_asks.append([source, data_type.value, metadata])
 
-        data_formatter = DataFormatter(data_type, config, kwargs)
+        data_formatter = DataFormatter(data_type, config, loader, chunker)
         documents, metadatas, _ids, new_chunks = self._load_and_embed(
-            data_formatter.loader, data_formatter.chunker, source, metadata, source_hash, dry_run
+            data_formatter.loader, data_formatter.chunker, source, metadata, source_hash, dry_run, **kwargs
         )
         if data_type in {DataType.DOCS_SITE}:
             self.is_docs_site_instance = True
@@ -212,7 +204,7 @@ class EmbedChain(JSONSerializable):
         # Insert the data into the 'data' table
         self.cursor.execute(
             """
-            INSERT INTO data_sources (hash, pipeline_id, type, value, metadata)
+            INSERT OR REPLACE INTO data_sources (hash, pipeline_id, type, value, metadata)
             VALUES (?, ?, ?, ?, ?)
         """,
             (source_hash, self.config.id, data_type.value, str(source), json.dumps(metadata)),
@@ -248,7 +240,7 @@ class EmbedChain(JSONSerializable):
         data_type: Optional[DataType] = None,
         metadata: Optional[Dict[str, Any]] = None,
         config: Optional[AddConfig] = None,
-        **kwargs: Dict[str, Any],
+        **kwargs: Optional[Dict[str, Any]],
     ):
         """
         Adds the data from the given URL to the vector db.
@@ -279,7 +271,7 @@ class EmbedChain(JSONSerializable):
             data_type=data_type,
             metadata=metadata,
             config=config,
-            kwargs=kwargs,
+            **kwargs,
         )
 
     def _get_existing_doc_id(self, chunker: BaseChunker, src: Any):
@@ -348,6 +340,7 @@ class EmbedChain(JSONSerializable):
         metadata: Optional[Dict[str, Any]] = None,
         source_hash: Optional[str] = None,
         dry_run=False,
+        **kwargs: Optional[Dict[str, Any]],
     ):
         """
         Loads the data from the given URL, chunks it, and adds it to database.
@@ -441,6 +434,7 @@ class EmbedChain(JSONSerializable):
             metadatas=metadatas,
             ids=ids,
             skip_embedding=(chunker.data_type == DataType.IMAGES),
+            **kwargs,
         )
         count_new_chunks = self.db.count() - chunks_before_addition
 
@@ -458,7 +452,12 @@ class EmbedChain(JSONSerializable):
         ]
 
     def _retrieve_from_database(
-        self, input_query: str, config: Optional[BaseLlmConfig] = None, where=None, citations: bool = False
+        self,
+        input_query: str,
+        config: Optional[BaseLlmConfig] = None,
+        where=None,
+        citations: bool = False,
+        **kwargs: Optional[Dict[str, Any]],
     ) -> Union[List[Tuple[str, str, str]], List[str]]:
         """
         Queries the vector database based on the given input query.
@@ -502,6 +501,7 @@ class EmbedChain(JSONSerializable):
             where=where,
             skip_embedding=(hasattr(config, "query_type") and config.query_type == "Images"),
             citations=citations,
+            **kwargs,
         )
 
         return contexts
@@ -537,8 +537,9 @@ class EmbedChain(JSONSerializable):
         :rtype: str, if citations is False, otherwise Tuple[str,List[Tuple[str,str,str]]]
         """
         citations = kwargs.get("citations", False)
+        db_kwargs = {key: value for key, value in kwargs.items() if key != "citations"}
         contexts = self._retrieve_from_database(
-            input_query=input_query, config=config, where=where, citations=citations
+            input_query=input_query, config=config, where=where, citations=citations, **db_kwargs
         )
         if citations and len(contexts) > 0 and isinstance(contexts[0], tuple):
             contexts_data_for_llm_query = list(map(lambda x: x[0], contexts))
@@ -589,9 +590,13 @@ class EmbedChain(JSONSerializable):
         or the dry run result
         :rtype: str, if citations is False, otherwise Tuple[str,List[Tuple[str,str,str]]]
         """
-        citations = kwargs.get("citations", False)
+        if "citations" in kwargs:
+            citations = kwargs.pop("citations")
+        else:
+            citations = False
+
         contexts = self._retrieve_from_database(
-            input_query=input_query, config=config, where=where, citations=citations
+            input_query=input_query, config=config, where=where, citations=citations, **kwargs
         )
         if citations and len(contexts) > 0 and isinstance(contexts[0], tuple):
             contexts_data_for_llm_query = list(map(lambda x: x[0], contexts))

embedchain/llm/huggingface.py

@@ -1,4 +1,5 @@
 import importlib
+import logging
 import os
 from typing import Optional
 
@@ -42,9 +43,11 @@ class HuggingFaceLlm(BaseLlm):
         else:
             raise ValueError("`top_p` must be > 0.0 and < 1.0")
 
+        model = config.model or "google/flan-t5-xxl"
+        logging.info(f"Using HuggingFaceHub with model {model}")
         llm = HuggingFaceHub(
             huggingfacehub_api_token=os.environ["HUGGINGFACE_ACCESS_TOKEN"],
-            repo_id=config.model or "google/flan-t5-xxl",
+            repo_id=model,
             model_kwargs=model_kwargs,
         )
 

embedchain/loaders/discord.py

@@ -0,0 +1,150 @@
+import hashlib
+import logging
+import os
+
+from embedchain.helpers.json_serializable import register_deserializable
+from embedchain.loaders.base_loader import BaseLoader
+
+
+@register_deserializable
+class DiscordLoader(BaseLoader):
+    """
+    Load data from a Discord Channel ID.
+    """
+
+    def __init__(self):
+        if not os.environ.get("DISCORD_TOKEN"):
+            raise ValueError("DISCORD_TOKEN is not set")
+
+        self.token = os.environ.get("DISCORD_TOKEN")
+
+    @staticmethod
+    def _format_message(message):
+        return {
+            "message_id": message.id,
+            "content": message.content,
+            "author": {
+                "id": message.author.id,
+                "name": message.author.name,
+                "discriminator": message.author.discriminator,
+            },
+            "created_at": message.created_at.isoformat(),
+            "attachments": [
+                {
+                    "id": attachment.id,
+                    "filename": attachment.filename,
+                    "size": attachment.size,
+                    "url": attachment.url,
+                    "proxy_url": attachment.proxy_url,
+                    "height": attachment.height,
+                    "width": attachment.width,
+                }
+                for attachment in message.attachments
+            ],
+            "embeds": [
+                {
+                    "title": embed.title,
+                    "type": embed.type,
+                    "description": embed.description,
+                    "url": embed.url,
+                    "timestamp": embed.timestamp.isoformat(),
+                    "color": embed.color,
+                    "footer": {
+                        "text": embed.footer.text,
+                        "icon_url": embed.footer.icon_url,
+                        "proxy_icon_url": embed.footer.proxy_icon_url,
+                    },
+                    "image": {
+                        "url": embed.image.url,
+                        "proxy_url": embed.image.proxy_url,
+                        "height": embed.image.height,
+                        "width": embed.image.width,
+                    },
+                    "thumbnail": {
+                        "url": embed.thumbnail.url,
+                        "proxy_url": embed.thumbnail.proxy_url,
+                        "height": embed.thumbnail.height,
+                        "width": embed.thumbnail.width,
+                    },
+                    "video": {
+                        "url": embed.video.url,
+                        "height": embed.video.height,
+                        "width": embed.video.width,
+                    },
+                    "provider": {
+                        "name": embed.provider.name,
+                        "url": embed.provider.url,
+                    },
+                    "author": {
+                        "name": embed.author.name,
+                        "url": embed.author.url,
+                        "icon_url": embed.author.icon_url,
+                        "proxy_icon_url": embed.author.proxy_icon_url,
+                    },
+                    "fields": [
+                        {
+                            "name": field.name,
+                            "value": field.value,
+                            "inline": field.inline,
+                        }
+                        for field in embed.fields
+                    ],
+                }
+                for embed in message.embeds
+            ],
+        }
+
+    def load_data(self, channel_id: str):
+        """Load data from a Discord Channel ID."""
+        import discord
+
+        messages = []
+
+        class DiscordClient(discord.Client):
+            async def on_ready(self) -> None:
+                logging.info("Logged on as {0}!".format(self.user))
+                try:
+                    channel = self.get_channel(int(channel_id))
+                    if not isinstance(channel, discord.TextChannel):
+                        raise ValueError(
+                            f"Channel {channel_id} is not a text channel. " "Only text channels are supported for now."
+                        )
+                    threads = {}
+
+                    for thread in channel.threads:
+                        threads[thread.id] = thread
+
+                    async for message in channel.history(limit=None):
+                        messages.append(DiscordLoader._format_message(message))
+                        if message.id in threads:
+                            async for thread_message in threads[message.id].history(limit=None):
+                                messages.append(DiscordLoader._format_message(thread_message))
+
+                except Exception as e:
+                    logging.error(e)
+                    await self.close()
+                finally:
+                    await self.close()
+
+        intents = discord.Intents.default()
+        intents.message_content = True
+        client = DiscordClient(intents=intents)
+        client.run(self.token)
+
+        meta_data = {
+            "url": channel_id,
+        }
+
+        messages = str(messages)
+
+        doc_id = hashlib.sha256((messages + channel_id).encode()).hexdigest()
+
+        return {
+            "doc_id": doc_id,
+            "data": [
+                {
+                    "content": messages,
+                    "meta_data": meta_data,
+                }
+            ],
+        }

embedchain/loaders/github.py

@@ -2,116 +2,296 @@ import concurrent.futures
 import hashlib
 import logging
 import os
+import re
+import shlex
+from typing import Any, Dict, Optional
 
 from tqdm import tqdm
 
 from embedchain.loaders.base_loader import BaseLoader
-from embedchain.loaders.json import JSONLoader
+from embedchain.utils import clean_string
-from embedchain.loaders.mdx import MdxLoader
-from embedchain.utils import detect_datatype
 
+GITHUB_URL = "https://github.com"
+GITHUB_API_URL = "https://api.github.com"
 
-def _load_file_data(path):
+VALID_SEARCH_TYPES = set(["code", "repo", "pr", "issue", "discussion"])
-    data = []
-    data_content = []
-    try:
-        with open(path, "rb") as f:
-            content = f.read().decode("utf-8")
-    except Exception as e:
-        print(f"Error reading file {path}: {e}")
-        raise ValueError(f"Failed to read file {path}")
-
-    meta_data = {}
-    meta_data["url"] = path
-    data.append(
-        {
-            "content": content,
-            "meta_data": meta_data,
-        }
-    )
-    data_content.append(content)
-    doc_id = hashlib.sha256((" ".join(data_content) + path).encode()).hexdigest()
-    return {
-        "doc_id": doc_id,
-        "data": data,
-    }
 
 
 class GithubLoader(BaseLoader):
-    def load_data(self, repo_url):
+    """Load data from github search query."""
-        """Load data from a git repo."""
+
+    def __init__(self, config: Optional[Dict[str, Any]] = None):
+        super().__init__()
+        if not config:
+            raise ValueError(
+                "GithubLoader requires a personal access token to use github api. Check - `https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens#creating-a-personal-access-token-classic`"  # noqa: E501
+            )
+
         try:
-            from git import Repo
+            from github import Github
         except ImportError as e:
             raise ValueError(
-                "GithubLoader requires extra dependencies. Install with `pip install --upgrade 'embedchain[git]'`"
+                "GithubLoader requires extra dependencies. Install with `pip install --upgrade 'embedchain[github]'`"
             ) from e
 
-        mdx_loader = MdxLoader()
+        self.config = config
-        json_loader = JSONLoader()
+        token = config.get("token")
+        if not token:
+            raise ValueError(
+                "GithubLoader requires a personal access token to use github api. Check - `https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens#creating-a-personal-access-token-classic`"  # noqa: E501
+            )
+
+        try:
+            self.client = Github(token)
+        except Exception as e:
+            logging.error(f"GithubLoader failed to initialize client: {e}")
+            self.client = None
+
+    def _github_search_code(self, query: str):
+        """Search github code."""
         data = []
-        data_urls = []
+        results = self.client.search_code(query)
+        for result in tqdm(results, total=results.totalCount, desc="Loading code files from github"):
+            url = result.html_url
+            logging.info(f"Added data from url: {url}")
+            content = result.decoded_content.decode("utf-8")
+            metadata = {
+                "url": url,
+            }
+            data.append(
+                {
+                    "content": clean_string(content),
+                    "meta_data": metadata,
+                }
+            )
+        return data
+
+    def _get_github_repo_data(self, repo_url: str):
+        local_hash = hashlib.sha256(repo_url.encode()).hexdigest()
+        local_path = f"/tmp/{local_hash}"
+        data = []
+
+        def _get_repo_tree(repo_url: str, local_path: str):
+            try:
+                from git import Repo
+            except ImportError as e:
+                raise ValueError(
+                    "GithubLoader requires extra dependencies. Install with `pip install --upgrade 'embedchain[github]'`"  # noqa: E501
+                ) from e
 
-        def _fetch_or_clone_repo(repo_url: str, local_path: str):
             if os.path.exists(local_path):
                 logging.info("Repository already exists. Fetching updates...")
                 repo = Repo(local_path)
-                origin = repo.remotes.origin
-                origin.fetch()
                 logging.info("Fetch completed.")
             else:
                 logging.info("Cloning repository...")
                 Repo.clone_from(repo_url, local_path)
+                repo = Repo.clone_from(repo_url, local_path)
                 logging.info("Clone completed.")
+            return repo.head.commit.tree
 
-        def _load_file(file_path: str):
+        def _get_repo_tree_contents(repo_path, tree, progress_bar):
-            try:
+            for subtree in tree:
-                data_type = detect_datatype(file_path).value
+                if subtree.type == "tree":
-            except Exception:
+                    _get_repo_tree_contents(repo_path, subtree, progress_bar)
-                data_type = "unstructured"
+                else:
-
+                    assert subtree.type == "blob"
-            if data_type == "mdx":
-                data = mdx_loader.load_data(file_path)
-            elif data_type == "json":
-                data = json_loader.load_data(file_path)
-            else:
-                data = _load_file_data(file_path)
-
-            return data.get("data", [])
-
-        def _is_file_empty(file_path):
-            return os.path.getsize(file_path) == 0
-
-        def _is_whitelisted(file_path):
-            whitelisted_extensions = ["md", "txt", "html", "json", "py", "js", "jsx", "ts", "tsx", "mdx", "rst"]
-            _, file_extension = os.path.splitext(file_path)
-            return file_extension[1:] in whitelisted_extensions
-
-        def _add_repo_files(repo_path: str):
-            with concurrent.futures.ThreadPoolExecutor(max_workers=4) as executor:
-                future_to_file = {
-                    executor.submit(_load_file, os.path.join(root, filename)): os.path.join(root, filename)
-                    for root, _, files in os.walk(repo_path)
-                    for filename in files
-                    if _is_whitelisted(os.path.join(root, filename))
-                    and not _is_file_empty(os.path.join(root, filename))  # noqa:E501
-                }
-                for future in tqdm(concurrent.futures.as_completed(future_to_file), total=len(future_to_file)):
-                    file = future_to_file[future]
                     try:
-                        results = future.result()
+                        contents = subtree.data_stream.read().decode("utf-8")
-                        if results:
+                    except Exception:
-                            data.extend(results)
+                        logging.warning(f"Failed to read file: {subtree.path}")
-                            data_urls.extend([result.get("meta_data").get("url") for result in results])
+                        progress_bar.update(1) if progress_bar else None
-                    except Exception as e:
+                        continue
-                        logging.warn(f"Failed to process {file}: {e}")
+
+                    url = f"{repo_url.rstrip('.git')}/blob/main/{subtree.path}"
+                    data.append(
+                        {
+                            "content": clean_string(contents),
+                            "meta_data": {
+                                "url": url,
+                            },
+                        }
+                    )
+                if progress_bar is not None:
+                    progress_bar.update(1)
+
+        repo_tree = _get_repo_tree(repo_url, local_path)
+        tree_list = list(repo_tree.traverse())
+        with tqdm(total=len(tree_list), desc="Loading files:", unit="item") as progress_bar:
+            _get_repo_tree_contents(local_path, repo_tree, progress_bar)
+
+        return data
+
+    def _github_search_repo(self, query: str):
+        """Search github repo."""
+        data = []
+        logging.info(f"Searching github repos with query: {query}")
+        results = self.client.search_repositories(query)
+        # Add repo urls and descriptions
+        urls = list(map(lambda x: x.html_url, results))
+        discriptions = list(map(lambda x: x.description, results))
+        data.append(
+            {
+                "content": clean_string(desc),
+                "meta_data": {
+                    "url": url,
+                },
+            }
+            for url, desc in zip(urls, discriptions)
+        )
+
+        # Add repo contents
+        for result in results:
+            clone_url = result.clone_url
+            logging.info(f"Cloning repository: {clone_url}")
+            data = self._get_github_repo_data(clone_url)
+        return data
+
+    def _github_search_issues_and_pr(self, query: str, type: str):
+        """Search github issues and PRs."""
+        data = []
+
+        query = f"{query} is:{type}"
+        logging.info(f"Searching github for query: {query}")
+
+        results = self.client.search_issues(query)
+
+        logging.info(f"Total results: {results.totalCount}")
+        for result in tqdm(results, total=results.totalCount, desc=f"Loading {type} from github"):
+            url = result.html_url
+            title = result.title
+            body = result.body
+            if not body:
+                logging.warn(f"Skipping issue because empty content for: {url}")
+                continue
+            labels = " ".join([label.name for label in result.labels])
+            issue_comments = result.get_comments()
+            comments = []
+            comments_created_at = []
+            for comment in issue_comments:
+                comments_created_at.append(str(comment.created_at))
+                comments.append(f"{comment.user.name}:{comment.body}")
+            content = "\n".join([title, labels, body, *comments])
+            metadata = {
+                "url": url,
+                "created_at": str(result.created_at),
+                "comments_created_at": " ".join(comments_created_at),
+            }
+            data.append(
+                {
+                    "content": clean_string(content),
+                    "meta_data": metadata,
+                }
+            )
+        return data
+
+    # need to test more for discussion
+    def _github_search_discussions(self, query: str):
+        """Search github discussions."""
+        data = []
+
+        query = f"{query} is:discussion"
+        logging.info(f"Searching github repo for query: {query}")
+        repos_results = self.client.search_repositories(query)
+        logging.info(f"Total repos found: {repos_results.totalCount}")
+        for repo_result in tqdm(repos_results, total=repos_results.totalCount, desc="Loading discussions from github"):
+            teams = repo_result.get_teams()
+            for team in teams:
+                team_discussions = team.get_discussions()
+                for discussion in team_discussions:
+                    url = discussion.html_url
+                    title = discussion.title
+                    body = discussion.body
+                    if not body:
+                        logging.warn(f"Skipping discussion because empty content for: {url}")
+                        continue
+                    comments = []
+                    comments_created_at = []
+                    print("Discussion comments: ", discussion.comments_url)
+                    content = "\n".join([title, body, *comments])
+                    metadata = {
+                        "url": url,
+                        "created_at": str(discussion.created_at),
+                        "comments_created_at": " ".join(comments_created_at),
+                    }
+                    data.append(
+                        {
+                            "content": clean_string(content),
+                            "meta_data": metadata,
+                        }
+                    )
+        return data
+
+    def _search_github_data(self, search_type: str, query: str):
+        """Search github data."""
+        if search_type == "code":
+            data = self._github_search_code(query)
+        elif search_type == "repo":
+            data = self._github_search_repo(query)
+        elif search_type == "issue":
+            data = self._github_search_issues_and_pr(query, search_type)
+        elif search_type == "pr":
+            data = self._github_search_issues_and_pr(query, search_type)
+        elif search_type == "discussion":
+            raise ValueError("GithubLoader does not support searching discussions yet.")
+
+        return data
+
+    def _get_valid_github_query(self, query: str):
+        """Check if query is valid and return search types and valid github query."""
+        query_terms = shlex.split(query)
+        # query must provide repo to load data from
+        if len(query_terms) < 1 or "repo:" not in query:
+            raise ValueError(
+                "GithubLoader requires a search query with `repo:` term. Refer docs - `https://docs.embedchain.ai/data-sources/github`"  # noqa: E501
+            )
+
+        github_query = []
+        types = set()
+        type_pattern = r"type:([a-zA-Z,]+)"
+        for term in query_terms:
+            term_match = re.search(type_pattern, term)
+            if term_match:
+                search_types = term_match.group(1).split(",")
+                types.update(search_types)
+            else:
+                github_query.append(term)
+
+        # query must provide search type
+        if len(types) == 0:
+            raise ValueError(
+                "GithubLoader requires a search query with `type:` term. Refer docs - `https://docs.embedchain.ai/data-sources/github`"  # noqa: E501
+            )
+
+        for search_type in search_types:
+            if search_type not in VALID_SEARCH_TYPES:
+                raise ValueError(
+                    f"Invalid search type: {search_type}. Valid types are: {', '.join(VALID_SEARCH_TYPES)}"
+                )
+
+        query = " ".join(github_query)
+
+        return types, query
+
+    def load_data(self, search_query: str, max_results: int = 1000):
+        """Load data from github search query."""
+
+        if not self.client:
+            raise ValueError(
+                "GithubLoader client is not initialized, data will not be loaded. Refer docs - `https://docs.embedchain.ai/data-sources/github`"  # noqa: E501
+            )
+
+        search_types, query = self._get_valid_github_query(search_query)
+        logging.info(f"Searching github for query: {query}, with types: {', '.join(search_types)}")
+
+        data = []
+
+        with concurrent.futures.ThreadPoolExecutor(max_workers=4) as executor:
+            futures_map = executor.map(self._search_github_data, search_types, [query] * len(search_types))
+            for search_data in tqdm(futures_map, total=len(search_types), desc="Searching data from github"):
+                data.extend(search_data)
 
-        source_hash = hashlib.sha256(repo_url.encode()).hexdigest()
-        repo_path = f"/tmp/{source_hash}"
-        _fetch_or_clone_repo(repo_url=repo_url, local_path=repo_path)
-        _add_repo_files(repo_path)
-        doc_id = hashlib.sha256((repo_url + ", ".join(data_urls)).encode()).hexdigest()
         return {
-            "doc_id": doc_id,
+            "doc_id": hashlib.sha256(query.encode()).hexdigest(),
             "data": data,
         }

embedchain/loaders/sitemap.py

@@ -1,6 +1,7 @@
 import concurrent.futures
 import hashlib
 import logging
+from urllib.parse import urlparse
 
 import requests
 from tqdm import tqdm
@@ -16,7 +17,6 @@ except ImportError:
 from embedchain.helpers.json_serializable import register_deserializable
 from embedchain.loaders.base_loader import BaseLoader
 from embedchain.loaders.web_page import WebPageLoader
-from embedchain.utils import is_readable
 
 
 @register_deserializable
@@ -30,29 +30,32 @@ class SitemapLoader(BaseLoader):
     def load_data(self, sitemap_url):
         output = []
         web_page_loader = WebPageLoader()
-        response = requests.get(sitemap_url)
-        response.raise_for_status()
 
-        soup = BeautifulSoup(response.text, "xml")
+        if urlparse(sitemap_url).scheme not in ["file", "http", "https"]:
+            raise ValueError("Not a valid URL.")
+
+        if urlparse(sitemap_url).scheme in ["http", "https"]:
+            response = requests.get(sitemap_url)
+            response.raise_for_status()
+        else:
+            with open(sitemap_url, "r") as file:
+                soup = BeautifulSoup(file, "xml")
         links = [link.text for link in soup.find_all("loc") if link.parent.name == "url"]
         if len(links) == 0:
             links = [link.text for link in soup.find_all("loc")]
 
         doc_id = hashlib.sha256((" ".join(links) + sitemap_url).encode()).hexdigest()
 
-        def load_link(link):
+        def load_web_page(link):
             try:
-                each_load_data = web_page_loader.load_data(link)
+                loader_data = web_page_loader.load_data(link)
-                if is_readable(each_load_data.get("data")[0].get("content")):
+                return loader_data.get("data")
-                    return each_load_data.get("data")
-                else:
-                    logging.warning(f"Page is not readable (too many invalid characters): {link}")
             except ParserRejectedMarkup as e:
                 logging.error(f"Failed to parse {link}: {e}")
             return None
 
         with concurrent.futures.ThreadPoolExecutor() as executor:
-            future_to_link = {executor.submit(load_link, link): link for link in links}
+            future_to_link = {executor.submit(load_web_page, link): link for link in links}
             for future in tqdm(concurrent.futures.as_completed(future_to_link), total=len(links), desc="Loading pages"):
                 link = future_to_link[future]
                 try:

embedchain/models/data_type.py

@@ -29,13 +29,10 @@ class IndirectDataType(Enum):
     JSON = "json"
     OPENAPI = "openapi"
     GMAIL = "gmail"
-    POSTGRES = "postgres"
-    MYSQL = "mysql"
-    SLACK = "slack"
-    DISCOURSE = "discourse"
     SUBSTACK = "substack"
-    GITHUB = "github"
     YOUTUBE_CHANNEL = "youtube_channel"
+    DISCORD = "discord"
+    CUSTOM = "custom"
 
 
 class SpecialDataType(Enum):
@@ -64,10 +61,7 @@ class DataType(Enum):
     JSON = IndirectDataType.JSON.value
     OPENAPI = IndirectDataType.OPENAPI.value
     GMAIL = IndirectDataType.GMAIL.value
-    POSTGRES = IndirectDataType.POSTGRES.value
-    MYSQL = IndirectDataType.MYSQL.value
-    SLACK = IndirectDataType.SLACK.value
-    DISCOURSE = IndirectDataType.DISCOURSE.value
     SUBSTACK = IndirectDataType.SUBSTACK.value
-    GITHUB = IndirectDataType.GITHUB.value
     YOUTUBE_CHANNEL = IndirectDataType.YOUTUBE_CHANNEL.value
+    DISCORD = IndirectDataType.DISCORD.value
+    CUSTOM = IndirectDataType.CUSTOM.value

embedchain/pipeline.py

@@ -4,6 +4,7 @@ import logging
 import os
 import sqlite3
 import uuid
+from typing import Any, Dict, Optional
 
 import requests
 import yaml
@@ -19,7 +20,7 @@ from embedchain.helpers.json_serializable import register_deserializable
 from embedchain.llm.base import BaseLlm
 from embedchain.llm.openai import OpenAILlm
 from embedchain.telemetry.posthog import AnonymousTelemetry
-from embedchain.utils import validate_yaml_config
+from embedchain.utils import validate_config
 from embedchain.vectordb.base import BaseVectorDB
 from embedchain.vectordb.chroma import ChromaDB
 
@@ -43,7 +44,7 @@ class Pipeline(EmbedChain):
         db: BaseVectorDB = None,
         embedding_model: BaseEmbedder = None,
         llm: BaseLlm = None,
-        yaml_path: str = None,
+        config_data: dict = None,
         log_level=logging.WARN,
         auto_deploy: bool = False,
         chunker: ChunkerConfig = None,
@@ -59,15 +60,15 @@ class Pipeline(EmbedChain):
         :type embedding_model: BaseEmbedder, optional
         :param llm: The LLM model used to calculate embeddings, defaults to None
         :type llm: BaseLlm, optional
-        :param yaml_path: Path to the YAML configuration file, defaults to None
+        :param config_data: Config dictionary, defaults to None
-        :type yaml_path: str, optional
+        :type config_data: dict, optional
         :param log_level: Log level to use, defaults to logging.WARN
         :type log_level: int, optional
         :param auto_deploy: Whether to deploy the pipeline automatically, defaults to False
         :type auto_deploy: bool, optional
         :raises Exception: If an error occurs while creating the pipeline
         """
-        if id and yaml_path:
+        if id and config_data:
             raise Exception("Cannot provide both id and config. Please provide only one of them.")
 
         if id and name:
@@ -79,8 +80,8 @@ class Pipeline(EmbedChain):
         logging.basicConfig(level=log_level, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s")
         self.logger = logging.getLogger(__name__)
         self.auto_deploy = auto_deploy
-        # Store the yaml config as an attribute to be able to send it
+        # Store the dict config as an attribute to be able to send it
-        self.yaml_config = None
+        self.config_data = config_data if (config_data and validate_config(config_data)) else None
         self.client = None
         # pipeline_id from the backend
         self.id = None
@@ -92,11 +93,6 @@ class Pipeline(EmbedChain):
         self.name = self.config.name
         self.config.id = self.local_id = str(uuid.uuid4()) if self.config.id is None else self.config.id
 
-        if yaml_path:
-            with open(yaml_path, "r") as file:
-                config_data = yaml.safe_load(file)
-                self.yaml_config = config_data
-
         if id is not None:
             # Init client first since user is trying to fetch the pipeline
             # details from the platform
@@ -187,9 +183,9 @@ class Pipeline(EmbedChain):
         Create a pipeline on the platform.
         """
         print("🛠️ Creating pipeline on the platform...")
-        # self.yaml_config is a dict. Pass it inside the key 'yaml_config' to the backend
+        # self.config_data is a dict. Pass it inside the key 'yaml_config' to the backend
         payload = {
-            "yaml_config": json.dumps(self.yaml_config),
+            "yaml_config": json.dumps(self.config_data),
             "name": self.name,
             "local_id": self.local_id,
         }
@@ -346,24 +342,57 @@ class Pipeline(EmbedChain):
         self.telemetry.capture(event_name="deploy", properties=self._telemetry_props)
 
     @classmethod
-    def from_config(cls, yaml_path: str, auto_deploy: bool = False):
+    def from_config(
+        cls,
+        config_path: Optional[str] = None,
+        config: Optional[Dict[str, Any]] = None,
+        auto_deploy: bool = False,
+        yaml_path: Optional[str] = None,
+    ):
         """
-        Instantiate a Pipeline object from a YAML configuration file.
+        Instantiate a Pipeline object from a configuration.
 
-        :param yaml_path: Path to the YAML configuration file.
+        :param config_path: Path to the YAML or JSON configuration file.
-        :type yaml_path: str
+        :type config_path: Optional[str]
+        :param config: A dictionary containing the configuration.
+        :type config: Optional[Dict[str, Any]]
         :param auto_deploy: Whether to deploy the pipeline automatically, defaults to False
         :type auto_deploy: bool, optional
+        :param yaml_path: (Deprecated) Path to the YAML configuration file. Use config_path instead.
+        :type yaml_path: Optional[str]
         :return: An instance of the Pipeline class.
         :rtype: Pipeline
         """
-        with open(yaml_path, "r") as file:
+        # Backward compatibility for yaml_path
-            config_data = yaml.safe_load(file)
+        if yaml_path and not config_path:
+            config_path = yaml_path
+
+        if config_path and config:
+            raise ValueError("Please provide only one of config_path or config.")
+
+        config_data = None
+
+        if config_path:
+            file_extension = os.path.splitext(config_path)[1]
+            with open(config_path, "r") as file:
+                if file_extension in [".yaml", ".yml"]:
+                    config_data = yaml.safe_load(file)
+                elif file_extension == ".json":
+                    config_data = json.load(file)
+                else:
+                    raise ValueError("config_path must be a path to a YAML or JSON file.")
+        elif config and isinstance(config, dict):
+            config_data = config
+        else:
+            logging.error(
+                "Please provide either a config file path (YAML or JSON) or a config dictionary. Falling back to defaults because no config is provided.",  # noqa: E501
+            )
+            config_data = {}
 
         try:
-            validate_yaml_config(config_data)
+            validate_config(config_data)
         except Exception as e:
-            raise Exception(f"❌ Error occurred while validating the YAML config. Error: {str(e)}")
+            raise Exception(f"Error occurred while validating the config. Error: {str(e)}")
 
         pipeline_config_data = config_data.get("app", {}).get("config", {})
         db_config_data = config_data.get("vectordb", {})
@@ -388,7 +417,7 @@ class Pipeline(EmbedChain):
         )
 
         # Send anonymous telemetry
-        event_properties = {"init_type": "yaml_config"}
+        event_properties = {"init_type": "config_data"}
         AnonymousTelemetry().capture(event_name="init", properties=event_properties)
 
         return cls(
@@ -396,7 +425,7 @@ class Pipeline(EmbedChain):
             llm=llm,
             db=db,
             embedding_model=embedding_model,
-            yaml_path=yaml_path,
+            config_data=config_data,
             auto_deploy=auto_deploy,
             chunker=chunker_config_data,
         )

embedchain/store/assistants.py

@@ -165,7 +165,7 @@ class AIAssistant:
         self.instructions = instructions
         self.assistant_id = assistant_id or str(uuid.uuid4())
         self.thread_id = thread_id or str(uuid.uuid4())
-        self.pipeline = Pipeline.from_config(yaml_path=yaml_path) if yaml_path else Pipeline()
+        self.pipeline = Pipeline.from_config(config_path=yaml_path) if yaml_path else Pipeline()
         self.pipeline.local_id = self.pipeline.config.id = self.thread_id
 
         if self.instructions:

embedchain/utils.py

@@ -1,3 +1,4 @@
+import itertools
 import json
 import logging
 import os
@@ -6,6 +7,7 @@ import string
 from typing import Any
 
 from schema import Optional, Or, Schema
+from tqdm import tqdm
 
 from embedchain.models.data_type import DataType
 
@@ -355,7 +357,7 @@ def is_valid_json_string(source: str):
         return False
 
 
-def validate_yaml_config(config_data):
+def validate_config(config_data):
     schema = Schema(
         {
             Optional("app"): {
@@ -422,3 +424,16 @@ def validate_yaml_config(config_data):
     )
 
     return schema.validate(config_data)
+
+
+def chunks(iterable, batch_size=100, desc="Processing chunks"):
+    """A helper function to break an iterable into chunks of size batch_size."""
+    it = iter(iterable)
+    total_size = len(iterable)
+
+    with tqdm(total=total_size, desc=desc, unit="batch") as pbar:
+        chunk = tuple(itertools.islice(it, batch_size))
+        while chunk:
+            yield chunk
+            pbar.update(len(chunk))
+            chunk = tuple(itertools.islice(it, batch_size))

embedchain/vectordb/chroma.py

@@ -133,6 +133,7 @@ class ChromaDB(BaseVectorDB):
         metadatas: List[object],
         ids: List[str],
         skip_embedding: bool,
+        **kwargs: Optional[Dict[str, Any]],
     ) -> Any:
         """
         Add vectors to chroma database
@@ -198,6 +199,7 @@ class ChromaDB(BaseVectorDB):
         where: Dict[str, any],
         skip_embedding: bool,
         citations: bool = False,
+        **kwargs: Optional[Dict[str, Any]],
     ) -> Union[List[Tuple[str, str, str]], List[str]]:
         """
         Query contents from vector database based on vector similarity
@@ -225,6 +227,7 @@ class ChromaDB(BaseVectorDB):
                     ],
                     n_results=n_results,
                     where=self._generate_where_clause(where),
+                    **kwargs,
                 )
             else:
                 result = self.collection.query(
@@ -233,6 +236,7 @@ class ChromaDB(BaseVectorDB):
                     ],
                     n_results=n_results,
                     where=self._generate_where_clause(where),
+                    **kwargs,
                 )
         except InvalidDimensionException as e:
             raise InvalidDimensionException(

embedchain/vectordb/elasticsearch.py

@@ -105,6 +105,7 @@ class ElasticsearchDB(BaseVectorDB):
         metadatas: List[object],
         ids: List[str],
         skip_embedding: bool,
+        **kwargs: Optional[Dict[str, any]],
     ) -> Any:
         """
         add data in vector database
@@ -142,6 +143,7 @@ class ElasticsearchDB(BaseVectorDB):
         where: Dict[str, any],
         skip_embedding: bool,
         citations: bool = False,
+        **kwargs: Optional[Dict[str, Any]],
     ) -> Union[List[Tuple[str, str, str]], List[str]]:
         """
         query contents from vector data base based on vector similarity

embedchain/vectordb/opensearch.py

@@ -1,6 +1,6 @@
 import logging
 import time
-from typing import Dict, List, Optional, Set, Tuple, Union
+from typing import Any, Dict, List, Optional, Set, Tuple, Union
 
 from tqdm import tqdm
 
@@ -121,42 +121,44 @@ class OpenSearchDB(BaseVectorDB):
         metadatas: List[object],
         ids: List[str],
         skip_embedding: bool,
+        **kwargs: Optional[Dict[str, any]],
     ):
-        """add data in vector database
+        """Add data in vector database.
 
-        :param embeddings: list of embeddings to add
+        Args:
-        :type embeddings: List[List[str]]
+            embeddings (List[List[str]]): List of embeddings to add.
-        :param documents: list of texts to add
+            documents (List[str]): List of texts to add.
-        :type documents: List[str]
+            metadatas (List[object]): List of metadata associated with docs.
-        :param metadatas: list of metadata associated with docs
+            ids (List[str]): IDs of docs.
-        :type metadatas: List[object]
+            skip_embedding (bool): If True, then embeddings are assumed to be already generated.
-        :param ids: ids of docs
-        :type ids: List[str]
-        :param skip_embedding: Optional. If True, then the embeddings are assumed to be already generated.
-        :type skip_embedding: bool
         """
+        for batch_start in tqdm(range(0, len(documents), self.BATCH_SIZE), desc="Inserting batches in opensearch"):
+            batch_end = batch_start + self.BATCH_SIZE
+            batch_documents = documents[batch_start:batch_end]
 
-        for i in tqdm(range(0, len(documents), self.BATCH_SIZE), desc="Inserting batches in opensearch"):
+            # Generate embeddings for the batch if not skipping embedding
             if not skip_embedding:
-                embeddings = self.embedder.embedding_fn(documents[i : i + self.BATCH_SIZE])
+                batch_embeddings = self.embedder.embedding_fn(batch_documents)
+            else:
+                batch_embeddings = embeddings[batch_start:batch_end]
 
-            docs = []
+            # Create document entries for bulk upload
-            for id, text, metadata, embeddings in zip(
+            batch_entries = [
-                ids[i : i + self.BATCH_SIZE],
+                {
-                documents[i : i + self.BATCH_SIZE],
+                    "_index": self._get_index(),
-                metadatas[i : i + self.BATCH_SIZE],
+                    "_id": doc_id,
-                embeddings[i : i + self.BATCH_SIZE],
+                    "_source": {"text": text, "metadata": metadata, "embeddings": embedding},
-            ):
+                }
-                docs.append(
+                for doc_id, text, metadata, embedding in zip(
-                    {
+                    ids[batch_start:batch_end], batch_documents, metadatas[batch_start:batch_end], batch_embeddings
-                        "_index": self._get_index(),
-                        "_id": id,
-                        "_source": {"text": text, "metadata": metadata, "embeddings": embeddings},
-                    }
                 )
-            bulk(self.client, docs)
+            ]
+
+            # Perform bulk operation
+            bulk(self.client, batch_entries, **kwargs)
             self.client.indices.refresh(index=self._get_index())
-            # Sleep for 0.1 seconds to avoid rate limiting
+
+            # Sleep to avoid rate limiting
             time.sleep(0.1)
 
     def query(
@@ -166,6 +168,7 @@ class OpenSearchDB(BaseVectorDB):
         where: Dict[str, any],
         skip_embedding: bool,
         citations: bool = False,
+        **kwargs: Optional[Dict[str, Any]],
     ) -> Union[List[Tuple[str, str, str]], List[str]]:
         """
         query contents from vector data base based on vector similarity
@@ -208,6 +211,7 @@ class OpenSearchDB(BaseVectorDB):
             metadata_field="metadata",
             pre_filter=pre_filter,
             k=n_results,
+            **kwargs,
         )
 
         contexts = []
@@ -250,7 +254,7 @@ class OpenSearchDB(BaseVectorDB):
         """
         # Delete all data from the database
         if self.client.indices.exists(index=self._get_index()):
-            # delete index in Es
+            # delete index in ES
             self.client.indices.delete(index=self._get_index())
 
     def delete(self, where):

embedchain/vectordb/pinecone.py

@@ -10,6 +10,7 @@ except ImportError:
 
 from embedchain.config.vectordb.pinecone import PineconeDBConfig
 from embedchain.helpers.json_serializable import register_deserializable
+from embedchain.utils import chunks
 from embedchain.vectordb.base import BaseVectorDB
 
 
@@ -92,6 +93,7 @@ class PineconeDB(BaseVectorDB):
         metadatas: List[object],
         ids: List[str],
         skip_embedding: bool,
+        **kwargs: Optional[Dict[str, any]],
     ):
         """add data in vector database
 
@@ -104,7 +106,6 @@ class PineconeDB(BaseVectorDB):
         """
         docs = []
         print("Adding documents to Pinecone...")
-
         embeddings = self.embedder.embedding_fn(documents)
         for id, text, metadata, embedding in zip(ids, documents, metadatas, embeddings):
             docs.append(
@@ -115,8 +116,8 @@ class PineconeDB(BaseVectorDB):
                 }
             )
 
-        for i in range(0, len(docs), self.BATCH_SIZE):
+        for chunk in chunks(docs, self.BATCH_SIZE, desc="Adding chunks in batches..."):
-            self.client.upsert(docs[i : i + self.BATCH_SIZE])
+            self.client.upsert(chunk, **kwargs)
 
     def query(
         self,
@@ -125,6 +126,7 @@ class PineconeDB(BaseVectorDB):
         where: Dict[str, any],
         skip_embedding: bool,
         citations: bool = False,
+        **kwargs: Optional[Dict[str, any]],
     ) -> Union[List[Tuple[str, str, str]], List[str]]:
         """
         query contents from vector database based on vector similarity
@@ -146,7 +148,7 @@ class PineconeDB(BaseVectorDB):
             query_vector = self.embedder.embedding_fn([input_query])[0]
         else:
             query_vector = input_query
-        data = self.client.query(vector=query_vector, filter=where, top_k=n_results, include_metadata=True)
+        data = self.client.query(vector=query_vector, filter=where, top_k=n_results, include_metadata=True, **kwargs)
         contexts = []
         for doc in data["matches"]:
             metadata = doc["metadata"]

embedchain/vectordb/qdrant.py

@@ -1,7 +1,7 @@
 import copy
 import os
 import uuid
-from typing import Dict, List, Optional, Tuple, Union
+from typing import Any, Dict, List, Optional, Tuple, Union
 
 try:
     from qdrant_client import QdrantClient
@@ -127,6 +127,7 @@ class QdrantDB(BaseVectorDB):
         metadatas: List[object],
         ids: List[str],
         skip_embedding: bool,
+        **kwargs: Optional[Dict[str, any]],
     ):
         """add data in vector database
         :param embeddings: list of embeddings for the corresponding documents to be added
@@ -158,6 +159,7 @@ class QdrantDB(BaseVectorDB):
                     payloads=payloads[i : i + self.BATCH_SIZE],
                     vectors=embeddings[i : i + self.BATCH_SIZE],
                 ),
+                **kwargs,
             )
 
     def query(
@@ -167,6 +169,7 @@ class QdrantDB(BaseVectorDB):
         where: Dict[str, any],
         skip_embedding: bool,
         citations: bool = False,
+        **kwargs: Optional[Dict[str, Any]],
     ) -> Union[List[Tuple[str, str, str]], List[str]]:
         """
         query contents from vector database based on vector similarity
@@ -208,6 +211,7 @@ class QdrantDB(BaseVectorDB):
             query_filter=models.Filter(must=qdrant_must_filters),
             query_vector=query_vector,
             limit=n_results,
+            **kwargs,
         )
 
         contexts = []

embedchain/vectordb/weaviate.py

@@ -1,6 +1,6 @@
 import copy
 import os
-from typing import Dict, List, Optional, Tuple, Union
+from typing import Any, Dict, List, Optional, Tuple, Union
 
 try:
     import weaviate
@@ -158,6 +158,7 @@ class WeaviateDB(BaseVectorDB):
         metadatas: List[object],
         ids: List[str],
         skip_embedding: bool,
+        **kwargs: Optional[Dict[str, any]],
     ):
         """add data in vector database
         :param embeddings: list of embeddings for the corresponding documents to be added
@@ -192,7 +193,9 @@ class WeaviateDB(BaseVectorDB):
                     class_name=self.index_name + "_metadata",
                     vector=embedding,
                 )
-                batch.add_reference(obj_uuid, self.index_name, "metadata", metadata_uuid, self.index_name + "_metadata")
+                batch.add_reference(
+                    obj_uuid, self.index_name, "metadata", metadata_uuid, self.index_name + "_metadata", **kwargs
+                )
 
     def query(
         self,
@@ -201,6 +204,7 @@ class WeaviateDB(BaseVectorDB):
         where: Dict[str, any],
         skip_embedding: bool,
         citations: bool = False,
+        **kwargs: Optional[Dict[str, Any]],
     ) -> Union[List[Tuple[str, str, str]], List[str]]:
         """
         query contents from vector database based on vector similarity

embedchain/vectordb/zilliz.py

@@ -1,5 +1,5 @@
 import logging
-from typing import Dict, List, Optional, Tuple, Union
+from typing import Any, Dict, List, Optional, Tuple, Union
 
 from embedchain.config import ZillizDBConfig
 from embedchain.helpers.json_serializable import register_deserializable
@@ -113,6 +113,7 @@ class ZillizVectorDB(BaseVectorDB):
         metadatas: List[object],
         ids: List[str],
         skip_embedding: bool,
+        **kwargs: Optional[Dict[str, any]],
     ):
         """Add to database"""
         if not skip_embedding:
@@ -120,7 +121,7 @@ class ZillizVectorDB(BaseVectorDB):
 
         for id, doc, metadata, embedding in zip(ids, documents, metadatas, embeddings):
             data = {**metadata, "id": id, "text": doc, "embeddings": embedding}
-            self.client.insert(collection_name=self.config.collection_name, data=data)
+            self.client.insert(collection_name=self.config.collection_name, data=data, **kwargs)
 
         self.collection.load()
         self.collection.flush()
@@ -133,6 +134,7 @@ class ZillizVectorDB(BaseVectorDB):
         where: Dict[str, any],
         skip_embedding: bool,
         citations: bool = False,
+        **kwargs: Optional[Dict[str, Any]],
     ) -> Union[List[Tuple[str, str, str]], List[str]]:
         """
         Query contents from vector data base based on vector similarity
@@ -165,6 +167,7 @@ class ZillizVectorDB(BaseVectorDB):
                 data=query_vector,
                 limit=n_results,
                 output_fields=output_fields,
+                **kwargs,
             )
 
         else:
@@ -176,6 +179,7 @@ class ZillizVectorDB(BaseVectorDB):
                 data=[query_vector],
                 limit=n_results,
                 output_fields=output_fields,
+                **kwargs,
             )
 
         contexts = []

examples/rest-api/main.py

@@ -108,7 +108,7 @@ async def get_datasources_associated_with_app_id(app_id: str, db: Session = Depe
         if db_app is None:
             raise HTTPException(detail=f"App with id {app_id} does not exist, please create it first.", status_code=400)
 
-        app = App.from_config(yaml_path=db_app.config)
+        app = App.from_config(config_path=db_app.config)
 
         response = app.get_data_sources()
         return {"results": response}
@@ -147,7 +147,7 @@ async def add_datasource_to_an_app(body: SourceApp, app_id: str, db: Session = D
         if db_app is None:
             raise HTTPException(detail=f"App with id {app_id} does not exist, please create it first.", status_code=400)
 
-        app = App.from_config(yaml_path=db_app.config)
+        app = App.from_config(config_path=db_app.config)
 
         response = app.add(source=body.source, data_type=body.data_type)
         return DefaultResponse(response=response)
@@ -185,7 +185,7 @@ async def query_an_app(body: QueryApp, app_id: str, db: Session = Depends(get_db
         if db_app is None:
             raise HTTPException(detail=f"App with id {app_id} does not exist, please create it first.", status_code=400)
 
-        app = App.from_config(yaml_path=db_app.config)
+        app = App.from_config(config_path=db_app.config)
 
         response = app.query(body.query)
         return DefaultResponse(response=response)
@@ -227,7 +227,7 @@ async def query_an_app(body: QueryApp, app_id: str, db: Session = Depends(get_db
 #               status_code=400
 #             )
 
-#         app = App.from_config(yaml_path=db_app.config)
+#         app = App.from_config(config_path=db_app.config)
 
 #         response = app.chat(body.message)
 #         return DefaultResponse(response=response)
@@ -264,7 +264,7 @@ async def deploy_app(body: DeployAppRequest, app_id: str, db: Session = Depends(
         if db_app is None:
             raise HTTPException(detail=f"App with id {app_id} does not exist, please create it first.", status_code=400)
 
-        app = App.from_config(yaml_path=db_app.config)
+        app = App.from_config(config_path=db_app.config)
 
         api_key = body.api_key
         # this will save the api key in the embedchain.db
@@ -305,7 +305,7 @@ async def delete_app(app_id: str, db: Session = Depends(get_db)):
         if db_app is None:
             raise HTTPException(detail=f"App with id {app_id} does not exist, please create it first.", status_code=400)
 
-        app = App.from_config(yaml_path=db_app.config)
+        app = App.from_config(config_path=db_app.config)
 
         # reset app.db
         app.db.reset()

notebooks/anthropic.ipynb

@@ -30,15 +30,6 @@
         "outputId": "efdce0dc-fb30-4e01-f5a8-ef1a7f4e8c09"
       },
       "outputs": [],
-      "source": [
-        "!pip install embedchain"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
       "source": [
         "!pip install embedchain[dataloaders]"
       ]
@@ -118,7 +109,7 @@
       },
       "outputs": [],
       "source": [
-        "app = App.from_config(yaml_path=\"anthropic.yaml\")"
+        "app = App.from_config(config_path=\"anthropic.yaml\")"
       ]
     },
     {

notebooks/azure-openai.ipynb

@@ -22,16 +22,6 @@
    "id": "b80ff15a",
    "metadata": {},
    "outputs": [],
-   "source": [
-    "!pip install embedchain"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "692ff37b",
-   "metadata": {},
-   "outputs": [],
    "source": [
     "!pip install embedchain[dataloaders]"
    ]
@@ -115,7 +105,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "app = App.from_config(yaml_path=\"azure_openai.yaml\")"
+    "app = App.from_config(config_path=\"azure_openai.yaml\")"
    ]
   },
   {
@@ -158,14 +148,6 @@
     "    answer = app.query(question)\n",
     "    print(answer)"
    ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "e1f2ead5",
-   "metadata": {},
-   "outputs": [],
-   "source": []
   }
  ],
  "metadata": {

notebooks/chromadb.ipynb

@@ -25,15 +25,6 @@
         "id": "-NbXjAdlh0vJ"
       },
       "outputs": [],
-      "source": [
-        "!pip install embedchain"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
       "source": [
         "!pip install embedchain[dataloaders]"
       ]
@@ -114,7 +105,7 @@
       },
       "outputs": [],
       "source": [
-        "app = App.from_config(yaml_path=\"chromadb.yaml\")"
+        "app = App.from_config(config_path=\"chromadb.yaml\")"
       ]
     },
     {

notebooks/cohere.ipynb

@@ -30,16 +30,7 @@
       },
       "outputs": [],
       "source": [
-        "!pip install embedchain"
+        "!pip install embedchain[dataloaders,cohere]"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "!pip install embedchain[dataloaders]"
       ]
     },
     {
@@ -48,27 +39,11 @@
         "id": "nGnpSYAAh2bQ"
       },
       "source": [
-        "### Step-2: Set Cohere related environment variables and install the dependencies\n",
+        "### Step-2: Set Cohere related environment variables\n",
         "\n",
         "You can find `OPENAI_API_KEY` on your [OpenAI dashboard](https://platform.openai.com/account/api-keys) and `COHERE_API_KEY` key on your [Cohere dashboard](https://dashboard.cohere.com/api-keys)."
       ]
     },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "colab": {
-          "base_uri": "https://localhost:8080/",
-          "height": 1000
-        },
-        "id": "S5jTywPZNtrj",
-        "outputId": "4a23c813-c9e5-4b6c-e3d9-b41e4fdbc54d"
-      },
-      "outputs": [],
-      "source": [
-        "!pip install embedchain[cohere]"
-      ]
-    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -139,7 +114,7 @@
       },
       "outputs": [],
       "source": [
-        "app = App.from_config(yaml_path=\"cohere.yaml\")"
+        "app = App.from_config(config_path=\"cohere.yaml\")"
       ]
     },
     {

notebooks/elasticsearch.ipynb

@@ -26,16 +26,7 @@
       },
       "outputs": [],
       "source": [
-        "!pip install embedchain"
+        "!pip install embedchain[dataloaders,elasticsearch]"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "!pip install embedchain[dataloaders]"
       ]
     },
     {
@@ -44,20 +35,9 @@
         "id": "nGnpSYAAh2bQ"
       },
       "source": [
-        "### Step-2: Set OpenAI environment variables and install the dependencies.\n",
+        "### Step-2: Set OpenAI environment variables.\n",
         "\n",
-        "You can find this env variable on your [OpenAI dashboard](https://platform.openai.com/account/api-keys). Now lets install the dependencies needed for Elasticsearch."
+        "You can find this env variable on your [OpenAI dashboard](https://platform.openai.com/account/api-keys)."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "-MUFRfxV7Jk7"
-      },
-      "outputs": [],
-      "source": [
-        "!pip install --upgrade 'embedchain[elasticsearch]'"
       ]
     },
     {
@@ -123,7 +103,7 @@
       },
       "outputs": [],
       "source": [
-        "app = App.from_config(yaml_path=\"elasticsearch.yaml\")"
+        "app = App.from_config(config_path=\"elasticsearch.yaml\")"
       ]
     },
     {

notebooks/gpt4all.ipynb

@@ -30,16 +30,7 @@
       },
       "outputs": [],
       "source": [
-        "!pip install embedchain"
+        "!pip install embedchain[dataloaders,opensource]"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "!pip install embedchain[dataloaders]"
       ]
     },
     {
@@ -48,24 +39,9 @@
         "id": "nGnpSYAAh2bQ"
       },
       "source": [
-        "### Step-2: Set GPT4ALL related environment variables and install dependencies\n",
+        "### Step-2: Set GPT4ALL related environment variables\n",
         "\n",
-        "GPT4All is free for all and doesn't require any API Key to use it. Just import the dependencies."
+        "GPT4All is free for all and doesn't require any API Key to use it. So you can use it for free!"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "colab": {
-          "base_uri": "https://localhost:8080/"
-        },
-        "id": "dGOE4u3dC6at",
-        "outputId": "c1c0087b-3f14-49fa-fb86-a4a3391ba14c"
-      },
-      "outputs": [],
-      "source": [
-        "!pip install --upgrade embedchain[opensource]"
       ]
     },
     {
@@ -138,7 +114,7 @@
       },
       "outputs": [],
       "source": [
-        "app = App.from_config(yaml_path=\"gpt4all.yaml\")"
+        "app = App.from_config(config_path=\"gpt4all.yaml\")"
       ]
     },
     {

notebooks/hugging_face_hub.ipynb

@@ -31,16 +31,7 @@
       },
       "outputs": [],
       "source": [
-        "!pip install embedchain"
+        "!pip install embedchain[dataloaders,huggingface_hub,opensource]"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "!pip install embedchain[dataloaders]"
       ]
     },
     {
@@ -49,39 +40,9 @@
         "id": "nGnpSYAAh2bQ"
       },
       "source": [
-        "### Step-2: Set Hugging Face Hub related environment variables and install dependencies\n",
+        "### Step-2: Set Hugging Face Hub related environment variables\n",
         "\n",
-        "You can find your `HUGGINGFACE_ACCESS_TOKEN` key on your [Hugging Face Hub dashboard](https://huggingface.co/settings/tokens) and install the dependencies"
+        "You can find your `HUGGINGFACE_ACCESS_TOKEN` key on your [Hugging Face Hub dashboard](https://huggingface.co/settings/tokens)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "colab": {
-          "base_uri": "https://localhost:8080/"
-        },
-        "id": "VfDNZJCqNfqo",
-        "outputId": "34894d35-7142-42ee-8564-2e9f718afcbb"
-      },
-      "outputs": [],
-      "source": [
-        "!pip install embedchain[huggingface-hub]"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "colab": {
-          "base_uri": "https://localhost:8080/"
-        },
-        "id": "SCNT8khqcR3G",
-        "outputId": "b789ee77-ef50-4330-8ac6-5da645dc36d6"
-      },
-      "outputs": [],
-      "source": [
-        "!pip install embedchain[opensource]"
       ]
     },
     {
@@ -153,7 +114,7 @@
       },
       "outputs": [],
       "source": [
-        "app = App.from_config(yaml_path=\"huggingface.yaml\")"
+        "app = App.from_config(config_path=\"huggingface.yaml\")"
       ]
     },
     {
@@ -209,15 +170,6 @@
         "    answer = app.query(question)\n",
         "    print(answer)"
       ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "HvZVn6gU5xB_"
-      },
-      "outputs": [],
-      "source": []
     }
   ],
   "metadata": {

notebooks/jina.ipynb

@@ -30,15 +30,6 @@
         "outputId": "69cb79a6-c758-4656-ccf7-9f3105c81d16"
       },
       "outputs": [],
-      "source": [
-        "!pip install embedchain"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
       "source": [
         "!pip install embedchain[dataloaders]"
       ]
@@ -123,7 +114,7 @@
       },
       "outputs": [],
       "source": [
-        "app = App.from_config(yaml_path=\"jina.yaml\")"
+        "app = App.from_config(config_path=\"jina.yaml\")"
       ]
     },
     {

notebooks/llama2.ipynb

@@ -30,16 +30,7 @@
       },
       "outputs": [],
       "source": [
-        "!pip install embedchain"
+        "!pip install embedchain[dataloaders,llama2]"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "!pip install embedchain[dataloaders]"
       ]
     },
     {
@@ -48,20 +39,9 @@
         "id": "nGnpSYAAh2bQ"
       },
       "source": [
-        "### Step-2: Set LLAMA2 related environment variables and install dependencies\n",
+        "### Step-2: Set LLAMA2 related environment variables\n",
         "\n",
-        "You can find `OPENAI_API_KEY` on your [OpenAI dashboard](https://platform.openai.com/account/api-keys) and `REPLICATE_API_TOKEN` key on your [Replicate dashboard](https://replicate.com/account/api-tokens). Now lets install the dependencies for LLAMA2."
+        "You can find `OPENAI_API_KEY` on your [OpenAI dashboard](https://platform.openai.com/account/api-keys) and `REPLICATE_API_TOKEN` key on your [Replicate dashboard](https://replicate.com/account/api-tokens)."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "qoBUbocNtUUD"
-      },
-      "outputs": [],
-      "source": [
-        "!pip install embedchain[llama2]"
       ]
     },
     {
@@ -129,7 +109,7 @@
       },
       "outputs": [],
       "source": [
-        "app = App.from_config(yaml_path=\"llama2.yaml\")"
+        "app = App.from_config(config_path=\"llama2.yaml\")"
       ]
     },
     {

notebooks/openai.ipynb

@@ -30,15 +30,6 @@
         "outputId": "6c630676-c7fc-4054-dc94-c613de58a037"
       },
       "outputs": [],
-      "source": [
-        "!pip install embedchain"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
       "source": [
         "!pip install embedchain[dataloaders]"
       ]
@@ -124,7 +115,7 @@
       },
       "outputs": [],
       "source": [
-        "app = App.from_config(yaml_path=\"openai.yaml\")"
+        "app = App.from_config(config_path=\"openai.yaml\")"
       ]
     },
     {

notebooks/opensearch.ipynb

@@ -26,16 +26,7 @@
       },
       "outputs": [],
       "source": [
-        "!pip install embedchain"
+        "!pip install embedchain[dataloaders,opensearch]"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "!pip install embedchain[dataloaders]"
       ]
     },
     {
@@ -49,17 +40,6 @@
         "You can find this env variable on your [OpenAI dashboard](https://platform.openai.com/account/api-keys). Now lets install the dependencies needed for Opensearch."
       ]
     },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "-MUFRfxV7Jk7"
-      },
-      "outputs": [],
-      "source": [
-        "!pip install --upgrade 'embedchain[opensearch]'"
-      ]
-    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -127,7 +107,7 @@
       },
       "outputs": [],
       "source": [
-        "app = App.from_config(yaml_path=\"opensearch.yaml\")"
+        "app = App.from_config(config_path=\"opensearch.yaml\")"
       ]
     },
     {

notebooks/pinecone.ipynb

@@ -26,16 +26,7 @@
       },
       "outputs": [],
       "source": [
-        "!pip install embedchain"
+        "!pip install embedchain[dataloaders,pinecone]"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "!pip install embedchain[dataloaders]"
       ]
     },
     {
@@ -44,20 +35,9 @@
         "id": "nGnpSYAAh2bQ"
       },
       "source": [
-        "### Step-2: Set environment variables needed for Pinecone and install the dependencies.\n",
+        "### Step-2: Set environment variables needed for Pinecone\n",
         "\n",
-        "You can find this env variable on your [OpenAI dashboard](https://platform.openai.com/account/api-keys) and [Pinecone dashboard](https://app.pinecone.io/). Now lets install the dependencies needed for Pinecone."
+        "You can find this env variable on your [OpenAI dashboard](https://platform.openai.com/account/api-keys) and [Pinecone dashboard](https://app.pinecone.io/)."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "-MUFRfxV7Jk7"
-      },
-      "outputs": [],
-      "source": [
-        "!pip install --upgrade 'embedchain[pinecone]'"
       ]
     },
     {
@@ -124,7 +104,7 @@
       },
       "outputs": [],
       "source": [
-        "app = App.from_config(yaml_path=\"pinecone.yaml\")"
+        "app = App.from_config(config_path=\"pinecone.yaml\")"
       ]
     },
     {

notebooks/vertex_ai.ipynb

@@ -30,16 +30,7 @@
       },
       "outputs": [],
       "source": [
-        "!pip install embedchain"
+        "!pip install embedchain[dataloaders,vertexai]"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {},
-      "outputs": [],
-      "source": [
-        "!pip install embedchain[dataloaders]"
       ]
     },
     {
@@ -48,20 +39,9 @@
         "id": "nGnpSYAAh2bQ"
       },
       "source": [
-        "### Step-2: Set VertexAI related environment variables and install dependencies.\n",
+        "### Step-2: Set VertexAI related environment variables\n",
         "\n",
-        "You can find `OPENAI_API_KEY` on your [OpenAI dashboard](https://platform.openai.com/account/api-keys). Now lets install the dependencies."
+        "You can find `OPENAI_API_KEY` on your [OpenAI dashboard](https://platform.openai.com/account/api-keys)."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "a_shbIFBtnwu"
-      },
-      "outputs": [],
-      "source": [
-        "!pip install embedchain[vertexai]"
       ]
     },
     {
@@ -137,7 +117,7 @@
       },
       "outputs": [],
       "source": [
-        "app = App.from_config(yaml_path=\"vertexai.yaml\")"
+        "app = App.from_config(config_path=\"vertexai.yaml\")"
       ]
     },
     {

poetry.lock

@@ -337,6 +337,26 @@ description = "The uncompromising code formatter."
 optional = false
 python-versions = ">=3.8"
 files = [
+    {file = "black-23.9.1-cp310-cp310-macosx_10_16_arm64.whl", hash = "sha256:d6bc09188020c9ac2555a498949401ab35bb6bf76d4e0f8ee251694664df6301"},
+    {file = "black-23.9.1-cp310-cp310-macosx_10_16_universal2.whl", hash = "sha256:13ef033794029b85dfea8032c9d3b92b42b526f1ff4bf13b2182ce4e917f5100"},
+    {file = "black-23.9.1-cp310-cp310-macosx_10_16_x86_64.whl", hash = "sha256:75a2dc41b183d4872d3a500d2b9c9016e67ed95738a3624f4751a0cb4818fe71"},
+    {file = "black-23.9.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:13a2e4a93bb8ca74a749b6974925c27219bb3df4d42fc45e948a5d9feb5122b7"},
+    {file = "black-23.9.1-cp310-cp310-win_amd64.whl", hash = "sha256:adc3e4442eef57f99b5590b245a328aad19c99552e0bdc7f0b04db6656debd80"},
+    {file = "black-23.9.1-cp311-cp311-macosx_10_16_arm64.whl", hash = "sha256:8431445bf62d2a914b541da7ab3e2b4f3bc052d2ccbf157ebad18ea126efb91f"},
+    {file = "black-23.9.1-cp311-cp311-macosx_10_16_universal2.whl", hash = "sha256:8fc1ddcf83f996247505db6b715294eba56ea9372e107fd54963c7553f2b6dfe"},
+    {file = "black-23.9.1-cp311-cp311-macosx_10_16_x86_64.whl", hash = "sha256:7d30ec46de88091e4316b17ae58bbbfc12b2de05e069030f6b747dfc649ad186"},
+    {file = "black-23.9.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:031e8c69f3d3b09e1aa471a926a1eeb0b9071f80b17689a655f7885ac9325a6f"},
+    {file = "black-23.9.1-cp311-cp311-win_amd64.whl", hash = "sha256:538efb451cd50f43aba394e9ec7ad55a37598faae3348d723b59ea8e91616300"},
+    {file = "black-23.9.1-cp38-cp38-macosx_10_16_arm64.whl", hash = "sha256:638619a559280de0c2aa4d76f504891c9860bb8fa214267358f0a20f27c12948"},
+    {file = "black-23.9.1-cp38-cp38-macosx_10_16_universal2.whl", hash = "sha256:a732b82747235e0542c03bf352c126052c0fbc458d8a239a94701175b17d4855"},
+    {file = "black-23.9.1-cp38-cp38-macosx_10_16_x86_64.whl", hash = "sha256:cf3a4d00e4cdb6734b64bf23cd4341421e8953615cba6b3670453737a72ec204"},
+    {file = "black-23.9.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cf99f3de8b3273a8317681d8194ea222f10e0133a24a7548c73ce44ea1679377"},
+    {file = "black-23.9.1-cp38-cp38-win_amd64.whl", hash = "sha256:14f04c990259576acd093871e7e9b14918eb28f1866f91968ff5524293f9c573"},
+    {file = "black-23.9.1-cp39-cp39-macosx_10_16_arm64.whl", hash = "sha256:c619f063c2d68f19b2d7270f4cf3192cb81c9ec5bc5ba02df91471d0b88c4c5c"},
+    {file = "black-23.9.1-cp39-cp39-macosx_10_16_universal2.whl", hash = "sha256:6a3b50e4b93f43b34a9d3ef00d9b6728b4a722c997c99ab09102fd5efdb88325"},
+    {file = "black-23.9.1-cp39-cp39-macosx_10_16_x86_64.whl", hash = "sha256:c46767e8df1b7beefb0899c4a95fb43058fa8500b6db144f4ff3ca38eb2f6393"},
+    {file = "black-23.9.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:50254ebfa56aa46a9fdd5d651f9637485068a1adf42270148cd101cdf56e0ad9"},
+    {file = "black-23.9.1-cp39-cp39-win_amd64.whl", hash = "sha256:403397c033adbc45c2bd41747da1f7fc7eaa44efbee256b53842470d4ac5a70f"},
     {file = "black-23.9.1-py3-none-any.whl", hash = "sha256:6ccd59584cc834b6d127628713e4b6b968e5f79572da66284532525a042549f9"},
     {file = "black-23.9.1.tar.gz", hash = "sha256:24b6b3ff5c6d9ea08a8888f6977eae858e1f340d7260cf56d70a49823236b62d"},
 ]
@@ -2050,7 +2070,7 @@ files = [
     {file = "greenlet-3.0.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0b72b802496cccbd9b31acea72b6f87e7771ccfd7f7927437d592e5c92ed703c"},
     {file = "greenlet-3.0.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:527cd90ba3d8d7ae7dceb06fda619895768a46a1b4e423bdb24c1969823b8362"},
     {file = "greenlet-3.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:37f60b3a42d8b5499be910d1267b24355c495064f271cfe74bf28b17b099133c"},
-    {file = "greenlet-3.0.0-cp311-universal2-macosx_10_9_universal2.whl", hash = "sha256:c3692ecf3fe754c8c0f2c95ff19626584459eab110eaab66413b1e7425cd84e9"},
+    {file = "greenlet-3.0.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:1482fba7fbed96ea7842b5a7fc11d61727e8be75a077e603e8ab49d24e234383"},
     {file = "greenlet-3.0.0-cp312-cp312-macosx_13_0_arm64.whl", hash = "sha256:be557119bf467d37a8099d91fbf11b2de5eb1fd5fc5b91598407574848dc910f"},
     {file = "greenlet-3.0.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:73b2f1922a39d5d59cc0e597987300df3396b148a9bd10b76a058a2f2772fc04"},
     {file = "greenlet-3.0.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d1e22c22f7826096ad503e9bb681b05b8c1f5a8138469b255eb91f26a76634f2"},
@@ -2060,7 +2080,6 @@ files = [
     {file = "greenlet-3.0.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:952256c2bc5b4ee8df8dfc54fc4de330970bf5d79253c863fb5e6761f00dda35"},
     {file = "greenlet-3.0.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:269d06fa0f9624455ce08ae0179430eea61085e3cf6457f05982b37fd2cefe17"},
     {file = "greenlet-3.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:9adbd8ecf097e34ada8efde9b6fec4dd2a903b1e98037adf72d12993a1c80b51"},
-    {file = "greenlet-3.0.0-cp312-universal2-macosx_10_9_universal2.whl", hash = "sha256:553d6fb2324e7f4f0899e5ad2c427a4579ed4873f42124beba763f16032959af"},
     {file = "greenlet-3.0.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c6b5ce7f40f0e2f8b88c28e6691ca6806814157ff05e794cdd161be928550f4c"},
     {file = "greenlet-3.0.0-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ecf94aa539e97a8411b5ea52fc6ccd8371be9550c4041011a091eb8b3ca1d810"},
     {file = "greenlet-3.0.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:80dcd3c938cbcac986c5c92779db8e8ce51a89a849c135172c88ecbdc8c056b7"},
@@ -3106,6 +3125,16 @@ files = [
     {file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:5bbe06f8eeafd38e5d0a4894ffec89378b6c6a625ff57e3028921f8ff59318ac"},
     {file = "MarkupSafe-2.1.3-cp311-cp311-win32.whl", hash = "sha256:dd15ff04ffd7e05ffcb7fe79f1b98041b8ea30ae9234aed2a9168b5797c3effb"},
     {file = "MarkupSafe-2.1.3-cp311-cp311-win_amd64.whl", hash = "sha256:134da1eca9ec0ae528110ccc9e48041e0828d79f24121a1a146161103c76e686"},
+    {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:f698de3fd0c4e6972b92290a45bd9b1536bffe8c6759c62471efaa8acb4c37bc"},
+    {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:aa57bd9cf8ae831a362185ee444e15a93ecb2e344c8e52e4d721ea3ab6ef1823"},
+    {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ffcc3f7c66b5f5b7931a5aa68fc9cecc51e685ef90282f4a82f0f5e9b704ad11"},
+    {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47d4f1c5f80fc62fdd7777d0d40a2e9dda0a05883ab11374334f6c4de38adffd"},
+    {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1f67c7038d560d92149c060157d623c542173016c4babc0c1913cca0564b9939"},
+    {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:9aad3c1755095ce347e26488214ef77e0485a3c34a50c5a5e2471dff60b9dd9c"},
+    {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:14ff806850827afd6b07a5f32bd917fb7f45b046ba40c57abdb636674a8b559c"},
+    {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8f9293864fe09b8149f0cc42ce56e3f0e54de883a9de90cd427f191c346eb2e1"},
+    {file = "MarkupSafe-2.1.3-cp312-cp312-win32.whl", hash = "sha256:715d3562f79d540f251b99ebd6d8baa547118974341db04f5ad06d5ea3eb8007"},
+    {file = "MarkupSafe-2.1.3-cp312-cp312-win_amd64.whl", hash = "sha256:1b8dd8c3fd14349433c79fa8abeb573a55fc0fdd769133baac1f5e07abf54aeb"},
     {file = "MarkupSafe-2.1.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8e254ae696c88d98da6555f5ace2279cf7cd5b3f52be2b5cf97feafe883b58d2"},
     {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb0932dc158471523c9637e807d9bfb93e06a95cbf010f1a38b98623b929ef2b"},
     {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9402b03f1a1b4dc4c19845e5c749e3ab82d5078d16a2a4c2cd2df62d57bb0707"},
@@ -4829,6 +4858,23 @@ files = [
 [package.dependencies]
 typing-extensions = ">=4.6.0,<4.7.0 || >4.7.0"
 
+[[package]]
+name = "pygithub"
+version = "1.59.1"
+description = "Use the full Github API v3"
+optional = true
+python-versions = ">=3.7"
+files = [
+    {file = "PyGithub-1.59.1-py3-none-any.whl", hash = "sha256:3d87a822e6c868142f0c2c4bf16cce4696b5a7a4d142a7bd160e1bdf75bc54a9"},
+    {file = "PyGithub-1.59.1.tar.gz", hash = "sha256:c44e3a121c15bf9d3a5cc98d94c9a047a5132a9b01d22264627f58ade9ddc217"},
+]
+
+[package.dependencies]
+deprecated = "*"
+pyjwt = {version = ">=2.4.0", extras = ["crypto"]}
+pynacl = ">=1.4.0"
+requests = ">=2.14.0"
+
 [[package]]
 name = "pyjwt"
 version = "2.8.0"
@@ -4840,6 +4886,9 @@ files = [
     {file = "PyJWT-2.8.0.tar.gz", hash = "sha256:57e28d156e3d5c10088e0c68abb90bfac3df82b40a71bd0daa20c65ccd5c23de"},
 ]
 
+[package.dependencies]
+cryptography = {version = ">=3.4.0", optional = true, markers = "extra == \"crypto\""}
+
 [package.extras]
 crypto = ["cryptography (>=3.4.0)"]
 dev = ["coverage[toml] (==5.0.4)", "cryptography (>=3.4.0)", "pre-commit", "pytest (>=6.0.0,<7.0.0)", "sphinx (>=4.5.0,<5.0.0)", "sphinx-rtd-theme", "zope.interface"]
@@ -4866,6 +4915,32 @@ protobuf = ">=3.20.0"
 requests = "*"
 ujson = ">=2.0.0"
 
+[[package]]
+name = "pynacl"
+version = "1.5.0"
+description = "Python binding to the Networking and Cryptography (NaCl) library"
+optional = true
+python-versions = ">=3.6"
+files = [
+    {file = "PyNaCl-1.5.0-cp36-abi3-macosx_10_10_universal2.whl", hash = "sha256:401002a4aaa07c9414132aaed7f6836ff98f59277a234704ff66878c2ee4a0d1"},
+    {file = "PyNaCl-1.5.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:52cb72a79269189d4e0dc537556f4740f7f0a9ec41c1322598799b0bdad4ef92"},
+    {file = "PyNaCl-1.5.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a36d4a9dda1f19ce6e03c9a784a2921a4b726b02e1c736600ca9c22029474394"},
+    {file = "PyNaCl-1.5.0-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:0c84947a22519e013607c9be43706dd42513f9e6ae5d39d3613ca1e142fba44d"},
+    {file = "PyNaCl-1.5.0-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:06b8f6fa7f5de8d5d2f7573fe8c863c051225a27b61e6860fd047b1775807858"},
+    {file = "PyNaCl-1.5.0-cp36-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:a422368fc821589c228f4c49438a368831cb5bbc0eab5ebe1d7fac9dded6567b"},
+    {file = "PyNaCl-1.5.0-cp36-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:61f642bf2378713e2c2e1de73444a3778e5f0a38be6fee0fe532fe30060282ff"},
+    {file = "PyNaCl-1.5.0-cp36-abi3-win32.whl", hash = "sha256:e46dae94e34b085175f8abb3b0aaa7da40767865ac82c928eeb9e57e1ea8a543"},
+    {file = "PyNaCl-1.5.0-cp36-abi3-win_amd64.whl", hash = "sha256:20f42270d27e1b6a29f54032090b972d97f0a1b0948cc52392041ef7831fee93"},
+    {file = "PyNaCl-1.5.0.tar.gz", hash = "sha256:8ac7448f09ab85811607bdd21ec2464495ac8b7c66d146bf545b0f08fb9220ba"},
+]
+
+[package.dependencies]
+cffi = ">=1.4.1"
+
+[package.extras]
+docs = ["sphinx (>=1.6.5)", "sphinx-rtd-theme"]
+tests = ["hypothesis (>=3.27.0)", "pytest (>=3.2.1,!=3.3.0)"]
+
 [[package]]
 name = "pypandoc"
 version = "1.11"
@@ -5215,6 +5290,7 @@ files = [
     {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"},
     {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"},
     {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"},
+    {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"},
     {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"},
     {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"},
     {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"},
@@ -5222,8 +5298,15 @@ files = [
     {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"},
     {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"},
     {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"},
+    {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"},
     {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"},
     {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"},
+    {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"},
+    {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"},
+    {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"},
+    {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"},
+    {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"},
+    {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"},
     {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"},
     {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"},
     {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"},
@@ -5240,6 +5323,7 @@ files = [
     {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"},
     {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"},
     {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"},
+    {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"},
     {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"},
     {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"},
     {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"},
@@ -5247,6 +5331,7 @@ files = [
     {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"},
     {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"},
     {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"},
+    {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"},
     {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"},
     {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"},
     {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"},
@@ -5774,6 +5859,11 @@ files = [
     {file = "scikit_learn-1.3.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f66eddfda9d45dd6cadcd706b65669ce1df84b8549875691b1f403730bdef217"},
     {file = "scikit_learn-1.3.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c6448c37741145b241eeac617028ba6ec2119e1339b1385c9720dae31367f2be"},
     {file = "scikit_learn-1.3.1-cp311-cp311-win_amd64.whl", hash = "sha256:c413c2c850241998168bbb3bd1bb59ff03b1195a53864f0b80ab092071af6028"},
+    {file = "scikit_learn-1.3.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:ef540e09873e31569bc8b02c8a9f745ee04d8e1263255a15c9969f6f5caa627f"},
+    {file = "scikit_learn-1.3.1-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:9147a3a4df4d401e618713880be023e36109c85d8569b3bf5377e6cd3fecdeac"},
+    {file = "scikit_learn-1.3.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d2cd3634695ad192bf71645702b3df498bd1e246fc2d529effdb45a06ab028b4"},
+    {file = "scikit_learn-1.3.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0c275a06c5190c5ce00af0acbb61c06374087949f643ef32d355ece12c4db043"},
+    {file = "scikit_learn-1.3.1-cp312-cp312-win_amd64.whl", hash = "sha256:0e1aa8f206d0de814b81b41d60c1ce31f7f2c7354597af38fae46d9c47c45122"},
     {file = "scikit_learn-1.3.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:52b77cc08bd555969ec5150788ed50276f5ef83abb72e6f469c5b91a0009bbca"},
     {file = "scikit_learn-1.3.1-cp38-cp38-macosx_12_0_arm64.whl", hash = "sha256:a683394bc3f80b7c312c27f9b14ebea7766b1f0a34faf1a2e9158d80e860ec26"},
     {file = "scikit_learn-1.3.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a15d964d9eb181c79c190d3dbc2fff7338786bf017e9039571418a1d53dab236"},
@@ -6088,13 +6178,54 @@ description = "Database Abstraction Library"
 optional = false
 python-versions = ">=3.7"
 files = [
+    {file = "SQLAlchemy-2.0.22-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f146c61ae128ab43ea3a0955de1af7e1633942c2b2b4985ac51cc292daf33222"},
+    {file = "SQLAlchemy-2.0.22-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:875de9414393e778b655a3d97d60465eb3fae7c919e88b70cc10b40b9f56042d"},
+    {file = "SQLAlchemy-2.0.22-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:13790cb42f917c45c9c850b39b9941539ca8ee7917dacf099cc0b569f3d40da7"},
+    {file = "SQLAlchemy-2.0.22-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e04ab55cf49daf1aeb8c622c54d23fa4bec91cb051a43cc24351ba97e1dd09f5"},
+    {file = "SQLAlchemy-2.0.22-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:a42c9fa3abcda0dcfad053e49c4f752eef71ecd8c155221e18b99d4224621176"},
+    {file = "SQLAlchemy-2.0.22-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:14cd3bcbb853379fef2cd01e7c64a5d6f1d005406d877ed9509afb7a05ff40a5"},
+    {file = "SQLAlchemy-2.0.22-cp310-cp310-win32.whl", hash = "sha256:d143c5a9dada696bcfdb96ba2de4a47d5a89168e71d05a076e88a01386872f97"},
+    {file = "SQLAlchemy-2.0.22-cp310-cp310-win_amd64.whl", hash = "sha256:ccd87c25e4c8559e1b918d46b4fa90b37f459c9b4566f1dfbce0eb8122571547"},
     {file = "SQLAlchemy-2.0.22-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4f6ff392b27a743c1ad346d215655503cec64405d3b694228b3454878bf21590"},
     {file = "SQLAlchemy-2.0.22-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f776c2c30f0e5f4db45c3ee11a5f2a8d9de68e81eb73ec4237de1e32e04ae81c"},
+    {file = "SQLAlchemy-2.0.22-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c8f1792d20d2f4e875ce7a113f43c3561ad12b34ff796b84002a256f37ce9437"},
+    {file = "SQLAlchemy-2.0.22-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d80eeb5189d7d4b1af519fc3f148fe7521b9dfce8f4d6a0820e8f5769b005051"},
+    {file = "SQLAlchemy-2.0.22-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:69fd9e41cf9368afa034e1c81f3570afb96f30fcd2eb1ef29cb4d9371c6eece2"},
+    {file = "SQLAlchemy-2.0.22-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:54bcceaf4eebef07dadfde424f5c26b491e4a64e61761dea9459103ecd6ccc95"},
+    {file = "SQLAlchemy-2.0.22-cp311-cp311-win32.whl", hash = "sha256:7ee7ccf47aa503033b6afd57efbac6b9e05180f492aeed9fcf70752556f95624"},
+    {file = "SQLAlchemy-2.0.22-cp311-cp311-win_amd64.whl", hash = "sha256:b560f075c151900587ade06706b0c51d04b3277c111151997ea0813455378ae0"},
     {file = "SQLAlchemy-2.0.22-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:2c9bac865ee06d27a1533471405ad240a6f5d83195eca481f9fc4a71d8b87df8"},
     {file = "SQLAlchemy-2.0.22-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:625b72d77ac8ac23da3b1622e2da88c4aedaee14df47c8432bf8f6495e655de2"},
+    {file = "SQLAlchemy-2.0.22-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b39a6e21110204a8c08d40ff56a73ba542ec60bab701c36ce721e7990df49fb9"},
+    {file = "SQLAlchemy-2.0.22-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:53a766cb0b468223cafdf63e2d37f14a4757476157927b09300c8c5832d88560"},
+    {file = "SQLAlchemy-2.0.22-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:0e1ce8ebd2e040357dde01a3fb7d30d9b5736b3e54a94002641dfd0aa12ae6ce"},
+    {file = "SQLAlchemy-2.0.22-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:505f503763a767556fa4deae5194b2be056b64ecca72ac65224381a0acab7ebe"},
+    {file = "SQLAlchemy-2.0.22-cp312-cp312-win32.whl", hash = "sha256:154a32f3c7b00de3d090bc60ec8006a78149e221f1182e3edcf0376016be9396"},
+    {file = "SQLAlchemy-2.0.22-cp312-cp312-win_amd64.whl", hash = "sha256:129415f89744b05741c6f0b04a84525f37fbabe5dc3774f7edf100e7458c48cd"},
     {file = "SQLAlchemy-2.0.22-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:3940677d341f2b685a999bffe7078697b5848a40b5f6952794ffcf3af150c301"},
+    {file = "SQLAlchemy-2.0.22-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:55914d45a631b81a8a2cb1a54f03eea265cf1783241ac55396ec6d735be14883"},
+    {file = "SQLAlchemy-2.0.22-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2096d6b018d242a2bcc9e451618166f860bb0304f590d205173d317b69986c95"},
+    {file = "SQLAlchemy-2.0.22-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:19c6986cf2fb4bc8e0e846f97f4135a8e753b57d2aaaa87c50f9acbe606bd1db"},
+    {file = "SQLAlchemy-2.0.22-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:6ac28bd6888fe3c81fbe97584eb0b96804bd7032d6100b9701255d9441373ec1"},
+    {file = "SQLAlchemy-2.0.22-cp37-cp37m-win32.whl", hash = "sha256:cb9a758ad973e795267da334a92dd82bb7555cb36a0960dcabcf724d26299db8"},
+    {file = "SQLAlchemy-2.0.22-cp37-cp37m-win_amd64.whl", hash = "sha256:40b1206a0d923e73aa54f0a6bd61419a96b914f1cd19900b6c8226899d9742ad"},
     {file = "SQLAlchemy-2.0.22-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:3aa1472bf44f61dd27987cd051f1c893b7d3b17238bff8c23fceaef4f1133868"},
     {file = "SQLAlchemy-2.0.22-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:56a7e2bb639df9263bf6418231bc2a92a773f57886d371ddb7a869a24919face"},
+    {file = "SQLAlchemy-2.0.22-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ccca778c0737a773a1ad86b68bda52a71ad5950b25e120b6eb1330f0df54c3d0"},
+    {file = "SQLAlchemy-2.0.22-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7c6c3e9350f9fb16de5b5e5fbf17b578811a52d71bb784cc5ff71acb7de2a7f9"},
+    {file = "SQLAlchemy-2.0.22-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:564e9f9e4e6466273dbfab0e0a2e5fe819eec480c57b53a2cdee8e4fdae3ad5f"},
+    {file = "SQLAlchemy-2.0.22-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:af66001d7b76a3fab0d5e4c1ec9339ac45748bc4a399cbc2baa48c1980d3c1f4"},
+    {file = "SQLAlchemy-2.0.22-cp38-cp38-win32.whl", hash = "sha256:9e55dff5ec115316dd7a083cdc1a52de63693695aecf72bc53a8e1468ce429e5"},
+    {file = "SQLAlchemy-2.0.22-cp38-cp38-win_amd64.whl", hash = "sha256:4e869a8ff7ee7a833b74868a0887e8462445ec462432d8cbeff5e85f475186da"},
+    {file = "SQLAlchemy-2.0.22-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9886a72c8e6371280cb247c5d32c9c8fa141dc560124348762db8a8b236f8692"},
+    {file = "SQLAlchemy-2.0.22-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a571bc8ac092a3175a1d994794a8e7a1f2f651e7c744de24a19b4f740fe95034"},
+    {file = "SQLAlchemy-2.0.22-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8db5ba8b7da759b727faebc4289a9e6a51edadc7fc32207a30f7c6203a181592"},
+    {file = "SQLAlchemy-2.0.22-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0b0b3f2686c3f162123adba3cb8b626ed7e9b8433ab528e36ed270b4f70d1cdb"},
+    {file = "SQLAlchemy-2.0.22-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:0c1fea8c0abcb070ffe15311853abfda4e55bf7dc1d4889497b3403629f3bf00"},
+    {file = "SQLAlchemy-2.0.22-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:4bb062784f37b2d75fd9b074c8ec360ad5df71f933f927e9e95c50eb8e05323c"},
+    {file = "SQLAlchemy-2.0.22-cp39-cp39-win32.whl", hash = "sha256:58a3aba1bfb32ae7af68da3f277ed91d9f57620cf7ce651db96636790a78b736"},
+    {file = "SQLAlchemy-2.0.22-cp39-cp39-win_amd64.whl", hash = "sha256:92e512a6af769e4725fa5b25981ba790335d42c5977e94ded07db7d641490a85"},
+    {file = "SQLAlchemy-2.0.22-py3-none-any.whl", hash = "sha256:3076740335e4aaadd7deb3fe6dcb96b3015f1613bd190a4e1634e1b99b02ec86"},
     {file = "SQLAlchemy-2.0.22.tar.gz", hash = "sha256:5434cc601aa17570d79e5377f5fd45ff92f9379e2abed0be5e8c2fba8d353d2b"},
 ]
 
@@ -7543,7 +7674,7 @@ community = ["llama-hub"]
 dataloaders = ["beautifulsoup4", "docx2txt", "duckduckgo-search", "pypdf", "pytube", "sentence-transformers", "unstructured", "youtube-transcript-api"]
 discord = ["discord"]
 elasticsearch = ["elasticsearch"]
-git = ["gitpython"]
+github = ["PyGithub", "gitpython"]
 gmail = ["llama-hub", "requests"]
 huggingface-hub = ["huggingface_hub"]
 images = ["ftfy", "pillow", "regex", "torch", "torchvision"]
@@ -7567,4 +7698,4 @@ youtube = ["youtube-transcript-api", "yt_dlp"]
 [metadata]
 lock-version = "2.0"
 python-versions = ">=3.9,<3.12"
-content-hash = "ea063cadfefd23d4c9b2a25c9096efe6bcedc367136d819ccb1fd2f510a91206"
+content-hash = "776ae7f49adab8a5dc98f6fe7c2887d2e700fd2d7c447383ea81ef05a463c8f3"

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "embedchain"
-version = "0.1.22"
+version = "0.1.27"
 description = "Data platform for LLMs - Load, index, retrieve and sync any unstructured data"
 authors = [
     "Taranjeet Singh <taranjeet@embedchain.ai>",
@@ -136,6 +136,7 @@ psycopg-pool = { version = "^3.1.8", optional = true }
 mysql-connector-python = { version = "^8.1.0", optional = true }
 gitpython = { version = "^3.1.38", optional = true }
 yt_dlp = { version = "^2023.11.14", optional = true }
+PyGithub = { version = "^1.59.1", optional = true }
 
 [tool.poetry.group.dev.dependencies]
 black = "^23.3.0"
@@ -192,7 +193,7 @@ gmail = [
 json = ["llama-hub"]
 postgres = ["psycopg", "psycopg-binary", "psycopg-pool"]
 mysql = ["mysql-connector-python"]
-git = ["gitpython"]
+github = ["PyGithub", "gitpython"]
 youtube = [
     "yt_dlp",
     "youtube-transcript-api",

tests/chunkers/test_chunkers.py

@@ -40,7 +40,7 @@ chunker_common_config = {
     PostgresChunker: {"chunk_size": 1000, "chunk_overlap": 0, "length_function": len},
     SlackChunker: {"chunk_size": 1000, "chunk_overlap": 0, "length_function": len},
     DiscourseChunker: {"chunk_size": 1000, "chunk_overlap": 0, "length_function": len},
-    CommonChunker: {"chunk_size": 1000, "chunk_overlap": 0, "length_function": len},
+    CommonChunker: {"chunk_size": 2000, "chunk_overlap": 0, "length_function": len},
 }
 
 

tests/loaders/test_github.py

@@ -0,0 +1,33 @@
+import pytest
+
+from embedchain.loaders.github import GithubLoader
+
+
+@pytest.fixture
+def mock_github_loader_config():
+    return {
+        "token": "your_mock_token",
+    }
+
+
+@pytest.fixture
+def mock_github_loader(mocker, mock_github_loader_config):
+    mock_github = mocker.patch("github.Github")
+    _ = mock_github.return_value
+    return GithubLoader(config=mock_github_loader_config)
+
+
+def test_github_loader_init(mocker, mock_github_loader_config):
+    mock_github = mocker.patch("github.Github")
+    GithubLoader(config=mock_github_loader_config)
+    mock_github.assert_called_once_with("your_mock_token")
+
+
+def test_github_loader_init_empty_config(mocker):
+    with pytest.raises(ValueError, match="requires a personal access token"):
+        GithubLoader()
+
+
+def test_github_loader_init_missing_token():
+    with pytest.raises(ValueError, match="requires a personal access token"):
+        GithubLoader(config={})

tests/test_utils.py

@@ -1,6 +1,6 @@
 import yaml
 
-from embedchain.utils import validate_yaml_config
+from embedchain.utils import validate_config
 
 CONFIG_YAMLS = [
     "configs/anthropic.yaml",
@@ -30,7 +30,7 @@ def test_all_config_yamls():
         assert config is not None
 
         try:
-            validate_yaml_config(config)
+            validate_config(config)
         except Exception as e:
             print(f"Error in {config_yaml}: {e}")
             raise e

tests/vectordb/test_pinecone.py

@@ -57,11 +57,11 @@ class TestPinecone:
         db.add(vectors, documents, metadatas, ids, True)
 
         expected_pinecone_upsert_args = [
-            {"id": "doc1", "metadata": {"text": "This is a document."}, "values": [0, 0, 0]},
+            {"id": "doc1", "values": [0, 0, 0], "metadata": {"text": "This is a document."}},
-            {"id": "doc2", "metadata": {"text": "This is another document."}, "values": [1, 1, 1]},
+            {"id": "doc2", "values": [1, 1, 1], "metadata": {"text": "This is another document."}},
         ]
         # Assert that the Pinecone client was called to upsert the documents
-        pinecone_client_mock.upsert.assert_called_once_with(expected_pinecone_upsert_args)
+        pinecone_client_mock.upsert.assert_called_once_with(tuple(expected_pinecone_upsert_args))
 
     @patch("embedchain.vectordb.pinecone.pinecone")
     def test_query_documents(self, pinecone_mock):