[version] Update langchain to v0.0.303 (#875)

README.md

@@ -64,7 +64,7 @@ For example, you can use Embedchain to create an Elon Musk bot using the followi
 
 ```python
 import os
-from embedchain import App
+from embedchain import Pipeline as App
 
 # Create a bot instance
 os.environ["OPENAI_API_KEY"] = "YOUR API KEY"
@@ -78,6 +78,17 @@ elon_bot.add("https://www.youtube.com/watch?v=RcYjXbSJBN8")
 # Query the bot
 elon_bot.query("How many companies does Elon Musk run and name those?")
 # Answer: Elon Musk currently runs several companies. As of my knowledge, he is the CEO and lead designer of SpaceX, the CEO and product architect of Tesla, Inc., the CEO and founder of Neuralink, and the CEO and founder of The Boring Company. However, please note that this information may change over time, so it's always good to verify the latest updates.
+
+# (Optional): Deploy app to Embedchain Platform
+app.deploy()
+# 🔑 Enter your Embedchain API key. You can find the API key at https://app.embedchain.ai/settings/keys/
+# ec-xxxxxx
+
+# 🛠️ Creating pipeline on the platform...
+# 🎉🎉🎉 Pipeline created successfully! View your pipeline: https://app.embedchain.ai/pipelines/xxxxx
+
+# 🛠️ Adding data to your pipeline...
+# ✅ Data of type: web_page, value: https://www.forbes.com/profile/elon-musk added successfully.
 ```
 
 ## Examples

docs/get-started/introduction.mdx

@@ -3,30 +3,106 @@ title: 📚 Introduction
 description: '📝 Embedchain is a Data Platform for LLMs - load, index, retrieve, and sync any unstructured data'
 ---
 
-## 🤔 What is Embedchain?
+## 🌐 What is Embedchain?
 
-Embedchain abstracts the entire process of loading data, chunking it, creating embeddings, and storing it in a vector database.
+Embedchain simplifies data handling by automatically processing unstructured data, breaking it into chunks, generating embeddings, and storing it in a vector database.
 
-You can add data from different data sources using the `.add()` method. Then, simply use the `.query()` method to find answers from the added datasets.
+Through various APIs, you can obtain contextual information for queries, find answers to specific questions, and engage in chat conversations using your data.
+## 🔍 Search
 
-If you want to create a Naval Ravikant bot with a YouTube video, a book in PDF format, two blog posts, and a question and answer pair, all you need to do is add the respective links. Embedchain will take care of the rest, creating a bot for you.
+Embedchain lets you get most relevant context by doing semantic search over your data sources for a provided query. See the example below:
 
 ```python
 from embedchain import Pipeline as App
 
-naval_bot = App()
-# Add online data
-naval_bot.add("https://www.youtube.com/watch?v=3qHkcs3kG44")
-naval_bot.add("https://navalmanack.s3.amazonaws.com/Eric-Jorgenson_The-Almanack-of-Naval-Ravikant_Final.pdf")
-naval_bot.add("https://nav.al/feedback")
-naval_bot.add("https://nav.al/agi")
-naval_bot.add("The Meanings of Life", 'text', metadata={'chapter': 'philosphy'})
+# Initialize app
+app = App()
 
-# Add local resources
-naval_bot.add(("Who is Naval Ravikant?", "Naval Ravikant is an Indian-American entrepreneur and investor."))
+# Add data source
+app.add("https://www.forbes.com/profile/elon-musk")
 
-naval_bot.query("What unique capacity does Naval argue humans possess when it comes to understanding explanations or concepts?")
-# Answer: Naval argues that humans possess the unique capacity to understand explanations or concepts to the maximum extent possible in this physical reality.
+# Get relevant context using semantic search
+context = app.search("What is the net worth of Elon?", num_documents=2)
+print(context)
+# Context:
+# [
+#     {
+#         'context': 'Elon Musk PROFILEElon MuskCEO, Tesla$221.9BReal Time Net Worthas of 10/29/23Reflects change since 5 pm ET of prior trading day. 1 in the world todayPhoto by Martin Schoeller for ForbesAbout Elon MuskElon Musk cofounded six companies, including electric car maker Tesla, rocket producer SpaceX and tunneling startup Boring Company.He owns about 21% of Tesla between stock and options, but has pledged more than half his shares as collateral for personal loans of up to $3.5 billion.SpaceX, founded in',
+#         'source': 'https://www.forbes.com/profile/elon-musk',
+#         'document_id': 'some_document_id'
+#     },
+#     {
+#         'context': 'company, which is now called X.Wealth HistoryHOVER TO REVEAL NET WORTH BY YEARForbes Lists 1Forbes 400 (2023)The Richest Person In Every State (2023) 2Billionaires (2023) 1Innovative Leaders (2019) 25Powerful People (2018) 12Richest In Tech (2017)Global Game Changers (2016)More ListsPersonal StatsAge52Source of WealthTesla, SpaceX, Self MadeSelf-Made Score8Philanthropy Score1ResidenceAustin, TexasCitizenshipUnited StatesMarital StatusSingleChildren11EducationBachelor of Arts/Science, University',
+#         'source': 'https://www.forbes.com/profile/elon-musk',
+#         'document_id': 'some_document_id'
+#     }
+# ]
+```
+
+## ❓Query
+
+Embedchain empowers developers to ask questions and receive relevant answers through a user-friendly query API. Refer to the following example to learn how to utilize the query API:
+
+```python
+from embedchain import Pipeline as App
+
+# Initialize app
+app = App()
+
+# Add data source
+app.add("https://www.forbes.com/profile/elon-musk")
+
+# Get relevant answer for your query
+answer = app.query("What is the net worth of Elon?")
+print(answer)
+# Answer: The net worth of Elon Musk is $221.9 billion.
+```
+
+## 💬 Chat
+
+Embedchain allows easy chatting over your data sources using a user-friendly chat API. Check out the example below to understand how to use the chat API:
+
+```python
+from embedchain import Pipeline as App
+
+# Initialize app
+app = App()
+
+# Add data source
+app.add("https://www.forbes.com/profile/elon-musk")
+
+# Chat on your data using `.chat()`
+answer = app.chat("How much did Elon pay for Twitter?")
+print(answer)
+# Answer: Elon Musk paid $44 billion for Twitter.
+```
+
+## 🚀 Deploy
+
+Embedchain enables developers to deploy their LLM-powered apps in production using the Embedchain platform. The platform offers free access to context on your data through its REST API. Once the pipeline is deployed, you can update your data sources anytime after deployment.
+
+See the example below on how to use the deploy API:
+
+```python
+from embedchain import Pipeline as App
+
+# Initialize app
+app = App()
+
+# Add data source
+app.add("https://www.forbes.com/profile/elon-musk")
+
+# Deploy your pipeline to Embedchain Platform
+app.deploy()
+
+# 🔑 Enter your Embedchain API key. You can find the API key at https://app.embedchain.ai/settings/keys/
+# ec-xxxxxx
+
+# 🛠️ Creating pipeline on the platform...
+# 🎉🎉🎉 Pipeline created successfully! View your pipeline: https://app.embedchain.ai/pipelines/xxxxx
+
+# 🛠️ Adding data to your pipeline...
+# ✅ Data of type: web_page, value: https://www.forbes.com/profile/elon-musk added successfully.
 ```
 
 ## 🚀 How it works?

docs/get-started/quickstart.mdx

@@ -33,6 +33,19 @@ app.add("https://www.forbes.com/profile/elon-musk")
 ```python
 app.query("What is the net worth of Elon Musk today?")
 # Answer: The net worth of Elon Musk today is $258.7 billion.
+```
+  </Step>
+  <Step title="🚀 (Optional) Deploy your pipeline to Embedchain Platform">
+```python
+app.deploy()
+# 🔑 Enter your Embedchain API key. You can find the API key at https://app.embedchain.ai/settings/keys/
+# ec-xxxxxx
+
+# 🛠️ Creating pipeline on the platform...
+# 🎉🎉🎉 Pipeline created successfully! View your pipeline: https://app.embedchain.ai/pipelines/xxxxx
+
+# 🛠️ Adding data to your pipeline...
+# ✅ Data of type: web_page, value: https://www.forbes.com/profile/elon-musk added successfully.
 ```
   </Step>
 </Steps>
@@ -55,4 +68,14 @@ app.add("https://www.forbes.com/profile/elon-musk")
 response = app.query("What is the net worth of Elon Musk today?")
 print(response)
 # Answer: The net worth of Elon Musk today is $258.7 billion.
+
+app.deploy()
+# 🔑 Enter your Embedchain API key. You can find the API key at https://app.embedchain.ai/settings/keys/
+# ec-xxxxxx
+
+# 🛠️ Creating pipeline on the platform...
+# 🎉🎉🎉 Pipeline created successfully! View your pipeline: https://app.embedchain.ai/pipelines/xxxxx
+
+# 🛠️ Adding data to your pipeline...
+# ✅ Data of type: web_page, value: https://www.forbes.com/profile/elon-musk added successfully.
 ```

docs/mint.json

@@ -7,9 +7,20 @@
   },
   "favicon": "/favicon.png",
   "colors": {
-    "primary": "#12A7D3",
-    "light": "#81D7F7",
-    "dark": "#004E7A"
+    "primary": "#2B48EE",
+    "light": "#2B48EE",
+    "dark": "#2B48EE",
+    "background": {
+      "dark": "#020415"
+    }
+  },
+  "metadata": {
+    "og:image": "/images/og.png",
+    "twitter:site": "@embedchain"
+  },
+  "topAnchor": {
+    "name": "Documentation",
+    "icon": "book-open"
   },
   "topbarLinks": [
     {
@@ -26,8 +37,11 @@
     }
   ],
   "topbarCtaButton": {
-    "name": "GitHub",
-    "url": "https://embedchain.ai"
+    "name": "Get started",
+    "url": "https://app.embedchain.ai"
+  },
+  "primaryTab": {
+    "name": "Docs"
   },
   "navigation": [
     {
@@ -100,7 +114,6 @@
     }
 
   ],
-
   "footerSocials": {
     "website": "https://embedchain.ai",
     "github": "https://github.com/embedchain/embedchain",
@@ -109,7 +122,19 @@
     "twitter": "https://twitter.com/embedchain",
     "linkedin": "https://www.linkedin.com/company/embedchain"
   },
-  "backgroundImage": "/background.png",
   "isWhiteLabeled": true,
-  "feedback.thumbsRating": true
+  "analytics": {
+    "posthog": {
+      "apiKey": "phc_MSQ1GVfzkm7dRpktoKUGzWPlpkYhmVpcUtmLk1RwkQs",
+      "apiHost": "https://app.embedchain.ai/ingest"
+    }
+  },
+  "feedback": {
+    "suggestEdit": true,
+    "raiseIssue": true,
+    "thumbsRating": true
+  },
+  "search": {
+    "prompt": "✨ Search embedchain docs..."
+  }
 }

embedchain/data_formatter/data_formatter.py

@@ -1,41 +1,9 @@
+from importlib import import_module
 from embedchain.chunkers.base_chunker import BaseChunker
-from embedchain.chunkers.docs_site import DocsSiteChunker
-from embedchain.chunkers.docx_file import DocxFileChunker
-from embedchain.chunkers.gmail import GmailChunker
-from embedchain.chunkers.images import ImagesChunker
-from embedchain.chunkers.json import JSONChunker
-from embedchain.chunkers.mdx import MdxChunker
-from embedchain.chunkers.notion import NotionChunker
-from embedchain.chunkers.openapi import OpenAPIChunker
-from embedchain.chunkers.pdf_file import PdfFileChunker
-from embedchain.chunkers.qna_pair import QnaPairChunker
-from embedchain.chunkers.sitemap import SitemapChunker
-from embedchain.chunkers.table import TableChunker
-from embedchain.chunkers.text import TextChunker
-from embedchain.chunkers.unstructured_file import UnstructuredFileChunker
-from embedchain.chunkers.web_page import WebPageChunker
-from embedchain.chunkers.xml import XmlChunker
-from embedchain.chunkers.youtube_video import YoutubeVideoChunker
 from embedchain.config import AddConfig
 from embedchain.config.add_config import ChunkerConfig, LoaderConfig
 from embedchain.helper.json_serializable import JSONSerializable
 from embedchain.loaders.base_loader import BaseLoader
-from embedchain.loaders.csv import CsvLoader
-from embedchain.loaders.docs_site_loader import DocsSiteLoader
-from embedchain.loaders.docx_file import DocxFileLoader
-from embedchain.loaders.gmail import GmailLoader
-from embedchain.loaders.images import ImagesLoader
-from embedchain.loaders.json import JSONLoader
-from embedchain.loaders.local_qna_pair import LocalQnaPairLoader
-from embedchain.loaders.local_text import LocalTextLoader
-from embedchain.loaders.mdx import MdxLoader
-from embedchain.loaders.openapi import OpenAPILoader
-from embedchain.loaders.pdf_file import PdfFileLoader
-from embedchain.loaders.sitemap import SitemapLoader
-from embedchain.loaders.unstructured_file import UnstructuredLoader
-from embedchain.loaders.web_page import WebPageLoader
-from embedchain.loaders.xml import XmlLoader
-from embedchain.loaders.youtube_video import YoutubeVideoLoader
 from embedchain.models.data_type import DataType
 
 
@@ -58,6 +26,11 @@ class DataFormatter(JSONSerializable):
         self.loader = self._get_loader(data_type=data_type, config=config.loader)
         self.chunker = self._get_chunker(data_type=data_type, config=config.chunker)
 
+    def _lazy_load(self, module_path: str):
+        module_path, class_name = module_path.rsplit(".", 1)
+        module = import_module(module_path)
+        return getattr(module, class_name)
+
     def _get_loader(self, data_type: DataType, config: LoaderConfig) -> BaseLoader:
         """
         Returns the appropriate data loader for the given data type.
@@ -71,71 +44,55 @@ class DataFormatter(JSONSerializable):
         :rtype: BaseLoader
         """
         loaders = {
-            DataType.YOUTUBE_VIDEO: YoutubeVideoLoader,
-            DataType.PDF_FILE: PdfFileLoader,
-            DataType.WEB_PAGE: WebPageLoader,
-            DataType.QNA_PAIR: LocalQnaPairLoader,
-            DataType.TEXT: LocalTextLoader,
-            DataType.DOCX: DocxFileLoader,
-            DataType.SITEMAP: SitemapLoader,
-            DataType.XML: XmlLoader,
-            DataType.DOCS_SITE: DocsSiteLoader,
-            DataType.CSV: CsvLoader,
-            DataType.MDX: MdxLoader,
-            DataType.IMAGES: ImagesLoader,
-            DataType.UNSTRUCTURED: UnstructuredLoader,
-            DataType.JSON: JSONLoader,
-            DataType.OPENAPI: OpenAPILoader,
-            DataType.GMAIL: GmailLoader,
+            DataType.YOUTUBE_VIDEO: "embedchain.loaders.youtube_video.YoutubeVideoLoader",
+            DataType.PDF_FILE: "embedchain.loaders.pdf_file.PdfFileLoader",
+            DataType.WEB_PAGE: "embedchain.loaders.web_page.WebPageLoader",
+            DataType.QNA_PAIR: "embedchain.loaders.local_qna_pair.LocalQnaPairLoader",
+            DataType.TEXT: "embedchain.loaders.local_text.LocalTextLoader",
+            DataType.DOCX: "embedchain.loaders.docx_file.DocxFileLoader",
+            DataType.SITEMAP: "embedchain.loaders.sitemap.SitemapLoader",
+            DataType.XML: "embedchain.loaders.xml.XmlLoader",
+            DataType.DOCS_SITE: "embedchain.loaders.docs_site_loader.DocsSiteLoader",
+            DataType.CSV: "embedchain.loaders.csv.CsvLoader",
+            DataType.MDX: "embedchain.loaders.mdx.MdxLoader",
+            DataType.IMAGES: "embedchain.loaders.images.ImagesLoader",
+            DataType.UNSTRUCTURED: "embedchain.loaders.unstructured_file.UnstructuredLoader",
+            DataType.JSON: "embedchain.loaders.json.JSONLoader",
+            DataType.OPENAPI: "embedchain.loaders.openapi.OpenAPILoader",
+            DataType.GMAIL: "embedchain.loaders.gmail.GmailLoader",
+            DataType.NOTION: "embedchain.loaders.notion.NotionLoader",
         }
-        lazy_loaders = {DataType.NOTION}
         if data_type in loaders:
-            loader_class: type = loaders[data_type]
-            loader: BaseLoader = loader_class()
-            return loader
-        elif data_type in lazy_loaders:
-            if data_type == DataType.NOTION:
-                from embedchain.loaders.notion import NotionLoader
-
-                return NotionLoader()
-            else:
-                raise ValueError(f"Unsupported data type: {data_type}")
+            loader_class: type = self._lazy_load(loaders[data_type])
+            return loader_class()
         else:
             raise ValueError(f"Unsupported data type: {data_type}")
 
     def _get_chunker(self, data_type: DataType, config: ChunkerConfig) -> BaseChunker:
-        """Returns the appropriate chunker for the given data type.
-
-        :param data_type: The type of the data to chunk.
-        :type data_type: DataType
-        :param config: Config to initialize the chunker with.
-        :type config: ChunkerConfig
-        :raises ValueError: If an unsupported data type is provided.
-        :return: The chunker for the given data type.
-        :rtype: BaseChunker
-        """
+        """Returns the appropriate chunker for the given data type (updated for lazy loading)."""
         chunker_classes = {
-            DataType.YOUTUBE_VIDEO: YoutubeVideoChunker,
-            DataType.PDF_FILE: PdfFileChunker,
-            DataType.WEB_PAGE: WebPageChunker,
-            DataType.QNA_PAIR: QnaPairChunker,
-            DataType.TEXT: TextChunker,
-            DataType.DOCX: DocxFileChunker,
-            DataType.DOCS_SITE: DocsSiteChunker,
-            DataType.SITEMAP: SitemapChunker,
-            DataType.NOTION: NotionChunker,
-            DataType.CSV: TableChunker,
-            DataType.MDX: MdxChunker,
-            DataType.IMAGES: ImagesChunker,
-            DataType.XML: XmlChunker,
-            DataType.UNSTRUCTURED: UnstructuredFileChunker,
-            DataType.JSON: JSONChunker,
-            DataType.OPENAPI: OpenAPIChunker,
-            DataType.GMAIL: GmailChunker,
+            DataType.YOUTUBE_VIDEO: "embedchain.chunkers.youtube_video.YoutubeVideoChunker",
+            DataType.PDF_FILE: "embedchain.chunkers.pdf_file.PdfFileChunker",
+            DataType.WEB_PAGE: "embedchain.chunkers.web_page.WebPageChunker",
+            DataType.QNA_PAIR: "embedchain.chunkers.qna_pair.QnaPairChunker",
+            DataType.TEXT: "embedchain.chunkers.text.TextChunker",
+            DataType.DOCX: "embedchain.chunkers.docx_file.DocxFileChunker",
+            DataType.SITEMAP: "embedchain.chunkers.sitemap.SitemapChunker",
+            DataType.XML: "embedchain.chunkers.xml.XmlChunker",
+            DataType.DOCS_SITE: "embedchain.chunkers.docs_site.DocsSiteChunker",
+            DataType.CSV: "embedchain.chunkers.table.TableChunker",
+            DataType.MDX: "embedchain.chunkers.mdx.MdxChunker",
+            DataType.IMAGES: "embedchain.chunkers.images.ImagesChunker",
+            DataType.UNSTRUCTURED: "embedchain.chunkers.unstructured_file.UnstructuredFileChunker",
+            DataType.JSON: "embedchain.chunkers.json.JSONChunker",
+            DataType.OPENAPI: "embedchain.chunkers.openapi.OpenAPIChunker",
+            DataType.GMAIL: "embedchain.chunkers.gmail.GmailChunker",
+            DataType.NOTION: "embedchain.chunkers.notion.NotionChunker",
         }
+
         if data_type in chunker_classes:
-            chunker_class: type = chunker_classes[data_type]
-            chunker: BaseChunker = chunker_class(config)
+            chunker_class = self._lazy_load(chunker_classes[data_type])
+            chunker = chunker_class(config)
             chunker.set_data_type(data_type)
             return chunker
         else:

embedchain/pipeline.py

@@ -229,12 +229,22 @@ class Pipeline(EmbedChain):
         # TODO: Search will call the endpoint rather than fetching the data from the db itself when deploy=True.
         if self.id is None:
             where = {"app_id": self.local_id}
-            return self.db.query(
+            context = self.db.query(
                 query,
                 n_results=num_documents,
                 where=where,
                 skip_embedding=False,
             )
+            result = []
+            for c in context:
+                result.append(
+                    {
+                        "context": c[0],
+                        "source": c[1],
+                        "document_id": c[2],
+                    }
+                )
+            return result
         else:
             # Make API call to the backend to get the results
             NotImplementedError("Search is not implemented yet for the prod mode.")

examples/full_stack/frontend/public/icons/twitter.svg

@@ -1,24 +1,10 @@
-<svg
-  fill="currentColor"
-  version="1.1"
-  xmlns="http://www.w3.org/2000/svg"
-  xmlns:xlink="http://www.w3.org/1999/xlink"
-  viewBox="0 0 512 512"
-  xml:space="preserve"
->
-  <g id="SVGRepo_bgCarrier" stroke-width="0"></g>
-  <g
-    id="SVGRepo_tracerCarrier"
-    stroke-linecap="round"
-    stroke-linejoin="round"
-  ></g>
-  <g id="SVGRepo_iconCarrier">
-    <g id="7935ec95c421cee6d86eb22ecd12f847">
-      <path
-        style="display: inline;"
-        d="M459.186,151.787c0.203,4.501,0.305,9.023,0.305,13.565 c0,138.542-105.461,298.285-298.274,298.285c-59.209,0-114.322-17.357-160.716-47.104c8.212,0.973,16.546,1.47,25.012,1.47 c49.121,0,94.318-16.759,130.209-44.884c-45.887-0.841-84.596-31.154-97.938-72.804c6.408,1.227,12.968,1.886,19.73,1.886 c9.55,0,18.816-1.287,27.617-3.68c-47.955-9.633-84.1-52.001-84.1-102.795c0-0.446,0-0.882,0.011-1.318 c14.133,7.847,30.294,12.562,47.488,13.109c-28.134-18.796-46.637-50.885-46.637-87.262c0-19.212,5.16-37.218,14.193-52.7 c51.707,63.426,128.941,105.156,216.072,109.536c-1.784-7.675-2.718-15.674-2.718-23.896c0-57.891,46.941-104.832,104.832-104.832 c30.173,0,57.404,12.734,76.525,33.102c23.887-4.694,46.313-13.423,66.569-25.438c-7.827,24.485-24.434,45.025-46.089,58.002 c21.209-2.535,41.426-8.171,60.222-16.505C497.448,118.542,479.666,137.004,459.186,151.787z"
-      > 
-      </path>
-    </g>
-  </g>
-</svg>
+<?xml version="1.0" encoding="utf-8"?>
+<!-- Generator: Adobe Illustrator 27.5.0, SVG Export Plug-In . SVG Version: 6.00 Build 0)  -->
+<svg version="1.1" id="svg5" xmlns:svg="http://www.w3.org/2000/svg"
+	 xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px" viewBox="0 0 1668.56 1221.19"
+	 style="enable-background:new 0 0 1668.56 1221.19;" xml:space="preserve">
+<g id="layer1" transform="translate(52.390088,-25.058597)">
+	<path id="path1009" d="M283.94,167.31l386.39,516.64L281.5,1104h87.51l340.42-367.76L984.48,1104h297.8L874.15,558.3l361.92-390.99
+		h-87.51l-313.51,338.7l-253.31-338.7H283.94z M412.63,231.77h136.81l604.13,807.76h-136.81L412.63,231.77z"/>
+</g>
+</svg>

notebooks/anthropic.ipynb

@@ -34,6 +34,15 @@
         "!pip install embedchain"
       ]
     },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "!pip install embedchain[dataloaders]"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -54,7 +63,7 @@
       "outputs": [],
       "source": [
         "import os\n",
-        "from embedchain import App\n",
+        "from embedchain import Pipeline as App\n",
         "\n",
         "os.environ[\"OPENAI_API_KEY\"] = \"sk-xxx\"\n",
         "os.environ[\"ANTHROPIC_API_KEY\"] = \"xxx\""

notebooks/azure-openai.ipynb

@@ -26,6 +26,16 @@
     "!pip install embedchain"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "692ff37b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!pip install embedchain[dataloaders]"
+   ]
+  },
   {
    "cell_type": "markdown",
    "id": "ac982a56",
@@ -44,7 +54,7 @@
    "outputs": [],
    "source": [
     "import os\n",
-    "from embedchain import App\n",
+    "from embedchain import Pipeline as App\n",
     "\n",
     "os.environ[\"OPENAI_API_TYPE\"] = \"azure\"\n",
     "os.environ[\"OPENAI_API_BASE\"] = \"https://xxx.openai.azure.com/\"\n",

notebooks/chromadb.ipynb

@@ -1,84 +1,84 @@
 {
-  "nbformat": 4,
-  "nbformat_minor": 0,
-  "metadata": {
-    "colab": {
-      "provenance": []
-    },
-    "kernelspec": {
-      "name": "python3",
-      "display_name": "Python 3"
-    },
-    "language_info": {
-      "name": "python"
-    }
-  },
   "cells": [
     {
       "cell_type": "markdown",
-      "source": [
-        "## Cookbook for using ChromaDB with Embedchain"
-      ],
       "metadata": {
         "id": "b02n_zJ_hl3d"
-      }
+      },
+      "source": [
+        "## Cookbook for using ChromaDB with Embedchain"
+      ]
     },
     {
       "cell_type": "markdown",
-      "source": [
-        "### Step-1: Install embedchain package"
-      ],
       "metadata": {
         "id": "gyJ6ui2vhtMY"
-      }
+      },
+      "source": [
+        "### Step-1: Install embedchain package"
+      ]
     },
     {
       "cell_type": "code",
-      "source": [
-        "!pip install embedchain"
-      ],
+      "execution_count": null,
       "metadata": {
         "id": "-NbXjAdlh0vJ"
       },
+      "outputs": [],
+      "source": [
+        "!pip install embedchain"
+      ]
+    },
+    {
+      "cell_type": "code",
       "execution_count": null,
-      "outputs": []
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "!pip install embedchain[dataloaders]"
+      ]
     },
     {
       "cell_type": "markdown",
+      "metadata": {
+        "id": "nGnpSYAAh2bQ"
+      },
       "source": [
         "### Step-2: Set OpenAI environment variables\n",
         "\n",
         "You can find this env variable on your [OpenAI dashboard](https://platform.openai.com/account/api-keys)."
-      ],
-      "metadata": {
-        "id": "nGnpSYAAh2bQ"
-      }
+      ]
     },
     {
       "cell_type": "code",
-      "source": [
-        "import os\n",
-        "from embedchain import App\n",
-        "\n",
-        "os.environ[\"OPENAI_API_KEY\"] = \"sk-xxx\""
-      ],
+      "execution_count": null,
       "metadata": {
         "id": "0fBdQ9GAiRvK"
       },
-      "execution_count": null,
-      "outputs": []
+      "outputs": [],
+      "source": [
+        "import os\n",
+        "from embedchain import Pipeline as App\n",
+        "\n",
+        "os.environ[\"OPENAI_API_KEY\"] = \"sk-xxx\""
+      ]
     },
     {
       "cell_type": "markdown",
-      "source": [
-        "### Step-3: Define your Vector Database config"
-      ],
       "metadata": {
         "id": "Ns6RhPfbiitr"
-      }
+      },
+      "source": [
+        "### Step-3: Define your Vector Database config"
+      ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "S9CkxVjriotB"
+      },
+      "outputs": [],
       "source": [
         "config = \"\"\"\n",
         "vectordb:\n",
@@ -95,64 +95,64 @@
         "# Write the multi-line string to a YAML file\n",
         "with open('chromadb.yaml', 'w') as file:\n",
         "    file.write(config)"
-      ],
-      "metadata": {
-        "id": "S9CkxVjriotB"
-      },
-      "execution_count": null,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
-      "source": [
-        "### Step-4 Create embedchain app based on the config"
-      ],
       "metadata": {
         "id": "PGt6uPLIi1CS"
-      }
+      },
+      "source": [
+        "### Step-4 Create embedchain app based on the config"
+      ]
     },
     {
       "cell_type": "code",
-      "source": [
-        "app = App.from_config(yaml_path=\"chromadb.yaml\")"
-      ],
+      "execution_count": null,
       "metadata": {
         "id": "Amzxk3m-i3tD"
       },
-      "execution_count": null,
-      "outputs": []
+      "outputs": [],
+      "source": [
+        "app = App.from_config(yaml_path=\"chromadb.yaml\")"
+      ]
     },
     {
       "cell_type": "markdown",
-      "source": [
-        "### Step-5: Add data sources to your app"
-      ],
       "metadata": {
         "id": "XNXv4yZwi7ef"
-      }
+      },
+      "source": [
+        "### Step-5: Add data sources to your app"
+      ]
     },
     {
       "cell_type": "code",
-      "source": [
-        "app.add(\"https://www.forbes.com/profile/elon-musk\")"
-      ],
+      "execution_count": null,
       "metadata": {
         "id": "Sn_0rx9QjIY9"
       },
-      "execution_count": null,
-      "outputs": []
+      "outputs": [],
+      "source": [
+        "app.add(\"https://www.forbes.com/profile/elon-musk\")"
+      ]
     },
     {
       "cell_type": "markdown",
-      "source": [
-        "### Step-6: All set. Now start asking questions related to your data"
-      ],
       "metadata": {
         "id": "_7W6fDeAjMAP"
-      }
+      },
+      "source": [
+        "### Step-6: All set. Now start asking questions related to your data"
+      ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "cvIK7dWRjN_f"
+      },
+      "outputs": [],
       "source": [
         "while(True):\n",
         "    question = input(\"Enter question: \")\n",
@@ -160,12 +160,21 @@
         "        break\n",
         "    answer = app.query(question)\n",
         "    print(answer)"
-      ],
-      "metadata": {
-        "id": "cvIK7dWRjN_f"
-      },
-      "execution_count": null,
-      "outputs": []
+      ]
     }
-  ]
-}
+  ],
+  "metadata": {
+    "colab": {
+      "provenance": []
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    },
+    "language_info": {
+      "name": "python"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

notebooks/cohere.ipynb

@@ -33,6 +33,15 @@
         "!pip install embedchain"
       ]
     },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "!pip install embedchain[dataloaders]"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -69,7 +78,7 @@
       "outputs": [],
       "source": [
         "import os\n",
-        "from embedchain import App\n",
+        "from embedchain import Pipeline as App\n",
         "\n",
         "os.environ[\"OPENAI_API_KEY\"] = \"sk-xxx\"\n",
         "os.environ[\"COHERE_API_KEY\"] = \"xxx\""

notebooks/elasticsearch.ipynb

@@ -1,95 +1,95 @@
 {
-  "nbformat": 4,
-  "nbformat_minor": 0,
-  "metadata": {
-    "colab": {
-      "provenance": []
-    },
-    "kernelspec": {
-      "name": "python3",
-      "display_name": "Python 3"
-    },
-    "language_info": {
-      "name": "python"
-    }
-  },
   "cells": [
     {
       "cell_type": "markdown",
-      "source": [
-        "## Cookbook for using ElasticSearchDB with Embedchain"
-      ],
       "metadata": {
         "id": "b02n_zJ_hl3d"
-      }
+      },
+      "source": [
+        "## Cookbook for using ElasticSearchDB with Embedchain"
+      ]
     },
     {
       "cell_type": "markdown",
-      "source": [
-        "### Step-1: Install embedchain package"
-      ],
       "metadata": {
         "id": "gyJ6ui2vhtMY"
-      }
+      },
+      "source": [
+        "### Step-1: Install embedchain package"
+      ]
     },
     {
       "cell_type": "code",
-      "source": [
-        "!pip install embedchain"
-      ],
+      "execution_count": null,
       "metadata": {
         "id": "-NbXjAdlh0vJ"
       },
+      "outputs": [],
+      "source": [
+        "!pip install embedchain"
+      ]
+    },
+    {
+      "cell_type": "code",
       "execution_count": null,
-      "outputs": []
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "!pip install embedchain[dataloaders]"
+      ]
     },
     {
       "cell_type": "markdown",
+      "metadata": {
+        "id": "nGnpSYAAh2bQ"
+      },
       "source": [
         "### Step-2: Set OpenAI environment variables and install the dependencies.\n",
         "\n",
         "You can find this env variable on your [OpenAI dashboard](https://platform.openai.com/account/api-keys). Now lets install the dependencies needed for Elasticsearch."
-      ],
-      "metadata": {
-        "id": "nGnpSYAAh2bQ"
-      }
+      ]
     },
     {
       "cell_type": "code",
-      "source": [
-        "!pip install --upgrade 'embedchain[elasticsearch]'"
-      ],
+      "execution_count": null,
       "metadata": {
         "id": "-MUFRfxV7Jk7"
       },
-      "execution_count": null,
-      "outputs": []
+      "outputs": [],
+      "source": [
+        "!pip install --upgrade 'embedchain[elasticsearch]'"
+      ]
     },
     {
       "cell_type": "code",
-      "source": [
-        "import os\n",
-        "from embedchain import App\n",
-        "\n",
-        "os.environ[\"OPENAI_API_KEY\"] = \"sk-xxx\""
-      ],
+      "execution_count": null,
       "metadata": {
         "id": "0fBdQ9GAiRvK"
       },
-      "execution_count": null,
-      "outputs": []
+      "outputs": [],
+      "source": [
+        "import os\n",
+        "from embedchain import Pipeline as App\n",
+        "\n",
+        "os.environ[\"OPENAI_API_KEY\"] = \"sk-xxx\""
+      ]
     },
     {
       "cell_type": "markdown",
-      "source": [
-        "### Step-3: Define your Vector Database config"
-      ],
       "metadata": {
         "id": "Ns6RhPfbiitr"
-      }
+      },
+      "source": [
+        "### Step-3: Define your Vector Database config"
+      ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "S9CkxVjriotB"
+      },
+      "outputs": [],
       "source": [
         "config = \"\"\"\n",
         "vectordb:\n",
@@ -104,64 +104,64 @@
         "# Write the multi-line string to a YAML file\n",
         "with open('elasticsearch.yaml', 'w') as file:\n",
         "    file.write(config)"
-      ],
-      "metadata": {
-        "id": "S9CkxVjriotB"
-      },
-      "execution_count": null,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
-      "source": [
-        "### Step-4 Create embedchain app based on the config"
-      ],
       "metadata": {
         "id": "PGt6uPLIi1CS"
-      }
+      },
+      "source": [
+        "### Step-4 Create embedchain app based on the config"
+      ]
     },
     {
       "cell_type": "code",
-      "source": [
-        "app = App.from_config(yaml_path=\"elasticsearch.yaml\")"
-      ],
+      "execution_count": null,
       "metadata": {
         "id": "Amzxk3m-i3tD"
       },
-      "execution_count": null,
-      "outputs": []
+      "outputs": [],
+      "source": [
+        "app = App.from_config(yaml_path=\"elasticsearch.yaml\")"
+      ]
     },
     {
       "cell_type": "markdown",
-      "source": [
-        "### Step-5: Add data sources to your app"
-      ],
       "metadata": {
         "id": "XNXv4yZwi7ef"
-      }
+      },
+      "source": [
+        "### Step-5: Add data sources to your app"
+      ]
     },
     {
       "cell_type": "code",
-      "source": [
-        "app.add(\"https://www.forbes.com/profile/elon-musk\")"
-      ],
+      "execution_count": null,
       "metadata": {
         "id": "Sn_0rx9QjIY9"
       },
-      "execution_count": null,
-      "outputs": []
+      "outputs": [],
+      "source": [
+        "app.add(\"https://www.forbes.com/profile/elon-musk\")"
+      ]
     },
     {
       "cell_type": "markdown",
-      "source": [
-        "### Step-6: All set. Now start asking questions related to your data"
-      ],
       "metadata": {
         "id": "_7W6fDeAjMAP"
-      }
+      },
+      "source": [
+        "### Step-6: All set. Now start asking questions related to your data"
+      ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "cvIK7dWRjN_f"
+      },
+      "outputs": [],
       "source": [
         "while(True):\n",
         "    question = input(\"Enter question: \")\n",
@@ -169,12 +169,21 @@
         "        break\n",
         "    answer = app.query(question)\n",
         "    print(answer)"
-      ],
-      "metadata": {
-        "id": "cvIK7dWRjN_f"
-      },
-      "execution_count": null,
-      "outputs": []
+      ]
     }
-  ]
-}
+  ],
+  "metadata": {
+    "colab": {
+      "provenance": []
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    },
+    "language_info": {
+      "name": "python"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

notebooks/embedchain-chromadb-server.ipynb

@@ -33,7 +33,7 @@
    "outputs": [],
    "source": [
     "import os\n",
-    "from embedchain import App\n",
+    "from embedchain import Pipeline as App\n",
     "from embedchain.config import AppConfig\n",
     "\n",
     "\n",

notebooks/embedchain-docs-site-example.ipynb

@@ -7,7 +7,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from embedchain import App\n",
+    "from embedchain import Pipeline as App\n",
     "\n",
     "embedchain_docs_bot = App()"
    ]

notebooks/gpt4all.ipynb

@@ -33,6 +33,15 @@
         "!pip install embedchain"
       ]
     },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "!pip install embedchain[dataloaders]"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -67,7 +76,7 @@
       },
       "outputs": [],
       "source": [
-        "from embedchain import App"
+        "from embedchain import Pipeline as App"
       ]
     },
     {

notebooks/hugging_face_hub.ipynb

@@ -34,6 +34,15 @@
         "!pip install embedchain"
       ]
     },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "!pip install embedchain[dataloaders]"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -84,7 +93,7 @@
       "outputs": [],
       "source": [
         "import os\n",
-        "from embedchain import App\n",
+        "from embedchain import Pipeline as App\n",
         "\n",
         "os.environ[\"HUGGINGFACE_ACCESS_TOKEN\"] = \"hf_xxx\""
       ]

notebooks/jina.ipynb

@@ -34,6 +34,15 @@
         "!pip install embedchain"
       ]
     },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "!pip install embedchain[dataloaders]"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -54,7 +63,7 @@
       "outputs": [],
       "source": [
         "import os\n",
-        "from embedchain import App\n",
+        "from embedchain import Pipeline as App\n",
         "\n",
         "os.environ[\"OPENAI_API_KEY\"] = \"sk-xxx\"\n",
         "os.environ[\"JINACHAT_API_KEY\"] = \"xxx\""

notebooks/llama2.ipynb

@@ -33,6 +33,15 @@
         "!pip install embedchain"
       ]
     },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "!pip install embedchain[dataloaders]"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -64,7 +73,7 @@
       "outputs": [],
       "source": [
         "import os\n",
-        "from embedchain import App\n",
+        "from embedchain import Pipeline as App\n",
         "\n",
         "os.environ[\"OPENAI_API_KEY\"] = \"sk-xxx\"\n",
         "os.environ[\"REPLICATE_API_TOKEN\"] = \"xxx\""

notebooks/openai.ipynb

@@ -34,6 +34,15 @@
         "!pip install embedchain"
       ]
     },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "!pip install embedchain[dataloaders]"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -54,7 +63,7 @@
       "outputs": [],
       "source": [
         "import os\n",
-        "from embedchain import App\n",
+        "from embedchain import Pipeline as App\n",
         "\n",
         "os.environ[\"OPENAI_API_KEY\"] = \"sk-xxx\""
       ]

notebooks/opensearch.ipynb

@@ -1,95 +1,95 @@
 {
-  "nbformat": 4,
-  "nbformat_minor": 0,
-  "metadata": {
-    "colab": {
-      "provenance": []
-    },
-    "kernelspec": {
-      "name": "python3",
-      "display_name": "Python 3"
-    },
-    "language_info": {
-      "name": "python"
-    }
-  },
   "cells": [
     {
       "cell_type": "markdown",
-      "source": [
-        "## Cookbook for using OpenSearchDB with Embedchain"
-      ],
       "metadata": {
         "id": "b02n_zJ_hl3d"
-      }
+      },
+      "source": [
+        "## Cookbook for using OpenSearchDB with Embedchain"
+      ]
     },
     {
       "cell_type": "markdown",
-      "source": [
-        "### Step-1: Install embedchain package"
-      ],
       "metadata": {
         "id": "gyJ6ui2vhtMY"
-      }
+      },
+      "source": [
+        "### Step-1: Install embedchain package"
+      ]
     },
     {
       "cell_type": "code",
-      "source": [
-        "!pip install embedchain"
-      ],
+      "execution_count": null,
       "metadata": {
         "id": "-NbXjAdlh0vJ"
       },
+      "outputs": [],
+      "source": [
+        "!pip install embedchain"
+      ]
+    },
+    {
+      "cell_type": "code",
       "execution_count": null,
-      "outputs": []
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "!pip install embedchain[dataloaders]"
+      ]
     },
     {
       "cell_type": "markdown",
+      "metadata": {
+        "id": "nGnpSYAAh2bQ"
+      },
       "source": [
         "### Step-2: Set OpenAI environment variables and install the dependencies.\n",
         "\n",
         "You can find this env variable on your [OpenAI dashboard](https://platform.openai.com/account/api-keys). Now lets install the dependencies needed for Opensearch."
-      ],
-      "metadata": {
-        "id": "nGnpSYAAh2bQ"
-      }
+      ]
     },
     {
       "cell_type": "code",
-      "source": [
-        "!pip install --upgrade 'embedchain[opensearch]'"
-      ],
+      "execution_count": null,
       "metadata": {
         "id": "-MUFRfxV7Jk7"
       },
-      "execution_count": null,
-      "outputs": []
+      "outputs": [],
+      "source": [
+        "!pip install --upgrade 'embedchain[opensearch]'"
+      ]
     },
     {
       "cell_type": "code",
-      "source": [
-        "import os\n",
-        "from embedchain import App\n",
-        "\n",
-        "os.environ[\"OPENAI_API_KEY\"] = \"sk-xxx\""
-      ],
+      "execution_count": null,
       "metadata": {
         "id": "0fBdQ9GAiRvK"
       },
-      "execution_count": null,
-      "outputs": []
+      "outputs": [],
+      "source": [
+        "import os\n",
+        "from embedchain import Pipeline as App\n",
+        "\n",
+        "os.environ[\"OPENAI_API_KEY\"] = \"sk-xxx\""
+      ]
     },
     {
       "cell_type": "markdown",
-      "source": [
-        "### Step-3: Define your Vector Database config"
-      ],
       "metadata": {
         "id": "Ns6RhPfbiitr"
-      }
+      },
+      "source": [
+        "### Step-3: Define your Vector Database config"
+      ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "S9CkxVjriotB"
+      },
+      "outputs": [],
       "source": [
         "config = \"\"\"\n",
         "vectordb:\n",
@@ -108,64 +108,64 @@
         "# Write the multi-line string to a YAML file\n",
         "with open('opensearch.yaml', 'w') as file:\n",
         "    file.write(config)"
-      ],
-      "metadata": {
-        "id": "S9CkxVjriotB"
-      },
-      "execution_count": null,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
-      "source": [
-        "### Step-4 Create embedchain app based on the config"
-      ],
       "metadata": {
         "id": "PGt6uPLIi1CS"
-      }
+      },
+      "source": [
+        "### Step-4 Create embedchain app based on the config"
+      ]
     },
     {
       "cell_type": "code",
-      "source": [
-        "app = App.from_config(yaml_path=\"opensearch.yaml\")"
-      ],
+      "execution_count": null,
       "metadata": {
         "id": "Amzxk3m-i3tD"
       },
-      "execution_count": null,
-      "outputs": []
+      "outputs": [],
+      "source": [
+        "app = App.from_config(yaml_path=\"opensearch.yaml\")"
+      ]
     },
     {
       "cell_type": "markdown",
-      "source": [
-        "### Step-5: Add data sources to your app"
-      ],
       "metadata": {
         "id": "XNXv4yZwi7ef"
-      }
+      },
+      "source": [
+        "### Step-5: Add data sources to your app"
+      ]
     },
     {
       "cell_type": "code",
-      "source": [
-        "app.add(\"https://www.forbes.com/profile/elon-musk\")"
-      ],
+      "execution_count": null,
       "metadata": {
         "id": "Sn_0rx9QjIY9"
       },
-      "execution_count": null,
-      "outputs": []
+      "outputs": [],
+      "source": [
+        "app.add(\"https://www.forbes.com/profile/elon-musk\")"
+      ]
     },
     {
       "cell_type": "markdown",
-      "source": [
-        "### Step-6: All set. Now start asking questions related to your data"
-      ],
       "metadata": {
         "id": "_7W6fDeAjMAP"
-      }
+      },
+      "source": [
+        "### Step-6: All set. Now start asking questions related to your data"
+      ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "cvIK7dWRjN_f"
+      },
+      "outputs": [],
       "source": [
         "while(True):\n",
         "    question = input(\"Enter question: \")\n",
@@ -173,12 +173,21 @@
         "        break\n",
         "    answer = app.query(question)\n",
         "    print(answer)"
-      ],
-      "metadata": {
-        "id": "cvIK7dWRjN_f"
-      },
-      "execution_count": null,
-      "outputs": []
+      ]
     }
-  ]
-}
+  ],
+  "metadata": {
+    "colab": {
+      "provenance": []
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    },
+    "language_info": {
+      "name": "python"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

notebooks/pinecone.ipynb

@@ -29,6 +29,15 @@
         "!pip install embedchain"
       ]
     },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "!pip install embedchain[dataloaders]"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -60,7 +69,7 @@
       "outputs": [],
       "source": [
         "import os\n",
-        "from embedchain import App\n",
+        "from embedchain import Pipeline as App\n",
         "\n",
         "os.environ[\"OPENAI_API_KEY\"] = \"sk-xxx\"\n",
         "os.environ[\"PINECONE_API_KEY\"] = \"xxx\"\n",

notebooks/vertex_ai.ipynb

@@ -33,6 +33,15 @@
         "!pip install embedchain"
       ]
     },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "!pip install embedchain[dataloaders]"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -64,7 +73,7 @@
       "outputs": [],
       "source": [
         "import os\n",
-        "from embedchain import App\n",
+        "from embedchain import Pipeline as App\n",
         "\n",
         "os.environ[\"OPENAI_API_KEY\"] = \"sk-xxx\""
       ]

poetry.lock

@@ -2620,6 +2620,31 @@ files = [
     {file = "jq-1.6.0.tar.gz", hash = "sha256:c7711f0c913a826a00990736efa6ffc285f8ef433414516bb14b7df971d6c1ea"},
 ]
 
+[[package]]
+name = "jsonpatch"
+version = "1.33"
+description = "Apply JSON-Patches (RFC 6902)"
+optional = false
+python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, !=3.6.*"
+files = [
+    {file = "jsonpatch-1.33-py2.py3-none-any.whl", hash = "sha256:0ae28c0cd062bbd8b8ecc26d7d164fbbea9652a1a3693f3b956c1eae5145dade"},
+    {file = "jsonpatch-1.33.tar.gz", hash = "sha256:9fcd4009c41e6d12348b4a0ff2563ba56a2923a7dfee731d004e212e1ee5030c"},
+]
+
+[package.dependencies]
+jsonpointer = ">=1.9"
+
+[[package]]
+name = "jsonpointer"
+version = "2.4"
+description = "Identify specific nodes in a JSON document (RFC 6901)"
+optional = false
+python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, !=3.6.*"
+files = [
+    {file = "jsonpointer-2.4-py2.py3-none-any.whl", hash = "sha256:15d51bba20eea3165644553647711d150376234112651b4f1811022aecad7d7a"},
+    {file = "jsonpointer-2.4.tar.gz", hash = "sha256:585cee82b70211fa9e6043b7bb89db6e1aa49524340dde8ad6b63206ea689d88"},
+]
+
 [[package]]
 name = "kiwisolver"
 version = "1.4.5"
@@ -2735,20 +2760,22 @@ files = [
 
 [[package]]
 name = "langchain"
-version = "0.0.279"
+version = "0.0.303"
 description = "Building applications with LLMs through composability"
 optional = false
 python-versions = ">=3.8.1,<4.0"
 files = [
-    {file = "langchain-0.0.279-py3-none-any.whl", hash = "sha256:d91e65b3a210e9b52e42ed67c65d76caeaa3195ff097217f5ab2ca477bd3ae27"},
-    {file = "langchain-0.0.279.tar.gz", hash = "sha256:344e33d73c76ae35cdd8c3ba16c965bac29e0e889c93064718c5982002cbe30c"},
+    {file = "langchain-0.0.303-py3-none-any.whl", hash = "sha256:1745961f66b60bc3b513820a34c560dd37c4ba4b7499ba82545dc4816d0133bd"},
+    {file = "langchain-0.0.303.tar.gz", hash = "sha256:84d2727eb8b3b27a9d0aa0da9f05408c2564a4a923c7d5b154a16e488430e725"},
 ]
 
 [package.dependencies]
 aiohttp = ">=3.8.3,<4.0.0"
+anyio = "<4.0"
 async-timeout = {version = ">=4.0.0,<5.0.0", markers = "python_version < \"3.11\""}
-dataclasses-json = ">=0.5.7,<0.6.0"
-langsmith = ">=0.0.21,<0.1.0"
+dataclasses-json = ">=0.5.7,<0.7"
+jsonpatch = ">=1.33,<2.0"
+langsmith = ">=0.0.38,<0.1.0"
 numexpr = ">=2.8.4,<3.0.0"
 numpy = ">=1,<2"
 pydantic = ">=1,<3"
@@ -2764,7 +2791,7 @@ clarifai = ["clarifai (>=9.1.0)"]
 cohere = ["cohere (>=4,<5)"]
 docarray = ["docarray[hnswlib] (>=0.32.0,<0.33.0)"]
 embeddings = ["sentence-transformers (>=2,<3)"]
-extended-testing = ["amazon-textract-caller (<2)", "assemblyai (>=0.17.0,<0.18.0)", "atlassian-python-api (>=3.36.0,<4.0.0)", "beautifulsoup4 (>=4,<5)", "bibtexparser (>=1.4.0,<2.0.0)", "cassio (>=0.0.7,<0.0.8)", "chardet (>=5.1.0,<6.0.0)", "esprima (>=4.0.1,<5.0.0)", "faiss-cpu (>=1,<2)", "feedparser (>=6.0.10,<7.0.0)", "geopandas (>=0.13.1,<0.14.0)", "gitpython (>=3.1.32,<4.0.0)", "gql (>=3.4.1,<4.0.0)", "html2text (>=2020.1.16,<2021.0.0)", "jinja2 (>=3,<4)", "jq (>=1.4.1,<2.0.0)", "lxml (>=4.9.2,<5.0.0)", "markdownify (>=0.11.6,<0.12.0)", "mwparserfromhell (>=0.6.4,<0.7.0)", "mwxml (>=0.3.3,<0.4.0)", "newspaper3k (>=0.2.8,<0.3.0)", "openai (>=0,<1)", "openapi-schema-pydantic (>=1.2,<2.0)", "pandas (>=2.0.1,<3.0.0)", "pdfminer-six (>=20221105,<20221106)", "pgvector (>=0.1.6,<0.2.0)", "psychicapi (>=0.8.0,<0.9.0)", "py-trello (>=0.19.0,<0.20.0)", "pymupdf (>=1.22.3,<2.0.0)", "pypdf (>=3.4.0,<4.0.0)", "pypdfium2 (>=4.10.0,<5.0.0)", "pyspark (>=3.4.0,<4.0.0)", "rank-bm25 (>=0.2.2,<0.3.0)", "rapidfuzz (>=3.1.1,<4.0.0)", "requests-toolbelt (>=1.0.0,<2.0.0)", "scikit-learn (>=1.2.2,<2.0.0)", "sqlite-vss (>=0.1.2,<0.2.0)", "streamlit (>=1.18.0,<2.0.0)", "sympy (>=1.12,<2.0)", "telethon (>=1.28.5,<2.0.0)", "tqdm (>=4.48.0)", "xata (>=1.0.0a7,<2.0.0)", "xmltodict (>=0.13.0,<0.14.0)"]
+extended-testing = ["amazon-textract-caller (<2)", "assemblyai (>=0.17.0,<0.18.0)", "atlassian-python-api (>=3.36.0,<4.0.0)", "beautifulsoup4 (>=4,<5)", "bibtexparser (>=1.4.0,<2.0.0)", "cassio (>=0.1.0,<0.2.0)", "chardet (>=5.1.0,<6.0.0)", "dashvector (>=1.0.1,<2.0.0)", "esprima (>=4.0.1,<5.0.0)", "faiss-cpu (>=1,<2)", "feedparser (>=6.0.10,<7.0.0)", "geopandas (>=0.13.1,<0.14.0)", "gitpython (>=3.1.32,<4.0.0)", "gql (>=3.4.1,<4.0.0)", "html2text (>=2020.1.16,<2021.0.0)", "jinja2 (>=3,<4)", "jq (>=1.4.1,<2.0.0)", "lxml (>=4.9.2,<5.0.0)", "markdownify (>=0.11.6,<0.12.0)", "mwparserfromhell (>=0.6.4,<0.7.0)", "mwxml (>=0.3.3,<0.4.0)", "newspaper3k (>=0.2.8,<0.3.0)", "openai (>=0,<1)", "openapi-schema-pydantic (>=1.2,<2.0)", "pandas (>=2.0.1,<3.0.0)", "pdfminer-six (>=20221105,<20221106)", "pgvector (>=0.1.6,<0.2.0)", "psychicapi (>=0.8.0,<0.9.0)", "py-trello (>=0.19.0,<0.20.0)", "pymupdf (>=1.22.3,<2.0.0)", "pypdf (>=3.4.0,<4.0.0)", "pypdfium2 (>=4.10.0,<5.0.0)", "pyspark (>=3.4.0,<4.0.0)", "rank-bm25 (>=0.2.2,<0.3.0)", "rapidfuzz (>=3.1.1,<4.0.0)", "requests-toolbelt (>=1.0.0,<2.0.0)", "scikit-learn (>=1.2.2,<2.0.0)", "sqlite-vss (>=0.1.2,<0.2.0)", "streamlit (>=1.18.0,<2.0.0)", "sympy (>=1.12,<2.0)", "telethon (>=1.28.5,<2.0.0)", "timescale-vector (>=0.0.1,<0.0.2)", "tqdm (>=4.48.0)", "xata (>=1.0.0a7,<2.0.0)", "xmltodict (>=0.13.0,<0.14.0)"]
 javascript = ["esprima (>=4.0.1,<5.0.0)"]
 llms = ["clarifai (>=9.1.0)", "cohere (>=4,<5)", "huggingface_hub (>=0,<1)", "manifest-ml (>=0.0.1,<0.0.2)", "nlpcloud (>=1,<2)", "openai (>=0,<1)", "openlm (>=0.0.5,<0.0.6)", "torch (>=1,<3)", "transformers (>=4,<5)"]
 openai = ["openai (>=0,<1)", "tiktoken (>=0.3.2,<0.4.0)"]
@@ -7196,4 +7223,4 @@ whatsapp = ["flask", "twilio"]
 [metadata]
 lock-version = "2.0"
 python-versions = ">=3.9,<3.13"
-content-hash = "fad388baca92d2669530c2d928920e9f7abbe0ad74786b34dc915603b4a08a17"
+content-hash = "17a067a3994a2f17c55e6ee48a3f8ec664a94112951183070ed34d40d25a7533"

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "embedchain"
-version = "0.0.83"
+version = "0.0.86"
 description = "Data platform for LLMs - Load, index, retrieve and sync any unstructured data"
 authors = [
     "Taranjeet Singh <taranjeet@embedchain.ai>",
@@ -90,7 +90,7 @@ color = true
 [tool.poetry.dependencies]
 python = ">=3.9,<3.13"
 python-dotenv = "^1.0.0"
-langchain = "^0.0.279"
+langchain = "^0.0.303"
 requests = "^2.31.0"
 openai = ">=0.28.0"
 chromadb = "^0.4.8"