[version] Update openai version to 0.28.0 (#866)

README.md

@@ -1,5 +1,7 @@
 # embedchain
 
+<a href="https://runacap.com/ross-index/q3-2023/" target="_blank" rel="noopener"><img style="width: 260px; height: 56px" src="https://runacap.com/wp-content/uploads/2023/10/ROSS_badge_black_Q3_2023.svg" alt="ROSS Index - Fastest Growing Open-Source Startups in Q3 2023 | Runa Capital" width="260" height="56"/></a>
+
 [![PyPI](https://img.shields.io/pypi/v/embedchain)](https://pypi.org/project/embedchain/)
 [![Slack](https://img.shields.io/badge/slack-embedchain-brightgreen.svg?logo=slack)](https://join.slack.com/t/embedchain/shared_invite/zt-22uwz3c46-Zg7cIh5rOBteT_xe1jwLDw)
 [![Discord](https://dcbadge.vercel.app/api/server/6PzXDgEjG5?style=flat)](https://discord.gg/CUU9FPhRNt)
@@ -38,20 +40,23 @@ The documentation for embedchain can be found at [docs.embedchain.ai](https://do
 
 Embedchain empowers you to create ChatGPT like apps, on your own dynamic dataset.
 
-### Data Types Supported
+### Data types supported
 
 * Youtube video
 * PDF file
+* CSV file
 * Web page
+* MDX file
+* XML file
 * Sitemap
 * Doc file
-* JSON file
-* Code documentation website loader
-* OpenAPI specs
 * Notion
+* JSON file
+* OpenAPI specs
+* Code docs website
 * Unstructured file loader and many more
 
-You can find the full list of data types on [our documentation](https://docs.embedchain.ai/data-sources/csv).
+You can find the full list of data types on [our documentation](https://docs.embedchain.ai/data-sources/).
 
 ### Queries
 

docs/data-sources/gmail.mdx

@@ -0,0 +1,35 @@
+---
+title: '📬 Gmail'
+---
+
+To use GmailLoader you must install the extra dependencies with `pip install --upgrade embedchain[gmail]`.
+
+The `source` must be a valid Gmail search query, you can refer `https://support.google.com/mail/answer/7190?hl=en` to build a query.
+
+To load Gmail messages, you MUST use the data_type as `gmail`. Otherwise the source will be detected as simple `text`.
+
+To use this you need to save `credentials.json` in the directory from where you will run the loader. Follow these steps to get the credentials
+
+1. Go to the [Google Cloud Console](https://console.cloud.google.com/apis/credentials).
+2. Create a project if you don't have one already.
+3. Create an `OAuth Consent Screen` in the project. You may need to select the `external` option.
+4. Make sure the consent screen is published.
+5. Enable the [Gmail API](https://console.cloud.google.com/apis/api/gmail.googleapis.com)
+6. Create credentials from the `Credentials` tab.
+7. Select the type `OAuth Client ID`.
+8. Choose the application type `Web application`. As a name you can choose `embedchain` or any other name as per your use case.
+9. Add an authorized redirect URI for `http://localhost:8080/`.
+10. You can leave everything else at default, finish the creation.
+11. When you are done, a modal opens where you can download the details in `json` format.
+12. Put the `.json` file in your current directory and rename it to `credentials.json`
+
+```python
+import os
+from embedchain.apps.app import App
+from embedchain.models.data_type import DataType
+app = App()
+
+query = "to: me label:inbox"
+app.add(query, data_type=DataType.GMAIL)
+app.query("Summarize my email conversations")
+```

docs/data-sources/notion.mdx

@@ -2,7 +2,7 @@
 title: '📓 Notion'
 ---
 
-To use notion you must install the extra dependencies with `pip install --upgrade embedchain[notion]`.
+To use notion you must install the extra dependencies with `pip install --upgrade embedchain[community]`.
 
 To load a notion page, use the data_type as `notion`. Since it is hard to automatically detect, it is advised to specify the `data_type` when adding a notion document.
 The next argument must **end** with the `notion page id`. The id is a 32-character string. Eg:

docs/data-sources/overview.mdx

@@ -17,8 +17,9 @@ Embedchain comes with built-in support for various data sources. We handle the c
   <Card title="📝 text" href="/data-sources/text"></Card>
   <Card title="🌐📄 web page" href="/data-sources/web-page"></Card>
   <Card title="🧾 xml" href="/data-sources/xml"></Card>
-  <Card title="🙌 OpenApi" href="/data-sources/openapi"></Card>
+  <Card title="🙌 OpenAPI" href="/data-sources/openapi"></Card>
   <Card title="🎥📺 youtube video" href="/data-sources/youtube-video"></Card>
+  <Card title="📬 Gmail" href="/data-sources/gmail"></Card>
 </CardGroup>
 
 <br/ >

docs/mint.json

@@ -71,6 +71,10 @@
       "group": "Examples",
       "pages": ["examples/full_stack", "examples/api_server", "examples/discord_bot", "examples/slack_bot", "examples/telegram_bot", "examples/whatsapp_bot", "examples/poe_bot"]
     },
+    {
+      "group": "Pipelines",
+      "pages": ["pipelines/quickstart"]
+    },
     {
       "group": "Community",
       "pages": [

docs/pipelines/quickstart.mdx

@@ -0,0 +1,44 @@
+---
+title: '🚀 Pipelines'
+description: '💡 Start building LLM powered data pipelines in 1 minute'
+---
+
+Embedchain lets you build data pipelines on your own data sources and deploy it in production in less than a minute. It can load, index, retrieve, and sync any unstructured data.
+
+Install embedchain python package:
+
+```bash
+pip install embedchain
+```
+
+Creating a pipeline involves 3 steps:
+
+<Steps>
+  <Step title="⚙️ Import pipeline instance">
+```python
+from embedchain import Pipeline
+p = Pipeline(name="Elon Musk")
+```
+  </Step>
+
+  <Step title="🗃️ Add data sources">
+```python
+# Add different data sources
+p.add("https://en.wikipedia.org/wiki/Elon_Musk")
+p.add("https://www.forbes.com/profile/elon-musk")
+# You can also add local data sources such as pdf, csv files etc.
+# p.add("/path/to/file.pdf")
+```
+  </Step>
+  <Step title="💬 Deploy your pipeline to Embedchain platform">
+```python
+p.deploy()
+```
+  </Step>
+</Steps>
+
+That's it. Now, head to the [Embedchain platform](https://app.embedchain.ai) and your pipeline is available there. Make sure to set the `OPENAI_API_KEY` 🔑 environment variable in the code.
+
+After you deploy your pipeline to Embedchain platform, you can still add more data sources and update the pipeline multiple times.
+
+Here is a Google Colab notebook for you to get started: [![Open in Colab](https://camo.githubusercontent.com/84f0493939e0c4de4e6dbe113251b4bfb5353e57134ffd9fcab6b8714514d4d1/68747470733a2f2f636f6c61622e72657365617263682e676f6f676c652e636f6d2f6173736574732f636f6c61622d62616467652e737667)](https://colab.research.google.com/drive/1YVXaBO4yqlHZY4ho67GCJ6aD4CHNiScD?usp=sharing)

embedchain/chunkers/gmail.py

@@ -0,0 +1,22 @@
+from typing import Optional
+
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+
+from embedchain.chunkers.base_chunker import BaseChunker
+from embedchain.config.add_config import ChunkerConfig
+from embedchain.helper.json_serializable import register_deserializable
+
+
+@register_deserializable
+class GmailChunker(BaseChunker):
+    """Chunker for gmail."""
+
+    def __init__(self, config: Optional[ChunkerConfig] = None):
+        if config is None:
+            config = ChunkerConfig(chunk_size=1000, chunk_overlap=0, length_function=len)
+        text_splitter = RecursiveCharacterTextSplitter(
+            chunk_size=config.chunk_size,
+            chunk_overlap=config.chunk_overlap,
+            length_function=config.length_function,
+        )
+        super().__init__(text_splitter)

embedchain/data_formatter/data_formatter.py

@@ -1,6 +1,7 @@
 from embedchain.chunkers.base_chunker import BaseChunker
 from embedchain.chunkers.docs_site import DocsSiteChunker
 from embedchain.chunkers.docx_file import DocxFileChunker
+from embedchain.chunkers.gmail import GmailChunker
 from embedchain.chunkers.images import ImagesChunker
 from embedchain.chunkers.json import JSONChunker
 from embedchain.chunkers.mdx import MdxChunker
@@ -22,6 +23,7 @@ from embedchain.loaders.base_loader import BaseLoader
 from embedchain.loaders.csv import CsvLoader
 from embedchain.loaders.docs_site_loader import DocsSiteLoader
 from embedchain.loaders.docx_file import DocxFileLoader
+from embedchain.loaders.gmail import GmailLoader
 from embedchain.loaders.images import ImagesLoader
 from embedchain.loaders.json import JSONLoader
 from embedchain.loaders.local_qna_pair import LocalQnaPairLoader
@@ -84,6 +86,7 @@ class DataFormatter(JSONSerializable):
             DataType.UNSTRUCTURED: UnstructuredLoader,
             DataType.JSON: JSONLoader,
             DataType.OPENAPI: OpenAPILoader,
+            DataType.GMAIL: GmailLoader,
         }
         lazy_loaders = {DataType.NOTION}
         if data_type in loaders:
@@ -128,6 +131,7 @@ class DataFormatter(JSONSerializable):
             DataType.UNSTRUCTURED: UnstructuredFileChunker,
             DataType.JSON: JSONChunker,
             DataType.OPENAPI: OpenAPIChunker,
+            DataType.GMAIL: GmailChunker,
         }
         if data_type in chunker_classes:
             chunker_class: type = chunker_classes[data_type]

embedchain/factory.py

@@ -50,6 +50,7 @@ class EmbedderFactory:
     provider_to_config_class = {
         "azure_openai": "embedchain.config.embedder.base.BaseEmbedderConfig",
         "openai": "embedchain.config.embedder.base.BaseEmbedderConfig",
+        "gpt4all": "embedchain.config.embedder.base.BaseEmbedderConfig",
     }
 
     @classmethod

embedchain/llm/gpt4all.py

@@ -15,7 +15,7 @@ class GPT4ALLLlm(BaseLlm):
         if self.config.model is None:
             self.config.model = "orca-mini-3b.ggmlv3.q4_0.bin"
         self.instance = GPT4ALLLlm._get_instance(self.config.model)
-        self.instance.streaming = config.stream
+        self.instance.streaming = self.config.stream
 
     def get_llm_model_answer(self, prompt):
         return self._get_answer(prompt=prompt, config=self.config)

embedchain/loaders/gmail.py

@@ -0,0 +1,123 @@
+import hashlib
+import logging
+import os
+import quopri
+from textwrap import dedent
+
+from bs4 import BeautifulSoup
+
+try:
+    from llama_hub.gmail.base import GmailReader
+except ImportError:
+    raise ImportError("Gmail requires extra dependencies. Install with `pip install embedchain[gmail]`") from None
+
+from embedchain.loaders.base_loader import BaseLoader
+from embedchain.utils import clean_string
+
+
+def get_header(text: str, header: str) -> str:
+    start_string_position = text.find(header)
+    pos_start = text.find(":", start_string_position) + 1
+    pos_end = text.find("\n", pos_start)
+    header = text[pos_start:pos_end]
+    return header.strip()
+
+
+class GmailLoader(BaseLoader):
+    def load_data(self, query):
+        """Load data from gmail."""
+        if not os.path.isfile("credentials.json"):
+            raise FileNotFoundError(
+                "You must download the valid credentials file from your google \
+                dev account. Refer this `https://cloud.google.com/docs/authentication/api-keys`"
+            )
+
+        loader = GmailReader(query=query, service=None, results_per_page=20)
+        documents = loader.load_data()
+        logging.info(f"Gmail Loader: {len(documents)} mails found for query- {query}")
+
+        data = []
+        data_contents = []
+        logging.info(f"Gmail Loader: {len(documents)} mails found")
+        for document in documents:
+            original_size = len(document.text)
+
+            snippet = document.metadata.get("snippet")
+            meta_data = {
+                "url": document.metadata.get("id"),
+                "date": get_header(document.text, "Date"),
+                "subject": get_header(document.text, "Subject"),
+                "from": get_header(document.text, "From"),
+                "to": get_header(document.text, "To"),
+                "search_query": query,
+            }
+
+            # Decode
+            decoded_bytes = quopri.decodestring(document.text)
+            decoded_str = decoded_bytes.decode("utf-8", errors="replace")
+
+            # Slice
+            mail_start = decoded_str.find("<!DOCTYPE")
+            email_data = decoded_str[mail_start:]
+
+            # Web Page HTML Processing
+            soup = BeautifulSoup(email_data, "html.parser")
+
+            tags_to_exclude = [
+                "nav",
+                "aside",
+                "form",
+                "header",
+                "noscript",
+                "svg",
+                "canvas",
+                "footer",
+                "script",
+                "style",
+            ]
+
+            for tag in soup(tags_to_exclude):
+                tag.decompose()
+
+            ids_to_exclude = ["sidebar", "main-navigation", "menu-main-menu"]
+            for id in ids_to_exclude:
+                tags = soup.find_all(id=id)
+                for tag in tags:
+                    tag.decompose()
+
+            classes_to_exclude = [
+                "elementor-location-header",
+                "navbar-header",
+                "nav",
+                "header-sidebar-wrapper",
+                "blog-sidebar-wrapper",
+                "related-posts",
+            ]
+
+            for class_name in classes_to_exclude:
+                tags = soup.find_all(class_=class_name)
+                for tag in tags:
+                    tag.decompose()
+
+            content = soup.get_text()
+            content = clean_string(content)
+
+            cleaned_size = len(content)
+            if original_size != 0:
+                logging.info(
+                    f"[{id}] Cleaned page size: {cleaned_size} characters, down from {original_size} (shrunk: {original_size-cleaned_size} chars, {round((1-(cleaned_size/original_size)) * 100, 2)}%)"  # noqa:E501
+                )
+
+            result = f"""
+            email from '{meta_data.get('from')}' to '{meta_data.get('to')}'
+            subject: {meta_data.get('subject')}
+            date: {meta_data.get('date')}
+            preview: {snippet}
+            content: f{content}
+            """
+            data_content = dedent(result)
+            data.append({"content": data_content, "meta_data": meta_data})
+            data_contents.append(data_content)
+        doc_id = hashlib.sha256((query + ", ".join(data_contents)).encode()).hexdigest()
+        response_data = {"doc_id": doc_id, "data": data}
+        return response_data

embedchain/models/data_type.py

@@ -28,6 +28,7 @@ class IndirectDataType(Enum):
     UNSTRUCTURED = "unstructured"
     JSON = "json"
     OPENAPI = "openapi"
+    GMAIL = "gmail"
 
 
 class SpecialDataType(Enum):
@@ -55,3 +56,4 @@ class DataType(Enum):
     UNSTRUCTURED = IndirectDataType.UNSTRUCTURED.value
     JSON = IndirectDataType.JSON.value
     OPENAPI = IndirectDataType.OPENAPI.value
+    GMAIL = IndirectDataType.GMAIL.value

embedchain/pipeline.py

@@ -34,6 +34,8 @@ class Pipeline(EmbedChain):
 
     def __init__(
         self,
+        id: str = None,
+        name: str = None,
         config: PipelineConfig = None,
         db: BaseVectorDB = None,
         embedding_model: BaseEmbedder = None,
@@ -61,6 +63,15 @@ class Pipeline(EmbedChain):
         :type auto_deploy: bool, optional
         :raises Exception: If an error occurs while creating the pipeline
         """
+        if id and yaml_path:
+            raise Exception("Cannot provide both id and config. Please provide only one of them.")
+
+        if id and name:
+            raise Exception("Cannot provide both id and name. Please provide only one of them.")
+
+        if name and config:
+            raise Exception("Cannot provide both name and config. Please provide only one of them.")
+
         logging.basicConfig(level=log_level, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s")
         self.logger = logging.getLogger(__name__)
 
@@ -71,16 +82,28 @@ class Pipeline(EmbedChain):
         self.client = None
         # pipeline_id from the backend
         self.id = None
-        if yaml_path:
-            with open(yaml_path, "r") as file:
-                config_data = yaml.safe_load(file)
-                self.yaml_config = config_data
 
         self.config = config or PipelineConfig()
         self.name = self.config.name
 
         self.config.id = self.local_id = str(uuid.uuid4()) if self.config.id is None else self.config.id
 
+        if yaml_path:
+            with open(yaml_path, "r") as file:
+                config_data = yaml.safe_load(file)
+                self.yaml_config = config_data
+
+        if id is not None:
+            # Init client first since user is trying to fetch the pipeline
+            # details from the platform
+            self._init_client()
+            pipeline_details = self._get_pipeline(id)
+            self.config.id = self.local_id = pipeline_details["metadata"]["local_id"]
+            self.id = id
+
+        if name is not None:
+            self.name = name
+
         self.embedding_model = embedding_model or OpenAIEmbedder()
         self.db = db or ChromaDB()
         self.llm = llm or OpenAILlm()
@@ -134,6 +157,24 @@ class Pipeline(EmbedChain):
             )
             self.client = Client(api_key=api_key)
 
+    def _get_pipeline(self, id):
+        """
+        Get existing pipeline
+        """
+        print("🛠️ Fetching pipeline details from the platform...")
+        url = f"{self.client.host}/api/v1/pipelines/{id}/cli/"
+        r = requests.get(
+            url,
+            headers={"Authorization": f"Token {self.client.api_key}"},
+        )
+        if r.status_code == 404:
+            raise Exception(f"❌ Pipeline with id {id} not found!")
+
+        print(
+            f"🎉 Pipeline loaded successfully! Pipeline url: https://app.embedchain.ai/pipelines/{r.json()['id']}\n"  # noqa: E501
+        )
+        return r.json()
+
     def _create_pipeline(self):
         """
         Create a pipeline on the platform.
@@ -154,9 +195,14 @@ class Pipeline(EmbedChain):
         if r.status_code not in [200, 201]:
             raise Exception(f"❌ Error occurred while creating pipeline. API response: {r.text}")
 
-        print(
+        if r.status_code == 200:
-            f"🎉🎉🎉 Pipeline created successfully! View your pipeline: https://app.embedchain.ai/pipelines/{r.json()['id']}\n"  # noqa: E501
+            print(
-        )
+                f"🎉🎉🎉 Existing pipeline found! View your pipeline: https://app.embedchain.ai/pipelines/{r.json()['id']}\n"  # noqa: E501
+            )  # noqa: E501
+        elif r.status_code == 201:
+            print(
+                f"🎉🎉🎉 Pipeline created successfully! View your pipeline: https://app.embedchain.ai/pipelines/{r.json()['id']}\n"  # noqa: E501
+            )
         return r.json()
 
     def _get_presigned_url(self, data_type, data_value):
@@ -257,7 +303,7 @@ class Pipeline(EmbedChain):
         self.id = pipeline_data["id"]
 
         results = self.cursor.execute(
-            "SELECT * FROM data_sources WHERE pipeline_id = ? AND is_uploaded = 0", (self.local_id,)
+            "SELECT * FROM data_sources WHERE pipeline_id = ? AND is_uploaded = 0", (self.local_id,)  # noqa:E501
         ).fetchall()
 
         if len(results) > 0:

embedchain/utils.py

@@ -259,6 +259,8 @@ def detect_datatype(source: Any) -> DataType:
     else:
         # Source is not a URL.
 
+        # TODO: check if source is gmail query
+
         # Use text as final fallback.
         logging.debug(f"Source of `{formatted_source}` detected as `text`.")
         return DataType.TEXT

examples/api_server/Dockerfile

@@ -8,4 +8,9 @@ COPY . .
 
 EXPOSE 5000
 
-CMD ["python", "api_server.py"]
+ENV FLASK_APP=api_server.py
+
+ENV FLASK_RUN_EXTRA_FILES=/usr/src/api/*
+ENV FLASK_ENV=development
+
+CMD ["flask", "run", "--host=0.0.0.0", "--reload"]

examples/api_server/api_server.py

@@ -1,3 +1,5 @@
+import logging
+
 from flask import Flask, jsonify, request
 
 from embedchain import App
@@ -5,11 +7,6 @@ from embedchain import App
 app = Flask(__name__)
 
 
-def initialize_chat_bot():
-    global chat_bot
-    chat_bot = App()
-
-
 @app.route("/add", methods=["POST"])
 def add():
     data = request.get_json()
@@ -17,9 +14,10 @@ def add():
     url_or_text = data.get("url_or_text")
     if data_type and url_or_text:
         try:
-            chat_bot.add(data_type, url_or_text)
+            App().add(url_or_text, data_type=data_type)
             return jsonify({"data": f"Added {data_type}: {url_or_text}"}), 200
         except Exception:
+            logging.exception(f"Failed to add {data_type=}: {url_or_text=}")
             return jsonify({"error": f"Failed to add {data_type}: {url_or_text}"}), 500
     return jsonify({"error": "Invalid request. Please provide 'data_type' and 'url_or_text' in JSON format."}), 400
 
@@ -30,9 +28,10 @@ def query():
     question = data.get("question")
     if question:
         try:
-            response = chat_bot.query(question)
+            response = App().query(question)
             return jsonify({"data": response}), 200
         except Exception:
+            logging.exception(f"Failed to query {question=}")
             return jsonify({"error": "An error occurred. Please try again!"}), 500
     return jsonify({"error": "Invalid request. Please provide 'question' in JSON format."}), 400
 
@@ -43,13 +42,13 @@ def chat():
     question = data.get("question")
     if question:
         try:
-            response = chat_bot.chat(question)
+            response = App().chat(question)
             return jsonify({"data": response}), 200
         except Exception:
+            logging.exception(f"Failed to chat {question=}")
             return jsonify({"error": "An error occurred. Please try again!"}), 500
     return jsonify({"error": "Invalid request. Please provide 'question' in JSON format."}), 400
 
 
 if __name__ == "__main__":
-    initialize_chat_bot()
     app.run(host="0.0.0.0", port=5000, debug=False)

examples/api_server/docker-compose.yml

@@ -10,4 +10,6 @@ services:
     env_file:
       - variables.env
     ports:
-      - "5000:5000"
+      - "5000:5000"
+    volumes:
+      - .:/usr/src/api

examples/api_server/requirements.txt

@@ -1,2 +1,2 @@
 flask==2.3.2
-embedchain==0.0.58
+embedchain[dataloaders]==0.0.78

poetry.lock

@@ -3455,40 +3455,6 @@ files = [
 [package.dependencies]
 numpy = ">=1.13.3"
 
-[[package]]
-name = "numpy"
-version = "1.25.2"
-description = "Fundamental package for array computing in Python"
-optional = false
-python-versions = ">=3.9"
-files = [
-    {file = "numpy-1.25.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:db3ccc4e37a6873045580d413fe79b68e47a681af8db2e046f1dacfa11f86eb3"},
-    {file = "numpy-1.25.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:90319e4f002795ccfc9050110bbbaa16c944b1c37c0baeea43c5fb881693ae1f"},
-    {file = "numpy-1.25.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dfe4a913e29b418d096e696ddd422d8a5d13ffba4ea91f9f60440a3b759b0187"},
-    {file = "numpy-1.25.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f08f2e037bba04e707eebf4bc934f1972a315c883a9e0ebfa8a7756eabf9e357"},
-    {file = "numpy-1.25.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bec1e7213c7cb00d67093247f8c4db156fd03075f49876957dca4711306d39c9"},
-    {file = "numpy-1.25.2-cp310-cp310-win32.whl", hash = "sha256:7dc869c0c75988e1c693d0e2d5b26034644399dd929bc049db55395b1379e044"},
-    {file = "numpy-1.25.2-cp310-cp310-win_amd64.whl", hash = "sha256:834b386f2b8210dca38c71a6e0f4fd6922f7d3fcff935dbe3a570945acb1b545"},
-    {file = "numpy-1.25.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c5462d19336db4560041517dbb7759c21d181a67cb01b36ca109b2ae37d32418"},
-    {file = "numpy-1.25.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c5652ea24d33585ea39eb6a6a15dac87a1206a692719ff45d53c5282e66d4a8f"},
-    {file = "numpy-1.25.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0d60fbae8e0019865fc4784745814cff1c421df5afee233db6d88ab4f14655a2"},
-    {file = "numpy-1.25.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:60e7f0f7f6d0eee8364b9a6304c2845b9c491ac706048c7e8cf47b83123b8dbf"},
-    {file = "numpy-1.25.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:bb33d5a1cf360304754913a350edda36d5b8c5331a8237268c48f91253c3a364"},
-    {file = "numpy-1.25.2-cp311-cp311-win32.whl", hash = "sha256:5883c06bb92f2e6c8181df7b39971a5fb436288db58b5a1c3967702d4278691d"},
-    {file = "numpy-1.25.2-cp311-cp311-win_amd64.whl", hash = "sha256:5c97325a0ba6f9d041feb9390924614b60b99209a71a69c876f71052521d42a4"},
-    {file = "numpy-1.25.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:b79e513d7aac42ae918db3ad1341a015488530d0bb2a6abcbdd10a3a829ccfd3"},
-    {file = "numpy-1.25.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:eb942bfb6f84df5ce05dbf4b46673ffed0d3da59f13635ea9b926af3deb76926"},
-    {file = "numpy-1.25.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e0746410e73384e70d286f93abf2520035250aad8c5714240b0492a7302fdca"},
-    {file = "numpy-1.25.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d7806500e4f5bdd04095e849265e55de20d8cc4b661b038957354327f6d9b295"},
-    {file = "numpy-1.25.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8b77775f4b7df768967a7c8b3567e309f617dd5e99aeb886fa14dc1a0791141f"},
-    {file = "numpy-1.25.2-cp39-cp39-win32.whl", hash = "sha256:2792d23d62ec51e50ce4d4b7d73de8f67a2fd3ea710dcbc8563a51a03fb07b01"},
-    {file = "numpy-1.25.2-cp39-cp39-win_amd64.whl", hash = "sha256:76b4115d42a7dfc5d485d358728cdd8719be33cc5ec6ec08632a5d6fca2ed380"},
-    {file = "numpy-1.25.2-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:1a1329e26f46230bf77b02cc19e900db9b52f398d6722ca853349a782d4cff55"},
-    {file = "numpy-1.25.2-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4c3abc71e8b6edba80a01a52e66d83c5d14433cbcd26a40c329ec7ed09f37901"},
-    {file = "numpy-1.25.2-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:1b9735c27cea5d995496f46a8b1cd7b408b3f34b6d50459d9ac8fe3a20cc17bf"},
-    {file = "numpy-1.25.2.tar.gz", hash = "sha256:fd608e19c8d7c55021dffd43bfe5492fab8cc105cc8986f813f8c3c048b38760"},
-]
-
 [[package]]
 name = "numpy"
 version = "1.26.1"
@@ -3820,13 +3786,13 @@ sympy = "*"
 
 [[package]]
 name = "openai"
-version = "0.27.10"
+version = "0.28.1"
 description = "Python client library for the OpenAI API"
 optional = false
 python-versions = ">=3.7.1"
 files = [
-    {file = "openai-0.27.10-py3-none-any.whl", hash = "sha256:beabd1757e3286fa166dde3b70ebb5ad8081af046876b47c14c41e203ed22a14"},
+    {file = "openai-0.28.1-py3-none-any.whl", hash = "sha256:d18690f9e3d31eedb66b57b88c2165d760b24ea0a01f150dd3f068155088ce68"},
-    {file = "openai-0.27.10.tar.gz", hash = "sha256:60e09edf7100080283688748c6803b7b3b52d5a55d21890f3815292a0552d83b"},
+    {file = "openai-0.28.1.tar.gz", hash = "sha256:4be1dad329a65b4ce1a660fe6d5431b438f429b5855c883435f0f7fcb6d2dcc8"},
 ]
 
 [package.dependencies]
@@ -5514,44 +5480,6 @@ docs = ["Pillow (>=7.1.2)", "matplotlib (>=3.1.3)", "memory-profiler (>=0.57.0)"
 examples = ["matplotlib (>=3.1.3)", "pandas (>=1.0.5)", "plotly (>=5.14.0)", "pooch (>=1.6.0)", "scikit-image (>=0.16.2)", "seaborn (>=0.9.0)"]
 tests = ["black (>=23.3.0)", "matplotlib (>=3.1.3)", "mypy (>=1.3)", "numpydoc (>=1.2.0)", "pandas (>=1.0.5)", "pooch (>=1.6.0)", "pyamg (>=4.0.0)", "pytest (>=7.1.2)", "pytest-cov (>=2.9.0)", "ruff (>=0.0.272)", "scikit-image (>=0.16.2)"]
 
-[[package]]
-name = "scipy"
-version = "1.9.3"
-description = "Fundamental algorithms for scientific computing in Python"
-optional = true
-python-versions = ">=3.8"
-files = [
-    {file = "scipy-1.9.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:1884b66a54887e21addf9c16fb588720a8309a57b2e258ae1c7986d4444d3bc0"},
-    {file = "scipy-1.9.3-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:83b89e9586c62e787f5012e8475fbb12185bafb996a03257e9675cd73d3736dd"},
-    {file = "scipy-1.9.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1a72d885fa44247f92743fc20732ae55564ff2a519e8302fb7e18717c5355a8b"},
-    {file = "scipy-1.9.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d01e1dd7b15bd2449c8bfc6b7cc67d630700ed655654f0dfcf121600bad205c9"},
-    {file = "scipy-1.9.3-cp310-cp310-win_amd64.whl", hash = "sha256:68239b6aa6f9c593da8be1509a05cb7f9efe98b80f43a5861cd24c7557e98523"},
-    {file = "scipy-1.9.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b41bc822679ad1c9a5f023bc93f6d0543129ca0f37c1ce294dd9d386f0a21096"},
-    {file = "scipy-1.9.3-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:90453d2b93ea82a9f434e4e1cba043e779ff67b92f7a0e85d05d286a3625df3c"},
-    {file = "scipy-1.9.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:83c06e62a390a9167da60bedd4575a14c1f58ca9dfde59830fc42e5197283dab"},
-    {file = "scipy-1.9.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:abaf921531b5aeaafced90157db505e10345e45038c39e5d9b6c7922d68085cb"},
-    {file = "scipy-1.9.3-cp311-cp311-win_amd64.whl", hash = "sha256:06d2e1b4c491dc7d8eacea139a1b0b295f74e1a1a0f704c375028f8320d16e31"},
-    {file = "scipy-1.9.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:5a04cd7d0d3eff6ea4719371cbc44df31411862b9646db617c99718ff68d4840"},
-    {file = "scipy-1.9.3-cp38-cp38-macosx_12_0_arm64.whl", hash = "sha256:545c83ffb518094d8c9d83cce216c0c32f8c04aaf28b92cc8283eda0685162d5"},
-    {file = "scipy-1.9.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0d54222d7a3ba6022fdf5773931b5d7c56efe41ede7f7128c7b1637700409108"},
-    {file = "scipy-1.9.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cff3a5295234037e39500d35316a4c5794739433528310e117b8a9a0c76d20fc"},
-    {file = "scipy-1.9.3-cp38-cp38-win_amd64.whl", hash = "sha256:2318bef588acc7a574f5bfdff9c172d0b1bf2c8143d9582e05f878e580a3781e"},
-    {file = "scipy-1.9.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:d644a64e174c16cb4b2e41dfea6af722053e83d066da7343f333a54dae9bc31c"},
-    {file = "scipy-1.9.3-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:da8245491d73ed0a994ed9c2e380fd058ce2fa8a18da204681f2fe1f57f98f95"},
-    {file = "scipy-1.9.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4db5b30849606a95dcf519763dd3ab6fe9bd91df49eba517359e450a7d80ce2e"},
-    {file = "scipy-1.9.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c68db6b290cbd4049012990d7fe71a2abd9ffbe82c0056ebe0f01df8be5436b0"},
-    {file = "scipy-1.9.3-cp39-cp39-win_amd64.whl", hash = "sha256:5b88e6d91ad9d59478fafe92a7c757d00c59e3bdc3331be8ada76a4f8d683f58"},
-    {file = "scipy-1.9.3.tar.gz", hash = "sha256:fbc5c05c85c1a02be77b1ff591087c83bc44579c6d2bd9fb798bb64ea5e1a027"},
-]
-
-[package.dependencies]
-numpy = ">=1.18.5,<1.26.0"
-
-[package.extras]
-dev = ["flake8", "mypy", "pycodestyle", "typing_extensions"]
-doc = ["matplotlib (>2)", "numpydoc", "pydata-sphinx-theme (==0.9.0)", "sphinx (!=4.1.0)", "sphinx-panels (>=0.5.2)", "sphinx-tabs"]
-test = ["asv", "gmpy2", "mpmath", "pytest", "pytest-cov", "pytest-xdist", "scikit-umfpack", "threadpoolctl"]
-
 [[package]]
 name = "scipy"
 version = "1.11.3"
@@ -7246,9 +7174,10 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p
 [extras]
 cohere = ["cohere"]
 community = ["llama-hub"]
-dataloaders = ["beautifulsoup4", "docx2txt", "duckduckgo-search", "jq", "pypdf", "pytube", "sentence-transformers", "unstructured"]
+dataloaders = ["beautifulsoup4", "docx2txt", "duckduckgo-search", "jq", "jq", "pypdf", "pytube", "sentence-transformers", "unstructured"]
 discord = ["discord"]
 elasticsearch = ["elasticsearch"]
+gmail = ["llama-hub", "requests"]
 huggingface-hub = ["huggingface_hub"]
 images = ["ftfy", "pillow", "regex", "torch", "torchvision"]
 llama2 = ["replicate"]
@@ -7267,4 +7196,4 @@ whatsapp = ["flask", "twilio"]
 [metadata]
 lock-version = "2.0"
 python-versions = ">=3.9,<3.13"
-content-hash = "4af6e1c42a582499a052face9ae1de4f152c6dc7e4137be36fa84f45ce7bf09b"
+content-hash = "4021d63d76a9128c8d7342bc482dfe90fa9878fde1359c2f0730fa9188e503af"

pyproject.toml

@@ -1,8 +1,11 @@
 [tool.poetry]
 name = "embedchain"
-version = "0.0.77"
+version = "0.0.82"
 description = "Data platform for LLMs - Load, index, retrieve and sync any unstructured data"
-authors = ["Taranjeet Singh, Deshraj Yadav"]
+authors = [
+    "Taranjeet Singh <taranjeet@embedchain.ai>",
+    "Deshraj Yadav <deshraj@embedchain.ai>",
+]
 license = "Apache License"
 readme = "README.md"
 exclude = [
@@ -89,7 +92,7 @@ python = ">=3.9,<3.13"
 python-dotenv = "^1.0.0"
 langchain = "^0.0.279"
 requests = "^2.31.0"
-openai = "^0.27.5"
+openai = ">=0.28.0"
 tiktoken = { version="^0.4.0", optional=true }
 chromadb ="^0.4.8"
 youtube-transcript-api = { version="^0.6.1", optional=true }
@@ -143,7 +146,7 @@ pytest-asyncio = "^0.21.1"
 [tool.poetry.extras]
 streamlit = ["streamlit"]
 community = ["llama-hub"]
-opensource = ["sentence-transformers", "torch", "gpt4all", "langchain"]
+opensource = ["sentence-transformers", "torch", "gpt4all"]
 elasticsearch = ["elasticsearch"]
 opensearch = ["opensearch-py"]
 poe = ["fastapi-poe"]
@@ -160,16 +163,26 @@ milvus = ["pymilvus"]
 dataloaders=[
     "youtube-transcripts-api",
     "beautifulsoup4",
+    "docx2txt",
+    "duckduckgo-search",
+    "jq",
     "pypdf",
     "pytube",
-    "duckduckgo-search",
-    "docx2txt",
-    "unstructured",
     "sentence-transformers",
+    "unstructured",
     "jq",
 ]
 vertexai = ["google-cloud-aiplatform"]
 llama2 = ["replicate"]
+gmail = [
+    "llama-hub",
+    "requests",
+    "google-api-python-client",
+    "google-auth",
+    "google-auth-oauthlib",
+    "google-auth-httplib2",
+    "google-api-core",
+]
 
 [tool.poetry.group.docs.dependencies]
 

tests/chunkers/test_chunkers.py

@@ -1,5 +1,6 @@
 from embedchain.chunkers.docs_site import DocsSiteChunker
 from embedchain.chunkers.docx_file import DocxFileChunker
+from embedchain.chunkers.gmail import GmailChunker
 from embedchain.chunkers.json import JSONChunker
 from embedchain.chunkers.mdx import MdxChunker
 from embedchain.chunkers.notion import NotionChunker
@@ -31,6 +32,7 @@ chunker_common_config = {
     YoutubeVideoChunker: {"chunk_size": 2000, "chunk_overlap": 0, "length_function": len},
     JSONChunker: {"chunk_size": 1000, "chunk_overlap": 0, "length_function": len},
     OpenAPIChunker: {"chunk_size": 1000, "chunk_overlap": 0, "length_function": len},
+    GmailChunker: {"chunk_size": 1000, "chunk_overlap": 0, "length_function": len},
 }
 
 

tests/llm/test_gpt4all.py

@@ -0,0 +1,60 @@
+import pytest
+from langchain.llms.gpt4all import GPT4All as LangchainGPT4All
+
+from embedchain.config import BaseLlmConfig
+from embedchain.llm.gpt4all import GPT4ALLLlm
+
+
+@pytest.fixture
+def config():
+    config = BaseLlmConfig(
+        temperature=0.7,
+        max_tokens=50,
+        top_p=0.8,
+        stream=False,
+        system_prompt="System prompt",
+        model="orca-mini-3b.ggmlv3.q4_0.bin",
+    )
+    yield config
+
+
+@pytest.fixture
+def gpt4all_with_config(config):
+    return GPT4ALLLlm(config=config)
+
+
+@pytest.fixture
+def gpt4all_without_config():
+    return GPT4ALLLlm()
+
+
+def test_gpt4all_init_with_config(config, gpt4all_with_config):
+    assert gpt4all_with_config.config.temperature == config.temperature
+    assert gpt4all_with_config.config.max_tokens == config.max_tokens
+    assert gpt4all_with_config.config.top_p == config.top_p
+    assert gpt4all_with_config.config.stream == config.stream
+    assert gpt4all_with_config.config.system_prompt == config.system_prompt
+    assert gpt4all_with_config.config.model == config.model
+
+    assert isinstance(gpt4all_with_config.instance, LangchainGPT4All)
+
+
+def test_gpt4all_init_without_config(gpt4all_without_config):
+    assert gpt4all_without_config.config.model == "orca-mini-3b.ggmlv3.q4_0.bin"
+    assert isinstance(gpt4all_without_config.instance, LangchainGPT4All)
+
+
+def test_get_llm_model_answer(mocker, gpt4all_with_config):
+    test_query = "Test query"
+    test_answer = "Test answer"
+
+    mocked_get_answer = mocker.patch("embedchain.llm.gpt4all.GPT4ALLLlm._get_answer", return_value=test_answer)
+    answer = gpt4all_with_config.get_llm_model_answer(test_query)
+
+    assert answer == test_answer
+    mocked_get_answer.assert_called_once_with(prompt=test_query, config=gpt4all_with_config.config)
+
+
+def test_gpt4all_model_switching(gpt4all_with_config):
+    with pytest.raises(RuntimeError, match="GPT4ALLLlm does not support switching models at runtime."):
+        gpt4all_with_config._get_answer("Test prompt", BaseLlmConfig(model="new_model"))

tests/loaders/test_gmail.py

@@ -0,0 +1,44 @@
+import pytest
+from llama_hub.readwise.base import Document
+
+from embedchain.loaders.gmail import GmailLoader
+
+
+@pytest.fixture
+def mock_quopri(mocker):
+    return mocker.patch("embedchain.loaders.gmail.quopri.decodestring", return_value=b"your_test_decoded_string")
+
+
+@pytest.fixture
+def mock_beautifulsoup(mocker):
+    return mocker.patch("embedchain.loaders.gmail.BeautifulSoup", return_value=mocker.MagicMock())
+
+
+@pytest.fixture
+def gmail_loader(mock_quopri, mock_beautifulsoup):
+    return GmailLoader()
+
+
+def test_load_data_file_not_found(gmail_loader, mocker):
+    with pytest.raises(FileNotFoundError):
+        with mocker.patch("os.path.isfile", return_value=False):
+            gmail_loader.load_data("your_query")
+
+
+@pytest.mark.skip(reason="TODO: Fix this test. Failing due to some googleapiclient import issue.")
+def test_load_data(gmail_loader, mocker):
+    mock_gmail_reader_instance = mocker.MagicMock()
+    text = "your_test_email_text"
+    metadata = {
+        "id": "your_test_id",
+        "snippet": "your_test_snippet",
+    }
+    mock_gmail_reader_instance.load_data.return_value = [Document(text=text, extra_info=metadata)]
+
+    with mocker.patch("os.path.isfile", return_value=True):
+        response_data = gmail_loader.load_data("your_query")
+
+    assert "doc_id" in response_data
+    assert "data" in response_data
+    assert isinstance(response_data["doc_id"], str)
+    assert isinstance(response_data["data"], list)