feat: bump version to 0.0.60 (#704)

Co-authored-by: Taranjeet Singh <reachtotj@gmail.com>

embedchain/__init__.py

@@ -3,9 +3,9 @@ import importlib.metadata
 __version__ = importlib.metadata.version(__package__ or __name__)
 
 from embedchain.apps.App import App  # noqa: F401
-from embedchain.apps.CustomApp import CustomApp  # noqa: F401
+from embedchain.apps.custom_app import CustomApp  # noqa: F401
 from embedchain.apps.Llama2App import Llama2App  # noqa: F401
-from embedchain.apps.OpenSourceApp import OpenSourceApp  # noqa: F401
+from embedchain.apps.open_source_app import OpenSourceApp  # noqa: F401
 from embedchain.apps.PersonApp import (PersonApp,  # noqa: F401
                                        PersonOpenSourceApp)
 from embedchain.vectordb.chroma import ChromaDB  # noqa: F401

embedchain/apps/Llama2App.py

@@ -1,6 +1,6 @@
 from typing import Optional
 
-from embedchain.apps.CustomApp import CustomApp
+from embedchain.apps.custom_app import CustomApp
 from embedchain.config import CustomAppConfig
 from embedchain.embedder.openai import OpenAiEmbedder
 from embedchain.helper.json_serializable import register_deserializable

embedchain/apps/PersonApp.py

@@ -1,9 +1,9 @@
 from string import Template
 
 from embedchain.apps.App import App
-from embedchain.apps.OpenSourceApp import OpenSourceApp
+from embedchain.apps.open_source_app import OpenSourceApp
 from embedchain.config import BaseLlmConfig
-from embedchain.config.apps.BaseAppConfig import BaseAppConfig
+from embedchain.config.apps.base_app_config import BaseAppConfig
 from embedchain.config.llm.base_llm_config import (DEFAULT_PROMPT,
                                                    DEFAULT_PROMPT_WITH_HISTORY)
 from embedchain.helper.json_serializable import register_deserializable

embedchain/config/__init__.py

@@ -1,9 +1,9 @@
 # flake8: noqa: F401
 
 from .AddConfig import AddConfig, ChunkerConfig
-from .apps.AppConfig import AppConfig
-from .apps.CustomAppConfig import CustomAppConfig
-from .apps.OpenSourceAppConfig import OpenSourceAppConfig
+from .apps.app_config import AppConfig
+from .apps.custom_app_config import CustomAppConfig
+from .apps.open_source_app_config import OpenSourceAppConfig
 from .BaseConfig import BaseConfig
 from .embedder.BaseEmbedderConfig import BaseEmbedderConfig
 from .embedder.BaseEmbedderConfig import BaseEmbedderConfig as EmbedderConfig

embedchain/config/apps/app_config.py

@@ -2,7 +2,7 @@ from typing import Optional
 
 from embedchain.helper.json_serializable import register_deserializable
 
-from .BaseAppConfig import BaseAppConfig
+from .base_app_config import BaseAppConfig
 
 
 @register_deserializable

embedchain/config/apps/custom_app_config.py

@@ -5,7 +5,7 @@ from dotenv import load_dotenv
 from embedchain.helper.json_serializable import register_deserializable
 from embedchain.vectordb.base import BaseVectorDB
 
-from .BaseAppConfig import BaseAppConfig
+from .base_app_config import BaseAppConfig
 
 load_dotenv()
 

embedchain/config/apps/open_source_app_config.py

@@ -2,7 +2,7 @@ from typing import Optional
 
 from embedchain.helper.json_serializable import register_deserializable
 
-from .BaseAppConfig import BaseAppConfig
+from .base_app_config import BaseAppConfig
 
 
 @register_deserializable

embedchain/embedchain.py

@@ -6,7 +6,7 @@ import os
 import threading
 import uuid
 from pathlib import Path
-from typing import Any, Dict, List, Optional, Tuple
+from typing import Any, Dict, List, Optional
 
 import requests
 from dotenv import load_dotenv
@@ -15,7 +15,7 @@ from tenacity import retry, stop_after_attempt, wait_fixed
 
 from embedchain.chunkers.base_chunker import BaseChunker
 from embedchain.config import AddConfig, BaseLlmConfig
-from embedchain.config.apps.BaseAppConfig import BaseAppConfig
+from embedchain.config.apps.base_app_config import BaseAppConfig
 from embedchain.data_formatter import DataFormatter
 from embedchain.embedder.base import BaseEmbedder
 from embedchain.helper.json_serializable import JSONSerializable
@@ -200,7 +200,7 @@ class EmbedChain(JSONSerializable):
 
         data_formatter = DataFormatter(data_type, config)
         self.user_asks.append([source, data_type.value, metadata])
-        documents, metadatas, _ids, new_chunks = self.load_and_embed_v2(
+        documents, metadatas, _ids, new_chunks = self.load_and_embed(
             data_formatter.loader, data_formatter.chunker, source, metadata, source_id, dry_run
         )
         if data_type in {DataType.DOCS_SITE}:
@@ -255,92 +255,6 @@ class EmbedChain(JSONSerializable):
         )
         return self.add(source=source, data_type=data_type, metadata=metadata, config=config)
 
-    def load_and_embed(
-        self,
-        loader: BaseLoader,
-        chunker: BaseChunker,
-        src: Any,
-        metadata: Optional[Dict[str, Any]] = None,
-        source_id: Optional[str] = None,
-        dry_run=False,
-    ) -> Tuple[List[str], Dict[str, Any], List[str], int]:
-        """The loader to use to load the data.
-
-        :param loader: The loader to use to load the data.
-        :type loader: BaseLoader
-        :param chunker: The chunker to use to chunk the data.
-        :type chunker: BaseChunker
-        :param src: The data to be handled by the loader.
-        Can be a URL for remote sources or local content for local loaders.
-        :type src: Any
-        :param metadata: Metadata associated with the data source., defaults to None
-        :type metadata: Dict[str, Any], optional
-        :param source_id: Hexadecimal hash of the source., defaults to None
-        :type source_id: str, optional
-        :param dry_run: Optional. A dry run returns chunks and doesn't update DB.
-        :type dry_run: bool, defaults to False
-        :return: (List) documents (embedded text), (List) metadata, (list) ids, (int) number of chunks
-        :rtype: Tuple[List[str], Dict[str, Any], List[str], int]
-        """
-        embeddings_data = chunker.create_chunks(loader, src)
-
-        # spread chunking results
-        documents = embeddings_data["documents"]
-        metadatas = embeddings_data["metadatas"]
-        ids = embeddings_data["ids"]
-
-        # get existing ids, and discard doc if any common id exist.
-        where = {"app_id": self.config.id} if self.config.id is not None else {}
-        # where={"url": src}
-        db_result = self.db.get(
-            ids=ids,
-            where=where,  # optional filter
-        )
-        existing_ids = set(db_result["ids"])
-
-        if len(existing_ids):
-            data_dict = {id: (doc, meta) for id, doc, meta in zip(ids, documents, metadatas)}
-            data_dict = {id: value for id, value in data_dict.items() if id not in existing_ids}
-
-            if not data_dict:
-                src_copy = src
-                if len(src_copy) > 50:
-                    src_copy = src[:50] + "..."
-                print(f"All data from {src_copy} already exists in the database.")
-                # Make sure to return a matching return type
-                return [], [], [], 0
-
-            ids = list(data_dict.keys())
-            documents, metadatas = zip(*data_dict.values())
-
-        # Loop though all metadatas and add extras.
-        new_metadatas = []
-        for m in metadatas:
-            # Add app id in metadatas so that they can be queried on later
-            if self.config.id:
-                m["app_id"] = self.config.id
-
-            # Add hashed source
-            m["hash"] = source_id
-
-            # Note: Metadata is the function argument
-            if metadata:
-                # Spread whatever is in metadata into the new object.
-                m.update(metadata)
-
-            new_metadatas.append(m)
-        metadatas = new_metadatas
-
-        if dry_run:
-            return list(documents), metadatas, ids, 0
-
-        # Count before, to calculate a delta in the end.
-        chunks_before_addition = self.db.count()
-
-        self.db.add(documents=documents, metadatas=metadatas, ids=ids)
-        count_new_chunks = self.db.count() - chunks_before_addition
-        print((f"Successfully saved {src} ({chunker.data_type}). New chunks count: {count_new_chunks}"))
-        return list(documents), metadatas, ids, count_new_chunks
 
     def _get_existing_doc_id(self, chunker: BaseChunker, src: Any):
         """
@@ -392,7 +306,7 @@ class EmbedChain(JSONSerializable):
                 "When it should be  DirectDataType, IndirectDataType or SpecialDataType."
             )
 
-    def load_and_embed_v2(
+    def load_and_embed(
         self,
         loader: BaseLoader,
         chunker: BaseChunker,

embedchain/models/__init__.py

@@ -1,4 +1,4 @@
 from .EmbeddingFunctions import EmbeddingFunctions  # noqa: F401
 from .Providers import Providers  # noqa: F401
-from .VectorDatabases import VectorDatabases  # noqa: F401
+from .vector_databases import VectorDatabases  # noqa: F401
 from .VectorDimensions import VectorDimensions  # noqa: F401

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "embedchain"
-version = "0.0.59"
+version = "0.0.60"
 description = "embedchain is a framework to easily create LLM powered bots over any dataset"
 authors = ["Taranjeet Singh"]
 license = "Apache License"
@@ -104,6 +104,7 @@ twilio = { version = "^8.5.0", optional = true }
 fastapi-poe = { version = "0.0.16", optional = true }
 discord = { version = "^2.3.2", optional = true }
 slack-sdk = { version = "3.21.3", optional = true }
+docx2txt = "^0.8"