[version] bump package version to v0.0.87 (#883)

Co-authored-by: Deven Patel <deven298@yahoo.com>

docs/mint.json

@@ -125,7 +125,7 @@
   "isWhiteLabeled": true,
   "analytics": {
     "posthog": {
-      "apiKey": "phc_MSQ1GVfzkm7dRpktoKUGzWPlpkYhmVpcUtmLk1RwkQs",
+      "apiKey": "phc_PHQDA5KwztijnSojsxJ2c1DuJd52QCzJzT2xnSGvjN2",
       "apiHost": "https://app.embedchain.ai/ingest"
     }
   },

embedchain/data_formatter/data_formatter.py

@@ -1,4 +1,5 @@
 from importlib import import_module
+
 from embedchain.chunkers.base_chunker import BaseChunker
 from embedchain.config import AddConfig
 from embedchain.config.add_config import ChunkerConfig, LoaderConfig

embedchain/loaders/json.py

@@ -1,24 +1,56 @@
 import hashlib
+import json
+import os
+import re
 
-from langchain.document_loaders.json_loader import \
-    JSONLoader as LangchainJSONLoader
+import requests
 
 from embedchain.loaders.base_loader import BaseLoader
 
-langchain_json_jq_schema = 'to_entries | map("\(.key): \(.value|tostring)") | .[]'
+VALID_URL_PATTERN = "^https:\/\/[0-9A-z.]+.[0-9A-z.]+.[a-z]+\/.*\.json$"
 
 
 class JSONLoader(BaseLoader):
     @staticmethod
     def load_data(content):
         """Load a json file. Each data point is a key value pair."""
+        try:
+            from llama_hub.jsondata.base import \
+                JSONDataReader as LLHBUBJSONLoader
+        except ImportError:
+            raise Exception(
+                f"Couldn't import the required packages to load {content}, \
+                Do `pip install --upgrade 'embedchain[json]`"
+            )
+
+        loader = LLHBUBJSONLoader()
+
+        if not isinstance(content, str):
+            print(f"Invaid content input. Provide the correct path to the json file saved locally in {content}")
+
         data = []
         data_content = []
-        loader = LangchainJSONLoader(content, text_content=False, jq_schema=langchain_json_jq_schema)
-        docs = loader.load()
+
+        # Load json data from various sources. TODO: add support for dictionary
+        if os.path.isfile(content):
+            with open(content, "r") as json_file:
+                json_data = json.load(json_file)
+        elif re.match(VALID_URL_PATTERN, content):
+            response = requests.get(content)
+            if response.status_code == 200:
+                json_data = response.json()
+            else:
+                raise ValueError(
+                    f"Loading data from the given url: {content} failed. \
+                    Make sure the url is working."
+                )
+        else:
+            raise ValueError(f"Invalid content to load json data from: {content}")
+
+        docs = loader.load_data(json_data)
         for doc in docs:
-            meta_data = doc.metadata
-            data.append({"content": doc.page_content, "meta_data": {"url": content, "row": meta_data["seq_num"]}})
-            data_content.append(doc.page_content)
+            doc_content = doc.text
+            data.append({"content": doc_content, "meta_data": {"url": content}})
+            data_content.append(doc_content)
         doc_id = hashlib.sha256((content + ", ".join(data_content)).encode()).hexdigest()
         return {"doc_id": doc_id, "data": data}

embedchain/telemetry/posthog.py

@@ -17,7 +17,7 @@ logger = logging.getLogger(__name__)
 
 class AnonymousTelemetry:
     def __init__(self, host="https://app.posthog.com", enabled=True):
-        self.project_api_key = "phc_XnMmNHzwxE7PVHX4mD2r8K6nfxVM48a2sq2U3N1p2lO"
+        self.project_api_key = "phc_PHQDA5KwztijnSojsxJ2c1DuJd52QCzJzT2xnSGvjN2"
         self.host = host
         self.posthog = Posthog(project_api_key=self.project_api_key, host=self.host)
         self.user_id = self.get_user_id()

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "embedchain"
-version = "0.0.86"
+version = "0.0.87"
 description = "Data platform for LLMs - Load, index, retrieve and sync any unstructured data"
 authors = [
     "Taranjeet Singh <taranjeet@embedchain.ai>",

tests/loaders/test_json.py

@@ -1,32 +1,91 @@
 import hashlib
-from unittest.mock import patch
 
-from langchain.docstore.document import Document
-from langchain.document_loaders.json_loader import \
-    JSONLoader as LangchainJSONLoader
+import pytest
+from llama_index.readers.schema.base import Document
 
 from embedchain.loaders.json import JSONLoader
 
 
-def test_load_data():
-    mock_document = [
-        Document(page_content="content1", metadata={"seq_num": 1}),
-        Document(page_content="content2", metadata={"seq_num": 2}),
+def test_load_data(mocker):
+    content = "temp.json"
+
+    mock_document = {
+        "doc_id": hashlib.sha256((content + ", ".join(["content1", "content2"])).encode()).hexdigest(),
+        "data": [
+            {"content": "content1", "meta_data": {"url": content}},
+            {"content": "content2", "meta_data": {"url": content}},
+        ],
+    }
+
+    mocker.patch("embedchain.loaders.json.JSONLoader.load_data", return_value=mock_document)
+
+    json_loader = JSONLoader()
+
+    result = json_loader.load_data(content)
+
+    assert "doc_id" in result
+    assert "data" in result
+
+    expected_data = [
+        {"content": "content1", "meta_data": {"url": content}},
+        {"content": "content2", "meta_data": {"url": content}},
     ]
-    with patch.object(LangchainJSONLoader, "load", return_value=mock_document):
-        content = "temp.json"
 
-        result = JSONLoader.load_data(content)
+    assert result["data"] == expected_data
 
-        assert "doc_id" in result
-        assert "data" in result
+    expected_doc_id = hashlib.sha256((content + ", ".join(["content1", "content2"])).encode()).hexdigest()
+    assert result["doc_id"] == expected_doc_id
 
-        expected_data = [
-            {"content": "content1", "meta_data": {"url": content, "row": 1}},
-            {"content": "content2", "meta_data": {"url": content, "row": 2}},
-        ]
 
-        assert result["data"] == expected_data
+def test_load_data_url(mocker):
+    content = "https://example.com/posts.json"
 
-        expected_doc_id = hashlib.sha256((content + ", ".join(["content1", "content2"])).encode()).hexdigest()
-        assert result["doc_id"] == expected_doc_id
+    mocker.patch("os.path.isfile", return_value=False)  # Mocking os.path.isfile to simulate a URL case
+    mocker.patch(
+        "llama_hub.jsondata.base.JSONDataReader.load_data",
+        return_value=[Document(text="content1"), Document(text="content2")],
+    )
+
+    mock_response = mocker.Mock()
+    mock_response.status_code = 200
+    mock_response.json.return_value = {"document1": "content1", "document2": "content2"}
+
+    mocker.patch("requests.get", return_value=mock_response)
+
+    result = JSONLoader.load_data(content)
+
+    assert "doc_id" in result
+    assert "data" in result
+
+    expected_data = [
+        {"content": "content1", "meta_data": {"url": content}},
+        {"content": "content2", "meta_data": {"url": content}},
+    ]
+
+    assert result["data"] == expected_data
+
+    expected_doc_id = hashlib.sha256((content + ", ".join(["content1", "content2"])).encode()).hexdigest()
+    assert result["doc_id"] == expected_doc_id
+
+
+def test_load_data_invalid_content(mocker):
+    mocker.patch("os.path.isfile", return_value=False)
+    mocker.patch("requests.get")
+
+    content = "123"
+
+    with pytest.raises(ValueError, match="Invalid content to load json data from"):
+        JSONLoader.load_data(content)
+
+
+def test_load_data_invalid_url(mocker):
+    mocker.patch("os.path.isfile", return_value=False)
+
+    mock_response = mocker.Mock()
+    mock_response.status_code = 404
+    mocker.patch("requests.get", return_value=mock_response)
+
+    content = "http://invalid-url.com/"
+
+    with pytest.raises(ValueError, match=f"Invalid content to load json data from: {content}"):
+        JSONLoader.load_data(content)

tests/telemetry/test_posthog.py

@@ -1,5 +1,6 @@
-import os
 import logging
+import os
+
 from embedchain.telemetry.posthog import AnonymousTelemetry