feat: Add embedding manager (#570)

2023-09-12 12:13:53 +05:30
parent ba208f5b48
commit 2bd6881361
16 changed files with 311 additions and 73 deletions
@@ -1,3 +1,4 @@
+import hashlib
 import logging
 import os

@@ -34,10 +35,13 @@ class NotionLoader(BaseLoader):

        # Clean text
        text = clean_string(raw_text)
-
-        return [
+        doc_id = hashlib.sha256((text + source).encode()).hexdigest()
+        return {
+            "doc_id": doc_id,
+            "data": [
            {
                "content": text,
                "meta_data": {"url": f"notion-{formatted_id}"},
            }
-        ]
+        ],
+        }