Rename embedchain to mem0 and open sourcing code for long term memory (#1474)
Co-authored-by: Deshraj Yadav <deshrajdry@gmail.com>
This commit is contained in:
Taranjeet Singh
co-authored by
Deshraj Yadav
Deshraj Yadav
parent
83e8c97295
commit
f842a92e25
@@ -0,0 +1,5 @@
|
||||
from embedchain.bots.poe import PoeBot # noqa: F401
|
||||
from embedchain.bots.whatsapp import WhatsAppBot # noqa: F401
|
||||
|
||||
# TODO: fix discord import
|
||||
# from embedchain.bots.discord import DiscordBot
|
||||
@@ -0,0 +1,46 @@
|
||||
from typing import Any
|
||||
|
||||
from embedchain import App
|
||||
from embedchain.config import AddConfig, AppConfig, BaseLlmConfig
|
||||
from embedchain.embedder.openai import OpenAIEmbedder
|
||||
from embedchain.helpers.json_serializable import (JSONSerializable,
|
||||
register_deserializable)
|
||||
from embedchain.llm.openai import OpenAILlm
|
||||
from embedchain.vectordb.chroma import ChromaDB
|
||||
|
||||
|
||||
@register_deserializable
|
||||
class BaseBot(JSONSerializable):
|
||||
def __init__(self):
|
||||
self.app = App(config=AppConfig(), llm=OpenAILlm(), db=ChromaDB(), embedding_model=OpenAIEmbedder())
|
||||
|
||||
def add(self, data: Any, config: AddConfig = None):
|
||||
"""
|
||||
Add data to the bot (to the vector database).
|
||||
Auto-dectects type only, so some data types might not be usable.
|
||||
|
||||
:param data: data to embed
|
||||
:type data: Any
|
||||
:param config: configuration class instance, defaults to None
|
||||
:type config: AddConfig, optional
|
||||
"""
|
||||
config = config if config else AddConfig()
|
||||
self.app.add(data, config=config)
|
||||
|
||||
def query(self, query: str, config: BaseLlmConfig = None) -> str:
|
||||
"""
|
||||
Query the bot
|
||||
|
||||
:param query: the user query
|
||||
:type query: str
|
||||
:param config: configuration class instance, defaults to None
|
||||
:type config: BaseLlmConfig, optional
|
||||
:return: Answer
|
||||
:rtype: str
|
||||
"""
|
||||
config = config
|
||||
return self.app.query(query, config=config)
|
||||
|
||||
def start(self):
|
||||
"""Start the bot's functionality."""
|
||||
raise NotImplementedError("Subclasses must implement the start method.")
|
||||
@@ -0,0 +1,128 @@
|
||||
import argparse
|
||||
import logging
|
||||
import os
|
||||
|
||||
from embedchain.helpers.json_serializable import register_deserializable
|
||||
|
||||
from .base import BaseBot
|
||||
|
||||
try:
|
||||
import discord
|
||||
from discord import app_commands
|
||||
from discord.ext import commands
|
||||
except ModuleNotFoundError:
|
||||
raise ModuleNotFoundError(
|
||||
"The required dependencies for Discord are not installed." "Please install with `pip install discord==2.3.2`"
|
||||
) from None
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
intents = discord.Intents.default()
|
||||
intents.message_content = True
|
||||
client = discord.Client(intents=intents)
|
||||
tree = app_commands.CommandTree(client)
|
||||
|
||||
# Invite link example
|
||||
# https://discord.com/api/oauth2/authorize?client_id={DISCORD_CLIENT_ID}&permissions=2048&scope=bot
|
||||
|
||||
|
||||
@register_deserializable
|
||||
class DiscordBot(BaseBot):
|
||||
def __init__(self, *args, **kwargs):
|
||||
BaseBot.__init__(self, *args, **kwargs)
|
||||
|
||||
def add_data(self, message):
|
||||
data = message.split(" ")[-1]
|
||||
try:
|
||||
self.add(data)
|
||||
response = f"Added data from: {data}"
|
||||
except Exception:
|
||||
logger.exception(f"Failed to add data {data}.")
|
||||
response = "Some error occurred while adding data."
|
||||
return response
|
||||
|
||||
def ask_bot(self, message):
|
||||
try:
|
||||
response = self.query(message)
|
||||
except Exception:
|
||||
logger.exception(f"Failed to query {message}.")
|
||||
response = "An error occurred. Please try again!"
|
||||
return response
|
||||
|
||||
def start(self):
|
||||
client.run(os.environ["DISCORD_BOT_TOKEN"])
|
||||
|
||||
|
||||
# @tree decorator cannot be used in a class. A global discord_bot is used as a workaround.
|
||||
|
||||
|
||||
@tree.command(name="question", description="ask embedchain")
|
||||
async def query_command(interaction: discord.Interaction, question: str):
|
||||
await interaction.response.defer()
|
||||
member = client.guilds[0].get_member(client.user.id)
|
||||
logger.info(f"User: {member}, Query: {question}")
|
||||
try:
|
||||
answer = discord_bot.ask_bot(question)
|
||||
if args.include_question:
|
||||
response = f"> {question}\n\n{answer}"
|
||||
else:
|
||||
response = answer
|
||||
await interaction.followup.send(response)
|
||||
except Exception as e:
|
||||
await interaction.followup.send("An error occurred. Please try again!")
|
||||
logger.error("Error occurred during 'query' command:", e)
|
||||
|
||||
|
||||
@tree.command(name="add", description="add new content to the embedchain database")
|
||||
async def add_command(interaction: discord.Interaction, url_or_text: str):
|
||||
await interaction.response.defer()
|
||||
member = client.guilds[0].get_member(client.user.id)
|
||||
logger.info(f"User: {member}, Add: {url_or_text}")
|
||||
try:
|
||||
response = discord_bot.add_data(url_or_text)
|
||||
await interaction.followup.send(response)
|
||||
except Exception as e:
|
||||
await interaction.followup.send("An error occurred. Please try again!")
|
||||
logger.error("Error occurred during 'add' command:", e)
|
||||
|
||||
|
||||
@tree.command(name="ping", description="Simple ping pong command")
|
||||
async def ping(interaction: discord.Interaction):
|
||||
await interaction.response.send_message("Pong", ephemeral=True)
|
||||
|
||||
|
||||
@tree.error
|
||||
async def on_app_command_error(interaction: discord.Interaction, error: discord.app_commands.AppCommandError) -> None:
|
||||
if isinstance(error, commands.CommandNotFound):
|
||||
await interaction.followup.send("Invalid command. Please refer to the documentation for correct syntax.")
|
||||
else:
|
||||
logger.error("Error occurred during command execution:", error)
|
||||
|
||||
|
||||
@client.event
|
||||
async def on_ready():
|
||||
# TODO: Sync in admin command, to not hit rate limits.
|
||||
# This might be overkill for most users, and it would require to set a guild or user id, where sync is allowed.
|
||||
await tree.sync()
|
||||
logger.debug("Command tree synced")
|
||||
logger.info(f"Logged in as {client.user.name}")
|
||||
|
||||
|
||||
def start_command():
|
||||
parser = argparse.ArgumentParser(description="EmbedChain DiscordBot command line interface")
|
||||
parser.add_argument(
|
||||
"--include-question",
|
||||
help="include question in query reply, otherwise it is hidden behind the slash command.",
|
||||
action="store_true",
|
||||
)
|
||||
global args
|
||||
args = parser.parse_args()
|
||||
|
||||
global discord_bot
|
||||
discord_bot = DiscordBot()
|
||||
discord_bot.start()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
start_command()
|
||||
@@ -0,0 +1,87 @@
|
||||
import argparse
|
||||
import logging
|
||||
import os
|
||||
from typing import Optional
|
||||
|
||||
from embedchain.helpers.json_serializable import register_deserializable
|
||||
|
||||
from .base import BaseBot
|
||||
|
||||
try:
|
||||
from fastapi_poe import PoeBot, run
|
||||
except ModuleNotFoundError:
|
||||
raise ModuleNotFoundError(
|
||||
"The required dependencies for Poe are not installed." "Please install with `pip install fastapi-poe==0.0.16`"
|
||||
) from None
|
||||
|
||||
|
||||
def start_command():
|
||||
parser = argparse.ArgumentParser(description="EmbedChain PoeBot command line interface")
|
||||
# parser.add_argument("--host", default="0.0.0.0", help="Host IP to bind")
|
||||
parser.add_argument("--port", default=8080, type=int, help="Port to bind")
|
||||
parser.add_argument("--api-key", type=str, help="Poe API key")
|
||||
# parser.add_argument(
|
||||
# "--history-length",
|
||||
# default=5,
|
||||
# type=int,
|
||||
# help="Set the max size of the chat history. Multiplies cost, but improves conversation awareness.",
|
||||
# )
|
||||
args = parser.parse_args()
|
||||
|
||||
# FIXME: Arguments are automatically loaded by Poebot's ArgumentParser which causes it to fail.
|
||||
# the port argument here is also just for show, it actually works because poe has the same argument.
|
||||
|
||||
run(PoeBot(), api_key=args.api_key or os.environ.get("POE_API_KEY"))
|
||||
|
||||
|
||||
@register_deserializable
|
||||
class PoeBot(BaseBot, PoeBot):
|
||||
def __init__(self):
|
||||
self.history_length = 5
|
||||
super().__init__()
|
||||
|
||||
async def get_response(self, query):
|
||||
last_message = query.query[-1].content
|
||||
try:
|
||||
history = (
|
||||
[f"{m.role}: {m.content}" for m in query.query[-(self.history_length + 1) : -1]]
|
||||
if len(query.query) > 0
|
||||
else None
|
||||
)
|
||||
except Exception as e:
|
||||
logging.error(f"Error when processing the chat history. Message is being sent without history. Error: {e}")
|
||||
answer = self.handle_message(last_message, history)
|
||||
yield self.text_event(answer)
|
||||
|
||||
def handle_message(self, message, history: Optional[list[str]] = None):
|
||||
if message.startswith("/add "):
|
||||
response = self.add_data(message)
|
||||
else:
|
||||
response = self.ask_bot(message, history)
|
||||
return response
|
||||
|
||||
# def add_data(self, message):
|
||||
# data = message.split(" ")[-1]
|
||||
# try:
|
||||
# self.add(data)
|
||||
# response = f"Added data from: {data}"
|
||||
# except Exception:
|
||||
# logging.exception(f"Failed to add data {data}.")
|
||||
# response = "Some error occurred while adding data."
|
||||
# return response
|
||||
|
||||
def ask_bot(self, message, history: list[str]):
|
||||
try:
|
||||
self.app.llm.set_history(history=history)
|
||||
response = self.query(message)
|
||||
except Exception:
|
||||
logging.exception(f"Failed to query {message}.")
|
||||
response = "An error occurred. Please try again!"
|
||||
return response
|
||||
|
||||
def start(self):
|
||||
start_command()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
start_command()
|
||||
@@ -0,0 +1,101 @@
|
||||
import argparse
|
||||
import logging
|
||||
import os
|
||||
import signal
|
||||
import sys
|
||||
|
||||
from embedchain import App
|
||||
from embedchain.helpers.json_serializable import register_deserializable
|
||||
|
||||
from .base import BaseBot
|
||||
|
||||
try:
|
||||
from flask import Flask, request
|
||||
from slack_sdk import WebClient
|
||||
except ModuleNotFoundError:
|
||||
raise ModuleNotFoundError(
|
||||
"The required dependencies for Slack are not installed."
|
||||
"Please install with `pip install slack-sdk==3.21.3 flask==2.3.3`"
|
||||
) from None
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
SLACK_BOT_TOKEN = os.environ.get("SLACK_BOT_TOKEN")
|
||||
|
||||
|
||||
@register_deserializable
|
||||
class SlackBot(BaseBot):
|
||||
def __init__(self):
|
||||
self.client = WebClient(token=SLACK_BOT_TOKEN)
|
||||
self.chat_bot = App()
|
||||
self.recent_message = {"ts": 0, "channel": ""}
|
||||
super().__init__()
|
||||
|
||||
def handle_message(self, event_data):
|
||||
message = event_data.get("event")
|
||||
if message and "text" in message and message.get("subtype") != "bot_message":
|
||||
text: str = message["text"]
|
||||
if float(message.get("ts")) > float(self.recent_message["ts"]):
|
||||
self.recent_message["ts"] = message["ts"]
|
||||
self.recent_message["channel"] = message["channel"]
|
||||
if text.startswith("query"):
|
||||
_, question = text.split(" ", 1)
|
||||
try:
|
||||
response = self.chat_bot.chat(question)
|
||||
self.send_slack_message(message["channel"], response)
|
||||
logger.info("Query answered successfully!")
|
||||
except Exception as e:
|
||||
self.send_slack_message(message["channel"], "An error occurred. Please try again!")
|
||||
logger.error("Error occurred during 'query' command:", e)
|
||||
elif text.startswith("add"):
|
||||
_, data_type, url_or_text = text.split(" ", 2)
|
||||
if url_or_text.startswith("<") and url_or_text.endswith(">"):
|
||||
url_or_text = url_or_text[1:-1]
|
||||
try:
|
||||
self.chat_bot.add(url_or_text, data_type)
|
||||
self.send_slack_message(message["channel"], f"Added {data_type} : {url_or_text}")
|
||||
except ValueError as e:
|
||||
self.send_slack_message(message["channel"], f"Error: {str(e)}")
|
||||
logger.error("Error occurred during 'add' command:", e)
|
||||
except Exception as e:
|
||||
self.send_slack_message(message["channel"], f"Failed to add {data_type} : {url_or_text}")
|
||||
logger.error("Error occurred during 'add' command:", e)
|
||||
|
||||
def send_slack_message(self, channel, message):
|
||||
response = self.client.chat_postMessage(channel=channel, text=message)
|
||||
return response
|
||||
|
||||
def start(self, host="0.0.0.0", port=5000, debug=True):
|
||||
app = Flask(__name__)
|
||||
|
||||
def signal_handler(sig, frame):
|
||||
logger.info("\nGracefully shutting down the SlackBot...")
|
||||
sys.exit(0)
|
||||
|
||||
signal.signal(signal.SIGINT, signal_handler)
|
||||
|
||||
@app.route("/", methods=["POST"])
|
||||
def chat():
|
||||
# Check if the request is a verification request
|
||||
if request.json.get("challenge"):
|
||||
return str(request.json.get("challenge"))
|
||||
|
||||
response = self.handle_message(request.json)
|
||||
return str(response)
|
||||
|
||||
app.run(host=host, port=port, debug=debug)
|
||||
|
||||
|
||||
def start_command():
|
||||
parser = argparse.ArgumentParser(description="EmbedChain SlackBot command line interface")
|
||||
parser.add_argument("--host", default="0.0.0.0", help="Host IP to bind")
|
||||
parser.add_argument("--port", default=5000, type=int, help="Port to bind")
|
||||
args = parser.parse_args()
|
||||
|
||||
slack_bot = SlackBot()
|
||||
slack_bot.start(host=args.host, port=args.port)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
start_command()
|
||||
@@ -0,0 +1,83 @@
|
||||
import argparse
|
||||
import importlib
|
||||
import logging
|
||||
import signal
|
||||
import sys
|
||||
|
||||
from embedchain.helpers.json_serializable import register_deserializable
|
||||
|
||||
from .base import BaseBot
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@register_deserializable
|
||||
class WhatsAppBot(BaseBot):
|
||||
def __init__(self):
|
||||
try:
|
||||
self.flask = importlib.import_module("flask")
|
||||
self.twilio = importlib.import_module("twilio")
|
||||
except ModuleNotFoundError:
|
||||
raise ModuleNotFoundError(
|
||||
"The required dependencies for WhatsApp are not installed. "
|
||||
"Please install with `pip install twilio==8.5.0 flask==2.3.3`"
|
||||
) from None
|
||||
super().__init__()
|
||||
|
||||
def handle_message(self, message):
|
||||
if message.startswith("add "):
|
||||
response = self.add_data(message)
|
||||
else:
|
||||
response = self.ask_bot(message)
|
||||
return response
|
||||
|
||||
def add_data(self, message):
|
||||
data = message.split(" ")[-1]
|
||||
try:
|
||||
self.add(data)
|
||||
response = f"Added data from: {data}"
|
||||
except Exception:
|
||||
logger.exception(f"Failed to add data {data}.")
|
||||
response = "Some error occurred while adding data."
|
||||
return response
|
||||
|
||||
def ask_bot(self, message):
|
||||
try:
|
||||
response = self.query(message)
|
||||
except Exception:
|
||||
logger.exception(f"Failed to query {message}.")
|
||||
response = "An error occurred. Please try again!"
|
||||
return response
|
||||
|
||||
def start(self, host="0.0.0.0", port=5000, debug=True):
|
||||
app = self.flask.Flask(__name__)
|
||||
|
||||
def signal_handler(sig, frame):
|
||||
logger.info("\nGracefully shutting down the WhatsAppBot...")
|
||||
sys.exit(0)
|
||||
|
||||
signal.signal(signal.SIGINT, signal_handler)
|
||||
|
||||
@app.route("/chat", methods=["POST"])
|
||||
def chat():
|
||||
incoming_message = self.flask.request.values.get("Body", "").lower()
|
||||
response = self.handle_message(incoming_message)
|
||||
twilio_response = self.twilio.twiml.messaging_response.MessagingResponse()
|
||||
twilio_response.message(response)
|
||||
return str(twilio_response)
|
||||
|
||||
app.run(host=host, port=port, debug=debug)
|
||||
|
||||
|
||||
def start_command():
|
||||
parser = argparse.ArgumentParser(description="EmbedChain WhatsAppBot command line interface")
|
||||
parser.add_argument("--host", default="0.0.0.0", help="Host IP to bind")
|
||||
parser.add_argument("--port", default=5000, type=int, help="Port to bind")
|
||||
args = parser.parse_args()
|
||||
|
||||
whatsapp_bot = WhatsAppBot()
|
||||
whatsapp_bot.start(host=args.host, port=args.port)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
start_command()
|
||||
Reference in New Issue
Block a user