diff --git a/fjerkroa_bot/ai_responder.py b/fjerkroa_bot/ai_responder.py index abf321b..ffbfcc3 100644 --- a/fjerkroa_bot/ai_responder.py +++ b/fjerkroa_bot/ai_responder.py @@ -10,7 +10,7 @@ from pathlib import Path from io import BytesIO from pprint import pformat from functools import lru_cache, wraps -from typing import Optional, List, Dict, Any, Tuple +from typing import Optional, List, Dict, Any, Tuple, Union def pp(*args, **kw): @@ -107,19 +107,21 @@ def same_channel(item1: Dict[str, Any], item2: Dict[str, Any]) -> bool: class AIMessageBase(object): def __init__(self) -> None: - pass + self.vars: List[str] = [] def __str__(self) -> str: - return json.dumps(vars(self)) + return json.dumps({k: v for k, v in vars(self).items() if k in self.vars}) class AIMessage(AIMessageBase): def __init__(self, user: str, message: str, channel: str = "chat", direct: bool = False, historise_question: bool = True) -> None: self.user = user self.message = message + self.urls: Optional[List[str]] = None self.channel = channel self.direct = direct self.historise_question = historise_question + self.vars = ['user', 'message', 'channel', 'direct'] class AIResponse(AIMessageBase): @@ -137,6 +139,7 @@ class AIResponse(AIMessageBase): self.staff = staff self.picture = picture self.hack = hack + self.vars = ['answer', 'answer_needed', 'channel', 'staff', 'picture', 'hack'] class AIResponderBase(object): @@ -182,7 +185,13 @@ class AIResponder(AIResponderBase): self.shrink_history_by_one() for msg in self.history: messages.append(msg) - messages.append({"role": "user", "content": str(message)}) + if not message.urls: + messages.append({"role": "user", "content": str(message)}) + else: + content: List[Dict[str, Union[str, Dict[str, str]]]] = [{"type": "text", "text": str(message)}] + for url in message.urls: + content.append({"type": "image_url", "image_url": {"url": url}}) + messages.append({"role": "user", "content": content}) return messages async def draw(self, description: str) -> BytesIO: @@ -270,6 +279,8 @@ class AIResponder(AIResponderBase): answer: Dict[str, Any], limit: int, historise_question: bool = True) -> None: + if type(question['content']) != str: + question['content'] = question['content'][0]['text'] if historise_question: self.history.append(question) self.history.append(answer) diff --git a/fjerkroa_bot/discord_bot.py b/fjerkroa_bot/discord_bot.py index 8476987..b7cd4e0 100644 --- a/fjerkroa_bot/discord_bot.py +++ b/fjerkroa_bot/discord_bot.py @@ -110,6 +110,8 @@ class FjerkroaBot(commands.Bot): await self.handle_message_through_responder(message) async def on_reaction_operation(self, reaction, user, operation): + if user.bot: + return logging.info(f'{operation} reaction {reaction} by {user}.') airesponder = self.get_ai_responder(self.get_channel_name(reaction.message.channel)) message = str(reaction.message.content) if reaction.message.content else '' @@ -126,6 +128,8 @@ class FjerkroaBot(commands.Bot): await self.on_reaction_operation(reaction, user, 'clearing') async def on_message_edit(self, before, after): + if before.author.bot or before.content == after.content: + return airesponder = self.get_ai_responder(self.get_channel_name(before.channel)) await airesponder.memoize(before.author.name, 'assistant', '\n> ' + before.content.replace('\n', '\n> '), @@ -202,6 +206,11 @@ class FjerkroaBot(commands.Bot): message_content = re.sub(f'[<][@][!]? *{uid} *[>]', f'@{user.name}', message_content) channel_name = self.get_channel_name(message.channel) msg = AIMessage(message.author.name, message_content, channel_name, self.user in message.mentions) + if message.attachments: + for attachment in message.attachments: + if not msg.urls: + msg.urls = [] + msg.urls.append(attachment.url) await self.respond(msg, message.channel) async def send_message_with_typing(self, airesponder, channel, message): diff --git a/fjerkroa_bot/openai_responder.py b/fjerkroa_bot/openai_responder.py index 4496ad9..6a1287f 100644 --- a/fjerkroa_bot/openai_responder.py +++ b/fjerkroa_bot/openai_responder.py @@ -37,7 +37,12 @@ class OpenAIResponder(AIResponder, LeonardoAIDrawMixIn): raise RuntimeError(f"Failed to generate image {repr(description)} after multiple retries") async def chat(self, messages: List[Dict[str, Any]], limit: int) -> Tuple[Optional[Dict[str, Any]], int]: - model = self.config["model"] + if type(messages[-1]['content']) == str: + model = self.config["model"] + elif 'model-vision' in self.config: + model = self.config["model-vision"] + else: + messages[-1]['content'] = messages[-1]['content'][0]['text'] try: result = await openai_chat(self.client, model=model, @@ -118,7 +123,7 @@ class OpenAIResponder(AIResponder, LeonardoAIDrawMixIn): {'role': 'user', 'content': f'Here is my previous memory:\n```\n{memory}\n```\n\n' f'Here is my conversanion:\n```\n{message_user}: {question}\n\n{answer_user}: {answer}\n```\n\n' f'Please rewrite the memory in a way, that it contain the content mentioned in conversation. ' - f'The whole memory should not be too long, summarize if required. ' + f'Summarize the memory if required, try to keep important information. ' f'Write just new memory data without any comments.'}] logging.info(f'Rewrite memory:\n{pp(messages)}') try: diff --git a/openai_chat.dat b/openai_chat.dat index 53f5a16..586ed71 100644 Binary files a/openai_chat.dat and b/openai_chat.dat differ