Fix image handling and do not react to bots or unchanged messages.

fjerkroa_bot/ai_responder.py

@@ -10,7 +10,7 @@ from pathlib import Path
 from io import BytesIO
 from pprint import pformat
 from functools import lru_cache, wraps
-from typing import Optional, List, Dict, Any, Tuple
+from typing import Optional, List, Dict, Any, Tuple, Union
 
 
 def pp(*args, **kw):
@@ -107,19 +107,21 @@ def same_channel(item1: Dict[str, Any], item2: Dict[str, Any]) -> bool:
 
 class AIMessageBase(object):
     def __init__(self) -> None:
-        pass
+        self.vars: List[str] = []
 
     def __str__(self) -> str:
-        return json.dumps(vars(self))
+        return json.dumps({k: v for k, v in vars(self).items() if k in self.vars})
 
 
 class AIMessage(AIMessageBase):
     def __init__(self, user: str, message: str, channel: str = "chat", direct: bool = False, historise_question: bool = True) -> None:
         self.user = user
         self.message = message
+        self.urls: Optional[List[str]] = None
         self.channel = channel
         self.direct = direct
         self.historise_question = historise_question
+        self.vars = ['user', 'message', 'channel', 'direct']
 
 
 class AIResponse(AIMessageBase):
@@ -137,6 +139,7 @@ class AIResponse(AIMessageBase):
         self.staff = staff
         self.picture = picture
         self.hack = hack
+        self.vars = ['answer', 'answer_needed', 'channel', 'staff', 'picture', 'hack']
 
 
 class AIResponderBase(object):
@@ -182,7 +185,13 @@ class AIResponder(AIResponderBase):
                 self.shrink_history_by_one()
         for msg in self.history:
             messages.append(msg)
-        messages.append({"role": "user", "content": str(message)})
+        if not message.urls:
+            messages.append({"role": "user", "content": str(message)})
+        else:
+            content: List[Dict[str, Union[str, Dict[str, str]]]] = [{"type": "text", "text": str(message)}]
+            for url in message.urls:
+                content.append({"type": "image_url", "image_url": {"url": url}})
+            messages.append({"role": "user", "content": content})
         return messages
 
     async def draw(self, description: str) -> BytesIO:
@@ -270,6 +279,8 @@ class AIResponder(AIResponderBase):
                        answer: Dict[str, Any],
                        limit: int,
                        historise_question: bool = True) -> None:
+        if type(question['content']) != str:
+            question['content'] = question['content'][0]['text']
         if historise_question:
             self.history.append(question)
         self.history.append(answer)

fjerkroa_bot/discord_bot.py

@@ -110,6 +110,8 @@ class FjerkroaBot(commands.Bot):
         await self.handle_message_through_responder(message)
 
     async def on_reaction_operation(self, reaction, user, operation):
+        if user.bot:
+            return
         logging.info(f'{operation} reaction {reaction} by {user}.')
         airesponder = self.get_ai_responder(self.get_channel_name(reaction.message.channel))
         message = str(reaction.message.content) if reaction.message.content else ''
@@ -126,6 +128,8 @@ class FjerkroaBot(commands.Bot):
         await self.on_reaction_operation(reaction, user, 'clearing')
 
     async def on_message_edit(self, before, after):
+        if before.author.bot or before.content == after.content:
+            return
         airesponder = self.get_ai_responder(self.get_channel_name(before.channel))
         await airesponder.memoize(before.author.name, 'assistant',
                                   '\n> ' + before.content.replace('\n', '\n> '),
@@ -202,6 +206,11 @@ class FjerkroaBot(commands.Bot):
                 message_content = re.sub(f'[<][@][!]? *{uid} *[>]', f'@{user.name}', message_content)
         channel_name = self.get_channel_name(message.channel)
         msg = AIMessage(message.author.name, message_content, channel_name, self.user in message.mentions)
+        if message.attachments:
+            for attachment in message.attachments:
+                if not msg.urls:
+                    msg.urls = []
+                msg.urls.append(attachment.url)
         await self.respond(msg, message.channel)
 
     async def send_message_with_typing(self, airesponder, channel, message):

fjerkroa_bot/openai_responder.py

@@ -37,7 +37,12 @@ class OpenAIResponder(AIResponder, LeonardoAIDrawMixIn):
         raise RuntimeError(f"Failed to generate image {repr(description)} after multiple retries")
 
     async def chat(self, messages: List[Dict[str, Any]], limit: int) -> Tuple[Optional[Dict[str, Any]], int]:
-        model = self.config["model"]
+        if type(messages[-1]['content']) == str:
+            model = self.config["model"]
+        elif 'model-vision' in self.config:
+            model = self.config["model-vision"]
+        else:
+            messages[-1]['content'] = messages[-1]['content'][0]['text']
         try:
             result = await openai_chat(self.client,
                                        model=model,
@@ -118,7 +123,7 @@ class OpenAIResponder(AIResponder, LeonardoAIDrawMixIn):
                     {'role': 'user', 'content': f'Here is my previous memory:\n```\n{memory}\n```\n\n'
                                      f'Here is my conversanion:\n```\n{message_user}: {question}\n\n{answer_user}: {answer}\n```\n\n'
                                      f'Please rewrite the memory in a way, that it contain the content mentioned in conversation. '
-                                     f'The whole memory should not be too long, summarize if required. '
+                                     f'Summarize the memory if required, try to keep important information. '
                                      f'Write just new memory data without any comments.'}]
         logging.info(f'Rewrite memory:\n{pp(messages)}')
         try: