Fix the direct message flag.

fjerkroa_bot/ai_responder.py

@@ -10,7 +10,7 @@ from pathlib import Path
 from io import BytesIO
 from pprint import pformat
 from functools import lru_cache, wraps
-from typing import Optional, List, Dict, Any, Tuple
+from typing import Optional, List, Dict, Any, Tuple, Union
 
 
 def pp(*args, **kw):
@@ -107,19 +107,21 @@ def same_channel(item1: Dict[str, Any], item2: Dict[str, Any]) -> bool:
 
 class AIMessageBase(object):
     def __init__(self) -> None:
-        pass
+        self.vars: List[str] = []
 
     def __str__(self) -> str:
-        return json.dumps(vars(self))
+        return json.dumps({k: v for k, v in vars(self).items() if k in self.vars})
 
 
 class AIMessage(AIMessageBase):
     def __init__(self, user: str, message: str, channel: str = "chat", direct: bool = False, historise_question: bool = True) -> None:
         self.user = user
         self.message = message
+        self.urls: Optional[List[str]] = None
         self.channel = channel
         self.direct = direct
         self.historise_question = historise_question
+        self.vars = ['user', 'message', 'channel', 'direct']
 
 
 class AIResponse(AIMessageBase):
@@ -137,6 +139,7 @@ class AIResponse(AIMessageBase):
         self.staff = staff
         self.picture = picture
         self.hack = hack
+        self.vars = ['answer', 'answer_needed', 'channel', 'staff', 'picture', 'hack']
 
 
 class AIResponderBase(object):
@@ -182,7 +185,13 @@ class AIResponder(AIResponderBase):
                 self.shrink_history_by_one()
         for msg in self.history:
             messages.append(msg)
-        messages.append({"role": "user", "content": str(message)})
+        if not message.urls:
+            messages.append({"role": "user", "content": str(message)})
+        else:
+            content: List[Dict[str, Union[str, Dict[str, str]]]] = [{"type": "text", "text": str(message)}]
+            for url in message.urls:
+                content.append({"type": "image_url", "image_url": {"url": url}})
+            messages.append({"role": "user", "content": content})
         return messages
 
     async def draw(self, description: str) -> BytesIO:
@@ -248,7 +257,7 @@ class AIResponder(AIResponderBase):
     async def fix(self, answer: str) -> str:
         raise NotImplementedError()
 
-    async def memory_rewrite(self, memory: str, user: str, question: str, answer: str) -> str:
+    async def memory_rewrite(self, memory: str, message_user: str, answer_user: str, question: str, answer: str) -> str:
         raise NotImplementedError()
 
     async def translate(self, text: str, language: str = "english") -> str:
@@ -270,6 +279,8 @@ class AIResponder(AIResponderBase):
                        answer: Dict[str, Any],
                        limit: int,
                        historise_question: bool = True) -> None:
+        if type(question['content']) != str:
+            question['content'] = question['content'][0]['text']
         if historise_question:
             self.history.append(question)
         self.history.append(answer)
@@ -292,13 +303,14 @@ class AIResponder(AIResponderBase):
             response["picture"] = await self.translate(response["picture"])
         return True
 
-    async def memoize(self, user: str, message: str, answer: str) -> None:
-        self.memory = await self.memory_rewrite(self.memory, user, message, answer)
+    async def memoize(self, message_user: str, answer_user: str, message: str, answer: str) -> None:
+        self.memory = await self.memory_rewrite(self.memory, message_user, answer_user, message, answer)
         self.update_memory(self.memory)
 
     async def memoize_reaction(self, message_user: str, reaction_user: str, operation: str, reaction: str, message: str) -> None:
         quoted_message = message.replace('\n', '\n> ')
-        await self.memoize(reaction_user, f'{message_user}:\n> {quoted_message}',
+        await self.memoize(message_user, 'assistant',
+                           f'\n> {quoted_message}',
                            f'User {reaction_user} has {operation} this raction: {reaction}')
 
     async def send(self, message: AIMessage) -> AIResponse:
@@ -352,7 +364,7 @@ class AIResponder(AIResponderBase):
 
             # Update memory
             if answer_message.answer is not None:
-                await self.memoize(message.user, message.message, answer_message.answer)
+                await self.memoize(message.user, 'assistant', message.message, answer_message.answer)
 
             # Return the updated answer message
             return answer_message

fjerkroa_bot/discord_bot.py

@@ -32,6 +32,7 @@ class FjerkroaBot(commands.Bot):
         intents = discord.Intents.default()
         intents.message_content = True
         intents.members = True
+        intents.reactions = True
         self._re_user = re.compile(r"[<][@][!]?\s*([0-9]+)[>]")
 
         self.init_observer()
@@ -108,21 +109,37 @@ class FjerkroaBot(commands.Bot):
             return
         await self.handle_message_through_responder(message)
 
-    async def on_reaction_add(self, reaction, user):
+    async def on_reaction_operation(self, reaction, user, operation):
         if user.bot:
             return
+        logging.info(f'{operation} reaction {reaction} by {user}.')
         airesponder = self.get_ai_responder(self.get_channel_name(reaction.message.channel))
         message = str(reaction.message.content) if reaction.message.content else ''
         if len(message) > 1:
-            await airesponder.memoize_reaction(reaction.message.user.name, user.name, 'adding', str(reaction.emoji), message)
+            await airesponder.memoize_reaction(reaction.message.author.name, user.name, operation, str(reaction.emoji), message)
+
+    async def on_reaction_add(self, reaction, user):
+        await self.on_reaction_operation(reaction, user, 'adding')
 
     async def on_reaction_remove(self, reaction, user):
-        if user.bot:
+        await self.on_reaction_operation(reaction, user, 'removing')
+
+    async def on_reaction_clear(self, reaction, user):
+        await self.on_reaction_operation(reaction, user, 'clearing')
+
+    async def on_message_edit(self, before, after):
+        if before.author.bot or before.content == after.content:
             return
-        airesponder = self.get_ai_responder(self.get_channel_name(reaction.message.channel))
-        message = str(reaction.message.content) if reaction.message.content else ''
-        if len(message) > 1:
-            await airesponder.memoize_reaction(reaction.message.user.name, user.name, 'removing', str(reaction.emoji), message)
+        airesponder = self.get_ai_responder(self.get_channel_name(before.channel))
+        await airesponder.memoize(before.author.name, 'assistant',
+                                  '\n> ' + before.content.replace('\n', '\n> '),
+                                  'User changed this message to:\n> ' + after.content.replace('\n', '\n> '))
+
+    async def on_message_delete(self, message):
+        airesponder = self.get_ai_responder(self.get_channel_name(message.channel))
+        await airesponder.memoize(message.author.name, 'assistant',
+                                  '\n> ' + message.content.replace('\n', '\n> '),
+                                  'User deleted this message.')
 
     def on_config_file_modified(self, event):
         if event.src_path == self.config_file:
@@ -188,7 +205,13 @@ class FjerkroaBot(commands.Bot):
             if user is not None:
                 message_content = re.sub(f'[<][@][!]? *{uid} *[>]', f'@{user.name}', message_content)
         channel_name = self.get_channel_name(message.channel)
-        msg = AIMessage(message.author.name, message_content, channel_name, self.user in message.mentions)
+        msg = AIMessage(message.author.name, message_content, channel_name,
+                        self.user in message.mentions or isinstance(message.channel, DMChannel))
+        if message.attachments:
+            for attachment in message.attachments:
+                if not msg.urls:
+                    msg.urls = []
+                msg.urls.append(attachment.url)
         await self.respond(msg, message.channel)
 
     async def send_message_with_typing(self, airesponder, channel, message):

fjerkroa_bot/openai_responder.py

@@ -37,7 +37,12 @@ class OpenAIResponder(AIResponder, LeonardoAIDrawMixIn):
         raise RuntimeError(f"Failed to generate image {repr(description)} after multiple retries")
 
     async def chat(self, messages: List[Dict[str, Any]], limit: int) -> Tuple[Optional[Dict[str, Any]], int]:
-        model = self.config["model"]
+        if type(messages[-1]['content']) == str:
+            model = self.config["model"]
+        elif 'model-vision' in self.config:
+            model = self.config["model-vision"]
+        else:
+            messages[-1]['content'] = messages[-1]['content'][0]['text']
         try:
             result = await openai_chat(self.client,
                                        model=model,
@@ -111,15 +116,16 @@ class OpenAIResponder(AIResponder, LeonardoAIDrawMixIn):
             logging.warning(f"failed to translate the text: {repr(err)}")
         return text
 
-    async def memory_rewrite(self, memory: str, user: str, question: str, answer: str) -> str:
+    async def memory_rewrite(self, memory: str, message_user: str, answer_user: str, question: str, answer: str) -> str:
         if 'memory-model' not in self.config:
             return memory
         messages = [{'role': 'system', 'content': self.config.get('memory-system', 'You are an memory assistant.')},
                     {'role': 'user', 'content': f'Here is my previous memory:\n```\n{memory}\n```\n\n'
-                                     f'Here is my conversanion:\n```\n{user}: {question}\n\nassistant: {answer}\n```\n\n'
+                                     f'Here is my conversanion:\n```\n{message_user}: {question}\n\n{answer_user}: {answer}\n```\n\n'
                                      f'Please rewrite the memory in a way, that it contain the content mentioned in conversation. '
-                                     f'The whole memory should not be too long, summarize if required. '
+                                     f'Summarize the memory if required, try to keep important information. '
                                      f'Write just new memory data without any comments.'}]
+        logging.info(f'Rewrite memory:\n{pp(messages)}')
         try:
             # logging.info(f'send this memory request:\n{pp(messages)}')
             result = await openai_chat(self.client,