Merge branch 'master' into remove-prometheus

6 files changed, 71 insertions, 41 deletions

diff --git a/bot/cogs/antispam.py b/bot/cogs/antispam.py
index f454061a6..f67ef6f05 100644
--- a/bot/cogs/antispam.py
+++ b/bot/cogs/antispam.py
@@ -19,6 +19,7 @@ from bot.constants import (
     STAFF_ROLES,
 )
 from bot.converters import Duration
+from bot.utils.messages import send_attachments
 
 
 log = logging.getLogger(__name__)
@@ -45,8 +46,9 @@ class DeletionContext:
     members: Dict[int, Member] = field(default_factory=dict)
     rules: Set[str] = field(default_factory=set)
     messages: Dict[int, Message] = field(default_factory=dict)
+    attachments: List[List[str]] = field(default_factory=list)
 
-    def add(self, rule_name: str, members: Iterable[Member], messages: Iterable[Message]) -> None:
+    async def add(self, rule_name: str, members: Iterable[Member], messages: Iterable[Message]) -> None:
         """Adds new rule violation events to the deletion context."""
         self.rules.add(rule_name)
 
@@ -58,6 +60,11 @@ class DeletionContext:
             if message.id not in self.messages:
                 self.messages[message.id] = message
 
+                # Re-upload attachments
+                destination = message.guild.get_channel(Channels.attachment_log)
+                urls = await send_attachments(message, destination, link_large=False)
+                self.attachments.append(urls)
+
     async def upload_messages(self, actor_id: int, modlog: ModLog) -> None:
         """Method that takes care of uploading the queue and posting modlog alert."""
         triggered_by_users = ", ".join(f"{m} (`{m.id}`)" for m in self.members.values())
@@ -70,7 +77,7 @@ class DeletionContext:
 
         # For multiple messages or those with excessive newlines, use the logs API
         if len(self.messages) > 1 or 'newlines' in self.rules:
-            url = await modlog.upload_log(self.messages.values(), actor_id)
+            url = await modlog.upload_log(self.messages.values(), actor_id, self.attachments)
             mod_alert_message += f"A complete log of the offending messages can be found [here]({url})"
         else:
             mod_alert_message += "Message:\n"
@@ -98,7 +105,7 @@ class DeletionContext:
 class AntiSpam(Cog):
     """Cog that controls our anti-spam measures."""
 
-    def __init__(self, bot: Bot, validation_errors: bool) -> None:
+    def __init__(self, bot: Bot, validation_errors: Dict[str, str]) -> None:
         self.bot = bot
         self.validation_errors = validation_errors
         role_id = AntiSpamConfig.punishment['role_id']
@@ -106,7 +113,6 @@ class AntiSpam(Cog):
         self.expiration_date_converter = Duration()
 
         self.message_deletion_queue = dict()
-        self.queue_consumption_tasks = dict()
 
         self.bot.loop.create_task(self.alert_on_validation_error())
 
@@ -180,15 +186,14 @@ class AntiSpam(Cog):
                 full_reason = f"`{rule_name}` rule: {reason}"
 
                 # If there's no spam event going on for this channel, start a new Message Deletion Context
-                if message.channel.id not in self.message_deletion_queue:
-                    log.trace(f"Creating queue for channel `{message.channel.id}`")
-                    self.message_deletion_queue[message.channel.id] = DeletionContext(channel=message.channel)
-                    self.queue_consumption_tasks = self.bot.loop.create_task(
-                        self._process_deletion_context(message.channel.id)
-                    )
+                channel = message.channel
+                if channel.id not in self.message_deletion_queue:
+                    log.trace(f"Creating queue for channel `{channel.id}`")
+                    self.message_deletion_queue[message.channel.id] = DeletionContext(channel)
+                    self.bot.loop.create_task(self._process_deletion_context(message.channel.id))
 
                 # Add the relevant of this trigger to the Deletion Context
-                self.message_deletion_queue[message.channel.id].add(
+                await self.message_deletion_queue[message.channel.id].add(
                     rule_name=rule_name,
                     members=members,
                     messages=relevant_messages
@@ -202,7 +207,7 @@ class AntiSpam(Cog):
                         self.punish(message, member, full_reason)
                     )
 
-                await self.maybe_delete_messages(message.channel, relevant_messages)
+                await self.maybe_delete_messages(channel, relevant_messages)
                 break
 
     async def punish(self, msg: Message, member: Member, reason: str) -> None:
@@ -255,10 +260,10 @@ class AntiSpam(Cog):
         await deletion_context.upload_messages(self.bot.user.id, self.mod_log)
 
 
-def validate_config(rules: Mapping = AntiSpamConfig.rules) -> Dict[str, str]:
+def validate_config(rules_: Mapping = AntiSpamConfig.rules) -> Dict[str, str]:
     """Validates the antispam configs."""
     validation_errors = {}
-    for name, config in rules.items():
+    for name, config in rules_.items():
         if name not in RULE_FUNCTION_MAPPING:
             log.error(
                 f"Unrecognized antispam rule `{name}`. "
diff --git a/bot/cogs/moderation/modlog.py b/bot/cogs/moderation/modlog.py
index c78eb24a7..e8ae0dbe6 100644
--- a/bot/cogs/moderation/modlog.py
+++ b/bot/cogs/moderation/modlog.py
@@ -4,6 +4,7 @@ import itertools
 import logging
 import typing as t
 from datetime import datetime
+from itertools import zip_longest
 
 import discord
 from dateutil.relativedelta import relativedelta
@@ -42,14 +43,16 @@ class ModLog(Cog, name="ModLog"):
         self._cached_deletes = []
         self._cached_edits = []
 
-    async def upload_log(self, messages: t.List[discord.Message], actor_id: int) -> str:
-        """
-        Uploads the log data to the database via an API endpoint for uploading logs.
-
-        Used in several mod log embeds.
+    async def upload_log(
+        self,
+        messages: t.Iterable[discord.Message],
+        actor_id: int,
+        attachments: t.Iterable[t.List[str]] = None
+    ) -> str:
+        """Upload message logs to the database and return a URL to a page for viewing the logs."""
+        if attachments is None:
+            attachments = []
 
-        Returns a URL that can be used to view the log.
-        """
         response = await self.bot.api_client.post(
             'bot/deleted-messages',
             json={
@@ -61,9 +64,10 @@ class ModLog(Cog, name="ModLog"):
                         'author': message.author.id,
                         'channel_id': message.channel.id,
                         'content': message.content,
-                        'embeds': [embed.to_dict() for embed in message.embeds]
+                        'embeds': [embed.to_dict() for embed in message.embeds],
+                        'attachments': attachment,
                     }
-                    for message in messages
+                    for message, attachment in zip_longest(messages, attachments)
                 ]
             }
         )
diff --git a/bot/cogs/site.py b/bot/cogs/site.py
index 2ea8c7a2e..853e29568 100644
--- a/bot/cogs/site.py
+++ b/bot/cogs/site.py
@@ -59,7 +59,7 @@ class Site(Cog):
     @site_group.command(name="tools")
     async def site_tools(self, ctx: Context) -> None:
         """Info about the site's Tools page."""
-        tools_url = f"{PAGES_URL}/tools"
+        tools_url = f"{PAGES_URL}/resources/tools"
 
         embed = Embed(title="Tools")
         embed.set_footer(text=f"{tools_url}")
@@ -74,7 +74,7 @@ class Site(Cog):
     @site_group.command(name="help")
     async def site_help(self, ctx: Context) -> None:
         """Info about the site's Getting Help page."""
-        url = f"{PAGES_URL}/asking-good-questions"
+        url = f"{PAGES_URL}/resources/guides/asking-good-questions"
 
         embed = Embed(title="Asking Good Questions")
         embed.set_footer(text=url)
diff --git a/bot/constants.py b/bot/constants.py
index 25c7856ba..629985bdf 100644
--- a/bot/constants.py
+++ b/bot/constants.py
@@ -359,6 +359,7 @@ class Channels(metaclass=YAMLGetter):
     admins: int
     admin_spam: int
     announcements: int
+    attachment_log: int
     big_brother_logs: int
     bot: int
     checkpoint_test: int
diff --git a/bot/utils/messages.py b/bot/utils/messages.py
index 549b33ca6..c4e2753e0 100644
--- a/bot/utils/messages.py
+++ b/bot/utils/messages.py
@@ -1,7 +1,8 @@
 import asyncio
 import contextlib
+import logging
 from io import BytesIO
-from typing import Optional, Sequence, Union
+from typing import List, Optional, Sequence, Union
 
 from discord import Client, Embed, File, Member, Message, Reaction, TextChannel, Webhook
 from discord.abc import Snowflake
@@ -9,7 +10,7 @@ from discord.errors import HTTPException
 
 from bot.constants import Emojis
 
-MAX_SIZE = 1024 * 1024 * 8  # 8 Mebibytes
+log = logging.getLogger(__name__)
 
 
 async def wait_for_deletion(
@@ -51,42 +52,58 @@ async def wait_for_deletion(
         await message.delete()
 
 
-async def send_attachments(message: Message, destination: Union[TextChannel, Webhook]) -> None:
+async def send_attachments(
+    message: Message,
+    destination: Union[TextChannel, Webhook],
+    link_large: bool = True
+) -> List[str]:
     """
-    Re-uploads each attachment in a message to the given channel or webhook.
+    Re-upload the message's attachments to the destination and return a list of their new URLs.
 
-    Each attachment is sent as a separate message to more easily comply with the 8 MiB request size limit.
-    If attachments are too large, they are instead grouped into a single embed which links to them.
+    Each attachment is sent as a separate message to more easily comply with the request/file size
+    limit. If link_large is True, attachments which are too large are instead grouped into a single
+    embed which links to them.
     """
     large = []
+    urls = []
     for attachment in message.attachments:
+        failure_msg = (
+            f"Failed to re-upload attachment {attachment.filename} from message {message.id}"
+        )
+
         try:
-            # This should avoid most files that are too large, but some may get through hence the try-catch.
             # Allow 512 bytes of leeway for the rest of the request.
-            if attachment.size <= MAX_SIZE - 512:
+            # This should avoid most files that are too large,
+            # but some may get through hence the try-catch.
+            if attachment.size <= destination.guild.filesize_limit - 512:
                 with BytesIO() as file:
-                    await attachment.save(file)
+                    await attachment.save(file, use_cached=True)
                     attachment_file = File(file, filename=attachment.filename)
 
                     if isinstance(destination, TextChannel):
-                        await destination.send(file=attachment_file)
+                        msg = await destination.send(file=attachment_file)
+                        urls.append(msg.attachments[0].url)
                     else:
                         await destination.send(
                             file=attachment_file,
                             username=message.author.display_name,
                             avatar_url=message.author.avatar_url
                         )
-            else:
+            elif link_large:
                 large.append(attachment)
+            else:
+                log.warning(f"{failure_msg} because it's too large.")
         except HTTPException as e:
-            if e.status == 413:
+            if link_large and e.status == 413:
                 large.append(attachment)
             else:
-                raise
+                log.warning(f"{failure_msg} with status {e.status}.")
 
-    if large:
-        embed = Embed(description=f"\n".join(f"[{attachment.filename}]({attachment.url})" for attachment in large))
+    if link_large and large:
+        desc = f"\n".join(f"[{attachment.filename}]({attachment.url})" for attachment in large)
+        embed = Embed(description=desc)
         embed.set_footer(text="Attachments exceed upload size limit.")
+
         if isinstance(destination, TextChannel):
             await destination.send(embed=embed)
         else:
@@ -95,3 +112,5 @@ async def send_attachments(message: Message, destination: Union[TextChannel, Web
                 username=message.author.display_name,
                 avatar_url=message.author.avatar_url
             )
+
+    return urls
diff --git a/config-default.yml b/config-default.yml
index 1a8aaedae..c113d3330 100644
--- a/config-default.yml
+++ b/config-default.yml
@@ -115,6 +115,7 @@ guild:
         admin_spam:        &ADMIN_SPAM    563594791770914816
         admins_voice:      &ADMINS_VOICE  500734494840717332
         announcements:                    354619224620138496
+        attachment_log:    &ATTCH_LOG     649243850006855680
         big_brother_logs:  &BBLOGS        468507907357409333
         bot:                              267659945086812160
         checkpoint_test:                  422077681434099723
@@ -152,7 +153,7 @@ guild:
         voice_log:                        640292421988646961
 
     staff_channels: [*ADMINS, *ADMIN_SPAM, *MOD_SPAM, *MODS, *HELPERS, *ORGANISATION, *DEFCON]
-    ignored: [*ADMINS, *MESSAGE_LOG, *MODLOG, *ADMINS_VOICE, *STAFF_VOICE]
+    ignored: [*ADMINS, *MESSAGE_LOG, *MODLOG, *ADMINS_VOICE, *STAFF_VOICE, *ATTCH_LOG]
 
     roles:
         admin:             &ADMIN_ROLE      267628507062992896