Merge branch 'master' into help_channel_rediscache

11 files changed, 134 insertions, 84 deletions

diff --git a/bot/cogs/clean.py b/bot/cogs/clean.py
index b5d9132cb..368d91c85 100644
--- a/bot/cogs/clean.py
+++ b/bot/cogs/clean.py
@@ -1,16 +1,16 @@
 import logging
 import random
 import re
-from typing import Optional
+from typing import Iterable, Optional
 
 from discord import Colour, Embed, Message, TextChannel, User
+from discord.ext import commands
 from discord.ext.commands import Cog, Context, group
 
 from bot.bot import Bot
 from bot.cogs.moderation import ModLog
 from bot.constants import (
-    Channels, CleanMessages, Colours, Event,
-    Icons, MODERATION_ROLES, NEGATIVE_REPLIES
+    Channels, CleanMessages, Colours, Event, Icons, MODERATION_ROLES, NEGATIVE_REPLIES
 )
 from bot.decorators import with_role
 
@@ -41,10 +41,10 @@ class Clean(Cog):
         self,
         amount: int,
         ctx: Context,
+        channels: Iterable[TextChannel],
         bots_only: bool = False,
         user: User = None,
         regex: Optional[str] = None,
-        channel: Optional[TextChannel] = None
     ) -> None:
         """A helper function that does the actual message cleaning."""
         def predicate_bots_only(message: Message) -> bool:
@@ -110,48 +110,39 @@ class Clean(Cog):
             predicate = None                     # Delete all messages
 
         # Default to using the invoking context's channel
-        if not channel:
-            channel = ctx.channel
+        if not channels:
+            channels = [ctx.channel]
+
+        # Delete the invocation first
+        self.mod_log.ignore(Event.message_delete, ctx.message.id)
+        await ctx.message.delete()
 
-        # Look through the history and retrieve message data
         messages = []
         message_ids = []
         self.cleaning = True
-        invocation_deleted = False
-
-        # To account for the invocation message, we index `amount + 1` messages.
-        async for message in channel.history(limit=amount + 1):
 
-            # If at any point the cancel command is invoked, we should stop.
-            if not self.cleaning:
-                return
+        # Find the IDs of the messages to delete. IDs are needed in order to ignore mod log events.
+        for channel in channels:
+            async for message in channel.history(limit=amount):
 
-            # Always start by deleting the invocation
-            if not invocation_deleted:
-                self.mod_log.ignore(Event.message_delete, message.id)
-                await message.delete()
-                invocation_deleted = True
-                continue
+                # If at any point the cancel command is invoked, we should stop.
+                if not self.cleaning:
+                    return
 
-            # If the message passes predicate, let's save it.
-            if predicate is None or predicate(message):
-                message_ids.append(message.id)
-                messages.append(message)
+                # If the message passes predicate, let's save it.
+                if predicate is None or predicate(message):
+                    message_ids.append(message.id)
 
         self.cleaning = False
 
-        # We should ignore the ID's we stored, so we don't get mod-log spam.
+        # Now let's delete the actual messages with purge.
         self.mod_log.ignore(Event.message_delete, *message_ids)
-
-        # Use bulk delete to actually do the cleaning. It's far faster.
-        await channel.purge(
-            limit=amount,
-            check=predicate
-        )
+        for channel in channels:
+            messages += await channel.purge(limit=amount, check=predicate)
 
         # Reverse the list to restore chronological order
         if messages:
-            messages = list(reversed(messages))
+            messages = reversed(messages)
             log_url = await self.mod_log.upload_log(messages, ctx.author.id)
         else:
             # Can't build an embed, nothing to clean!
@@ -163,8 +154,10 @@ class Clean(Cog):
             return
 
         # Build the embed and send it
+        target_channels = ", ".join(channel.mention for channel in channels)
+
         message = (
-            f"**{len(message_ids)}** messages deleted in <#{channel.id}> by **{ctx.author.name}**\n\n"
+            f"**{len(message_ids)}** messages deleted in {target_channels} by **{ctx.author.name}**\n\n"
             f"A log of the deleted messages can be found [here]({log_url})."
         )
 
@@ -189,10 +182,10 @@ class Clean(Cog):
         ctx: Context,
         user: User,
         amount: Optional[int] = 10,
-        channel: TextChannel = None
+        channels: commands.Greedy[TextChannel] = None
     ) -> None:
         """Delete messages posted by the provided user, stop cleaning after traversing `amount` messages."""
-        await self._clean_messages(amount, ctx, user=user, channel=channel)
+        await self._clean_messages(amount, ctx, user=user, channels=channels)
 
     @clean_group.command(name="all", aliases=["everything"])
     @with_role(*MODERATION_ROLES)
@@ -200,10 +193,10 @@ class Clean(Cog):
         self,
         ctx: Context,
         amount: Optional[int] = 10,
-        channel: TextChannel = None
+        channels: commands.Greedy[TextChannel] = None
     ) -> None:
         """Delete all messages, regardless of poster, stop cleaning after traversing `amount` messages."""
-        await self._clean_messages(amount, ctx, channel=channel)
+        await self._clean_messages(amount, ctx, channels=channels)
 
     @clean_group.command(name="bots", aliases=["bot"])
     @with_role(*MODERATION_ROLES)
@@ -211,10 +204,10 @@ class Clean(Cog):
         self,
         ctx: Context,
         amount: Optional[int] = 10,
-        channel: TextChannel = None
+        channels: commands.Greedy[TextChannel] = None
     ) -> None:
         """Delete all messages posted by a bot, stop cleaning after traversing `amount` messages."""
-        await self._clean_messages(amount, ctx, bots_only=True, channel=channel)
+        await self._clean_messages(amount, ctx, bots_only=True, channels=channels)
 
     @clean_group.command(name="regex", aliases=["word", "expression"])
     @with_role(*MODERATION_ROLES)
@@ -223,10 +216,10 @@ class Clean(Cog):
         ctx: Context,
         regex: str,
         amount: Optional[int] = 10,
-        channel: TextChannel = None
+        channels: commands.Greedy[TextChannel] = None
     ) -> None:
         """Delete all messages that match a certain regex, stop cleaning after traversing `amount` messages."""
-        await self._clean_messages(amount, ctx, regex=regex, channel=channel)
+        await self._clean_messages(amount, ctx, regex=regex, channels=channels)
 
     @clean_group.command(name="stop", aliases=["cancel", "abort"])
     @with_role(*MODERATION_ROLES)
diff --git a/bot/cogs/information.py b/bot/cogs/information.py
index f0eb3a1ea..f0bd1afdb 100644
--- a/bot/cogs/information.py
+++ b/bot/cogs/information.py
@@ -6,7 +6,8 @@ from collections import Counter, defaultdict
 from string import Template
 from typing import Any, Mapping, Optional, Union
 
-from discord import Colour, Embed, Member, Message, Role, Status, utils
+from discord import ChannelType, Colour, Embed, Guild, Member, Message, Role, Status, utils
+from discord.abc import GuildChannel
 from discord.ext.commands import BucketType, Cog, Context, Paginator, command, group
 from discord.utils import escape_markdown
 
@@ -26,6 +27,49 @@ class Information(Cog):
     def __init__(self, bot: Bot):
         self.bot = bot
 
+    @staticmethod
+    def role_can_read(channel: GuildChannel, role: Role) -> bool:
+        """Return True if `role` can read messages in `channel`."""
+        overwrites = channel.overwrites_for(role)
+        return overwrites.read_messages is True
+
+    def get_staff_channel_count(self, guild: Guild) -> int:
+        """
+        Get the number of channels that are staff-only.
+
+        We need to know two things about a channel:
+        - Does the @everyone role have explicit read deny permissions?
+        - Do staff roles have explicit read allow permissions?
+
+        If the answer to both of these questions is yes, it's a staff channel.
+        """
+        channel_ids = set()
+        for channel in guild.channels:
+            if channel.type is ChannelType.category:
+                continue
+
+            everyone_can_read = self.role_can_read(channel, guild.default_role)
+
+            for role in constants.STAFF_ROLES:
+                role_can_read = self.role_can_read(channel, guild.get_role(role))
+                if role_can_read and not everyone_can_read:
+                    channel_ids.add(channel.id)
+                    break
+
+        return len(channel_ids)
+
+    @staticmethod
+    def get_channel_type_counts(guild: Guild) -> str:
+        """Return the total amounts of the various types of channels in `guild`."""
+        channel_counter = Counter(c.type for c in guild.channels)
+        channel_type_list = []
+        for channel, count in channel_counter.items():
+            channel_type = str(channel).title()
+            channel_type_list.append(f"{channel_type} channels: {count}")
+
+        channel_type_list = sorted(channel_type_list)
+        return "\n".join(channel_type_list)
+
     @with_role(*constants.MODERATION_ROLES)
     @command(name="roles")
     async def roles_info(self, ctx: Context) -> None:
@@ -102,15 +146,16 @@ class Information(Cog):
 
         roles = len(ctx.guild.roles)
         member_count = ctx.guild.member_count
-
-        # How many of each type of channel?
-        channels = Counter(c.type for c in ctx.guild.channels)
-        channel_counts = "".join(sorted(f"{str(ch).title()} channels: {channels[ch]}\n" for ch in channels)).strip()
+        channel_counts = self.get_channel_type_counts(ctx.guild)
 
         # How many of each user status?
         statuses = Counter(member.status for member in ctx.guild.members)
         embed = Embed(colour=Colour.blurple())
 
+        # How many staff members and staff channels do we have?
+        staff_member_count = len(ctx.guild.get_role(constants.Roles.helpers).members)
+        staff_channel_count = self.get_staff_channel_count(ctx.guild)
+
         # Because channel_counts lacks leading whitespace, it breaks the dedent if it's inserted directly by the
         # f-string. While this is correctly formated by Discord, it makes unit testing difficult. To keep the formatting
         # without joining a tuple of strings we can use a Template string to insert the already-formatted channel_counts
@@ -122,12 +167,16 @@ class Information(Cog):
                 Voice region: {region}
                 Features: {features}
 
-                **Counts**
+                **Channel counts**
+                $channel_counts
+                Staff channels: {staff_channel_count}
+
+                **Member counts**
                 Members: {member_count:,}
+                Staff members: {staff_member_count}
                 Roles: {roles}
-                $channel_counts
 
-                **Members**
+                **Member statuses**
                 {constants.Emojis.status_online} {statuses[Status.online]:,}
                 {constants.Emojis.status_idle} {statuses[Status.idle]:,}
                 {constants.Emojis.status_dnd} {statuses[Status.dnd]:,}
diff --git a/bot/cogs/moderation/scheduler.py b/bot/cogs/moderation/scheduler.py
index f0a3ad1b1..b03d89537 100644
--- a/bot/cogs/moderation/scheduler.py
+++ b/bot/cogs/moderation/scheduler.py
@@ -106,6 +106,27 @@ class InfractionScheduler(Scheduler):
         log_content = None
         failed = False
 
+        # DM the user about the infraction if it's not a shadow/hidden infraction.
+        # This needs to happen before we apply the infraction, as the bot cannot
+        # send DMs to user that it doesn't share a guild with. If we were to
+        # apply kick/ban infractions first, this would mean that we'd make it
+        # impossible for us to deliver a DM. See python-discord/bot#982.
+        if not infraction["hidden"]:
+            dm_result = f"{constants.Emojis.failmail} "
+            dm_log_text = "\nDM: **Failed**"
+
+            # Sometimes user is a discord.Object; make it a proper user.
+            try:
+                if not isinstance(user, (discord.Member, discord.User)):
+                    user = await self.bot.fetch_user(user.id)
+            except discord.HTTPException as e:
+                log.error(f"Failed to DM {user.id}: could not fetch user (status {e.status})")
+            else:
+                # Accordingly display whether the user was successfully notified via DM.
+                if await utils.notify_infraction(user, infr_type, expiry, reason, icon):
+                    dm_result = ":incoming_envelope: "
+                    dm_log_text = "\nDM: Sent"
+
         if infraction["actor"] == self.bot.user.id:
             log.trace(
                 f"Infraction #{id_} actor is bot; including the reason in the confirmation message."
@@ -150,27 +171,7 @@ class InfractionScheduler(Scheduler):
                     log.exception(log_msg)
                 failed = True
 
-        # DM the user about the infraction if it's not a shadow/hidden infraction.
-        # Don't send DM when applying failed.
-        if not infraction["hidden"] and not failed:
-            dm_result = f"{constants.Emojis.failmail} "
-            dm_log_text = "\nDM: **Failed**"
-
-            # Sometimes user is a discord.Object; make it a proper user.
-            try:
-                if not isinstance(user, (discord.Member, discord.User)):
-                    user = await self.bot.fetch_user(user.id)
-            except discord.HTTPException as e:
-                log.error(f"Failed to DM {user.id}: could not fetch user (status {e.status})")
-            else:
-                # Accordingly display whether the user was successfully notified via DM.
-                if await utils.notify_infraction(user, infr_type, expiry, reason, icon):
-                    dm_result = ":incoming_envelope: "
-                    dm_log_text = "\nDM: Sent"
-
         if failed:
-            dm_log_text = "\nDM: **Canceled**"
-            dm_result = f"{constants.Emojis.failmail} "
             log.trace(f"Deleted infraction {infraction['id']} from database because applying infraction failed.")
             try:
                 await self.bot.api_client.delete(f"bot/infractions/{id_}")
diff --git a/bot/cogs/moderation/utils.py b/bot/cogs/moderation/utils.py
index 1b716b2ea..fb55287b6 100644
--- a/bot/cogs/moderation/utils.py
+++ b/bot/cogs/moderation/utils.py
@@ -41,7 +41,6 @@ async def post_user(ctx: Context, user: UserSnowflake) -> t.Optional[dict]:
         log.debug("The user being added to the DB is not a Member or User object.")
 
     payload = {
-        'avatar_hash': getattr(user, 'avatar', 0),
         'discriminator': int(getattr(user, 'discriminator', 0)),
         'id': user.id,
         'in_guild': False,
diff --git a/bot/cogs/site.py b/bot/cogs/site.py
index 7fc2a9c34..e61cd5003 100644
--- a/bot/cogs/site.py
+++ b/bot/cogs/site.py
@@ -133,6 +133,9 @@ class Site(Cog):
             await ctx.send(f":x: Invalid rule indices: {indices}")
             return
 
+        for rule in rules:
+            self.bot.stats.incr(f"rule_uses.{rule}")
+
         final_rules = tuple(f"**{pick}.** {full_rules[pick - 1]}" for pick in rules)
 
         await LinePaginator.paginate(final_rules, ctx, rules_embed, max_lines=3)
diff --git a/bot/cogs/sync/cog.py b/bot/cogs/sync/cog.py
index 5708be3f4..7cc3726b2 100644
--- a/bot/cogs/sync/cog.py
+++ b/bot/cogs/sync/cog.py
@@ -94,7 +94,6 @@ class Sync(Cog):
         the database, the user is added.
         """
         packed = {
-            'avatar_hash': member.avatar,
             'discriminator': int(member.discriminator),
             'id': member.id,
             'in_guild': True,
@@ -135,12 +134,11 @@ class Sync(Cog):
     @Cog.listener()
     async def on_user_update(self, before: User, after: User) -> None:
         """Update the user information in the database if a relevant change is detected."""
-        attrs = ("name", "discriminator", "avatar")
+        attrs = ("name", "discriminator")
         if any(getattr(before, attr) != getattr(after, attr) for attr in attrs):
             updated_information = {
                 "name": after.name,
                 "discriminator": int(after.discriminator),
-                "avatar_hash": after.avatar,
             }
             await self.patch_user(after.id, updated_information=updated_information)
 
diff --git a/bot/cogs/sync/syncers.py b/bot/cogs/sync/syncers.py
index e55bf27fd..536455668 100644
--- a/bot/cogs/sync/syncers.py
+++ b/bot/cogs/sync/syncers.py
@@ -17,7 +17,7 @@ log = logging.getLogger(__name__)
 # These objects are declared as namedtuples because tuples are hashable,
 # something that we make use of when diffing site roles against guild roles.
 _Role = namedtuple('Role', ('id', 'name', 'colour', 'permissions', 'position'))
-_User = namedtuple('User', ('id', 'name', 'discriminator', 'avatar_hash', 'roles', 'in_guild'))
+_User = namedtuple('User', ('id', 'name', 'discriminator', 'roles', 'in_guild'))
 _Diff = namedtuple('Diff', ('created', 'updated', 'deleted'))
 
 
@@ -298,7 +298,6 @@ class UserSyncer(Syncer):
                 id=member.id,
                 name=member.name,
                 discriminator=int(member.discriminator),
-                avatar_hash=member.avatar,
                 roles=tuple(sorted(role.id for role in member.roles)),
                 in_guild=True
             )
diff --git a/bot/resources/tags/modmail.md b/bot/resources/tags/modmail.md
new file mode 100644
index 000000000..7545419ee
--- /dev/null
+++ b/bot/resources/tags/modmail.md
@@ -0,0 +1,9 @@
+**Contacting the moderation team via ModMail**
+
+<@!683001325440860340> is a bot that will relay your messages to our moderation team, so that you can start a conversation with the moderation team. Your messages will be relayed to the entire moderator team, who will be able to respond to you via the bot.
+
+It supports attachments, codeblocks, and reactions. As communication happens over direct messages, the conversation will stay between you and the mod team.
+
+**To use it, simply send a direct message to the bot.**
+
+Should there be an urgent and immediate need for a moderator or admin to look at a channel, feel free to ping the <@&267629731250176001> or <@&267628507062992896> role instead.
diff --git a/tests/bot/cogs/sync/test_cog.py b/tests/bot/cogs/sync/test_cog.py
index 81398c61f..14fd909c4 100644
--- a/tests/bot/cogs/sync/test_cog.py
+++ b/tests/bot/cogs/sync/test_cog.py
@@ -247,14 +247,12 @@ class SyncCogListenerTests(SyncCogTestCase):
         before_data = {
             "name": "old name",
             "discriminator": "1234",
-            "avatar": "old avatar",
             "bot": False,
         }
 
         subtests = (
             (True, "name", "name", "new name", "new name"),
             (True, "discriminator", "discriminator", "8765", 8765),
-            (True, "avatar", "avatar_hash", "9j2e9", "9j2e9"),
             (False, "bot", "bot", True, True),
         )
 
@@ -295,7 +293,6 @@ class SyncCogListenerTests(SyncCogTestCase):
         )
 
         data = {
-            "avatar_hash": member.avatar,
             "discriminator": int(member.discriminator),
             "id": member.id,
             "in_guild": True,
diff --git a/tests/bot/cogs/sync/test_users.py b/tests/bot/cogs/sync/test_users.py
index 818883012..002a947ad 100644
--- a/tests/bot/cogs/sync/test_users.py
+++ b/tests/bot/cogs/sync/test_users.py
@@ -10,7 +10,6 @@ def fake_user(**kwargs):
     kwargs.setdefault("id", 43)
     kwargs.setdefault("name", "bob the test man")
     kwargs.setdefault("discriminator", 1337)
-    kwargs.setdefault("avatar_hash", None)
     kwargs.setdefault("roles", (666,))
     kwargs.setdefault("in_guild", True)
 
@@ -32,7 +31,6 @@ class UserSyncerDiffTests(unittest.IsolatedAsyncioTestCase):
 
         for member in members:
             member = member.copy()
-            member["avatar"] = member.pop("avatar_hash")
             del member["in_guild"]
 
             mock_member = helpers.MockMember(**member)
diff --git a/tests/bot/cogs/test_information.py b/tests/bot/cogs/test_information.py
index aca6b594f..79c0e0ad3 100644
--- a/tests/bot/cogs/test_information.py
+++ b/tests/bot/cogs/test_information.py
@@ -148,14 +148,18 @@ class InformationCogTests(unittest.TestCase):
                 Voice region: {self.ctx.guild.region}
                 Features: {', '.join(self.ctx.guild.features)}
 
-                **Counts**
-                Members: {self.ctx.guild.member_count:,}
-                Roles: {len(self.ctx.guild.roles)}
+                **Channel counts**
                 Category channels: 1
                 Text channels: 1
                 Voice channels: 1
+                Staff channels: 0
+
+                **Member counts**
+                Members: {self.ctx.guild.member_count:,}
+                Staff members: 0
+                Roles: {len(self.ctx.guild.roles)}
 
-                **Members**
+                **Member statuses**
                 {constants.Emojis.status_online} 2
                 {constants.Emojis.status_idle} 1
                 {constants.Emojis.status_dnd} 4