Merge branch 'main' into voice_ver_dm

13 files changed, 212 insertions, 143 deletions

diff --git a/bot/constants.py b/bot/constants.py
index 885b5c822..ab55da482 100644
--- a/bot/constants.py
+++ b/bot/constants.py
@@ -602,7 +602,6 @@ class HelpChannels(metaclass=YAMLGetter):
     section = 'help_channels'
 
     enable: bool
-    claim_minutes: int
     cmd_whitelist: List[int]
     idle_minutes_claimant: int
     idle_minutes_others: int
diff --git a/bot/exts/filters/pixels_token_remover.py b/bot/exts/filters/pixels_token_remover.py
new file mode 100644
index 000000000..2356491e5
--- /dev/null
+++ b/bot/exts/filters/pixels_token_remover.py
@@ -0,0 +1,108 @@
+import logging
+import re
+import typing as t
+
+from discord import Colour, Message, NotFound
+from discord.ext.commands import Cog
+
+from bot.bot import Bot
+from bot.constants import Channels, Colours, Event, Icons
+from bot.exts.moderation.modlog import ModLog
+from bot.utils.messages import format_user
+
+log = logging.getLogger(__name__)
+
+LOG_MESSAGE = "Censored a valid Pixels token sent by {author} in {channel}, token was `{token}`"
+DELETION_MESSAGE_TEMPLATE = (
+    "Hey {mention}! I noticed you posted a valid Pixels API "
+    "token in your message and have removed your message. "
+    "This means that your token has been **compromised**. "
+    "I have taken the liberty of invalidating the token for you. "
+    "You can go to <https://pixels.pythondiscord.com/authorize> to get a new key."
+)
+
+PIXELS_TOKEN_RE = re.compile(r"[A-Za-z0-9-_=]{30,}\.[A-Za-z0-9-_=]{50,}\.[A-Za-z0-9-_.+\=]{30,}")
+
+
+class PixelsTokenRemover(Cog):
+    """Scans messages for Pixels API tokens, removes and invalidates them."""
+
+    def __init__(self, bot: Bot):
+        self.bot = bot
+
+    @property
+    def mod_log(self) -> ModLog:
+        """Get currently loaded ModLog cog instance."""
+        return self.bot.get_cog("ModLog")
+
+    @Cog.listener()
+    async def on_message(self, msg: Message) -> None:
+        """Check each message for a string that matches the RS-256 token pattern."""
+        # Ignore DMs; can't delete messages in there anyway.
+        if not msg.guild or msg.author.bot:
+            return
+
+        found_token = await self.find_token_in_message(msg)
+        if found_token:
+            await self.take_action(msg, found_token)
+
+    @Cog.listener()
+    async def on_message_edit(self, before: Message, after: Message) -> None:
+        """Check each edit for a string that matches the RS-256 token pattern."""
+        await self.on_message(after)
+
+    async def take_action(self, msg: Message, found_token: str) -> None:
+        """Remove the `msg` containing the `found_token` and send a mod log message."""
+        self.mod_log.ignore(Event.message_delete, msg.id)
+
+        try:
+            await msg.delete()
+        except NotFound:
+            log.debug(f"Failed to remove token in message {msg.id}: message already deleted.")
+            return
+
+        await msg.channel.send(DELETION_MESSAGE_TEMPLATE.format(mention=msg.author.mention))
+
+        log_message = self.format_log_message(msg, found_token)
+        log.debug(log_message)
+
+        # Send pretty mod log embed to mod-alerts
+        await self.mod_log.send_log_message(
+            icon_url=Icons.token_removed,
+            colour=Colour(Colours.soft_red),
+            title="Token removed!",
+            text=log_message,
+            thumbnail=msg.author.avatar_url_as(static_format="png"),
+            channel_id=Channels.mod_alerts,
+            ping_everyone=False,
+        )
+
+        self.bot.stats.incr("tokens.removed_pixels_tokens")
+
+    @staticmethod
+    def format_log_message(msg: Message, token: str) -> str:
+        """Return the generic portion of the log message to send for `token` being censored in `msg`."""
+        return LOG_MESSAGE.format(
+            author=format_user(msg.author),
+            channel=msg.channel.mention,
+            token=token
+        )
+
+    async def find_token_in_message(self, msg: Message) -> t.Optional[str]:
+        """Return a seemingly valid token found in `msg` or `None` if no token is found."""
+        # Use finditer rather than search to guard against method calls prematurely returning the
+        # token check (e.g. `message.channel.send` also matches our token pattern)
+        for match in PIXELS_TOKEN_RE.finditer(msg.content):
+            auth_header = {"Authorization": f"Bearer {match[0]}"}
+            async with self.bot.http_session.delete("https://pixels.pythondiscord.com/token", headers=auth_header) as r:
+                if r.status == 204:
+                    # Short curcuit on first match.
+                    return match[0]
+
+        # No matching substring
+        return
+
+
+def setup(bot: Bot) -> None:
+    """Load the PixelsTokenRemover cog."""
+    bot.add_cog(PixelsTokenRemover(bot))
diff --git a/bot/exts/help_channels/_cog.py b/bot/exts/help_channels/_cog.py
index 262b18e16..5c410a0a1 100644
--- a/bot/exts/help_channels/_cog.py
+++ b/bot/exts/help_channels/_cog.py
@@ -12,7 +12,7 @@ from discord.ext import commands
 
 from bot import constants
 from bot.bot import Bot
-from bot.exts.help_channels import _caches, _channel, _cooldown, _message, _name, _stats
+from bot.exts.help_channels import _caches, _channel, _message, _name, _stats
 from bot.utils import channel as channel_utils, lock, scheduling
 
 log = logging.getLogger(__name__)
@@ -94,6 +94,24 @@ class HelpChannels(commands.Cog):
 
         self.scheduler.cancel_all()
 
+    async def _handle_role_change(self, member: discord.Member, coro: t.Callable[..., t.Coroutine]) -> None:
+        """
+        Change `member`'s cooldown role via awaiting `coro` and handle errors.
+
+        `coro` is intended to be `discord.Member.add_roles` or `discord.Member.remove_roles`.
+        """
+        try:
+            await coro(self.bot.get_guild(constants.Guild.id).get_role(constants.Roles.help_cooldown))
+        except discord.NotFound:
+            log.debug(f"Failed to change role for {member} ({member.id}): member not found")
+        except discord.Forbidden:
+            log.debug(
+                f"Forbidden to change role for {member} ({member.id}); "
+                f"possibly due to role hierarchy"
+            )
+        except discord.HTTPException as e:
+            log.error(f"Failed to change role for {member} ({member.id}): {e.status} {e.code}")
+
     @lock.lock_arg(NAMESPACE, "message", attrgetter("channel.id"))
     @lock.lock_arg(NAMESPACE, "message", attrgetter("author.id"))
     @lock.lock_arg(f"{NAMESPACE}.unclaim", "message", attrgetter("author.id"), wait=True)
@@ -106,9 +124,10 @@ class HelpChannels(commands.Cog):
         """
         log.info(f"Channel #{message.channel} was claimed by `{message.author.id}`.")
         await self.move_to_in_use(message.channel)
-        await _cooldown.revoke_send_permissions(message.author, self.scheduler)
+        await self._handle_role_change(message.author, message.author.add_roles)
 
         await _message.pin(message)
+
         try:
             await _message.dm_on_open(message)
         except Exception as e:
@@ -276,7 +295,6 @@ class HelpChannels(commands.Cog):
 
         log.trace("Initialising the cog.")
         await self.init_categories()
-        await _cooldown.check_cooldowns(self.scheduler)
 
         self.channel_queue = self.create_channel_queue()
         self.name_queue = _name.create_name_queue(
@@ -407,16 +425,11 @@ class HelpChannels(commands.Cog):
         """Actual implementation of `unclaim_channel`. See that for full documentation."""
         await _caches.claimants.delete(channel.id)
 
-        # Ignore missing tasks because a channel may still be dormant after the cooldown expires.
-        if claimant_id in self.scheduler:
-            self.scheduler.cancel(claimant_id)
-
         claimant = self.bot.get_guild(constants.Guild.id).get_member(claimant_id)
         if claimant is None:
             log.info(f"{claimant_id} left the guild during their help session; the cooldown role won't be removed")
-        elif not any(claimant.id == user_id for _, user_id in await _caches.claimants.items()):
-            # Remove the cooldown role if the claimant has no other channels left
-            await _cooldown.remove_cooldown_role(claimant)
+        else:
+            await self._handle_role_change(claimant, claimant.remove_roles)
 
         await _message.unpin(channel)
         await _stats.report_complete_session(channel.id, closed_on)
diff --git a/bot/exts/help_channels/_cooldown.py b/bot/exts/help_channels/_cooldown.py
deleted file mode 100644
index c5c39297f..000000000
--- a/bot/exts/help_channels/_cooldown.py
+++ /dev/null
@@ -1,95 +0,0 @@
-import logging
-from typing import Callable, Coroutine
-
-import discord
-
-import bot
-from bot import constants
-from bot.exts.help_channels import _caches, _channel
-from bot.utils.scheduling import Scheduler
-
-log = logging.getLogger(__name__)
-CoroutineFunc = Callable[..., Coroutine]
-
-
-async def add_cooldown_role(member: discord.Member) -> None:
-    """Add the help cooldown role to `member`."""
-    log.trace(f"Adding cooldown role for {member} ({member.id}).")
-    await _change_cooldown_role(member, member.add_roles)
-
-
-async def check_cooldowns(scheduler: Scheduler) -> None:
-    """Remove expired cooldowns and re-schedule active ones."""
-    log.trace("Checking all cooldowns to remove or re-schedule them.")
-    guild = bot.instance.get_guild(constants.Guild.id)
-    cooldown = constants.HelpChannels.claim_minutes * 60
-
-    for channel_id, member_id in await _caches.claimants.items():
-        member = guild.get_member(member_id)
-        if not member:
-            continue  # Member probably left the guild.
-
-        in_use_time = await _channel.get_in_use_time(channel_id)
-
-        if not in_use_time or in_use_time.seconds > cooldown:
-            # Remove the role if no claim time could be retrieved or if the cooldown expired.
-            # Since the channel is in the claimants cache, it is definitely strange for a time
-            # to not exist. However, it isn't a reason to keep the user stuck with a cooldown.
-            await remove_cooldown_role(member)
-        else:
-            # The member is still on a cooldown; re-schedule it for the remaining time.
-            delay = cooldown - in_use_time.seconds
-            scheduler.schedule_later(delay, member.id, remove_cooldown_role(member))
-
-
-async def remove_cooldown_role(member: discord.Member) -> None:
-    """Remove the help cooldown role from `member`."""
-    log.trace(f"Removing cooldown role for {member} ({member.id}).")
-    await _change_cooldown_role(member, member.remove_roles)
-
-
-async def revoke_send_permissions(member: discord.Member, scheduler: Scheduler) -> None:
-    """
-    Disallow `member` to send messages in the Available category for a certain time.
-
-    The time until permissions are reinstated can be configured with
-    `HelpChannels.claim_minutes`.
-    """
-    log.trace(
-        f"Revoking {member}'s ({member.id}) send message permissions in the Available category."
-    )
-
-    await add_cooldown_role(member)
-
-    # Cancel the existing task, if any.
-    # Would mean the user somehow bypassed the lack of permissions (e.g. user is guild owner).
-    if member.id in scheduler:
-        scheduler.cancel(member.id)
-
-    delay = constants.HelpChannels.claim_minutes * 60
-    scheduler.schedule_later(delay, member.id, remove_cooldown_role(member))
-
-
-async def _change_cooldown_role(member: discord.Member, coro_func: CoroutineFunc) -> None:
-    """
-    Change `member`'s cooldown role via awaiting `coro_func` and handle errors.
-
-    `coro_func` is intended to be `discord.Member.add_roles` or `discord.Member.remove_roles`.
-    """
-    guild = bot.instance.get_guild(constants.Guild.id)
-    role = guild.get_role(constants.Roles.help_cooldown)
-    if role is None:
-        log.warning(f"Help cooldown role ({constants.Roles.help_cooldown}) could not be found!")
-        return
-
-    try:
-        await coro_func(role)
-    except discord.NotFound:
-        log.debug(f"Failed to change role for {member} ({member.id}): member not found")
-    except discord.Forbidden:
-        log.debug(
-            f"Forbidden to change role for {member} ({member.id}); "
-            f"possibly due to role hierarchy"
-        )
-    except discord.HTTPException as e:
-        log.error(f"Failed to change role for {member} ({member.id}): {e.status} {e.code}")
diff --git a/bot/exts/info/doc/_cog.py b/bot/exts/info/doc/_cog.py
index 2a8016fb8..c54a3ee1c 100644
--- a/bot/exts/info/doc/_cog.py
+++ b/bot/exts/info/doc/_cog.py
@@ -27,11 +27,12 @@ log = logging.getLogger(__name__)
 
 # symbols with a group contained here will get the group prefixed on duplicates
 FORCE_PREFIX_GROUPS = (
-    "2to3fixer",
-    "token",
+    "term",
     "label",
+    "token",
+    "doc",
     "pdbcommand",
-    "term",
+    "2to3fixer",
 )
 NOT_FOUND_DELETE_DELAY = RedirectOutput.delete_delay
 # Delay to wait before trying to reach a rescheduled inventory again, in minutes
@@ -181,22 +182,26 @@ class DocCog(commands.Cog):
             else:
                 return new_name
 
-        # Certain groups are added as prefixes to disambiguate the symbols.
-        if group_name in FORCE_PREFIX_GROUPS:
-            return rename(group_name)
-
-        # The existing symbol with which the current symbol conflicts should have a group prefix.
-        # It currently doesn't have the group prefix because it's only added once there's a conflict.
-        elif item.group in FORCE_PREFIX_GROUPS:
-            return rename(item.group, rename_extant=True)
+        # When there's a conflict, and the package names of the items differ, use the package name as a prefix.
+        if package_name != item.package:
+            if package_name in PRIORITY_PACKAGES:
+                return rename(item.package, rename_extant=True)
+            else:
+                return rename(package_name)
 
-        elif package_name in PRIORITY_PACKAGES:
-            return rename(item.package, rename_extant=True)
+        # If the symbol's group is a non-priority group from FORCE_PREFIX_GROUPS,
+        # add it as a prefix to disambiguate the symbols.
+        elif group_name in FORCE_PREFIX_GROUPS:
+            if item.group in FORCE_PREFIX_GROUPS:
+                needs_moving = FORCE_PREFIX_GROUPS.index(group_name) < FORCE_PREFIX_GROUPS.index(item.group)
+            else:
+                needs_moving = False
+            return rename(item.group if needs_moving else group_name, rename_extant=needs_moving)
 
-        # If we can't specially handle the symbol through its group or package,
-        # fall back to prepending its package name to the front.
+        # If the above conditions didn't pass, either the existing symbol has its group in FORCE_PREFIX_GROUPS,
+        # or deciding which item to rename would be arbitrary, so we rename the existing symbol.
         else:
-            return rename(package_name)
+            return rename(item.group, rename_extant=True)
 
     async def refresh_inventories(self) -> None:
         """Refresh internal documentation inventories."""
diff --git a/bot/exts/moderation/infraction/_scheduler.py b/bot/exts/moderation/infraction/_scheduler.py
index 988fb7220..8286d3635 100644
--- a/bot/exts/moderation/infraction/_scheduler.py
+++ b/bot/exts/moderation/infraction/_scheduler.py
@@ -47,12 +47,33 @@ class InfractionScheduler:
         log.trace(f"Rescheduling infractions for {self.__class__.__name__}.")
 
         infractions = await self.bot.api_client.get(
-            'bot/infractions',
-            params={'active': 'true'}
+            "bot/infractions",
+            params={
+                "active": "true",
+                "ordering": "expires_at",
+                "permanent": "false",
+                "types": ",".join(supported_infractions),
+            },
         )
-        for infraction in infractions:
-            if infraction["expires_at"] is not None and infraction["type"] in supported_infractions:
-                self.schedule_expiration(infraction)
+
+        to_schedule = [i for i in infractions if i["id"] not in self.scheduler]
+
+        for infraction in to_schedule:
+            log.trace("Scheduling %r", infraction)
+            self.schedule_expiration(infraction)
+
+        # Call ourselves again when the last infraction would expire. This will be the "oldest" infraction we've seen
+        # from the database so far, and new ones are scheduled as part of application.
+        # We make sure to fire this
+        if to_schedule:
+            next_reschedule_point = max(
+                dateutil.parser.isoparse(infr["expires_at"]).replace(tzinfo=None) for infr in to_schedule
+            )
+            log.trace("Will reschedule remaining infractions at %s", next_reschedule_point)
+
+            self.scheduler.schedule_at(next_reschedule_point, -1, self.reschedule_infractions(supported_infractions))
+
+        log.trace("Done rescheduling")
 
     async def reapply_infraction(
         self,
diff --git a/bot/exts/recruitment/talentpool/_review.py b/bot/exts/recruitment/talentpool/_review.py
index d53c3b074..b9ff61986 100644
--- a/bot/exts/recruitment/talentpool/_review.py
+++ b/bot/exts/recruitment/talentpool/_review.py
@@ -120,7 +120,8 @@ class Reviewer:
         opening = f"<@&{Roles.mod_team}> <@&{Roles.admins}>\n{member.mention} ({member}) for Helper!"
 
         current_nominations = "\n\n".join(
-            f"**<@{entry['actor']}>:** {entry['reason'] or '*no reason given*'}" for entry in nomination['entries']
+            f"**<@{entry['actor']}>:** {entry['reason'] or '*no reason given*'}"
+            for entry in nomination['entries'][::-1]
         )
         current_nominations = f"**Nominated by:**\n{current_nominations}"
 
diff --git a/bot/resources/tags/async-await.md b/bot/resources/tags/async-await.md
new file mode 100644
index 000000000..ff71ace07
--- /dev/null
+++ b/bot/resources/tags/async-await.md
@@ -0,0 +1,28 @@
+**Concurrency in Python**
+
+Python provides the ability to run multiple tasks and coroutines simultaneously with the use of the `asyncio` library, which is included in the Python standard library.
+
+This works by running these coroutines in an event loop, where the context of which coroutine is being run is switches periodically to allow all of them to run, giving the appearance of running at the same time. This is different to using threads or processes in that all code is run in the main process and thread, although it is possible to run coroutines in threads.
+
+To call an async function we can either `await` it, or run it in an event loop which we get from `asyncio`.
+
+To create a coroutine that can be used with asyncio we need to define a function using the async keyword:
+```py
+async def main():
+    await something_awaitable()
+```
+Which means we can call `await something_awaitable()` directly from within the function. If this were a non-async function this would have raised an exception like: `SyntaxError: 'await' outside async function`
+
+To run the top level async function from outside of the event loop we can get an event loop from `asyncio`, and then use that loop to run the function:
+```py
+from asyncio import get_event_loop
+
+async def main():
+    await something_awaitable()
+
+loop = get_event_loop()
+loop.run_until_complete(main())
+```
+Note that in the `run_until_complete()` where we appear to be calling `main()`, this does not execute the code in `main`, rather it returns a `coroutine` object which is then handled and run by the event loop via `run_until_complete()`.
+
+To learn more about asyncio and its use, see the [asyncio documentation](https://docs.python.org/3/library/asyncio.html).
diff --git a/bot/resources/tags/floats.md b/bot/resources/tags/floats.md
index 7129b91bb..03fcd7268 100644
--- a/bot/resources/tags/floats.md
+++ b/bot/resources/tags/floats.md
@@ -5,7 +5,7 @@ You may have noticed that when doing arithmetic with floats in Python you someti
 0.30000000000000004
 ```
 **Why this happens**
-Internally your computer stores floats as as binary fractions. Many decimal values cannot be stored as exact binary fractions, which means an approximation has to be used.
+Internally your computer stores floats as binary fractions. Many decimal values cannot be stored as exact binary fractions, which means an approximation has to be used.
 
 **How you can avoid this**
  You can use [math.isclose](https://docs.python.org/3/library/math.html#math.isclose) to check if two floats are close, or to get an exact decimal representation, you can use the [decimal](https://docs.python.org/3/library/decimal.html) or [fractions](https://docs.python.org/3/library/fractions.html) module. Here are some examples:
diff --git a/bot/resources/tags/modmail.md b/bot/resources/tags/modmail.md
index 7545419ee..412468174 100644
--- a/bot/resources/tags/modmail.md
+++ b/bot/resources/tags/modmail.md
@@ -6,4 +6,4 @@ It supports attachments, codeblocks, and reactions. As communication happens ove
 
 **To use it, simply send a direct message to the bot.**
 
-Should there be an urgent and immediate need for a moderator or admin to look at a channel, feel free to ping the <@&267629731250176001> or <@&267628507062992896> role instead.
+Should there be an urgent and immediate need for a moderator or admin to look at a channel, feel free to ping the <@&831776746206265384> or <@&267628507062992896> role instead.
diff --git a/bot/resources/tags/star-imports.md b/bot/resources/tags/star-imports.md
index 2be6aab6e..3b1b6a858 100644
--- a/bot/resources/tags/star-imports.md
+++ b/bot/resources/tags/star-imports.md
@@ -16,33 +16,24 @@ Example:
 >>> from math import *
 >>> sin(pi / 2)  # uses sin from math rather than your custom sin
 ```
-
 • Potential namespace collision. Names defined from a previous import might get shadowed by a wildcard import.
-
 • Causes ambiguity. From the example, it is unclear which `sin` function is actually being used. From the Zen of Python **[3]**: `Explicit is better than implicit.`
-
 • Makes import order significant, which they shouldn't. Certain IDE's `sort import` functionality may end up breaking code due to namespace collision.
 
 **How should you import?**
 
 • Import the module under the module's namespace (Only import the name of the module, and names defined in the module can be used by prefixing the module's name)
-
 ```python
 >>> import math
 >>> math.sin(math.pi / 2)
 ```
-
 • Explicitly import certain names from the module
-
 ```python
 >>> from math import sin, pi
 >>> sin(pi / 2)
 ```
-
 Conclusion: Namespaces are one honking great idea -- let's do more of those! *[3]*
 
 **[1]** If the module defines the variable `__all__`, the names defined in `__all__` will get imported by the wildcard import, otherwise all the names in the module get imported (except for names with a leading underscore)
-
 **[2]** [Namespaces and scopes](https://www.programiz.com/python-programming/namespace)
-
 **[3]** [Zen of Python](https://www.python.org/dev/peps/pep-0020/)
diff --git a/bot/utils/regex.py b/bot/utils/regex.py
index 0d2068f90..a8efe1446 100644
--- a/bot/utils/regex.py
+++ b/bot/utils/regex.py
@@ -5,6 +5,7 @@ INVITE_RE = re.compile(
     r"discord(?:[\.,]|dot)com(?:\/|slash)invite|"     # or discord.com/invite/
     r"discordapp(?:[\.,]|dot)com(?:\/|slash)invite|"  # or discordapp.com/invite/
     r"discord(?:[\.,]|dot)me|"                        # or discord.me
+    r"discord(?:[\.,]|dot)li|"                        # or discord.li
     r"discord(?:[\.,]|dot)io"                         # or discord.io.
     r")(?:[\/]|slash)"                                # / or 'slash'
     r"([a-zA-Z0-9\-]+)",                              # the invite code itself
diff --git a/config-default.yml b/config-default.yml
index 394c51c26..55388247c 100644
--- a/config-default.yml
+++ b/config-default.yml
@@ -463,9 +463,6 @@ free:
 help_channels:
     enable: true
 
-    # Minimum interval before allowing a certain user to claim a new help channel
-    claim_minutes: 15
-
     # Roles which are allowed to use the command which makes channels dormant
     cmd_whitelist:
         - *HELPERS_ROLE
@@ -511,7 +508,7 @@ redirect_output:
 
 
 duck_pond:
-    threshold: 5
+    threshold: 7
     channel_blacklist:
         - *ANNOUNCEMENTS
         - *PYNEWS_CHANNEL