Merge branch 'master' into ks123/raw-fix

10 files changed, 131 insertions, 71 deletions

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 876d32b15..1597592ca 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -21,6 +21,6 @@ repos:
         name: Flake8
         description: This hook runs flake8 within our project's pipenv environment.
         entry: pipenv run flake8
-        language: python
+        language: system
         types: [python]
         require_serial: true
diff --git a/Pipfile b/Pipfile
index 3ff653749..1a9c271b4 100644
--- a/Pipfile
+++ b/Pipfile
@@ -14,7 +14,7 @@ beautifulsoup4 = "~=4.9"
 colorama = {version = "~=0.4.3",sys_platform = "== 'win32'"}
 coloredlogs = "~=14.0"
 deepdiff = "~=4.0"
-"discord.py" = {git = "https://github.com/Rapptz/discord.py.git", ref = "93f102ca907af6722ee03638766afd53dfe93a7f"}
+"discord.py" = {git = "https://github.com/Rapptz/discord.py.git", ref = "94f76e63947b102e5de6dae9a2cd687b308033"}
 feedparser = "~=5.2"
 fuzzywuzzy = "~=0.17"
 lxml = "~=4.4"
diff --git a/Pipfile.lock b/Pipfile.lock
index 085d3d829..17d2f81ba 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "1ba637e521c654a23bcc82950e155f5366219eae00bbf809170a371122961a4f"
+            "sha256": "8cc7415371be66ebc4dbfc3f3f27f19f743f4f1a9952ca30abf385a06047439b"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -232,11 +232,11 @@
         },
         "discord-py": {
             "git": "https://github.com/Rapptz/discord.py.git",
-            "ref": "93f102ca907af6722ee03638766afd53dfe93a7f"
+            "ref": "94f76e63947b102e5de6dae9a2cd687b308033dd"
         },
         "discord.py": {
             "git": "https://github.com/Rapptz/discord.py.git",
-            "ref": "93f102ca907af6722ee03638766afd53dfe93a7f"
+            "ref": "94f76e63947b102e5de6dae9a2cd687b308033"
         },
         "docutils": {
             "hashes": [
@@ -592,10 +592,10 @@
         },
         "pytz": {
             "hashes": [
-                "sha256:3e6b7dd2d1e0a59084bcee14a17af60c5c562cdc16d828e8eba2e683d3a7e268",
-                "sha256:5c55e189b682d420be27c6995ba6edce0c0a77dd67bfbe2ae6607134d5851ffd"
+                "sha256:16962c5fb8db4a8f63a26646d8886e9d769b6c511543557bc84e9569fb9a9cb4",
+                "sha256:180befebb1927b16f6b57101720075a984c019ac16b1b7575673bea42c6c3da5"
             ],
-            "version": "==2020.4"
+            "version": "==2020.5"
         },
         "pyyaml": {
             "hashes": [
@@ -834,43 +834,58 @@
         },
         "coverage": {
             "hashes": [
-                "sha256:0203acd33d2298e19b57451ebb0bed0ab0c602e5cf5a818591b4918b1f97d516",
-                "sha256:0f313707cdecd5cd3e217fc68c78a960b616604b559e9ea60cc16795c4304259",
-                "sha256:1c6703094c81fa55b816f5ae542c6ffc625fec769f22b053adb42ad712d086c9",
-                "sha256:1d44bb3a652fed01f1f2c10d5477956116e9b391320c94d36c6bf13b088a1097",
-                "sha256:280baa8ec489c4f542f8940f9c4c2181f0306a8ee1a54eceba071a449fb870a0",
-                "sha256:29a6272fec10623fcbe158fdf9abc7a5fa032048ac1d8631f14b50fbfc10d17f",
-                "sha256:2b31f46bf7b31e6aa690d4c7a3d51bb262438c6dcb0d528adde446531d0d3bb7",
-                "sha256:2d43af2be93ffbad25dd959899b5b809618a496926146ce98ee0b23683f8c51c",
-                "sha256:381ead10b9b9af5f64646cd27107fb27b614ee7040bb1226f9c07ba96625cbb5",
-                "sha256:47a11bdbd8ada9b7ee628596f9d97fbd3851bd9999d398e9436bd67376dbece7",
-                "sha256:4d6a42744139a7fa5b46a264874a781e8694bb32f1d76d8137b68138686f1729",
-                "sha256:50691e744714856f03a86df3e2bff847c2acede4c191f9a1da38f088df342978",
-                "sha256:530cc8aaf11cc2ac7430f3614b04645662ef20c348dce4167c22d99bec3480e9",
-                "sha256:582ddfbe712025448206a5bc45855d16c2e491c2dd102ee9a2841418ac1c629f",
-                "sha256:63808c30b41f3bbf65e29f7280bf793c79f54fb807057de7e5238ffc7cc4d7b9",
-                "sha256:71b69bd716698fa62cd97137d6f2fdf49f534decb23a2c6fc80813e8b7be6822",
-                "sha256:7858847f2d84bf6e64c7f66498e851c54de8ea06a6f96a32a1d192d846734418",
-                "sha256:78e93cc3571fd928a39c0b26767c986188a4118edc67bc0695bc7a284da22e82",
-                "sha256:7f43286f13d91a34fadf61ae252a51a130223c52bfefb50310d5b2deb062cf0f",
-                "sha256:86e9f8cd4b0cdd57b4ae71a9c186717daa4c5a99f3238a8723f416256e0b064d",
-                "sha256:8f264ba2701b8c9f815b272ad568d555ef98dfe1576802ab3149c3629a9f2221",
-                "sha256:9342dd70a1e151684727c9c91ea003b2fb33523bf19385d4554f7897ca0141d4",
-                "sha256:9361de40701666b034c59ad9e317bae95c973b9ff92513dd0eced11c6adf2e21",
-                "sha256:9669179786254a2e7e57f0ecf224e978471491d660aaca833f845b72a2df3709",
-                "sha256:aac1ba0a253e17889550ddb1b60a2063f7474155465577caa2a3b131224cfd54",
-                "sha256:aef72eae10b5e3116bac6957de1df4d75909fc76d1499a53fb6387434b6bcd8d",
-                "sha256:bd3166bb3b111e76a4f8e2980fa1addf2920a4ca9b2b8ca36a3bc3dedc618270",
-                "sha256:c1b78fb9700fc961f53386ad2fd86d87091e06ede5d118b8a50dea285a071c24",
-                "sha256:c3888a051226e676e383de03bf49eb633cd39fc829516e5334e69b8d81aae751",
-                "sha256:c5f17ad25d2c1286436761b462e22b5020d83316f8e8fcb5deb2b3151f8f1d3a",
-                "sha256:c851b35fc078389bc16b915a0a7c1d5923e12e2c5aeec58c52f4aa8085ac8237",
-                "sha256:cb7df71de0af56000115eafd000b867d1261f786b5eebd88a0ca6360cccfaca7",
-                "sha256:cedb2f9e1f990918ea061f28a0f0077a07702e3819602d3507e2ff98c8d20636",
-                "sha256:e8caf961e1b1a945db76f1b5fa9c91498d15f545ac0ababbe575cfab185d3bd8"
+                "sha256:08b3ba72bd981531fd557f67beee376d6700fba183b167857038997ba30dd297",
+                "sha256:2757fa64e11ec12220968f65d086b7a29b6583d16e9a544c889b22ba98555ef1",
+                "sha256:3102bb2c206700a7d28181dbe04d66b30780cde1d1c02c5f3c165cf3d2489497",
+                "sha256:3498b27d8236057def41de3585f317abae235dd3a11d33e01736ffedb2ef8606",
+                "sha256:378ac77af41350a8c6b8801a66021b52da8a05fd77e578b7380e876c0ce4f528",
+                "sha256:38f16b1317b8dd82df67ed5daa5f5e7c959e46579840d77a67a4ceb9cef0a50b",
+                "sha256:3911c2ef96e5ddc748a3c8b4702c61986628bb719b8378bf1e4a6184bbd48fe4",
+                "sha256:3a3c3f8863255f3c31db3889f8055989527173ef6192a283eb6f4db3c579d830",
+                "sha256:3b14b1da110ea50c8bcbadc3b82c3933974dbeea1832e814aab93ca1163cd4c1",
+                "sha256:535dc1e6e68fad5355f9984d5637c33badbdc987b0c0d303ee95a6c979c9516f",
+                "sha256:6f61319e33222591f885c598e3e24f6a4be3533c1d70c19e0dc59e83a71ce27d",
+                "sha256:723d22d324e7997a651478e9c5a3120a0ecbc9a7e94071f7e1954562a8806cf3",
+                "sha256:76b2775dda7e78680d688daabcb485dc87cf5e3184a0b3e012e1d40e38527cc8",
+                "sha256:782a5c7df9f91979a7a21792e09b34a658058896628217ae6362088b123c8500",
+                "sha256:7e4d159021c2029b958b2363abec4a11db0ce8cd43abb0d9ce44284cb97217e7",
+                "sha256:8dacc4073c359f40fcf73aede8428c35f84639baad7e1b46fce5ab7a8a7be4bb",
+                "sha256:8f33d1156241c43755137288dea619105477961cfa7e47f48dbf96bc2c30720b",
+                "sha256:8ffd4b204d7de77b5dd558cdff986a8274796a1e57813ed005b33fd97e29f059",
+                "sha256:93a280c9eb736a0dcca19296f3c30c720cb41a71b1f9e617f341f0a8e791a69b",
+                "sha256:9a4f66259bdd6964d8cf26142733c81fb562252db74ea367d9beb4f815478e72",
+                "sha256:9a9d4ff06804920388aab69c5ea8a77525cf165356db70131616acd269e19b36",
+                "sha256:a2070c5affdb3a5e751f24208c5c4f3d5f008fa04d28731416e023c93b275277",
+                "sha256:a4857f7e2bc6921dbd487c5c88b84f5633de3e7d416c4dc0bb70256775551a6c",
+                "sha256:a607ae05b6c96057ba86c811d9c43423f35e03874ffb03fbdcd45e0637e8b631",
+                "sha256:a66ca3bdf21c653e47f726ca57f46ba7fc1f260ad99ba783acc3e58e3ebdb9ff",
+                "sha256:ab110c48bc3d97b4d19af41865e14531f300b482da21783fdaacd159251890e8",
+                "sha256:b239711e774c8eb910e9b1ac719f02f5ae4bf35fa0420f438cdc3a7e4e7dd6ec",
+                "sha256:be0416074d7f253865bb67630cf7210cbc14eb05f4099cc0f82430135aaa7a3b",
+                "sha256:c46643970dff9f5c976c6512fd35768c4a3819f01f61169d8cdac3f9290903b7",
+                "sha256:c5ec71fd4a43b6d84ddb88c1df94572479d9a26ef3f150cef3dacefecf888105",
+                "sha256:c6e5174f8ca585755988bc278c8bb5d02d9dc2e971591ef4a1baabdf2d99589b",
+                "sha256:c89b558f8a9a5a6f2cfc923c304d49f0ce629c3bd85cb442ca258ec20366394c",
+                "sha256:cc44e3545d908ecf3e5773266c487ad1877be718d9dc65fc7eb6e7d14960985b",
+                "sha256:cc6f8246e74dd210d7e2b56c76ceaba1cc52b025cd75dbe96eb48791e0250e98",
+                "sha256:cd556c79ad665faeae28020a0ab3bda6cd47d94bec48e36970719b0b86e4dcf4",
+                "sha256:ce6f3a147b4b1a8b09aae48517ae91139b1b010c5f36423fa2b866a8b23df879",
+                "sha256:ceb499d2b3d1d7b7ba23abe8bf26df5f06ba8c71127f188333dddcf356b4b63f",
+                "sha256:cef06fb382557f66d81d804230c11ab292d94b840b3cb7bf4450778377b592f4",
+                "sha256:e448f56cfeae7b1b3b5bcd99bb377cde7c4eb1970a525c770720a352bc4c8044",
+                "sha256:e52d3d95df81c8f6b2a1685aabffadf2d2d9ad97203a40f8d61e51b70f191e4e",
+                "sha256:ee2f1d1c223c3d2c24e3afbb2dd38be3f03b1a8d6a83ee3d9eb8c36a52bee899",
+                "sha256:f2c6888eada180814b8583c3e793f3f343a692fc802546eed45f40a001b1169f",
+                "sha256:f51dbba78d68a44e99d484ca8c8f604f17e957c1ca09c3ebc2c7e3bbd9ba0448",
+                "sha256:f54de00baf200b4539a5a092a759f000b5f45fd226d6d25a76b0dff71177a714",
+                "sha256:fa10fee7e32213f5c7b0d6428ea92e3a3fdd6d725590238a3f92c0de1c78b9d2",
+                "sha256:fabeeb121735d47d8eab8671b6b031ce08514c86b7ad8f7d5490a7b6dcd6267d",
+                "sha256:fac3c432851038b3e6afe086f777732bcf7f6ebbfd90951fa04ee53db6d0bcdd",
+                "sha256:fda29412a66099af6d6de0baa6bd7c52674de177ec2ad2630ca264142d69c6c7",
+                "sha256:ff1330e8bc996570221b450e2d539134baa9465f5cb98aff0e0f73f34172e0ae"
             ],
             "index": "pypi",
-            "version": "==5.3"
+            "version": "==5.3.1"
         },
         "coveralls": {
             "hashes": [
@@ -972,11 +987,11 @@
         },
         "identify": {
             "hashes": [
-                "sha256:943cd299ac7f5715fcb3f684e2fc1594c1e0f22a90d15398e5888143bd4144b5",
-                "sha256:cc86e6a9a390879dcc2976cef169dd9cc48843ed70b7380f321d1b118163c60e"
+                "sha256:7aef7a5104d6254c162990e54a203cdc0fd202046b6c415bd5d636472f6565c4",
+                "sha256:b2c71bf9f5c482c389cef816f3a15f1c9d7429ad70f497d4a2e522442d80c6de"
             ],
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
-            "version": "==1.5.10"
+            "version": "==1.5.11"
         },
         "idna": {
             "hashes": [
diff --git a/bot/bot.py b/bot/bot.py
index 4ebe0a5c3..d5f108575 100644
--- a/bot/bot.py
+++ b/bot/bot.py
@@ -3,7 +3,8 @@ import logging
 import socket
 import warnings
 from collections import defaultdict
-from typing import Dict, Optional
+from contextlib import suppress
+from typing import Dict, List, Optional
 
 import aiohttp
 import discord
@@ -70,6 +71,9 @@ class Bot(commands.Bot):
                 attempt + 1
             )
 
+        # All tasks that need to block closing until finished
+        self.closing_tasks: List[asyncio.Task] = []
+
     async def cache_filter_list_data(self) -> None:
         """Cache all the data in the FilterList on the site."""
         full_cache = await self.api_client.get('bot/filter-lists')
@@ -145,6 +149,20 @@ class Bot(commands.Bot):
 
     async def close(self) -> None:
         """Close the Discord connection and the aiohttp session, connector, statsd client, and resolver."""
+        # Done before super().close() to allow tasks finish before the HTTP session closes.
+        for ext in list(self.extensions):
+            with suppress(Exception):
+                self.unload_extension(ext)
+
+        for cog in list(self.cogs):
+            with suppress(Exception):
+                self.remove_cog(cog)
+
+        # Wait until all tasks that have to be completed before bot is closing is done
+        log.trace("Waiting for tasks before closing.")
+        await asyncio.gather(*self.closing_tasks)
+
+        # Now actually do full close of bot
         await super().close()
 
         if self.api_client:
diff --git a/bot/exts/info/information.py b/bot/exts/info/information.py
index 94c207ee3..38e760ee3 100644
--- a/bot/exts/info/information.py
+++ b/bot/exts/info/information.py
@@ -229,9 +229,9 @@ class Information(Cog):
         if on_server:
             joined = time_since(user.joined_at, max_units=3)
             roles = ", ".join(role.mention for role in user.roles[1:])
-            membership = {"Joined": joined, "Pending": user.pending, "Roles": roles or None}
+            membership = {"Joined": joined, "Verified": not user.pending, "Roles": roles or None}
             if not is_mod_channel(ctx.channel):
-                membership.pop("Pending")
+                membership.pop("Verified")
 
             membership = textwrap.dedent("\n".join([f"{key}: {value}" for key, value in membership.items()]))
         else:
diff --git a/bot/exts/info/reddit.py b/bot/exts/info/reddit.py
index bad4c504d..6790be762 100644
--- a/bot/exts/info/reddit.py
+++ b/bot/exts/info/reddit.py
@@ -45,7 +45,7 @@ class Reddit(Cog):
         """Stop the loop task and revoke the access token when the cog is unloaded."""
         self.auto_poster_loop.cancel()
         if self.access_token and self.access_token.expires_at > datetime.utcnow():
-            asyncio.create_task(self.revoke_access_token())
+            self.bot.closing_tasks.append(asyncio.create_task(self.revoke_access_token()))
 
     async def init_reddit_ready(self) -> None:
         """Sets the reddit webhook when the cog is loaded."""
diff --git a/bot/exts/moderation/infraction/_scheduler.py b/bot/exts/moderation/infraction/_scheduler.py
index c062ae7f8..242b2d30f 100644
--- a/bot/exts/moderation/infraction/_scheduler.py
+++ b/bot/exts/moderation/infraction/_scheduler.py
@@ -74,8 +74,21 @@ class InfractionScheduler:
             return
 
         # Allowing mod log since this is a passive action that should be logged.
-        await apply_coro
-        log.info(f"Re-applied {infraction['type']} to user {infraction['user']} upon rejoining.")
+        try:
+            await apply_coro
+        except discord.HTTPException as e:
+            # When user joined and then right after this left again before action completed, this can't apply roles
+            if e.code == 10007 or e.status == 404:
+                log.info(
+                    f"Can't reapply {infraction['type']} to user {infraction['user']} because user left the guild."
+                )
+            else:
+                log.exception(
+                    f"Got unexpected HTTPException (HTTP {e.status}, Discord code {e.code})"
+                    f"when awaiting {infraction['type']} coroutine for {infraction['user']}."
+                )
+        else:
+            log.info(f"Re-applied {infraction['type']} to user {infraction['user']} upon rejoining.")
 
     async def apply_infraction(
         self,
@@ -178,6 +191,10 @@ class InfractionScheduler:
                 log_msg = f"Failed to apply {' '.join(infr_type.split('_'))} infraction #{id_} to {user}"
                 if isinstance(e, discord.Forbidden):
                     log.warning(f"{log_msg}: bot lacks permissions.")
+                elif e.code == 10007 or e.status == 404:
+                    log.info(
+                        f"Can't apply {infraction['type']} to user {infraction['user']} because user left from guild."
+                    )
                 else:
                     log.exception(log_msg)
                 failed = True
@@ -352,9 +369,16 @@ class InfractionScheduler:
             log_text["Failure"] = "The bot lacks permissions to do this (role hierarchy?)"
             log_content = mod_role.mention
         except discord.HTTPException as e:
-            log.exception(f"Failed to deactivate infraction #{id_} ({type_})")
-            log_text["Failure"] = f"HTTPException with status {e.status} and code {e.code}."
-            log_content = mod_role.mention
+            if e.code == 10007 or e.status == 404:
+                log.info(
+                    f"Can't pardon {infraction['type']} for user {infraction['user']} because user left the guild."
+                )
+                log_text["Failure"] = "User left the guild."
+                log_content = mod_role.mention
+            else:
+                log.exception(f"Failed to deactivate infraction #{id_} ({type_})")
+                log_text["Failure"] = f"HTTPException with status {e.status} and code {e.code}."
+                log_content = mod_role.mention
 
         # Check if the user is currently being watched by Big Brother.
         try:
diff --git a/bot/exts/moderation/watchchannels/_watchchannel.py b/bot/exts/moderation/watchchannels/_watchchannel.py
index 7118dee02..f9fc12dc3 100644
--- a/bot/exts/moderation/watchchannels/_watchchannel.py
+++ b/bot/exts/moderation/watchchannels/_watchchannel.py
@@ -342,11 +342,14 @@ class WatchChannel(metaclass=CogABCMeta):
         """Takes care of unloading the cog and canceling the consumption task."""
         self.log.trace("Unloading the cog")
         if self._consume_task and not self._consume_task.done():
+            def done_callback(task: asyncio.Task) -> None:
+                """Send exception when consuming task have been cancelled."""
+                try:
+                    task.result()
+                except asyncio.CancelledError:
+                    self.log.info(
+                        f"The consume task of {type(self).__name__} was canceled. Messages may be lost."
+                    )
+
+            self._consume_task.add_done_callback(done_callback)
             self._consume_task.cancel()
-            try:
-                self._consume_task.result()
-            except asyncio.CancelledError as e:
-                self.log.exception(
-                    "The consume task was canceled. Messages may be lost.",
-                    exc_info=e
-                )
diff --git a/bot/exts/moderation/watchchannels/talentpool.py b/bot/exts/moderation/watchchannels/talentpool.py
index a77dbe156..df2ce586e 100644
--- a/bot/exts/moderation/watchchannels/talentpool.py
+++ b/bot/exts/moderation/watchchannels/talentpool.py
@@ -64,12 +64,12 @@ class TalentPool(WatchChannel, Cog, name="Talentpool"):
 
     @nomination_group.command(name='watch', aliases=('w', 'add', 'a'), root_aliases=("nominate",))
     @has_any_role(*STAFF_ROLES)
-    async def watch_command(self, ctx: Context, user: FetchedMember, *, reason: str) -> None:
+    async def watch_command(self, ctx: Context, user: FetchedMember, *, reason: str = '') -> None:
         """
         Relay messages sent by the given `user` to the `#talent-pool` channel.
 
-        A `reason` for adding the user to the talent pool is required and will be displayed
-        in the header when relaying messages of this user to the channel.
+        A `reason` for adding the user to the talent pool is optional.
+        If given, it will be displayed in the header when relaying messages of this user to the channel.
         """
         if user.bot:
             await ctx.send(f":x: I'm sorry {ctx.author}, I'm afraid I can't do that. I only watch humans.")
@@ -202,7 +202,7 @@ class TalentPool(WatchChannel, Cog, name="Talentpool"):
             f"{self.api_endpoint}/{nomination_id}",
             json={field: reason}
         )
-
+        await self.fetch_user_cache()  # Update cache.
         await ctx.send(f":white_check_mark: Updated the {field} of the nomination!")
 
     @Cog.listener()
@@ -243,8 +243,8 @@ class TalentPool(WatchChannel, Cog, name="Talentpool"):
         actor = guild.get_member(actor_id)
 
         active = nomination_object["active"]
-        log.debug(active)
-        log.debug(type(nomination_object["inserted_at"]))
+
+        reason = nomination_object["reason"] or "*None*"
 
         start_date = time.format_infraction(nomination_object["inserted_at"])
         if active:
@@ -254,7 +254,7 @@ class TalentPool(WatchChannel, Cog, name="Talentpool"):
                 Status: **Active**
                 Date: {start_date}
                 Actor: {actor.mention if actor else actor_id}
-                Reason: {nomination_object["reason"]}
+                Reason: {reason}
                 Nomination ID: `{nomination_object["id"]}`
                 ===============
                 """
@@ -267,7 +267,7 @@ class TalentPool(WatchChannel, Cog, name="Talentpool"):
                 Status: Inactive
                 Date: {start_date}
                 Actor: {actor.mention if actor else actor_id}
-                Reason: {nomination_object["reason"]}
+                Reason: {reason}
 
                 End date: {end_date}
                 Unwatch reason: {nomination_object["end_reason"]}
diff --git a/tests/bot/exts/info/test_information.py b/tests/bot/exts/info/test_information.py
index 043cce8de..d077be960 100644
--- a/tests/bot/exts/info/test_information.py
+++ b/tests/bot/exts/info/test_information.py
@@ -355,7 +355,7 @@ class UserEmbedTests(unittest.IsolatedAsyncioTestCase):
         self.assertEqual(
             textwrap.dedent(f"""
                 Joined: {"1 year ago"}
-                Pending: {"False"}
+                Verified: {"True"}
                 Roles: &Moderators
             """).strip(),
             embed.fields[1].value